simulate_hepmc.sh

#!/bin/bash
#SBATCH --output=log/simulate_%A_%4a.out
#SBATCH --time=04:00:00
#SBATCH --mem-per-cpu=4G
#SBATCH --cpus-per-task=1
#SBATCH --mail-user=wdconinc@gmail.com
#SBATCH --mail-type=ALL

if [ -n "$SLURM_JOB_ID" ] ;  then
  BASEDIR=$(realpath ${SLURM_SUBMIT_DIR})
else
  BASEDIR=$(realpath $(dirname ${0}))
fi
EICSHELL=${BASEDIR}/eic-shell

# check arguments
if [ $# -lt 1 ] ; then
  echo "Usage: "
  echo "  $0 <input> [n_chunk=10000]"
  echo "  sbatch --array 1-N $0 <input> [n_chunk=10000]"
  echo
  echo "This job is intended to be run as an array job:"
  echo "  sbatch --array 1-<N> $0 <input> [n_chunk=10000]"
  echo "where N tasks of n_chunk events will be dispatched."
  echo
  echo "Note: The job with task ID 1 will start with the first event."
  echo
  echo "In interactive use, the number of events per task is still honored,"
  echo "but the sequence always starts from the first event."
  echo "  $0 <input> [n_chunk=10000]"
  echo
  echo "A typical npsim run requires from 0.5 to 1.5 core-seconds per event,"
  echo "and uses slightly less than 0.5 GB memory. The output ROOT file for"
  echo "10k events take up about 2 GB in disk space."
  exit
fi

# startup
date

# data ingress
INPUT_FILE=${1}

# number of events and chunks
EVENTS_PER_TASK=${2:-10000}

if [ -z ${SLURM_ARRAY_TASK_ID} ] ; then
  # interactive
  TASK=""
  SKIP_N_EVENTS=0
else
  # array job
  TASK=$(printf ".%04d" ${SLURM_ARRAY_TASK_ID})
  SKIP_N_EVENTS=$(((SLURM_ARRAY_TASK_ID-1)*EVENTS_PER_TASK))
fi

# data egress
BASENAME=$(basename ${INPUT_FILE} .hepmc)
INPUT_DIR=$(dirname $(realpath --relative-to=${BASEDIR} ${INPUT_FILE}))
INPUT_PREFIX=${INPUT_DIR/\/*/}
TAG=${INPUT_DIR/${INPUT_PREFIX}\//}
mkdir -p  ${BASEDIR}/full/${TAG}
FULL_FILE=${BASEDIR}/full/${TAG}/${BASENAME}${TASK}.root
mkdir -p  ${BASEDIR}/geom/${TAG}
GEOM_ROOT=${BASEDIR}/geom/${TAG}/${BASENAME}${TASK}.geom
mkdir -p  ${BASEDIR}/reco/${TAG}
RECO_FILE=${BASEDIR}/reco/${TAG}/${BASENAME}${TASK}.root

# detector description
COMPACT_FILE=/opt/detector/share/athena/athena.xml

# start container
cat << EOF | ${EICSHELL}

# Run geant4 simulations
if [ ! -f ${FULL_FILE} -o ! -d ${GEOM_ROOT} ] ; then
  source /opt/detector/setup.sh

  npsim \
      --runType batch \
      --printLevel WARNING \
      --skipNEvents ${SKIP_N_EVENTS} \
      --numberOfEvents ${EVENTS_PER_TASK} \
      --part.minimalKineticEnergy 10*MeV \
      --compactFile ${COMPACT_FILE} \
      --inputFiles ${INPUT_FILE} \
      --outputFile ${FULL_FILE}

  # Take snapshot of geometry and versions
  mkdir -p ${GEOM_ROOT}
  cp -r /opt/detector/* ${GEOM_ROOT}
  eic-info > ${GEOM_ROOT}/eic-info.txt
  echo -n "export LD_LIBRARY_PATH=${GEOM_ROOT}/lib:$" > ${GEOM_ROOT}/setup.sh
  echo "LD_LIBRARY_PATH" >> ${GEOM_ROOT}/setup.sh

  # Data egress
  if [ -x /usr/local/bin/mc ] ; then
    mc cp ${FULL_FILE} S3rw/eictest/ATHENA/FULL/${TAG}/${BASENAME}${TASK}.root
  fi
else
  ls -al ${FULL_FILE}
fi

# Run reconstruction
source ${GEOM_ROOT}/setup.sh
export JUGGLER_SIM_FILE="${FULL_FILE}"
export JUGGLER_REC_FILE="${RECO_FILE}"
export JUGGLER_N_EVENTS=2147483647
export JUGGLER_DETECTOR=athena
export JUGGLER_DETECTOR_PATH="${GEOM_ROOT}/share/athena"
xenv -x /usr/local/Juggler.xenv \
  gaudirun.py /opt/benchmarks/reconstruction_benchmarks/benchmarks/full/options/full_reconstruction.py

# Data egress
if [ -x /usr/local/bin/mc ] ; then
  /usr/local/bin/mc cp ${RECO_FILE} S3rw/eictest/ATHENA/RECO/${TAG}/${BASENAME}${TASK}.root
fi

EOF

# closeout
ls -al ${FULL_FILE}
ls -al ${RECO_FILE}
date