simulate_gps.sh

#!/bin/bash
#SBATCH --output=log/simulate_%A_%4a.out
#SBATCH --time=06:00:00
#SBATCH --mem-per-cpu=4G
#SBATCH --cpus-per-task=1
#SBATCH --mail-user=wdconinc@gmail.com
#SBATCH --mail-type=ALL

if [ -n "$SLURM_JOB_ID" ] ;  then
  BASEDIR=$(realpath ${SLURM_SUBMIT_DIR})
else
  BASEDIR=$(realpath $(dirname ${0}))
fi

if [ -x ${BASEDIR}/eic-shel ] ; then
  EICSHELL=${BASEDIR}/eic-shell
else
  EICSHELL="/cvmfs/oasis.opensciencegrid.org/mis/singularity/bin/singularity run /cvmfs/singularity.opensciencegrid.org/eicweb/jug_xl:nightly/"
fi

# check arguments
if [ $# -lt 1 ] ; then
  echo "Usage: "
  echo "  $0 <input> [n_chunk=10000]"
  echo "  sbatch --array 1-<N> $0 <input> [n_chunk=10000]"
  echo
  echo "This job is intended to be run as an array job:"
  echo "  sbatch --array 1-<N> $0 <input> [n_chunk=10000]"
  echo "where N tasks of n_chunk events will be dispatched."
  echo
  echo "Note: The job with task ID 1 will start with the first event."
  echo
  echo "In interactive use, the number of events per task is still honored,"
  echo "but the sequence always starts from the first event."
  echo "  $0 <input> [n_chunk=10000]"
  echo
  echo "A typical npsim run requires from 0.5 to 1.5 core-seconds per event,"
  echo "and uses slightly less than 0.5 GB memory. The output ROOT file for"
  echo "10k events take up about 2 GB in disk space."
  exit
fi

# startup
date
ls -al

# data ingress
INPUT_FILE=${1}

# number of events and chunks
EVENTS_PER_TASK=${2:-10000}

if [ -z ${SLURM_ARRAY_TASK_ID} ] ; then
  # interactive
  TASK=""
else
  # array job
  TASK=$(printf ".%04d" ${SLURM_ARRAY_TASK_ID})
fi

# data egress
BASENAME=$(basename ${INPUT_FILE} .steer)
TAG=${BASENAME//_/\/}
mkdir -p  ${BASEDIR}/full/SINGLE/${TAG}
FULL_FILE=${BASEDIR}/full/SINGLE/${TAG}/${BASENAME}${TASK}.root
mkdir -p  ${BASEDIR}/geom/SINGLE/${TAG}
GEOM_ROOT=${BASEDIR}/geom/SINGLE/${TAG}/${BASENAME}${TASK}.geom
mkdir -p  ${BASEDIR}/reco/SINGLE/${TAG}
RECO_FILE=${BASEDIR}/reco/SINGLE/${TAG}/${BASENAME}${TASK}.root

# detector description
COMPACT_FILE=/opt/detector/share/athena/athena.xml

# start container
cat << EOF | ${EICSHELL}

source /opt/detector/setup.sh

if [ ! -f ${FULL_FILE} -o ! -d ${GEOM_ROOT} ] ; then
  # Run geant4 simulations
  /usr/bin/time -v \
  npsim \
      --runType run \
      --printLevel WARNING \
      --enableGun \
      --steeringFile ${INPUT_FILE} \
      --numberOfEvents ${EVENTS_PER_TASK} \
      --compactFile ${COMPACT_FILE} \
      --outputFile ${FULL_FILE}
  rootls -t "${FULL_FILE}"

  # Take snapshot of geometry and versions
  mkdir -p ${GEOM_ROOT}
  cp -r /opt/detector/* ${GEOM_ROOT}
  eic-info > ${GEOM_ROOT}/eic-info.txt
  echo -n "export LD_LIBRARY_PATH=${GEOM_ROOT}/lib:$" > ${GEOM_ROOT}/setup.sh
  echo "LD_LIBRARY_PATH" >> ${GEOM_ROOT}/setup.sh

  # Data egress
  if [ -x /usr/local/bin/mc -a -r config.json ] ; then
    if ping -c 1 -w 5 google.com > /dev/null ; then
      /usr/local/bin/mc -C $PWD cp ${FULL_FILE} S3rw/eictest/ATHENA/FULL/SINGLE/${TAG}/${BASENAME}${TASK}.root
    else
      echo "No internet connection."
    fi
  fi
else
  ls -al ${FULL_FILE}
fi

# Run reconstruction
source ${GEOM_ROOT}/setup.sh
export JUGGLER_SIM_FILE="${FULL_FILE}"
export JUGGLER_REC_FILE="${RECO_FILE}"
export JUGGLER_N_EVENTS=2147483647
export JUGGLER_DETECTOR=athena
export DETECTOR_PATH="${GEOM_ROOT}/share/athena"
/usr/bin/time -v \
xenv -x /usr/local/Juggler.xenv \
  gaudirun.py /opt/benchmarks/reconstruction_benchmarks/benchmarks/full/options/full_reconstruction.py
rootls -t "${RECO_FILE}"

# S3 egress
if [ -x /usr/local/bin/mc -a -r config.json ] ; then
  if ping -c 1 -w 5 google.com > /dev/null ; then
    /usr/local/bin/mc -C $PWD cp ${RECO_FILE} S3rw/eictest/ATHENA/RECO/SINGLE/${TAG}/${BASENAME}${TASK}.root
  else
    echo "No internet connection."
  fi
fi
EOF

# Condor egress
if [ ! -z "${_CONDOR_SCRATCH_DIR}" ] ; then
  mv ${RECO_FILE} ${_CONDOR_SCRATCH_DIR}
fi

# closeout
ls -al ${FULL_FILE}
ls -al ${RECO_FILE}
date