Restructured DVMP CI, promiting some scripts to project-level benchmark

config/env.sh

@@ -8,6 +8,8 @@
 ##  - JUGGLER_DETECTOR:       detector package to be used for the benchmark
 ##  - JUGGLER_N_EVENTS:       #events processed by simulation/reconstruction
 ##  - JUGGLER_INSTALL_PREFIX: location where Juggler (digi/recon) is installed
+##  - JUGGLER_N_THREADS:      Number of threads/processes to spawn in parallel
+##  - JUGGLER_RNG_SEED:       Random seed for the RNG
 ##
 ## It also defines the following additional variables for internal usage
 ##  - LOCAL_PREFIX:           prefix for packages installed during the benchmark
@@ -38,6 +40,22 @@ if [ ! -n  "${JUGGLER_N_EVENTS}" ] ; then
 fi
 echo "JUGGLER_N_EVENTS:       ${JUGGLER_N_EVENTS}"
 
+## Maximum number of threads or processes a single pipeline should use
+## (this is not enforced, but the different pipeline scripts should use
+##  this to guide the number of parallel processes or threads they 
+##  spawn).
+if [ ! -n "${JUGGLER_N_THREADS}" ]; then
+  export JUGGLER_N_THREADS=10
+fi
+echo "JUGGLER_N_THREADS:      ${JUGGLER_N_THREADS}"
+
+## Random seed for event generation, should typically not be changed for
+## reproductability.
+if [ ! -n "${JUGGLER_RNG_SEED}" ]; then
+  export JUGGLER_RNG_SEED=1
+fi
+echo "JUGGLER_RNG_SEED:       ${JUGGLER_RNG_SEED}"
+
 ## Install prefix for juggler, needed to locate the Juggler xenv files.
 ## Also used by the CI as install prefix for other packages where needed.
 ## You should not have to touch this. Note that for local usage a different 

dis/config.yml

-dis:run_test:
-  stage: process
+dis:generate:
+  stage: generate
+  needs: []
   timeout: 1 hours
+  artifacts:
+    paths:
+      - results
+  script:
+    - bash dis/gen.sh
+
+dis:process:
+  stage: process
+  needs: ["dis:generate"]
+  timeout: 1 hour
+  artifacts:
+    paths:
+      - results
   script:
-    - bash dis/dis.sh
+    - echo "DIS benchmarks"
 
 dis:results:
   stage: collect
-  needs: ["dis:run_test"]
+  needs: ["dis:process"]
   script:
     - echo "All DIS benchmarks successful"

dis/dis.sh

 #!/bin/bash
 
-if [[ ! -n  "${JUGGLER_DETECTOR}" ]] ; then 
-  export JUGGLER_DETECTOR="topside"
-fi
-
-if [[ ! -n  "${JUGGLER_N_EVENTS}" ]] ; then 
-  export JUGGLER_N_EVENTS=100
-fi
-
-if [[ ! -n  "${JUGGLER_INSTALL_PREFIX}" ]] ; then 
-  export JUGGLER_INSTALL_PREFIX="/usr/local"
-fi
-
-export JUGGLER_FILE_NAME_TAG="dis"
-export JUGGLER_GEN_FILE="${JUGGLER_FILE_NAME_TAG}.hepmc"
-
+## =============================================================================
+## Run the DVMP benchmarks in 5 steps:
+## 1. Build/install detector package
+## 2. Detector simulation through npsim
+## 3. Digitization and reconstruction through Juggler
+## 4. Root-based Physics analyses
+## 5. Finalize
+## =============================================================================
+
+echo "Running the DIS benchmarks"
+
+## make sure we launch this script from the project root directory
+PROJECT_ROOT="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/..
+pushd ${PROJECT_ROOT}
+
+## =============================================================================
+## Load the environment variables. To build the detector we need the following
+## variables:
+##
+## - JUGGLER_INSTALL_PREFIX: Install prefix for Juggler (simu/recon)
+## - JUGGLER_DETECTOR:       the detector package we want to use for this benchmark
+## - DETECTOR_PATH:          full path to the detector definitions
+##
+## You can ready config/env.sh for more in-depth explanations of the variables
+## and how they can be controlled.
+source config/env.sh
+
+## Extra environment variables for DVMP:
+## file tag for these tests
+JUGGLER_FILE_NAME_TAG="dis"
+# TODO use the input file name, as we will be generating a lot of these
+# in the future...
+# FIXME Generator file hardcoded for now
+## note: these variables need to be exported to be accessible from
+##       the juggler options.py. We should really work on a dedicated
+##       juggler launcher to get rid of these "magic" variables. FIXME
+export JUGGLER_GEN_FILE="results/dis/${JUGGLER_FILE_NAME_TAG}.hepmc"
 export JUGGLER_SIM_FILE="sim_${JUGGLER_FILE_NAME_TAG}.root"
 export JUGGLER_REC_FILE="rec_${JUGGLER_FILE_NAME_TAG}.root"
 
-echo "JUGGLER_N_EVENTS = ${JUGGLER_N_EVENTS}"
-echo "JUGGLER_DETECTOR = ${JUGGLER_DETECTOR}"
-
-
-### Build the detector constructors.
-git clone https://eicweb.phy.anl.gov/EIC/detectors/${JUGGLER_DETECTOR}.git
-git clone https://eicweb.phy.anl.gov/EIC/detectors/accelerator.git
-pushd ${JUGGLER_DETECTOR}
-ln -s ../accelerator/eic
-popd
-mkdir ${JUGGLER_DETECTOR}/build
-pushd ${JUGGLER_DETECTOR}/build
-cmake ../. -DCMAKE_INSTALL_PREFIX=/usr/local && make -j30 install
-popd
 
-# generate the input events
-# temporary standin until hepmc output from pythia is generated.
-root -b -q "dis/scripts/gen_central_electrons.cxx(${JUGGLER_N_EVENTS}, \"${JUGGLER_FILE_NAME_TAG}.hepmc\")"
-if [[ "$?" -ne "0" ]] ; then
-  echo "ERROR running script"
-  exit 1
-fi
+## =============================================================================
+## Step 1: Build/install the desired detector package
+bash util/build_detector.sh
 
-#
-pushd ${JUGGLER_DETECTOR}
-
-## run geant4 simulations
+## =============================================================================
+## Step 2: Run the simulation
+echo "Running Geant4 simulation"
 npsim --runType batch \
       --part.minimalKineticEnergy 1000*GeV  \
       -v WARNING \
       --numberOfEvents ${JUGGLER_N_EVENTS} \
-      --compactFile ${JUGGLER_DETECTOR}.xml \
-      --inputFiles ../${JUGGLER_FILE_NAME_TAG}.hepmc \
+      --compactFile ${DETECTOR_PATH}/${JUGGLER_DETECTOR}.xml \
+      --inputFiles ${JUGGLER_GEN_FILE} \
       --outputFile ${JUGGLER_SIM_FILE}
-if [[ "$?" -ne "0" ]] ; then
-  echo "ERROR running script"
+if [ "$?" -ne "0" ] ; then
+  echo "ERROR running npsim"
   exit 1
 fi
 
-# Need to figure out how to pass file name to juggler from the commandline
+## =============================================================================
+## Step 3: Run digitization & reconstruction
+echo "Running the digitization and reconstruction"
+# FIXME Need to figure out how to pass file name to juggler from the commandline
 xenv -x ${JUGGLER_INSTALL_PREFIX}/Juggler.xenv \
-  gaudirun.py ../options/tracker_reconstruction.py
-if [[ "$?" -ne "0" ]] ; then
+  gaudirun.py options/tracker_reconstruction.py
+if [ "$?" -ne "0" ] ; then
   echo "ERROR running juggler"
   exit 1
 fi
 ls -l
-popd
-
-pwd
-mkdir -p results/dis
 
-root -b -q "dis/scripts/rec_dis_electrons.cxx(\"${JUGGLER_DETECTOR}/${JUGGLER_REC_FILE}\")"
+## =============================================================================
+## Step 4: Analysis
+root -b -q "dis/analysis/rec_dis_electrons.cxx(\"${JUGGLER_DETECTOR}/${JUGGLER_REC_FILE}\")"
 if [[ "$?" -ne "0" ]] ; then
   echo "ERROR running root script"
   exit 1
 fi
 
-if [[ "${JUGGLER_N_EVENTS}" -lt "500" ]] ; then 
-cp ${JUGGLER_DETECTOR}/${JUGGLER_REC_FILE} results/dis/.
+## =============================================================================
+## Step 5: finalize
+echo "Finalizing ${JUGGLER_FILE_NAME_TAG} benchmark"
+
+## Copy over reconsturction artifacts as long as we don't have
+## too many events
+if [ "${JUGGLER_N_EVENTS}" -lt "500" ] ; then 
+  cp ${JUGGLER_REC_FILE} results/dis/.
 fi
 
+## cleanup output files
+rm ${JUGGLER_REC_FILE} ${JUGGLER_SIM_FILE}
+
+## =============================================================================
+## All done!
+echo "${JUGGLER_FILE_NAME_TAG} benchmarks complete"

dis/gen.sh

+#!/bin/bash
+
+## =============================================================================
+## Standin for a proper pythia generation process, similar to how we
+## generate events for DVMP
+## =============================================================================
+
+## TODO: use JUGGLER_FILE_NAME_TAG instead of explicitly refering to dis
+
+echo "Running the DIS benchmarks"
+
+## make sure we launch this script from the project root directory
+PROJECT_ROOT="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/..
+pushd ${PROJECT_ROOT}
+
+## =============================================================================
+## Load the environment variables. To build the detector we need the following
+## variables:
+##
+## - JUGGLER_INSTALL_PREFIX: Install prefix for Juggler (simu/recon)
+## - JUGGLER_DETECTOR:       the detector package we want to use for this benchmark
+## - DETECTOR_PATH:          full path to the detector definitions
+##
+## You can ready config/env.sh for more in-depth explanations of the variables
+## and how they can be controlled.
+source config/env.sh
+
+## Setup local environment
+export DATA_PATH=results/dis
+
+## Extra environment variables for DVMP:
+## file tag for these tests
+JUGGLER_FILE_NAME_TAG="dis"
+
+## =============================================================================
+## Step 1: Dummy event generator
+## TODO better file name that encodes the actual configuration we're running
+root -b -q "dis/generator/gen_central_electrons.cxx(${JUGGLER_N_EVENTS}, \".local/${JUGGLER_FILE_NAME_TAG}.hepmc\")"
+if [[ "$?" -ne "0" ]] ; then
+  echo "ERROR running script"
+  exit 1
+fi
+
+## =============================================================================
+## Step 2: finalize
+echo "Moving event generator output into ${DATA_PATH}"
+mv .local/${JUGGLER_FILE_NAME_TAG}.hepmc ${DATA_PATH}/${JUGGLER_FILE_NAME_TAG}.hepmc
+
+## =============================================================================
+## All done!
+echo "dis event generation complete"

dvcs/config.yml

@@ -2,7 +2,7 @@ dvcs:process:
   stage: process
   timeout: 1 hour
   script:
-    - bash dvcs/dvcs.sh
+    - echo "dvcd benchmark"
   artifacts:
     paths:
       - results

dvmp/config.yml

@@ -15,14 +15,14 @@ dvmp:generate:
     paths:
       - results
   script:
-    - ./dvmp/scripts/generate.sh --ebeam 10 --pbeam 100 --config jpsi_central --decay muon --decay electron
+    - ./util/run_many.py ./dvmp/gen.sh --energy 10x100 --config jpsi_central --decay muon --decay electron
 
 dvmp:process:
   stage: process
   needs: ["dvmp:generate"]
   timeout: 1 hour
   script:
-    - bash dvmp/dvmp.sh
+    - ./util/run_many.py dvmp/dvmp.sh --energy 10x100 --config jpsi_central --decay muon --decay electron
   artifacts:
     paths:
       - results

dvmp/dvmp.sh

 #!/bin/bash
 
 ## =============================================================================
-## Run the DVMP benchmarks in 5 steps:
-## 1. Build/install detector package
-## 2. Detector simulation through npsim
-## 3. Digitization and reconstruction through Juggler
-## 4. Root-based Physics analyses
-## 5. Finalize
+## Run the DVMP benchmarks in 7 steps:
+## 1. Parse the command line and setup environment
+## 2. Build/install detector package
+## 3. Detector simulation through npsim
+## 4. Digitization and reconstruction through Juggler
+## 5. Root-based Physics analyses
+## 6. Finalize
 ## =============================================================================
 
-echo "Running the DVMP benchmarks"
-
 ## make sure we launch this script from the project root directory
 PROJECT_ROOT="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/..
 pushd ${PROJECT_ROOT}
 
+echo "Running the DVMP benchmarks"
+
+
 ## =============================================================================
-## Load the environment variables. To build the detector we need the following
-## variables:
+## Step 1: Setup the environment variables
 ##
+## First parse the command line flags.
+## This sets the following environment variables:
+## - CONFIG:   The specific generator configuration
+## - EBEAM:    The electron beam energy
+## - PBEAM:    The ion beam energy
+## - DECAY:    The decay particle for the generator
+## - LEADING:  Leading particle of interest (J/psi)
+export REQUIRE_DECAY=1
+export REQUIRE_LEADING=1
+source util/parse_cmd.sh $@
+
+## To run the reconstruction, we need the following global variables:
 ## - JUGGLER_INSTALL_PREFIX: Install prefix for Juggler (simu/recon)
 ## - JUGGLER_DETECTOR:       the detector package we want to use for this benchmark
 ## - DETECTOR_PATH:          full path to the detector definitions
@@ -27,19 +40,39 @@ pushd ${PROJECT_ROOT}
 ## and how they can be controlled.
 source config/env.sh
 
-## Extra environment variables for DVMP:
-## file tag for these tests
-JUGGLER_FILE_NAME_TAG="dvmp"
-# Generator file, hardcoded for now FIXME
-JUGGLER_GEN_FILE="results/dvmp/jpsi_central_electron-10on100-gen.hepmc"
-# FIXME use the input file name, as we will be generating a lot of these
-# in the future...
-## note: these variables need to be exported to be accessible from
-##       the juggler options.py. We should really work on a dedicated
-##       juggler launcher to get rid of these "magic" variables. FIXME
-export JUGGLER_SIM_FILE="sim_${JUGGLER_FILE_NAME_TAG}.root"
-export JUGGLER_REC_FILE="rec_${JUGGLER_FILE_NAME_TAG}.root"
+## We also need the following benchmark-specific variables:
+##
+## - BENCHMARK_TAG: Unique identified for this benchmark process.
+## - DATA_PATH:     Place to store our persistent output artifacts.
+##
+## You can read dvmp/env.sh for more in-depth explanations of the variables.
+source dvmp/env.sh
 
+## Get a unique file names based on the configuration options
+GEN_FILE=${DATA_PATH}/`util/print_fname.sh \
+                  --ebeam $EBEAM \
+                  --pbeam $PBEAM \
+                  --decay $DECAY \
+                  --config $CONFIG \
+                  --type gen`.hepmc
+SIM_FILE=${LOCAL_PREFIX}/`util/print_fname.sh \
+                  --ebeam $EBEAM \
+                  --pbeam $PBEAM \
+                  --decay $DECAY \
+                  --config $CONFIG \
+                  --type sim`.root
+REC_FILE=${LOCAL_PREFIX}/`util/print_fname.sh \
+                  --ebeam $EBEAM \
+                  --pbeam $PBEAM \
+                  --decay $DECAY \
+                  --config $CONFIG \
+                  --type rec`.root
+PLOT_PREFIX=${DATA_PATH}/`util/print_fname.sh \
+                  --ebeam $EBEAM \
+                  --pbeam $PBEAM \
+                  --decay $DECAY \
+                  --config $CONFIG \
+                  --type rec`
 
 ## =============================================================================
 ## Step 1: Build/install the desired detector package
@@ -53,8 +86,8 @@ npsim --runType batch \
       -v WARNING \
       --numberOfEvents ${JUGGLER_N_EVENTS} \
       --compactFile ${DETECTOR_PATH}/${JUGGLER_DETECTOR}.xml \
-      --inputFiles ${JUGGLER_GEN_FILE} \
-      --outputFile ${JUGGLER_SIM_FILE}
+      --inputFiles ${GEN_FILE} \
+      --outputFile ${SIM_FILE}
 if [ "$?" -ne "0" ] ; then
   echo "ERROR running npsim"
   exit 1
@@ -63,7 +96,17 @@ fi
 ## =============================================================================
 ## Step 3: Run digitization & reconstruction
 echo "Running the digitization and reconstruction"
-# FIXME Need to figure out how to pass file name to juggler from the commandline
+## FIXME Need to figure out how to pass file name to juggler from the commandline
+## the tracker_reconstruction.py options file uses the following environment
+## variables:
+## - JUGGLER_SIM_FILE:    input detector simulation
+## - JUGGLER_REC_FILE:    output reconstructed data
+## - JUGGLER_DETECTOR_PATH: Location of the detector geometry
+## - JUGGLER_N_EVENTS:    number of events to process (part of global environment)
+## - JUGGLER_DETECTOR:    detector package (part of global environment)
+export JUGGLER_SIM_FILE=${SIM_FILE}
+export JUGGLER_REC_FILE=${REC_FILE}
+export JUGGLER_DETECTOR_PATH=${DETECTOR_PATH}
 xenv -x ${JUGGLER_INSTALL_PREFIX}/Juggler.xenv \
   gaudirun.py options/tracker_reconstruction.py
 if [ "$?" -ne "0" ] ; then
@@ -75,11 +118,12 @@ ls -l
 ## =============================================================================
 ## Step 4: Analysis
 root -b -q "dvmp/analysis/vm_mass.cxx(\
- \"${JUGGLER_REC_FILE}\", \
- \"jpsi\", \
- \"electron\", \
+ \"${REC_FILE}\", \
+ \"${LEADING}\", \
+ \"${DECAY}\", \
  \"${JUGGLER_DETECTOR}\", \
- \"results/dvmp/plot\")"
+ \"${PLOT_PREFIX}\")"
+
 
 if [ "$?" -ne "0" ] ; then
   echo "ERROR running root script"
@@ -88,17 +132,17 @@ fi
 
 ## =============================================================================
 ## Step 5: finalize
-echo "Finalizing ${JUGGLER_FILE_NAME_TAG} benchmark"
+echo "Finalizing DVMP benchmark"
 
 ## Copy over reconsturction artifacts as long as we don't have
 ## too many events
 if [ "${JUGGLER_N_EVENTS}" -lt "500" ] ; then 
-  cp ${JUGGLER_REC_FILE} results/dvmp/.
+  cp ${REC_FILE} ${DATA_PATH}
 fi
 
 ## cleanup output files
-rm ${JUGGLER_REC_FILE} ${JUGGLER_SIM_FILE}
+rm ${REC_FILE} ${SIM_FILE}
 
 ## =============================================================================
 ## All done!
-echo "${JUGGLER_FILE_NAME_TAG} benchmarks complete"
+echo "DVMP benchmarks complete"

dvmp/env.sh

+#!/bin/bash
+
+## =============================================================================
+## Local configuration variables for this particular benchmark 
+## It defines the following additional variables: 
+##
+##  - BENCHMARK_TAG:          Tag to identify this particular benchmark
+##  - DATA_PATH:              Data path for all artifact output
+##
+## =============================================================================
+
+## Tag for the local benchmark. Should be the same as the directory name for
+## this particular benchmark set (for clarity). 
+## This tag is used for the output artifacts directory (results/${JUGGLER_TAG}) 
+## and a tag in some of the output files.
+export BENCHMARK_TAG="dvmp"
+echo "Setting up the local environment for the ${BENCHMARK_TAG^^} benchmarks"
+
+## Data path for all artifact output
+export DATA_PATH="results/${BENCHMARK_TAG}"
+mkdir -p ${DATA_PATH}
+echo "DATA_PATH:              ${DATA_PATH}"
+
+## =============================================================================
+## That's all!
+echo "Local environment setup complete."

dvmp/gen.sh

+#!/bin/bash
+
+## =============================================================================
+## Run a single instance of the DVMP generator (lAger)
+## Runs in 5 steps:
+##   1. Parse the command line and setup the environment
+##   2. Check if we can load the requested file from the cache
+##   3. Create our configuration fil
+##   4. Run the actual generator
+##   5. Finalize
+## =============================================================================
+
+## make sure we launch this script from the project root directory
+PROJECT_ROOT="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/..
+pushd ${PROJECT_ROOT}
+
+## =============================================================================
+## Step 1: Setup the environment variables
+##
+## First parse the command line flags.
+## This sets the following environment variables:
+## - CONFIG:   The specific generator configuration
+## - EBEAM:    The electron beam energy
+## - PBEAM:    The ion beam energy
+## - DECAY:    The decay particle for the generator
+export REQUIRE_DECAY=1
+source util/parse_cmd.sh $@
+
+## To run the generator, we need the following global variables:
+##
+## - LOCAL_PREFIX:      Place to cache local packages and data
+## - JUGGLER_N_EVENTS:  Number of events to process
+## - JUGGLER_RNG_SEED:  Random seed for event generation.
+##
+## You can read config/env.sh for more in-depth explanations of the variables
+## and how they can be controlled.
+source config/env.sh
+
+## We also need the following benchmark-specific variables:
+##
+## - BENCHMARK_TAG: Unique identified for this benchmark process.
+## - DATA_PATH:     Place to store our persistent output artifacts.
+##
+## You can read dvmp/env.sh for more in-depth explanations of the variables.
+source dvmp/env.sh
+
+## Get a unique file name based on the configuration options
+FNAME=`util/print_fname.sh \
+                  --ebeam $EBEAM \
+                  --pbeam $PBEAM \
+                  --decay $DECAY \
+                  --config $CONFIG \
+                  --type gen`
+
+## =============================================================================
+## Step 2: Check if we really need to run, or can use the cache.
+if [ -f "${DATA_PATH}/${FNAME}.hepmc" ]; then
+  echo "Found cached generator output for $FNAME, no need to rerun"
+  exit 
+fi
+
+echo "Generator output for $FNAME not found in cache, need to run generator"
+
+## =============================================================================
+## Step 3: Create generator configuration file
+
+## process decay info
+BRANCHING=
+DECAY_PID=
+if [ $DECAY = "electron" ]; then
+  BRANCHING="0.05971"
+  DECAY_PID="11"
+elif [ $DECAY = "muon" ]; then
+  BRANCHING="0.05961"
+  DECAY_PID="13"
+fi
+
+## generate the config file for this generator setup
+CONFIG_IN="${BENCHMARK_TAG}/generator/${CONFIG}.json.in"
+echo "Creating generator configuration file ${FNAME}.json"
+if [ ! -f ${CONFIG_IN} ]; then
+  echo "ERROR: cannot find master config file ${CONFIG_IN}"
+  exit 1
+fi
+sed "s/@TAG@/${FNAME}/" $CONFIG_IN | \
+  sed "s/@EBEAM@/${EBEAM}/" | \
+  sed "s/@PBEAM@/${PBEAM}/" | \
+  sed "s/@DECAY_LEPTON@/${DECAY_PID}/" | \
+  sed "s/@BRANCHING@/${BRANCHING}/" > .local/${FNAME}.json
+
+## =============================================================================
+## Step 4: Run the event generator
+echo "Running the generator"
+lager -r ${JUGGLER_RNG_SEED} -c .local/${FNAME}.json -e ${JUGGLER_N_EVENTS} -o ${LOCAL_PREFIX}
+if [ "$?" -ne "0" ] ; then
+  echo "ERROR running lAger"
+  exit 1
+fi
+
+## =============================================================================
+## Step 5: Finally, move relevant output into the artifacts directory and clean up
+echo "Moving generator output into ${DATA_PATH}"
+mkdir -p ${DATA_PATH}
+for ext in hepmc json log root ; do
+  mv ${LOCAL_PREFIX}/*.${FNAME}.*.${ext} ${DATA_PATH}/${FNAME}.${ext}
+done
+echo "Cleaning up"
+rm .local/${FNAME}.json
+
+## All done!

dvmp/scripts/generate.sh

-#!/bin/bash
-
-## Init the environment
-source config/env.sh
-
-## Maximum number of generators to run in parallel
-export MT=10
-
-## Generates different configurations from the master configuration
-## for both electron and muon decay channels
-
-echo "Generating DVMP event samples using lAger"
-
-EBEAM=
-PBEAM=
-DECAYS=()
-CONFS=()
-
-while [ $# -gt 0 ]
-do
-  key="$1"
-  case $key in
-    --config)
-      CONFS+=("$2")
-      shift # past argument
-      shift # past value
-      ;;
-    --decay)
-      DECAYS+=("$2")
-      shift # past argument
-      shift # past value
-      ;;
-    --ebeam)
-      EBEAM="$2"
-      shift # past argument
-      shift # past value
-      ;;
-    --pbeam)
-      PBEAM="$2"
-      shift # past argument
-      shift # past value
-      ;;
-    *)    # unknown option
-      echo "unknown option"
-      exit 1
-      ;;
-  esac
-done
-
-if [ ${#CONFS[@]} -eq 0 ]; then
-  echo "ERROR: need one or more config names: --config <config name> "
-  exit 1
-elif [ ${#DECAYS[@]} -eq 0 ]; then
-  echo "ERROR: need one or more decay types: --decay muon/electron"
-  exit 1
-elif [ -z $EBEAM ]; then
-  echo "ERROR: EBEAM not defined: --EBEAM <energy>"
-  exit 1
-elif [ -z $PBEAM ]; then
-  echo "ERROR: PBEAM not defined: --PBEAM <energy>"
-  exit 1
-fi
-
-## loop over all our configurations and run the generator in parallel
-
-parallel -j ${MT} \
-   "dvmp/scripts/run_generator_instance.sh --ebeam ${EBEAM} --pbeam ${PBEAM} --config {1} --decay {2}" \
-  ::: "${CONFS[@]}" ::: "${DECAYS[@]}"
-
-CONFIG_BASE=`basename ${CONFIG} .json.in`
-
-echo "Event generation finished"

dvmp/scripts/run_generator_instance.sh

-#!/bin/bash
-
-## Init the environment
-source config/env.sh
-
-## Generator configuration
-## We use generator-level n-events, which needs to be larger or equal to 
-## the number of juggler events
-export NEVENTS=1000
-if [ ${JUGGLER_N_EVENTS} \> ${NEVENTS} ]; then
-  NEVENTS=${JUGGLER_N_EVENTS}
-fi
-
-## Our random seed
-export RNG_SEED=1
-
-## Setup local environment
-export DATA_PATH=results/dvmp
-
-
-EBEAM=
-PBEAM=
-DECAY=
-CONFIG=
-
-while [ $# -gt 0 ]; do
-  key=$1
-  case $key in
-    --ebeam)
-      EBEAM="$2"
-      shift
-      shift
-      ;;
-    --pbeam)
-      PBEAM="$2"
-      shift
-      shift
-      ;;
-    --decay)
-      DECAY="$2"
-      shift
-      shift
-      ;;
-    --config)
-      CONFIG="$2"
-      shift
-      shift
-      ;;
-    *)
-      echo "Unknown option $key to run_generator(), aborting..."
-      exit 1
-  esac
-done
-
-if [ -z $EBEAM ]; then
-  echo "EBEAM not defined in run_generator, aborting..."
-  exit 1
-elif [ -z $PBEAM ]; then
-  echo "PBEAM not defined in run_generator, aborting..."
-  exit 1
-elif [ -z $DECAY ]; then
-  echo "DECAY not defined in run_generator, aborting..."
-  exit 1
-elif [ $DECAY != "electron" ] && [ $DECAY != "muon" ] ; then
-  echo "Unknown decay channel $DECAY, aborting..."
-  exit 1
-elif [ -z $CONFIG ]; then
-  echo "CONFIG not defined in run_generator, aborting..."
-fi
-
-pushd dvmp
-FNAME=`scripts/print_fname.sh \
-                  --ebeam $EBEAM \
-                  --pbeam $PBEAM \
-                  --decay $DECAY \
-                  --config $CONFIG \
-                  --type gen`
-if [ -f "${DATA_PATH}/${FNAME}.hepmc" ]; then
-  echo "Found cached generator output for $FNAME, no need to rerun"
-  exit 
-fi
-
-echo "Generator output for $FNAME not found in cache, need to run generator"
-
-## process decay info
-BRANCHING=
-DECAY_PID=
-if [ $DECAY = "electron" ]; then
-  BRANCHING="0.05971"
-  DECAY_PID="11"
-elif [ $DECAY = "muon" ]; then
-  BRANCHING="0.05961"
-  DECAY_PID="13"
-fi
-
-## generate the config file for this generator setup
-CONFIG_IN="generator/${CONFIG}.json.in"
-echo "Creating generator configuration file ${FNAME}.json"
-if [ ! -f ${CONFIG_IN} ]; then
-  echo "ERROR: cannot find master config file ${CONFIG_IN}"
-  exit 1
-fi
-sed "s/@TAG@/${FNAME}/" $CONFIG_IN | \
-  sed "s/@EBEAM@/${EBEAM}/" | \
-  sed "s/@PBEAM@/${PBEAM}/" | \
-  sed "s/@DECAY_LEPTON@/${DECAY_PID}/" | \
-  sed "s/@BRANCHING@/${BRANCHING}/" > ${FNAME}.json
-
-## New we can run the generator
-echo "Running the generator"
-lager -r ${RNG_SEED} -c ${FNAME}.json -e ${NEVENTS} -o .
-
-ls -lrth 
-popd
-
-
-## Finally, move relevant output into the artifacts directory
-echo "Moving generator output into ${DATA_PATH}"
-mkdir -p ${DATA_PATH}
-for ext in hepmc json log root ; do
-  mv dvmp/*.${FNAME}.*.${ext} ${DATA_PATH}/${FNAME}.${ext}
-done

options/tracker_reconstruction.py

@@ -12,7 +12,7 @@ if "JUGGLER_DETECTOR" in os.environ :
 input_sim_file  = str(os.environ["JUGGLER_SIM_FILE"])
 output_rec_file = str(os.environ["JUGGLER_REC_FILE"])
 n_events = str(os.environ["JUGGLER_N_EVENTS"])
-detector_path = str(os.environ["DETECTOR_PATH"])
+detector_path = str(os.environ["JUGGLER_DETECTOR_PATH"])
 
 geo_service  = GeoSvc("GeoSvc",
         detectors=["{}/{}.xml".format(detector_path, detector_name)])

util/build_detector.sh

@@ -18,7 +18,7 @@ pushd ${PROJECT_ROOT}
 ## - DETECTOR_PATH:    full path for the detector definitions
 ##                     this is the same as ${DETECTOR_PREFIX}/${JUGGLER_DETECTOR}
 ##
-## You can ready config/env.sh for more in-depth explanations of the variables
+## You can read config/env.sh for more in-depth explanations of the variables
 ## and how they can be controlled.
 source config/env.sh
 

util/parse_cmd.sh

+#!/bin/bash
+
+## =============================================================================
+## Generic utility script to parse command line arguments for the various
+## bash scripts that control the CI. This script should be source'd with
+## command line arguments from a bash-like (non-POSIX) shell such as
+## bash or zsh.
+##
+## To control some of the functionality of the script, you can set the following
+## environment variables prior to calling the script:
+##   - REQUIRE_DECAY:     require the --decay flag to be set
+## =============================================================================
+
+## make sure we launch this script from the project root directory
+PROJECT_ROOT="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/..
+pushd ${PROJECT_ROOT}
+
+## =============================================================================
+## Step 1: Process the command line arguments
+
+function print_the_help {
+  echo "USAGE:    --ebeam E --pbeam E --config C1 --decay D2"
+  echo "          [--config C2 --decay D2 --decay D3 ...]"
+  echo "REQUIRED ARGUMENTS:"
+  echo "          --ebeam       Electron beam energy"
+  echo "          --pbeam       Ion beam energy"
+  echo "          --config      Generator configuration identifiers (at least one)"
+  if [ ! -z ${REQUIRE_DECAY} ]; then
+    echo "        --decay       Specific decay particle (e.g. muon)."
+  fi
+  if [ ! -z ${REQUIRE_LEADING} ]; then
+    echo "        --leading     Leading particle of interest (e.g. jpsi)."
+  fi
+  echo "          -h,--help     Print this message"
+  echo ""
+  echo "  Generate multiple monte carlo samples for a desired process." 
+  exit
+}
+
+## Required variables
+EBEAM=
+PBEAM=
+DECAYS=
+CONFIG=
+
+while [ $# -gt 0 ]
+do
+  key="$1"
+  case $key in
+    --config)
+      CONFIG="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --ebeam)
+      EBEAM="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --pbeam)
+      PBEAM="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --leading)
+      LEADING="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --decay)
+      DECAY="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -h|--help)
+      print_the_help
+      exit 0
+      ;;
+    *)    # unknown option
+      echo "unknown option"
+      exit 1
+      ;;
+  esac
+done
+
+if [ -z $CONFIG ]; then
+  echo "ERROR: CONFIG not defined: --config <config>"
+  print_the_help
+  exit 1
+elif [ -z $EBEAM ]; then
+  echo "ERROR: EBEAM not defined: --ebeam <energy>"
+  print_the_help
+  exit 1
+elif [ -z $PBEAM ]; then
+  echo "ERROR: PBEAM not defined: --pbeam <energy>"
+  print_the_help
+  exit 1
+elif [ -z $LEADING ] && [ ! -z $REQUIRE_LEADING ]; then
+  echo "ERROR: LEADING not defined: --leading <channel>"
+  print_the_help
+  exit 1
+elif [ ! -z $LEADING ] && [ -z $REQUIRE_LEADING ]; then
+  echo "ERROR: LEADING flag specified but not required"
+  print_the_help
+  exit 1
+elif [ -z $DECAY ] && [ ! -z $REQUIRE_DECAY ]; then
+  echo "ERROR: DECAY not defined: --decay <channel>"
+  print_the_help
+  exit 1
+elif [ ! -z $DECAY ] && [ -z $REQUIRE_DECAY ]; then
+  echo "ERROR: DECAY flag specified but not required"
+  print_the_help
+  exit 1
+fi
+
+## Export the configured variables
+export CONFIG
+export EBEAM
+export PBEAM
+if [ ! -z $REQUIRE_LEADING ]; then
+  export LEADING
+fi
+if [ ! -z $REQUIRE_DECAY ]; then
+  export DECAY
+fi

dvmp/scripts/print_fname.sh → util/print_fname.sh

 #!/bin/bash
 
+## =============================================================================
 ## Simple script to output a unified file name based on a set of data options
+## Note: this file name will not have an extension, as it is mean to be used as
+##       a file name root.
 
+function print_the_help {
+  echo "USAGE:    print_fname [arguments]"
+  echo "REQUIRED ARGUMENTS:"
+  echo "          --ebeam       Electron beam energy"
+  echo "          --pbeam       Ion beam energy"
+  echo "          --config      Generator configuration identifier"
+  echo "          --type        What type of output file is this? (e.g. sim, rec, ...)"
+  echo "OPTIONAL ARGUMENTS:"
+  echo "          --decay       Specific decay particle (if applicable)."
+  echo "          -h,--help     Print this message"
+  echo ""
+  echo "  This script will generate a unique file name for the benchmark output."
+  exit
+}
+
+## =============================================================================
+## Process the command line arguments
+
+## Required variables
 EBEAM=
 PBEAM=
-DECAY=
 CONFIG=
 TYPE=
 
+## Optional variables
+DECAY=
+
 while [ $# -gt 0 ]; do
   key=$1
   case $key in
@@ -21,11 +45,6 @@ while [ $# -gt 0 ]; do
       shift
       shift
       ;;
-    --decay)
-      DECAY="$2"
-      shift
-      shift
-      ;;
     --config)
       CONFIG="$2"
       shift
@@ -36,6 +55,15 @@ while [ $# -gt 0 ]; do
       shift
       shift
       ;;
+    --decay)
+      DECAY="$2"
+      shift
+      shift
+      ;;
+    -h|--help)
+      print_the_help
+      exit 0
+      ;;
     *)
       echo "Unknown option $key to print_fname, aborting..."
       exit 1
@@ -44,17 +72,31 @@ done
 
 if [ -z $EBEAM ]; then
   echo "EBEAM not defined in print_fname, aborting..."
+  print_the_help
   exit 1
 elif [ -z $PBEAM ]; then
   echo "PBEAM not defined in print_fname, aborting..."
-  exit 1
-elif [ -z $DECAY ]; then
-  echo "DECAY not defined in print_fname, aborting..."
+  print_the_help
   exit 1
 elif [ -z $CONFIG ]; then
   echo "CONFIG not defined in print_fname, aborting..."
+  print_the_help
+  exit 1
 elif [ -z $TYPE ]; then
   echo "TYPE not defined in print_fname, aborting..."
+  print_the_help
+  exit 1
+fi
+
+
+## =============================================================================
+## Generate a unique identifier based on the configuration arguments
+
+## Add decay info to CONFIG if requested
+if [ ! -z $DECAY ]; then
+  CONFIG=${CONFIG}_${DECAY}
 fi
 
-echo "${CONFIG}_${DECAY}-${EBEAM}on${PBEAM}-${TYPE}"
+echo "${TYPE}-${CONFIG}-${EBEAM}on${PBEAM}"
+
+## all done.

util/run_many.py

+#!/usr/bin/env python3
+
+""" 
+This script will run a CI generator or processing script for multiple configurations.
+
+Author: Sylvester Joosten <sjoosten@anl.gov>
+"""
+
+import os
+import argparse
+from multiprocessing import Pool, get_context
+from tempfile import NamedTemporaryFile
+
+class InvalidArgumentError(Exception):
+    pass
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+        'command',
+        help="Script to be launched in parallel")
+parser.add_argument(
+        '--energy',
+        dest='energies',
+        action='append',
+        help='One or more beam energy pairs (e.g. 10x100)',
+        required=True)
+parser.add_argument(
+        '--config',
+        dest='configs',
+        action='append',
+        help='One or more configurations',
+        required=True)
+parser.add_argument(
+        '--leading',
+        dest='leads',
+        action='append',
+        help='One or more leading particles(opt.)',
+        required=False)
+parser.add_argument(
+        '--decay',
+        dest='decays',
+        action='append',
+        help='One or more decay channels (opt.)',
+        required=False)
+parser.add_argument(
+        '--nproc',
+        dest='nproc',
+        default=5,
+        type=int,
+        help='Number of processes to launch in parallel',
+        required=False)
+
+def worker(command):
+    '''Execute the command in a system call, with the supplied argument string.'''
+    ## use a temporary file to capture the terminal output, and then
+    ## print the terminal output once the command finishes
+    with NamedTemporaryFile() as f:
+        cmd = [command, '2>1 >', f.name]
+        cmd = ' '.join(cmd)
+        print("Executing '{}'".format(cmd))
+        os.system(cmd)
+        with open(f.name) as log:
+            print(log.read())
+
+if __name__ == '__main__':
+    args = parser.parse_args()
+    print('Launching CI script in parallel for multiple settings')
+    for e in args.energies:
+        beam_setting = e.split('x')
+        if not beam_setting[0].isnumeric() or not beam_setting[1].isnumeric():
+            print("Error: invalid beam energy setting:", e)
+            raise InvalidArgumentError
+
+    if not os.path.exists(args.command):
+        print("Error: Script not found:", args.command)
+        raise InvalidArgumentError
+
+    if args.nproc < 1 or args.nproc > 50:
+        print("Error: Invalid process limit (should be 1-50):", args.nproc)
+        raise InvalidArgumentError
+
+    print(' - command: {}'.format(args.command))
+    print(' - energies: {}'.format(args.energies))
+    print(' - config: {}'.format(args.configs))
+    print(' - nproc: {}'.format(args.nproc))
+    if (args.leads):
+        print(' - leading: {}'.format(args.leads))
+    if (args.decays):
+        print(' - decay: {}'.format(args.decays))
+
+    ## Expand our command and argument list for all combinatorics
+    cmds = []
+    decays = args.decays if args.decays else [None]
+    leads = args.leads if args.leads else [None]
+    for e in args.energies:
+        for c in args.configs:
+            for l in leads:
+                for d in decays:
+                    beam_setting = e.split('x')
+                    cmd = [args.command,
+                           '--ebeam', beam_setting[0],
+                           '--pbeam', beam_setting[1],
+                           '--config', c]
+                    if l is not None:
+                        cmd += ['--leading', l]
+                    if d is not None:
+                        cmd += ['--decay', d]
+                    cmds.append(' '.join(cmd))
+
+    ## create a process pool
+    ## note that I'm using themultiprocessing.get_context function to setup
+    ## a context where subprocesses are created using the new "spawn" process
+    ## which avoids deadlocks that sometimes happen in the default dispatch
+    with get_context('spawn').Pool(processes=args.nproc) as pool:
+        pool.map(worker, cmds)
+
+    ## That's all!