Got rid of fname script for dvmp, and split up generator artifacts from...

Got rid of fname script for dvmp, and split up generator artifacts from reconstruction artifacts (input versus results directories

.gitignore

@@ -41,8 +41,9 @@ calorimeters/test/
 *.d
 *.pcm
 
-# output files
-results/*
+# input/output files
+input
+results
 
 # ROOT files
 *.root

dvmp/config.yml

@@ -10,10 +10,10 @@ dvmp:generate:
         - dvmp/scripts/jpsi_central-generate.sh
       prefix: "$CI_COMMIT_REF_SLUG"
     paths:
-      - results/dvmp
+      - input/dvmp
   artifacts:
     paths:
-      - results
+      - input
   script:
     - ./util/run_many.py ./dvmp/gen.sh --energy 10x100 --config jpsi_central --decay muon --decay electron
 

dvmp/dvmp.sh

 #!/bin/bash
 
 ## =============================================================================
-## Run the DVMP benchmarks in 7 steps:
+## Run the DVMP benchmarks in 5 steps:
 ## 1. Parse the command line and setup environment
-## 2. Build/install detector package
-## 3. Detector simulation through npsim
-## 4. Digitization and reconstruction through Juggler
-## 5. Root-based Physics analyses
-## 6. Finalize
+## 2. Detector simulation through npsim
+## 3. Digitization and reconstruction through Juggler
+## 4. Root-based Physics analyses
+## 5. Finalize
 ## =============================================================================
 
 ## make sure we launch this script from the project root directory
@@ -43,41 +42,18 @@ source config/env.sh
 ## We also need the following benchmark-specific variables:
 ##
 ## - BENCHMARK_TAG: Unique identified for this benchmark process.
-## - DATA_PATH:     Place to store our persistent output artifacts.
+## - INPUT_PATH:    Path for generator-level input to the benchmarks
+## - TMP_PATH:      Path for temporary data (not exported as artifacts)
+## - RESULTS_PATH:  Path for benchmark output figures and files
 ##
 ## You can read dvmp/env.sh for more in-depth explanations of the variables.
 source dvmp/env.sh
 
 ## Get a unique file names based on the configuration options
-GEN_FILE=${DATA_PATH}/`util/print_fname.sh \
-                  --ebeam $EBEAM \
-                  --pbeam $PBEAM \
-                  --decay $DECAY \
-                  --config $CONFIG \
-                  --type gen`.hepmc
-SIM_FILE=${LOCAL_PREFIX}/`util/print_fname.sh \
-                  --ebeam $EBEAM \
-                  --pbeam $PBEAM \
-                  --decay $DECAY \
-                  --config $CONFIG \
-                  --type sim`.root
-REC_FILE=${LOCAL_PREFIX}/`util/print_fname.sh \
-                  --ebeam $EBEAM \
-                  --pbeam $PBEAM \
-                  --decay $DECAY \
-                  --config $CONFIG \
-                  --type rec`.root
-PLOT_PREFIX=${DATA_PATH}/`util/print_fname.sh \
-                  --ebeam $EBEAM \
-                  --pbeam $PBEAM \
-                  --decay $DECAY \
-                  --config $CONFIG \
-                  --type rec`
-
-## =============================================================================
-## Step 1: Build/install the desired detector package
-# moved to different CI step TODO remove
-#bash util/build_detector.sh
+GEN_FILE=${INPUT_PATH}/gen-${CONFIG}_${DECAY}.hepmc
+SIM_FILE=${TMP_PATH}/sim-${CONFIG}_${DECAY}.root
+REC_FILE=${TMP_PATH}/rec-${CONFIG}_${DECAY}.root
+PLOT_PREFIX=${CONFIG}_${DECAY}
 
 ## =============================================================================
 ## Step 2: Run the simulation
@@ -123,7 +99,7 @@ root -b -q "dvmp/analysis/vm_mass.cxx(\
  \"${LEADING}\", \
  \"${DECAY}\", \
  \"${JUGGLER_DETECTOR}\", \
- \"${PLOT_PREFIX}\")"
+ \"${RESULTS_PATH}/${PLOT_PREFIX}\")"
 
 
 if [ "$?" -ne "0" ] ; then
@@ -138,11 +114,11 @@ echo "Finalizing DVMP benchmark"
 ## Copy over reconsturction artifacts as long as we don't have
 ## too many events
 if [ "${JUGGLER_N_EVENTS}" -lt "500" ] ; then 
-  cp ${REC_FILE} ${DATA_PATH}
+  cp ${REC_FILE} ${RESULTS_PATH}
 fi
 
 ## cleanup output files
-rm ${REC_FILE} ${SIM_FILE}
+rm -f ${REC_FILE} ${SIM_FILE}
 
 ## =============================================================================
 ## All done!

dvmp/env.sh

@@ -5,7 +5,12 @@
 ## It defines the following additional variables: 
 ##
 ##  - BENCHMARK_TAG:          Tag to identify this particular benchmark
-##  - DATA_PATH:              Data path for all artifact output
+##  - INPUT_PATH:             Path for generator-level input to the benchmarks
+##  - TMP_PATH:               Path for temporary data (not exported as artifacts)
+##  - RESULTS_PATH:           Path for benchmark output figures and files
+##
+## This script assumes that EBEAM and PBEAM are set as part of the
+## calling script (usually as command line argument).
 ##
 ## =============================================================================
 
@@ -16,10 +21,25 @@
 export BENCHMARK_TAG="dvmp"
 echo "Setting up the local environment for the ${BENCHMARK_TAG^^} benchmarks"
 
-## Data path for all artifact output
-export DATA_PATH="results/${BENCHMARK_TAG}"
-mkdir -p ${DATA_PATH}
-echo "DATA_PATH:              ${DATA_PATH}"
+## Data path for input data (generator-level hepmc file)
+INPUT_PATH="input/${BENCHMARK_TAG}/${EBEAM}on${PBEAM}"
+mkdir -p ${INPUT_PATH}
+export INPUT_PATH=`realpath ${INPUT_PATH}`
+echo "INPUT_PATH:             ${INPUT_PATH}"
+
+
+## Data path for temporary data (not exported as artifacts)
+TMP_PATH=${LOCAL_PREFIX}
+mkdir -p ${TMP_PATH}
+export TMP_PATH=`realpath ${TMP_PATH}`
+echo "TMP_PATH:               ${TMP_PATH}"
+
+## Data path for benchmark output (plots and reconstructed files
+## if not too big).
+RESULTS_PATH="results/${BENCHMARK_TAG}/${EBEAM}on${PBEAM}"
+mkdir -p ${RESULTS_PATH}
+export RESULTS_PATH=`realpath ${RESULTS_PATH}`
+echo "RESULTS_PATH:           ${RESULTS_PATH}"
 
 ## =============================================================================
 ## That's all!

dvmp/gen.sh

@@ -39,27 +39,23 @@ source config/env.sh
 ## We also need the following benchmark-specific variables:
 ##
 ## - BENCHMARK_TAG: Unique identified for this benchmark process.
-## - DATA_PATH:     Place to store our persistent output artifacts.
+## - INPUT_PATH:    Path for generator-level input to the benchmarks
+## - TMP_PATH:      Path for temporary data (not exported as artifacts)
 ##
 ## You can read dvmp/env.sh for more in-depth explanations of the variables.
 source dvmp/env.sh
 
-## Get a unique file name based on the configuration options
-FNAME=`util/print_fname.sh \
-                  --ebeam $EBEAM \
-                  --pbeam $PBEAM \
-                  --decay $DECAY \
-                  --config $CONFIG \
-                  --type gen`
+## Get a unique file name prefix based on the configuration options
+GEN_PREFIX=gen-${CONFIG}_${DECAY}
 
 ## =============================================================================
 ## Step 2: Check if we really need to run, or can use the cache.
-if [ -f "${DATA_PATH}/${FNAME}.hepmc" ]; then
-  echo "Found cached generator output for $FNAME, no need to rerun"
-  exit 
+if [ -f "${INPUT_PATH}/${GEN_PREFIX}.hepmc" ]; then
+  echo "Found cached generator output for $GEN_PREFIX, no need to rerun"
+  exit 0
 fi
 
-echo "Generator output for $FNAME not found in cache, need to run generator"
+echo "Generator output for $GEN_PREFIX not found in cache, need to run generator"
 
 ## =============================================================================
 ## Step 3: Create generator configuration file
@@ -77,21 +73,24 @@ fi
 
 ## generate the config file for this generator setup
 CONFIG_IN="${BENCHMARK_TAG}/generator/${CONFIG}.json.in"
-echo "Creating generator configuration file ${FNAME}.json"
+echo "Creating generator configuration file ${GEN_PREFIX}.json"
 if [ ! -f ${CONFIG_IN} ]; then
   echo "ERROR: cannot find master config file ${CONFIG_IN}"
   exit 1
 fi
-sed "s/@TAG@/${FNAME}/" $CONFIG_IN | \
+sed "s/@TAG@/${GEN_PREFIX}/" $CONFIG_IN | \
   sed "s/@EBEAM@/${EBEAM}/" | \
   sed "s/@PBEAM@/${PBEAM}/" | \
   sed "s/@DECAY_LEPTON@/${DECAY_PID}/" | \
-  sed "s/@BRANCHING@/${BRANCHING}/" > .local/${FNAME}.json
+  sed "s/@BRANCHING@/${BRANCHING}/" > ${TMP_PATH}/${GEN_PREFIX}.json
 
 ## =============================================================================
 ## Step 4: Run the event generator
 echo "Running the generator"
-lager -r ${JUGGLER_RNG_SEED} -c .local/${FNAME}.json -e ${JUGGLER_N_EVENTS} -o ${LOCAL_PREFIX}
+lager -r ${JUGGLER_RNG_SEED} \
+      -c ${TMP_PATH}/${GEN_PREFIX}.json \
+      -e ${JUGGLER_N_EVENTS} \
+      -o ${TMP_PATH}
 if [ "$?" -ne "0" ] ; then
   echo "ERROR running lAger"
   exit 1
@@ -99,12 +98,12 @@ fi
 
 ## =============================================================================
 ## Step 5: Finally, move relevant output into the artifacts directory and clean up
-echo "Moving generator output into ${DATA_PATH}"
-mkdir -p ${DATA_PATH}
+echo "Moving generator output into ${INPUT_PATH}"
 for ext in hepmc json log root ; do
-  mv ${LOCAL_PREFIX}/*.${FNAME}.*.${ext} ${DATA_PATH}/${FNAME}.${ext}
+  mv ${TMP_PATH}/*.${GEN_PREFIX}.*.${ext} ${INPUT_PATH}/${GEN_PREFIX}.${ext}
 done
+## this step only matters for local execution
 echo "Cleaning up"
-rm .local/${FNAME}.json
+rm ${TMP_PATH}/${GEN_PREFIX}.json
 
 ## All done!

util/print_fname.sh

-#!/bin/bash
-
-## =============================================================================
-## Simple script to output a unified file name based on a set of data options
-## Note: this file name will not have an extension, as it is mean to be used as
-##       a file name root.
-
-function print_the_help {
-  echo "USAGE:    print_fname [arguments]"
-  echo "REQUIRED ARGUMENTS:"
-  echo "          --ebeam       Electron beam energy"
-  echo "          --pbeam       Ion beam energy"
-  echo "          --config      Generator configuration identifier"
-  echo "          --type        What type of output file is this? (e.g. sim, rec, ...)"
-  echo "OPTIONAL ARGUMENTS:"
-  echo "          --decay       Specific decay particle (if applicable)."
-  echo "          -h,--help     Print this message"
-  echo ""
-  echo "  This script will generate a unique file name for the benchmark output."
-  exit
-}
-
-## =============================================================================
-## Process the command line arguments
-
-## Required variables
-EBEAM=
-PBEAM=
-CONFIG=
-TYPE=
-
-## Optional variables
-DECAY=
-
-while [ $# -gt 0 ]; do
-  key=$1
-  case $key in
-    --ebeam)
-      EBEAM="$2"
-      shift
-      shift
-      ;;
-    --pbeam)
-      PBEAM="$2"
-      shift
-      shift
-      ;;
-    --config)
-      CONFIG="$2"
-      shift
-      shift
-      ;;
-    --type)
-      TYPE="$2"
-      shift
-      shift
-      ;;
-    --decay)
-      DECAY="$2"
-      shift
-      shift
-      ;;
-    -h|--help)
-      print_the_help
-      exit 0
-      ;;
-    *)
-      echo "Unknown option $key to print_fname, aborting..."
-      exit 1
-  esac
-done
-
-if [ -z $EBEAM ]; then
-  echo "EBEAM not defined in print_fname, aborting..."
-  print_the_help
-  exit 1
-elif [ -z $PBEAM ]; then
-  echo "PBEAM not defined in print_fname, aborting..."
-  print_the_help
-  exit 1
-elif [ -z $CONFIG ]; then
-  echo "CONFIG not defined in print_fname, aborting..."
-  print_the_help
-  exit 1
-elif [ -z $TYPE ]; then
-  echo "TYPE not defined in print_fname, aborting..."
-  print_the_help
-  exit 1
-fi
-
-
-## =============================================================================
-## Generate a unique identifier based on the configuration arguments
-
-## Add decay info to CONFIG if requested
-if [ ! -z $DECAY ]; then
-  CONFIG=${CONFIG}_${DECAY}
-fi
-
-echo "${TYPE}-${CONFIG}-${EBEAM}on${PBEAM}"
-
-## all done.