From 8ab2f69f40299f20a1baad56c3d6b14f1f6e87d1 Mon Sep 17 00:00:00 2001
From: Sylvester Joosten <sylvester.joosten@gmail.com>
Date: Sat, 21 Nov 2020 17:39:04 -0600
Subject: [PATCH] Got rid of fname script for dvmp, and split up generator
artifacts from reconstruction artifacts (input versus results directories
---
.gitignore | 5 ++-
dvmp/config.yml | 4 +-
dvmp/dvmp.sh | 54 +++++++----------------
dvmp/env.sh | 30 ++++++++++---
dvmp/gen.sh | 39 +++++++++--------
util/print_fname.sh | 102 --------------------------------------------
6 files changed, 64 insertions(+), 170 deletions(-)
delete mode 100755 util/print_fname.sh
diff --git a/.gitignore b/.gitignore
index d8d55fee..5cf7b521 100644
--- a/.gitignore
+++ b/.gitignore
@@ -41,8 +41,9 @@ calorimeters/test/
*.d
*.pcm
-# output files
-results/*
+# input/output files
+input
+results
# ROOT files
*.root
diff --git a/dvmp/config.yml b/dvmp/config.yml
index 3266b84f..c646d6dd 100644
--- a/dvmp/config.yml
+++ b/dvmp/config.yml
@@ -10,10 +10,10 @@ dvmp:generate:
- dvmp/scripts/jpsi_central-generate.sh
prefix: "$CI_COMMIT_REF_SLUG"
paths:
- - results/dvmp
+ - input/dvmp
artifacts:
paths:
- - results
+ - input
script:
- ./util/run_many.py ./dvmp/gen.sh --energy 10x100 --config jpsi_central --decay muon --decay electron
diff --git a/dvmp/dvmp.sh b/dvmp/dvmp.sh
index 09bdac80..6930fc02 100755
--- a/dvmp/dvmp.sh
+++ b/dvmp/dvmp.sh
@@ -1,13 +1,12 @@
#!/bin/bash
## =============================================================================
-## Run the DVMP benchmarks in 7 steps:
+## Run the DVMP benchmarks in 5 steps:
## 1. Parse the command line and setup environment
-## 2. Build/install detector package
-## 3. Detector simulation through npsim
-## 4. Digitization and reconstruction through Juggler
-## 5. Root-based Physics analyses
-## 6. Finalize
+## 2. Detector simulation through npsim
+## 3. Digitization and reconstruction through Juggler
+## 4. Root-based Physics analyses
+## 5. Finalize
## =============================================================================
## make sure we launch this script from the project root directory
@@ -43,41 +42,18 @@ source config/env.sh
## We also need the following benchmark-specific variables:
##
## - BENCHMARK_TAG: Unique identified for this benchmark process.
-## - DATA_PATH: Place to store our persistent output artifacts.
+## - INPUT_PATH: Path for generator-level input to the benchmarks
+## - TMP_PATH: Path for temporary data (not exported as artifacts)
+## - RESULTS_PATH: Path for benchmark output figures and files
##
## You can read dvmp/env.sh for more in-depth explanations of the variables.
source dvmp/env.sh
## Get a unique file names based on the configuration options
-GEN_FILE=${DATA_PATH}/`util/print_fname.sh \
- --ebeam $EBEAM \
- --pbeam $PBEAM \
- --decay $DECAY \
- --config $CONFIG \
- --type gen`.hepmc
-SIM_FILE=${LOCAL_PREFIX}/`util/print_fname.sh \
- --ebeam $EBEAM \
- --pbeam $PBEAM \
- --decay $DECAY \
- --config $CONFIG \
- --type sim`.root
-REC_FILE=${LOCAL_PREFIX}/`util/print_fname.sh \
- --ebeam $EBEAM \
- --pbeam $PBEAM \
- --decay $DECAY \
- --config $CONFIG \
- --type rec`.root
-PLOT_PREFIX=${DATA_PATH}/`util/print_fname.sh \
- --ebeam $EBEAM \
- --pbeam $PBEAM \
- --decay $DECAY \
- --config $CONFIG \
- --type rec`
-
-## =============================================================================
-## Step 1: Build/install the desired detector package
-# moved to different CI step TODO remove
-#bash util/build_detector.sh
+GEN_FILE=${INPUT_PATH}/gen-${CONFIG}_${DECAY}.hepmc
+SIM_FILE=${TMP_PATH}/sim-${CONFIG}_${DECAY}.root
+REC_FILE=${TMP_PATH}/rec-${CONFIG}_${DECAY}.root
+PLOT_PREFIX=${CONFIG}_${DECAY}
## =============================================================================
## Step 2: Run the simulation
@@ -123,7 +99,7 @@ root -b -q "dvmp/analysis/vm_mass.cxx(\
\"${LEADING}\", \
\"${DECAY}\", \
\"${JUGGLER_DETECTOR}\", \
- \"${PLOT_PREFIX}\")"
+ \"${RESULTS_PATH}/${PLOT_PREFIX}\")"
if [ "$?" -ne "0" ] ; then
@@ -138,11 +114,11 @@ echo "Finalizing DVMP benchmark"
## Copy over reconsturction artifacts as long as we don't have
## too many events
if [ "${JUGGLER_N_EVENTS}" -lt "500" ] ; then
- cp ${REC_FILE} ${DATA_PATH}
+ cp ${REC_FILE} ${RESULTS_PATH}
fi
## cleanup output files
-rm ${REC_FILE} ${SIM_FILE}
+rm -f ${REC_FILE} ${SIM_FILE}
## =============================================================================
## All done!
diff --git a/dvmp/env.sh b/dvmp/env.sh
index db5500d4..9c543b34 100644
--- a/dvmp/env.sh
+++ b/dvmp/env.sh
@@ -5,7 +5,12 @@
## It defines the following additional variables:
##
## - BENCHMARK_TAG: Tag to identify this particular benchmark
-## - DATA_PATH: Data path for all artifact output
+## - INPUT_PATH: Path for generator-level input to the benchmarks
+## - TMP_PATH: Path for temporary data (not exported as artifacts)
+## - RESULTS_PATH: Path for benchmark output figures and files
+##
+## This script assumes that EBEAM and PBEAM are set as part of the
+## calling script (usually as command line argument).
##
## =============================================================================
@@ -16,10 +21,25 @@
export BENCHMARK_TAG="dvmp"
echo "Setting up the local environment for the ${BENCHMARK_TAG^^} benchmarks"
-## Data path for all artifact output
-export DATA_PATH="results/${BENCHMARK_TAG}"
-mkdir -p ${DATA_PATH}
-echo "DATA_PATH: ${DATA_PATH}"
+## Data path for input data (generator-level hepmc file)
+INPUT_PATH="input/${BENCHMARK_TAG}/${EBEAM}on${PBEAM}"
+mkdir -p ${INPUT_PATH}
+export INPUT_PATH=`realpath ${INPUT_PATH}`
+echo "INPUT_PATH: ${INPUT_PATH}"
+
+
+## Data path for temporary data (not exported as artifacts)
+TMP_PATH=${LOCAL_PREFIX}
+mkdir -p ${TMP_PATH}
+export TMP_PATH=`realpath ${TMP_PATH}`
+echo "TMP_PATH: ${TMP_PATH}"
+
+## Data path for benchmark output (plots and reconstructed files
+## if not too big).
+RESULTS_PATH="results/${BENCHMARK_TAG}/${EBEAM}on${PBEAM}"
+mkdir -p ${RESULTS_PATH}
+export RESULTS_PATH=`realpath ${RESULTS_PATH}`
+echo "RESULTS_PATH: ${RESULTS_PATH}"
## =============================================================================
## That's all!
diff --git a/dvmp/gen.sh b/dvmp/gen.sh
index c1998e7e..0eaefda7 100755
--- a/dvmp/gen.sh
+++ b/dvmp/gen.sh
@@ -39,27 +39,23 @@ source config/env.sh
## We also need the following benchmark-specific variables:
##
## - BENCHMARK_TAG: Unique identified for this benchmark process.
-## - DATA_PATH: Place to store our persistent output artifacts.
+## - INPUT_PATH: Path for generator-level input to the benchmarks
+## - TMP_PATH: Path for temporary data (not exported as artifacts)
##
## You can read dvmp/env.sh for more in-depth explanations of the variables.
source dvmp/env.sh
-## Get a unique file name based on the configuration options
-FNAME=`util/print_fname.sh \
- --ebeam $EBEAM \
- --pbeam $PBEAM \
- --decay $DECAY \
- --config $CONFIG \
- --type gen`
+## Get a unique file name prefix based on the configuration options
+GEN_PREFIX=gen-${CONFIG}_${DECAY}
## =============================================================================
## Step 2: Check if we really need to run, or can use the cache.
-if [ -f "${DATA_PATH}/${FNAME}.hepmc" ]; then
- echo "Found cached generator output for $FNAME, no need to rerun"
- exit
+if [ -f "${INPUT_PATH}/${GEN_PREFIX}.hepmc" ]; then
+ echo "Found cached generator output for $GEN_PREFIX, no need to rerun"
+ exit 0
fi
-echo "Generator output for $FNAME not found in cache, need to run generator"
+echo "Generator output for $GEN_PREFIX not found in cache, need to run generator"
## =============================================================================
## Step 3: Create generator configuration file
@@ -77,21 +73,24 @@ fi
## generate the config file for this generator setup
CONFIG_IN="${BENCHMARK_TAG}/generator/${CONFIG}.json.in"
-echo "Creating generator configuration file ${FNAME}.json"
+echo "Creating generator configuration file ${GEN_PREFIX}.json"
if [ ! -f ${CONFIG_IN} ]; then
echo "ERROR: cannot find master config file ${CONFIG_IN}"
exit 1
fi
-sed "s/@TAG@/${FNAME}/" $CONFIG_IN | \
+sed "s/@TAG@/${GEN_PREFIX}/" $CONFIG_IN | \
sed "s/@EBEAM@/${EBEAM}/" | \
sed "s/@PBEAM@/${PBEAM}/" | \
sed "s/@DECAY_LEPTON@/${DECAY_PID}/" | \
- sed "s/@BRANCHING@/${BRANCHING}/" > .local/${FNAME}.json
+ sed "s/@BRANCHING@/${BRANCHING}/" > ${TMP_PATH}/${GEN_PREFIX}.json
## =============================================================================
## Step 4: Run the event generator
echo "Running the generator"
-lager -r ${JUGGLER_RNG_SEED} -c .local/${FNAME}.json -e ${JUGGLER_N_EVENTS} -o ${LOCAL_PREFIX}
+lager -r ${JUGGLER_RNG_SEED} \
+ -c ${TMP_PATH}/${GEN_PREFIX}.json \
+ -e ${JUGGLER_N_EVENTS} \
+ -o ${TMP_PATH}
if [ "$?" -ne "0" ] ; then
echo "ERROR running lAger"
exit 1
@@ -99,12 +98,12 @@ fi
## =============================================================================
## Step 5: Finally, move relevant output into the artifacts directory and clean up
-echo "Moving generator output into ${DATA_PATH}"
-mkdir -p ${DATA_PATH}
+echo "Moving generator output into ${INPUT_PATH}"
for ext in hepmc json log root ; do
- mv ${LOCAL_PREFIX}/*.${FNAME}.*.${ext} ${DATA_PATH}/${FNAME}.${ext}
+ mv ${TMP_PATH}/*.${GEN_PREFIX}.*.${ext} ${INPUT_PATH}/${GEN_PREFIX}.${ext}
done
+## this step only matters for local execution
echo "Cleaning up"
-rm .local/${FNAME}.json
+rm ${TMP_PATH}/${GEN_PREFIX}.json
## All done!
diff --git a/util/print_fname.sh b/util/print_fname.sh
deleted file mode 100755
index 7c7bcce7..00000000
--- a/util/print_fname.sh
+++ /dev/null
@@ -1,102 +0,0 @@
-#!/bin/bash
-
-## =============================================================================
-## Simple script to output a unified file name based on a set of data options
-## Note: this file name will not have an extension, as it is mean to be used as
-## a file name root.
-
-function print_the_help {
- echo "USAGE: print_fname [arguments]"
- echo "REQUIRED ARGUMENTS:"
- echo " --ebeam Electron beam energy"
- echo " --pbeam Ion beam energy"
- echo " --config Generator configuration identifier"
- echo " --type What type of output file is this? (e.g. sim, rec, ...)"
- echo "OPTIONAL ARGUMENTS:"
- echo " --decay Specific decay particle (if applicable)."
- echo " -h,--help Print this message"
- echo ""
- echo " This script will generate a unique file name for the benchmark output."
- exit
-}
-
-## =============================================================================
-## Process the command line arguments
-
-## Required variables
-EBEAM=
-PBEAM=
-CONFIG=
-TYPE=
-
-## Optional variables
-DECAY=
-
-while [ $# -gt 0 ]; do
- key=$1
- case $key in
- --ebeam)
- EBEAM="$2"
- shift
- shift
- ;;
- --pbeam)
- PBEAM="$2"
- shift
- shift
- ;;
- --config)
- CONFIG="$2"
- shift
- shift
- ;;
- --type)
- TYPE="$2"
- shift
- shift
- ;;
- --decay)
- DECAY="$2"
- shift
- shift
- ;;
- -h|--help)
- print_the_help
- exit 0
- ;;
- *)
- echo "Unknown option $key to print_fname, aborting..."
- exit 1
- esac
-done
-
-if [ -z $EBEAM ]; then
- echo "EBEAM not defined in print_fname, aborting..."
- print_the_help
- exit 1
-elif [ -z $PBEAM ]; then
- echo "PBEAM not defined in print_fname, aborting..."
- print_the_help
- exit 1
-elif [ -z $CONFIG ]; then
- echo "CONFIG not defined in print_fname, aborting..."
- print_the_help
- exit 1
-elif [ -z $TYPE ]; then
- echo "TYPE not defined in print_fname, aborting..."
- print_the_help
- exit 1
-fi
-
-
-## =============================================================================
-## Generate a unique identifier based on the configuration arguments
-
-## Add decay info to CONFIG if requested
-if [ ! -z $DECAY ]; then
- CONFIG=${CONFIG}_${DECAY}
-fi
-
-echo "${TYPE}-${CONFIG}-${EBEAM}on${PBEAM}"
-
-## all done.
--
GitLab