Using common_bench scripts.

- The artifacts shouldn't deviate from the default, ie don't set them without including the default values.
- Use `mkdir_local_data_link` to create a symlink to the LOCAL_DATA_PATH
- still need to remove a lot of the old environment config. Should be careful to not not mix benchmark-specific stuff into the envrionment setup.
- Should remove `TEMP_DIR` as a container is inherently temporary. Just use, `LOCAL_DATA_PATH`, or current directory.

.gitlab-ci.yml

 image: eicweb.phy.anl.gov:4567/eic/juggler/juggler:$JUGGLER_TAG
 
 default:
-  tags:
-    - silicon
   before_script:
-    - |
-      if [[ -n "${EIC_DATA_MODEL_VERISON}" ]]; then
-        echo "Triggered from upstream EICD pipeline."
-        git clone -b ${EIC_DATA_MODEL_VERSION} https://eicweb.phy.anl.gov/EIC/eicd.git 
-        mkdir eicd/build && cd eicd/build && cmake ../. -DCMAKE_INSTALL_PREFIX=/usr/local && make -j20 install && cd ../..
-      fi
+    - source .local/bin/env.sh
   artifacts:
     expire_in: 3 days
+    paths:
+      - .local/detector
+      - .local/lib
+      - .local/bin
+      - .local/include
+      - .local/share
+      - results
+      - config
 
 stages:
   - config
   - initialize
+  - generate
   - process
   - collect
   - finish
 
-detector:
+common:setup:
+  stage: config
+  before_script:
+    - git clone https://eicweb.phy.anl.gov/EIC/benchmarks/common_bench.git setup 
+  script:
+    - source setup/bin/env.sh && ./setup/bin/install_common.sh
+
+common:detector:
   stage: initialize
-  needs: []
-  timeout: 1 hours
-  cache:
-    key:
-      files:
-        - config/env.sh
-        - util/build_detector.sh
-      prefix: "$CI_COMMIT_REF_SLUG"
-    paths:
-      - .local/detector
-      - .local/lib
-  artifacts:
-    paths:
-      - .local/detector
-      - .local/lib
+  needs: ["common:setup"]
   script:
-    - ./util/print_env.sh
-    - ./util/build_detector.sh
+    - source .local/bin/env.sh && build_detector.sh
+    - mkdir_local_data_link sim_output
+    - mkdir -p results
+    - mkdir -p config
+    - print_env.sh
 
 include:
   - local: 'benchmarks/dis/config.yml'

benchmarks/dis/analysis-only.sh

@@ -30,9 +30,6 @@ source util/parse_cmd.sh $@
 ## - JUGGLER_DETECTOR:       the detector package we want to use for this benchmark
 ## - DETECTOR_PATH:          full path to the detector definitions
 ##
-## You can ready options/env.sh for more in-depth explanations of the variables
-## and how they can be controlled.
-source options/env.sh
 
 ## We also need the following benchmark-specific variables:
 ##

benchmarks/dis/config.yml

 dis:generate:
-  stage: initialize
-  needs: []
+  stage: generate
+  needs: ["common:detector"]
   timeout: 1 hours
   cache:
     key:
@@ -9,23 +9,16 @@ dis:generate:
       prefix: "$CI_COMMIT_REF_SLUG"
     paths:
       - input/dis
-  artifacts:
-    paths:
-      - input
   script:
     - bash benchmarks/dis/gen.sh --config barrel --ebeam 18 --pbeam 275
 
 dis:process:
   stage: process
-  needs: ["detector", "dis:generate"]
+  needs: ["common:detector", "dis:generate"]
   timeout: 1 hour
   script:
-    - source options/env.sh
     - ./util/compile_analyses.py dis
     - ./benchmarks/dis/dis.sh --config barrel --ebeam 18 --pbeam 275
-  artifacts:
-    paths:
-      - results
   retry:
     max: 2
     when:
@@ -44,7 +37,3 @@ dis:results:
   needs: ["dis:process"]
   script:
     - ./util/collect_tests.py dis
-  artifacts:
-    paths:
-      - results/dis.json
-      - results/dis

benchmarks/dis/dis.sh

@@ -30,9 +30,9 @@ source util/parse_cmd.sh $@
 ## - JUGGLER_DETECTOR:       the detector package we want to use for this benchmark
 ## - DETECTOR_PATH:          full path to the detector definitions
 ##
-## You can ready options/env.sh for more in-depth explanations of the variables
+## defined in common_bench repo
+## You can ready bin/env.sh for more in-depth explanations of the variables
 ## and how they can be controlled.
-source options/env.sh
 
 ## We also need the following benchmark-specific variables:
 ##

benchmarks/dis/gen.sh

@@ -31,9 +31,10 @@ source util/parse_cmd.sh $@
 ## - JUGGLER_N_EVENTS:  Number of events to process
 ## - JUGGLER_RNG_SEED:  Random seed for event generation.
 ##
-## You can read options/env.sh for more in-depth explanations of the variables
+## defined in common_bench repo
+## You can ready bin/env.sh for more in-depth explanations of the variables
 ## and how they can be controlled.
-source options/env.sh
+
 
 ## We also need the following benchmark-specific variables:
 ##

benchmarks/dvcs/config.yml

 dvcs:process:
   stage: process
   timeout: 1 hour
-  needs: ["detector"]
+  needs: ["common:detector"]
   script:
     - bash benchmarks/dvcs/dvcs.sh
-  artifacts:
-    paths:
-      - results
 
 dvcs:results:
   stage: collect
@@ -14,7 +11,3 @@ dvcs:results:
   script:
     - pip install junitparser
       #- python dvcs/scripts/merge_results.py
-  artifacts:
-    paths:
-      - results
-        #  reports:

benchmarks/dvcs/dvcs.sh

@@ -14,10 +14,6 @@ echo "JUGGLER_FILE_NAME_TAG = ${JUGGLER_FILE_NAME_TAG}"
 ## - JUGGLER_INSTALL_PREFIX: Install prefix for Juggler (simu/recon)
 ## - JUGGLER_DETECTOR:       the detector package we want to use for this benchmark
 ## - DETECTOR_PATH:          full path to the detector definitions
-##
-## You can ready options/env.sh for more in-depth explanations of the variables
-## and how they can be controlled.
-source options/env.sh
 
 
 curl -o test_proton_dvcs_eic.hepmc "https://eicweb.phy.anl.gov/api/v4/projects/345/jobs/artifacts/master/raw/data/test_proton_dvcs_eic.hepmc?job=compile"

benchmarks/dvmp/config.yml

 dvmp:generate:
+  needs: ["common:detector"]
   image: eicweb.phy.anl.gov:4567/monte_carlo/lager/lager:unstable
-  stage: initialize
-  needs: []
+  stage: generate
   timeout: 1 hours
-  cache:
-    key:
-      files:
-        - benchmarks/dvmp/generator/jpsi_central.json
-        - benchmarks/dvmp/scripts/jpsi_central-generate.sh
-      prefix: "$CI_COMMIT_REF_SLUG"
-    paths:
-      - input/dvmp
-  artifacts:
-    paths:
-      - input
+  #cache:
+  #  key:
+  #    files:
+  #      - benchmarks/dvmp/generator/jpsi_central.json
+  #      - benchmarks/dvmp/scripts/jpsi_central-generate.sh
+  #    prefix: "$CI_COMMIT_REF_SLUG"
+  #  paths:
+  #    - input/dvmp
   script:
     - ./util/run_many.py ./benchmarks/dvmp/gen.sh 
           -c jpsi_barrel 
@@ -23,20 +20,18 @@ dvmp:generate:
 
 dvmp:process:
   stage: process
-  needs: ["detector", "dvmp:generate"]
+  needs: ["common:detector", "dvmp:generate"]
   timeout: 1 hour
   script:
-    - source options/env.sh
+    - ls -lrth
     - ./util/compile_analyses.py dvmp
+    - ls -lrth
     - ./util/run_many.py ./benchmarks/dvmp/dvmp.sh 
           -c jpsi_barrel 
           -e 10x100 
           --decay muon --decay electron
           --leading jpsi
           --nproc 5
-  artifacts:
-    paths:
-      - results
   retry:
     max: 2
     when:
@@ -55,7 +50,3 @@ dvmp:results:
   needs: ["dvmp:process"]
   script:
     - ./util/collect_tests.py dvmp
-  artifacts:
-    paths:
-      - results/dvmp.json
-      - results/dvmp

benchmarks/dvmp/dvmp.sh

@@ -30,15 +30,6 @@ export REQUIRE_DECAY=1
 export REQUIRE_LEADING=1
 source util/parse_cmd.sh $@
 
-## To run the reconstruction, we need the following global variables:
-## - JUGGLER_INSTALL_PREFIX: Install prefix for Juggler (simu/recon)
-## - JUGGLER_DETECTOR:       the detector package we want to use for this benchmark
-## - DETECTOR_PATH:          full path to the detector definitions
-##
-## You can ready options/env.sh for more in-depth explanations of the variables
-## and how they can be controlled.
-source options/env.sh
-
 ## We also need the following benchmark-specific variables:
 ##
 ## - BENCHMARK_TAG: Unique identified for this benchmark process.
@@ -65,14 +56,17 @@ PLOT_TAG=${CONFIG}_${DECAY}
 ## =============================================================================
 ## Step 2: Run the simulation
 echo "Running Geant4 simulation"
+ls -lrth 
+ls -lrth input
+echo ${TMP_PATH}
+ls -lrth ${TMP_PATH}
 npsim --runType batch \
       --part.minimalKineticEnergy 100*GeV  \
       -v WARNING \
       --numberOfEvents ${JUGGLER_N_EVENTS} \
       --compactFile ${DETECTOR_PATH}/${JUGGLER_DETECTOR}.xml \
       --inputFiles ${GEN_FILE} \
-      --outputFile ${SIM_FILE} \
-  2>&1 > ${SIM_LOG}
+      --outputFile ${SIM_FILE}
 if [ "$?" -ne "0" ] ; then
   echo "ERROR running npsim"
   exit 1

benchmarks/dvmp/env.sh

@@ -25,17 +25,22 @@ echo "Setting up the local environment for the ${BENCHMARK_TAG^^} benchmarks"
 ## Extra beam tag to identify the desired beam configuration
 export BEAM_TAG="${EBEAM}on${PBEAM}"
 
+if [[ ! -d "input" ]] ; then
+  echo " making local link to input "
+  mkdir_local_data_link  input
+fi
+
 ## Data path for input data (generator-level hepmc file)
 INPUT_PATH="input/${BENCHMARK_TAG}/${BEAM_TAG}"
+#export INPUT_PATH=`realpath ${INPUT_PATH}`
 mkdir -p ${INPUT_PATH}
-export INPUT_PATH=`realpath ${INPUT_PATH}`
 echo "INPUT_PATH:             ${INPUT_PATH}"
 
 
 ## Data path for temporary data (not exported as artifacts)
-TMP_PATH=${LOCAL_PREFIX}/tmp/${BEAM_TAG}
+TMP_PATH=${LOCAL_DATA_PATH}/tmp/${BEAM_TAG}
+#export TMP_PATH=`realpath ${TMP_PATH}`
 mkdir -p ${TMP_PATH}
-export TMP_PATH=`realpath ${TMP_PATH}`
 echo "TMP_PATH:               ${TMP_PATH}"
 
 ## Data path for benchmark output (plots and reconstructed files

benchmarks/dvmp/gen.sh

@@ -33,9 +33,8 @@ source util/parse_cmd.sh $@
 ## - JUGGLER_N_EVENTS:  Number of events to process
 ## - JUGGLER_RNG_SEED:  Random seed for event generation.
 ##
-## You can read options/env.sh for more in-depth explanations of the variables
+## You can read common_bench repo for more in-depth explanations of the variables
 ## and how they can be controlled.
-source options/env.sh
 
 ## We also need the following benchmark-specific variables:
 ##

options/env.sh

-#!/bin/bash
-
-## =============================================================================
-## Global configuration variables for the benchmark scripts
-## The script defines the following environment variables that are meant to
-## be overriden by the Gitlab continuous integration (CI)
-##
-##  - JUGGLER_DETECTOR:       detector package to be used for the benchmark
-##  - JUGGLER_N_EVENTS:       #events processed by simulation/reconstruction
-##  - JUGGLER_INSTALL_PREFIX: location where Juggler (digi/recon) is installed
-##  - JUGGLER_N_THREADS:      Number of threads/processes to spawn in parallel
-##  - JUGGLER_RNG_SEED:       Random seed for the RNG
-##
-## It also defines the following additional variables for internal usage
-##  - LOCAL_PREFIX:           prefix for packages installed during the benchmark
-##  - DETECTOR_PREFIX:        prefix for the detector definitions
-##  - DETECTOR_PATH:          actual path with the detector definitions
-##
-## Finally, it makes sure LOCAL_PREFIX and JUGGLER_PREFIX are added to PATH
-## and LD_LIBRARY_PATH
-## =============================================================================
-
-echo "Setting up the Physics Benchmarks environment"
-
-## =============================================================================
-## Default variable definitions, normally these should be set
-## by the CI. In case of local development you may want to change these
-## in case you would like to modify the detector package or
-## number of events to be analyzed during the benchmark
-
-## Detector package to be used during the benchmark process
-if [ ! -n  "${JUGGLER_DETECTOR}" ] ; then 
-  export JUGGLER_DETECTOR="topside"
-fi
-
-if [ ! -n  "${JUGGLER_DETECTOR_VERSION}" ] ; then 
-  export JUGGLER_DETECTOR_VERSION="master"
-fi
-
-
-## Number of events that will be processed by the reconstruction
-if [ ! -n  "${JUGGLER_N_EVENTS}" ] ; then 
-  export JUGGLER_N_EVENTS=100
-fi
-
-## Maximum number of threads or processes a single pipeline should use
-## (this is not enforced, but the different pipeline scripts should use
-##  this to guide the number of parallel processes or threads they 
-##  spawn).
-if [ ! -n "${JUGGLER_N_THREADS}" ]; then
-  export JUGGLER_N_THREADS=10
-fi
-
-## Random seed for event generation, should typically not be changed for
-## reproductability.
-if [ ! -n "${JUGGLER_RNG_SEED}" ]; then
-  export JUGGLER_RNG_SEED=1
-fi
-
-## Install prefix for juggler, needed to locate the Juggler xenv files.
-## Also used by the CI as install prefix for other packages where needed.
-## You should not have to touch this. Note that for local usage a different 
-## prefix structure is automatically used.
-if [ ! -n  "${JUGGLER_INSTALL_PREFIX}" ] ; then 
-  export JUGGLER_INSTALL_PREFIX="/usr/local"
-fi
-## Ensure the juggler prefix is an absolute path
-export JUGGLER_INSTALL_PREFIX=`realpath ${JUGGLER_INSTALL_PREFIX}`
-
-
-## =============================================================================
-## Other utility variables that govern how some of the dependent packages
-## are built and installed. You should not have to change these.
-
-## local prefix to be used for local storage of packages
-## downloaded/installed during the benchmark process
-LOCAL_PREFIX=".local"
-mkdir -p ${LOCAL_PREFIX}
-export LOCAL_PREFIX=`realpath ${LOCAL_PREFIX}`
-
-## detector prefix: prefix for the detector definitions
-export DETECTOR_PREFIX="${LOCAL_PREFIX}/detector"
-mkdir -p ${DETECTOR_PREFIX}
-
-## detector path: actual detector definition path
-export DETECTOR_PATH="${DETECTOR_PREFIX}/${JUGGLER_DETECTOR}"
-
-## build dir for ROOT to put its binaries etc.
-export ROOT_BUILD_DIR=$LOCAL_PREFIX/root_build
-
-echo "JUGGLER_DETECTOR:           ${JUGGLER_DETECTOR}"
-echo "JUGGLER_DETECTOR_VERSION:   ${JUGGLER_DETECTOR_VERSION}"
-echo "JUGGLER_N_EVENTS:           ${JUGGLER_N_EVENTS}"
-echo "JUGGLER_N_THREADS:          ${JUGGLER_N_THREADS}"
-echo "JUGGLER_RNG_SEED:           ${JUGGLER_RNG_SEED}"
-echo "JUGGLER_INSTALL_PREFIX:     ${JUGGLER_INSTALL_PREFIX}"
-echo "LOCAL_PREFIX:               ${LOCAL_PREFIX}"
-echo "DETECTOR_PREFIX:            ${DETECTOR_PREFIX}"
-echo "DETECTOR_PATH:              ${DETECTOR_PATH}"
-echo "ROOT_BUILD_DIR:             ${ROOT_BUILD_DIR}"
-
-## =============================================================================
-## Setup PATH and LD_LIBRARY_PATH to include our prefixes
-echo "Adding JUGGLER_INSTALL_PREFIX and LOCAL_PREFIX to PATH and LD_LIBRARY_PATH"
-export PATH=${JUGGLER_INSTALL_PREFIX}/bin:${LOCAL_PREFIX}/bin:${PATH}
-export LD_LIBRARY_PATH=${JUGGLER_INSTALL_PREFIX}/lib:${LOCAL_PREFIX}/lib:${LD_LIBRARY_PATH}
-
-## =============================================================================
-## That's all!
-echo "Environment setup complete."