diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d3f5a613844e52040e53e6916788ddd5807548e7..48ca285fda4710351d839b82536e864d44abe7ef 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,45 +1,43 @@ image: eicweb.phy.anl.gov:4567/eic/juggler/juggler:$JUGGLER_TAG default: - tags: - - silicon before_script: - - | - if [[ -n "${EIC_DATA_MODEL_VERISON}" ]]; then - echo "Triggered from upstream EICD pipeline." - git clone -b ${EIC_DATA_MODEL_VERSION} https://eicweb.phy.anl.gov/EIC/eicd.git - mkdir eicd/build && cd eicd/build && cmake ../. -DCMAKE_INSTALL_PREFIX=/usr/local && make -j20 install && cd ../.. - fi + - source .local/bin/env.sh artifacts: expire_in: 3 days + paths: + - .local/detector + - .local/lib + - .local/bin + - .local/include + - .local/share + - results + - config stages: - config - initialize + - generate - process - collect - finish -detector: +common:setup: + stage: config + before_script: + - git clone https://eicweb.phy.anl.gov/EIC/benchmarks/common_bench.git setup + script: + - source setup/bin/env.sh && ./setup/bin/install_common.sh + +common:detector: stage: initialize - needs: [] - timeout: 1 hours - cache: - key: - files: - - config/env.sh - - util/build_detector.sh - prefix: "$CI_COMMIT_REF_SLUG" - paths: - - .local/detector - - .local/lib - artifacts: - paths: - - .local/detector - - .local/lib + needs: ["common:setup"] script: - - ./util/print_env.sh - - ./util/build_detector.sh + - source .local/bin/env.sh && build_detector.sh + - mkdir_local_data_link sim_output + - mkdir -p results + - mkdir -p config + - print_env.sh include: - local: 'benchmarks/dis/config.yml' diff --git a/benchmarks/dis/analysis-only.sh b/benchmarks/dis/analysis-only.sh index 89a455890c768366d7cbfaeb65f7c46af765e5c1..a8f023d479cbfb24fa88f9f8db01b2ee8ae724be 100755 --- a/benchmarks/dis/analysis-only.sh +++ b/benchmarks/dis/analysis-only.sh @@ -30,9 +30,6 @@ source util/parse_cmd.sh $@ ## - JUGGLER_DETECTOR: the detector package we want to use for this benchmark ## - DETECTOR_PATH: full path to the detector definitions ## -## You can ready options/env.sh for more in-depth explanations of the variables -## and how they can be controlled. -source options/env.sh ## We also need the following benchmark-specific variables: ## diff --git a/benchmarks/dis/config.yml b/benchmarks/dis/config.yml index 6331fd56f4bde5000a7108c53c1296ccf7df4831..cdc65bada797dc914e8a46b81113fda7be47f836 100644 --- a/benchmarks/dis/config.yml +++ b/benchmarks/dis/config.yml @@ -1,6 +1,6 @@ dis:generate: - stage: initialize - needs: [] + stage: generate + needs: ["common:detector"] timeout: 1 hours cache: key: @@ -9,23 +9,16 @@ dis:generate: prefix: "$CI_COMMIT_REF_SLUG" paths: - input/dis - artifacts: - paths: - - input script: - bash benchmarks/dis/gen.sh --config barrel --ebeam 18 --pbeam 275 dis:process: stage: process - needs: ["detector", "dis:generate"] + needs: ["common:detector", "dis:generate"] timeout: 1 hour script: - - source options/env.sh - ./util/compile_analyses.py dis - ./benchmarks/dis/dis.sh --config barrel --ebeam 18 --pbeam 275 - artifacts: - paths: - - results retry: max: 2 when: @@ -44,7 +37,3 @@ dis:results: needs: ["dis:process"] script: - ./util/collect_tests.py dis - artifacts: - paths: - - results/dis.json - - results/dis diff --git a/benchmarks/dis/dis.sh b/benchmarks/dis/dis.sh index dd3d24f806f6790670dd66249413b1794cc596b4..b0a5c21e67dafdb93d73d60162aae9fe703a7348 100755 --- a/benchmarks/dis/dis.sh +++ b/benchmarks/dis/dis.sh @@ -30,9 +30,9 @@ source util/parse_cmd.sh $@ ## - JUGGLER_DETECTOR: the detector package we want to use for this benchmark ## - DETECTOR_PATH: full path to the detector definitions ## -## You can ready options/env.sh for more in-depth explanations of the variables +## defined in common_bench repo +## You can ready bin/env.sh for more in-depth explanations of the variables ## and how they can be controlled. -source options/env.sh ## We also need the following benchmark-specific variables: ## diff --git a/benchmarks/dis/gen.sh b/benchmarks/dis/gen.sh index 173fd4ed7ac890c78c19b40b6f54f88c72ec6608..a6f3dbe5783c5f4d0df578a474d40db811bbe102 100755 --- a/benchmarks/dis/gen.sh +++ b/benchmarks/dis/gen.sh @@ -31,9 +31,10 @@ source util/parse_cmd.sh $@ ## - JUGGLER_N_EVENTS: Number of events to process ## - JUGGLER_RNG_SEED: Random seed for event generation. ## -## You can read options/env.sh for more in-depth explanations of the variables +## defined in common_bench repo +## You can ready bin/env.sh for more in-depth explanations of the variables ## and how they can be controlled. -source options/env.sh + ## We also need the following benchmark-specific variables: ## diff --git a/benchmarks/dvcs/config.yml b/benchmarks/dvcs/config.yml index d2c1a44710489b1e0e2af7ba590e70e2545b5a7a..9b0f56cd113b956718a7244df21e04c019dc9f3e 100644 --- a/benchmarks/dvcs/config.yml +++ b/benchmarks/dvcs/config.yml @@ -1,12 +1,9 @@ dvcs:process: stage: process timeout: 1 hour - needs: ["detector"] + needs: ["common:detector"] script: - bash benchmarks/dvcs/dvcs.sh - artifacts: - paths: - - results dvcs:results: stage: collect @@ -14,7 +11,3 @@ dvcs:results: script: - pip install junitparser #- python dvcs/scripts/merge_results.py - artifacts: - paths: - - results - # reports: diff --git a/benchmarks/dvcs/dvcs.sh b/benchmarks/dvcs/dvcs.sh index 35bbcafeddfe3e2e218ba286f209464cf8a297d7..8e7e3fbd7a8243c273eb0d89fe335ed731c0f096 100644 --- a/benchmarks/dvcs/dvcs.sh +++ b/benchmarks/dvcs/dvcs.sh @@ -14,10 +14,6 @@ echo "JUGGLER_FILE_NAME_TAG = ${JUGGLER_FILE_NAME_TAG}" ## - JUGGLER_INSTALL_PREFIX: Install prefix for Juggler (simu/recon) ## - JUGGLER_DETECTOR: the detector package we want to use for this benchmark ## - DETECTOR_PATH: full path to the detector definitions -## -## You can ready options/env.sh for more in-depth explanations of the variables -## and how they can be controlled. -source options/env.sh curl -o test_proton_dvcs_eic.hepmc "https://eicweb.phy.anl.gov/api/v4/projects/345/jobs/artifacts/master/raw/data/test_proton_dvcs_eic.hepmc?job=compile" diff --git a/benchmarks/dvmp/config.yml b/benchmarks/dvmp/config.yml index 941bafd42481774bee08e8d6017051b5777926ce..e260e8fb6433909c716f8e069613e1cc9285f13c 100644 --- a/benchmarks/dvmp/config.yml +++ b/benchmarks/dvmp/config.yml @@ -1,19 +1,16 @@ dvmp:generate: + needs: ["common:detector"] image: eicweb.phy.anl.gov:4567/monte_carlo/lager/lager:unstable - stage: initialize - needs: [] + stage: generate timeout: 1 hours - cache: - key: - files: - - benchmarks/dvmp/generator/jpsi_central.json - - benchmarks/dvmp/scripts/jpsi_central-generate.sh - prefix: "$CI_COMMIT_REF_SLUG" - paths: - - input/dvmp - artifacts: - paths: - - input + #cache: + # key: + # files: + # - benchmarks/dvmp/generator/jpsi_central.json + # - benchmarks/dvmp/scripts/jpsi_central-generate.sh + # prefix: "$CI_COMMIT_REF_SLUG" + # paths: + # - input/dvmp script: - ./util/run_many.py ./benchmarks/dvmp/gen.sh -c jpsi_barrel @@ -23,20 +20,18 @@ dvmp:generate: dvmp:process: stage: process - needs: ["detector", "dvmp:generate"] + needs: ["common:detector", "dvmp:generate"] timeout: 1 hour script: - - source options/env.sh + - ls -lrth - ./util/compile_analyses.py dvmp + - ls -lrth - ./util/run_many.py ./benchmarks/dvmp/dvmp.sh -c jpsi_barrel -e 10x100 --decay muon --decay electron --leading jpsi --nproc 5 - artifacts: - paths: - - results retry: max: 2 when: @@ -55,7 +50,3 @@ dvmp:results: needs: ["dvmp:process"] script: - ./util/collect_tests.py dvmp - artifacts: - paths: - - results/dvmp.json - - results/dvmp diff --git a/benchmarks/dvmp/dvmp.sh b/benchmarks/dvmp/dvmp.sh index 1746a8600c1a3e20a8968b43db51884d2311e2fc..c1cd683bf22a66398e9e40906d6e97b1494ed88e 100755 --- a/benchmarks/dvmp/dvmp.sh +++ b/benchmarks/dvmp/dvmp.sh @@ -30,15 +30,6 @@ export REQUIRE_DECAY=1 export REQUIRE_LEADING=1 source util/parse_cmd.sh $@ -## To run the reconstruction, we need the following global variables: -## - JUGGLER_INSTALL_PREFIX: Install prefix for Juggler (simu/recon) -## - JUGGLER_DETECTOR: the detector package we want to use for this benchmark -## - DETECTOR_PATH: full path to the detector definitions -## -## You can ready options/env.sh for more in-depth explanations of the variables -## and how they can be controlled. -source options/env.sh - ## We also need the following benchmark-specific variables: ## ## - BENCHMARK_TAG: Unique identified for this benchmark process. @@ -65,14 +56,17 @@ PLOT_TAG=${CONFIG}_${DECAY} ## ============================================================================= ## Step 2: Run the simulation echo "Running Geant4 simulation" +ls -lrth +ls -lrth input +echo ${TMP_PATH} +ls -lrth ${TMP_PATH} npsim --runType batch \ --part.minimalKineticEnergy 100*GeV \ -v WARNING \ --numberOfEvents ${JUGGLER_N_EVENTS} \ --compactFile ${DETECTOR_PATH}/${JUGGLER_DETECTOR}.xml \ --inputFiles ${GEN_FILE} \ - --outputFile ${SIM_FILE} \ - 2>&1 > ${SIM_LOG} + --outputFile ${SIM_FILE} if [ "$?" -ne "0" ] ; then echo "ERROR running npsim" exit 1 diff --git a/benchmarks/dvmp/env.sh b/benchmarks/dvmp/env.sh index 17ef386b0a67b964e75470576f5374566f471621..ac51b1746c8b915b3d8927f79fa3184880b0ae11 100644 --- a/benchmarks/dvmp/env.sh +++ b/benchmarks/dvmp/env.sh @@ -25,17 +25,22 @@ echo "Setting up the local environment for the ${BENCHMARK_TAG^^} benchmarks" ## Extra beam tag to identify the desired beam configuration export BEAM_TAG="${EBEAM}on${PBEAM}" +if [[ ! -d "input" ]] ; then + echo " making local link to input " + mkdir_local_data_link input +fi + ## Data path for input data (generator-level hepmc file) INPUT_PATH="input/${BENCHMARK_TAG}/${BEAM_TAG}" +#export INPUT_PATH=`realpath ${INPUT_PATH}` mkdir -p ${INPUT_PATH} -export INPUT_PATH=`realpath ${INPUT_PATH}` echo "INPUT_PATH: ${INPUT_PATH}" ## Data path for temporary data (not exported as artifacts) -TMP_PATH=${LOCAL_PREFIX}/tmp/${BEAM_TAG} +TMP_PATH=${LOCAL_DATA_PATH}/tmp/${BEAM_TAG} +#export TMP_PATH=`realpath ${TMP_PATH}` mkdir -p ${TMP_PATH} -export TMP_PATH=`realpath ${TMP_PATH}` echo "TMP_PATH: ${TMP_PATH}" ## Data path for benchmark output (plots and reconstructed files diff --git a/benchmarks/dvmp/gen.sh b/benchmarks/dvmp/gen.sh index 8bef0a116f14014869d134c38720e7717ad5e49d..3dd319ad7e22943f797ca31056285de0dbb70e41 100755 --- a/benchmarks/dvmp/gen.sh +++ b/benchmarks/dvmp/gen.sh @@ -33,9 +33,8 @@ source util/parse_cmd.sh $@ ## - JUGGLER_N_EVENTS: Number of events to process ## - JUGGLER_RNG_SEED: Random seed for event generation. ## -## You can read options/env.sh for more in-depth explanations of the variables +## You can read common_bench repo for more in-depth explanations of the variables ## and how they can be controlled. -source options/env.sh ## We also need the following benchmark-specific variables: ## diff --git a/options/env.sh b/options/env.sh deleted file mode 100755 index c65d9f20e0fae381156d00fc3deae6d7dbed7f92..0000000000000000000000000000000000000000 --- a/options/env.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash - -## ============================================================================= -## Global configuration variables for the benchmark scripts -## The script defines the following environment variables that are meant to -## be overriden by the Gitlab continuous integration (CI) -## -## - JUGGLER_DETECTOR: detector package to be used for the benchmark -## - JUGGLER_N_EVENTS: #events processed by simulation/reconstruction -## - JUGGLER_INSTALL_PREFIX: location where Juggler (digi/recon) is installed -## - JUGGLER_N_THREADS: Number of threads/processes to spawn in parallel -## - JUGGLER_RNG_SEED: Random seed for the RNG -## -## It also defines the following additional variables for internal usage -## - LOCAL_PREFIX: prefix for packages installed during the benchmark -## - DETECTOR_PREFIX: prefix for the detector definitions -## - DETECTOR_PATH: actual path with the detector definitions -## -## Finally, it makes sure LOCAL_PREFIX and JUGGLER_PREFIX are added to PATH -## and LD_LIBRARY_PATH -## ============================================================================= - -echo "Setting up the Physics Benchmarks environment" - -## ============================================================================= -## Default variable definitions, normally these should be set -## by the CI. In case of local development you may want to change these -## in case you would like to modify the detector package or -## number of events to be analyzed during the benchmark - -## Detector package to be used during the benchmark process -if [ ! -n "${JUGGLER_DETECTOR}" ] ; then - export JUGGLER_DETECTOR="topside" -fi - -if [ ! -n "${JUGGLER_DETECTOR_VERSION}" ] ; then - export JUGGLER_DETECTOR_VERSION="master" -fi - - -## Number of events that will be processed by the reconstruction -if [ ! -n "${JUGGLER_N_EVENTS}" ] ; then - export JUGGLER_N_EVENTS=100 -fi - -## Maximum number of threads or processes a single pipeline should use -## (this is not enforced, but the different pipeline scripts should use -## this to guide the number of parallel processes or threads they -## spawn). -if [ ! -n "${JUGGLER_N_THREADS}" ]; then - export JUGGLER_N_THREADS=10 -fi - -## Random seed for event generation, should typically not be changed for -## reproductability. -if [ ! -n "${JUGGLER_RNG_SEED}" ]; then - export JUGGLER_RNG_SEED=1 -fi - -## Install prefix for juggler, needed to locate the Juggler xenv files. -## Also used by the CI as install prefix for other packages where needed. -## You should not have to touch this. Note that for local usage a different -## prefix structure is automatically used. -if [ ! -n "${JUGGLER_INSTALL_PREFIX}" ] ; then - export JUGGLER_INSTALL_PREFIX="/usr/local" -fi -## Ensure the juggler prefix is an absolute path -export JUGGLER_INSTALL_PREFIX=`realpath ${JUGGLER_INSTALL_PREFIX}` - - -## ============================================================================= -## Other utility variables that govern how some of the dependent packages -## are built and installed. You should not have to change these. - -## local prefix to be used for local storage of packages -## downloaded/installed during the benchmark process -LOCAL_PREFIX=".local" -mkdir -p ${LOCAL_PREFIX} -export LOCAL_PREFIX=`realpath ${LOCAL_PREFIX}` - -## detector prefix: prefix for the detector definitions -export DETECTOR_PREFIX="${LOCAL_PREFIX}/detector" -mkdir -p ${DETECTOR_PREFIX} - -## detector path: actual detector definition path -export DETECTOR_PATH="${DETECTOR_PREFIX}/${JUGGLER_DETECTOR}" - -## build dir for ROOT to put its binaries etc. -export ROOT_BUILD_DIR=$LOCAL_PREFIX/root_build - -echo "JUGGLER_DETECTOR: ${JUGGLER_DETECTOR}" -echo "JUGGLER_DETECTOR_VERSION: ${JUGGLER_DETECTOR_VERSION}" -echo "JUGGLER_N_EVENTS: ${JUGGLER_N_EVENTS}" -echo "JUGGLER_N_THREADS: ${JUGGLER_N_THREADS}" -echo "JUGGLER_RNG_SEED: ${JUGGLER_RNG_SEED}" -echo "JUGGLER_INSTALL_PREFIX: ${JUGGLER_INSTALL_PREFIX}" -echo "LOCAL_PREFIX: ${LOCAL_PREFIX}" -echo "DETECTOR_PREFIX: ${DETECTOR_PREFIX}" -echo "DETECTOR_PATH: ${DETECTOR_PATH}" -echo "ROOT_BUILD_DIR: ${ROOT_BUILD_DIR}" - -## ============================================================================= -## Setup PATH and LD_LIBRARY_PATH to include our prefixes -echo "Adding JUGGLER_INSTALL_PREFIX and LOCAL_PREFIX to PATH and LD_LIBRARY_PATH" -export PATH=${JUGGLER_INSTALL_PREFIX}/bin:${LOCAL_PREFIX}/bin:${PATH} -export LD_LIBRARY_PATH=${JUGGLER_INSTALL_PREFIX}/lib:${LOCAL_PREFIX}/lib:${LD_LIBRARY_PATH} - -## ============================================================================= -## That's all! -echo "Environment setup complete."