From 637a8f71ee91eeb69263527219b8b4cc3e2f66a3 Mon Sep 17 00:00:00 2001
From: Whitney Armstrong <warmstrong@anl.gov>
Date: Sun, 2 May 2021 23:13:19 +0000
Subject: [PATCH] modified: benchmarks/clustering/config.yml
modified: benchmarks/ecal/config.yml modified:
benchmarks/rich/config.yml modified: benchmarks/tracking/config.yml
---
.gitlab-ci.yml | 45 +++----
benchmarks/clustering/config.yml | 10 +-
benchmarks/ecal/config.yml | 10 +-
benchmarks/ecal/emcal_electrons.sh | 2 +-
benchmarks/ecal/emcal_pi0s.sh | 12 +-
benchmarks/ecal/full_emcal_electrons.sh | 2 +-
.../ecal/options/crystal_calorimeter_reco.py | 7 +-
benchmarks/ecal/run_emcal_barrel_electrons.sh | 2 +-
benchmarks/ecal/run_emcal_barrel_pions.sh | 2 +-
benchmarks/rich/config.yml | 2 +-
benchmarks/tracking/config.yml | 2 +-
options/env.sh | 110 ------------------
12 files changed, 44 insertions(+), 162 deletions(-)
delete mode 100755 options/env.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index beb4aa32..449c20c8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,12 +1,21 @@
image: eicweb.phy.anl.gov:4567/eic/juggler/juggler:$JUGGLER_TAG
default:
+ before_script:
+ - source .local/bin/env.sh
artifacts:
expire_in: 3 days
paths:
- - results/
+ - .local/detector
+ - .local/lib
+ - .local/bin
+ - .local/include
+ - .local/share
+ - results
+ - config
stages:
+ - config
- initialize
- run
- benchmarks1
@@ -15,30 +24,22 @@ stages:
- collect
- finish
-detector:
+common:setup:
+ stage: config
+ before_script:
+ - git clone https://eicweb.phy.anl.gov/EIC/benchmarks/common_bench.git setup
+ script:
+ - source setup/bin/env.sh && ./setup/bin/install_common.sh
+
+common:detector:
stage: initialize
- needs: []
- timeout: 1 hours
- cache:
- key:
- files:
- - config/env.sh
- - util/build_detector.sh
- prefix: "$CI_COMMIT_REF_SLUG"
- paths:
- - .local/detector
- - .local/lib
- artifacts:
- paths:
- - .local/detector
- - .local/lib
- - results
- - config
+ needs: ["common:setup"]
script:
- - ./util/print_env.sh
- - ./util/build_detector.sh
+ - source .local/bin/env.sh && build_detector.sh
+ - mkdir_local_data_link sim_output
- mkdir -p results
- mkdir -p config
+ - print_env.sh
include:
- local: 'benchmarks/ecal/config.yml'
@@ -49,7 +50,7 @@ include:
final_report:
stage: finish
- needs: ["ecal_1_emcal_crystal_electrons", "tracking_central_electrons"]
+ needs: ["ecal_1_emcal_crystal_electrons", "tracking_central_electrons","clustering:results"]
script:
- mkdir -p results/views && cd results/views && bash ../../bin/download_views
- echo "It was a success!"
diff --git a/benchmarks/clustering/config.yml b/benchmarks/clustering/config.yml
index f4223df7..70e88ca4 100644
--- a/benchmarks/clustering/config.yml
+++ b/benchmarks/clustering/config.yml
@@ -1,11 +1,7 @@
clustering:process :
stage: process
timeout: 8 hour
- needs: ["detector"]
- artifacts:
- expire_in: 20 weeks
- paths:
- - results/
+ needs: ["common:detector"]
script:
- bash benchmarks/clustering/barrel_clusters.sh
@@ -15,8 +11,4 @@ clustering:results:
script:
- ls -lrth
#- python dvcs/scripts/merge_results.py
- artifacts:
- paths:
- - results
- # reports:
diff --git a/benchmarks/ecal/config.yml b/benchmarks/ecal/config.yml
index 814f77c7..f4171a0c 100644
--- a/benchmarks/ecal/config.yml
+++ b/benchmarks/ecal/config.yml
@@ -1,7 +1,7 @@
ecal_1_emcal_crystal_electrons:
image: eicweb.phy.anl.gov:4567/eic/juggler/juggler:$JUGGLER_TAG
stage: process
- needs: ["detector"]
+ needs: ["common:detector"]
timeout: 8 hours
artifacts:
expire_in: 20 weeks
@@ -12,7 +12,7 @@ ecal_1_emcal_crystal_electrons:
ecal_2_emcal_crystal_pi0s:
image: eicweb.phy.anl.gov:4567/eic/juggler/juggler:$JUGGLER_TAG
- needs: ["detector"]
+ needs: ["common:detector"]
timeout: 12 hours
artifacts:
expire_in: 20 weeks
@@ -24,7 +24,7 @@ ecal_2_emcal_crystal_pi0s:
ecal_3_emcal_barrel_electrons:
image: eicweb.phy.anl.gov:4567/eic/juggler/juggler:$JUGGLER_TAG
- needs: ["detector"]
+ needs: ["common:detector"]
timeout: 48 hours
artifacts:
expire_in: 20 weeks
@@ -36,7 +36,7 @@ ecal_3_emcal_barrel_electrons:
ecal_4_emcal_barrel_pions:
image: eicweb.phy.anl.gov:4567/eic/juggler/juggler:$JUGGLER_TAG
- needs: ["detector"]
+ needs: ["common:detector"]
timeout: 48 hours
artifacts:
expire_in: 20 weeks
@@ -49,7 +49,7 @@ ecal_4_emcal_barrel_pions:
## Not fully yet...
full_emcal_barrel_electrons:
image: eicweb.phy.anl.gov:4567/eic/juggler/juggler:$JUGGLER_TAG
- needs: ["detector"]
+ needs: ["common:detector"]
timeout: 48 hours
artifacts:
expire_in: 20 weeks
diff --git a/benchmarks/ecal/emcal_electrons.sh b/benchmarks/ecal/emcal_electrons.sh
index 717b68b5..de2e3bbc 100644
--- a/benchmarks/ecal/emcal_electrons.sh
+++ b/benchmarks/ecal/emcal_electrons.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-./util/print_env.sh
+print_env.sh
## To run the reconstruction, we need the following global variables:
## - JUGGLER_INSTALL_PREFIX: Install prefix for Juggler (simu/recon)
diff --git a/benchmarks/ecal/emcal_pi0s.sh b/benchmarks/ecal/emcal_pi0s.sh
index 6dac4a0d..fd09b8d8 100644
--- a/benchmarks/ecal/emcal_pi0s.sh
+++ b/benchmarks/ecal/emcal_pi0s.sh
@@ -1,7 +1,7 @@
#!/bin/bash
# Based on emcal_electrons.sh script
-./util/print_env.sh
+print_env.sh
## To run the reconstruction, we need the following global variables:
## - JUGGLER_INSTALL_PREFIX: Install prefix for Juggler (simu/recon)
@@ -9,10 +9,8 @@
## - JUGGLER_DETECTOR_VERSION: the detector package we want to use for this benchmark
## - DETECTOR_PATH: full path to the detector definitions
##
-## You can ready options/env.sh for more in-depth explanations of the variables
+## You can read .local/bin/env.sh for more in-depth explanations of the variables
## and how they can be controlled.
-source options/env.sh
-export JUGGLER_DETECTOR_PATH=${DETECTOR_PATH}
if [[ ! -n "${JUGGLER_N_EVENTS}" ]] ; then
export JUGGLER_N_EVENTS=100
@@ -42,10 +40,10 @@ fi
echo "CHECK POINT FOR GEANT4 SIMULATION"
-echo "JUGGLER_N_EVENTS = ${JUGGLER_N_EVENTS}"
-echo "JUGGLER_DETECTOR = ${JUGGLER_DETECTOR}"
+echo "JUGGLER_N_EVENTS = ${JUGGLER_N_EVENTS}"
+echo "JUGGLER_DETECTOR = ${JUGGLER_DETECTOR}"
echo "JUGGLER_FILE_NAME_TAG = ${JUGGLER_FILE_NAME_TAG}"
-echo "JUGGLER_SIM_FILE = ${JUGGLER_SIM_FILE}"
+echo "JUGGLER_SIM_FILE = ${JUGGLER_SIM_FILE}"
# run geant4 simulations
npsim --runType batch \
--part.minimalKineticEnergy 1000*GeV \
diff --git a/benchmarks/ecal/full_emcal_electrons.sh b/benchmarks/ecal/full_emcal_electrons.sh
index 68ad8688..e30364ab 100644
--- a/benchmarks/ecal/full_emcal_electrons.sh
+++ b/benchmarks/ecal/full_emcal_electrons.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-./util/print_env.sh
+print_env.sh
## To run the reconstruction, we need the following global variables:
## - JUGGLER_INSTALL_PREFIX: Install prefix for Juggler (simu/recon)
diff --git a/benchmarks/ecal/options/crystal_calorimeter_reco.py b/benchmarks/ecal/options/crystal_calorimeter_reco.py
index 218ccc84..7892b0fb 100644
--- a/benchmarks/ecal/options/crystal_calorimeter_reco.py
+++ b/benchmarks/ecal/options/crystal_calorimeter_reco.py
@@ -9,8 +9,9 @@ detector_name = "topside"
if "JUGGLER_DETECTOR" in os.environ :
detector_name = str(os.environ["JUGGLER_DETECTOR"])
-if "JUGGLER_DETECTOR_PATH" in os.environ :
- detector_name = str(os.environ["JUGGLER_DETECTOR_PATH"])+"/"+detector_name
+detector_path = "topside"
+if "DETECTOR_PATH" in os.environ :
+ detector_path = str(os.environ["DETECTOR_PATH"])
# todo add checks
input_sim_file = "jug_input.root"
@@ -30,7 +31,7 @@ n_events = 100
if "JUGGLER_N_EVENTS" in os.environ :
n_events = str(os.environ["JUGGLER_N_EVENTS"])
-geo_service = GeoSvc("GeoSvc", detectors=["{}.xml".format(detector_name)])
+geo_service = GeoSvc("GeoSvc", detectors=["{}/{}.xml".format(detector_path,detector_name)])
podioevent = EICDataSvc("EventDataSvc", inputs=[input_sim_file], OutputLevel=DEBUG)
from Configurables import PodioInput
diff --git a/benchmarks/ecal/run_emcal_barrel_electrons.sh b/benchmarks/ecal/run_emcal_barrel_electrons.sh
index 95b7e270..c4482b21 100755
--- a/benchmarks/ecal/run_emcal_barrel_electrons.sh
+++ b/benchmarks/ecal/run_emcal_barrel_electrons.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-./util/print_env.sh
+print_env.sh
## To run the reconstruction, we need the following global variables:
## - JUGGLER_INSTALL_PREFIX: Install prefix for Juggler (simu/recon)
diff --git a/benchmarks/ecal/run_emcal_barrel_pions.sh b/benchmarks/ecal/run_emcal_barrel_pions.sh
index 05d3db92..da409133 100755
--- a/benchmarks/ecal/run_emcal_barrel_pions.sh
+++ b/benchmarks/ecal/run_emcal_barrel_pions.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-./util/print_env.sh
+print_env.sh
## To run the reconstruction, we need the following global variables:
## - JUGGLER_INSTALL_PREFIX: Install prefix for Juggler (simu/recon)
diff --git a/benchmarks/rich/config.yml b/benchmarks/rich/config.yml
index e76e89d0..dae1855d 100644
--- a/benchmarks/rich/config.yml
+++ b/benchmarks/rich/config.yml
@@ -1,6 +1,6 @@
rich_job_x:
image: eicweb.phy.anl.gov:4567/eic/juggler/juggler:$JUGGLER_TAG
- needs: ["detector"]
+ needs: ["common:detector"]
timeout: 24 hours
artifacts:
expire_in: 20 weeks
diff --git a/benchmarks/tracking/config.yml b/benchmarks/tracking/config.yml
index f8abdc0e..0d63dc39 100644
--- a/benchmarks/tracking/config.yml
+++ b/benchmarks/tracking/config.yml
@@ -1,6 +1,6 @@
tracking_central_electrons:
image: eicweb.phy.anl.gov:4567/eic/juggler/juggler:$JUGGLER_TAG
- needs: ["detector"]
+ needs: ["common:detector"]
timeout: 24 hours
artifacts:
expire_in: 20 weeks
diff --git a/options/env.sh b/options/env.sh
deleted file mode 100755
index c65d9f20..00000000
--- a/options/env.sh
+++ /dev/null
@@ -1,110 +0,0 @@
-#!/bin/bash
-
-## =============================================================================
-## Global configuration variables for the benchmark scripts
-## The script defines the following environment variables that are meant to
-## be overriden by the Gitlab continuous integration (CI)
-##
-## - JUGGLER_DETECTOR: detector package to be used for the benchmark
-## - JUGGLER_N_EVENTS: #events processed by simulation/reconstruction
-## - JUGGLER_INSTALL_PREFIX: location where Juggler (digi/recon) is installed
-## - JUGGLER_N_THREADS: Number of threads/processes to spawn in parallel
-## - JUGGLER_RNG_SEED: Random seed for the RNG
-##
-## It also defines the following additional variables for internal usage
-## - LOCAL_PREFIX: prefix for packages installed during the benchmark
-## - DETECTOR_PREFIX: prefix for the detector definitions
-## - DETECTOR_PATH: actual path with the detector definitions
-##
-## Finally, it makes sure LOCAL_PREFIX and JUGGLER_PREFIX are added to PATH
-## and LD_LIBRARY_PATH
-## =============================================================================
-
-echo "Setting up the Physics Benchmarks environment"
-
-## =============================================================================
-## Default variable definitions, normally these should be set
-## by the CI. In case of local development you may want to change these
-## in case you would like to modify the detector package or
-## number of events to be analyzed during the benchmark
-
-## Detector package to be used during the benchmark process
-if [ ! -n "${JUGGLER_DETECTOR}" ] ; then
- export JUGGLER_DETECTOR="topside"
-fi
-
-if [ ! -n "${JUGGLER_DETECTOR_VERSION}" ] ; then
- export JUGGLER_DETECTOR_VERSION="master"
-fi
-
-
-## Number of events that will be processed by the reconstruction
-if [ ! -n "${JUGGLER_N_EVENTS}" ] ; then
- export JUGGLER_N_EVENTS=100
-fi
-
-## Maximum number of threads or processes a single pipeline should use
-## (this is not enforced, but the different pipeline scripts should use
-## this to guide the number of parallel processes or threads they
-## spawn).
-if [ ! -n "${JUGGLER_N_THREADS}" ]; then
- export JUGGLER_N_THREADS=10
-fi
-
-## Random seed for event generation, should typically not be changed for
-## reproductability.
-if [ ! -n "${JUGGLER_RNG_SEED}" ]; then
- export JUGGLER_RNG_SEED=1
-fi
-
-## Install prefix for juggler, needed to locate the Juggler xenv files.
-## Also used by the CI as install prefix for other packages where needed.
-## You should not have to touch this. Note that for local usage a different
-## prefix structure is automatically used.
-if [ ! -n "${JUGGLER_INSTALL_PREFIX}" ] ; then
- export JUGGLER_INSTALL_PREFIX="/usr/local"
-fi
-## Ensure the juggler prefix is an absolute path
-export JUGGLER_INSTALL_PREFIX=`realpath ${JUGGLER_INSTALL_PREFIX}`
-
-
-## =============================================================================
-## Other utility variables that govern how some of the dependent packages
-## are built and installed. You should not have to change these.
-
-## local prefix to be used for local storage of packages
-## downloaded/installed during the benchmark process
-LOCAL_PREFIX=".local"
-mkdir -p ${LOCAL_PREFIX}
-export LOCAL_PREFIX=`realpath ${LOCAL_PREFIX}`
-
-## detector prefix: prefix for the detector definitions
-export DETECTOR_PREFIX="${LOCAL_PREFIX}/detector"
-mkdir -p ${DETECTOR_PREFIX}
-
-## detector path: actual detector definition path
-export DETECTOR_PATH="${DETECTOR_PREFIX}/${JUGGLER_DETECTOR}"
-
-## build dir for ROOT to put its binaries etc.
-export ROOT_BUILD_DIR=$LOCAL_PREFIX/root_build
-
-echo "JUGGLER_DETECTOR: ${JUGGLER_DETECTOR}"
-echo "JUGGLER_DETECTOR_VERSION: ${JUGGLER_DETECTOR_VERSION}"
-echo "JUGGLER_N_EVENTS: ${JUGGLER_N_EVENTS}"
-echo "JUGGLER_N_THREADS: ${JUGGLER_N_THREADS}"
-echo "JUGGLER_RNG_SEED: ${JUGGLER_RNG_SEED}"
-echo "JUGGLER_INSTALL_PREFIX: ${JUGGLER_INSTALL_PREFIX}"
-echo "LOCAL_PREFIX: ${LOCAL_PREFIX}"
-echo "DETECTOR_PREFIX: ${DETECTOR_PREFIX}"
-echo "DETECTOR_PATH: ${DETECTOR_PATH}"
-echo "ROOT_BUILD_DIR: ${ROOT_BUILD_DIR}"
-
-## =============================================================================
-## Setup PATH and LD_LIBRARY_PATH to include our prefixes
-echo "Adding JUGGLER_INSTALL_PREFIX and LOCAL_PREFIX to PATH and LD_LIBRARY_PATH"
-export PATH=${JUGGLER_INSTALL_PREFIX}/bin:${LOCAL_PREFIX}/bin:${PATH}
-export LD_LIBRARY_PATH=${JUGGLER_INSTALL_PREFIX}/lib:${LOCAL_PREFIX}/lib:${LD_LIBRARY_PATH}
-
-## =============================================================================
-## That's all!
-echo "Environment setup complete."
--
GitLab