Skip to content
Snippets Groups Projects
Commit ff148744 authored by Sylvester Joosten's avatar Sylvester Joosten
Browse files

Polished environment script, and made utility scripts executable from any directory

parent 8da4d540
No related branches found
No related tags found
1 merge request!8First DVMP analysis
This commit is part of merge request !8. Comments created here will be created in the context of that merge request.
...@@ -44,10 +44,5 @@ calorimeters/test/ ...@@ -44,10 +44,5 @@ calorimeters/test/
# output files # output files
results/* results/*
#intermediary files for testign # local runtime files
detector/* .local
detector-build/*
topside
accelerator
*.sif
#!/bin/bash #!/bin/bash
if [[ ! -n "${JUGGLER_DETECTOR}" ]] ; then ## =============================================================================
## Global configuration variables for the benchmark scripts
## The script defines the following environment variables that are meant to
## be overriden by the Gitlab continuous integration (CI)
##
## - JUGGLER_DETECTOR: detector package to be used for the benchmark
## - JUGGLER_N_EVENTS: #events processed by simulation/reconstruction
## - JUGGLER_INSTALL_PREFIX: location where Juggler (digi/recon) is installed
##
## It also defines the following additional variables for internal usage
## - LOCAL_PREFIX: prefix for packages installed during the benchmark
## - DETECTOR_PREFIX: prefix for the detector definitions
## - DETECTOR_PATH: actual path with the detector definitions
##
## Finally, it makes sure LOCAL_PREFIX and JUGGLER_PREFIX are added to PATH
## and LD_LIBRARY_PATH
## =============================================================================
echo "Setting up the Physics Benchmarks environment"
## =============================================================================
## Default variable definitions, normally these should be set
## by the CI. In case of local development you may want to change these
## in case you would like to modify the detector package or
## number of events to be analyzed during the benchmark
## Detector package to be used during the benchmark process
if [ ! -n "${JUGGLER_DETECTOR}" ] ; then
export JUGGLER_DETECTOR="topside" export JUGGLER_DETECTOR="topside"
fi fi
echo "JUGGLER_DETECTOR: ${JUGGLER_DETECTOR}"
if [[ ! -n "${JUGGLER_N_EVENTS}" ]] ; then ## Number of events that will be processed by the reconstruction
if [ ! -n "${JUGGLER_N_EVENTS}" ] ; then
export JUGGLER_N_EVENTS=100 export JUGGLER_N_EVENTS=100
fi fi
echo "JUGGLER_N_EVENTS: ${JUGGLER_N_EVENTS}"
if [[ ! -n "${JUGGLER_INSTALL_PREFIX}" ]] ; then ## Install prefix for juggler, needed to locate the Juggler xenv files.
## Also used by the CI as install prefix for other packages where needed.
## You should not have to touch this. Note that for local usage a different
## prefix structure is automatically used.
if [ ! -n "${JUGGLER_INSTALL_PREFIX}" ] ; then
export JUGGLER_INSTALL_PREFIX="/usr/local" export JUGGLER_INSTALL_PREFIX="/usr/local"
fi fi
## Ensure the juggler prefix is an absolute path
export JUGGLER_INSTALL_PREFIX=`realpath ${JUGGLER_INSTALL_PREFIX}`
echo "JUGGLER_INSTALL_PREFIX: ${JUGGLER_INSTALL_PREFIX}"
# not sure this is needed ## =============================================================================
if [[ ! -n "${DETECTOR_INSTALL_PREFIX}" ]]; then ## Other utility variables that govern how some of the dependent packages
# reuse the custom juggler install prefix for detector if in CI ## are built and installed. You should not have to change these.
if [ ! -n ${CI} ]; then
echo "Setting up detector prefix for CI usage"
export DETECTOR_INSTALL_PREFIX=${JUGGLER_INSTALL_PREFIX}
# for local builds, just use a "detector" directory
else
echo "Setting up detector prefix for local usage"
export DETECTOR_INSTALL_PREFIX="detector"
fi
fi
## ensure absolute paths ## local prefix to be used for local storage of packages
# not sure this is needed either ## downloaded/installed during the benchmark process
export JUGGLER_INSTALL_PREFIX=`realpath ${JUGGLER_INSTALL_PREFIX}` LOCAL_PREFIX=".local"
export DETECTOR_INSTALL_PREFIX=`realpath ${DETECTOR_INSTALL_PREFIX}` mkdir -p ${LOCAL_PREFIX}
export LOCAL_PREFIX=`realpath ${LOCAL_PREFIX}`
echo "LOCAL_PREFIX: ${LOCAL_PREFIX}"
## detector prefix: prefix for the detector definitions
export DETECTOR_PREFIX="${LOCAL_PREFIX}/detector"
mkdir -p ${DETECTOR_PREFIX}
echo "DETECTOR_PREFIX: ${DETECTOR_PREFIX}"
## Ensure Juggler and Detector prefixes are in our paths ## detector path: actual detector definition path
export PATH=${JUGGLER_INSTALL_PREFIX}/bin:${DETECTOR_INSTALL_PREFIX}/bin:${PATH} export DETECTOR_PATH="${DETECTOR_PREFIX}/${JUGGLER_DETECTOR}"
export LD_LIBRARY_PATH=${JUGGLER_INSTALL_PREFIX}/lib:${DETECTOR_INSTALL_PREFIX}/lib:${LD_LIBRARY_PATH} echo "DETECTOR_PATH: ${DETECTOR_PATH}"
## =============================================================================
## Setup PATH and LD_LIBRARY_PATH to include our prefixes
echo "Adding JUGGLER_INSTALL_PREFIX and LOCAL_PREFIX to PATH and LD_LIBRARY_PATH"
export PATH=${JUGGLER_INSTALL_PREFIX}/bin:${LOCAL_PREFIX}/bin:${PATH}
export LD_LIBRARY_PATH=${JUGGLER_INSTALL_PREFIX}/lib:${LOCAL_PREFIX}/lib:${LD_LIBRARY_PATH}
## setup root results artifact path ## =============================================================================
# this should be in the CI File instead ## That's all!
# https://docs.gitlab.com/ee/ci/yaml/README.html#variables echo "Environment setup complete."
# export RESULTS_PATH=`realpath results`
#!/bin/bash #!/bin/bash
## Init the environment ## make sure we launch this script from the project root directory
PROJECT_ROOT="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/..
pushd ${PROJECT_ROOT}
## =============================================================================
## Build and install the JUGGLER_DETECTOR detector package into our local prefix
## This script will only work when executed from the project root directory.
## =============================================================================
## =============================================================================
## Load the environment variables. To build the detector we need the following
## variables:
##
## - JUGGLER_DETECTOR: the detector package we want to use for this benchmark
## - LOCAL_PREFIX: location where local packages should be installed
## - DETECTOR_PREFIX: prefix for the detector definitions
## - DETECTOR_PATH: full path for the detector definitions
## this is the same as ${DETECTOR_PREFIX}/${JUGGLER_DETECTOR}
##
## You can ready config/env.sh for more in-depth explanations of the variables
## and how they can be controlled.
source config/env.sh source config/env.sh
## Build and install the detector plugins. ## =============================================================================
if [[ ! -d ${JUGGLER_DETECTOR} ]]; then ## Step 1: download/update the detector definitions (if needed)
pushd ${DETECTOR_PREFIX}
## We need an up-to-date copy of the detector
if [ ! -d ${JUGGLER_DETECTOR} ]; then
echo "Fetching ${JUGGLER_DETECTOR}"
git clone https://eicweb.phy.anl.gov/EIC/detectors/${JUGGLER_DETECTOR}.git git clone https://eicweb.phy.anl.gov/EIC/detectors/${JUGGLER_DETECTOR}.git
# this might be temporary. There are multiple solutions here but this is the simple pattern for now
# I do not want to use git submodules here -whit
git clone https://eicweb.phy.anl.gov/EIC/detectors/accelerator.git
pushd ${JUGGLER_DETECTOR}
ln -s ../accelerator/eic
popd
else else
echo "Updating ${JUGGLER_DETECTOR}"
pushd ${JUGGLER_DETECTOR} pushd ${JUGGLER_DETECTOR}
git pull --ff-only git pull --ff-only
popd popd
fi
## We also need an up-to-date copy of the accelerator. For now this is done
## manually. Down the road we could maybe automize this with cmake
if [ ! -d accelerator ]; then
echo "Fetching accelerator"
git clone https://eicweb.phy.anl.gov/EIC/detectors/accelerator.git
else
echo "Updating accelerator"
pushd accelerator pushd accelerator
git pull --ff-only git pull --ff-only
popd popd
fi fi
mkdir -p detector-build ## Now symlink the accelerator definition into the detector definition
pushd detector-build echo "Linking accelerator definition into detector definition"
# Always keep the detector directory at the top level. ln -s -f ${DETECTOR_PREFIX}/accelerator/eic ${DETECTOR_PATH}/eic
echo cmake ../${JUGGLER_DETECTOR} -DCMAKE_INSTALL_PREFIX=${DETECTOR_INSTALL_PREFIX} && make -j30 install
cmake ../${JUGGLER_DETECTOR} -DCMAKE_INSTALL_PREFIX=${DETECTOR_INSTALL_PREFIX} && make -j30 install ## =============================================================================
popd ## Step 2: Compile and install the detector definition
echo "Building and installing the ${JUGGLER_DETECTOR} package"
mkdir -p ${DETECTOR_PREFIX}/build
pushd ${DETECTOR_PREFIX}/build
cmake ${DETECTOR_PATH} -DCMAKE_INSTALL_PREFIX=${LOCAL_PREFIX} && make -j30 install
## =============================================================================
## Step 3: That's all!
echo "Detector build/install complete!"
#!/bin/bash #!/bin/bash
## Init the environment ## =============================================================================
source config/env.sh ## Download generator & reconstruction artifacts for one or more physics
## processes.
## =============================================================================
## Generates different configurations from the master configuration ## make sure we launch this script from the project root directory
## for both electron and muon decay channels PROJECT_ROOT="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/..
pushd ${PROJECT_ROOT}
echo "Download generator or reconstruction artifacts for one or more of the physics processes"
PROCS=() PROCS=()
BRANCH="master" BRANCH="master"
...@@ -25,7 +26,6 @@ function print_the_help { ...@@ -25,7 +26,6 @@ function print_the_help {
exit exit
} }
while [ $# -gt 0 ] while [ $# -gt 0 ]
do do
key="$1" key="$1"
...@@ -51,6 +51,8 @@ do ...@@ -51,6 +51,8 @@ do
esac esac
done done
echo "Downloading generator & reconstruction artifacts for one or more physics processes"
if [ ${#PROCS[@]} -eq 0 ]; then if [ ${#PROCS[@]} -eq 0 ]; then
echo "ERROR: need one or more processes: -p <process name> " echo "ERROR: need one or more processes: -p <process name> "
exit 1 exit 1
...@@ -62,9 +64,10 @@ for proc in ${PROCS[@]}; do ...@@ -62,9 +64,10 @@ for proc in ${PROCS[@]}; do
## FIXME this needs to be smarter, probably through more flags... ## FIXME this needs to be smarter, probably through more flags...
wget https://eicweb.phy.anl.gov/EIC/benchmarks/physics_benchmarks/-/jobs/artifacts/$BRANCH/download?job=${proc}:jpsi_central:process -O results_rec.zip wget https://eicweb.phy.anl.gov/EIC/benchmarks/physics_benchmarks/-/jobs/artifacts/$BRANCH/download?job=${proc}:jpsi_central:process -O results_rec.zip
echo "Unpacking artifacts..." echo "Unpacking artifacts..."
unzip -u results_gen.zip unzip -u -o results_gen.zip
unzip -u results_rec.zip unzip -u -o results_rec.zip
echo "Cleaning up..." echo "Cleaning up..."
rm results_???.zip rm results_???.zip
done done
popd
echo "All done" echo "All done"
#!/bin/bash #!/bin/bash
## =============================================================================
## Setup (if needed) and start a development shell environment on Linux or MacOS
## This script will only work when executed from the project root directory.
## =============================================================================
## make sure we launch this script from the project root directory
PROJECT_ROOT="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"/..
pushd ${PROJECT_ROOT}
## We do not load the global development environment here, as this script is
## to be executed on a "naked" system outside of any container
## =============================================================================
## Step 1: Parse command line options
## do we want to force-update the container (only affects Linux)
## default: we do not want to do this.
FORCE_UPDATE=
function print_the_help {
echo "USAGE: ./util/start_dev_shell [-f]"
echo "OPTIONS:"
echo " -f,--force Force-update container (Only affects Linux)"
echo " -h,--help Print this message"
echo ""
echo " This script will setup and launch a containerized development
environment"
exit
}
while [ $# -gt 0 ]
do
key="$1"
case $key in
-f|--force)
FORCE_UPDATE="true"
shift # past value
;;
-h|--help)
print_the_help
shift
;;
*) # unknown option
echo "unknown option $1"
exit 1
;;
esac
done
## get OS type
OS=`uname -s` OS=`uname -s`
if [ "${OS}" = "Linux" ]; then ## =============================================================================
echo "Detected OS: Linux" ## Step 2: Update container and launch shell
if [ ! -f juggler_latest.sif ]; then echo "Launching a containerized development shell"
echo "Need to fetch singularity image"
wget https://eicweb.phy.anl.gov/eic/juggler/-/jobs/artifacts/master/raw/build/juggler.sif?job=docker:singularity -O juggler_latest.sif case ${OS} in
fi Linux)
echo "Launching dev shell (through singularity)..." echo " - Detected OS: Linux"
singularity exec juggler_latest.sif eic-shell ## Use the same prefix as we use for other local packages
elif [ "${OS}" = "Darwin" ]; then export PREFIX=.local/lib
echo "Detector OS: MacOS" if [ ! -f $PREFIX/juggler_latest.sif ] || [ ! -z ${FORCE_UPDATE} ]; then
echo "Syncing docker container" echo " - Fetching singularity image"
docker pull sly2j/juggler:latest mkdir -p $PREFIX
echo "Launching dev shell (through docker)..." wget https://eicweb.phy.anl.gov/eic/juggler/-/jobs/artifacts/master/raw/build/juggler.sif?job=docker:singularity
docker run -v /Users:/Users -w=$PWD -i -t --rm sly2j/juggler:latest eic-shell -O $PREFIX/juggler_latest.sif
else fi
echo "ERROR: dev shell not available for this OS (${OS})" echo " - Using singularity to launch shell..."
fi singularity exec $PREFIX/juggler_latest.sif eic-shell
;;
Darwin)
echo " - Detector OS: MacOS"
echo " - Syncing docker container"
docker pull sly2j/juggler:latest
echo " - Using docker to launch shell..."
docker run -v /Users:/Users -w=$PWD -i -t --rm sly2j/juggler:latest eic-shell
;;
*)
echo "ERROR: dev shell not available for this OS (${OS})"
exit 1
esac
## =============================================================================
## Step 3: All done
echo "Exiting development environment..."
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment