Refactor CI setup

.gitlab-ci.yml

-image: eicweb.phy.anl.gov:4567/containers/eic_container/eic:latest
+image: eicweb.phy.anl.gov:4567/eic/juggler/juggler:latest
 
 default:
+  tags:
+    - silicon
   artifacts:
-    expire_in: 10 mins
+    expire_in: 3 days
     paths:
-      - config/
+      - detector/
       - results/
-      - sim_output/
-      - data
-        #    exclude:
-        #      - .git/
-        #      - datasets/.git/
-        #before_script:
-        #  - git clone https://eicweb.phy.anl.gov/EIC/NPDet.git
-        #      #    - cd NPDet/build && cmake ../. -DCMAKE_INSTALL_PREFIX=/usr/local && make -j10 && make install
-        #      #    - cd ../.. 
 
 stages:
-  - build
-  - data_init
+  - init
   - config
-  - run
+  - generate
+  - process
+  - analyze
+  - collect
+  - cleanup
   - finish
 
-get_data:
-  stage: data_init
-  tags:
-    - sodium
-  script:
-    - git clone https://eicweb.phy.anl.gov/EIC/datasets.git datasets
-    - mkdir -p data
-    - mkdir -p results
-    - mkdir -p sim_output
-
-generate_config:
-  stage: config
-  needs: ["get_data"]
-  tags:
-    - sodium
+detector:
+  stage: init
   script:
-    - mkdir -p config && ./bin/gen_ci_config -p test_ -i dummy > config/dummy_config.yml
-    - mkdir -p config && ./bin/gen_ci_config -p dis_ -i dis > config/dis_config.yml
-
-dummy-pipeline:
-  stage: run
-  needs: ["generate_config"]
-  trigger:
-    include:
-      - artifact: config/dummy_config.yml
-        job: generate_config
-    strategy: depend
+    - mkdir -p detector
+    - bash init/build_detector.sh
 
-dis-pipeline:
-  stage: run
-  needs: ["generate_config"]
-  trigger:
-    include:
-      - artifact: config/dis_config.yml
-        job: generate_config
-    strategy: depend
+include:
+  - local: 'dis/config.yml'
+  - local: 'dummy/config.yml'
+  - local: 'dvmp/config.yml'
 
-final_report:
+summary:
   stage: finish
-  tags:
-    - sodium
-  needs: ["dis-pipeline","dummy-pipeline"]
+  needs: ["dis:results", "dummy:results", "dvmp:results"]
   script:
-    - echo "It was a success!"
-
+    - echo "All benchmarks passed"

bin/gen_ci_config

-#!/bin/bash
-set -o nounset
-set -o errexit
-
-BENCHMARK_SCRIPT_DIR=./dummy
-CI_TAG=sodium
-CI_JOB_PREFIX=test_
-CI_FAILURE="true"
-
-function print_the_help {
-  echo "USAGE: $0 [-t <runner_tag>] "
-  echo "  OPTIONS: "
-  echo "            -i,--input         Input scripts directory "
-  echo "            -t,--tag           Gitlab Runner tag"
-  echo "            -p,--prefix        job name prefix"
-  echo "            --no-failure       do not allow failures"
-  exit 
-}
-
-POSITIONAL=()
-while [[ $# -gt 0 ]]
-do
-  key="$1"
-  case $key in
-    -h|--help)
-      shift # past argument
-      print_the_help
-      ;;
-    -i|--input)
-      BENCHMARK_SCRIPT_DIR="$2"
-      shift # past argument
-      shift # past value
-      ;;
-    -t|--tag)
-      CI_TAG="$2"
-      shift # past argument
-      shift # past value
-      ;;
-    -p|--prefix)
-      CI_JOB_PREFIX="$2"
-      shift # past argument
-      shift # past value
-      ;;
-    --no-failure)
-      CI_FAILURE="false"
-      shift # past argument
-      ;;
-    *)    # unknown option
-      #POSITIONAL+=("$1") # save it in an array for later
-      echo "unknown option $1"
-      print_the_help
-      shift # past argument
-      ;;
-  esac
-done
-set -- "${POSITIONAL[@]}" # restore positional parameters
-
-shopt -s nullglob
-ifile=0
-
-
-cat <<EOF 
-stages:
-  #- simulate
-  - benchmarks
-EOF
-
-for script_name in ${BENCHMARK_SCRIPT_DIR}/*.sh
-do
-  filename=$(basename ${script_name})
-  filename_noext="${filename%%.*}"
-  ifile=$((ifile+1))
-  cat <<EOF 
-${CI_JOB_PREFIX}${ifile}_${filename_noext}:
-  tags:
-    - ${CI_TAG}
-  stage: benchmarks
-  script:
-    - bash ${script_name}
-  allow_failure: ${CI_FAILURE}
-
-EOF
-done
-
-for script_name in ${BENCHMARK_SCRIPT_DIR}/*.cxx
-do
-  filename=$(basename ${script_name})
-  filename_noext="${filename%%.*}"
-  ifile=$((ifile+1))
-  cat <<EOF 
-${CI_JOB_PREFIX}${ifile}_${filename_noext}:
-  tags:
-    - ${CI_TAG}
-  stage: benchmarks
-  script:
-    - root -b -q ${script_name}
-  allow_failure: ${CI_FAILURE}
-
-EOF
-done
-        

config/env.sh

+#!/bin/bash
+
+if [[ ! -n  "${JUGGLER_DETECTOR}" ]] ; then 
+  export JUGGLER_DETECTOR="topside"
+fi
+
+if [[ ! -n  "${JUGGLER_N_EVENTS}" ]] ; then 
+  export JUGGLER_N_EVENTS=100
+fi
+
+if [[ ! -n  "${JUGGLER_INSTALL_PREFIX}" ]] ; then 
+  export JUGGLER_INSTALL_PREFIX="/usr/local"
+fi
+
+if [[ ! -n "${DETECTOR_PREFIX}" ]]; then
+  export DETECTOR_PREFIX=detector
+fi
+
+## ensure absolute paths
+export JUGGLER_INSTALL_PREFIX=`realpath ${JUGGLER_INSTALL_PREFIX}`
+export DETECTOR_PREFIX=`realpath ${DETECTOR_PREFIX}`
+
+## setup detector paths
+export LD_LIBRARY_PATH=${DETECTOR_PREFIX}/lib:$LD_LIBRARY_PATH
+export DETECTOR_SOURCE_PATH=${DETECTOR_PREFIX}/src

dis/config.yml

+dis:dummy_test:
+  stage: analyze
+  needs: ["detector"]
+  timeout: 1 hours
+  script:
+    - dis/dummy_test.sh
+
+dis:dummy_test2:
+  stage: analyze
+  needs: ["detector"]
+  timeout: 1 hours
+  script:
+    - dis/dummy_test2.sh
+
+dis:dummy_fail_test:
+  stage: analyze
+  needs: ["detector"]
+  timeout: 1 hours
+  allow_failure: true
+  script:
+    - dis/dummy_fail_test.sh
+
+dis:results:
+  stage: collect
+  needs: ["dis:dummy_test", "dis:dummy_test2", "dis:dummy_fail_test"]
+  script:
+    - echo "All DIS benchmarks successful"

dummy/config.yml

+dummy:dummy_test:
+  stage: analyze
+  needs: ["detector"]
+  timeout: 1 hours
+  script:
+    - dummy/dummy_test.sh
+
+dummy:dummy_test2:
+  stage: analyze
+  needs: ["detector"]
+  timeout: 1 hours
+  script:
+    - dummy/dummy_test2.sh
+
+dummy:dummy_fail_test:
+  stage: analyze
+  needs: ["detector"]
+  timeout: 1 hours
+  allow_failure: true
+  script:
+    - dummy/dummy_fail_test.sh
+
+dummy:results:
+  stage: collect
+  needs: ["dummy:dummy_test", "dummy:dummy_test2", "dummy:dummy_fail_test"]
+  script:
+    - echo "All DIS benchmarks successful"

dvmp/config.yml

+dvmp:dummy_test:
+  stage: analyze
+  needs: ["detector"]
+  timeout: 1 hours
+  script:
+    - dvmp/dummy_test.sh
+
+dvmp:dummy_test2:
+  stage: analyze
+  needs: ["detector"]
+  timeout: 1 hours
+  script:
+    - dvmp/dummy_test2.sh
+
+dvmp:dummy_fail_test:
+  stage: analyze
+  needs: ["detector"]
+  timeout: 1 hours
+  allow_failure: true
+  script:
+    - dvmp/dummy_fail_test.sh
+
+dvmp:results:
+  stage: collect
+  needs: ["dvmp:dummy_test", "dvmp:dummy_test2", "dvmp:dummy_fail_test"]
+  script:
+    - echo "All DIS benchmarks successful"

dvmp/dummy_fail_test.sh

+#!/bin/bash
+
+echo "Dummy Test..."
+echo "..."
+echo "Fails!"
+
+exit 1

dvmp/dummy_test.sh

+#!/bin/bash
+
+echo "Dummy Test..."
+echo "..."
+echo "Passes!"
+
+#exit 1

dvmp/dummy_test2.sh

+#!/bin/bash
+
+echo "Dummy Test number 2..."
+echo "..."
+echo "Passes!"
+
+#exit 1

init/build_detector.sh

+#!/bin/bash
+
+## Init the environment
+source config/env.sh
+
+## Build and install the detector plugins.
+if [[ ! -d ${DETECTOR_SOURCE_PATH} ]]; then
+  git clone https://eicweb.phy.anl.gov/EIC/detectors/${JUGGLER_DETECTOR}.git ${DETECTOR_SOURCE_PATH}
+else
+  pushd ${DETECTOR_SOURCE_PATH}
+  git pull --ff-only
+  popd
+fi
+mkdir -p detector-build
+pushd detector-build
+echo cmake ${DETECTOR_SOURCE_PATH} -DCMAKE_INSTALL_PREFIX=${DETECTOR_PREFIX} && make -j30 install
+cmake ${DETECTOR_SOURCE_PATH} -DCMAKE_INSTALL_PREFIX=${DETECTOR_PREFIX} && make -j30 install
+popd