From 1443e856da3c339a6e64b4df9e97acb10ed59725 Mon Sep 17 00:00:00 2001
From: Sylvester Joosten <sjoosten@anl.gov>
Date: Wed, 4 Nov 2020 16:30:15 +0000
Subject: [PATCH] Refactor CI setup
---
.gitlab-ci.yml | 77 +++++++++--------------------
bin/gen_ci_config | 101 ---------------------------------------
config/env.sh | 25 ++++++++++
dis/config.yml | 27 +++++++++++
dis/dummy_fail_test.sh | 0
dis/dummy_test.sh | 0
dis/dummy_test2.sh | 0
dummy/config.yml | 27 +++++++++++
dummy/dummy_fail_test.sh | 0
dummy/dummy_test.sh | 0
dummy/dummy_test2.sh | 0
dvmp/.gitignore | 0
dvmp/config.yml | 27 +++++++++++
dvmp/dummy_fail_test.sh | 7 +++
dvmp/dummy_test.sh | 7 +++
dvmp/dummy_test2.sh | 7 +++
init/build_detector.sh | 18 +++++++
17 files changed, 167 insertions(+), 156 deletions(-)
delete mode 100755 bin/gen_ci_config
create mode 100755 config/env.sh
create mode 100644 dis/config.yml
mode change 100644 => 100755 dis/dummy_fail_test.sh
mode change 100644 => 100755 dis/dummy_test.sh
mode change 100644 => 100755 dis/dummy_test2.sh
create mode 100644 dummy/config.yml
mode change 100644 => 100755 dummy/dummy_fail_test.sh
mode change 100644 => 100755 dummy/dummy_test.sh
mode change 100644 => 100755 dummy/dummy_test2.sh
create mode 100644 dvmp/.gitignore
create mode 100644 dvmp/config.yml
create mode 100755 dvmp/dummy_fail_test.sh
create mode 100755 dvmp/dummy_test.sh
create mode 100755 dvmp/dummy_test2.sh
create mode 100755 init/build_detector.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 42332400..7be2a15e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,70 +1,37 @@
-image: eicweb.phy.anl.gov:4567/containers/eic_container/eic:latest
+image: eicweb.phy.anl.gov:4567/eic/juggler/juggler:latest
default:
+ tags:
+ - silicon
artifacts:
- expire_in: 10 mins
+ expire_in: 3 days
paths:
- - config/
+ - detector/
- results/
- - sim_output/
- - data
- # exclude:
- # - .git/
- # - datasets/.git/
- #before_script:
- # - git clone https://eicweb.phy.anl.gov/EIC/NPDet.git
- # # - cd NPDet/build && cmake ../. -DCMAKE_INSTALL_PREFIX=/usr/local && make -j10 && make install
- # # - cd ../..
stages:
- - build
- - data_init
+ - init
- config
- - run
+ - generate
+ - process
+ - analyze
+ - collect
+ - cleanup
- finish
-get_data:
- stage: data_init
- tags:
- - sodium
- script:
- - git clone https://eicweb.phy.anl.gov/EIC/datasets.git datasets
- - mkdir -p data
- - mkdir -p results
- - mkdir -p sim_output
-
-generate_config:
- stage: config
- needs: ["get_data"]
- tags:
- - sodium
+detector:
+ stage: init
script:
- - mkdir -p config && ./bin/gen_ci_config -p test_ -i dummy > config/dummy_config.yml
- - mkdir -p config && ./bin/gen_ci_config -p dis_ -i dis > config/dis_config.yml
-
-dummy-pipeline:
- stage: run
- needs: ["generate_config"]
- trigger:
- include:
- - artifact: config/dummy_config.yml
- job: generate_config
- strategy: depend
+ - mkdir -p detector
+ - bash init/build_detector.sh
-dis-pipeline:
- stage: run
- needs: ["generate_config"]
- trigger:
- include:
- - artifact: config/dis_config.yml
- job: generate_config
- strategy: depend
+include:
+ - local: 'dis/config.yml'
+ - local: 'dummy/config.yml'
+ - local: 'dvmp/config.yml'
-final_report:
+summary:
stage: finish
- tags:
- - sodium
- needs: ["dis-pipeline","dummy-pipeline"]
+ needs: ["dis:results", "dummy:results", "dvmp:results"]
script:
- - echo "It was a success!"
-
+ - echo "All benchmarks passed"
diff --git a/bin/gen_ci_config b/bin/gen_ci_config
deleted file mode 100755
index 781607b0..00000000
--- a/bin/gen_ci_config
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/bin/bash
-set -o nounset
-set -o errexit
-
-BENCHMARK_SCRIPT_DIR=./dummy
-CI_TAG=sodium
-CI_JOB_PREFIX=test_
-CI_FAILURE="true"
-
-function print_the_help {
- echo "USAGE: $0 [-t <runner_tag>] "
- echo " OPTIONS: "
- echo " -i,--input Input scripts directory "
- echo " -t,--tag Gitlab Runner tag"
- echo " -p,--prefix job name prefix"
- echo " --no-failure do not allow failures"
- exit
-}
-
-POSITIONAL=()
-while [[ $# -gt 0 ]]
-do
- key="$1"
- case $key in
- -h|--help)
- shift # past argument
- print_the_help
- ;;
- -i|--input)
- BENCHMARK_SCRIPT_DIR="$2"
- shift # past argument
- shift # past value
- ;;
- -t|--tag)
- CI_TAG="$2"
- shift # past argument
- shift # past value
- ;;
- -p|--prefix)
- CI_JOB_PREFIX="$2"
- shift # past argument
- shift # past value
- ;;
- --no-failure)
- CI_FAILURE="false"
- shift # past argument
- ;;
- *) # unknown option
- #POSITIONAL+=("$1") # save it in an array for later
- echo "unknown option $1"
- print_the_help
- shift # past argument
- ;;
- esac
-done
-set -- "${POSITIONAL[@]}" # restore positional parameters
-
-shopt -s nullglob
-ifile=0
-
-
-cat <<EOF
-stages:
- #- simulate
- - benchmarks
-EOF
-
-for script_name in ${BENCHMARK_SCRIPT_DIR}/*.sh
-do
- filename=$(basename ${script_name})
- filename_noext="${filename%%.*}"
- ifile=$((ifile+1))
- cat <<EOF
-${CI_JOB_PREFIX}${ifile}_${filename_noext}:
- tags:
- - ${CI_TAG}
- stage: benchmarks
- script:
- - bash ${script_name}
- allow_failure: ${CI_FAILURE}
-
-EOF
-done
-
-for script_name in ${BENCHMARK_SCRIPT_DIR}/*.cxx
-do
- filename=$(basename ${script_name})
- filename_noext="${filename%%.*}"
- ifile=$((ifile+1))
- cat <<EOF
-${CI_JOB_PREFIX}${ifile}_${filename_noext}:
- tags:
- - ${CI_TAG}
- stage: benchmarks
- script:
- - root -b -q ${script_name}
- allow_failure: ${CI_FAILURE}
-
-EOF
-done
-
diff --git a/config/env.sh b/config/env.sh
new file mode 100755
index 00000000..95b30d84
--- /dev/null
+++ b/config/env.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+if [[ ! -n "${JUGGLER_DETECTOR}" ]] ; then
+ export JUGGLER_DETECTOR="topside"
+fi
+
+if [[ ! -n "${JUGGLER_N_EVENTS}" ]] ; then
+ export JUGGLER_N_EVENTS=100
+fi
+
+if [[ ! -n "${JUGGLER_INSTALL_PREFIX}" ]] ; then
+ export JUGGLER_INSTALL_PREFIX="/usr/local"
+fi
+
+if [[ ! -n "${DETECTOR_PREFIX}" ]]; then
+ export DETECTOR_PREFIX=detector
+fi
+
+## ensure absolute paths
+export JUGGLER_INSTALL_PREFIX=`realpath ${JUGGLER_INSTALL_PREFIX}`
+export DETECTOR_PREFIX=`realpath ${DETECTOR_PREFIX}`
+
+## setup detector paths
+export LD_LIBRARY_PATH=${DETECTOR_PREFIX}/lib:$LD_LIBRARY_PATH
+export DETECTOR_SOURCE_PATH=${DETECTOR_PREFIX}/src
diff --git a/dis/config.yml b/dis/config.yml
new file mode 100644
index 00000000..0e7aa279
--- /dev/null
+++ b/dis/config.yml
@@ -0,0 +1,27 @@
+dis:dummy_test:
+ stage: analyze
+ needs: ["detector"]
+ timeout: 1 hours
+ script:
+ - dis/dummy_test.sh
+
+dis:dummy_test2:
+ stage: analyze
+ needs: ["detector"]
+ timeout: 1 hours
+ script:
+ - dis/dummy_test2.sh
+
+dis:dummy_fail_test:
+ stage: analyze
+ needs: ["detector"]
+ timeout: 1 hours
+ allow_failure: true
+ script:
+ - dis/dummy_fail_test.sh
+
+dis:results:
+ stage: collect
+ needs: ["dis:dummy_test", "dis:dummy_test2", "dis:dummy_fail_test"]
+ script:
+ - echo "All DIS benchmarks successful"
diff --git a/dis/dummy_fail_test.sh b/dis/dummy_fail_test.sh
old mode 100644
new mode 100755
diff --git a/dis/dummy_test.sh b/dis/dummy_test.sh
old mode 100644
new mode 100755
diff --git a/dis/dummy_test2.sh b/dis/dummy_test2.sh
old mode 100644
new mode 100755
diff --git a/dummy/config.yml b/dummy/config.yml
new file mode 100644
index 00000000..682fd84e
--- /dev/null
+++ b/dummy/config.yml
@@ -0,0 +1,27 @@
+dummy:dummy_test:
+ stage: analyze
+ needs: ["detector"]
+ timeout: 1 hours
+ script:
+ - dummy/dummy_test.sh
+
+dummy:dummy_test2:
+ stage: analyze
+ needs: ["detector"]
+ timeout: 1 hours
+ script:
+ - dummy/dummy_test2.sh
+
+dummy:dummy_fail_test:
+ stage: analyze
+ needs: ["detector"]
+ timeout: 1 hours
+ allow_failure: true
+ script:
+ - dummy/dummy_fail_test.sh
+
+dummy:results:
+ stage: collect
+ needs: ["dummy:dummy_test", "dummy:dummy_test2", "dummy:dummy_fail_test"]
+ script:
+ - echo "All DIS benchmarks successful"
diff --git a/dummy/dummy_fail_test.sh b/dummy/dummy_fail_test.sh
old mode 100644
new mode 100755
diff --git a/dummy/dummy_test.sh b/dummy/dummy_test.sh
old mode 100644
new mode 100755
diff --git a/dummy/dummy_test2.sh b/dummy/dummy_test2.sh
old mode 100644
new mode 100755
diff --git a/dvmp/.gitignore b/dvmp/.gitignore
new file mode 100644
index 00000000..e69de29b
diff --git a/dvmp/config.yml b/dvmp/config.yml
new file mode 100644
index 00000000..bd7b4eab
--- /dev/null
+++ b/dvmp/config.yml
@@ -0,0 +1,27 @@
+dvmp:dummy_test:
+ stage: analyze
+ needs: ["detector"]
+ timeout: 1 hours
+ script:
+ - dvmp/dummy_test.sh
+
+dvmp:dummy_test2:
+ stage: analyze
+ needs: ["detector"]
+ timeout: 1 hours
+ script:
+ - dvmp/dummy_test2.sh
+
+dvmp:dummy_fail_test:
+ stage: analyze
+ needs: ["detector"]
+ timeout: 1 hours
+ allow_failure: true
+ script:
+ - dvmp/dummy_fail_test.sh
+
+dvmp:results:
+ stage: collect
+ needs: ["dvmp:dummy_test", "dvmp:dummy_test2", "dvmp:dummy_fail_test"]
+ script:
+ - echo "All DIS benchmarks successful"
diff --git a/dvmp/dummy_fail_test.sh b/dvmp/dummy_fail_test.sh
new file mode 100755
index 00000000..f8c506b3
--- /dev/null
+++ b/dvmp/dummy_fail_test.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "Dummy Test..."
+echo "..."
+echo "Fails!"
+
+exit 1
diff --git a/dvmp/dummy_test.sh b/dvmp/dummy_test.sh
new file mode 100755
index 00000000..5af1dc34
--- /dev/null
+++ b/dvmp/dummy_test.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "Dummy Test..."
+echo "..."
+echo "Passes!"
+
+#exit 1
diff --git a/dvmp/dummy_test2.sh b/dvmp/dummy_test2.sh
new file mode 100755
index 00000000..dfede272
--- /dev/null
+++ b/dvmp/dummy_test2.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "Dummy Test number 2..."
+echo "..."
+echo "Passes!"
+
+#exit 1
diff --git a/init/build_detector.sh b/init/build_detector.sh
new file mode 100755
index 00000000..7a1477f8
--- /dev/null
+++ b/init/build_detector.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+## Init the environment
+source config/env.sh
+
+## Build and install the detector plugins.
+if [[ ! -d ${DETECTOR_SOURCE_PATH} ]]; then
+ git clone https://eicweb.phy.anl.gov/EIC/detectors/${JUGGLER_DETECTOR}.git ${DETECTOR_SOURCE_PATH}
+else
+ pushd ${DETECTOR_SOURCE_PATH}
+ git pull --ff-only
+ popd
+fi
+mkdir -p detector-build
+pushd detector-build
+echo cmake ${DETECTOR_SOURCE_PATH} -DCMAKE_INSTALL_PREFIX=${DETECTOR_PREFIX} && make -j30 install
+cmake ${DETECTOR_SOURCE_PATH} -DCMAKE_INSTALL_PREFIX=${DETECTOR_PREFIX} && make -j30 install
+popd
--
GitLab