Compare revisions

Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Kolja Kauder
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,7 +8,7 @@ variables:
  ## Application versions used for the main release 
  ## note: nightly builds will always use the master/main branch
  JUGGLER_VERSION: "v10.1.0"
-  EICRECON_VERSION: "v1.5.1"
+  EICRECON_VERSION: "v1.7.0"

  ## Local registry
  CI_PUSH: 1
@@ -44,6 +44,7 @@ variables:
  INTERNAL_TAG: "pipeline-${CI_PIPELINE_ID}"

 stages:
+  - status-pending
  - config
  - base            ## base OS image
  - jug             ## jug container images
@@ -51,6 +52,7 @@ stages:
  - benchmarks
  - test
  - finalize
+  - status-report

 ## only run CI for in the following cases:
 ## master, stable branch, release tag, MR event and nightly builds
@@ -61,8 +63,8 @@ workflow:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_PIPELINE_SOURCE == "web"'
    - if: '$CI_COMMIT_BRANCH == "master"'
-    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-stable/'          ## main stable branch: vX.Y-stable
-    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-[a-z]+-stable/'   ## special stable branch: vX.Y-acadia-stable (etc)
+    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-(alpha|beta|stable)/'          ## main stable branch: vX.Y-stable
+    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-[a-z]+-(alpha|beta|stable)/'   ## special stable branch: vX.Y-acadia-stable (etc)
    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+/'            ## commit tags start with vX.Y.Z with optional suffix

 ## plan:
@@ -105,10 +107,10 @@ version:
      elif [[ "$CI_COMMIT_TAG" =~ ^v[0-9.]+\.[0-9]+\.[0-9]+ ]]; then
        VERSION="${CI_COMMIT_TAG:1}"
        EXPORT_TAG="${VERSION}"
-      elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+\.[0-9]+-stable ]]; then
+      elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+\.[0-9]+-\(alpha\|beta\|stable\) ]]; then
        VERSION="${CI_COMMIT_BRANCH:1}"
        EXPORT_TAG="${VERSION}"
-      elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+\.[0-9]+-[a-z]+-stable ]]; then
+      elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+\.[0-9]+-[a-z]+-\(alpha\|beta\|stable\) ]]; then
        VERSION="${CI_COMMIT_BRANCH:1}"
        EXPORT_TAG="${VERSION}"
      elif [[ "$CI_COMMIT_BRANCH" =~ ^master$ ]]; then
@@ -128,6 +130,32 @@ version:
    reports:
      dotenv: build.env

+.status:
+  image: curlimages/curl:latest
+  before_script: []
+  script:
+    - |
+      if [ -n "${GITHUB_SHA}" ] ; then
+          curl \
+            -X POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: token ${GITHUB_REPO_STATUS_TOKEN}" \
+            "https://api.github.com/repos/${GITHUB_REPOSITORY}/statuses/${GITHUB_SHA}" \
+            -d '{"state":"'"${STATE}"'",
+                 "target_url":"'"${CI_PIPELINE_URL}"'",
+                 "description":"'"${DESCRIPTION} $(TZ=America/New_York date)"'",
+                 "context":"eicweb/eic_container"
+                }' ;
+      fi
+
+status:pending:
+  stage: status-pending
+  extends: .status
+  variables:
+    STATE: "pending"
+    DESCRIPTION: "Started..."
+  when: always
+
 ## base job settings for all docker interactions
 .docker:
  image: ${DOCKER_IMAGE}
@@ -244,21 +272,25 @@ jug_dev:
        BASE_IMAGE: debian_stable_base
        BUILD_IMAGE: jug_
        PLATFORM: linux/amd64,linux/arm64/v8
-        ENV: [dev, prod]
+        ENV:
+        - dev
+        - prod
 #      - BUILD_TYPE:
 #        - default
 #        - nightly
 #        BASE_IMAGE: oneapi_base
 #        BUILD_IMAGE: oneapi_
 #        PLATFORM: linux/amd64
-#        ENV: [prod]
+#        ENV:
+#        - prod
 #      - BUILD_TYPE:
 #        - default
 #        - nightly
 #        BASE_IMAGE: cuda_base_new
 #        BUILD_IMAGE: cuda_
 #        PLATFORM: linux/amd64
-#        ENV: [prod]
+#        ENV:
+#        - prod
  extends: .build
  stage: jug
  needs:
@@ -329,7 +361,16 @@ jug_xl:default:
  stage: jug
  needs:
    - version
-    - jug_dev
+    - job: jug_dev
+      parallel:
+        matrix:
+        - BUILD_TYPE:
+          - default
+          BASE_IMAGE: debian_stable_base
+          BUILD_IMAGE: jug_
+          PLATFORM: linux/amd64,linux/arm64/v8
+          ENV:
+          - dev
  script:
    - apk add git
    - docker buildx build --push ${BUILD_OPTIONS}
@@ -369,7 +410,16 @@ jug_xl:nightly:
    - when: never
  needs:
    - version
-    - jug_dev
+    - job: jug_dev
+      parallel:
+        matrix:
+        - BUILD_TYPE:
+          - nightly
+          BASE_IMAGE: debian_stable_base
+          BUILD_IMAGE: jug_
+          PLATFORM: linux/amd64,linux/arm64/v8
+          ENV:
+          - dev
  script:
    - |
      PUSH_NIGHTLY_WITH_DATE=""
@@ -442,36 +492,24 @@ jug_xl:singularity:nightly:
    - version
    - jug_xl:nightly

-benchmarks:reconstruction:default:
+benchmarks:detector:default:
  stage: benchmarks
  needs: 
    - version
    - jug_xl:default
  variables:
+    DETECTOR: "epic"
+    DETECTOR_CONFIG: epic_craterlake
+    DETECTOR_REPOSITORYURL: 'https://github.com/eic/epic.git'
    BENCHMARKS_TAG: "${INTERNAL_TAG}-default"
    BENCHMARKS_CONTAINER: "jug_xl"
    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
  trigger:
-    project: EIC/benchmarks/reconstruction_benchmarks
+    project: EIC/benchmarks/detector_benchmarks
    strategy: depend
  allow_failure: true
  
-benchmarks:physics:default:
-  stage: benchmarks
-  needs: 
-    - version
-    - jug_xl:default
-  variables:
-    RECO: "eicrecon"
-    BENCHMARKS_TAG: "${INTERNAL_TAG}-default"
-    BENCHMARKS_CONTAINER: "jug_xl"
-    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
-  trigger:
-    project: EIC/benchmarks/physics_benchmarks
-    strategy: depend
-  allow_failure: false
-
-benchmarks:reconstruction:nightly:
+benchmarks:detector:nightly:
  stage: benchmarks
  rules:
    - !reference ['.nightly', rules]
@@ -479,36 +517,20 @@ benchmarks:reconstruction:nightly:
    - version
    - jug_xl:nightly
  variables:
+    DETECTOR: "epic"
+    DETECTOR_CONFIG: epic_craterlake
+    DETECTOR_REPOSITORYURL: 'https://github.com/eic/epic.git'
    BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly"
    BENCHMARKS_CONTAINER: "jug_xl"
    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
  trigger:
-    project: EIC/benchmarks/reconstruction_benchmarks
+    project: EIC/benchmarks/detector_benchmarks
    strategy: depend
  allow_failure: true
  
-benchmarks:physics:nightly:
-  stage: benchmarks
-  rules:
-    - !reference ['.nightly', rules]
-  needs: 
-    - version
-    - jug_xl:nightly
-  variables:
-    RECO: "eicrecon"
-    BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly"
-    BENCHMARKS_CONTAINER: "jug_xl"
-    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
-  trigger:
-    project: EIC/benchmarks/physics_benchmarks
-    strategy: depend
-  allow_failure: false
-
 clean_internal_tag:
  image: alpine/curl
  stage: finalize
-  dependencies:
-    - version
  when: always
  script:
    - |
@@ -521,8 +543,6 @@ clean_internal_tag:
 clean_unstable_mr:
  extends: .docker
  stage: finalize
-  dependencies:
-    - version
  tags:
    - docker-new
  when: always
@@ -554,7 +574,7 @@ clean_unstable_mr:
 clean_pipeline:
  extends: .docker
  stage: finalize
-  dependencies:
+  needs:
    - version
    - clean_internal_tag
  tags:
@@ -583,3 +603,20 @@ clean_pipeline:
    - docker system prune --filter until=24h --force
    - docker system df
    - docker images
+
+status:success:
+  stage: status-report
+  extends: .status
+  variables:
+    STATE: "success"
+    DESCRIPTION: "Succeeded!"
+  when: on_success
+
+status:failure:
+  stage: status-report
+  extends: .status
+  variables:
+    STATE: "failure"
+    DESCRIPTION: "Failed!"
+  when: on_failure
+
--- a/containers/debian/base.Dockerfile
+++ b/containers/debian/base.Dockerfile
@@ -10,6 +10,10 @@ LABEL maintainer="Sylvester Joosten <sjoosten@anl.gov>" \

 COPY bashrc /root/.bashrc

+## With heredocs for multi-line scripts, we want to fail on error and the print failing line.
+## Ref: https://docs.docker.com/engine/reference/builder/#example-running-a-multi-line-script
+SHELL ["bash", "-ex", "-c"]
+
 ENV CLICOLOR_FORCE=1                                                    \
    LANGUAGE=en_US.UTF-8                                                \
    LANG=en_US.UTF-8                                                    \
@@ -17,13 +21,13 @@ ENV CLICOLOR_FORCE=1                                                    \

 ## Install additional packages. Remove the auto-cleanup functionality
 ## for docker, as we're using the new buildkit cache instead.
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked             \
-    --mount=type=cache,target=/var/lib/apt/lists,sharing=locked         \
-    rm -f /etc/apt/apt.conf.d/docker-clean                              \
- && ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime          \
- && echo "US/Eastern" > /etc/timezone                                   \
- && apt-get -yqq update                                                 \
- && apt-get -yqq install --no-install-recommends                        \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt/lists,sharing=locked <<EOF
+rm -f /etc/apt/apt.conf.d/docker-clean
+ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime
+echo "US/Eastern" > /etc/timezone
+apt-get -yqq update
+apt-get -yqq install --no-install-recommends                            \
        bc                                                              \
        bzip2                                                           \
        ca-certificates                                                 \
@@ -53,50 +57,40 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked             \
        time                                                            \
        unzip                                                           \
        vim-nox                                                         \
-        wget                                                            \
- && apt-get -yqq autoremove                                             \
- && localedef -i en_US -f UTF-8 en_US.UTF-8
+        wget
+apt-get -yqq autoremove
+localedef -i en_US -f UTF-8 en_US.UTF-8
+EOF

 # Install updated compilers, with support for multiple base images
 ## Ubuntu: latest gcc from toolchain ppa, latest stable clang
 ## Debian: default gcc with distribution, latest stable clang
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked             \
-    --mount=type=cache,target=/var/lib/apt/lists,sharing=locked         \
-    . /etc/os-release                                                   \
- && mkdir -p /etc/apt/source.list.d                                     \
- && if [ "${ID}" = "ubuntu" ] ; then                                    \
-      echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/ppa/ubuntu/ \
-            ${VERSION_CODENAME} main"                                   \
-      > /etc/apt/source.list.d/ubuntu-toolchain.list                    \
-   && if [ "${VERSION_ID}" = "20.04" ] ; then GCC="-10" CLANG="-12" ; fi\
-   && if [ "${VERSION_ID}" = "22.04" ] ; then GCC="-12" CLANG="-14" ; fi\
-   && curl -s https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -\
-   && echo "deb http://apt.llvm.org/${VERSION_CODENAME}                 \
-            llvm-toolchain-${VERSION_CODENAME}${CLANG} main"            \
-      > /etc/apt/source.list.d/llvm.list                                \
-   && apt-get -yqq update                                               \
-   && apt-get -yqq install                                              \
-          gcc${GCC} g++${GCC} gfortran${GCC}                            \
-   && apt-get -yqq install                                              \
-          clang${CLANG} clang-tidy${CLANG} clang-format${CLANG}         \
-          iwyu                                                          \
-   && update-alternatives --install /usr/bin/gcc gcc                    \
-                                    /usr/bin/gcc${GCC} 100              \
-   && update-alternatives --install /usr/bin/g++ g++                    \
-                                    /usr/bin/g++${GCC} 100              \
-   && update-alternatives --install /usr/bin/gfortran gfortran          \
-                                    /usr/bin/gfortran${GCC} 100         \
-   && update-alternatives --install /usr/bin/clang clang                \
-                                    /usr/bin/clang${CLANG} 100          \
-   && update-alternatives --install /usr/bin/clang++ clang++            \
-                                    /usr/bin/clang++${CLANG} 100        \
- ; else                                                                 \
-      apt-get -yqq update                                               \
-   && apt-get -yqq install                                              \
-          gcc g++ gfortran                                              \
-          clang clang-tidy clang-format                                 \
-          iwyu                                                          \
- ; fi                                                                   \
- && apt-get -yqq autoremove                                             \
- && gcc --version                                                       \
- && clang --version
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt/lists,sharing=locked <<EOF
+. /etc/os-release
+mkdir -p /etc/apt/source.list.d
+if [ "${ID}" = "ubuntu" ] ; then
+  echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/ppa/ubuntu/${VERSION_CODENAME} main" > /etc/apt/source.list.d/ubuntu-toolchain.list
+  if [ "${VERSION_ID}" = "20.04" ] ; then GCC="-10" CLANG="-12" ; fi
+  if [ "${VERSION_ID}" = "22.04" ] ; then GCC="-12" CLANG="-14" ; fi
+  curl -s https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
+  echo "deb http://apt.llvm.org/${VERSION_CODENAME} llvm-toolchain-${VERSION_CODENAME}${CLANG} main" > /etc/apt/source.list.d/llvm.list
+  apt-get -yqq update
+  apt-get -yqq install gcc${GCC} g++${GCC} gfortran${GCC}
+  apt-get -yqq install clang${CLANG} clang-tidy${CLANG} clang-format${CLANG}
+  apt-get -yqq install iwyu
+  update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc${GCC} 100
+  update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++${GCC} 100
+  update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran${GCC} 100
+  update-alternatives --install /usr/bin/clang clang /usr/bin/clang${CLANG} 100
+  update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++${CLANG} 100
+else
+  apt-get -yqq update
+  apt-get -yqq install gcc g++ gfortran
+  apt-get -yqq install clang clang-tidy clang-format
+  apt-get -yqq install iwyu
+fi
+apt-get -yqq autoremove
+gcc --version
+clang --version
+EOF
--- a/containers/jug/dev.Dockerfile
+++ b/containers/jug/dev.Dockerfile
--- a/containers/jug/xl.Dockerfile
+++ b/containers/jug/xl.Dockerfile
@@ -17,8 +17,7 @@ ARG jobs=8
 ## the date
 ARG JUG_VERSION=1

-RUN cd /tmp                                                                     \
- && echo " - jug_xl: ${JUG_VERSION}" >> /etc/jug_info
+RUN echo " - jug_xl: ${JUG_VERSION}" >> /etc/jug_info

 ## also install detector/ip geometries into opt
 ARG NIGHTLY=''
@@ -31,11 +30,11 @@ ADD https://api.github.com/repos/eic/ip6 /tmp/ip6.json
 ADD https://api.github.com/repos/eic/epic /tmp/epic.json
 COPY setup_detectors.py /tmp
 COPY --from=detectors detectors.yaml /tmp
-RUN --mount=type=cache,target=/ccache/,sharing=locked,id=${TARGETPLATFORM}      \
-    cd /tmp                                                                     \
- && export CCACHE_DIR=/ccache                                                   \
- && [ "z$NIGHTLY" = "z1" ] && NIGHTLY_FLAG="--nightly" || NIGHTLY_FLAG=""       \
- && /tmp/setup_detectors.py --prefix /opt/detector --config /tmp/detectors.yaml \
-                         $NIGHTLY_FLAG                                          \
- && ccache --show-stats                                                         \
- && rm /tmp/setup_detectors.py
+RUN --mount=type=cache,target=/ccache/,sharing=locked,id=${TARGETPLATFORM} <<EOF
+cd /tmp
+export CCACHE_DIR=/ccache
+[ "z$NIGHTLY" = "z1" ] && NIGHTLY_FLAG="--nightly" || NIGHTLY_FLAG=""
+/tmp/setup_detectors.py --prefix /opt/detector --config /tmp/detectors.yaml $NIGHTLY_FLAG
+ccache --show-stats
+rm /tmp/setup_detectors.py
+EOF
--- a/detectors.yaml
+++ b/detectors.yaml
@@ -5,18 +5,6 @@ detectors:
      version: main
    main:
      version: main
-    23.05.0:
-      version: 23.05.0
-      patches:
-      - https://github.com/eic/epic/pull/449.patch
-    23.05.1:
-      version: 23.05.1
-      patches:
-      - https://github.com/eic/epic/pull/449.patch
-    23.05.2:
-      version: 23.05.2
-      patches:
-      - https://github.com/eic/epic/pull/449.patch
    23.06.0:
      version: 23.06.0
      patches:
@@ -37,3 +25,7 @@ detectors:
      version: 23.09.0
    23.09.1:
      version: 23.09.1
+    23.10.0:
+      version: 23.10.0
+    23.11.0:
+      version: 23.11.0
--- a/eic-spack.sh
+++ b/eic-spack.sh
 ## EIC spack organization and repository, e.g. eic/eic-spack
 EICSPACK_ORGREPO="eic/eic-spack"

-## EIC spack github version, e.g. v0.19.7 or commit hash
-## note: nightly builds will use e.g. releases/v0.19
-EICSPACK_VERSION="v0.20.20"
+## EIC spack commit hash or github version, e.g. v0.19.7
+## note: nightly builds could use a branch e.g. releases/v0.19
+EICSPACK_VERSION="1d2956a5f87ada2bf04ed1bdee761baf81b85feb"

 ## Space-separated list of eic-spack cherry-picks
 read -r -d '' EICSPACK_CHERRYPICKS <<- \

--- a/spack-environment/dev/spack.yaml
+++ b/spack-environment/dev/spack.yaml
@@ -52,6 +52,7 @@ spack:
    - py-boto3
    - py-dask
    - py-deepdiff
+    - py-epic-capybara
    - py-ipython
    - py-jinja2
    - py-jinja2-cli
@@ -75,6 +76,7 @@ spack:
    - py-wurlitzer
    - py-yapf
    - root
+    - snakemake
    - spdlog
    - stow
    - tensorflow-lite

--- a/spack-environment/packages.yaml
+++ b/spack-environment/packages.yaml
@@ -32,7 +32,7 @@ packages:
    - +root +zlib
  algorithms:
    require:
-    - '@main'
+    - '@git.ebc8f85709bc7233574843b843ee0191a5bf3c1d'
  cairo:
    require:
    - '@1.16.0'
@@ -80,7 +80,7 @@ packages:
    - '@656aa3192b097a631ddd1e0380e80c26fd6644a7'
  edm4eic:
    require:
-    - '@3.0.1'
+    - '@4.0.0'
    - cxxstd=17
  edm4hep:
    require:
@@ -88,7 +88,7 @@ packages:
    - cxxstd=17
  eic-smear:
    require:
-    - '@1.1.10'
+    - '@1.1.12'
  eigen:
    require:
    - '@3.4.0'
@@ -141,7 +141,7 @@ packages:
    - '@2020.3'
  irt:
    require:
-    - '@1.0.6'
+    - '@1.0.7'
  jana2:
    require:
    - '@2.1.1'
@@ -210,6 +210,9 @@ packages:
  py-deepdiff:
    require:
    - '@6.3.0'
+  py-epic-capybara:
+    require:
+    - '@main'
  py-ipython:
    require:
    - '@8.11.0'
@@ -285,6 +288,10 @@ packages:
    - '@6.28.04'
    - cxxstd=17 +fftw +fortran +gdml +http -ipo +mlp +pythia8 +root7 +tmva +vc -webgui +xrootd +ssl
    - any_of: [+opengl +x, -opengl -x]
+  snakemake:
+    require:
+    - '@7.22.0'
+    - +http +s3
  spdlog:
    require:
    - '@1.11.0'

--- a/spack.sh
+++ b/spack.sh
@@ -27,6 +27,7 @@ ed76eab6943221f17776fd8d128ade6ba69e492c
 6c5d125cb06a86ce05bec27ae9fb9b07103bc1c5
 e3e7609af4903be7df42b6ae5ccf9a20293503d2
 df4a2457a41e7ab634e86d3148d8b22a9f433a6a
+53a45c820c983867e8b05cab3df34e98b468f932
 ---
 ## Ref: https://github.com/spack/spack/commit/[hash]
 ## [hash]: [description]
@@ -49,3 +50,4 @@ df4a2457a41e7ab634e86d3148d8b22a9f433a6a
 ## 6c5d125cb06a86ce05bec27ae9fb9b07103bc1c5: cernlib: new variant shared
 ## e3e7609af4903be7df42b6ae5ccf9a20293503d2: edm4hep: Add version 0.9
 ## df4a2457a41e7ab634e86d3148d8b22a9f433a6a: Fix broken semver regex
+## 53a45c820c983867e8b05cab3df34e98b468f932: docker entrypoint.sh: fail multi-line RUN on first error with set -e
No results found