Compare revisions

Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,7 +4,7 @@ variables:
  ## Application versions used for the main release 
  ## note: nightly builds will always use the master/main branch
  JUGGLER_VERSION: "v9.4.0"
-  EICRECON_VERSION: "v1.3.1"
+  EICRECON_VERSION: "v1.3.2"

  ## Local registry
  CI_PUSH: 1
@@ -27,19 +27,17 @@ variables:

  ## is this nightly or not?
  NIGHTLY:
+  ## Add to tag
+  NIGHTLY_TAG: "nightly"

  ## Additional options for docker build (e.g. --no-cache)
  BUILD_OPTIONS:

-  ## Default version and tags
-  ## - internal tag used for the CI. Also temporarily tagged
-  ##   on eicweb to communicate between jobs (removed in cleanup job)
-  ## - export tag to public registries, optional secondary export tag
-  VERSION: "${CI_COMMIT_REF_NAME}"
-  NIGHTLY_TAG: "nightly"
-  TESTING_TAG: "testing"
-  INTERNAL_TAG: "${TESTING_TAG}-${VERSION}"
-  EXPORT_TAG: ""
+  ## Version to be set by external trigger
+  VERSION:
+
+  ## Internal tag used for the CI
+  INTERNAL_TAG: "pipeline-${CI_PIPELINE_ID}"

 stages:
  - config
@@ -98,30 +96,39 @@ workflow:
 ## caching strategy for dispatch to multiple nodes and to avoid
 ##  --> try this strategy: https://medium.com/titansoft-engineering/docker-build-cache-sharing-on-multi-hosts-with-buildkit-and-buildx-eb8f7005918e

+## Default version and tags
+## - internal tag used for the CI. Also temporarily tagged
+##   on eicweb to communicate between jobs (removed in cleanup job)
+## - export tag to public registries, optional secondary export tag
 version:
  stage: config
  script:
    - |
-      if [ "x${CI_PIPELINE_SOURCE}" = "xmerge_request_event" ]; then
+      if [ -n "${VERSION}" ]; then
+        # external trigger with specified version
+        EXPORT_TAG="${VERSION}"
+      elif [ "x${CI_PIPELINE_SOURCE}" = "xmerge_request_event" ]; then
        VERSION="unstable-mr-${CI_MERGE_REQUEST_PROJECT_ID}-${CI_MERGE_REQUEST_IID}"
-        INTERNAL_TAG="${VERSION}"
        NIGHTLY_TAG="${VERSION}-nightly"
        EXPORT_TAG="${VERSION}"
      elif [[ "$CI_COMMIT_TAG" =~ ^v[0-9.]+\.[0-9]+\.[0-9]+ ]]; then
        VERSION="${CI_COMMIT_TAG:1}"
-        INTERNAL_TAG="stable-tag-${VERSION}"
        EXPORT_TAG="${VERSION}"
-      elif [[ "$CI_COMMIT_BRANCH" = ^v[0-9.]+\.[0-9]+-stable ]]; then
+      elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+\.[0-9]+-stable ]]; then
        VERSION="${CI_COMMIT_BRANCH:1}"
-        INTERNAL_TAG="stable-br-${VERSION}"
        EXPORT_TAG="${VERSION}"
      elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+\.[0-9]+-[a-z]+-stable ]]; then
        VERSION="${CI_COMMIT_BRANCH:1}"
-        INTERNAL_TAG="stable-br-${VERSION}"
        EXPORT_TAG="${VERSION}"
+      elif [[ "$CI_COMMIT_BRANCH" =~ ^master$ ]]; then
+        VERSION="${CI_COMMIT_BRANCH}"
+        EXPORT_TAG="${VERSION}"
+      else
+        VERSION="${CI_COMMIT_REF_NAME}-${CI_PIPELINE_ID}"
+        EXPORT_TAG=""
      fi
-      echo "VERSION=$VERSION" >> build.env
      echo "INTERNAL_TAG=$INTERNAL_TAG" >> build.env
+      echo "VERSION=$VERSION" >> build.env
      echo "NIGHTLY_TAG=$NIGHTLY_TAG" >> build.env
      echo "EXPORT_TAG=$EXPORT_TAG" >> build.env
      cat build.env
@@ -146,6 +153,15 @@ version:
      done
    - docker buildx create --name builder --driver docker-container --bootstrap --use

+## rules for nightly jobs
+.nightly:
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master"'
+      when: on_success
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: on_success
+    - when: never
+
 ## Images:
 ## debian_testing_base --> jug_dev  --> jug_xl
 ##             ---------------> jug_sim
@@ -246,11 +262,10 @@ jug_dev:default:
    - version
    - base
  script:
-    ## Setup mirrors.yaml with rw mirror secrets
-    - cat mirrors.yaml.in | envsubst > mirrors.yaml
-    ## Build targets
    - source spack.sh ; 
      source eic-spack.sh ;
+      export SPACK_VERSION ;
+      cat mirrors.yaml.in | envsubst > mirrors.yaml ;
      docker buildx build --push ${BUILD_OPTIONS}
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
                   ${EXPORT_TAG:+
@@ -338,7 +353,12 @@ jug_xl:nightly:
        PUSH_NIGHTLY_WITH_DATE="1"
      fi
    - docker buildx build --push ${BUILD_OPTIONS}
-                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
+                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}-nightly
+                   ${EXPORT_TAG:+
+                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}-nightly}
+                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}-nightly}
+                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}-nightly}
+                   }
                   ${NIGHTLY:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
@@ -439,10 +459,8 @@ jug_xl:singularity:nightly:
    - mkdir build
    - singularity pull build/${BUILD_IMAGE}.sif docker://${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${NIGHTLY_TAG}

-benchmarks:reconstruction:
+benchmarks:reconstruction:default:
  stage: benchmarks
-  rules:
-    - if:  '$CI_SERVER_HOST == "eicweb.phy.anl.gov"'
  needs: 
    - version
    - jug_xl:default
@@ -455,10 +473,8 @@ benchmarks:reconstruction:
    strategy: depend
  allow_failure: true
  
-benchmarks:physics:
+benchmarks:physics:default:
  stage: benchmarks
-  rules:
-    - if:  '$CI_SERVER_HOST == "eicweb.phy.anl.gov"'
  needs: 
    - version
    - jug_xl:default
@@ -472,6 +488,39 @@ benchmarks:physics:
    strategy: depend
  allow_failure: false

+benchmarks:reconstruction:nightly:
+  stage: benchmarks
+  rules:
+    - !reference ['.nightly', rules]
+  needs: 
+    - version
+    - jug_xl:nightly
+  variables:
+    BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly"
+    BENCHMARKS_CONTAINER: "jug_xl"
+    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
+  trigger:
+    project: EIC/benchmarks/reconstruction_benchmarks
+    strategy: depend
+  allow_failure: true
+  
+benchmarks:physics:nightly:
+  stage: benchmarks
+  rules:
+    - !reference ['.nightly', rules]
+  needs: 
+    - version
+    - jug_xl:nightly
+  variables:
+    RECO: "eicrecon"
+    BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly"
+    BENCHMARKS_CONTAINER: "jug_xl"
+    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
+  trigger:
+    project: EIC/benchmarks/physics_benchmarks
+    strategy: depend
+  allow_failure: false
+
 clean:
  stage: finalize
  dependencies:
@@ -482,13 +531,17 @@ clean:
  script:
    ## remove the pipeline specific export from eicweb if needed
    - echo "Cleaning up pipeline specific docker tags if needed"
-    - ./gitlab-ci/cleanup_registry.sh -i debian_testing_base -r 66 ${INTERNAL_TAG}
+    - ./gitlab-ci/cleanup_registry.sh -i debian_base -r 66 ${INTERNAL_TAG}
+    - ./gitlab-ci/cleanup_registry.sh -i ubuntu_base -r 80 ${INTERNAL_TAG}
+    - ./gitlab-ci/cleanup_registry.sh -i debian_testing_base -r 83 ${INTERNAL_TAG}
+    - ./gitlab-ci/cleanup_registry.sh -i debian_stable_base -r 92 ${INTERNAL_TAG}
    - ./gitlab-ci/cleanup_registry.sh -i jug_dev -r 68 ${INTERNAL_TAG}
    - ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${INTERNAL_TAG}
    - |
      if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
        ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${NIGHTLY_TAG}
      fi
+    - ./gitlab-ci/cleanup_registry.sh -i dind -r 93 ${INTERNAL_TAG}

 clean_unstable_mr:
  stage: finalize

--- a/containers/dind/Dockerfile
+++ b/containers/dind/Dockerfile
@@ -54,8 +54,7 @@ ADD https://get.docker.com/ /tmp/get-docker.sh
 RUN bash /tmp/get-docker.sh --version ${DOCKER_VERSION}

 # Install the magic wrapper.
-ADD ./wrapdocker /usr/local/bin/wrapdocker
-RUN chmod +x /usr/local/bin/wrapdocker
+ADD --chmod=0755 wrapdocker /usr/local/bin/wrapdocker

 # Define additional metadata for our image.
 VOLUME /var/lib/docker

--- a/containers/jug/dev.Dockerfile
+++ b/containers/jug/dev.Dockerfile
@@ -193,14 +193,13 @@ COPY profile.d/z11_jug_env.sh /etc/profile.d
 COPY singularity.d /.singularity.d

 ## Add minio client into /usr/local/bin
-ADD https://dl.min.io/client/mc/release/linux-amd64/mc /usr/local/bin/mc-amd64
-ADD https://dl.min.io/client/mc/release/linux-arm64/mc /usr/local/bin/mc-arm64
+ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-amd64/mc /usr/local/bin/mc-amd64
+ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-arm64/mc /usr/local/bin/mc-arm64
 RUN declare -A target=(                                                 \
      ["linux/amd64"]="amd64"                                           \
      ["linux/arm64"]="arm64"                                           \
    )                                                                   \
 && mv /usr/local/bin/mc-${target[${TARGETPLATFORM}]} /usr/local/bin/mc \
- && chmod a+x /usr/local/bin/mc                                         \
 && unset target[${TARGETPLATFORM}]                                     \
 && for t in ${target[*]} ; do                                          \
      rm /usr/local/bin/mc-${t} ;                                       \

--- a/containers/jug/eic-news
+++ b/containers/jug/eic-news
+#!/bin/bash
+
+news=$HOME/.eic-news
+
+# run only if stdin and sterr
+if [[ -t 0 && -t 2 && ! -f $HOME/.hushlogin && -f $news ]] ; then
+
+  if [[ $- == *i* ]] ; then
+    # sourced: print once a day
+    if [[ -z $(find $HOME/.eic-news -atime -1) ]] ; then
+      cat $news
+    fi
+  else
+    # else: print always
+    cat $news
+  fi
+
+  # update once a day
+  if [[ -z $(find $HOME/.eic-news -mtime -1) ]] ; then
+    (&>/dev/null curl --silent --connect-timeout 5 --location https://eic.github.io/motd --output $news &)
+  fi
+fi
--- a/containers/jug/xl.Dockerfile
+++ b/containers/jug/xl.Dockerfile
@@ -140,5 +140,10 @@ RUN --mount=type=cache,target=/ccache/,sharing=locked,id=${TARGETPLATFORM}
 && ccache --show-stats                                                         \
 && rm /tmp/setup_detectors.py

+## eic-news
+COPY --chmod=0755 eic-news /usr/local/bin/eic-news
+RUN echo "test -f $HOME/.eic-news && source /usr/local/bin/eic-news"            \
+    > /etc/profile.d/z13_eic-news.sh 
+
 ## Hotfix for misbehaving OSG nodes
 RUN mkdir /hadoop
--- a/detectors.yaml
+++ b/detectors.yaml
@@ -37,3 +37,5 @@ detectors:
      version: 23.07.0
    23.07.1:
      version: 23.07.1
+    23.07.2:
+      version: 23.07.2
--- a/eic-spack.sh
+++ b/eic-spack.sh
@@ -3,7 +3,7 @@ EICSPACK_ORGREPO="eic/eic-spack"

 ## EIC spack github version, e.g. v0.19.7 or commit hash
 ## note: nightly builds will use e.g. releases/v0.19
-EICSPACK_VERSION="v0.20.7"
+EICSPACK_VERSION="v0.20.9"

 ## Space-separated list of eic-spack cherry-picks
 read -r -d '' EICSPACK_CHERRYPICKS <<- \

--- a/spack-environment/dev/spack.yaml
+++ b/spack-environment/dev/spack.yaml
@@ -20,7 +20,7 @@ spack:
    - dd4hep@1.25.1 +ddg4 +ddcad -frames +hepmc3 +lcio +xercesc
    - dpmjet@19.3.5
    - edm4eic@2.0.0 cxxstd=17
-    - edm4hep@=0.7 cxxstd=17
+    - edm4hep@0.7.2 cxxstd=17
    - eic-smear@1.1.10
    - eigen@3.4.0
    - emacs@28.2
@@ -30,6 +30,7 @@ spack:
    - gaudi@36.10 +aida
    - gdb@12.1 -debuginfod
    - geant4@11.1.1 cxxstd=17 +opengl -vecgeom +x11 +qt +threads
+    - graphviz@8.0.1
    - hepmc3@3.2.6 +python +rootio
    - heppdt@2.06.01
    - imagemagick@7.0.8-7
@@ -51,8 +52,9 @@ spack:
    - pythia8@8.309 +fastjet
    - python@3.10.10
    - py-awkward@2.1.1
-    - py-dask@2021.6.2
    - py-boto3@1.26.26
+    - py-dask@2021.6.2
+    - py-deepdiff@6.3.0
    - py-ipython@8.11.0
    - py-jinja2@3.1.2
    - py-jinja2-cli@0.8.2

--- a/spack.sh
+++ b/spack.sh
@@ -3,7 +3,7 @@ SPACK_ORGREPO="spack/spack"

 ## Spack github version, e.g. v0.18.1 or commit hash
 ## note: nightly builds will use e.g. releases/v0.19
-SPACK_VERSION="v0.20.0"
+SPACK_VERSION="v0.20.1"

 ## Space-separated list of spack cherry-picks
 read -r -d '' SPACK_CHERRYPICKS <<- \
No results found