Compare revisions

Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,14 +4,14 @@ variables:
  ## Application versions used for the main release 
  ## note: nightly builds will always use the master/main branch
  JUGGLER_VERSION: "v9.3.0"
-  EICRECON_VERSION: "v0.5.4"
+  EICRECON_VERSION: "v0.6.2"

  ## Spack organization and repository, e.g. spack/spack
  SPACK_ORGREPO: "spack/spack"
  ## Spack github version, e.g. v0.18.1 or commit hash
-  SPACK_VERSION: "v0.19.0"
+  SPACK_VERSION: "v0.19.1"
  ## Space-separated list of spack cherry-picks
-  SPACK_CHERRYPICKS: "b5ef5c2eb5145020f9de1bcb964626ce6ac2d02e 99056e03bd3e903b222e300636ec484d85d4b3fb f3f8b31be54280c6ef0b5f95ab85604aba3dff30 0ea81affd18820933640279bbc687038b3296a4e dceb4c9d655d0529e112b8929558be60973b39f7 f2b0c1deab87da3b5aa4c1f2ef4d6af06fa4e32d 2f057d729da61e8c12828df44377f0a93fed820f 39a1f1462b0abf12dfaccd33f84142c852c4b56e 4b12d015e1c316b4837e02ae37e2c69a8a647180 7f1467e795b1cab8b4855e019910c509896ea0e1 62da76cb5dca4d52c43bee06230cca6a5882f05d cc2ae9f270befa554ba8b09c68e89bb8248ea650 ae98d2ba2fcefa9d027e2d6ccc6e7558a32e7228"
+  SPACK_CHERRYPICKS: "b5ef5c2eb5145020f9de1bcb964626ce6ac2d02e 99056e03bd3e903b222e300636ec484d85d4b3fb f3f8b31be54280c6ef0b5f95ab85604aba3dff30 0ea81affd18820933640279bbc687038b3296a4e dceb4c9d655d0529e112b8929558be60973b39f7 f2b0c1deab87da3b5aa4c1f2ef4d6af06fa4e32d 2f057d729da61e8c12828df44377f0a93fed820f 39a1f1462b0abf12dfaccd33f84142c852c4b56e 4b12d015e1c316b4837e02ae37e2c69a8a647180 7f1467e795b1cab8b4855e019910c509896ea0e1 62da76cb5dca4d52c43bee06230cca6a5882f05d cc2ae9f270befa554ba8b09c68e89bb8248ea650 ae98d2ba2fcefa9d027e2d6ccc6e7558a32e7228 ae189cfab8d9036e8d39bbd3f1b61b400d1fcd5b"
  ## Ref: https://github.com/spack/spack/commit/[hash]
  ## [hash]: [description]
  ## b5ef5c2eb5145020f9de1bcb964626ce6ac2d02e: geant4: version bumps for Geant4 11.1.0
@@ -27,6 +27,7 @@ variables:
  ## 62da76cb5dca4d52c43bee06230cca6a5882f05d: directives: depends_on should not admit anonymous specs
  ## cc2ae9f270befa554ba8b09c68e89bb8248ea650: Add a "maintainer" directive
  ## ae98d2ba2fcefa9d027e2d6ccc6e7558a32e7228: Support packages for using scitokens on OSG
+  ## ae189cfab8d9036e8d39bbd3f1b61b400d1fcd5b: geant4: new version 11.1.1

  ## EIC spack organization and repository, e.g. eic/eic-spack
  EICSPACK_ORGREPO: "eic/eic-spack"
@@ -38,15 +39,15 @@ variables:
  ## [hash]: [description]
  ## f892e2b5d7ea9d1f2e43741499e899ce21dd3d5a: py-minkowskiengine: new package

-  ## We need to enable Docker Buildkit to use cache mounts and better
-  ## build performance overall
-  DOCKER_BUILDKIT: 1
-
+  ## Local registry
+  CI_PUSH: 1
  ## Dockerhub registry
+  DH_REGISTRY: docker.io
  DH_REGISTRY_USER: eicweb
  DH_PUSH: 1
  ## GitHub registry
  GH_REGISTRY: ghcr.io
+  GH_REGISTRY_USER: eic
  GH_PUSH: 1

  ## Also export the raw builder image, will be stored on DH only
@@ -58,16 +59,27 @@ variables:
  DOCKER_WAIT_TIME: 5

  ## is this nightly or not?
-  NIGHTLY: 0
+  NIGHTLY:

  ## Force a rebuild without using cache
  FORCE_NOCACHE: 0

+  ## Default version and tags
+  ## - internal tag used for the CI. Also temporarily tagged
+  ##   on eicweb to communicate between jobs (removed in cleanup job)
+  ## - export tag to public registries, optional secondary export tag
+  VERSION: "${CI_COMMIT_REF_NAME}"
+  NIGHTLY_TAG: "nightly"
+  TESTING_TAG: "testing"
+  INTERNAL_TAG: "${TESTING_TAG}-${VERSION}"
+  EXPORT_TAG: ""
+
 stages:
  - config
  - base            ## base OS image
  - jug             ## jug container images
  - deploy          ## build/deploy singularity images
+  - benchmarks
  - test
  - finalize

@@ -91,6 +103,7 @@ default:
 workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+    - if: '$CI_PIPELINE_SOURCE == "web"'
    - if: '$CI_COMMIT_BRANCH == "master"'
    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-stable/'          ## main stable branch: vX.Y-stable
    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-[a-z]+-stable/'   ## special stable branch: vX.Y-acadia-stable (etc)
@@ -121,76 +134,40 @@ version:
  stage: config
  script:
    - |
-      VERSION=`head -n1 VERSION`
-      VERSION_FULL=${VERSION}
-      VERSION_SHORT=${VERSION%.*}
-      TESTING="testing"
-      NIGHTLY="nightly"
-    ## determine appropriate major docker tag for this scenario
-    - |
-      ## internal tag used for the CI. Also temporarily tagged
-      ## on eicweb to communicate between jobs (removed in cleanup job)
-      INTERNAL_TAG="testing-$VERSION"
-      ## main export tag, optional secondary export tag,
-      EXPORT_TAG=${TESTING}
-      EXPORT_TAG2=
-      ## nightly tag, only used in master
-      NIGHTLY_TAG=${NIGHTLY}
      if [ "x${CI_PIPELINE_SOURCE}" = "xmerge_request_event" ]; then
-        INTERNAL_TAG="unstable-mr-${CI_MERGE_REQUEST_ID}"
-        NIGHTLY_TAG="${INTERNAL_TAG}-nightly"
-        EXPORT_TAG="${VERSION_SHORT}-unstable"
-        EXPORT_TAG2="unstable-mr-${CI_MERGE_REQUEST_PROJECT_ID}-${CI_MERGE_REQUEST_IID}"
-      elif [[ "$CI_COMMIT_TAG" =~ ^v${VERSION} ]]; then
-        VERSION_FULL="${CI_COMMIT_TAG:1}"
-        INTERNAL_TAG="stable-tag-${VERSION_FULL}"
-        NIGHTLY_TAG=
-        EXPORT_TAG=${VERSION_FULL}
-        EXPORT_TAG2=
-      elif [ "$CI_COMMIT_BRANCH" = "v${VERSION_SHORT}-stable" ]; then
+        VERSION="unstable-mr-${CI_MERGE_REQUEST_PROJECT_ID}-${CI_MERGE_REQUEST_IID}"
+        INTERNAL_TAG="${VERSION}"
+        NIGHTLY_TAG="${VERSION}-nightly"
+        EXPORT_TAG="${VERSION}"
+      elif [[ "$CI_COMMIT_TAG" =~ ^v[0-9.]+ ]]; then
+        VERSION="${CI_COMMIT_TAG:1}"
+        INTERNAL_TAG="stable-tag-${VERSION}"
+        EXPORT_TAG="${VERSION}"
+      elif [[ "$CI_COMMIT_BRANCH" = "v[0-9.]+-stable" ]]; then
+        VERSION="${CI_COMMIT_BRANCH:1}"
+        INTERNAL_TAG="stable-br-${VERSION}"
+        EXPORT_TAG="${VERSION}"
+      elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+-[a-z]+-stable ]]; then
+        VERSION="${CI_COMMIT_BRANCH:1}"
        INTERNAL_TAG="stable-br-${VERSION}"
-        NIGHTLY_TAG=
-        EXPORT_TAG=${VERSION_SHORT}-stable
-        EXPORT_TAG2=
-      elif [[ "$CI_COMMIT_BRANCH" =~ ^v${VERSION_SHORT}-[a-z]+-stable ]]; then
-        VERSION_FULL="${CI_COMMIT_BRANCH:1}"
-        INTERNAL_TAG="stable-br-${VERSION_FULL}"
-        NIGHTLY_TAG=
-        EXPORT_TAG=${VERSION_FULL}
-        EXPORT_TAG2=
+        EXPORT_TAG="${VERSION}"
      fi
-      echo "VERSION_FULL=$VERSION_FULL" >> build.env
-      echo "VERSION_SHORT=$VERSION_SHORT" >> build.env
+      echo "VERSION=$VERSION" >> build.env
      echo "INTERNAL_TAG=$INTERNAL_TAG" >> build.env
      echo "NIGHTLY_TAG=$NIGHTLY_TAG" >> build.env
      echo "EXPORT_TAG=$EXPORT_TAG" >> build.env
-      echo "EXPORT_TAG2=$EXPORT_TAG2" >> build.env
      cat build.env
-
  artifacts:
    reports:
      dotenv: build.env

-## base job for all build jobs. Dependent jobs are expected to set
-## the BUILD_IMAGE environment variable, and take care of the actual
-## docker build during the "script" step
-## note that the nightly builds use a different pipeline
+## base job settings for all build jobs
 .build:
  rules:
    - when: on_success
  resource_group: ${CI_COMMIT_REF_NAME}
-  ## Use docker runner for docker builds
  tags:
    - docker-new
-  ## cookie-cutter docker push code, to be included at the
-  ## end of the regular job scripts
-  script:
-    - ./gitlab-ci/docker_push.sh -i ${BUILD_IMAGE} -l ${INTERNAL_TAG}
-                                 -n $DOCKER_NTRIES -t $DOCKER_WAIT_TIME
-                                 ${EXPORT_TAG} ${EXPORT_TAG2}
-    - ./gitlab-ci/docker_push.sh -i ${BUILD_IMAGE} -l ${INTERNAL_TAG}
-                                 -n $DOCKER_NTRIES -t $DOCKER_WAIT_TIME
-                                 ${INTERNAL_TAG} --eicweb

 ## Images:
 ## debian_testing_base --> jug_dev  --> jug_xl
@@ -205,24 +182,18 @@ version:
 base:
  parallel:
    matrix:
-      - BASE_IMAGE: amd64/debian:testing-20220822-slim
-        BUILD_IMAGE: amd64_debian_testing_base
-        PLATFORM: linux/amd64
-      - BASE_IMAGE: amd64/ubuntu:20.04
-        BUILD_IMAGE: amd64_ubuntu_base
-        PLATFORM: linux/amd64
+      - BASE_IMAGE: debian:testing-20220822-slim
+        BUILD_IMAGE: debian_testing_base
+        PLATFORM: linux/amd64,linux/arm64/v8
+      - BASE_IMAGE: ubuntu:20.04
+        BUILD_IMAGE: ubuntu_base
+        PLATFORM: linux/amd64,linux/arm64/v8
      - BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04
        BUILD_IMAGE: oneapi_base
        PLATFORM: linux/amd64
 #      - BASE_IMAGE: nvidia/cuda:11.8.0-devel-ubuntu20.04
 #        BUILD_IMAGE: cuda_base
 #        PLATFORM: linux/amd64
-      - BASE_IMAGE: arm64v8/debian:testing-20220822-slim
-        BUILD_IMAGE: arm64v8_debian_testing_base
-        PLATFORM: linux/arm64/v8
-#      - BASE_IMAGE: debian:testing-20220822-slim
-#        BUILD_IMAGE: debian_testing_base_multiarch
-#        PLATFORM: linux/amd64,linux/arm64/v8
  extends: .build
  stage: base
  needs:
@@ -230,23 +201,25 @@ base:
  script:
    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
    - docker buildx create --name builder --driver docker-container --bootstrap --use
-    - docker buildx build --load -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
+    - docker buildx build --push
+                   -t ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
+                   ${EXPORT_TAG:+
+                     ${CI_PUSH:+-t ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
+                     ${DH_PUSH:+-t ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
+                     ${GH_PUSH:+-t ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
+                   }
                   -f containers/debian/base.Dockerfile
-                   ${PLATFORM:+--platform ${PLATFORM}}
+                   --platform ${PLATFORM}
                   --build-arg BASE_IMAGE=${BASE_IMAGE}
                   --build-arg BUILD_IMAGE=${BUILD_IMAGE}
                   containers/debian
-    - !reference [.build, script]

 jug_dev:default:
  parallel:
    matrix:
-      - BASE_IMAGE: amd64_debian_testing_base
+      - BASE_IMAGE: debian_testing_base
        BUILD_IMAGE: jug_dev
-        PLATFORM: linux/amd64
-      - BASE_IMAGE: arm64v8_debian_testing_base
-        BUILD_IMAGE: jug_dev_arm64v8
-        PLATFORM: linux/arm64/v8
+        PLATFORM: linux/amd64,linux/arm64/v8
 #      - BASE_IMAGE: oneapi_base
 #        BUILD_IMAGE: jug_dev_oneapi
 #        PLATFORM: linux/amd64
@@ -260,44 +233,36 @@ jug_dev:default:
    - base
  script:
    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-    ## calculate a hash based on the spack.yaml file and the spack directory
-    ## and use this spack as a docker variable to force a rebuild when there
-    ## is a change (versus rerun from cache)
-    - PACKAGE_HASH=$(sha1sum spack.yaml | head -c40)
-    - echo "PACKAGE_HASH=${PACKAGE_HASH}"
-    ## move spack.yaml into the container build directory
-    - cp -r spack.yaml containers/jug
+    - docker buildx create --name builder --driver docker-container --bootstrap --use
+    ## Which targets to build
+    - TARGETS="export"
+    - |
+      if [ ${EXPORT_BUILDER} = 1 ]; then
+        TARGETS="builder ${TARGETS}"
+      fi
+    ## Disable caching
    - CACHE_FLAG=""
    - |
      if [ $FORCE_NOCACHE = 1 ]; then
        echo "FORCE_NOCACHE set"
        export CACHE_FLAG="--no-cache"
      fi
-    ## Optionally build the raw builder image
-    - test ${EXPORT_BUILDER} = 1 && docker buildx build --load ${CACHE_FLAG}
-                   -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:builder-${INTERNAL_TAG}
-                   --target=builder
-                   -f containers/jug/dev.Dockerfile
-                   ${PLATFORM:+--platform ${PLATFORM}}
-                   --build-arg BASE_IMAGE="${BASE_IMAGE}"
-                   --build-arg SPACK_ORGREPO="${SPACK_ORGREPO}"
-                   --build-arg SPACK_VERSION="${SPACK_VERSION}"
-                   --build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
-                   --build-arg EICSPACK_ORGREPO="${EICSPACK_ORGREPO}"
-                   --build-arg EICSPACK_VERSION="${EICSPACK_VERSION}"
-                   --build-arg EICSPACK_CHERRYPICKS="${EICSPACK_CHERRYPICKS}"
-                   --build-arg S3_ACCESS_KEY="${S3_ACCESS_KEY}"
-                   --build-arg S3_SECRET_KEY="${S3_SECRET_KEY}"
-                   --build-arg S3RW_ACCESS_KEY="${S3RW_ACCESS_KEY}"
-                   --build-arg S3RW_SECRET_KEY="${S3RW_SECRET_KEY}"
-                   --build-arg CACHE_BUST=${PACKAGE_HASH}
-                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
-                   --build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD)
-                   containers/jug
-    ## now build our image
-    - docker buildx build --load -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG} ${CACHE_FLAG}
+    ## Build targets
+    - for target in ${TARGETS} ; do
+        if [ ! ${target} = export ] ; then
+          suffix=-${target} ;
+        fi ;
+        docker buildx build --push
+                   --target ${target}
+                   ${CACHE_FLAG}
+                   -t ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}${suffix}
+                   ${EXPORT_TAG:+
+                     ${CI_PUSH:+-t ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}${suffix}}
+                     ${DH_PUSH:+-t ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}${suffix}}
+                     ${GH_PUSH:+-t ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}${suffix}}
+                   }
                   -f containers/jug/dev.Dockerfile
-                   ${PLATFORM:+--platform ${PLATFORM}}
+                   --platform ${PLATFORM}
                   --build-arg BASE_IMAGE="${BASE_IMAGE}"
                   --build-arg SPACK_ORGREPO="${SPACK_ORGREPO}"
                   --build-arg SPACK_VERSION="${SPACK_VERSION}"
@@ -309,39 +274,18 @@ jug_dev:default:
                   --build-arg S3_SECRET_KEY="${S3_SECRET_KEY}"
                   --build-arg S3RW_ACCESS_KEY="${S3RW_ACCESS_KEY}"
                   --build-arg S3RW_SECRET_KEY="${S3RW_SECRET_KEY}"
-                   --build-arg CACHE_BUST=${PACKAGE_HASH}
                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
                   --build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD)
+                   --build-context spack=.
                   containers/jug
-    ## push builder image do DH if desired
-    - test ${EXPORT_BUILDER} = 1 && ./gitlab-ci/docker_push.sh
-                                 -i ${BUILD_IMAGE} -l builder-${INTERNAL_TAG}
-                                 -n ${DOCKER_NTRIES} -t ${DOCKER_WAIT_TIME}
-                                 builder-${EXPORT_TAG} --dockerhub
-    ## standard exports
-    - !reference [.build, script]
-
-jug_dev:multiarch:
-  parallel:
-    matrix:
-      - IMAGE: jug_dev
-        ARCHS: "amd64 arm64v8"
-  stage: jug
-  needs:
-    - version 
-    - jug_dev:default
-  script:
-    - echo docker manifest create ${IMAGE}:${EXPORT_TAG} $(echo ${ARCHS} | xargs -d " " -I "{}" echo -n "--amend ${IMAGE}:${EXPORT_TAG}-{}")
+      ; done

 jug_xl:default:
  parallel:
    matrix:
      - BASE_IMAGE: jug_dev
        BUILD_IMAGE: jug_xl
-        PLATFORM: linux/amd64
-      - BASE_IMAGE: jug_dev_arm64v8
-        BUILD_IMAGE: jug_xl_arm64v8
-        PLATFORM: linux/arm64/v8
+        PLATFORM: linux/amd64,linux/arm64/v8
 #      - BASE_IMAGE: jug_dev_oneapi
 #        BUILD_IMAGE: jug_xl_oneapi
 #        PLATFORM: linux/amd64
@@ -352,27 +296,30 @@ jug_xl:default:
    - jug_dev:default
  script:
    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+    - docker buildx create --name builder --driver docker-container --bootstrap --use
    - cp detectors.yaml containers/jug
-    - docker buildx build --load -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
+    - docker buildx build --push
+                   -t ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
+                   ${EXPORT_TAG:+
+                     ${CI_PUSH:+-t ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
+                     ${DH_PUSH:+-t ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
+                     ${GH_PUSH:+-t ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
+                   }
                   -f containers/jug/xl.Dockerfile
-                   ${PLATFORM:+--platform ${PLATFORM}}
+                   --platform ${PLATFORM}
                   --build-arg BASE_IMAGE=${BASE_IMAGE}
                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
                   --build-arg JUGGLER_VERSION=${JUGGLER_VERSION}
                   --build-arg EICRECON_VERSION=${EICRECON_VERSION}
                   --build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD)
                   containers/jug
-    - !reference [.build, script]

 jug_xl:nightly:
  parallel:
    matrix:
      - BASE_IMAGE: jug_dev
        BUILD_IMAGE: jug_xl
-        PLATFORM: linux/amd64
-      - BASE_IMAGE: jug_dev_arm64v8
-        BUILD_IMAGE: jug_xl_arm64v8
-        PLATFORM: linux/arm64/v8
+        PLATFORM: linux/amd64,linux/arm64/v8
 #      - BASE_IMAGE: jug_dev_oneapi
 #        BUILD_IMAGE: jug_xl_oneapi
 #        PLATFORM: linux/amd64
@@ -389,32 +336,31 @@ jug_xl:nightly:
    - jug_dev:default
  script:
    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+    - docker buildx create --name builder --driver docker-container --bootstrap --use
    - cp detectors.yaml containers/jug
-    - docker buildx build --load -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${NIGHTLY_TAG}
+    - |
+      PUSH_NIGHTLY_WITH_DATE=""
+      if [ "$CI_COMMIT_BRANCH" == "master" ]; then
+        PUSH_NIGHTLY_WITH_DATE="1"
+      fi
+    - docker buildx build --push
+                   -t ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
+                   ${NIGHTLY:+
+                     ${CI_PUSH:+-t ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
+                     ${DH_PUSH:+-t ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
+                     ${GH_PUSH:+-t ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
+                   }
+                   ${PUSH_NIGHTLY_WITH_DATE:+
+                     ${DH_PUSH:+-t ${DH_REGISTRY}/${DH_REGISTRY_USER}/${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
+                     ${GH_PUSH:+-t ${GH_REGISTRY}/${GH_REGISTRY_USER}/${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
+                   }
                   -f containers/jug/xl.Dockerfile
-                   ${PLATFORM:+--platform ${PLATFORM}}
+                   --platform ${PLATFORM}
                   --build-arg BASE_IMAGE=${BASE_IMAGE}
                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
-                   --build-arg EICRECON_VERSION=${EICRECON_VERSION}
                   --build-arg JUG_VERSION=nightly-$(date +%Y-%m-%d_%H-%M-%S)-${INTERNAL_TAG}-$(git rev-parse HEAD)
                   --build-arg NIGHTLY=1
                   containers/jug
-    - |
-      PUSH_FLAG=""
-      if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
-        PUSH_FLAG="--eicweb"
-      fi
-    - ./gitlab-ci/docker_push.sh -i ${BUILD_IMAGE} -l ${NIGHTLY_TAG}
-                                 -n $DOCKER_NTRIES -t $DOCKER_WAIT_TIME
-                                 ${NIGHTLY_TAG} ${PUSH_FLAG}
-    - if [ -z "${PUSH_FLAG}" ] ; then
-      ./gitlab-ci/docker_push.sh -i ${BUILD_IMAGE} -l ${NIGHTLY_TAG}
-                                 -n $DOCKER_NTRIES -t $DOCKER_WAIT_TIME
-                                 ${NIGHTLY_TAG}-$(date +%Y-%m-%d) --ghcr ;
-      ./gitlab-ci/docker_push.sh -i ${BUILD_IMAGE} -l ${NIGHTLY_TAG}
-                                 -n $DOCKER_NTRIES -t $DOCKER_WAIT_TIME
-                                 ${NIGHTLY_TAG}-$(date +%Y-%m-%d) --dockerhub ;
-      fi

 jug_xl:feature:
  extends: .build
@@ -446,7 +392,7 @@ jug_xl:feature:
      if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
        PUSH_FLAG="--eicweb"
      fi
-      FEATURE_EXPORT_TAG="${VERSION_SHORT}-${DETECTOR_BRANCH}-stable"
+      FEATURE_EXPORT_TAG="${VERSION}-${DETECTOR}-${DETECTOR_BRANCH}-stable"
    - ./gitlab-ci/docker_push.sh -i ${BUILD_IMAGE} -l ${INTERNAL_TAG}-${DETECTOR}-${DETECTOR_BRANCH}
                                 -n ${DOCKER_NTRIES} -t ${DOCKER_WAIT_TIME}
                                 ${FEATURE_EXPORT_TAG} ${PUSH_FLAG}
@@ -517,7 +463,41 @@ juggler:main:
    project: EIC/juggler
  allow_failure: true

-cleanup:
+benchmarks:reconstruction:
+  stage: benchmarks
+  rules:
+    - if:  '$CI_SERVER_HOST == "eicweb.phy.anl.gov"'
+  needs: 
+    - version
+    - jug_xl:default
+  variables:
+    BENCHMARKS_TAG: "$INTERNAL_TAG"
+    BENCHMARKS_CONTAINER: "jug_xl"
+    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
+  trigger:
+    project: EIC/benchmarks/reconstruction_benchmarks
+    strategy: depend
+  ## for now allow failure, as we need to update those pipelines
+  allow_failure: false
+  
+benchmarks:physics:
+  stage: benchmarks
+  rules:
+    - if:  '$CI_SERVER_HOST == "eicweb.phy.anl.gov"'
+  needs: 
+    - version
+    - jug_xl:default
+  variables:
+    RECO: "juggler"
+    BENCHMARKS_TAG: "$INTERNAL_TAG"
+    BENCHMARKS_CONTAINER: "jug_xl"
+    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
+  trigger:
+    project: EIC/benchmarks/physics_benchmarks
+    strategy: depend
+  allow_failure: false
+
+clean:
  stage: finalize
  dependencies:
    - version
@@ -532,3 +512,13 @@ cleanup:
      if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
        ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${NIGHTLY_TAG}
      fi
+    - docker images
+
+deep_clean:
+  stage: finalize
+  dependencies:
+    - clean
+  when: manual
+  script:
+    - docker system prune --force
+    - docker images
--- a/VERSION
+++ b/VERSION
-23.02.0
--- a/containers/jug/dev.Dockerfile
+++ b/containers/jug/dev.Dockerfile
-#syntax=docker/dockerfile:1.2
+#syntax=docker/dockerfile:1.4
 ARG DOCKER_REGISTRY="eicweb.phy.anl.gov:4567/containers/eic_container/"
 ARG BASE_IMAGE="debian_base"
 ARG INTERNAL_TAG="testing"
@@ -97,7 +97,7 @@ RUN git clone https://github.com/${EICSPACK_ORGREPO}.git ${EICSPACK_ROOT}     \
 && spack repo add --scope site "${EICSPACK_ROOT}"

 ## Setup our custom environment
-COPY spack.yaml /opt/spack-environment/
+COPY --from=spack spack.yaml /opt/spack-environment/
 RUN rm -r /usr/local                                                    \
 && source $SPACK_ROOT/share/spack/setup-env.sh                         \
 && spack env activate /opt/spack-environment/                          \
@@ -124,7 +124,7 @@ RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
 && status=0                                                            \
 && spack install -j64 --no-check-signature                             \
    || spack install -j64 --no-check-signature                          \
-    || spack install -j64 --no-check-signature                          \
+    || spack install -j64 --no-check-signature --show-log-on-error      \
    || status=$?                                                        \
 && spack mirror rm --scope site eic-spack                              \
 && [ -z "${CACHE_NUKE}" ]                                              \
@@ -222,6 +222,13 @@ RUN cd /opt/spack-environment                                           \
 && spack env activate .                                                \
 && spack gc -y

+# Garbage collect in git
+RUN cd $SPACK_ROOT                                                      \
+ && du -sh $SPACK_ROOT                                                  \
+ && git fetch --depth=1                                                 \
+ && git gc --prune=all --aggressive                                     \
+ && du -sh $SPACK_ROOT
+
 ## Bugfix to address issues loading the Qt5 libraries on Linux kernels prior to 3.15
 ## See
 #https://askubuntu.com/questions/1034313/ubuntu-18-4-libqt5core-so-5-cannot-open-shared-object-file-no-such-file-or-dir
@@ -252,7 +259,7 @@ RUN chmod a+x /usr/local/bin/mc
 ## STAGE 3
 ## Lean target image
 ## ========================================================================================
-FROM ${DOCKER_REGISTRY}${BASE_IMAGE}:${INTERNAL_TAG}
+FROM ${DOCKER_REGISTRY}${BASE_IMAGE}:${INTERNAL_TAG} as export
 ARG TARGETPLATFORM

 LABEL maintainer="Sylvester Joosten <sjoosten@anl.gov>" \
@@ -262,6 +269,7 @@ LABEL maintainer="Sylvester Joosten <sjoosten@anl.gov>" \
 ## copy over everything we need from staging in a single layer :-)
 RUN --mount=from=staging,target=/staging                                \
    rm -rf /usr/local                                                   \
+ && cp -r /staging/opt/spack /opt/spack                                 \
 && cp -r /staging/opt/spack-environment /opt/spack-environment         \
 && cp -r /staging/opt/software /opt/software                           \
 && cp -r /staging/usr/._local /usr/._local                             \

--- a/containers/jug/setup_detectors.py
+++ b/containers/jug/setup_detectors.py
@@ -122,6 +122,8 @@ if __name__ == '__main__':
                            detector=proj),
                       '&&',
                       'cmake -B /tmp/build -S /tmp/det -DCMAKE_CXX_STANDARD=17',
+                       '-DCMAKE_CXX_FLAGS="-Wno-psabi"',
+                       '-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache',
                       '-DCMAKE_INSTALL_PREFIX={prefix}'.format(prefix=prefix),
                       '&&',
                       'cmake --build /tmp/build -j$(($(($(nproc)/4))+1)) -- install']

--- a/containers/jug/xl.Dockerfile
+++ b/containers/jug/xl.Dockerfile
@@ -8,6 +8,7 @@ ARG INTERNAL_TAG="testing"
 ## EIC builder image with spack
 ## ========================================================================================
 FROM ${DOCKER_REGISTRY}${BASE_IMAGE}:${INTERNAL_TAG}
+ARG TARGETPLATFORM

 ARG EICWEB="https://eicweb.phy.anl.gov/api/v4/projects"
 ARG JUGGLER_VERSION="main"
@@ -21,14 +22,18 @@ RUN cd /tmp
 && echo " - jug_xl: ${JUG_VERSION}" >> /etc/jug_info

 ADD ${EICWEB}/369/repository/tree?ref=${JUGGLER_VERSION} /tmp/369.json
-RUN cd /tmp                                                                     \
+RUN --mount=type=cache,target=/ccache/                                          \
+    cd /tmp                                                                     \
 && echo "INSTALLING JUGGLER"                                                   \
 && git clone -b ${JUGGLER_VERSION} --depth 1                                   \
        https://eicweb.phy.anl.gov/EIC/juggler.git                              \
+ && export CCACHE_DIR=/ccache/$TARGETPLATFORM                                   \     
 && cmake -B build -S juggler                                                   \
+          -DCMAKE_CXX_FLAGS="-Wno-psabi"                                        \
          -DCMAKE_CXX_STANDARD=17                                               \
          -DCMAKE_INSTALL_PREFIX=/usr/local                                     \
          -DCMAKE_BUILD_TYPE=Release                                            \
+          -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
 && cmake --build build -j12 -- install                                         \
 && pushd juggler                                                               \
 && echo " - juggler: ${JUGGLER_VERSION}-$(git rev-parse HEAD)"                 \
@@ -37,14 +42,18 @@ RUN cd /tmp
 && rm -rf build juggler

 ADD https://api.github.com/repos/eic/eicrecon/commits/${EICRECON_VERSION} /tmp/eicrecon.json
-RUN cd /tmp                                                                     \
+RUN --mount=type=cache,target=/ccache/                                          \
+    cd /tmp                                                                     \
 && echo "INSTALLING EICRECON"                                                  \
 && git clone -b ${EICRECON_VERSION} --depth 1                                  \
        https://github.com/eic/eicrecon.git                                     \
+ && export CCACHE_DIR=/ccache/$TARGETPLATFORM                                   \
 && cmake -B build -S eicrecon                                                  \
+          -DCMAKE_CXX_FLAGS="-Wno-psabi"                                        \
          -DCMAKE_CXX_STANDARD=17                                               \
          -DCMAKE_INSTALL_PREFIX=/usr/local                                     \
          -DCMAKE_BUILD_TYPE=Release                                            \
+          -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
 && cmake --build build -j12 -- install                                         \
 && pushd eicrecon                                                              \
 && echo " - eicrecon: ${EICRECON_VERSION}-$(git rev-parse HEAD)"               \
@@ -121,10 +130,13 @@ ADD https://api.github.com/repos/eic/ip6 /tmp/ip6.json
 ADD https://api.github.com/repos/eic/epic /tmp/epic.json
 COPY setup_detectors.py /tmp
 COPY detectors.yaml /tmp
-RUN cd /tmp                                                                     \
+RUN --mount=type=cache,target=/ccache/                                          \
+    cd /tmp                                                                     \
+ && export CCACHE_DIR=/ccache/$TARGETPLATFORM                                   \
 && [ "z$NIGHTLY" = "z1" ] && NIGHTLY_FLAG="--nightly" || NIGHTLY_FLAG=""       \
 && /tmp/setup_detectors.py --prefix /opt/detector --config /tmp/detectors.yaml \
                         $NIGHTLY_FLAG                                          \
+ && ccache --show-stats                                                         \
 && rm /tmp/setup_detectors.py

 ## Hotfix for misbehaving OSG nodes

--- a/detectors.yaml
+++ b/detectors.yaml
@@ -39,3 +39,5 @@ detectors:
      version: 22.12.0
    23.01.0:
      version: 23.01.0
+    23.03.0:
+      version: 23.03.0
--- a/spack.yaml
+++ b/spack.yaml
@@ -17,17 +17,18 @@ spack:
    - edm4hep@0.7 cxxstd=17
    - eic-smear@1.1.10
    - eigen@3.4.0
-    - emacs
+    - emacs@28.2
    - fastjet@3.4.0
    - fmt@8.1.1 +shared cxxstd=17
    - gaudi@36.7 +aida
-    - geant4@11.1.0 cxxstd=17 +opengl +vecgeom +x11 +qt +threads
+    - geant4@11.1.1 cxxstd=17 +opengl +vecgeom +x11 +qt +threads
    - hepmc3@3.2.5 +python +rootio
    - heppdt@2.06.01
    - imagemagick@7.0.8-7
    - intel-tbb@2020.3
    - irt@1.0.0
-    - jana2@2.0.8 -ipo +root +zmq
+    - jana2@2.0.9 -ipo +root +zmq
+    - lcov@1.15
    - madx@5.08.01
    - mesa@22.1.2 +glx -llvm +opengl +opengles +osmesa
    - nlohmann-json@3.11.2
@@ -35,7 +36,7 @@ spack:
    - opencascade@7.6.3
    - osg-ca-certs@1.109.igtf.1.117
    - pkg-config@0.29.2
-    - podio@0.16
+    - podio@0.16.2
    - prmon@3.0.2 +plot
    - pythia8@8.306 +fastjet
    - python@3.10.8
No results found