Compare revisions

Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,15 +3,43 @@ image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
 variables:
  ## Application versions used for the main release 
  ## note: nightly builds will always use the master/main branch
-  JUGGLER_VERSION: "v9.3.0"
+  JUGGLER_VERSION: "v9.4.0"
-  EICRECON_VERSION: "v0.6.2"
+  EICRECON_VERSION: "v1.0.0"
  ## Spack organization and repository, e.g. spack/spack
  SPACK_ORGREPO: "spack/spack"
  ## Spack github version, e.g. v0.18.1 or commit hash
-  SPACK_VERSION: "v0.19.1"
+  SPACK_VERSION: "v0.19.2"
  ## Space-separated list of spack cherry-picks
-  SPACK_CHERRYPICKS: "b5ef5c2eb5145020f9de1bcb964626ce6ac2d02e 99056e03bd3e903b222e300636ec484d85d4b3fb f3f8b31be54280c6ef0b5f95ab85604aba3dff30 0ea81affd18820933640279bbc687038b3296a4e dceb4c9d655d0529e112b8929558be60973b39f7 f2b0c1deab87da3b5aa4c1f2ef4d6af06fa4e32d 2f057d729da61e8c12828df44377f0a93fed820f 39a1f1462b0abf12dfaccd33f84142c852c4b56e 4b12d015e1c316b4837e02ae37e2c69a8a647180 7f1467e795b1cab8b4855e019910c509896ea0e1 62da76cb5dca4d52c43bee06230cca6a5882f05d cc2ae9f270befa554ba8b09c68e89bb8248ea650 ae98d2ba2fcefa9d027e2d6ccc6e7558a32e7228 ae189cfab8d9036e8d39bbd3f1b61b400d1fcd5b"
+  SPACK_CHERRYPICKS: >
+    b5ef5c2eb5145020f9de1bcb964626ce6ac2d02e
+    99056e03bd3e903b222e300636ec484d85d4b3fb
+    f3f8b31be54280c6ef0b5f95ab85604aba3dff30
+    0ea81affd18820933640279bbc687038b3296a4e
+    dceb4c9d655d0529e112b8929558be60973b39f7
+    f2b0c1deab87da3b5aa4c1f2ef4d6af06fa4e32d
+    2f057d729da61e8c12828df44377f0a93fed820f
+    39a1f1462b0abf12dfaccd33f84142c852c4b56e
+    4b12d015e1c316b4837e02ae37e2c69a8a647180
+    f83d47442dade012b1019840181b8dd459fd8edd
+    7f1467e795b1cab8b4855e019910c509896ea0e1
+    ebc24b7063ba9a8eb43b4424aac5143cf958d76f
+    a47ebe57841f13239e881ed69eab4949b1d98c32
+    ab999d5af90f1bff644b5134bb370b2716e1bcf0
+    62da76cb5dca4d52c43bee06230cca6a5882f05d
+    cc2ae9f270befa554ba8b09c68e89bb8248ea650
+    ae98d2ba2fcefa9d027e2d6ccc6e7558a32e7228
+    ae189cfab8d9036e8d39bbd3f1b61b400d1fcd5b
+    3afe6f1adcc24335cbca9a9c03ffea188f802766
+    559c3de213707b5d52d899fd0382495f2cc8508d
+    8e84dcd7ef999e2659822b34372515175f1723c4
+    65bd9b9ac556480b4a9dcc60f7539492af195d4a
+    1a32cea11495cbdd699fea4fe622babab83e630d
+    6edc4807369a05786e36f63b5d959588ae94a1fa
+    af74680405c931dab16c6674f9b97a32bf3f1122
+    0a952f8b7bf6f70009dd5821bccbaf9170c73d07
+    f050b1cf7835fd31992b020e1061c52294ff7330
+    a419ffcf501134faed24253ccc83e6c71f9659f9
  ## Ref: https://github.com/spack/spack/commit/[hash]
  ## [hash]: [description]
  ## b5ef5c2eb5145020f9de1bcb964626ce6ac2d02e: geant4: version bumps for Geant4 11.1.0
@@ -23,11 +51,25 @@ variables:
  ## 2f057d729da61e8c12828df44377f0a93fed820f: py-scipy: add v1.9
  ## 39a1f1462b0abf12dfaccd33f84142c852c4b56e: SIP build system: fix "python not defined in builder"
  ## 4b12d015e1c316b4837e02ae37e2c69a8a647180: py-jinja2-cli: new package
+  ## f83d47442dade012b1019840181b8dd459fd8edd: dd4hep: depends_on root +x +opengl when +utilityapps
  ## 7f1467e795b1cab8b4855e019910c509896ea0e1: dd4hep: new version 1.24, depends_on podio@0.16:
+  ## ebc24b7063ba9a8eb43b4424aac5143cf958d76f: dd4hep: extend conflict on CMake
+  ## a47ebe57841f13239e881ed69eab4949b1d98c32: dd4hep: new versions 1.25, 1.25.1
+  ## ab999d5af90f1bff644b5134bb370b2716e1bcf0: dd4hep: depends_on root +webgui when +ddeve ^root @6.28:
  ## 62da76cb5dca4d52c43bee06230cca6a5882f05d: directives: depends_on should not admit anonymous specs
  ## cc2ae9f270befa554ba8b09c68e89bb8248ea650: Add a "maintainer" directive
  ## ae98d2ba2fcefa9d027e2d6ccc6e7558a32e7228: Support packages for using scitokens on OSG
  ## ae189cfab8d9036e8d39bbd3f1b61b400d1fcd5b: geant4: new version 11.1.1
+  ## 3afe6f1adcc24335cbca9a9c03ffea188f802766: ROOT: add math/gsl conflict and change version-dependent features to conditional variants
+  ## 559c3de213707b5d52d899fd0382495f2cc8508d: ROOT: new versions and associated dependency constraints
+  ## 8e84dcd7ef999e2659822b34372515175f1723c4: root: new version 6.28.00
+  ## 65bd9b9ac556480b4a9dcc60f7539492af195d4a: podio, edm4hep: add v0.7.2 and v0.16.1 respectively
+  ## 1a32cea11495cbdd699fea4fe622babab83e630d: podio: add v0.16.2
+  ## 6edc4807369a05786e36f63b5d959588ae94a1fa: podio: Add version 0.16.3
+  ## af74680405c931dab16c6674f9b97a32bf3f1122: depfile: improve tab completion
+  ## 0a952f8b7bf6f70009dd5821bccbaf9170c73d07: docs updates for spack env depfile
+  ## f050b1cf7835fd31992b020e1061c52294ff7330: depfile: variable with all identifiers
+  ## a419ffcf501134faed24253ccc83e6c71f9659f9: osg-ca-certs: igtf link should point to version, not 'current'
  ## EIC spack organization and repository, e.g. eic/eic-spack
  EICSPACK_ORGREPO: "eic/eic-spack"
@@ -165,9 +207,18 @@ version:
 .build:
  rules:
    - when: on_success
-  resource_group: ${CI_COMMIT_REF_NAME}
+  resource_group: ${CI_COMMIT_REF_NAME}-${VERSION}
  tags:
    - docker-new
+  before_script:
+    - !reference [default, before_script]
+    - mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc ;
+      for arch in aarch64 ; do
+        if ! grep --quiet enabled /proc/sys/fs/binfmt_misc/qemu-$arch ; then
+          docker run --rm --privileged multiarch/qemu-user-static --persistent yes ;
+        fi ;
+      done
+    - docker buildx create --name builder --driver docker-container --bootstrap --use
 ## Images:
 ## debian_testing_base --> jug_dev  --> jug_xl
@@ -199,8 +250,6 @@ base:
  needs:
    - version
  script:
-    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-    - docker buildx create --name builder --driver docker-container --bootstrap --use
    - docker buildx build --push
                   -t ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
                   ${EXPORT_TAG:+
@@ -232,8 +281,6 @@ jug_dev:default:
    - version
    - base
  script:
-    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-    - docker buildx create --name builder --driver docker-container --bootstrap --use
    ## Which targets to build
    - TARGETS="export"
    - |
@@ -295,9 +342,6 @@ jug_xl:default:
    - version
    - jug_dev:default
  script:
-    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-    - docker buildx create --name builder --driver docker-container --bootstrap --use
-    - cp detectors.yaml containers/jug
    - docker buildx build --push
                   -t ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
                   ${EXPORT_TAG:+
@@ -312,6 +356,7 @@ jug_xl:default:
                   --build-arg JUGGLER_VERSION=${JUGGLER_VERSION}
                   --build-arg EICRECON_VERSION=${EICRECON_VERSION}
                   --build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD)
+                   --build-context detectors=.
                   containers/jug
 jug_xl:nightly:
@@ -335,9 +380,6 @@ jug_xl:nightly:
    - version
    - jug_dev:default
  script:
-    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-    - docker buildx create --name builder --driver docker-container --bootstrap --use
-    - cp detectors.yaml containers/jug
    - |
      PUSH_NIGHTLY_WITH_DATE=""
      if [ "$CI_COMMIT_BRANCH" == "master" ]; then
@@ -349,10 +391,10 @@ jug_xl:nightly:
                     ${CI_PUSH:+-t ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
                     ${DH_PUSH:+-t ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
                     ${GH_PUSH:+-t ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
-                   }
+                     ${PUSH_NIGHTLY_WITH_DATE:+
-                   ${PUSH_NIGHTLY_WITH_DATE:+
+                       ${DH_PUSH:+-t ${DH_REGISTRY}/${DH_REGISTRY_USER}/${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
-                     ${DH_PUSH:+-t ${DH_REGISTRY}/${DH_REGISTRY_USER}/${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
+                       ${GH_PUSH:+-t ${GH_REGISTRY}/${GH_REGISTRY_USER}/${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
-                     ${GH_PUSH:+-t ${GH_REGISTRY}/${GH_REGISTRY_USER}/${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
+                     }
                   }
                   -f containers/jug/xl.Dockerfile
                   --platform ${PLATFORM}
@@ -360,6 +402,7 @@ jug_xl:nightly:
                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
                   --build-arg JUG_VERSION=nightly-$(date +%Y-%m-%d_%H-%M-%S)-${INTERNAL_TAG}-$(git rev-parse HEAD)
                   --build-arg NIGHTLY=1
+                   --build-context detectors=.
                   containers/jug
 jug_xl:feature:
@@ -380,7 +423,6 @@ jug_xl:feature:
        DETECTOR_BRANCH:
          - main
  script:
-    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
    - docker buildx build --load -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}-${DETECTOR}-${DETECTOR_BRANCH}
                   -f containers/jug/feature.Dockerfile
                   --build-arg INTERNAL_TAG=${NIGHTLY_TAG}
@@ -400,7 +442,6 @@ jug_xl:feature:
 .singularity:
  stage: deploy
  interruptible: true
-  resource_group: ${CI_COMMIT_REF_NAME}
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never
@@ -444,25 +485,6 @@ jug_xl:singularity:nightly:
    - mkdir build
    - singularity pull build/${BUILD_IMAGE}.sif docker://${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${NIGHTLY_TAG}
-## trigger juggler rebuild on nightly schedule to ensure both images remain in sync
-juggler:main:
-  stage: deploy
-  rules:
-    - if: '$NIGHTLY != "0" && $CI_COMMIT_BRANCH == "master"'
-      when: on_success
-    - when: never
-  needs:
-    - version
-    - jug_xl:nightly
-  variables:
-    TRIGGERED_BY_NIGHTLY: 1
-    JUGGLER_VERSION: main
-    DETECTOR_VERSION: master
-    IP6_VERSION: master
-  trigger:
-    project: EIC/juggler
-  allow_failure: true
 benchmarks:reconstruction:
  stage: benchmarks
  rules:
@@ -477,7 +499,6 @@ benchmarks:reconstruction:
  trigger:
    project: EIC/benchmarks/reconstruction_benchmarks
    strategy: depend
-  ## for now allow failure, as we need to update those pipelines
  allow_failure: false
 benchmarks:physics:
@@ -488,7 +509,7 @@ benchmarks:physics:
    - version
    - jug_xl:default
  variables:
-    RECO: "juggler"
+    RECO: "eicrecon"
    BENCHMARKS_TAG: "$INTERNAL_TAG"
    BENCHMARKS_CONTAINER: "jug_xl"
    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
@@ -501,6 +522,8 @@ clean:
  stage: finalize
  dependencies:
    - version
+  tags:
+    - docker-new
  when: always
  script:
    ## remove the pipeline specific export from eicweb if needed
@@ -512,13 +535,68 @@ clean:
      if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
        ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${NIGHTLY_TAG}
      fi
-    - docker images
-deep_clean:
+clean_unstable_mr:
  stage: finalize
  dependencies:
    - clean
-  when: manual
+  tags:
+    - docker-new
+  when: always
+  variables:
+    API: "https://eicweb.phy.anl.gov/api/v4"
  script:
+    - apt update -q
+    - apt install -yq jq
+    - for registry in
+        eicweb
+        eicweb.phy.anl.gov:4567/eic/juggler
+        eicweb.phy.anl.gov:4567/containers/eic_container
+      ; do 
+        docker images 
+          --filter=reference=${registry}/*:unstable-mr*
+          --format "{{.ID}} {{.Repository}} {{.Tag}}" ;
+      done | while read id repository tag ; do
+        if [[ ${tag} =~ ^unstable-mr-([0-9]+)-([0-9]+)(-nightly){0,1}$ ]] ; then
+          project_id=${BASH_REMATCH[1]} ;
+          merge_request_iid=${BASH_REMATCH[2]} ;
+          if [[ $(curl --silent -L
+                  "${API}/projects/${project_id}/merge_requests?scope=all&state=merged&per_page=100"
+                  "${API}/projects/${project_id}/merge_requests?scope=all&state=closed&per_page=100"
+                  | jq -r ".[]|select(.iid == ${merge_request_iid})|.state") =~ ^(merged|closed)$ ]] ; then
+            docker rmi $repository:$tag ;
+          else
+            echo "Image $repository:$tag still in use (project ${project_id}, idd ${merge_request_iid})." ;
+          fi ;
+        fi ;
+        if [[ ${tag} =~ ^unstable-mr-([0-9]+)(-nightly){0,1}$ ]] ; then
+          merge_request_id=${BASH_REMATCH[1]} ;
+          suffix=${BASH_REMATCH[2]} ;
+          if [[ $(curl --silent -L
+                  "${API}/projects/290/merge_requests?scope=all&state=merged&per_page=100"
+                  "${API}/projects/369/merge_requests?scope=all&state=merged&per_page=100"
+                  "${API}/projects/290/merge_requests?scope=all&state=closed&per_page=100"
+                  "${API}/projects/369/merge_requests?scope=all&state=closed&per_page=100"
+                  | jq -r ".[]|select(.id == ${merge_request_id})|.state") =~ ^(merged|closed)$ ]] ; then
+            docker rmi $repository:$tag ;
+          else
+            project_id=$(curl --silent -L
+                        "${API}/projects/290/merge_requests?scope=all&state=opened&per_page=100"
+                        "${API}/projects/369/merge_requests?scope=all&state=opened&per_page=100"
+                        | jq -r ".[]|select(.id == ${merge_request_id})|.project_id") ;
+            merge_request_iid=$(curl --silent -L 
+                        "${API}/projects/290/merge_requests?scope=all&state=opened&per_page=100"
+                        "${API}/projects/369/merge_requests?scope=all&state=opened&per_page=100"
+                        | jq -r ".[]|select(.id == ${merge_request_id})|.iid") ;
+            echo "Image $repository:$tag still in use (project ${project_id}, idd ${merge_request_iid})." ;
+            if [[ ${project_id} == "" || ${merge_request_iid} == "" ]] ; then
+              echo "$tag not found" ;
+            else
+              docker tag $repository:$tag $repository:unstable-mr-${project_id}-${merge_request_iid}${suffix} ;
+              docker rmi $repository:$tag ;
+            fi ;
+          fi ;
+        fi ;
+      done
    - docker system prune --force
    - docker images
--- a/containers/debian/bashrc
+++ b/containers/debian/bashrc
@@ -394,7 +394,7 @@ function __setprompt
 	fi
 	# Date
-	PS1+="\[${DARKGRAY}\](\[${CYAN}\]\$(date +%a) $(date +%b-'%-m')" # Date
+	PS1+="\[${DARKGRAY}\](\[${CYAN}\]\$(date +%a) $(date +%b-'%-d')" # Date
 	PS1+="${BLUE} $(date +'%-I':%M:%S%P)\[${DARKGRAY}\])-" # Time
 	# CPU

--- a/containers/jug/dev.Dockerfile
+++ b/containers/jug/dev.Dockerfile
@@ -11,7 +11,8 @@ FROM ${DOCKER_REGISTRY}${BASE_IMAGE}:${INTERNAL_TAG} as builder
 ARG TARGETPLATFORM
 ## install some extra spack dependencies
-RUN --mount=type=cache,target=/var/cache/apt                            \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${TARGETPLATFORM} \
+    --mount=type=cache,target=/var/lib/apt/lists,sharing=locked,id=${TARGETPLATFORM} \
    rm -f /etc/apt/apt.conf.d/docker-clean                              \
 && apt-get -yqq update                                                 \
 && apt-get -yqq install --no-install-recommends                        \
@@ -43,6 +44,7 @@ RUN git clone https://github.com/${SPACK_ORGREPO}.git ${SPACK_ROOT}     \
 SHELL ["docker-shell"]
+ARG jobs=64
 RUN declare -A arch=(                                                   \
      ["linux/amd64"]="x86_64"                                          \
      ["linux/arm64"]="aarch64"                                         \
@@ -51,9 +53,11 @@ RUN declare -A arch=(                                                   \
 && spack config --scope site add "packages:all:require:arch=${arch}"   \
 && spack config blame packages                                         \
 && spack config --scope site add "config:suppress_gpg_warnings:true"   \
- && spack config --scope site add "config:build_jobs:64"                \
+ && spack config --scope site add "config:build_jobs:${jobs}"           \
 && spack config --scope site add "config:install_tree:root:/opt/software" \
- && spack config blame config
+ && spack config blame config                                           \
+ && spack compiler find --scope site                                    \
+ && spack config blame compilers
 ## Setup spack buildcache mirrors, including an internal
 ## spack mirror using the docker build cache, and
@@ -97,86 +101,42 @@ RUN git clone https://github.com/${EICSPACK_ORGREPO}.git ${EICSPACK_ROOT}     \
 && spack repo add --scope site "${EICSPACK_ROOT}"
 ## Setup our custom environment
-COPY --from=spack spack.yaml /opt/spack-environment/
+COPY --from=spack spack-environment/ /opt/spack-environment/
-RUN rm -r /usr/local                                                    \
+ARG ENV=dev
+RUN --mount=type=cache,target=/var/cache/spack-mirror,sharing=locked    \
+    cd /opt/spack-environment                                           \
 && source $SPACK_ROOT/share/spack/setup-env.sh                         \
- && spack env activate /opt/spack-environment/                          \
+ && spack env activate --dir /opt/spack-environment/${ENV}              \
- && spack concretize --fresh
+ && make --jobs ${jobs} --keep-going --directory /opt/spack-environment \
+    SPACK_ENV=${ENV}                                                    \
+    BUILDCACHE_DIR=/var/cache/spack-mirror
+# FIXME disabled S3 buildcache until multipart upload fixed
+#                              \
+#    BUILDCACHE_MIRROR=eic-spack
+## Create view at /usr/local
+RUN --mount=type=cache,target=/var/cache/spack-mirror,sharing=locked    \
+    cd /opt/spack-environment                                           \
+ && source $SPACK_ROOT/share/spack/setup-env.sh                         \
+ && spack env activate --dir /opt/spack-environment/${ENV}              \
+ && rm -r /usr/local                                                    \
+ && spack env view enable /usr/local
-## Now execute the main build (or fetch from cache if possible)
-## note, no-check-signature is needed to allow the quicker signature-less
-## packages from the internal (docker) buildcache
-##
 ## Optional, nuke the buildcache after install, before (re)caching
 ## This is useful when going to completely different containers,
 ## or intermittently to keep the buildcache step from taking too much time
-##
+RUN --mount=type=cache,target=/var/cache/spack-mirror,sharing=locked    \
-## Update the local build cache if needed. Consists of 3 steps:
+    [ -z "${CACHE_NUKE}" ]                                              \
-## 1. Remove the eic-spack buildcache on S3
+    || rm -rf /var/cache/spack-mirror/build_cache/*
-## 2. Get a list of all packages, and compare with what is already on
-##    the buildcache (using package hash)
-## 3. Add packages that need to be added to buildcache if any
-RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
-    cd /opt/spack-environment                                           \
- && source $SPACK_ROOT/share/spack/setup-env.sh                         \
- && spack env activate .                                                \
- && status=0                                                            \
- && spack install -j64 --no-check-signature                             \
-    || spack install -j64 --no-check-signature                          \
-    || spack install -j64 --no-check-signature --show-log-on-error      \
-    || status=$?                                                        \
- && spack mirror rm --scope site eic-spack                              \
- && [ -z "${CACHE_NUKE}" ]                                              \
-    || rm -rf /var/cache/spack-mirror/build_cache/*                     \
- && mkdir -p /var/cache/spack-mirror/build_cache                        \
- && spack buildcache update-index -d /var/cache/spack-mirror            \
- && spack buildcache list --allarch --very-long                         \
-    | sed '/^$/d;/^--/d;s/@.\+//;s/\([a-z0-9]*\) \(.*\)/\2\/\1/'        \
-    | sort > buildcache.local.txt                                       \
- && spack find --format {name}/{hash} | sort                            \
-    | comm -23 - buildcache.local.txt                                   \
-    | xargs --no-run-if-empty                                           \
-      spack buildcache create --allow-root --only package --unsigned    \
-                              --directory /var/cache/spack-mirror       \
-                              --rebuild-index                           \
- && spack clean -a                                                      \
- && exit $status
-## Update the S3 build cache (without local cache mount)
-ARG S3RW_ACCESS_KEY=""
-ARG S3RW_SECRET_KEY=""
-RUN cd /opt/spack-environment                                           \
- && source $SPACK_ROOT/share/spack/setup-env.sh                         \
- && spack env activate .                                                \
- && if [ -n "${S3RW_ACCESS_KEY}" ] ; then                               \
-    spack mirror add --scope site                                       \
-      --s3-endpoint-url https://eics3.sdcc.bnl.gov:9000                 \
-      --s3-access-key-id "${S3RW_ACCESS_KEY}"                           \
-      --s3-access-key-secret "${S3RW_SECRET_KEY}"                       \
-      eic-spack s3://eictest/EPIC/spack                                 \
- && spack mirror list                                                   \
- && spack buildcache list --allarch --very-long                         \
-    | sed '/^$/d;/^--/d;s/@.\+//;s/\([a-z0-9]*\) \(.*\)/\2\/\1/'        \
-    | sort > buildcache.eic-spack.txt                                   \
- && spack find --format {name}/{hash} | sort                            \
-    | comm -23 - buildcache.eic-spack.txt                               \
-    | xargs --no-run-if-empty                                           \
-      spack buildcache create --allow-root --only package --unsigned    \
-                              --mirror-name eic-spack                   \
- && spack buildcache update-index --mirror-url eic-spack                \
- && spack mirror rm --scope site eic-spack                              \
-    ; fi                                                                \
- && spack mirror list
 ## Extra post-spack steps:
 ##   - Python packages
 COPY requirements.txt /usr/local/etc/requirements.txt
-RUN --mount=type=cache,target=/var/cache/pip                            \
+RUN --mount=type=cache,target=/var/cache/pip,sharing=locked,id=${TARGETPLATFORM} \
    echo "Installing additional python packages"                        \
 && cd /opt/spack-environment                                           \
 && source $SPACK_ROOT/share/spack/setup-env.sh                         \
- && spack env activate .                                                \
+ && spack env activate --dir /opt/spack-environment/${ENV}              \
 && python -m pip install                                               \
    --trusted-host pypi.org                                             \
    --trusted-host files.pythonhosted.org                               \
@@ -185,25 +145,12 @@ RUN --mount=type=cache,target=/var/cache/pip                            \
    --no-warn-script-location
    # ^ Supress not on PATH Warnings
-## Including some small fixes:
+## Including some small fixes
-##   - Somehow PODIO env isn't automatically set, 
-##   - and Gaudi likes BINARY_TAG to be set
 RUN cd /opt/spack-environment                                           \
 && source $SPACK_ROOT/share/spack/setup-env.sh                         \
- && echo -n ""                                                          \
 && echo "Grabbing environment info"                                    \
- && spack env activate --sh -d .                                        \
+ && spack env activate --sh --dir /opt/spack-environment/${ENV}         \
-        | sed "s?LD_LIBRARY_PATH=?&/lib/x86_64-linux-gnu:?"             \
+    > /etc/profile.d/z10_spack_environment.sh
-        | sed '/MANPATH/ s/;$/:;/'                                      \
-    > /etc/profile.d/z10_spack_environment.sh                           \
- && cd /opt/spack-environment && spack env activate .                   \
- && echo -n ""                                                          \
- && echo "Add extra environment variables for Jug, Podio and Gaudi"     \
- && echo "export PODIO=$(spack location -i podio);"                     \
-        >> /etc/profile.d/z10_spack_environment.sh                      \
- && echo -n ""                                                          \
- && echo "Executing cmake patch for dd4hep 16.1"                        \                
- && sed -i "s/FIND_PACKAGE(Python/#&/" /usr/local/cmake/DD4hepBuild.cmake
 ## make sure we have the entrypoints setup correctly
 ENTRYPOINT []
@@ -219,7 +166,7 @@ FROM builder as staging
 # Garbage collect in environment
 RUN cd /opt/spack-environment                                           \
 && source $SPACK_ROOT/share/spack/setup-env.sh                         \
- && spack env activate .                                                \
+ && spack env activate --dir /opt/spack-environment/${ENV}              \
 && spack gc -y
 # Garbage collect in git

--- a/containers/jug/setup_detectors.py
+++ b/containers/jug/setup_detectors.py
 #!/usr/bin/env python3
 import os
+import sys
 import yaml
 import argparse
+import subprocess
+from datetime import datetime
 DETECTOR_REPO_GROUP = 'https://github.com/eic'
 DETECTOR_BEAMLINE_ENV ='''
@@ -97,6 +100,7 @@ if __name__ == '__main__':
                        default_found = True
                print('    - {}: {}{}'.format(det, branch, default_str))
    print(' --> Building and installing detector/ip libraries')
+    process_list = []
    for det in detectors:
        if not args.nightly and 'nightly' in detectors[det]:
            del detectors[det]['nightly']    
@@ -128,7 +132,7 @@ if __name__ == '__main__':
                       '&&',
                       'cmake --build /tmp/build -j$(($(($(nproc)/4))+1)) -- install']
                print(' '.join(cmd))
-                os.system(' '.join(cmd))
+                subprocess.check_call(' '.join(cmd), shell=True)
                ## write version info to jug_info if available
                if os.path.exists('/etc/jug_info'):
                    cmd = ['cd /tmp/det',
@@ -141,17 +145,17 @@ if __name__ == '__main__':
                           '&&',
                           'cd -']
                    print(' '.join(cmd))
-                    os.system(' '.join(cmd))
+                    subprocess.check_call(' '.join(cmd), shell=True)
                ## also copy over IP configuration to the detector
                if 'ip' in cfg and os.path.exists('/tmp/det/{ip}'.format(ip=cfg['ip']['config'])):
                    cmd = 'cp -r /tmp/det/{ip} {data_dir}'.format(
                                    ip=ip['config'], data_dir=data_dir)
                    print(cmd)
-                    os.system(cmd)
+                    subprocess.check_call(cmd, shell=True)
                ## cleanup
                cmd = 'rm -rf /tmp/det /tmp/build'
                print(cmd)
-                os.system(cmd)
+                subprocess.check_call(cmd, shell=True)
            # be resilient against failures
            if os.path.exists(prefix):
                ## create a shortcut for the prefix if desired
@@ -160,7 +164,7 @@ if __name__ == '__main__':
                                prefix=prefix,
                                shortcut='{}/{}-{}'.format(args.prefix, det, branch))
                    print(cmd)
-                    os.system(cmd)
+                    subprocess.check_call(cmd, shell=True)
                ## write an environment file for this detector
                with open('{prefix}/setup.sh'.format(prefix=prefix), 'w') as f:
                    if 'ip' in cfg:
@@ -183,15 +187,31 @@ if __name__ == '__main__':
                            file=f)
                ## run once inside global prefix to initialize artifacts in /opt/detectors
                os.environ['DETECTOR_PATH'] = args.prefix
-                cmd = f'bash -c \'cd {args.prefix} && source {prefix}/setup.sh && checkGeometry -c {prefix}/share/{det}/{det}.xml\''
+                cmd = f'cd {args.prefix} && source {prefix}/setup.sh && checkGeometry -c {prefix}/share/{det}/{det}.xml'
                print(cmd)
-                os.system(cmd)
+                process_list.append(subprocess.Popen(cmd, shell=True, executable='/bin/bash', stdout=subprocess.PIPE, stderr=subprocess.STDOUT))
                ## run once inside specific prefix to initialize artifacts in $DETECTOR_PATH
                os.environ['DETECTOR_PATH'] = args.prefix
-                cmd = f'bash -c \'cd {prefix}/share/{det} && source {prefix}/setup.sh && checkGeometry -c {prefix}/share/{det}/{det}.xml\''
+                cmd = f'cd {prefix}/share/{det} && source {prefix}/setup.sh && checkGeometry -c {prefix}/share/{det}/{det}.xml'
                print(cmd)
-                os.system(cmd)
+                process_list.append(subprocess.Popen(cmd, shell=True, executable='/bin/bash', stdout=subprocess.PIPE, stderr=subprocess.STDOUT))
+    while len(process_list) > 0:
+        now = datetime.now()
+        current_time = now.strftime("%H:%M:%S")
+        print("{} processes running... ({})".format(len(process_list), current_time))
+        (out, err) = process_list[-1].communicate()
+        if process_list[-1].wait() != 0:
+            print(process_list[-1].args)
+            if out is not None:
+                print("stdout:")
+                print(out.decode())
+            if err is not None:
+                print("stderr:")
+                print(err.decode())
+            sys.exit(1)
+        process_list.pop()
    if not default_found and not args.nightly:
        # Skip symlinking if no defaults present and its not a nightly build
        pass
@@ -204,6 +224,6 @@ if __name__ == '__main__':
            '&&',
            'ln -sf {full_prefix}/setup.sh {short_prefix}']
        print(' '.join(cmd))
-        os.system(' '.join(cmd).format(full_prefix=full_prefix, short_prefix=args.prefix))
+        subprocess.check_call(' '.join(cmd).format(full_prefix=full_prefix, short_prefix=args.prefix), shell=True)
    print('All done!')
--- a/containers/jug/xl.Dockerfile
+++ b/containers/jug/xl.Dockerfile
-#syntax=docker/dockerfile:1.2
+#syntax=docker/dockerfile:1.4
 ARG DOCKER_REGISTRY="eicweb.phy.anl.gov:4567/containers/eic_container/"
 ARG BASE_IMAGE="jug_dev"
 ARG INTERNAL_TAG="testing"
@@ -22,12 +22,12 @@ RUN cd /tmp
 && echo " - jug_xl: ${JUG_VERSION}" >> /etc/jug_info
 ADD ${EICWEB}/369/repository/tree?ref=${JUGGLER_VERSION} /tmp/369.json
-RUN --mount=type=cache,target=/ccache/                                          \
+RUN --mount=type=cache,target=/ccache/,sharing=locked,id=${TARGETPLATFORM}      \
    cd /tmp                                                                     \
 && echo "INSTALLING JUGGLER"                                                   \
 && git clone -b ${JUGGLER_VERSION} --depth 1                                   \
        https://eicweb.phy.anl.gov/EIC/juggler.git                              \
- && export CCACHE_DIR=/ccache/$TARGETPLATFORM                                   \     
+ && export CCACHE_DIR=/ccache                                                   \     
 && cmake -B build -S juggler                                                   \
          -DCMAKE_CXX_FLAGS="-Wno-psabi"                                        \
          -DCMAKE_CXX_STANDARD=17                                               \
@@ -42,12 +42,12 @@ RUN --mount=type=cache,target=/ccache/
 && rm -rf build juggler
 ADD https://api.github.com/repos/eic/eicrecon/commits/${EICRECON_VERSION} /tmp/eicrecon.json
-RUN --mount=type=cache,target=/ccache/                                          \
+RUN --mount=type=cache,target=/ccache/,sharing=locked,id=${TARGETPLATFORM}      \
    cd /tmp                                                                     \
 && echo "INSTALLING EICRECON"                                                  \
 && git clone -b ${EICRECON_VERSION} --depth 1                                  \
        https://github.com/eic/eicrecon.git                                     \
- && export CCACHE_DIR=/ccache/$TARGETPLATFORM                                   \
+ && export CCACHE_DIR=/ccache                                                   \
 && cmake -B build -S eicrecon                                                  \
          -DCMAKE_CXX_FLAGS="-Wno-psabi"                                        \
          -DCMAKE_CXX_STANDARD=17                                               \
@@ -129,10 +129,10 @@ ADD ${EICWEB}/452/repository/tree?ref=master /tmp/452.json
 ADD https://api.github.com/repos/eic/ip6 /tmp/ip6.json
 ADD https://api.github.com/repos/eic/epic /tmp/epic.json
 COPY setup_detectors.py /tmp
-COPY detectors.yaml /tmp
+COPY --from=detectors detectors.yaml /tmp
-RUN --mount=type=cache,target=/ccache/                                          \
+RUN --mount=type=cache,target=/ccache/,sharing=locked,id=${TARGETPLATFORM}      \
    cd /tmp                                                                     \
- && export CCACHE_DIR=/ccache/$TARGETPLATFORM                                   \
+ && export CCACHE_DIR=/ccache                                                   \
 && [ "z$NIGHTLY" = "z1" ] && NIGHTLY_FLAG="--nightly" || NIGHTLY_FLAG=""       \
 && /tmp/setup_detectors.py --prefix /opt/detector --config /tmp/detectors.yaml \
                         $NIGHTLY_FLAG                                          \

--- a/detectors.yaml
+++ b/detectors.yaml
@@ -41,3 +41,5 @@ detectors:
      version: 23.01.0
    23.03.0:
      version: 23.03.0
+    23.03.0-alt-img:
+      version: alternative-geometries-img-ecal
--- a/spack-environment/Makefile
+++ b/spack-environment/Makefile
+MAKEFLAGS += -Orecurse
+SPACK ?= spack
+SPACK_INSTALL_FLAGS += --no-check-signature
+export SPACK_COLOR = always
+SPACK_ENV ?= dev
+BUILDCACHE_DIR := $(SPACK_ENV)/cache
+BUILDCACHE_MIRROR :=
+.PHONY: all clean
+all: $(SPACK_ENV)/push
+ifeq (,$(filter clean,$(MAKECMDGOALS)))
+include $(SPACK_ENV)/spack.mk
+endif
+$(SPACK_ENV)/push: $(addprefix $(SPACK_ENV)/push/,$($(SPACK_ENV)/SPACK_PACKAGE_IDS))
+	@mkdir -p $(BUILDCACHE_DIR)
+	$(foreach buildcache, $(BUILDCACHE_DIR), $(SPACK) buildcache update-index --mirror-url $(buildcache) ;)
+	$(foreach buildcache, $(BUILDCACHE_MIRROR), $(SPACK) buildcache update-index --mirror-url $(buildcache) ;)
+$(SPACK_ENV)/push/%: $(SPACK_ENV)/install/%
+	@mkdir -p $(dir $@)
+	$(foreach buildcache, $(BUILDCACHE_DIR), $(SPACK) buildcache create --allow-root --only=package --unsigned --directory $(buildcache) /$(HASH) ;) # push $(SPEC)
+	$(foreach buildcache, $(BUILDCACHE_MIRROR), $(SPACK) buildcache create --allow-root --only=package --unsigned --mirror-name $(buildcache) /$(HASH) ;) # push $(SPEC)
+	@touch $@
+$(SPACK_ENV)/spack.lock: $(SPACK_ENV)/spack.yaml Makefile
+	$(SPACK) concretize --force --fresh
+$(SPACK_ENV)/spack.mk: $(SPACK_ENV)/spack.lock Makefile
+	$(SPACK) env depfile --output $@ --make-target-prefix $(SPACK_ENV)
+clean:
+	rm -rf $(SPACK_ENV)/spack.lock $(SPACK_ENV)/spack.mk
--- a/spack-environment/concretizer.yaml
+++ b/spack-environment/concretizer.yaml
+concretizer: 
+  unify: true
--- a/spack.yaml
+++ b/spack.yaml
 spack:
+  include: 
+  - ../concretizer.yaml
+  - ../packages.yaml
  specs:
    - acts@21.1.0 +dd4hep +identification +json +tgeo +examples +fatras +geant4 +python
    - actsvg@0.4.26 +examples
    - acts-dd4hep@1.0.1
    - afterburner@0.1.2 +root +zlib
    - cairo@1.16.0 +fc+ft+X+pdf+gobject
+    - catch2@3.0.1 -ipo
    - clhep@2.4.6.0 cxxstd=17
    - cli11@2.1.1
    - cmake@3.24.3
@@ -12,31 +16,34 @@ spack:
    - cppcoro@git.10bbcdbf2be3ad3aa56febcf4c7662d771460a99
    - dawn@3_91a
    - dawncut@1_54a
-    - dd4hep@1.24 +ddg4 +ddcad -frames +hepmc3 +lcio
+    - dd4hep@1.25.1 +ddg4 +ddcad +hepmc3 +lcio +xercesc
-    - edm4eic@1.2.2 cxxstd=17
+    - edm4eic@2.0.0 cxxstd=17
    - edm4hep@0.7 cxxstd=17
    - eic-smear@1.1.10
    - eigen@3.4.0
    - emacs@28.2
    - fastjet@3.4.0
+    - fjcontrib@1.045
    - fmt@8.1.1 +shared cxxstd=17
    - gaudi@36.7 +aida
-    - geant4@11.1.1 cxxstd=17 +opengl +vecgeom +x11 +qt +threads
+    - geant4@11.1.1 cxxstd=17 +opengl -vecgeom +x11 +qt +threads
    - hepmc3@3.2.5 +python +rootio
    - heppdt@2.06.01
    - imagemagick@7.0.8-7
    - intel-tbb@2020.3
-    - irt@1.0.0
+    - irt@1.0.6
-    - jana2@2.0.9 -ipo +root +zmq
+    - jana2@2.1.0 -ipo +podio +root +zmq
    - lcov@1.15
    - madx@5.08.01
    - mesa@22.1.2 +glx -llvm +opengl +opengles +osmesa
    - nlohmann-json@3.11.2
    - npsim@1.1.0 +geocad +http
+    - onnx@1.12.0
    - opencascade@7.6.3
    - osg-ca-certs@1.109.igtf.1.117
+    - phonebook-cli@1.0.0
    - pkg-config@0.29.2
-    - podio@0.16.2
+    - podio@0.16.3
    - prmon@3.0.2 +plot
    - pythia8@8.306 +fastjet
    - python@3.10.8
@@ -55,6 +62,7 @@ spack:
    - py-numpy@1.23.4
    - py-pandas@1.5.1
    - py-pip@22.2.2
+    - py-pre-commit@2.20.0
    - py-pycairo@1.20.0
    - py-pyyaml@6.0
    - py-scipy@1.9.3
@@ -63,18 +71,11 @@ spack:
    - py-uproot@4.3.5
    - py-yapf@0.30.0
    - qt@5.15.5 +opengl
-    - root@6.26.06 cxxstd=17 
+    - root@6.26.10 cxxstd=17 
          +fftw +fortran +gdml +http -ipo +mlp +pythia8 
          +root7 +tmva +vc -webgui +xrootd +ssl 
    - spdlog@1.10.0 +fmt_external
    - stow@2.3.1
    - tensorflow-lite@2.8.0 -xnnpack
    - xrootd@5.5.1 cxxstd=17 +python +scitokens-cpp
-  concretizer: 
+  view: false
-    unify: true
-  packages:
-    all:
-      compiler: [gcc]
-      variants:
-        +ipo build_type=Release
-  view: /usr/local
--- a/spack-environment/packages.yaml
+++ b/spack-environment/packages.yaml
+packages:
+  all:
+    compiler: [gcc]
+    variants:
+      +ipo build_type=Release
No results found