.github/workflows/mirror.yaml

+name: Mirror and Trigger EICweb
+
+on:
+  pull_request:
+  push:
+  workflow_dispatch:
+
+concurrency:
+  group: mirror
+  cancel-in-progress: false
+
+jobs:
+  build:
+    name: Mirror and Trigger EICweb
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      contents: read
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Push to EICweb
+      uses: eic/gitlab-sync@master
+      with:
+        url: https://eicweb.phy.anl.gov/containers/eic_container.git/
+        token: ${{ secrets.GITLAB_TOKEN }}
+        username: ${{ secrets.GITLAB_USERNAME }}
+        ciskip: true
+    - name: Trigger EICweb
+      uses: eic/trigger-gitlab-ci@v2
+      with:
+        url: https://eicweb.phy.anl.gov
+        project_id: 290
+        token: ${{ secrets.EICWEB_EIC_CONTAINER_TRIGGER }}
+        ref_name: ${{ github.event.pull_request.head.ref || github.ref }}
+        variables:
+          GITHUB_REPOSITORY=${{ github.repository }}
+          GITHUB_SHA=${{ github.event.pull_request.head.sha || github.sha }}
+          GITHUB_PR=${{ github.event.pull_request.number }}

.gitlab-ci.yml

@@ -2,13 +2,13 @@ image: alpine
 
 variables:
   ## External images
-  DOCKER_IMAGE: docker.io/docker:24.0.5
-  SINGULARITY_IMAGE: quay.io/singularity/singularity:v3.11.4
+  DOCKER_IMAGE: docker.io/docker:24.0.7
+  SINGULARITY_IMAGE: quay.io/singularity/singularity:v3.11.5
 
   ## Application versions used for the main release 
   ## note: nightly builds will always use the master/main branch
-  JUGGLER_VERSION: "v11.0.0"
-  EICRECON_VERSION: "v1.9.0"
+  JUGGLER_VERSION: "v13.0.0"
+  EICRECON_VERSION: "v1.9.1"
 
   ## Local registry
   CI_PUSH: 1
@@ -159,19 +159,19 @@ status:pending:
 ## base job settings for all docker interactions
 .docker:
   image: ${DOCKER_IMAGE}
-  services: 
+  services:
     - docker:dind
   before_script:
     - # Login to registries
       if [[ -n "${EXPORT_TAG}" ]] ; then
         if [[ -n "${DH_REGISTRY}" ]] ; then
-          docker login -u ${DH_REGISTRY_USER} -p ${DH_EICWEB_TOKEN} ${DH_REGISTRY} ;
+          echo ${DH_EICWEB_TOKEN} | docker login -u ${DH_REGISTRY_USER} --password-stdin ${DH_REGISTRY} ;
         fi ;
         if [[ -n "${GH_REGISTRY}" ]] ; then
-          docker login -u ${GITHUB_REGISTRY_USER} -p ${GITHUB_REGISTRY_TOKEN} ${GH_REGISTRY} ;
+          echo ${GITHUB_REGISTRY_TOKEN} | docker login -u ${GITHUB_REGISTRY_USER} --password-stdin ${GH_REGISTRY} ;
         fi ;
         if [[ -n "${CI_REGISTRY}" ]] ; then
-          docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} ;
+          echo ${CI_REGISTRY_PASSWORD} | docker login -u ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY} ;
         fi ;
       fi
   after_script:
@@ -527,7 +527,50 @@ benchmarks:detector:nightly:
     project: EIC/benchmarks/detector_benchmarks
     strategy: depend
   allow_failure: false
-  
+
+df:
+  extends: .build
+  stage: config
+  tags:
+    - docker-new
+  needs: 
+    - version
+  script:
+    - docker images
+    - docker ps --size
+    - docker stats --all --no-stream --no-trunc > docker-stats-all.txt
+    - docker context ls
+    - docker volume ls
+    - docker volume ls --filter=name=buildx_buildkit_builder
+    - docker buildx ls
+    - docker buildx inspect
+    - docker buildx du
+    - docker buildx du --verbose > docker-buildx-du-verbose.txt
+    - docker system df
+    - docker system df --verbose > docker-system-df-verbose.txt
+  artifacts:
+    paths:
+      - docker-stats-all.txt
+      - docker-buildx-du-verbose.txt
+      - docker-system-df-verbose.txt
+
+prune: 
+  extends: .build
+  stage: finalize
+  tags:
+    - docker-new
+  needs:
+    - clean_internal_tag
+    - clean_unstable_mr
+    - clean_pipeline
+  rules:
+    - when: manual
+  script:
+    - docker buildx prune --all --force --filter=until=24h --verbose
+    - docker system prune --all --force --filter=until=24h
+    - docker system prune --volumes --force
+  allow_failure: true
+
 clean_internal_tag:
   image: alpine/curl
   stage: finalize
@@ -567,9 +610,6 @@ clean_unstable_mr:
         if [ "$state" == "merged" ] ; then docker rmi $repository:$tag ; fi ;
         if [ "$state" == "closed" ] ; then docker rmi $repository:$tag ; fi ;
       done
-    - docker system prune --filter until=24h --force
-    - docker system df
-    - docker images
 
 clean_pipeline:
   extends: .docker
@@ -600,9 +640,6 @@ clean_pipeline:
         if [ "$status" == "failed" ] ; then docker rmi $repository:$tag ; fi ;
         if [ "$status" == "canceled" ] ; then docker rmi $repository:$tag ; fi ;
       done
-    - docker system prune --filter until=24h --force
-    - docker system df
-    - docker images
 
 status:success:
   stage: status-report

containers/jug/dev.Dockerfile

@@ -127,7 +127,7 @@ FROM spack as builder
 COPY --from=spack-environment . /opt/spack-environment/
 ARG ENV=dev
 ARG JUGGLER_VERSION="main"
-ADD https://eicweb.phy.anl.gov/api/v4/projects/EIC%2Fjuggler/repository/commits/${JUGGLER_VERSION} /tmp/juggler.json
+ADD https://api.github.com/repos/eic/juggler/commits/${JUGGLER_VERSION} /tmp/juggler.json
 ARG EICRECON_VERSION="main"
 ADD https://api.github.com/repos/eic/eicrecon/commits/${EICRECON_VERSION} /tmp/eicrecon.json
 ENV SPACK_ENV=/opt/spack-environment/${ENV}
@@ -137,15 +137,20 @@ RUN --mount=type=cache,target=/ccache,id=${TARGETPLATFORM}              \
     <<EOF
 set -e
 export CCACHE_DIR=/ccache
+mkdir -p /var/cache/spack/blobs/sha256/
 find /var/cache/spack/blobs/sha256/ -atime +7 -delete
-JUGGLER_VERSION=$(jq -r .id /tmp/juggler.json)
+JUGGLER_VERSION=$(jq -r .sha /tmp/juggler.json)
 EICRECON_VERSION=$(jq -r .sha /tmp/eicrecon.json)
 spack buildcache update-index eics3rw
 spack env activate --dir ${SPACK_ENV}
 spack add juggler@git.${JUGGLER_VERSION}
 spack add eicrecon@git.${EICRECON_VERSION}
 spack concretize --fresh --force --quiet
-make --jobs ${jobs} --keep-going --directory /opt/spack-environment SPACK_ENV=${SPACK_ENV} BUILDCACHE_MIRROR_ONLY_PACKAGE="eics3rw" BUILDCACHE_MIRROR_DEPENDENCIES="eicweb ghcr"
+make --jobs ${jobs} --keep-going --directory /opt/spack-environment \
+  SPACK_ENV=${SPACK_ENV} \
+  BUILDCACHE_OCI_PROMPT="eicweb" \
+  BUILDCACHE_OCI_FINAL="ghcr" \
+  BUILDCACHE_S3_FINAL="eics3rw"
 ccache --show-stats
 ccache --zero-stats
 EOF

eic-spack.sh

@@ -3,7 +3,7 @@ EICSPACK_ORGREPO="eic/eic-spack"
 
 ## EIC spack commit hash or github version, e.g. v0.19.7
 ## note: nightly builds could use a branch e.g. releases/v0.19
-EICSPACK_VERSION="f3e18d6a2273029d16462575fe6cc4f4e1efeee0"
+EICSPACK_VERSION="5d8ba97ea6b021a50aa11e924a55a4726c689d65"
 
 ## Space-separated list of eic-spack cherry-picks
 read -r -d '' EICSPACK_CHERRYPICKS <<- \

spack-environment/Makefile

@@ -7,8 +7,10 @@ export SPACK_COLOR = always
 
 SPACK_ENV ?= dev
 
-BUILDCACHE_MIRROR_ONLY_PACKAGE :=
-BUILDCACHE_MIRROR_DEPENDENCIES :=
+BUILDCACHE_S3_PROMPT :=
+BUILDCACHE_S3_FINAL :=
+BUILDCACHE_OCI_PROMPT :=
+BUILDCACHE_OCI_FINAL :=
 BUILDCACHE_OCI_BASE_IMAGE := ghcr.io/eic/debian:stable-slim
 
 .PHONY: all clean
@@ -20,13 +22,15 @@ include $(SPACK_ENV)/spack.mk
 endif
 
 $(SPACK_ENV)/push: $(addprefix $(SPACK_ENV)/push/,$($(SPACK_ENV)/SPACK_PACKAGE_IDS))
-	$(foreach buildcache, $(BUILDCACHE_MIRROR_ONLY_PACKAGE), $(SPACK) buildcache update-index $(buildcache) ;)
-	$(foreach buildcache, $(BUILDCACHE_MIRROR_DEPENDENCIES), $(SPACK) buildcache update-index $(buildcache) ;)
+	$(foreach buildcache, $(BUILDCACHE_S3_FINAL), $(SPACK) buildcache push --unsigned $(buildcache) ;)
+	$(foreach buildcache, $(BUILDCACHE_S3_FINAL), $(SPACK) buildcache update-index $(buildcache) ;)
+	$(foreach buildcache, $(BUILDCACHE_S3_PROMPT), $(SPACK) buildcache update-index $(buildcache) ;)
+	$(foreach buildcache, $(BUILDCACHE_OCI_FINAL), $(SPACK) buildcache push --unsigned --base-image $(BUILDCACHE_OCI_BASE_IMAGE) $(buildcache) ;)
 
 $(SPACK_ENV)/push/%: $(SPACK_ENV)/install/%
 	@mkdir -p $(dir $@)
-	$(foreach buildcache, $(BUILDCACHE_MIRROR_ONLY_PACKAGE), $(SPACK) buildcache push --only=package --unsigned $(buildcache) /$(HASH) ;) # push $(SPEC)
-	$(foreach buildcache, $(BUILDCACHE_MIRROR_DEPENDENCIES), $(SPACK) buildcache push --unsigned --base-image $(BUILDCACHE_OCI_BASE_IMAGE) $(buildcache) /$(HASH) | grep -v "Using cached archive" ;) # push $(SPEC)
+	$(foreach buildcache, $(BUILDCACHE_S3_PROMPT), $(SPACK) buildcache push --unsigned --only=package $(buildcache) /$(HASH) ;) # push $(SPEC)
+	$(foreach buildcache, $(BUILDCACHE_OCI_PROMPT), $(SPACK) buildcache push --unsigned --base-image $(BUILDCACHE_OCI_BASE_IMAGE) $(buildcache) /$(HASH) | grep -v "Using cached archive" ;) # push $(SPEC)
 	@touch $@
 
 $(SPACK_ENV)/spack.lock: $(SPACK_ENV)/spack.yaml Makefile

spack-environment/packages.yaml

@@ -17,7 +17,7 @@ packages:
     - any_of: [build_type=Release, '@:']
   acts:
     require:
-    - '@30.3.2'
+    - '@31.2.0'
     - cxxstd=20 +binaries +dd4hep +examples +fatras +geant4 +identification +json +python +tgeo
   actsvg:
     require:
@@ -312,6 +312,7 @@ packages:
   py-torch:
     require:
     - '@2.1.0'
+    - -mkldnn
   py-uproot:
     require:
     - '@5.0.5'

spack.sh

@@ -23,6 +23,8 @@ b111064e221aae83e62226672cd8bf9a7524423d
 c07ddf83c32b7129247fe90eed486dd844047087
 00875fbc749a7e4e979c35d59c0a8d60d32d4cd7
 64cf67822471ec2d3df2625a6d713f80cbe7ff43
+7a0c4e8017033430e5f15ed628be6b539e935ba9
+48fcfda1e7c1781cab4cada6d099823b263ab0cc
 ---
 ## Ref: https://github.com/spack/spack/commit/[hash]
 ## [hash]: [description]
@@ -41,3 +43,5 @@ c07ddf83c32b7129247fe90eed486dd844047087
 ## c07ddf83c32b7129247fe90eed486dd844047087: py-torch: set env OpenBLAS_HOME
 ## 00875fbc749a7e4e979c35d59c0a8d60d32d4cd7: py-torch: patch for ${OpenBLAS_HOME}/include/openblas
 ## 64cf67822471ec2d3df2625a6d713f80cbe7ff43: py-torch: move patch before def patch
+## 7a0c4e8017033430e5f15ed628be6b539e935ba9: acts: new versions 31.*
+## 48fcfda1e7c1781cab4cada6d099823b263ab0cc: setup-env.sh: if exe contains qemu, use /proc/$$/comm instead