.gitlab-ci.yml

@@ -4,86 +4,7 @@ variables:
   ## Application versions used for the main release 
   ## note: nightly builds will always use the master/main branch
   JUGGLER_VERSION: "v9.4.0"
-  EICRECON_VERSION: "v1.1.0"
-
-  ## Spack organization and repository, e.g. spack/spack
-  SPACK_ORGREPO: "spack/spack"
-  ## Spack github version, e.g. v0.18.1 or commit hash
-  ## note: nightly builds will use e.g. releases/v0.19
-  SPACK_VERSION: "v0.19.2"
-  ## Space-separated list of spack cherry-picks
-  SPACK_CHERRYPICKS: >
-    b5ef5c2eb5145020f9de1bcb964626ce6ac2d02e
-    99056e03bd3e903b222e300636ec484d85d4b3fb
-    f3f8b31be54280c6ef0b5f95ab85604aba3dff30
-    0ea81affd18820933640279bbc687038b3296a4e
-    dceb4c9d655d0529e112b8929558be60973b39f7
-    f2b0c1deab87da3b5aa4c1f2ef4d6af06fa4e32d
-    2f057d729da61e8c12828df44377f0a93fed820f
-    39a1f1462b0abf12dfaccd33f84142c852c4b56e
-    4b12d015e1c316b4837e02ae37e2c69a8a647180
-    f83d47442dade012b1019840181b8dd459fd8edd
-    7f1467e795b1cab8b4855e019910c509896ea0e1
-    ebc24b7063ba9a8eb43b4424aac5143cf958d76f
-    a47ebe57841f13239e881ed69eab4949b1d98c32
-    ab999d5af90f1bff644b5134bb370b2716e1bcf0
-    62da76cb5dca4d52c43bee06230cca6a5882f05d
-    cc2ae9f270befa554ba8b09c68e89bb8248ea650
-    ae98d2ba2fcefa9d027e2d6ccc6e7558a32e7228
-    ae189cfab8d9036e8d39bbd3f1b61b400d1fcd5b
-    3afe6f1adcc24335cbca9a9c03ffea188f802766
-    559c3de213707b5d52d899fd0382495f2cc8508d
-    8e84dcd7ef999e2659822b34372515175f1723c4
-    65bd9b9ac556480b4a9dcc60f7539492af195d4a
-    1a32cea11495cbdd699fea4fe622babab83e630d
-    6edc4807369a05786e36f63b5d959588ae94a1fa
-    af74680405c931dab16c6674f9b97a32bf3f1122
-    0a952f8b7bf6f70009dd5821bccbaf9170c73d07
-    f050b1cf7835fd31992b020e1061c52294ff7330
-    a419ffcf501134faed24253ccc83e6c71f9659f9
-    c3e41153ac92f6ef92414024a8386d4ceec2615c
-  ## Ref: https://github.com/spack/spack/commit/[hash]
-  ## [hash]: [description]
-  ## b5ef5c2eb5145020f9de1bcb964626ce6ac2d02e: geant4: version bumps for Geant4 11.1.0
-  ## 99056e03bd3e903b222e300636ec484d85d4b3fb: acts: new versions 19.11.0, 21.0.0, 21.1.0
-  ## f3f8b31be54280c6ef0b5f95ab85604aba3dff30: XRootD: add checksum + patch for 5.5.1 
-  ## 0ea81affd18820933640279bbc687038b3296a4e: py-torch: fix build with gcc@12:
-  ## dceb4c9d655d0529e112b8929558be60973b39f7: Update PyTorch ecosystem
-  ## f2b0c1deab87da3b5aa4c1f2ef4d6af06fa4e32d: py-minkowskiengine: new package (sparse tensor autodiff by Nvidia)
-  ## 2f057d729da61e8c12828df44377f0a93fed820f: py-scipy: add v1.9
-  ## 39a1f1462b0abf12dfaccd33f84142c852c4b56e: SIP build system: fix "python not defined in builder"
-  ## 4b12d015e1c316b4837e02ae37e2c69a8a647180: py-jinja2-cli: new package
-  ## f83d47442dade012b1019840181b8dd459fd8edd: dd4hep: depends_on root +x +opengl when +utilityapps
-  ## 7f1467e795b1cab8b4855e019910c509896ea0e1: dd4hep: new version 1.24, depends_on podio@0.16:
-  ## ebc24b7063ba9a8eb43b4424aac5143cf958d76f: dd4hep: extend conflict on CMake
-  ## a47ebe57841f13239e881ed69eab4949b1d98c32: dd4hep: new versions 1.25, 1.25.1
-  ## ab999d5af90f1bff644b5134bb370b2716e1bcf0: dd4hep: depends_on root +webgui when +ddeve ^root @6.28:
-  ## 62da76cb5dca4d52c43bee06230cca6a5882f05d: directives: depends_on should not admit anonymous specs
-  ## cc2ae9f270befa554ba8b09c68e89bb8248ea650: Add a "maintainer" directive
-  ## ae98d2ba2fcefa9d027e2d6ccc6e7558a32e7228: Support packages for using scitokens on OSG
-  ## ae189cfab8d9036e8d39bbd3f1b61b400d1fcd5b: geant4: new version 11.1.1
-  ## 3afe6f1adcc24335cbca9a9c03ffea188f802766: ROOT: add math/gsl conflict and change version-dependent features to conditional variants
-  ## 559c3de213707b5d52d899fd0382495f2cc8508d: ROOT: new versions and associated dependency constraints
-  ## 8e84dcd7ef999e2659822b34372515175f1723c4: root: new version 6.28.00
-  ## 65bd9b9ac556480b4a9dcc60f7539492af195d4a: podio, edm4hep: add v0.7.2 and v0.16.1 respectively
-  ## 1a32cea11495cbdd699fea4fe622babab83e630d: podio: add v0.16.2
-  ## 6edc4807369a05786e36f63b5d959588ae94a1fa: podio: Add version 0.16.3
-  ## af74680405c931dab16c6674f9b97a32bf3f1122: depfile: improve tab completion
-  ## 0a952f8b7bf6f70009dd5821bccbaf9170c73d07: docs updates for spack env depfile
-  ## f050b1cf7835fd31992b020e1061c52294ff7330: depfile: variable with all identifiers
-  ## a419ffcf501134faed24253ccc83e6c71f9659f9: osg-ca-certs: igtf link should point to version, not 'current'
-  ## c3e41153ac92f6ef92414024a8386d4ceec2615c: Package requirements: allow single specs in requirement lists
-
-  ## EIC spack organization and repository, e.g. eic/eic-spack
-  EICSPACK_ORGREPO: "eic/eic-spack"
-  ## EIC spack github version, e.g. v0.19.7 or commit hash
-  ## note: nightly builds will use e.g. releases/v0.19
-  EICSPACK_VERSION: "v0.19.0"
-  ## Space-separated list of eic-spack cherry-picks
-  EICSPACK_CHERRYPICKS: "f892e2b5d7ea9d1f2e43741499e899ce21dd3d5a"
-  ## Ref: https://github.com/eic/eic-spack/commit/[hash]
-  ## [hash]: [description]
-  ## f892e2b5d7ea9d1f2e43741499e899ce21dd3d5a: py-minkowskiengine: new package
+  EICRECON_VERSION: "v1.2.1"
 
   ## Local registry
   CI_PUSH: 1
@@ -104,6 +25,9 @@ variables:
   DOCKER_NTRIES: 5
   DOCKER_WAIT_TIME: 5
 
+  ## Number of jobs to start during container builds
+  JOBS: 128
+
   ## is this nightly or not?
   NIGHTLY:
 
@@ -122,6 +46,7 @@ variables:
 
 stages:
   - config
+  - dind
   - base            ## base OS image
   - jug             ## jug container images
   - deploy          ## build/deploy singularity images
@@ -234,13 +159,31 @@ version:
 ## TODO
 ## oneapi_runtime + jug_xl --> oneapi_prod 
 
+## Docker-in-docker image
+dind:
+  parallel:
+    matrix:
+      - BUILD_IMAGE: dind
+  extends: .build
+  stage: dind
+  needs:
+    - version
+  script:
+    - docker buildx build --push
+        --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
+        --file containers/dind/Dockerfile
+        containers/dind
+
 base:
   parallel:
     matrix:
-      - BASE_IMAGE: debian:testing-20220822-slim
+      - BASE_IMAGE: debian:testing-slim
         BUILD_IMAGE: debian_testing_base
         PLATFORM: linux/amd64,linux/arm64/v8
-      - BASE_IMAGE: ubuntu:20.04
+      - BASE_IMAGE: debian:stable-slim
+        BUILD_IMAGE: debian_stable_base
+        PLATFORM: linux/amd64,linux/arm64/v8
+      - BASE_IMAGE: ubuntu:22.04
         BUILD_IMAGE: ubuntu_base
         PLATFORM: linux/amd64,linux/arm64/v8
       - BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04
@@ -270,7 +213,7 @@ base:
 jug_dev:default:
   parallel:
     matrix:
-      - BASE_IMAGE: debian_testing_base
+      - BASE_IMAGE: debian_stable_base
         BUILD_IMAGE: jug_dev
         PLATFORM: linux/amd64,linux/arm64/v8
 #      - BASE_IMAGE: oneapi_base
@@ -301,7 +244,9 @@ jug_dev:default:
     ## Setup mirrors.yaml with rw mirror secrets
     - cat mirrors.yaml.in | envsubst > mirrors.yaml
     ## Build targets
-    - for target in ${TARGETS} ; do
+    - source spack.sh ; 
+      source eic-spack.sh ; 
+      for target in ${TARGETS} ; do
         if [ ! ${target} = export ] ; then
           suffix=-${target} ;
         fi ;
@@ -316,18 +261,20 @@ jug_dev:default:
                    }
                    --file containers/jug/dev.Dockerfile
                    --platform ${PLATFORM}
-                   --build-arg BASE_IMAGE="${BASE_IMAGE}"
-                   --build-arg SPACK_ORGREPO="${SPACK_ORGREPO}"
-                   --build-arg SPACK_VERSION="${SPACK_VERSION}"
+                   --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
+                   --build-arg BASE_IMAGE=${BASE_IMAGE}
+                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
+                   --build-arg SPACK_ORGREPO=${SPACK_ORGREPO}
+                   --build-arg SPACK_VERSION=${SPACK_VERSION}
                    --build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
-                   --build-arg EICSPACK_ORGREPO="${EICSPACK_ORGREPO}"
-                   --build-arg EICSPACK_VERSION="${EICSPACK_VERSION}"
+                   --build-arg EICSPACK_ORGREPO=${EICSPACK_ORGREPO}
+                   --build-arg EICSPACK_VERSION=${EICSPACK_VERSION}
                    --build-arg EICSPACK_CHERRYPICKS="${EICSPACK_CHERRYPICKS}"
-                   --build-arg S3_ACCESS_KEY="${S3_ACCESS_KEY}"
-                   --build-arg S3_SECRET_KEY="${S3_SECRET_KEY}"
-                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
+                   --build-arg S3_ACCESS_KEY=${S3_ACCESS_KEY}
+                   --build-arg S3_SECRET_KEY=${S3_SECRET_KEY}
                    --build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD)
-                   --build-context spack=.
+                   --build-arg jobs=${JOBS}
+                   --build-context spack-environment=spack-environment
                    --secret id=mirrors,src=mirrors.yaml
                    containers/jug
       ; done
@@ -356,12 +303,13 @@ jug_xl:default:
                    }
                    --file containers/jug/xl.Dockerfile
                    --platform ${PLATFORM}
+                   --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
                    --build-arg BASE_IMAGE=${BASE_IMAGE}
                    --build-arg INTERNAL_TAG=${INTERNAL_TAG}
                    --build-arg JUGGLER_VERSION=${JUGGLER_VERSION}
                    --build-arg EICRECON_VERSION=${EICRECON_VERSION}
                    --build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD)
-                   --build-arg jobs=64
+                   --build-arg jobs=${JOBS}
                    --build-context detectors=.
                    containers/jug
 
@@ -404,6 +352,7 @@ jug_xl:nightly:
                    }
                    --file containers/jug/xl.Dockerfile
                    --platform ${PLATFORM}
+                   --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
                    --build-arg BASE_IMAGE=${BASE_IMAGE}
                    --build-arg INTERNAL_TAG=${INTERNAL_TAG}
                    --build-arg JUG_VERSION=nightly-$(date +%Y-%m-%d_%H-%M-%S)-${INTERNAL_TAG}-$(git rev-parse HEAD)
@@ -604,5 +553,5 @@ clean_unstable_mr:
           fi ;
         fi ;
       done
-    - docker system prune --force
+    - docker system prune --filter until=144h --force
     - docker images

containers/debian/base.Dockerfile

-#syntax=docker/dockerfile:1.2
-ARG BASE_IMAGE="amd64/debian:testing-20220822-slim"
-ARG BUILD_IMAGE="debian_base"
+#syntax=docker/dockerfile:1.4
+ARG BASE_IMAGE="amd64/debian:stable-slim"
+ARG BUILD_IMAGE="debian_stable_base"
 
 # Minimal container based on Debian base systems for up-to-date packages. 
 FROM  ${BASE_IMAGE}

containers/dind/Dockerfile

+FROM ubuntu:22.04
+LABEL maintainer="Whitney Armstrong <warmstrong@anl.gov>" \
+      name="ubuntu_dind" \
+      group="ubuntu_dind" \
+      march="native" \
+      base="ubuntu" \
+      version="22.04"
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    apt-transport-https \
+    binfmt-support \
+    build-essential \
+    ca-certificates \
+    cryptsetup \
+    curl \
+    gettext \
+    git \
+    iptables \
+    libgpgme11-dev \
+    libseccomp-dev \
+    libssl-dev \
+    lxc \
+    make \
+    pkg-config \
+    qemu \
+    qemu-user-static \
+    uuid-dev \
+    squashfs-tools \
+    wget
+
+RUN cd /tmp \
+    && export VERSION=1.15.6 OS=linux ARCH=amd64 && \
+    wget https://dl.google.com/go/go$VERSION.$OS-$ARCH.tar.gz && \
+    tar -C /usr/local -xzvf go$VERSION.$OS-$ARCH.tar.gz && \
+    rm go$VERSION.$OS-$ARCH.tar.gz 
+
+RUN cd /tmp \
+    && export PATH=/usr/local/go/bin:$PATH \
+    && export VERSION=3.7.4 && \
+    wget https://github.com/sylabs/singularity/releases/download/v${VERSION}/singularity-${VERSION}.tar.gz && \ 
+    tar -xzf singularity-${VERSION}.tar.gz && \
+    cd singularity \
+    && ./mconfig && \
+    make -C builddir && \
+    make -C builddir install
+
+# Install Docker from Docker Inc. repositories. 
+RUN export VERSION=20.10.17 \
+ && curl -sSL https://get.docker.com/ | sh
+# Install the magic wrapper. 
+ADD ./wrapdocker /usr/local/bin/wrapdocker
+RUN chmod +x /usr/local/bin/wrapdocker
+# Define additional metadata for our image. 
+VOLUME /var/lib/docker
+CMD ["wrapdocker"]

containers/dind/wrapdocker

+#!/bin/bash
+
+# Ensure that all nodes in /dev/mapper correspond to mapped devices currently loaded by the device-mapper kernel driver
+dmsetup mknodes
+
+# First, make sure that cgroups are mounted correctly.
+CGROUP=/sys/fs/cgroup
+: {LOG:=stdio}
+
+[ -d $CGROUP ] ||
+	mkdir $CGROUP
+
+mountpoint -q $CGROUP ||
+	mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || {
+		echo "Could not make a tmpfs mount. Did you use --privileged?"
+		exit 1
+	}
+
+if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security
+then
+    mount -t securityfs none /sys/kernel/security || {
+        echo "Could not mount /sys/kernel/security."
+        echo "AppArmor detection and --privileged mode might break."
+    }
+fi
+
+# Mount the cgroup hierarchies exactly as they are in the parent system.
+for SUBSYS in $(cut -d: -f2 /proc/1/cgroup)
+do
+        [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS
+        mountpoint -q $CGROUP/$SUBSYS ||
+                mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS
+
+        # The two following sections address a bug which manifests itself
+        # by a cryptic "lxc-start: no ns_cgroup option specified" when
+        # trying to start containers withina container.
+        # The bug seems to appear when the cgroup hierarchies are not
+        # mounted on the exact same directories in the host, and in the
+        # container.
+
+        # Named, control-less cgroups are mounted with "-o name=foo"
+        # (and appear as such under /proc/<pid>/cgroup) but are usually
+        # mounted on a directory named "foo" (without the "name=" prefix).
+        # Systemd and OpenRC (and possibly others) both create such a
+        # cgroup. To avoid the aforementioned bug, we symlink "foo" to
+        # "name=foo". This shouldn't have any adverse effect.
+        echo $SUBSYS | grep -q ^name= && {
+                NAME=$(echo $SUBSYS | sed s/^name=//)
+                ln -s $SUBSYS $CGROUP/$NAME
+        }
+
+        # Likewise, on at least one system, it has been reported that
+        # systemd would mount the CPU and CPU accounting controllers
+        # (respectively "cpu" and "cpuacct") with "-o cpuacct,cpu"
+        # but on a directory called "cpu,cpuacct" (note the inversion
+        # in the order of the groups). This tries to work around it.
+        [ $SUBSYS = cpuacct,cpu ] && ln -s $SUBSYS $CGROUP/cpu,cpuacct
+done
+
+# Note: as I write those lines, the LXC userland tools cannot setup
+# a "sub-container" properly if the "devices" cgroup is not in its
+# own hierarchy. Let's detect this and issue a warning.
+grep -q :devices: /proc/1/cgroup ||
+	echo "WARNING: the 'devices' cgroup should be in its own hierarchy."
+grep -qw devices /proc/1/cgroup ||
+	echo "WARNING: it looks like the 'devices' cgroup is not mounted."
+
+# Now, close extraneous file descriptors.
+pushd /proc/self/fd >/dev/null
+for FD in *
+do
+	case "$FD" in
+	# Keep stdin/stdout/stderr
+	[012])
+		;;
+	# Nuke everything else
+	*)
+		eval exec "$FD>&-"
+		;;
+	esac
+done
+popd >/dev/null
+
+
+# If a pidfile is still around (for example after a container restart),
+# delete it so that docker can start.
+rm -rf /var/run/docker.pid
+
+# If we were given a PORT environment variable, start as a simple daemon;
+# otherwise, spawn a shell as well
+if [ "$PORT" ]
+then
+	exec dockerd -H 0.0.0.0:$PORT -H unix:///var/run/docker.sock \
+		$DOCKER_DAEMON_ARGS
+else
+	if [ "$LOG" == "file" ]
+	then
+		dockerd $DOCKER_DAEMON_ARGS &>/var/log/docker.log &
+	else
+		dockerd $DOCKER_DAEMON_ARGS &
+	fi
+	(( timeout = 60 + SECONDS ))
+	until docker info >/dev/null 2>&1
+	do
+		if (( SECONDS >= timeout )); then
+			echo 'Timed out trying to connect to internal docker host.' >&2
+			break
+		fi
+		sleep 1
+	done
+	[[ $1 ]] && exec "$@"
+	exec bash --login
+fi

containers/jug/dev.Dockerfile

 #syntax=docker/dockerfile:1.4
-ARG DOCKER_REGISTRY="eicweb.phy.anl.gov:4567/containers/eic_container/"
-ARG BASE_IMAGE="debian_base"
+ARG DOCKER_REGISTRY="eicweb/"
+ARG BASE_IMAGE="debian_stable_base"
 ARG INTERNAL_TAG="testing"
 
 ## ========================================================================================
@@ -26,7 +26,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${TARGETPLATFORM}
 ## Setup spack
 ENV SPACK_ROOT=/opt/spack
 ARG SPACK_ORGREPO="spack/spack"
-ARG SPACK_VERSION="releases/v0.19"
+ARG SPACK_VERSION="releases/v0.20"
 ARG SPACK_CHERRYPICKS=""
 ADD https://api.github.com/repos/${SPACK_ORGREPO}/commits/${SPACK_VERSION} /tmp/spack.json
 RUN git clone https://github.com/${SPACK_ORGREPO}.git ${SPACK_ROOT}     \
@@ -56,17 +56,18 @@ RUN declare -A target=(                                                 \
  && spack config --scope site add "config:db_lock_timeout:${jobs}0"     \
  && spack config --scope site add "config:install_tree:root:/opt/software" \
  && spack config --scope site add "config:source_cache:/var/cache/spack" \
+ && spack config --scope site add "config:ccache:true"                  \
  && spack config blame config                                           \
  && spack compiler find --scope site                                    \
  && spack config blame compilers
 
-## Setup spack buildcache mirrors
+## Setup local buildcache mirrors
 RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
-    spack mirror add docker /var/cache/spack-mirror                     \
- && spack buildcache update-index -d /var/cache/spack-mirror            \
+    spack mirror add local /var/cache/spack-mirror/${SPACK_VERSION}     \
+ && spack buildcache update-index local                                 \
  && spack mirror list
 
-## Setup eic-spack buildcache mirrors
+## Setup eics3 buildcache mirrors
 ## - this always adds the read-only mirror to the container
 ## - the write-enabled mirror is provided later as a secret mount
 ARG S3_ACCESS_KEY=""
@@ -77,7 +78,7 @@ RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
       --s3-endpoint-url https://eics3.sdcc.bnl.gov:9000                 \
       --s3-access-key-id "${S3_ACCESS_KEY}"                             \
       --s3-access-key-secret "${S3_SECRET_KEY}"                         \
-      eic-spack s3://eictest/EPIC/spack                                 \
+      eics3 s3://eictest/EPIC/spack/${SPACK_VERSION}                    \
     ; fi                                                                \
  && spack mirror list
 
@@ -92,7 +93,7 @@ ARG EICSPACK_ORGREPO="eic/eic-spack"
 ARG EICSPACK_VERSION="$SPACK_VERSION"
 ARG EICSPACK_CHERRYPICKS=""
 ADD https://api.github.com/repos/${EICSPACK_ORGREPO}/commits/${EICSPACK_VERSION} /tmp/eic-spack.json
-RUN git clone https://github.com/${EICSPACK_ORGREPO}.git ${EICSPACK_ROOT}     \
+RUN git clone https://github.com/${EICSPACK_ORGREPO}.git ${EICSPACK_ROOT} \
  && git -C ${EICSPACK_ROOT} checkout ${EICSPACK_VERSION}                \
  && if [ -n "${EICSPACK_CHERRYPICKS}" ] ; then                          \
       git -C ${EICSPACK_ROOT} cherry-pick -n ${EICSPACK_CHERRYPICKS} ;  \
@@ -100,17 +101,19 @@ RUN git clone https://github.com/${EICSPACK_ORGREPO}.git ${EICSPACK_ROOT}     \
  && spack repo add --scope site "${EICSPACK_ROOT}"
 
 ## Setup our custom environment (secret mount for write-enabled mirror)
-COPY --from=spack spack-environment/ /opt/spack-environment/
+COPY --from=spack-environment . /opt/spack-environment/
 ARG ENV=dev
 ENV SPACK_ENV=/opt/spack-environment/${ENV}
-RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
+RUN --mount=type=cache,target=/ccache,id=${TARGETPLATFORM}              \
+    --mount=type=cache,target=/var/cache/spack-mirror                   \
     --mount=type=secret,id=mirrors,target=/opt/spack/etc/spack/mirrors.yaml \
     source $SPACK_ROOT/share/spack/setup-env.sh                         \
+ && export CCACHE_DIR=/ccache                                           \
  && spack env activate --dir ${SPACK_ENV}                               \
  && make --jobs ${jobs} --keep-going --directory /opt/spack-environment \
     SPACK_ENV=${SPACK_ENV}                                              \
-    BUILDCACHE_DIR=/var/cache/spack-mirror                              \
-    BUILDCACHE_MIRROR=eic-spack
+    BUILDCACHE_MIRROR="local eics3rw"                                   \
+ && ccache --show-stats
 
 ## Create view at /usr/local
 RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
@@ -124,7 +127,7 @@ RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
 ## or intermittently to keep the buildcache step from taking too much time
 RUN --mount=type=cache,target=/var/cache/spack-mirror,sharing=locked    \
     [ -z "${CACHE_NUKE}" ]                                              \
-    || rm -rf /var/cache/spack-mirror/build_cache/*
+    || rm -rf /var/cache/spack-mirror/${SPACK_VERSION}/build_cache/*
 
 ## Extra post-spack steps:
 ##   - Python packages
@@ -188,9 +191,18 @@ COPY profile.d/z11_jug_env.sh /etc/profile.d
 COPY singularity.d /.singularity.d
 
 ## Add minio client into /usr/local/bin
-## FIXME: This should download .../linux-arm64/mc for arm64.
-ADD https://dl.min.io/client/mc/release/linux-amd64/mc /usr/local/bin
-RUN chmod a+x /usr/local/bin/mc
+ADD https://dl.min.io/client/mc/release/linux-amd64/mc /usr/local/bin/mc-amd64
+ADD https://dl.min.io/client/mc/release/linux-arm64/mc /usr/local/bin/mc-arm64
+RUN declare -A target=(                                                 \
+      ["linux/amd64"]="amd64"                                           \
+      ["linux/arm64"]="arm64"                                           \
+    )                                                                   \
+ && mv /usr/local/bin/mc-${target[${TARGETPLATFORM}]} /usr/local/bin/mc \
+ && chmod a+x /usr/local/bin/mc                                         \
+ && unset target[${TARGETPLATFORM}]                                     \
+ && for t in ${target[*]} ; do                                          \
+      rm /usr/local/bin/mc-${t} ;                                       \
+    done
 
 ## ========================================================================================
 ## STAGE 3
@@ -221,9 +233,7 @@ RUN --mount=from=staging,target=/staging                                \
  && cp -r /staging/.singularity.d /.singularity.d                        
 
 ## set the jug_dev version and add the afterburner
-## TODO: move afterburner to spack when possible
 ARG JUG_VERSION=1
-ARG AFTERBURNER_VERSION=main
 RUN echo "" >> /etc/jug_info                                            \
  && echo " - jug_dev: ${JUG_VERSION}" >> /etc/jug_info
 

containers/jug/profile.d/z11_jug_env.sh

@@ -3,11 +3,10 @@
 # FIXME: add ROOT libraries to LD_LIBRARY_PATH until removal can be tested 
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/root
 
-export BINARY_TAG=x86_64-linux-gcc9-opt
 if [ ! -z ${EIC_SHELL_PREFIX} ]; then
   if [  "$LD_LIBRARY_PATH" != *"${EIC_SHELL_PREFIX}/lib"* ]; then
-    export LD_LIBRARY_PATH=$EIC_SHELL_PREFIX/lib:$LD_LIBRARY_PATH
-    export PATH=$EIC_SHELL_PREFIX/bin:$PATH
+    export LD_LIBRARY_PATH=$EIC_SHELL_PREFIX/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
+    export PATH=$EIC_SHELL_PREFIX/bin${PATH:+:$PATH}
   fi
 fi
 

containers/jug/xl.Dockerfile

 #syntax=docker/dockerfile:1.4
-ARG DOCKER_REGISTRY="eicweb.phy.anl.gov:4567/containers/eic_container/"
+ARG DOCKER_REGISTRY="eicweb/"
 ARG BASE_IMAGE="jug_dev"
 ARG INTERNAL_TAG="testing"
 

detectors.yaml

@@ -5,36 +5,6 @@ detectors:
       version: main
     main:
       version: main
-    22.10.0:
-      version: 22.10.0
-      ip:
-        config: ip6
-        version: 22.10.0
-    22.10.1:
-      version: 22.10.1
-      ip:
-        config: ip6
-        version: 22.10.0
-    22.11.0:
-      version: 22.11.0
-      ip:
-        config: ip6
-        version: 22.11.0
-    22.11.1:
-      version: 22.11.1
-      ip:
-        config: ip6
-        version: 22.11.0
-    22.11.2:
-      version: 22.11.2
-      ip:
-        config: ip6
-        version: 22.11.0
-    22.11.3:
-      version: 22.11.3
-      ip:
-        config: ip6
-        version: 22.11.0
     22.12.0:
       version: 22.12.0
     23.01.0:
@@ -45,3 +15,11 @@ detectors:
       version: alternative-geometries-img-ecal
     23.05.0:
       version: 23.05.0
+    23.05.1:
+      version: 23.05.1
+    23.05.2:
+      version: 23.05.2
+    23.06.0:
+      version: 23.06.0
+    23.06.1:
+      version: 23.06.1

eic-spack.sh

+## EIC spack organization and repository, e.g. eic/eic-spack
+EICSPACK_ORGREPO="eic/eic-spack"
+
+## EIC spack github version, e.g. v0.19.7 or commit hash
+## note: nightly builds will use e.g. releases/v0.19
+EICSPACK_VERSION="v0.20.3"
+
+## Space-separated list of eic-spack cherry-picks
+read -r -d '' EICSPACK_CHERRYPICKS <<- \
+--- || true
+---
+## Ref: https://github.com/eic/eic-spack/commit/[hash]
+## [hash]: [description]

mirrors.yaml.in

 mirrors:
-  eic-spack:
+  eics3rw:
     fetch:
-      url: s3://eictest/EPIC/spack
+      url: s3://eictest/EPIC/spack/${SPACK_VERSION}
       access_pair:
       - ${S3RW_ACCESS_KEY}
       - ${S3RW_SECRET_KEY}
@@ -9,7 +9,7 @@ mirrors:
       profile: null
       endpoint_url: https://eics3.sdcc.bnl.gov:9000
     push:
-      url: s3://eictest/EPIC/spack
+      url: s3://eictest/EPIC/spack/${SPACK_VERSION}
       access_pair:
       - ${S3RW_ACCESS_KEY}
       - ${S3RW_SECRET_KEY}

spack-environment/Makefile

@@ -7,7 +7,6 @@ export SPACK_COLOR = always
 
 SPACK_ENV ?= dev
 
-BUILDCACHE_DIR := $(SPACK_ENV)/cache
 BUILDCACHE_MIRROR :=
 
 .PHONY: all clean
@@ -19,14 +18,11 @@ include $(SPACK_ENV)/spack.mk
 endif
 
 $(SPACK_ENV)/push: $(addprefix $(SPACK_ENV)/push/,$($(SPACK_ENV)/SPACK_PACKAGE_IDS))
-	@mkdir -p $(BUILDCACHE_DIR)
-	$(foreach buildcache, $(BUILDCACHE_DIR), $(SPACK) buildcache update-index --mirror-url $(buildcache) ;)
-	$(foreach buildcache, $(BUILDCACHE_MIRROR), $(SPACK) buildcache update-index --mirror-url $(buildcache) ;)
+	$(foreach buildcache, $(BUILDCACHE_MIRROR), $(SPACK) buildcache update-index $(buildcache) ;)
 
 $(SPACK_ENV)/push/%: $(SPACK_ENV)/install/%
 	@mkdir -p $(dir $@)
-	$(foreach buildcache, $(BUILDCACHE_DIR), $(SPACK) buildcache create --allow-root --only=package --unsigned --directory $(buildcache) /$(HASH) ;) # push $(SPEC)
-	$(foreach buildcache, $(BUILDCACHE_MIRROR), $(SPACK) buildcache create --allow-root --only=package --unsigned --mirror-name $(buildcache) /$(HASH) ;) # push $(SPEC)
+	$(foreach buildcache, $(BUILDCACHE_MIRROR), $(SPACK) buildcache create --allow-root --only=package --unsigned $(buildcache) /$(HASH) ;) # push $(SPEC)
 	@touch $@
 
 $(SPACK_ENV)/spack.lock: $(SPACK_ENV)/spack.yaml Makefile

spack-environment/dev/spack.yaml

 spack:
-  include: 
+  include:
   - ../concretizer.yaml
   - ../packages.yaml
   specs:
-    - acts@21.1.0 +dd4hep +identification +json +tgeo +examples +fatras +geant4 +python
+    - acts@21.1.1 +dd4hep +examples +fatras +geant4 +identification +json +python +tgeo
     - actsvg@0.4.26 +examples
     - acts-dd4hep@1.0.1
     - afterburner@0.1.2 +root +zlib
     - cairo@1.16.0 +fc+ft+X+pdf+gobject
     - catch2@3.0.1 -ipo
-    - clhep@2.4.6.0 cxxstd=17
-    - cli11@2.1.1
-    - cmake@3.24.3
+    - cernlib@2022.11.08.0-free
+    - clhep@2.4.6.4 cxxstd=17
+    - cli11@2.3.2
+    - cmake@3.26.3
     - cnpy@master
-    - cppcoro@git.10bbcdbf2be3ad3aa56febcf4c7662d771460a99
+    #- cppcoro@git.10bbcdbf2be3ad3aa56febcf4c7662d771460a99
     - dawn@3_91a
     - dawncut@1_54a
-    - dd4hep@1.25.1 +ddg4 +ddcad +hepmc3 +lcio +xercesc
+    - dd4hep@1.25.1 +ddg4 +ddcad -frames +hepmc3 +lcio +xercesc
+    - dpmjet@19.3.5
     - edm4eic@2.0.0 cxxstd=17
-    - edm4hep@0.7 cxxstd=17
+    - edm4hep@=0.7 cxxstd=17
     - eic-smear@1.1.10
     - eigen@3.4.0
     - emacs@28.2
     - fastjet@3.4.0
-    - fjcontrib@1.045
-    - fmt@8.1.1 +shared cxxstd=17
-    - gaudi@36.7 +aida
+    - fjcontrib@1.051
+    - fmt@9.1.0 +shared cxxstd=17
+    - gaudi@36.10 +aida
+    - gdb@12.1 -debuginfod
     - geant4@11.1.1 cxxstd=17 +opengl -vecgeom +x11 +qt +threads
-    - hepmc3@3.2.5 +python +rootio
+    - hepmc3@3.2.6 +python +rootio
     - heppdt@2.06.01
     - imagemagick@7.0.8-7
     - intel-tbb@2020.3
     - irt@1.0.6
     - jana2@2.1.0 -ipo +podio +root +zmq
-    - lcov@1.15
+    - lcov@1.16
+    - lhapdf@6.5.1 +python
     - madx@5.08.01
-    - mesa@22.1.2 +glx -llvm +opengl +opengles +osmesa
+    - mesa@22.1.6 +glx -llvm +opengl +opengles +osmesa
     - nlohmann-json@3.11.2
     - npsim@1.1.0 +geocad +http
-    - onnx@1.12.0
-    - opencascade@7.6.3
+    - onnx@1.13.1
+    - opencascade@7.7.1
     - osg-ca-certs@1.109.igtf.1.117
     - phonebook-cli@1.0.0
-    - pkg-config@0.29.2
-    - podio@0.16.3
+    - podio@0.16.5
     - prmon@3.0.2 +plot
-    - pythia8@8.306 +fastjet
-    - python@3.10.8
-    - py-awkward@1.9.0
-    - py-boto3@1.18.12
-    - py-ipython@8.5.0
+    - pythia8@8.309 +fastjet
+    - python@3.10.10
+    - py-awkward@2.1.1
+    - py-dask@2021.6.2
+    - py-boto3@1.26.26
+    - py-ipython@8.11.0
     - py-jinja2@3.1.2
     - py-jinja2-cli@0.8.2
-    - py-jupyter@1.0.0
-    - py-jupyterlab@3.2.9
+    - py-jupyter-console@6.4.4
+    - py-jupyterlab@3.4.8
     - py-lmfit@1.0.2
     - py-lxml@4.9.1
-    - py-matplotlib@3.6.2
+    - py-matplotlib@3.7.1
     - py-mplhep@0.3.26
     - py-htgettoken@1.16-33-g3788bb4
     - py-numpy@1.23.4
-    - py-pandas@1.5.1
-    - py-pip@22.2.2
+    - py-pandas@1.5.3
+    - py-pip@23.0
     - py-pre-commit@2.20.0
     - py-pycairo@1.20.0
     - py-pyyaml@6.0
-    - py-scipy@1.9.3
-    - py-seaborn@0.12.0
+    - py-scipy@1.10.1
+    - py-seaborn@0.12.2
     - py-toml@0.10.2
-    - py-uproot@4.3.5
+    - py-uproot@5.0.5
     - py-yapf@0.30.0
-    - qt@5.15.5 +opengl
+    - qt@5.15.9 +opengl
     - root@6.26.10 cxxstd=17 
           +fftw +fortran +gdml +http -ipo +mlp +pythia8 
           +root7 +tmva +vc -webgui +xrootd +ssl 
-    - spdlog@1.10.0 +fmt_external
+    - spdlog@1.11.0 +fmt_external
     - stow@2.3.1
     - tensorflow-lite@2.8.0 -xnnpack
-    - xrootd@5.5.1 cxxstd=17 +python +scitokens-cpp
+    - xrootd@5.5.5 cxxstd=17 -davix +python +scitokens-cpp
   view: false

spack.sh

+## Spack organization and repository, e.g. spack/spack
+SPACK_ORGREPO="spack/spack"
+
+## Spack github version, e.g. v0.18.1 or commit hash
+## note: nightly builds will use e.g. releases/v0.19
+SPACK_VERSION="v0.20.0"
+
+## Space-separated list of spack cherry-picks
+read -r -d '' SPACK_CHERRYPICKS <<- \
+--- || true
+9ee2d79de172de14477a78e5d407548a63eea33a
+776ab132760d63eab0703b7c0ebebc72a8443f5b
+188168c476eabe99764634db8d78eb3f9ea2a927
+---
+## Ref: https://github.com/spack/spack/commit/[hash]
+## [hash]: [description]
+## 9ee2d79de172de14477a78e5d407548a63eea33a: libxpm package: fix RHEL8 build with libintl
+## 776ab132760d63eab0703b7c0ebebc72a8443f5b: [xrootd] New variants, new version, improve build config
+## 188168c476eabe99764634db8d78eb3f9ea2a927: podio: Add 0.16.5 tag