feat: install most dependencies in cacheable layer, before adding main

Instead of requiring the whole stack to be reinstalled whenever a specific version (e.g. main) causes a cache fail, this aims to have stable default version with a second-stage installation of the potentially-modified versions of eicrecon and juggler. This should allow all builds to cache the first installation in virtually all cases, leading to faster builds.

containers/jug/dev.Dockerfile

@@ -20,6 +20,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${TARGETPLATFORM}
 rm -f /etc/apt/apt.conf.d/docker-clean
 apt-get -yqq update
 apt-get -yqq install --no-install-recommends                            \
+        jq                                                              \
         python3                                                         \
         python3-dev                                                     \
         python3-distutils                                               \
@@ -70,14 +71,6 @@ spack compiler find --scope site
 spack config blame compilers
 EOF
 
-## Setup local buildcache mirrors
-RUN --mount=type=cache,target=/var/cache/spack <<EOF
-set -e
-spack mirror add local /var/cache/spack/mirror/${SPACK_VERSION}
-spack buildcache update-index local
-spack mirror list
-EOF
-
 ## Setup eics3 buildcache mirrors
 ## - this always adds the read-only mirror to the container
 ## - the write-enabled mirror is provided later as a secret mount
@@ -140,37 +133,40 @@ RUN --mount=type=cache,target=/ccache,id=${TARGETPLATFORM}              \
     <<EOF
 set -e
 export CCACHE_DIR=/ccache
-spack buildcache update-index local
+source ${SPACK_ROOT}/share/spack/setup-env.sh
+find /var/cache/spack/blobs/sha256/ -atime +7 -delete
 spack buildcache update-index eics3rw
 spack env activate --dir ${SPACK_ENV}
 spack concretize --fresh --force --quiet
-make --jobs ${jobs} --keep-going --directory /opt/spack-environment SPACK_ENV=${SPACK_ENV} BUILDCACHE_MIRROR="local eics3rw"
+make --jobs ${jobs} --keep-going --directory /opt/spack-environment SPACK_ENV=${SPACK_ENV} BUILDCACHE_MIRROR_ONLY_PACKAGE="eics3rw" BUILDCACHE_MIRROR_DEPENDENCIES="eicweb ghcr"
 ccache --show-stats
 ccache --zero-stats
 EOF
 
 ## 2. Setup our environment with custom versions (on top of cached layer)
-ARG JUGGLER_VERSION=""
-ARG EICRECON_VERSION=""
-ADD https://eicweb.phy.anl.gov/api/v4/projects/EIC%2Fjuggler/repository/tree?ref=${JUGGLER_VERSION} /tmp/juggler.json
+## Note: these default versions are just the very first commit.
+ARG JUGGLER_VERSION="df87bf1f8643afa8e80bece9d36d6dc26dfe8132"
+ARG EICRECON_VERSION="28108da4a1e8919a05dfdb5f11e114800a2cbe96"
+ADD https://eicweb.phy.anl.gov/api/v4/projects/EIC%2Fjuggler/repository/commits/${JUGGLER_VERSION} /tmp/juggler.json
 ADD https://api.github.com/repos/eic/eicrecon/commits/${EICRECON_VERSION} /tmp/eicrecon.json
 RUN --mount=type=cache,target=/ccache,id=${TARGETPLATFORM}              \
     --mount=type=cache,target=/var/cache/spack                          \
     --mount=type=secret,id=mirrors,target=/opt/spack/etc/spack/mirrors.yaml \
     <<EOF
-source $SPACK_ROOT/share/spack/setup-env.sh
+source ${SPACK_ROOT}/share/spack/setup-env.sh
 export CCACHE_DIR=/ccache
-spack buildcache update-index local
 spack buildcache update-index eics3rw
 spack env activate --dir ${SPACK_ENV}
-if [ -n ${JUGGLER_VERSION} ] ; then
+if [ "${JUGGLER_VERSION}" != "df87bf1f8643afa8e80bece9d36d6dc26dfe8132" ] ; then
+  export JUGGLER_VERSION=$(jq -r .id /tmp/juggler.json)
   spack config add "packages:juggler::require:['@git.${JUGGLER_VERSION}']"
 fi
-if [ -n ${EICRECON_VERSION} ] ; then
+if [ "${EICRECON_VERSION}" != "28108da4a1e8919a05dfdb5f11e114800a2cbe96" ] ; then
+  export EICRECON_VERSION=$(jq -r .sha /tmp/eicrecon.json)
   spack config add "packages:eicrecon::require:['@git.${EICRECON_VERSION}']"
 fi
 spack concretize --fresh --force --quiet
-make --jobs ${jobs} --keep-going --directory /opt/spack-environment SPACK_ENV=${SPACK_ENV} BUILDCACHE_MIRROR="local eics3rw"
+make --jobs ${jobs} --keep-going --directory /opt/spack-environment SPACK_ENV=${SPACK_ENV} BUILDCACHE_MIRROR_ONLY_PACKAGE="eics3rw" BUILDCACHE_MIRROR_DEPENDENCIES="eicweb ghcr"
 ccache --show-stats
 ccache --zero-stats
 EOF
@@ -178,9 +174,8 @@ EOF
 ## Create view at /usr/local
 RUN --mount=type=cache,target=/var/cache/spack <<EOF
 set -e
-spack env activate --dir ${SPACK_ENV}
 rm -r /usr/local
-spack env view enable /usr/local
+spack -e ${SPACK_ENV} env view enable /usr/local
 EOF
 
 ## Optional, nuke the buildcache after install, before (re)caching
@@ -318,10 +313,10 @@ ARG BENCHMARK_DET_VERSION="master"
 ARG BENCHMARK_REC_VERSION="master"
 ARG BENCHMARK_PHY_VERSION="master"
 ## cache bust when updated repositories
-ADD ${EICWEB}/458/repository/tree?ref=${BENCHMARK_COM_VERSION} /tmp/485.json
-ADD ${EICWEB}/399/repository/tree?ref=${BENCHMARK_DET_VERSION} /tmp/399.json
-ADD ${EICWEB}/408/repository/tree?ref=${BENCHMARK_REC_VERSION} /tmp/408.json 
-ADD ${EICWEB}/400/repository/tree?ref=${BENCHMARK_PHY_VERSION} /tmp/400.json
+ADD ${EICWEB}/458/repository/commits/${BENCHMARK_COM_VERSION} /tmp/485.json
+ADD ${EICWEB}/399/repository/commits/${BENCHMARK_DET_VERSION} /tmp/399.json
+ADD ${EICWEB}/408/repository/commits/${BENCHMARK_REC_VERSION} /tmp/408.json 
+ADD ${EICWEB}/400/repository/commits/${BENCHMARK_PHY_VERSION} /tmp/400.json
 RUN <<EOF
 set -ex
 mkdir -p /opt/benchmarks