diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8e6f61a8d94a6d07ed6982fd50cc563c0cfa8a4e..17b05fbb2ec991442827cec5e6ed6497548d08d4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -36,6 +36,9 @@ variables:
65bd9b9ac556480b4a9dcc60f7539492af195d4a
1a32cea11495cbdd699fea4fe622babab83e630d
6edc4807369a05786e36f63b5d959588ae94a1fa
+ af74680405c931dab16c6674f9b97a32bf3f1122
+ 0a952f8b7bf6f70009dd5821bccbaf9170c73d07
+ f050b1cf7835fd31992b020e1061c52294ff7330
## Ref: https://github.com/spack/spack/commit/[hash]
## [hash]: [description]
## b5ef5c2eb5145020f9de1bcb964626ce6ac2d02e: geant4: version bumps for Geant4 11.1.0
@@ -62,6 +65,9 @@ variables:
## 65bd9b9ac556480b4a9dcc60f7539492af195d4a: podio, edm4hep: add v0.7.2 and v0.16.1 respectively
## 1a32cea11495cbdd699fea4fe622babab83e630d: podio: add v0.16.2
## 6edc4807369a05786e36f63b5d959588ae94a1fa: podio: Add version 0.16.3
+ ## af74680405c931dab16c6674f9b97a32bf3f1122: depfile: improve tab completion
+ ## 0a952f8b7bf6f70009dd5821bccbaf9170c73d07: docs updates for spack env depfile
+ ## f050b1cf7835fd31992b020e1061c52294ff7330: depfile: variable with all identifiers
## EIC spack organization and repository, e.g. eic/eic-spack
EICSPACK_ORGREPO: "eic/eic-spack"
diff --git a/containers/jug/dev.Dockerfile b/containers/jug/dev.Dockerfile
index b6a79e5f81f02952ff4321eedb2cb5fa90a1999b..98008cf0012b141bc3af1e9397df002f36582bc8 100644
--- a/containers/jug/dev.Dockerfile
+++ b/containers/jug/dev.Dockerfile
@@ -100,77 +100,21 @@ RUN git clone https://github.com/${EICSPACK_ORGREPO}.git ${EICSPACK_ROOT} \
## Setup our custom environment
COPY --from=spack spack-environment/ /opt/spack-environment/
ARG ENV=dev
-RUN rm -r /usr/local \
- && cd /opt/spack-environment \
+RUN --mount=type=cache,target=/var/cache/spack-mirror,sharing=locked \
+ cd /opt/spack-environment \
+ && rm -r /usr/local \
&& source $SPACK_ROOT/share/spack/setup-env.sh \
&& spack env activate --dir /opt/spack-environment/${ENV} \
- && spack concretize --fresh
-
+ && make -C /opt/spack-environment SPACK_ENV=${ENV} \
+ BUILDCACHE_DIR=/var/cache/spack-mirror \
+ BUILDCACHE_MIRROR=eic-spack
-## Now execute the main build (or fetch from cache if possible)
-## note, no-check-signature is needed to allow the quicker signature-less
-## packages from the internal (docker) buildcache
-##
## Optional, nuke the buildcache after install, before (re)caching
## This is useful when going to completely different containers,
## or intermittently to keep the buildcache step from taking too much time
-##
-## Update the local build cache if needed. Consists of 3 steps:
-## 1. Remove the eic-spack buildcache on S3
-## 2. Get a list of all packages, and compare with what is already on
-## the buildcache (using package hash)
-## 3. Add packages that need to be added to buildcache if any
-RUN --mount=type=cache,target=/var/cache/spack-mirror \
- cd /opt/spack-environment \
- && source $SPACK_ROOT/share/spack/setup-env.sh \
- && spack env activate --dir /opt/spack-environment/${ENV} \
- && status=0 \
- && spack install -j64 --no-check-signature \
- || spack install -j64 --no-check-signature \
- || spack install -j64 --no-check-signature --show-log-on-error \
- || status=$? \
- && spack mirror rm --scope site eic-spack \
- && [ -z "${CACHE_NUKE}" ] \
- || rm -rf /var/cache/spack-mirror/build_cache/* \
- && mkdir -p /var/cache/spack-mirror/build_cache \
- && spack buildcache update-index -d /var/cache/spack-mirror \
- && spack buildcache list --allarch --very-long \
- | sed '/^$/d;/^--/d;s/@.\+//;s/\([a-z0-9]*\) \(.*\)/\2\/\1/' \
- | sort > buildcache.local.txt \
- && spack find --format {name}/{hash} | sort \
- | comm -23 - buildcache.local.txt \
- | xargs --verbose --no-run-if-empty \
- spack buildcache create --allow-root --only package --unsigned \
- --directory /var/cache/spack-mirror \
- --rebuild-index \
- && spack clean -a \
- && exit $status
-
-## Update the S3 build cache (without local cache mount)
-ARG S3RW_ACCESS_KEY=""
-ARG S3RW_SECRET_KEY=""
-RUN cd /opt/spack-environment \
- && source $SPACK_ROOT/share/spack/setup-env.sh \
- && spack env activate --dir /opt/spack-environment/${ENV} \
- && if [ -n "${S3RW_ACCESS_KEY}" ] ; then \
- spack mirror add --scope site \
- --s3-endpoint-url https://eics3.sdcc.bnl.gov:9000 \
- --s3-access-key-id "${S3RW_ACCESS_KEY}" \
- --s3-access-key-secret "${S3RW_SECRET_KEY}" \
- eic-spack s3://eictest/EPIC/spack \
- && spack mirror list \
- && spack buildcache list --allarch --very-long \
- | sed '/^$/d;/^--/d;s/@.\+//;s/\([a-z0-9]*\) \(.*\)/\2\/\1/' \
- | sort > buildcache.eic-spack.txt \
- && spack find --format {name}/{hash} | sort \
- | comm -23 - buildcache.eic-spack.txt \
- | xargs --verbose --no-run-if-empty \
- spack buildcache create --allow-root --only package --unsigned \
- --mirror-name eic-spack \
- && spack buildcache update-index --mirror-url eic-spack \
- && spack mirror rm --scope site eic-spack \
- ; fi \
- && spack mirror list
+RUN --mount=type=cache,target=/var/cache/spack-mirror,sharing=locked \
+ [ -z "${CACHE_NUKE}" ] \
+ || rm -rf /var/cache/spack-mirror/build_cache/*
## Extra post-spack steps:
## - Python packages
diff --git a/spack-environment/Makefile b/spack-environment/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..8f4dba4125f41e88733c651e7932e193e8a68475
--- /dev/null
+++ b/spack-environment/Makefile
@@ -0,0 +1,39 @@
+MAKEFLAGS += -Orecurse
+
+SPACK ?= spack
+SPACK_INSTALL_FLAGS += --no-check-signature
+
+export SPACK_COLOR = always
+
+SPACK_ENV ?= dev
+
+BUILDCACHE_DIR := $(SPACK_ENV)/cache
+BUILDCACHE_MIRROR :=
+
+.PHONY: all clean
+
+all: $(SPACK_ENV)/push
+
+ifeq (,$(filter clean,$(MAKECMDGOALS)))
+include $(SPACK_ENV)/spack.mk
+endif
+
+$(SPACK_ENV)/push: $(addprefix $(SPACK_ENV)/push/,$($(SPACK_ENV)/SPACK_PACKAGE_IDS))
+ @mkdir -p $(BUILDCACHE_DIR)
+ $(foreach buildcache, $(BUILDCACHE_DIR), $(SPACK) buildcache update-index --mirror-url $(buildcache) ;)
+ $(foreach buildcache, $(BUILDCACHE_MIRROR), $(SPACK) buildcache update-index --mirror-url $(buildcache) ;)
+
+$(SPACK_ENV)/push/%: $(SPACK_ENV)/install/%
+ @mkdir -p $(dir $@)
+ $(foreach buildcache, $(BUILDCACHE_DIR), $(SPACK) buildcache create --allow-root --only=package --unsigned --directory $(buildcache) /$(HASH) ;) # push $(SPEC)
+ $(foreach buildcache, $(BUILDCACHE_MIRROR), $(SPACK) buildcache create --allow-root --only=package --unsigned --mirror-name $(buildcache) /$(HASH) ;) # push $(SPEC)
+ @touch $@
+
+$(SPACK_ENV)/spack.lock: $(SPACK_ENV)/spack.yaml Makefile
+ $(SPACK) concretize --force --fresh
+
+$(SPACK_ENV)/spack.mk: $(SPACK_ENV)/spack.lock Makefile
+ $(SPACK) env depfile --output $@ --make-target-prefix $(SPACK_ENV)
+
+clean:
+ rm -rf $(SPACK_ENV)/spack.lock $(SPACK_ENV)/spack.mk