feat: avoid large copy into runtime image

This MR removes the many COPYs from the builder to runtime image, which make it hard to reuse the default environment layer and causes large downloads.

The container is now built with two parallel tracks: one builder and one runtime.

## This docker build follows two tracks, in order to ensure that we build all packages
## in a builder image, but install them in a runtime image, while at the same time
## avoiding a expensive filesystem copy operation at the end that breaks layering.
##
## The build is split in an infrequently-changing default environment, upon which
## an environment with custom versions (e.g. individual commits) is layered. The
## custom environment will change frequently but layers will be smaller, allowing
## for easier deployment with smaller delta layers.
##
## The separation in a builder and runtime image is particularly relevant to end up with
## lightweight images for expensive build dependencies, such as for example CUDA.
##
## builder track                         runtime track
## ----------------------------------------------------------------------
## builder_image                         runtime_image
## builder_concretization_default   
## builder_installation_default     ->   runtime_concretization_default  (copy spack.lock)
##                                 \->   runtime_installation_default    (from buildcache)
## builder_concretization_custom
## builder_installation_custom      ->   runtime_concretization_custom   (copy spack.lock)
##                                 \->   runtime_installation_custom     (from buildcache)

containers/jug/dev.Dockerfile

@@ -5,14 +5,38 @@ ARG BUILDER_IMAGE="debian_stable_base"
 ARG RUNTIME_IMAGE="debian_stable_base"
 ARG INTERNAL_TAG="testing"
 
+##
+## This docker build follows two tracks, in order to ensure that we build all packages
+## in a builder image, but install them in a runtime image, while at the same time
+## avoiding a expensive filesystem copy operation at the end that breaks layering.
+##
+## The build is split in an infrequently-changing default environment, upon which
+## an environment with custom versions (e.g. individual commits) is layered. The
+## custom environment will change frequently but layers will be smaller, allowing
+## for easier deployment with smaller delta layers.
+##
+## The separation in a builder and runtime image is particularly relevant to end up with
+## lightweight images for expensive build dependencies, such as for example CUDA.
+##
+## builder track                         runtime track
+## ----------------------------------------------------------------------
+## builder_image                         runtime_image
+## builder_concretization_default   
+## builder_installation_default     ->   runtime_concretization_default  (copy spack.lock)
+##                                 \->   runtime_installation_default    (from buildcache)
+## builder_concretization_custom
+## builder_installation_custom      ->   runtime_concretization_custom   (copy spack.lock)
+##                                 \->   runtime_installation_custom     (from buildcache)
+##
+
 ## ========================================================================================
-## STAGE 1: builder
-## EIC builder image with spack environment
+## builder_concretization_default
+## - builder base with concretization of default versions
 ## ========================================================================================
-FROM ${DOCKER_REGISTRY}${BUILDER_IMAGE}:${INTERNAL_TAG} as builder
+FROM ${DOCKER_REGISTRY}${BUILDER_IMAGE}:${INTERNAL_TAG} as builder_concretization_default
 ARG TARGETPLATFORM
 
-## 1. Setup our default environment (secret mount for write-enabled mirror)
+## Copy our default environment
 COPY --from=spack-environment . /opt/spack-environment/
 ARG ENV=dev
 ENV SPACK_ENV=/opt/spack-environment/${ENV}
@@ -24,6 +48,14 @@ spack env activate --dir ${SPACK_ENV}
 spack concretize --fresh --force
 EOF
 
+
+## ========================================================================================
+## builder_installation_default
+## - builder base with installation of default versions
+## ========================================================================================
+FROM builder_concretization_default as builder_installation_default
+ARG TARGETPLATFORM
+
 # Installation (default environment)
 RUN --mount=type=cache,target=/ccache,id=${TARGETPLATFORM}              \
     --mount=type=cache,target=/var/cache/spack                          \
@@ -47,6 +79,41 @@ ccache --show-stats
 ccache --zero-stats
 EOF
 
+
+## ========================================================================================
+## runtime_concretization_default
+## - runtime base with concretization of default versions (taken from equivalent builder)
+## ========================================================================================
+FROM ${DOCKER_REGISTRY}${RUNTIME_IMAGE}:${INTERNAL_TAG} as runtime_concretization_default
+ARG TARGETPLATFORM
+
+COPY --from=builder_installation_default /opt/spack-environment /opt/spack-environment
+ARG ENV=dev
+ENV SPACK_ENV=/opt/spack-environment/${ENV}
+
+
+## ========================================================================================
+## runtime_installation_default
+## - runtime base with installation of default versions (buildcache populated by builder)
+## ========================================================================================
+FROM runtime_concretization_default as runtime_installation_default
+ARG TARGETPLATFORM
+
+# Installation (default environment, from buildcache)
+RUN --mount=type=cache,target=/var/cache/spack                          \
+    --mount=type=secret,id=mirrors,target=/opt/spack/etc/spack/mirrors.yaml \
+    <<EOF
+make --jobs ${jobs} --keep-going --directory /opt/spack-environment \
+  SPACK_ENV=${SPACK_ENV} SPACK_INSTALL_FLAGS="--use-buildcache only"
+EOF
+
+## ========================================================================================
+## builder_concretization_custom
+## - builder base with concretization of custom versions
+## ========================================================================================
+FROM builder_installation_default as builder_concretization_custom
+ARG TARGETPLATFORM
+
 ## 2. Setup our environment with custom versions (on top of cached layer)
 ## Note: these default versions are just the very first commit.
 ARG EDM4EIC_VERSION="8aeb507f93a93257c99985efbce0ec1371e0b331"
@@ -85,6 +152,13 @@ fi
 spack concretize --fresh --force
 EOF
 
+## ========================================================================================
+## builder_installation_custom
+## - builder base with installation of custom versions
+## ========================================================================================
+FROM builder_concretization_custom as builder_installation_custom
+ARG TARGETPLATFORM
+
 # Installation (custom environment)
 RUN --mount=type=cache,target=/ccache,id=${TARGETPLATFORM}              \
     --mount=type=cache,target=/var/cache/spack                          \
@@ -105,6 +179,34 @@ ccache --show-stats
 ccache --zero-stats
 EOF
 
+
+## ========================================================================================
+## runtime_concretization_custom
+## - runtime base with concretization of custom versions (taken from equivalent builder)
+## ========================================================================================
+FROM runtime_installation_default as runtime_concretization_custom
+ARG TARGETPLATFORM
+
+ARG ENV=dev
+ENV SPACK_ENV=/opt/spack-environment/${ENV}
+COPY --from=builder_installation_custom /opt/spack-environment /opt/spack-environment
+
+
+## ========================================================================================
+## runtime_installation_custom
+## - runtime base with installation of custom versions (buildcache populated by builder)
+## ========================================================================================
+FROM runtime_concretization_custom as runtime_installation_custom
+ARG TARGETPLATFORM
+
+# Installation (default environment, from buildcache)
+RUN --mount=type=cache,target=/var/cache/spack                          \
+    --mount=type=secret,id=mirrors,target=/opt/spack/etc/spack/mirrors.yaml \
+    <<EOF
+make --jobs ${jobs} --keep-going --directory /opt/spack-environment \
+  SPACK_ENV=${SPACK_ENV} SPACK_INSTALL_FLAGS="--use-buildcache only"
+EOF
+
 ## Create views at /opt/local and /opt/detector
 RUN <<EOF
 set -e