feat: use spack ci generate to setup build pipeline

Rather than having a single long build, we can split it up in different jobs with spack ci generate. This will be more resilient against failures. We can also run the builds inside a regular job instead of inside the container, and then only pull from the build cache once we build the container.

.gitlab-ci.yml

@@ -23,6 +23,12 @@ variables:
   ## Ref: https://github.com/eic/eic-spack/commit/[hash]
   ## [hash]: [description]
 
+  ## S3 buildcache access
+  S3_ENDPOINT_URL: "https://dtn01.sdcc.bnl.gov:9000"
+  ## S3 credentials (through CI/CD variables)
+  AWS_ACCESS_KEY_ID: "${S3_ACCESS_KEY}"
+  AWS_SECRET_ACCESS_KEY: "${S3_SECRET_KEY}"
+
   ## We need to enable Docker Buildkit to use cache mounts and better
   ## build performance overall
   DOCKER_BUILDKIT: 1
@@ -50,6 +56,8 @@ variables:
 
 stages:
   - config
+  - generate        ## spack ci generate
+  - build           ## spack build jobs
   - base            ## base OS image
   - jug             ## jug container images
   - deploy          ## build/deploy singularity images
@@ -212,6 +220,53 @@ base:
                    containers/debian
     - !reference [.build, script]
 
+.generate:
+  stage: generate
+  tags:
+    - docker-new
+  image:
+    name: ${CI_REGISTRY_IMAGE}/${BASE_IMAGE}:${INTERNAL_TAG}
+    entrypoint: [""]
+  before_script:
+    - apt-get update -yqq && apt-get install -yqq python3-boto3
+    - git clone https://github.com/${SPACK_ORGREPO}.git spack
+    - git -C spack checkout ${SPACK_VERSION}
+    - . "./spack/share/spack/setup-env.sh"
+    - git clone https://github.com/${EICSPACK_ORGREPO}.git ${SPACK_ROOT}/var/spack/repos/eic-spack
+    - git -C ${SPACK_ROOT}/var/spack/repos/eic-spack checkout ${EICSPACK_VERSION}
+  script:
+    - spack env activate --without-view .
+    - spack mirror list
+    - spack mirror rm --scope defaults spack-public
+    - find /usr/local
+    - rm -r /usr/local
+    - spack ci generate
+      --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir/"
+      --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/pipeline.yml"
+      --check-index-only
+      --prune-dag
+      --optimize
+  after_script: [ ]
+  artifacts:
+    paths:
+      - "${CI_PROJECT_DIR}/jobs_scratch_dir"
+
+generate-pipeline-debian-testing:
+  extends: .generate
+  variables:
+    BASE_IMAGE: debian_testing_base
+
+build-jobs:
+  stage: build
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/pipeline.yml
+        job: generate-pipeline-debian-testing
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: generate-pipeline-debian-testing
+
 jug_dev:default:
   parallel:
     matrix: