.gitlab-ci.yml

image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest

variables:
  ## Application versions used for the main release 
  ## note: nightly builds will always use the master/main branch
  JUGGLER_VERSION: "v9.4.0"
  EICRECON_VERSION: "v1.1.1"

  ## Spack organization and repository, e.g. spack/spack
  SPACK_ORGREPO: "spack/spack"
  ## Spack github version, e.g. v0.18.1 or commit hash
  ## note: nightly builds will use e.g. releases/v0.19
  SPACK_VERSION: "v0.19.2"
  ## Space-separated list of spack cherry-picks
  SPACK_CHERRYPICKS: >
    b5ef5c2eb5145020f9de1bcb964626ce6ac2d02e
    99056e03bd3e903b222e300636ec484d85d4b3fb
    f3f8b31be54280c6ef0b5f95ab85604aba3dff30
    0ea81affd18820933640279bbc687038b3296a4e
    dceb4c9d655d0529e112b8929558be60973b39f7
    f2b0c1deab87da3b5aa4c1f2ef4d6af06fa4e32d
    2f057d729da61e8c12828df44377f0a93fed820f
    39a1f1462b0abf12dfaccd33f84142c852c4b56e
    4b12d015e1c316b4837e02ae37e2c69a8a647180
    f83d47442dade012b1019840181b8dd459fd8edd
    7f1467e795b1cab8b4855e019910c509896ea0e1
    ebc24b7063ba9a8eb43b4424aac5143cf958d76f
    a47ebe57841f13239e881ed69eab4949b1d98c32
    ab999d5af90f1bff644b5134bb370b2716e1bcf0
    62da76cb5dca4d52c43bee06230cca6a5882f05d
    cc2ae9f270befa554ba8b09c68e89bb8248ea650
    ae98d2ba2fcefa9d027e2d6ccc6e7558a32e7228
    ae189cfab8d9036e8d39bbd3f1b61b400d1fcd5b
    3afe6f1adcc24335cbca9a9c03ffea188f802766
    559c3de213707b5d52d899fd0382495f2cc8508d
    8e84dcd7ef999e2659822b34372515175f1723c4
    65bd9b9ac556480b4a9dcc60f7539492af195d4a
    1a32cea11495cbdd699fea4fe622babab83e630d
    6edc4807369a05786e36f63b5d959588ae94a1fa
    af74680405c931dab16c6674f9b97a32bf3f1122
    0a952f8b7bf6f70009dd5821bccbaf9170c73d07
    f050b1cf7835fd31992b020e1061c52294ff7330
    a419ffcf501134faed24253ccc83e6c71f9659f9
    c3e41153ac92f6ef92414024a8386d4ceec2615c
  ## Ref: https://github.com/spack/spack/commit/[hash]
  ## [hash]: [description]
  ## b5ef5c2eb5145020f9de1bcb964626ce6ac2d02e: geant4: version bumps for Geant4 11.1.0
  ## 99056e03bd3e903b222e300636ec484d85d4b3fb: acts: new versions 19.11.0, 21.0.0, 21.1.0
  ## f3f8b31be54280c6ef0b5f95ab85604aba3dff30: XRootD: add checksum + patch for 5.5.1 
  ## 0ea81affd18820933640279bbc687038b3296a4e: py-torch: fix build with gcc@12:
  ## dceb4c9d655d0529e112b8929558be60973b39f7: Update PyTorch ecosystem
  ## f2b0c1deab87da3b5aa4c1f2ef4d6af06fa4e32d: py-minkowskiengine: new package (sparse tensor autodiff by Nvidia)
  ## 2f057d729da61e8c12828df44377f0a93fed820f: py-scipy: add v1.9
  ## 39a1f1462b0abf12dfaccd33f84142c852c4b56e: SIP build system: fix "python not defined in builder"
  ## 4b12d015e1c316b4837e02ae37e2c69a8a647180: py-jinja2-cli: new package
  ## f83d47442dade012b1019840181b8dd459fd8edd: dd4hep: depends_on root +x +opengl when +utilityapps
  ## 7f1467e795b1cab8b4855e019910c509896ea0e1: dd4hep: new version 1.24, depends_on podio@0.16:
  ## ebc24b7063ba9a8eb43b4424aac5143cf958d76f: dd4hep: extend conflict on CMake
  ## a47ebe57841f13239e881ed69eab4949b1d98c32: dd4hep: new versions 1.25, 1.25.1
  ## ab999d5af90f1bff644b5134bb370b2716e1bcf0: dd4hep: depends_on root +webgui when +ddeve ^root @6.28:
  ## 62da76cb5dca4d52c43bee06230cca6a5882f05d: directives: depends_on should not admit anonymous specs
  ## cc2ae9f270befa554ba8b09c68e89bb8248ea650: Add a "maintainer" directive
  ## ae98d2ba2fcefa9d027e2d6ccc6e7558a32e7228: Support packages for using scitokens on OSG
  ## ae189cfab8d9036e8d39bbd3f1b61b400d1fcd5b: geant4: new version 11.1.1
  ## 3afe6f1adcc24335cbca9a9c03ffea188f802766: ROOT: add math/gsl conflict and change version-dependent features to conditional variants
  ## 559c3de213707b5d52d899fd0382495f2cc8508d: ROOT: new versions and associated dependency constraints
  ## 8e84dcd7ef999e2659822b34372515175f1723c4: root: new version 6.28.00
  ## 65bd9b9ac556480b4a9dcc60f7539492af195d4a: podio, edm4hep: add v0.7.2 and v0.16.1 respectively
  ## 1a32cea11495cbdd699fea4fe622babab83e630d: podio: add v0.16.2
  ## 6edc4807369a05786e36f63b5d959588ae94a1fa: podio: Add version 0.16.3
  ## af74680405c931dab16c6674f9b97a32bf3f1122: depfile: improve tab completion
  ## 0a952f8b7bf6f70009dd5821bccbaf9170c73d07: docs updates for spack env depfile
  ## f050b1cf7835fd31992b020e1061c52294ff7330: depfile: variable with all identifiers
  ## a419ffcf501134faed24253ccc83e6c71f9659f9: osg-ca-certs: igtf link should point to version, not 'current'
  ## c3e41153ac92f6ef92414024a8386d4ceec2615c: Package requirements: allow single specs in requirement lists

  ## EIC spack organization and repository, e.g. eic/eic-spack
  EICSPACK_ORGREPO: "eic/eic-spack"
  ## EIC spack github version, e.g. v0.19.7 or commit hash
  ## note: nightly builds will use e.g. releases/v0.19
  EICSPACK_VERSION: "v0.19.1"
  ## Space-separated list of eic-spack cherry-picks
  EICSPACK_CHERRYPICKS: "f892e2b5d7ea9d1f2e43741499e899ce21dd3d5a"
  ## Ref: https://github.com/eic/eic-spack/commit/[hash]
  ## [hash]: [description]
  ## f892e2b5d7ea9d1f2e43741499e899ce21dd3d5a: py-minkowskiengine: new package

  ## Local registry
  CI_PUSH: 1
  ## Dockerhub registry
  DH_REGISTRY: docker.io
  DH_REGISTRY_USER: eicweb
  DH_PUSH: 1
  ## GitHub registry
  GH_REGISTRY: ghcr.io
  GH_REGISTRY_USER: eic
  GH_PUSH: 1

  ## Also export the raw builder image, will be stored on DH only
  EXPORT_BUILDER: 0

  ## TLS error resiliency: number of retries and second wait between tries 
  ## (wait time is doubled with each attempt)
  DOCKER_NTRIES: 5
  DOCKER_WAIT_TIME: 5

  ## Number of jobs to start during container builds
  JOBS: 128

  ## is this nightly or not?
  NIGHTLY:

  ## Force a rebuild without using cache
  FORCE_NOCACHE: 0

  ## Default version and tags
  ## - internal tag used for the CI. Also temporarily tagged
  ##   on eicweb to communicate between jobs (removed in cleanup job)
  ## - export tag to public registries, optional secondary export tag
  VERSION: "${CI_COMMIT_REF_NAME}"
  NIGHTLY_TAG: "nightly"
  TESTING_TAG: "testing"
  INTERNAL_TAG: "${TESTING_TAG}-${VERSION}"
  EXPORT_TAG: ""

stages:
  - config
  - base            ## base OS image
  - jug             ## jug container images
  - deploy          ## build/deploy singularity images
  - benchmarks
  - test
  - finalize

default:
  before_script:
    - ./gitlab-ci/docker_login.sh -u ${DH_REGISTRY_USER} -p ${DH_EICWEB_TOKEN}
                                  -n ${DOCKER_NTRIES} -t ${DOCKER_WAIT_TIME}
    - ./gitlab-ci/docker_login.sh -u ${GITHUB_REGISTRY_USER} -p ${GITHUB_REGISTRY_TOKEN}
                                  -n ${DOCKER_NTRIES} -t ${DOCKER_WAIT_TIME}
                                  ${GH_REGISTRY}
    - ./gitlab-ci/docker_login.sh --ci -n ${DOCKER_NTRIES} -t ${DOCKER_WAIT_TIME}
  after_script:
    - docker logout
    - docker logout ${CI_REGISTRY}
    - docker logout ${GH_REGISTRY}

## only run CI for in the following cases:
## master, stable branch, release tag, MR event and nightly builds
## nightly builds are now part of the regular master build in order to keep
## all artifacts available at all times.
workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_PIPELINE_SOURCE == "web"'
    - if: '$CI_COMMIT_BRANCH == "master"'
    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-stable/'          ## main stable branch: vX.Y-stable
    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-[a-z]+-stable/'   ## special stable branch: vX.Y-acadia-stable (etc)
    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+/'            ## commit tags start with vX.Y.Z with optional suffix

## plan:
##  Workflows:
##  - master      --> config + all build stages + singularity
#                            + nightly jug + nightly singularity
##  - v4.0-stable --> config + all build stages + singularity
##  - v4.0.0      --> config + all build stages + singularity
##  - MR          --> config + all build stages
##
##  Container images tags
##  - master      --> testing
##  - <nightly>   --> nightly (run as part of master)
##  - v4.0-stable --> 4.0-stable
##  - v4.0.0      --> 4.0-stable, 4.0.0
##  - acadia     
##  - MR          --> 4.0-unstable (on all registries)
##                --> unstable-mr-XXX (on eicweb only, untag at end of pipeline)
##  - all other   --> do nothing
##
## caching strategy for dispatch to multiple nodes and to avoid
##  --> try this strategy: https://medium.com/titansoft-engineering/docker-build-cache-sharing-on-multi-hosts-with-buildkit-and-buildx-eb8f7005918e

version:
  stage: config
  script:
    - |
      if [ "x${CI_PIPELINE_SOURCE}" = "xmerge_request_event" ]; then
        VERSION="unstable-mr-${CI_MERGE_REQUEST_PROJECT_ID}-${CI_MERGE_REQUEST_IID}"
        INTERNAL_TAG="${VERSION}"
        NIGHTLY_TAG="${VERSION}-nightly"
        EXPORT_TAG="${VERSION}"
      elif [[ "$CI_COMMIT_TAG" =~ ^v[0-9.]+\.[0-9]+\.[0-9]+ ]]; then
        VERSION="${CI_COMMIT_TAG:1}"
        INTERNAL_TAG="stable-tag-${VERSION}"
        EXPORT_TAG="${VERSION}"
      elif [[ "$CI_COMMIT_BRANCH" = ^v[0-9.]+\.[0-9]+-stable ]]; then
        VERSION="${CI_COMMIT_BRANCH:1}"
        INTERNAL_TAG="stable-br-${VERSION}"
        EXPORT_TAG="${VERSION}"
      elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+\.[0-9]+-[a-z]+-stable ]]; then
        VERSION="${CI_COMMIT_BRANCH:1}"
        INTERNAL_TAG="stable-br-${VERSION}"
        EXPORT_TAG="${VERSION}"
      fi
      echo "VERSION=$VERSION" >> build.env
      echo "INTERNAL_TAG=$INTERNAL_TAG" >> build.env
      echo "NIGHTLY_TAG=$NIGHTLY_TAG" >> build.env
      echo "EXPORT_TAG=$EXPORT_TAG" >> build.env
      cat build.env
  artifacts:
    reports:
      dotenv: build.env

## base job settings for all build jobs
.build:
  rules:
    - when: on_success
  resource_group: ${CI_COMMIT_REF_NAME}-${VERSION}
  tags:
    - docker-new
  before_script:
    - !reference [default, before_script]
    - mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc ;
      for arch in aarch64 ; do
        if ! grep --quiet enabled /proc/sys/fs/binfmt_misc/qemu-$arch ; then
          docker run --rm --privileged multiarch/qemu-user-static --persistent yes ;
        fi ;
      done
    - docker buildx create --name builder --driver docker-container --bootstrap --use

## Images:
## debian_testing_base --> jug_dev  --> jug_xl
##             ---------------> jug_sim
##             ---------------> jug_ml
## debian_stable_base --> oneapi_jug_dev
## oneapi_jug_dev + jug_xl   --> oneapi_jug_xl

## TODO
## oneapi_runtime + jug_xl --> oneapi_prod 

base:
  parallel:
    matrix:
      - BASE_IMAGE: debian:testing-20220822-slim
        BUILD_IMAGE: debian_testing_base
        PLATFORM: linux/amd64,linux/arm64/v8
      - BASE_IMAGE: ubuntu:20.04
        BUILD_IMAGE: ubuntu_base
        PLATFORM: linux/amd64,linux/arm64/v8
      - BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04
        BUILD_IMAGE: oneapi_base
        PLATFORM: linux/amd64
#      - BASE_IMAGE: nvidia/cuda:11.8.0-devel-ubuntu20.04
#        BUILD_IMAGE: cuda_base
#        PLATFORM: linux/amd64
  extends: .build
  stage: base
  needs:
    - version
  script:
    - docker buildx build --push
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
                   ${EXPORT_TAG:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
                   }
                   --file containers/debian/base.Dockerfile
                   --platform ${PLATFORM}
                   --build-arg BASE_IMAGE=${BASE_IMAGE}
                   --build-arg BUILD_IMAGE=${BUILD_IMAGE}
                   containers/debian

jug_dev:default:
  parallel:
    matrix:
      - BASE_IMAGE: debian_testing_base
        BUILD_IMAGE: jug_dev
        PLATFORM: linux/amd64,linux/arm64/v8
#      - BASE_IMAGE: oneapi_base
#        BUILD_IMAGE: jug_dev_oneapi
#        PLATFORM: linux/amd64
#      - BASE_IMAGE: cuda_base_new
#        BUILD_IMAGE: jug_dev_cuda
#        PLATFORM: linux/amd64
  extends: .build
  stage: jug
  needs:
    - version
    - base
  script:
    ## Which targets to build
    - TARGETS="export"
    - |
      if [ ${EXPORT_BUILDER} = 1 ]; then
        TARGETS="builder ${TARGETS}"
      fi
    ## Disable caching
    - CACHE_FLAG=""
    - |
      if [ $FORCE_NOCACHE = 1 ]; then
        echo "FORCE_NOCACHE set"
        export CACHE_FLAG="--no-cache"
      fi
    ## Setup mirrors.yaml with rw mirror secrets
    - cat mirrors.yaml.in | envsubst > mirrors.yaml
    ## Build targets
    - for target in ${TARGETS} ; do
        if [ ! ${target} = export ] ; then
          suffix=-${target} ;
        fi ;
        docker buildx build --push
                   --target ${target}
                   ${CACHE_FLAG}
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}${suffix}
                   ${EXPORT_TAG:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}${suffix}}
                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}${suffix}}
                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}${suffix}}
                   }
                   --file containers/jug/dev.Dockerfile
                   --platform ${PLATFORM}
                   --build-arg BASE_IMAGE="${BASE_IMAGE}"
                   --build-arg SPACK_ORGREPO="${SPACK_ORGREPO}"
                   --build-arg SPACK_VERSION="${SPACK_VERSION}"
                   --build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
                   --build-arg EICSPACK_ORGREPO="${EICSPACK_ORGREPO}"
                   --build-arg EICSPACK_VERSION="${EICSPACK_VERSION}"
                   --build-arg EICSPACK_CHERRYPICKS="${EICSPACK_CHERRYPICKS}"
                   --build-arg S3_ACCESS_KEY="${S3_ACCESS_KEY}"
                   --build-arg S3_SECRET_KEY="${S3_SECRET_KEY}"
                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
                   --build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD)
                   --build-arg jobs=${JOBS}
                   --build-context spack=.
                   --secret id=mirrors,src=mirrors.yaml
                   containers/jug
      ; done

jug_xl:default:
  parallel:
    matrix:
      - BASE_IMAGE: jug_dev
        BUILD_IMAGE: jug_xl
        PLATFORM: linux/amd64,linux/arm64/v8
#      - BASE_IMAGE: jug_dev_oneapi
#        BUILD_IMAGE: jug_xl_oneapi
#        PLATFORM: linux/amd64
  extends: .build
  stage: jug
  needs:
    - version
    - jug_dev:default
  script:
    - docker buildx build --push
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
                   ${EXPORT_TAG:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
                   }
                   --file containers/jug/xl.Dockerfile
                   --platform ${PLATFORM}
                   --build-arg BASE_IMAGE=${BASE_IMAGE}
                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
                   --build-arg JUGGLER_VERSION=${JUGGLER_VERSION}
                   --build-arg EICRECON_VERSION=${EICRECON_VERSION}
                   --build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD)
                   --build-arg jobs=${JOBS}
                   --build-context detectors=.
                   containers/jug

jug_xl:nightly:
  parallel:
    matrix:
      - BASE_IMAGE: jug_dev
        BUILD_IMAGE: jug_xl
        PLATFORM: linux/amd64,linux/arm64/v8
#      - BASE_IMAGE: jug_dev_oneapi
#        BUILD_IMAGE: jug_xl_oneapi
#        PLATFORM: linux/amd64
  extends: .build
  stage: jug
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
      when: on_success
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: on_success
    - when: never
  needs:
    - version
    - jug_dev:default
  script:
    - |
      PUSH_NIGHTLY_WITH_DATE=""
      if [ "$CI_COMMIT_BRANCH" == "master" ]; then
        PUSH_NIGHTLY_WITH_DATE="1"
      fi
    - docker buildx build --push
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
                   ${NIGHTLY:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
                     ${PUSH_NIGHTLY_WITH_DATE:+
                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
                     }
                   }
                   --file containers/jug/xl.Dockerfile
                   --platform ${PLATFORM}
                   --build-arg BASE_IMAGE=${BASE_IMAGE}
                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
                   --build-arg JUG_VERSION=nightly-$(date +%Y-%m-%d_%H-%M-%S)-${INTERNAL_TAG}-$(git rev-parse HEAD)
                   --build-arg NIGHTLY=1
                   --build-context detectors=.
                   containers/jug

jug_xl:feature:
  extends: .build
  stage: jug
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
      when: on_success
    - when: never
  needs:
    - version
    - jug_xl:nightly
  variables:
    BUILD_IMAGE: "jug_xl"
  parallel:
    matrix:
      - DETECTOR: epic
        DETECTOR_BRANCH:
          - main
  script:
    - docker buildx build --load --tag ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}-${DETECTOR}-${DETECTOR_BRANCH}
                   --file containers/jug/feature.Dockerfile
                   --build-arg INTERNAL_TAG=${NIGHTLY_TAG}
                   --build-arg DETECTOR=${DETECTOR}
                   --build-arg DETECTOR_BRANCH=${DETECTOR_BRANCH}
                   containers/jug
    - |
      PUSH_FLAG=""
      if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
        PUSH_FLAG="--eicweb"
      fi
      FEATURE_EXPORT_TAG="${VERSION}-${DETECTOR}-${DETECTOR_BRANCH}-stable"
    - ./gitlab-ci/docker_push.sh -i ${BUILD_IMAGE} -l ${INTERNAL_TAG}-${DETECTOR}-${DETECTOR_BRANCH}
                                 -n ${DOCKER_NTRIES} -t ${DOCKER_WAIT_TIME}
                                 ${FEATURE_EXPORT_TAG} ${PUSH_FLAG}

.singularity:
  stage: deploy
  interruptible: true
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never
  artifacts:
      expire_in: 1 days
      paths:
        - build/${BUILD_IMAGE}.sif
  script:
    - mkdir build
    - singularity pull build/${BUILD_IMAGE}.sif docker://${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}

jug_dev:singularity:default:
  extends: .singularity
  needs:
    - version
    - jug_dev:default
  variables:
    BUILD_IMAGE: "jug_dev"

jug_xl:singularity:default:
  extends: .singularity
  needs:
    - version
    - jug_xl:default
  variables:
    BUILD_IMAGE: "jug_xl"

jug_xl:singularity:nightly:
  extends: .singularity
  rules:
    - !reference ['.singularity', rules]
    - if: '$CI_COMMIT_BRANCH == "master"'
      when: on_success
    - when: never
  needs:
    - version
    - jug_xl:nightly
  variables:
    BUILD_IMAGE: "jug_xl"
  script:
    - mkdir build
    - singularity pull build/${BUILD_IMAGE}.sif docker://${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${NIGHTLY_TAG}

benchmarks:reconstruction:
  stage: benchmarks
  rules:
    - if:  '$CI_SERVER_HOST == "eicweb.phy.anl.gov"'
  needs: 
    - version
    - jug_xl:default
  variables:
    BENCHMARKS_TAG: "$INTERNAL_TAG"
    BENCHMARKS_CONTAINER: "jug_xl"
    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
  trigger:
    project: EIC/benchmarks/reconstruction_benchmarks
    strategy: depend
  allow_failure: true
  
benchmarks:physics:
  stage: benchmarks
  rules:
    - if:  '$CI_SERVER_HOST == "eicweb.phy.anl.gov"'
  needs: 
    - version
    - jug_xl:default
  variables:
    RECO: "eicrecon"
    BENCHMARKS_TAG: "$INTERNAL_TAG"
    BENCHMARKS_CONTAINER: "jug_xl"
    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
  trigger:
    project: EIC/benchmarks/physics_benchmarks
    strategy: depend
  allow_failure: false

clean:
  stage: finalize
  dependencies:
    - version
  tags:
    - docker-new
  when: always
  script:
    ## remove the pipeline specific export from eicweb if needed
    - echo "Cleaning up pipeline specific docker tags if needed"
    - ./gitlab-ci/cleanup_registry.sh -i debian_testing_base -r 66 ${INTERNAL_TAG}
    - ./gitlab-ci/cleanup_registry.sh -i jug_dev -r 68 ${INTERNAL_TAG}
    - ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${INTERNAL_TAG}
    - |
      if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
        ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${NIGHTLY_TAG}
      fi

clean_unstable_mr:
  stage: finalize
  dependencies:
    - clean
  tags:
    - docker-new
  when: always
  variables:
    API: "https://eicweb.phy.anl.gov/api/v4"
  script:
    - apt update -q
    - apt install -yq jq
    - for registry in
        eicweb
        eicweb.phy.anl.gov:4567/eic/juggler
        eicweb.phy.anl.gov:4567/containers/eic_container
      ; do 
        docker images 
          --filter=reference=${registry}/*:unstable-mr*
          --format "{{.ID}} {{.Repository}} {{.Tag}}" ;
      done | while read id repository tag ; do
        if [[ ${tag} =~ ^unstable-mr-([0-9]+)-([0-9]+)(-nightly){0,1}$ ]] ; then
          project_id=${BASH_REMATCH[1]} ;
          merge_request_iid=${BASH_REMATCH[2]} ;
          if [[ $(curl --silent -L
                  "${API}/projects/${project_id}/merge_requests?scope=all&state=merged&per_page=100"
                  "${API}/projects/${project_id}/merge_requests?scope=all&state=closed&per_page=100"
                  | jq -r ".[]|select(.iid == ${merge_request_iid})|.state") =~ ^(merged|closed)$ ]] ; then
            docker rmi $repository:$tag ;
          else
            echo "Image $repository:$tag still in use (project ${project_id}, idd ${merge_request_iid})." ;
          fi ;
        fi ;
        if [[ ${tag} =~ ^unstable-mr-([0-9]+)(-nightly){0,1}$ ]] ; then
          merge_request_id=${BASH_REMATCH[1]} ;
          suffix=${BASH_REMATCH[2]} ;
          if [[ $(curl --silent -L
                  "${API}/projects/290/merge_requests?scope=all&state=merged&per_page=100"
                  "${API}/projects/369/merge_requests?scope=all&state=merged&per_page=100"
                  "${API}/projects/290/merge_requests?scope=all&state=closed&per_page=100"
                  "${API}/projects/369/merge_requests?scope=all&state=closed&per_page=100"
                  | jq -r ".[]|select(.id == ${merge_request_id})|.state") =~ ^(merged|closed)$ ]] ; then
            docker rmi $repository:$tag ;
          else
            project_id=$(curl --silent -L
                        "${API}/projects/290/merge_requests?scope=all&state=opened&per_page=100"
                        "${API}/projects/369/merge_requests?scope=all&state=opened&per_page=100"
                        | jq -r ".[]|select(.id == ${merge_request_id})|.project_id") ;
            merge_request_iid=$(curl --silent -L 
                        "${API}/projects/290/merge_requests?scope=all&state=opened&per_page=100"
                        "${API}/projects/369/merge_requests?scope=all&state=opened&per_page=100"
                        | jq -r ".[]|select(.id == ${merge_request_id})|.iid") ;
            echo "Image $repository:$tag still in use (project ${project_id}, idd ${merge_request_iid})." ;
            if [[ ${project_id} == "" || ${merge_request_iid} == "" ]] ; then
              echo "$tag not found" ;
            else
              docker tag $repository:$tag $repository:unstable-mr-${project_id}-${merge_request_iid}${suffix} ;
              docker rmi $repository:$tag ;
            fi ;
          fi ;
        fi ;
      done
    - docker system prune --force
    - docker images