.gitlab-ci.yml

image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest

variables:
  ## Application versions used for the main release 
  ## note: nightly builds will always use the master/main branch
  JUGGLER_VERSION: "v10.0.1"
  EICRECON_VERSION: "v1.4.1"

  ## Local registry
  CI_PUSH: 1
  ## Dockerhub registry
  DH_REGISTRY: docker.io
  DH_REGISTRY_USER: eicweb
  DH_PUSH: 1
  ## GitHub registry
  GH_REGISTRY: ghcr.io
  GH_REGISTRY_USER: eic
  GH_PUSH: 1

  ## TLS error resiliency: number of retries and second wait between tries 
  ## (wait time is doubled with each attempt)
  DOCKER_NTRIES: 5
  DOCKER_WAIT_TIME: 5

  ## Number of jobs to start during container builds
  JOBS: 128

  ## is this nightly or not?
  NIGHTLY:
  ## Add to tag
  NIGHTLY_TAG: "nightly"

  ## Additional options for docker build (e.g. --no-cache)
  BUILD_OPTIONS:

  ## Version to be set by external trigger
  VERSION:

  ## Internal tag used for the CI
  INTERNAL_TAG: "pipeline-${CI_PIPELINE_ID}"

stages:
  - config
  - dind
  - base            ## base OS image
  - jug             ## jug container images
  - deploy          ## build/deploy singularity images
  - benchmarks
  - test
  - finalize

default:
  before_script:
    - ./gitlab-ci/docker_login.sh -u ${DH_REGISTRY_USER} -p ${DH_EICWEB_TOKEN}
                                  -n ${DOCKER_NTRIES} -t ${DOCKER_WAIT_TIME}
    - ./gitlab-ci/docker_login.sh -u ${GITHUB_REGISTRY_USER} -p ${GITHUB_REGISTRY_TOKEN}
                                  -n ${DOCKER_NTRIES} -t ${DOCKER_WAIT_TIME}
                                  ${GH_REGISTRY}
    - ./gitlab-ci/docker_login.sh --ci -n ${DOCKER_NTRIES} -t ${DOCKER_WAIT_TIME}
  after_script:
    - docker logout
    - docker logout ${CI_REGISTRY}
    - docker logout ${GH_REGISTRY}

## only run CI for in the following cases:
## master, stable branch, release tag, MR event and nightly builds
## nightly builds are now part of the regular master build in order to keep
## all artifacts available at all times.
workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_PIPELINE_SOURCE == "web"'
    - if: '$CI_COMMIT_BRANCH == "master"'
    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-stable/'          ## main stable branch: vX.Y-stable
    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-[a-z]+-stable/'   ## special stable branch: vX.Y-acadia-stable (etc)
    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+/'            ## commit tags start with vX.Y.Z with optional suffix

## plan:
##  Workflows:
##  - master      --> config + all build stages + singularity
#                            + nightly jug + nightly singularity
##  - v4.0-stable --> config + all build stages + singularity
##  - v4.0.0      --> config + all build stages + singularity
##  - MR          --> config + all build stages
##
##  Container images tags
##  - master      --> testing
##  - <nightly>   --> nightly (run as part of master)
##  - v4.0-stable --> 4.0-stable
##  - v4.0.0      --> 4.0-stable, 4.0.0
##  - acadia     
##  - MR          --> 4.0-unstable (on all registries)
##                --> unstable-mr-XXX (on eicweb only, untag at end of pipeline)
##  - all other   --> do nothing
##
## caching strategy for dispatch to multiple nodes and to avoid
##  --> try this strategy: https://medium.com/titansoft-engineering/docker-build-cache-sharing-on-multi-hosts-with-buildkit-and-buildx-eb8f7005918e

## Default version and tags
## - internal tag used for the CI. Also temporarily tagged
##   on eicweb to communicate between jobs (removed in cleanup job)
## - export tag to public registries, optional secondary export tag
version:
  stage: config
  script:
    - |
      if [ -n "${VERSION}" ]; then
        # external trigger with specified version
        EXPORT_TAG="${VERSION}"
      elif [ "x${CI_PIPELINE_SOURCE}" = "xmerge_request_event" ]; then
        VERSION="unstable-mr-${CI_MERGE_REQUEST_PROJECT_ID}-${CI_MERGE_REQUEST_IID}"
        NIGHTLY_TAG="${VERSION}-nightly"
        EXPORT_TAG="${VERSION}"
      elif [[ "$CI_COMMIT_TAG" =~ ^v[0-9.]+\.[0-9]+\.[0-9]+ ]]; then
        VERSION="${CI_COMMIT_TAG:1}"
        EXPORT_TAG="${VERSION}"
      elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+\.[0-9]+-stable ]]; then
        VERSION="${CI_COMMIT_BRANCH:1}"
        EXPORT_TAG="${VERSION}"
      elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+\.[0-9]+-[a-z]+-stable ]]; then
        VERSION="${CI_COMMIT_BRANCH:1}"
        EXPORT_TAG="${VERSION}"
      elif [[ "$CI_COMMIT_BRANCH" =~ ^master$ ]]; then
        VERSION="${CI_COMMIT_BRANCH}"
        EXPORT_TAG="${VERSION}"
      else
        VERSION="${CI_COMMIT_REF_NAME}-${CI_PIPELINE_ID}"
        EXPORT_TAG=""
      fi
      echo "INTERNAL_TAG=$INTERNAL_TAG" >> build.env
      echo "VERSION=$VERSION" >> build.env
      echo "NIGHTLY_TAG=$NIGHTLY_TAG" >> build.env
      echo "EXPORT_TAG=$EXPORT_TAG" >> build.env
      cat build.env
  artifacts:
    reports:
      dotenv: build.env

## base job settings for all build jobs
.build:
  rules:
    - when: on_success
  resource_group: ${CI_COMMIT_REF_NAME}-${VERSION}
  tags:
    - docker-new
  before_script:
    - !reference [default, before_script]
    - mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc ;
      for arch in aarch64 ; do
        if ! grep --quiet enabled /proc/sys/fs/binfmt_misc/qemu-$arch ; then
          docker run --rm --privileged multiarch/qemu-user-static --persistent yes ;
        fi ;
      done
    - docker buildx create --name builder --driver docker-container --bootstrap --use

## rules for nightly jobs
.nightly:
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
      when: on_success
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: on_success
    - when: never

## Images:
## debian_testing_base --> jug_dev  --> jug_xl
##             ---------------> jug_sim
##             ---------------> jug_ml
## debian_stable_base --> oneapi_jug_dev
## oneapi_jug_dev + jug_xl   --> oneapi_jug_xl

## TODO
## oneapi_runtime + jug_xl --> oneapi_prod 

## Docker-in-docker image
dind-build:
  parallel:
    matrix:
      - BUILD_IMAGE: dind
  extends: .build
  stage: dind
  needs:
    - version
  script:
    - docker buildx build --push
        --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}     
        --file containers/dind/Dockerfile
        containers/dind

dind-publish:
  image: ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
  parallel:
    matrix:
      - BUILD_IMAGE: dind
  extends: .build
  stage: dind
  needs:
    - version
    - dind-build
  script:
    - docker buildx build --push
        --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
        ${EXPORT_TAG:+
          ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
          ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
          ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
        }        
        --file containers/dind/Dockerfile
        containers/dind

base:
  parallel:
    matrix:
      - BASE_IMAGE: debian:testing-slim
        BUILD_IMAGE: debian_testing_base
        PLATFORM: linux/amd64,linux/arm64/v8
      - BASE_IMAGE: debian:stable-slim
        BUILD_IMAGE: debian_stable_base
        PLATFORM: linux/amd64,linux/arm64/v8
      - BASE_IMAGE: ubuntu:22.04
        BUILD_IMAGE: ubuntu_base
        PLATFORM: linux/amd64,linux/arm64/v8
#      - BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04
#        BUILD_IMAGE: oneapi_base
#        PLATFORM: linux/amd64
#      - BASE_IMAGE: nvidia/cuda:11.8.0-devel-ubuntu20.04
#        BUILD_IMAGE: cuda_base
#        PLATFORM: linux/amd64
  extends: .build
  stage: base
  needs:
    - version
  script:
    - docker buildx build --push ${BUILD_OPTIONS}
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
                   ${EXPORT_TAG:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
                   }
                   --file containers/debian/base.Dockerfile
                   --platform ${PLATFORM}
                   --build-arg BASE_IMAGE=${BASE_IMAGE}
                   --build-arg BUILD_IMAGE=${BUILD_IMAGE}
                   containers/debian

jug_dev:
  parallel:
    matrix:
      - BUILD_TYPE:
        - default
        - nightly
        BASE_IMAGE: debian_stable_base
        BUILD_IMAGE: jug_
        PLATFORM: linux/amd64,linux/arm64/v8
        ENV: [dev, prod]
#      - BUILD_TYPE:
#        - default
#        - nightly
#        BASE_IMAGE: oneapi_base
#        BUILD_IMAGE: oneapi_
#        PLATFORM: linux/amd64
#        ENV: [prod]
#      - BUILD_TYPE:
#        - default
#        - nightly
#        BASE_IMAGE: cuda_base_new
#        BUILD_IMAGE: cuda_
#        PLATFORM: linux/amd64
#        ENV: [prod]
  extends: .build
  stage: jug
  needs:
    - version
    - base
  script:
    - if [ "${BUILD_TYPE}" == "nightly" ] ; then IF_BUILD_NIGHTLY=1 ; fi
    - if [ "${BUILD_TYPE}" == "default" ] ; then IF_BUILD_DEFAULT=1 ; fi
    - source spack.sh ; 
      source eic-spack.sh ;
      export SPACK_VERSION ;
      cat mirrors.yaml.in | envsubst > mirrors.yaml ;
      docker buildx build --push ${BUILD_OPTIONS}
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE}
                   ${EXPORT_TAG:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
                   }
                   ${IF_BUILD_NIGHTLY:+
                     ${NIGHLTY:+
                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
                       ${PUSH_NIGHTLY_WITH_DATE:+
                         ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
                         ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
                       }
                     }
                   }
                   --file containers/jug/dev.Dockerfile
                   --platform ${PLATFORM}
                   --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
                   --build-arg BASE_IMAGE=${BASE_IMAGE}
                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
                   --build-arg SPACK_ORGREPO=${SPACK_ORGREPO}
                   --build-arg SPACK_VERSION=${SPACK_VERSION}
                   --build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
                   --build-arg EICSPACK_ORGREPO=${EICSPACK_ORGREPO}
                   --build-arg EICSPACK_VERSION=${EICSPACK_VERSION}
                   --build-arg EICSPACK_CHERRYPICKS="${EICSPACK_CHERRYPICKS}"
                   --build-arg S3_ACCESS_KEY=${S3_ACCESS_KEY}
                   --build-arg S3_SECRET_KEY=${S3_SECRET_KEY}
                   --build-arg JUG_VERSION=${EXPORT_TAG}-${BUILD_TYPE}-$(git rev-parse HEAD)-$(date +%Y-%m-%d_%H-%M-%S)
                   --build-arg ENV=${ENV}
                   --build-arg jobs=${JOBS}
                   --build-context spack-environment=spack-environment
                   --secret id=mirrors,src=mirrors.yaml
                   containers/jug

jug_xl:default:
  parallel:
    matrix:
      - BASE_IMAGE: jug_dev
        BUILD_IMAGE: jug_xl
        PLATFORM: linux/amd64
#      - BASE_IMAGE: jug_dev_oneapi
#        BUILD_IMAGE: jug_xl_oneapi
#        PLATFORM: linux/amd64
  extends: .build
  stage: jug
  needs:
    - version
    - jug_dev
  script:
    - docker buildx build --push ${BUILD_OPTIONS}
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}-default
                   ${EXPORT_TAG:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
                   }
                   --file containers/jug/xl.Dockerfile
                   --platform ${PLATFORM}
                   --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
                   --build-arg BASE_IMAGE=${BASE_IMAGE}
                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}-default
                   --build-arg JUGGLER_VERSION=${JUGGLER_VERSION}
                   --build-arg EICRECON_VERSION=${EICRECON_VERSION}
                   --build-arg JUG_VERSION=${EXPORT_TAG}-$(git rev-parse HEAD)-$(date +%Y-%m-%d_%H-%M-%S)
                   --build-arg jobs=${JOBS}
                   --build-context detectors=.
                   containers/jug

jug_xl:nightly:
  parallel:
    matrix:
      - BASE_IMAGE: jug_dev
        BUILD_IMAGE: jug_xl
        PLATFORM: linux/amd64
#      - BASE_IMAGE: jug_dev_oneapi
#        BUILD_IMAGE: jug_xl_oneapi
#        PLATFORM: linux/amd64
  extends: .build
  stage: jug
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
      when: on_success
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: on_success
    - when: never
  needs:
    - version
    - jug_dev
  script:
    - |
      PUSH_NIGHTLY_WITH_DATE=""
      if [ "$CI_COMMIT_BRANCH" == "master" ]; then
        PUSH_NIGHTLY_WITH_DATE="1"
      fi
    - docker buildx build --push ${BUILD_OPTIONS}
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}-nightly
                   ${EXPORT_TAG:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}-nightly}
                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}-nightly}
                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}-nightly}
                   }
                   ${NIGHTLY:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
                     ${PUSH_NIGHTLY_WITH_DATE:+
                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
                     }
                   }
                   --file containers/jug/xl.Dockerfile
                   --platform ${PLATFORM}
                   --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
                   --build-arg BASE_IMAGE=${BASE_IMAGE}
                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}-nightly
                   --build-arg JUG_VERSION=${EXPORT_TAG}-nightly-$(git rev-parse HEAD)-$(date +%Y-%m-%d_%H-%M-%S)
                   --build-arg NIGHTLY=1
                   --build-context detectors=.
                   containers/jug

.singularity:
  stage: deploy
  interruptible: true
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: never
  artifacts:
      expire_in: 1 days
      paths:
        - build/${BUILD_IMAGE}.sif
  script:
    - mkdir build
    - singularity pull build/${BUILD_IMAGE}.sif docker://${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}

jug_dev:singularity:default:
  extends: .singularity
  needs:
    - version
    - jug_dev
  variables:
    BUILD_IMAGE: "jug_dev"

jug_xl:singularity:default:
  extends: .singularity
  needs:
    - version
    - jug_xl:default
  variables:
    BUILD_IMAGE: "jug_xl"

jug_xl:singularity:nightly:
  extends: .singularity
  rules:
    - !reference ['.singularity', rules]
    - if: '$CI_COMMIT_BRANCH == "master"'
      when: on_success
    - when: never
  needs:
    - version
    - jug_xl:nightly
  variables:
    BUILD_IMAGE: "jug_xl"
  script:
    - mkdir build
    - singularity pull build/${BUILD_IMAGE}.sif docker://${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${NIGHTLY_TAG}

benchmarks:reconstruction:default:
  stage: benchmarks
  needs: 
    - version
    - jug_xl:default
  variables:
    BENCHMARKS_TAG: "${INTERNAL_TAG}-default"
    BENCHMARKS_CONTAINER: "jug_xl"
    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
  trigger:
    project: EIC/benchmarks/reconstruction_benchmarks
    strategy: depend
  allow_failure: true
  
benchmarks:physics:default:
  stage: benchmarks
  needs: 
    - version
    - jug_xl:default
  variables:
    RECO: "eicrecon"
    BENCHMARKS_TAG: "${INTERNAL_TAG}-default"
    BENCHMARKS_CONTAINER: "jug_xl"
    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
  trigger:
    project: EIC/benchmarks/physics_benchmarks
    strategy: depend
  allow_failure: false

benchmarks:reconstruction:nightly:
  stage: benchmarks
  rules:
    - !reference ['.nightly', rules]
  needs: 
    - version
    - jug_xl:nightly
  variables:
    BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly"
    BENCHMARKS_CONTAINER: "jug_xl"
    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
  trigger:
    project: EIC/benchmarks/reconstruction_benchmarks
    strategy: depend
  allow_failure: true
  
benchmarks:physics:nightly:
  stage: benchmarks
  rules:
    - !reference ['.nightly', rules]
  needs: 
    - version
    - jug_xl:nightly
  variables:
    RECO: "eicrecon"
    BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly"
    BENCHMARKS_CONTAINER: "jug_xl"
    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
  trigger:
    project: EIC/benchmarks/physics_benchmarks
    strategy: depend
  allow_failure: false

clean:
  stage: finalize
  dependencies:
    - version
  tags:
    - docker-new
  when: always
  script:
    ## remove the pipeline specific export from eicweb if needed
    - echo "Cleaning up pipeline specific docker tags if needed"
    - ./gitlab-ci/cleanup_registry.sh -i debian_base -r 66 ${INTERNAL_TAG}
    - ./gitlab-ci/cleanup_registry.sh -i ubuntu_base -r 80 ${INTERNAL_TAG}
    - ./gitlab-ci/cleanup_registry.sh -i debian_testing_base -r 83 ${INTERNAL_TAG}
    - ./gitlab-ci/cleanup_registry.sh -i debian_stable_base -r 92 ${INTERNAL_TAG}
    - ./gitlab-ci/cleanup_registry.sh -i jug_prod -r 91 ${INTERNAL_TAG}
    - ./gitlab-ci/cleanup_registry.sh -i jug_dev -r 68 ${INTERNAL_TAG}
    - ./gitlab-ci/cleanup_registry.sh -i jug_dev -r 68 ${INTERNAL_TAG}-nightly
    - ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${INTERNAL_TAG}
    - ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${INTERNAL_TAG}-nightly
    - |
      if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
        ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${NIGHTLY_TAG}
      fi
    - ./gitlab-ci/cleanup_registry.sh -i dind -r 93 ${INTERNAL_TAG}

clean_unstable_mr:
  stage: finalize
  dependencies:
    - clean
  tags:
    - docker-new
  when: always
  variables:
    API: "https://eicweb.phy.anl.gov/api/v4"
  script:
    - apt update -q
    - apt install -yq jq
    - for registry in
        eicweb
        eicweb.phy.anl.gov:4567/eic/juggler
        eicweb.phy.anl.gov:4567/containers/eic_container
      ; do 
        docker images 
          --filter=reference=${registry}/*:unstable-mr*
          --format "{{.ID}} {{.Repository}} {{.Tag}}" ;
      done | while read id repository tag ; do
        if [[ ${tag} =~ ^unstable-mr-([0-9]+)-([0-9]+)(-nightly){0,1}$ ]] ; then
          project_id=${BASH_REMATCH[1]} ;
          merge_request_iid=${BASH_REMATCH[2]} ;
          if [[ $(curl --silent -L
                  "${API}/projects/${project_id}/merge_requests?scope=all&state=merged&per_page=100"
                  "${API}/projects/${project_id}/merge_requests?scope=all&state=closed&per_page=100"
                  | jq -r ".[]|select(.iid == ${merge_request_iid})|.state") =~ ^(merged|closed)$ ]] ; then
            docker rmi $repository:$tag ;
          else
            echo "Image $repository:$tag still in use (project ${project_id}, idd ${merge_request_iid})." ;
          fi ;
        fi ;
        if [[ ${tag} =~ ^unstable-mr-([0-9]+)(-nightly){0,1}$ ]] ; then
          merge_request_id=${BASH_REMATCH[1]} ;
          suffix=${BASH_REMATCH[2]} ;
          if [[ $(curl --silent -L
                  "${API}/projects/290/merge_requests?scope=all&state=merged&per_page=100"
                  "${API}/projects/369/merge_requests?scope=all&state=merged&per_page=100"
                  "${API}/projects/290/merge_requests?scope=all&state=closed&per_page=100"
                  "${API}/projects/369/merge_requests?scope=all&state=closed&per_page=100"
                  | jq -r ".[]|select(.id == ${merge_request_id})|.state") =~ ^(merged|closed)$ ]] ; then
            docker rmi $repository:$tag ;
          else
            project_id=$(curl --silent -L
                        "${API}/projects/290/merge_requests?scope=all&state=opened&per_page=100"
                        "${API}/projects/369/merge_requests?scope=all&state=opened&per_page=100"
                        | jq -r ".[]|select(.id == ${merge_request_id})|.project_id") ;
            merge_request_iid=$(curl --silent -L 
                        "${API}/projects/290/merge_requests?scope=all&state=opened&per_page=100"
                        "${API}/projects/369/merge_requests?scope=all&state=opened&per_page=100"
                        | jq -r ".[]|select(.id == ${merge_request_id})|.iid") ;
            echo "Image $repository:$tag still in use (project ${project_id}, idd ${merge_request_iid})." ;
            if [[ ${project_id} == "" || ${merge_request_iid} == "" ]] ; then
              echo "$tag not found" ;
            else
              docker tag $repository:$tag $repository:unstable-mr-${project_id}-${merge_request_iid}${suffix} ;
              docker rmi $repository:$tag ;
            fi ;
          fi ;
        fi ;
      done
    - docker system prune --filter until=24h --force
    - docker images