image: alpine

variables:
  ## External images
  DOCKER_IMAGE: docker.io/docker:24.0.5
  SINGULARITY_IMAGE: quay.io/singularity/singularity:v3.11.4

  ## Application versions used for the main release 
  ## note: nightly builds will always use the master/main branch
  JUGGLER_VERSION: "v10.0.1"
  EICRECON_VERSION: "v1.4.1"

  ## Local registry
  CI_PUSH: 1
  ## Dockerhub registry
  DH_REGISTRY: docker.io
  DH_REGISTRY_USER: eicweb
  DH_PUSH: 1
  ## GitHub registry
  GH_REGISTRY: ghcr.io
  GH_REGISTRY_USER: eic
  GH_PUSH: 1

  ## TLS error resiliency: number of retries and second wait between tries 
  ## (wait time is doubled with each attempt)
  DOCKER_NTRIES: 5
  DOCKER_WAIT_TIME: 5

  ## Number of jobs to start during container builds
  JOBS: 128

  ## is this nightly or not?
  NIGHTLY: ""
  ## Add to tag
  NIGHTLY_TAG: "nightly"

  ## Additional options for docker build (e.g. --no-cache)
  BUILD_OPTIONS: ""

  ## Version to be set by external trigger
  VERSION: ""

  ## Internal tag used for the CI
  INTERNAL_TAG: "pipeline-${CI_PIPELINE_ID}"

stages:
  - config
  - base            ## base OS image
  - jug             ## jug container images
  - deploy          ## build/deploy singularity images
  - benchmarks
  - test
  - finalize

## only run CI for in the following cases:
## master, stable branch, release tag, MR event and nightly builds
## nightly builds are now part of the regular master build in order to keep
## all artifacts available at all times.
workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_PIPELINE_SOURCE == "web"'
    - if: '$CI_COMMIT_BRANCH == "master"'
    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-stable/'          ## main stable branch: vX.Y-stable
    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-[a-z]+-stable/'   ## special stable branch: vX.Y-acadia-stable (etc)
    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+/'            ## commit tags start with vX.Y.Z with optional suffix

## plan:
##  Workflows:
##  - master      --> config + all build stages + singularity
#                            + nightly jug + nightly singularity
##  - v4.0-stable --> config + all build stages + singularity
##  - v4.0.0      --> config + all build stages + singularity
##  - MR          --> config + all build stages
##
##  Container images tags
##  - master      --> testing
##  - <nightly>   --> nightly (run as part of master)
##  - v4.0-stable --> 4.0-stable
##  - v4.0.0      --> 4.0-stable, 4.0.0
##  - acadia     
##  - MR          --> 4.0-unstable (on all registries)
##                --> unstable-mr-XXX (on eicweb only, untag at end of pipeline)
##  - all other   --> do nothing
##
## caching strategy for dispatch to multiple nodes and to avoid
##  --> try this strategy: https://medium.com/titansoft-engineering/docker-build-cache-sharing-on-multi-hosts-with-buildkit-and-buildx-eb8f7005918e

## Default version and tags
## - internal tag used for the CI. Also temporarily tagged
##   on eicweb to communicate between jobs (removed in cleanup job)
## - export tag to public registries, optional secondary export tag
version:
  stage: config
  script:
    - |
      if [ -n "${VERSION}" ]; then
        # external trigger with specified version
        EXPORT_TAG="${VERSION}"
      elif [ "x${CI_PIPELINE_SOURCE}" = "xmerge_request_event" ]; then
        VERSION="unstable-mr-${CI_MERGE_REQUEST_PROJECT_ID}-${CI_MERGE_REQUEST_IID}"
        NIGHTLY_TAG="${VERSION}-nightly"
        EXPORT_TAG="${VERSION}"
        DH_PUSH=""
      elif [[ "$CI_COMMIT_TAG" =~ ^v[0-9.]+\.[0-9]+\.[0-9]+ ]]; then
        VERSION="${CI_COMMIT_TAG:1}"
        EXPORT_TAG="${VERSION}"
      elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+\.[0-9]+-stable ]]; then
        VERSION="${CI_COMMIT_BRANCH:1}"
        EXPORT_TAG="${VERSION}"
      elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+\.[0-9]+-[a-z]+-stable ]]; then
        VERSION="${CI_COMMIT_BRANCH:1}"
        EXPORT_TAG="${VERSION}"
      elif [[ "$CI_COMMIT_BRANCH" =~ ^master$ ]]; then
        VERSION="${CI_COMMIT_BRANCH}"
        EXPORT_TAG="${VERSION}"
      else
        VERSION="${CI_COMMIT_REF_NAME}-${CI_PIPELINE_ID}"
        EXPORT_TAG=""
      fi
      echo "INTERNAL_TAG=$INTERNAL_TAG" > build.env
      echo "VERSION=$VERSION" >> build.env
      echo "NIGHTLY_TAG=$NIGHTLY_TAG" >> build.env
      echo "EXPORT_TAG=$EXPORT_TAG" >> build.env
      echo "DH_PUSH=$DH_PUSH" >> build.env
      cat build.env
  artifacts:
    reports:
      dotenv: build.env

## base job settings for all docker interactions
.docker:
  image: ${DOCKER_IMAGE}
  services: 
    - docker:dind
  before_script:
    - # Login to registries
      if [[ -n "${EXPORT_TAG}" ]] ; then
        if [[ -n "${DH_REGISTRY}" ]] ; then
          docker login -u ${DH_REGISTRY_USER} -p ${DH_EICWEB_TOKEN} ${DH_REGISTRY} ;
        fi ;
        if [[ -n "${GH_REGISTRY}" ]] ; then
          docker login -u ${GITHUB_REGISTRY_USER} -p ${GITHUB_REGISTRY_TOKEN} ${GH_REGISTRY} ;
        fi ;
        if [[ -n "${CI_REGISTRY}" ]] ; then
          docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} ;
        fi ;
      fi
  after_script:
    - # Logout from registries
      if [[ -n "${EXPORT_TAG}" ]] ; then
        if [[ -n "${DH_REGISTRY}" ]] ; then
          docker logout ${DH_REGISTRY} ;
        fi ;
        if [[ -n "${GH_REGISTRY}" ]] ; then
          docker logout ${GH_REGISTRY} ;
        fi ;
        if [[ -n "${CI_REGISTRY}" ]] ; then
          docker logout ${CI_REGISTRY} ;
        fi ;
      fi

## base job settings for all docker build jobs
.build:
  extends: .docker 
  rules:
    - when: on_success
  resource_group: ${CI_COMMIT_REF_NAME}-${VERSION}
  tags:
    - docker-new
  before_script:
    - !reference [.docker, before_script]
    - mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc ;
      for arch in aarch64 ; do
        if ! grep -q enabled /proc/sys/fs/binfmt_misc/qemu-$arch ; then
          docker run --rm --privileged multiarch/qemu-user-static --persistent yes ;
        fi ;
      done
    - docker context create context
    - docker context use context
    - docker buildx create --name builder --driver docker-container --bootstrap --use context

## rules for nightly jobs
.nightly:
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
      when: on_success
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: on_success
    - when: never

## Images:
## debian_testing_base --> jug_dev  --> jug_xl
##             ---------------> jug_sim
##             ---------------> jug_ml
## debian_stable_base --> oneapi_jug_dev
## oneapi_jug_dev + jug_xl   --> oneapi_jug_xl

## TODO
## oneapi_runtime + jug_xl --> oneapi_prod 

base:
  parallel:
    matrix:
      - BASE_IMAGE: debian:testing-slim
        BUILD_IMAGE: debian_testing_base
        PLATFORM: linux/amd64,linux/arm64/v8
      - BASE_IMAGE: debian:stable-slim
        BUILD_IMAGE: debian_stable_base
        PLATFORM: linux/amd64,linux/arm64/v8
      - BASE_IMAGE: ubuntu:22.04
        BUILD_IMAGE: ubuntu_base
        PLATFORM: linux/amd64,linux/arm64/v8
#      - BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04
#        BUILD_IMAGE: oneapi_base
#        PLATFORM: linux/amd64
#      - BASE_IMAGE: nvidia/cuda:11.8.0-devel-ubuntu20.04
#        BUILD_IMAGE: cuda_base
#        PLATFORM: linux/amd64
  extends: .build
  stage: base
  needs:
    - version
  script:
    - docker buildx build --push ${BUILD_OPTIONS}
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
                   ${EXPORT_TAG:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
                   }
                   --file containers/debian/base.Dockerfile
                   --platform ${PLATFORM}
                   --build-arg BASE_IMAGE=${BASE_IMAGE}
                   --build-arg BUILD_IMAGE=${BUILD_IMAGE}
                   containers/debian

jug_dev:
  parallel:
    matrix:
      - BUILD_TYPE:
        - default
        - nightly
        BASE_IMAGE: debian_stable_base
        BUILD_IMAGE: jug_
        PLATFORM: linux/amd64,linux/arm64/v8
        ENV: [dev, prod]
#      - BUILD_TYPE:
#        - default
#        - nightly
#        BASE_IMAGE: oneapi_base
#        BUILD_IMAGE: oneapi_
#        PLATFORM: linux/amd64
#        ENV: [prod]
#      - BUILD_TYPE:
#        - default
#        - nightly
#        BASE_IMAGE: cuda_base_new
#        BUILD_IMAGE: cuda_
#        PLATFORM: linux/amd64
#        ENV: [prod]
  extends: .build
  stage: jug
  needs:
    - version
    - base
  script:
    - if [ "${BUILD_TYPE}" == "nightly" ] ; then IF_BUILD_NIGHTLY=1 ; fi
    - if [ "${BUILD_TYPE}" == "default" ] ; then IF_BUILD_DEFAULT=1 ; fi
    - apk add envsubst git
    - source spack.sh ; 
      source eic-spack.sh ;
      export SPACK_VERSION ;
      cat mirrors.yaml.in | envsubst > mirrors.yaml
    - docker buildx build --push ${BUILD_OPTIONS}
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE}
                   ${EXPORT_TAG:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
                   }
                   ${IF_BUILD_NIGHTLY:+
                     ${NIGHLTY:+
                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
                       ${PUSH_NIGHTLY_WITH_DATE:+
                         ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
                         ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
                       }
                     }
                   }
                   --file containers/jug/dev.Dockerfile
                   --platform ${PLATFORM}
                   --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
                   --build-arg BASE_IMAGE=${BASE_IMAGE}
                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
                   --build-arg SPACK_ORGREPO=${SPACK_ORGREPO}
                   --build-arg SPACK_VERSION=${SPACK_VERSION}
                   --build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
                   --build-arg EICSPACK_ORGREPO=${EICSPACK_ORGREPO}
                   --build-arg EICSPACK_VERSION=${EICSPACK_VERSION}
                   --build-arg EICSPACK_CHERRYPICKS="${EICSPACK_CHERRYPICKS}"
                   --build-arg S3_ACCESS_KEY=${S3_ACCESS_KEY}
                   --build-arg S3_SECRET_KEY=${S3_SECRET_KEY}
                   --build-arg JUG_VERSION=${EXPORT_TAG}-${BUILD_TYPE}-$(git rev-parse HEAD)
                   ${IF_BUILD_DEFAULT:+
                     --build-arg JUGGLER_VERSION=${JUGGLER_VERSION}
                     --build-arg EICRECON_VERSION=${EICRECON_VERSION}
                   }
                   --build-arg ENV=${ENV}
                   --build-arg jobs=${JOBS}
                   --build-context spack-environment=spack-environment
                   --secret id=mirrors,src=mirrors.yaml
                   containers/jug

jug_xl:default:
  parallel:
    matrix:
      - BASE_IMAGE: jug_dev
        BUILD_IMAGE: jug_xl
        PLATFORM: linux/amd64
#      - BASE_IMAGE: jug_dev_oneapi
#        BUILD_IMAGE: jug_xl_oneapi
#        PLATFORM: linux/amd64
  extends: .build
  stage: jug
  needs:
    - version
    - jug_dev
  script:
    - apk add git
    - docker buildx build --push ${BUILD_OPTIONS}
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}-default
                   ${EXPORT_TAG:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
                   }
                   --file containers/jug/xl.Dockerfile
                   --platform ${PLATFORM}
                   --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
                   --build-arg BASE_IMAGE=${BASE_IMAGE}
                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}-default
                   --build-arg JUG_VERSION=${EXPORT_TAG}-$(git rev-parse HEAD)
                   --build-arg jobs=${JOBS}
                   --build-context detectors=.
                   containers/jug

jug_xl:nightly:
  parallel:
    matrix:
      - BASE_IMAGE: jug_dev
        BUILD_IMAGE: jug_xl
        PLATFORM: linux/amd64
#      - BASE_IMAGE: jug_dev_oneapi
#        BUILD_IMAGE: jug_xl_oneapi
#        PLATFORM: linux/amd64
  extends: .build
  stage: jug
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
      when: on_success
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: on_success
    - when: never
  needs:
    - version
    - jug_dev
  script:
    - |
      PUSH_NIGHTLY_WITH_DATE=""
      if [ "$CI_COMMIT_BRANCH" == "master" ]; then
        PUSH_NIGHTLY_WITH_DATE="1"
      fi
    - apk add git
    - docker buildx build --push ${BUILD_OPTIONS}
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}-nightly
                   ${EXPORT_TAG:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}-nightly}
                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}-nightly}
                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}-nightly}
                   }
                   ${NIGHTLY:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
                     ${PUSH_NIGHTLY_WITH_DATE:+
                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
                     }
                   }
                   --file containers/jug/xl.Dockerfile
                   --platform ${PLATFORM}
                   --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
                   --build-arg BASE_IMAGE=${BASE_IMAGE}
                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}-nightly
                   --build-arg JUG_VERSION=${EXPORT_TAG}-nightly-$(git rev-parse HEAD)-$(date +%Y-%m-%d_%H-%M-%S)
                   --build-arg NIGHTLY=1
                   --build-context detectors=.
                   containers/jug


.singularity:
  image:
    name: ${SINGULARITY_IMAGE}
    entrypoint: [""]
  stage: deploy
  interruptible: true
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
      when: on_success
    - when: never
  artifacts:
      expire_in: 1 days
      paths:
        - build/${BUILD_IMAGE}.sif
  script:
    - mkdir build
    - singularity pull build/${BUILD_IMAGE}.sif docker://${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}-${BUILD_TYPE}

jug_xl:singularity:default:
  extends: .singularity
  variables:
    BUILD_TYPE: default
    BUILD_IMAGE: jug_xl
  needs:
    - version
    - jug_xl:default

# eic-shell expects build/jug_xl.sif from the job with name 'jug_xl:singularity:nightly'
jug_xl:singularity:nightly:
  extends: .singularity
  variables:
    BUILD_TYPE: nightly
    BUILD_IMAGE: jug_xl
  needs:
    - version
    - jug_xl:nightly

benchmarks:reconstruction:default:
  stage: benchmarks
  needs: 
    - version
    - jug_xl:default
  variables:
    BENCHMARKS_TAG: "${INTERNAL_TAG}-default"
    BENCHMARKS_CONTAINER: "jug_xl"
    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
  trigger:
    project: EIC/benchmarks/reconstruction_benchmarks
    strategy: depend
  allow_failure: true
  
benchmarks:physics:default:
  stage: benchmarks
  needs: 
    - version
    - jug_xl:default
  variables:
    RECO: "eicrecon"
    BENCHMARKS_TAG: "${INTERNAL_TAG}-default"
    BENCHMARKS_CONTAINER: "jug_xl"
    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
  trigger:
    project: EIC/benchmarks/physics_benchmarks
    strategy: depend
  allow_failure: false

benchmarks:reconstruction:nightly:
  stage: benchmarks
  rules:
    - !reference ['.nightly', rules]
  needs: 
    - version
    - jug_xl:nightly
  variables:
    BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly"
    BENCHMARKS_CONTAINER: "jug_xl"
    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
  trigger:
    project: EIC/benchmarks/reconstruction_benchmarks
    strategy: depend
  allow_failure: true
  
benchmarks:physics:nightly:
  stage: benchmarks
  rules:
    - !reference ['.nightly', rules]
  needs: 
    - version
    - jug_xl:nightly
  variables:
    RECO: "eicrecon"
    BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly"
    BENCHMARKS_CONTAINER: "jug_xl"
    BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
  trigger:
    project: EIC/benchmarks/physics_benchmarks
    strategy: depend
  allow_failure: false

clean_internal_tag:
  image: alpine/curl
  stage: finalize
  dependencies:
    - version
  when: always
  script:
    - |
      for id in 66 68 69 80 83 91 92 ; do
        for tag in ${INTERNAL_TAG} ${INTERNAL_TAG}-default ${INTERNAL_TAG}-nightly ; do 
          curl --request DELETE --header "PRIVATE-TOKEN: ${REG_CLEANUP_TOKEN}" ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/registry/repositories/${id}/tags/${tag} ;
        done ;
      done

clean_unstable_mr:
  extends: .docker
  stage: finalize
  dependencies:
    - version
  tags:
    - docker-new
  when: always
  script:
    - apk add curl jq
    - for registry in
        eicweb
        eicweb.phy.anl.gov:4567/containers/eic_container
      ; do 
        docker images 
          --filter=reference=${registry}/*:unstable-mr*
          --format "{{.ID}} {{.Repository}} {{.Tag}}" ;
      done
      | sed -n -E 's/unstable-mr-([0-9]+)-([0-9]+)(-nightly|-default){0,1}$/& \1 \2/p'
      | while read id repository tag project_id merge_request_iid ; do
        state=$(curl --silent -L
                "${CI_API_V4_URL}/projects/${project_id}/merge_requests?scope=all&state=opened&per_page=100"
                "${CI_API_V4_URL}/projects/${project_id}/merge_requests?scope=all&state=merged&per_page=100"
                "${CI_API_V4_URL}/projects/${project_id}/merge_requests?scope=all&state=closed&per_page=100"
                | jq -r ".[]|select(.iid == ${merge_request_iid})|.state") ;
        echo "Image $repository:$tag is ${state:-undetermined} (project ${project_id}, iid ${merge_request_iid})." ;
        if [ "$state" == "merged" ] ; then docker rmi $repository:$tag ; fi ;
        if [ "$state" == "closed" ] ; then docker rmi $repository:$tag ; fi ;
      done
    - docker system prune --filter until=24h --force
    - docker system df
    - docker images

clean_pipeline:
  extends: .docker
  stage: finalize
  dependencies:
    - version
    - clean_internal_tag
  tags:
    - docker-new
  when: always
  script:
    - apk add curl jq
    - for registry in
        eicweb
        eicweb.phy.anl.gov:4567/containers/eic_container
      ; do 
        docker images 
          --filter=reference=${registry}/*:pipeline-*
          --format "{{.ID}} {{.Repository}} {{.Tag}}" ;
      done
      | sed -n -E 's/pipeline-([0-9]+)(-.*){0,1}$/& \1/p'
      | while read id repository tag pipeline_id ; do
        status=$(curl --silent -L
                "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${pipeline_id}"
                | jq -r ".status") ;
        echo "Image $repository:$tag is ${status:-undetermined} (pipeline ${pipeline_id})." ;
        if [ "$status" == "success" ] ; then docker rmi $repository:$tag ; fi ;
        if [ "$status" == "failed" ] ; then docker rmi $repository:$tag ; fi ;
        if [ "$status" == "canceled" ] ; then docker rmi $repository:$tag ; fi ;
      done
    - docker system prune --filter until=24h --force
    - docker system df
    - docker images