Newer
Older

Sylvester Joosten
committed
SINGULARITY_IMAGE: quay.io/singularity/singularity:v3.11.4
## Application versions used for the main release
## note: nightly builds will always use the master/main branch
Wouter Deconinck
committed
JUGGLER_VERSION: "v10.1.0"
## Dockerhub registry
DH_REGISTRY_USER: eicweb
## GitHub registry
GH_REGISTRY: ghcr.io
## TLS error resiliency: number of retries and second wait between tries
## (wait time is doubled with each attempt)
DOCKER_NTRIES: 5
DOCKER_WAIT_TIME: 5
## Number of jobs to start during container builds
## is this nightly or not?
## Add to tag
NIGHTLY_TAG: "nightly"
## Additional options for docker build (e.g. --no-cache)
## Version to be set by external trigger
INTERNAL_TAG: "pipeline-${CI_PIPELINE_ID}"
- config
- base ## base OS image
- jug ## jug container images
- deploy ## build/deploy singularity images
## only run CI for in the following cases:
## master, stable branch, release tag, MR event and nightly builds
## nightly builds are now part of the regular master build in order to keep
## all artifacts available at all times.

Sylvester Joosten
committed
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_PIPELINE_SOURCE == "web"'
- if: '$CI_COMMIT_BRANCH == "master"'
- if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-stable/' ## main stable branch: vX.Y-stable
- if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-[a-z]+-stable/' ## special stable branch: vX.Y-acadia-stable (etc)
- if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+/' ## commit tags start with vX.Y.Z with optional suffix
## plan:
## Workflows:
## - master --> config + all build stages + singularity
# + nightly jug + nightly singularity
## - v4.0-stable --> config + all build stages + singularity
## - v4.0.0 --> config + all build stages + singularity
## - MR --> config + all build stages
##
## Container images tags
## - master --> testing
## - <nightly> --> nightly (run as part of master)
## - v4.0-stable --> 4.0-stable
## - v4.0.0 --> 4.0-stable, 4.0.0
## --> unstable-mr-XXX (on eicweb only, untag at end of pipeline)
## - all other --> do nothing
##
## caching strategy for dispatch to multiple nodes and to avoid
## --> try this strategy: https://medium.com/titansoft-engineering/docker-build-cache-sharing-on-multi-hosts-with-buildkit-and-buildx-eb8f7005918e
## Default version and tags
## - internal tag used for the CI. Also temporarily tagged
## on eicweb to communicate between jobs (removed in cleanup job)
## - export tag to public registries, optional secondary export tag
version:
stage: config

Sylvester Joosten
committed
script:
if [ -n "${VERSION}" ]; then
# external trigger with specified version
EXPORT_TAG="${VERSION}"
elif [ "x${CI_PIPELINE_SOURCE}" = "xmerge_request_event" ]; then
Wouter Deconinck
committed
VERSION="unstable-mr-${CI_MERGE_REQUEST_PROJECT_ID}-${CI_MERGE_REQUEST_IID}"
NIGHTLY_TAG="${VERSION}-nightly"
EXPORT_TAG="${VERSION}"
elif [[ "$CI_COMMIT_TAG" =~ ^v[0-9.]+\.[0-9]+\.[0-9]+ ]]; then
Wouter Deconinck
committed
EXPORT_TAG="${VERSION}"
elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+\.[0-9]+-stable ]]; then
Wouter Deconinck
committed
EXPORT_TAG="${VERSION}"
elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+\.[0-9]+-[a-z]+-stable ]]; then
Wouter Deconinck
committed
EXPORT_TAG="${VERSION}"
elif [[ "$CI_COMMIT_BRANCH" =~ ^master$ ]]; then
VERSION="${CI_COMMIT_BRANCH}"
EXPORT_TAG="${VERSION}"
else
VERSION="${CI_COMMIT_REF_NAME}-${CI_PIPELINE_ID}"
EXPORT_TAG=""
echo "INTERNAL_TAG=$INTERNAL_TAG" > build.env
echo "VERSION=$VERSION" >> build.env
echo "NIGHTLY_TAG=$NIGHTLY_TAG" >> build.env
echo "EXPORT_TAG=$EXPORT_TAG" >> build.env
echo "DH_PUSH=$DH_PUSH" >> build.env
cat build.env
artifacts:
reports:
dotenv: build.env
## base job settings for all docker interactions
.docker:
image: ${DOCKER_IMAGE}
before_script:
- # Login to registries
if [[ -n "${EXPORT_TAG}" ]] ; then
if [[ -n "${DH_REGISTRY}" ]] ; then
docker login -u ${DH_REGISTRY_USER} -p ${DH_EICWEB_TOKEN} ${DH_REGISTRY} ;
fi ;
if [[ -n "${GH_REGISTRY}" ]] ; then
docker login -u ${GITHUB_REGISTRY_USER} -p ${GITHUB_REGISTRY_TOKEN} ${GH_REGISTRY} ;
fi ;
if [[ -n "${CI_REGISTRY}" ]] ; then
docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} ;
fi ;
fi
after_script:
- # Logout from registries
if [[ -n "${EXPORT_TAG}" ]] ; then
if [[ -n "${DH_REGISTRY}" ]] ; then
docker logout ${DH_REGISTRY} ;
fi ;
if [[ -n "${GH_REGISTRY}" ]] ; then
docker logout ${GH_REGISTRY} ;
fi ;
if [[ -n "${CI_REGISTRY}" ]] ; then
docker logout ${CI_REGISTRY} ;
fi ;
fi
## base job settings for all docker build jobs

Sylvester Joosten
committed
rules:
- when: on_success
- !reference [.docker, before_script]
- mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc ;
for arch in aarch64 ; do
if ! grep -q enabled /proc/sys/fs/binfmt_misc/qemu-$arch ; then
docker run --rm --privileged multiarch/qemu-user-static --persistent yes ;
fi ;
done
- docker context create context
- docker context use context
- docker buildx create --name builder --driver docker-container --bootstrap --use context

Sylvester Joosten
committed
## rules for nightly jobs
.nightly:
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
when: on_success
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: on_success
- when: never
## debian_testing_base --> jug_dev --> jug_xl
## ---------------> jug_sim
## ---------------> jug_ml
## debian_stable_base --> oneapi_jug_dev
## oneapi_jug_dev + jug_xl --> oneapi_jug_xl
## TODO
## oneapi_runtime + jug_xl --> oneapi_prod
base:
parallel:
matrix:
PLATFORM: linux/amd64,linux/arm64/v8
- BASE_IMAGE: debian:stable-slim
BUILD_IMAGE: debian_stable_base
PLATFORM: linux/amd64,linux/arm64/v8
- BASE_IMAGE: ubuntu:22.04
PLATFORM: linux/amd64,linux/arm64/v8
# - BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04
# BUILD_IMAGE: oneapi_base
# PLATFORM: linux/amd64
# - BASE_IMAGE: nvidia/cuda:11.8.0-devel-ubuntu20.04
# BUILD_IMAGE: cuda_base
# PLATFORM: linux/amd64
needs:
- version
- docker buildx build --push ${BUILD_OPTIONS}
--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
--file containers/debian/base.Dockerfile
--build-arg BASE_IMAGE=${BASE_IMAGE}
--build-arg BUILD_IMAGE=${BUILD_IMAGE}
- BUILD_TYPE:
- default
- nightly
BASE_IMAGE: debian_stable_base
PLATFORM: linux/amd64,linux/arm64/v8
# - BUILD_TYPE:
# - default
# - nightly
# BASE_IMAGE: oneapi_base
# BUILD_IMAGE: oneapi_
# PLATFORM: linux/amd64
# - BUILD_TYPE:
# - default
# - nightly
# BASE_IMAGE: cuda_base_new
# BUILD_IMAGE: cuda_
# PLATFORM: linux/amd64
needs:
- version
- if [ "${BUILD_TYPE}" == "nightly" ] ; then IF_BUILD_NIGHTLY=1 ; fi
- if [ "${BUILD_TYPE}" == "default" ] ; then IF_BUILD_DEFAULT=1 ; fi
- source spack.sh ;
source eic-spack.sh ;
cat mirrors.yaml.in | envsubst > mirrors.yaml
- docker buildx build --push ${BUILD_OPTIONS}
--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE}
${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
}
${IF_BUILD_NIGHTLY:+
${NIGHLTY:+
${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}}
${PUSH_NIGHTLY_WITH_DATE:+
${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
}
}
--build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
--build-arg BASE_IMAGE=${BASE_IMAGE}
--build-arg INTERNAL_TAG=${INTERNAL_TAG}
--build-arg SPACK_ORGREPO=${SPACK_ORGREPO}
--build-arg SPACK_VERSION=${SPACK_VERSION}
--build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
--build-arg EICSPACK_ORGREPO=${EICSPACK_ORGREPO}
--build-arg EICSPACK_VERSION=${EICSPACK_VERSION}
--build-arg EICSPACK_CHERRYPICKS="${EICSPACK_CHERRYPICKS}"
--build-arg S3_ACCESS_KEY=${S3_ACCESS_KEY}
--build-arg S3_SECRET_KEY=${S3_SECRET_KEY}
--build-arg JUG_VERSION=${EXPORT_TAG}-${BUILD_TYPE}-$(git rev-parse HEAD)
${IF_BUILD_DEFAULT:+
--build-arg JUGGLER_VERSION=${JUGGLER_VERSION}
--build-arg EICRECON_VERSION=${EICRECON_VERSION}
}
--build-context spack-environment=spack-environment
Wouter Deconinck
committed
--secret id=mirrors,src=mirrors.yaml
jug_xl:default:
parallel:
matrix:
- BASE_IMAGE: jug_dev
BUILD_IMAGE: jug_xl
# - BASE_IMAGE: jug_dev_oneapi
# BUILD_IMAGE: jug_xl_oneapi
# PLATFORM: linux/amd64
needs:
- version
- docker buildx build --push ${BUILD_OPTIONS}
--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}-default
${EXPORT_TAG:+
${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
}
--file containers/jug/xl.Dockerfile
--platform ${PLATFORM}
--build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
--build-arg BASE_IMAGE=${BASE_IMAGE}
--build-arg INTERNAL_TAG=${INTERNAL_TAG}-default
--build-arg JUG_VERSION=${EXPORT_TAG}-$(git rev-parse HEAD)
--build-arg jobs=${JOBS}
--build-context detectors=.
containers/jug
jug_xl:nightly:
parallel:
matrix:
- BASE_IMAGE: jug_dev
BUILD_IMAGE: jug_xl
# - BASE_IMAGE: jug_dev_oneapi
# BUILD_IMAGE: jug_xl_oneapi
# PLATFORM: linux/amd64
extends: .build
stage: jug
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
when: on_success
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: on_success
- when: never
needs:
- version
PUSH_NIGHTLY_WITH_DATE=""
if [ "$CI_COMMIT_BRANCH" == "master" ]; then
PUSH_NIGHTLY_WITH_DATE="1"
- docker buildx build --push ${BUILD_OPTIONS}
--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}-nightly
${EXPORT_TAG:+
${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}-nightly}
${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}-nightly}
${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}-nightly}
}
${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
${PUSH_NIGHTLY_WITH_DATE:+
${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
--build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
--build-arg BASE_IMAGE=${BASE_IMAGE}
--build-arg INTERNAL_TAG=${INTERNAL_TAG}-nightly
--build-arg JUG_VERSION=${EXPORT_TAG}-nightly-$(git rev-parse HEAD)-$(date +%Y-%m-%d_%H-%M-%S)
--build-arg NIGHTLY=1
--build-context detectors=.
containers/jug
image:
name: ${SINGULARITY_IMAGE}
entrypoint: [""]
stage: deploy
- if: '$CI_COMMIT_BRANCH == "master"'
when: on_success
- when: never
artifacts:
paths:
- build/${BUILD_IMAGE}.sif
script:
- mkdir build
- singularity pull build/${BUILD_IMAGE}.sif docker://${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}-${BUILD_TYPE}
jug_xl:singularity:default:
extends: .singularity
variables:
BUILD_TYPE: default
BUILD_IMAGE: jug_xl
- jug_xl:default
# eic-shell expects build/jug_xl.sif from the job with name 'jug_xl:singularity:nightly'
jug_xl:singularity:nightly:
variables:
BUILD_TYPE: nightly
BUILD_IMAGE: jug_xl
- version
- jug_xl:nightly
benchmarks:reconstruction:default:
stage: benchmarks
needs:
- version
- jug_xl:default
BENCHMARKS_TAG: "${INTERNAL_TAG}-default"
BENCHMARKS_CONTAINER: "jug_xl"
BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
trigger:
project: EIC/benchmarks/reconstruction_benchmarks
strategy: depend
stage: benchmarks
needs:
- version
- jug_xl:default
BENCHMARKS_TAG: "${INTERNAL_TAG}-default"
BENCHMARKS_CONTAINER: "jug_xl"
BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
trigger:
project: EIC/benchmarks/physics_benchmarks
strategy: depend
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
allow_failure: false
benchmarks:reconstruction:nightly:
stage: benchmarks
rules:
- !reference ['.nightly', rules]
needs:
- version
- jug_xl:nightly
variables:
BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly"
BENCHMARKS_CONTAINER: "jug_xl"
BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
trigger:
project: EIC/benchmarks/reconstruction_benchmarks
strategy: depend
allow_failure: true
benchmarks:physics:nightly:
stage: benchmarks
rules:
- !reference ['.nightly', rules]
needs:
- version
- jug_xl:nightly
variables:
RECO: "eicrecon"
BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly"
BENCHMARKS_CONTAINER: "jug_xl"
BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE"
trigger:
project: EIC/benchmarks/physics_benchmarks
strategy: depend
clean_internal_tag:
image: alpine/curl
stage: finalize
dependencies:
- version
for id in 66 68 69 80 83 91 92 ; do
for tag in ${INTERNAL_TAG} ${INTERNAL_TAG}-default ${INTERNAL_TAG}-nightly ; do
curl --request DELETE --header "PRIVATE-TOKEN: ${REG_CLEANUP_TOKEN}" ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/registry/repositories/${id}/tags/${tag} ;
done ;
done
stage: finalize
dependencies:
- for registry in
eicweb
eicweb.phy.anl.gov:4567/containers/eic_container
; do
docker images
--filter=reference=${registry}/*:unstable-mr*
--format "{{.ID}} {{.Repository}} {{.Tag}}" ;
done
| sed -n -E 's/unstable-mr-([0-9]+)-([0-9]+)(-nightly|-default){0,1}$/& \1 \2/p'
| while read id repository tag project_id merge_request_iid ; do
state=$(curl --silent -L
"${CI_API_V4_URL}/projects/${project_id}/merge_requests?scope=all&state=opened&per_page=100"
"${CI_API_V4_URL}/projects/${project_id}/merge_requests?scope=all&state=merged&per_page=100"
"${CI_API_V4_URL}/projects/${project_id}/merge_requests?scope=all&state=closed&per_page=100"
| jq -r ".[]|select(.iid == ${merge_request_iid})|.state") ;
echo "Image $repository:$tag is ${state:-undetermined} (project ${project_id}, iid ${merge_request_iid})." ;
if [ "$state" == "merged" ] ; then docker rmi $repository:$tag ; fi ;
if [ "$state" == "closed" ] ; then docker rmi $repository:$tag ; fi ;
- docker system prune --filter until=24h --force
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
- docker system df
- docker images
clean_pipeline:
extends: .docker
stage: finalize
dependencies:
- version
- clean_internal_tag
tags:
- docker-new
when: always
script:
- apk add curl jq
- for registry in
eicweb
eicweb.phy.anl.gov:4567/containers/eic_container
; do
docker images
--filter=reference=${registry}/*:pipeline-*
--format "{{.ID}} {{.Repository}} {{.Tag}}" ;
done
| sed -n -E 's/pipeline-([0-9]+)(-.*){0,1}$/& \1/p'
| while read id repository tag pipeline_id ; do
status=$(curl --silent -L
"${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${pipeline_id}"
| jq -r ".status") ;
echo "Image $repository:$tag is ${status:-undetermined} (pipeline ${pipeline_id})." ;
if [ "$status" == "success" ] ; then docker rmi $repository:$tag ; fi ;
if [ "$status" == "failed" ] ; then docker rmi $repository:$tag ; fi ;
if [ "$status" == "canceled" ] ; then docker rmi $repository:$tag ; fi ;
done
- docker system prune --filter until=24h --force
- docker system df