Skip to content
Snippets Groups Projects
Commit 0f0cbf52 authored by Wouter Deconinck's avatar Wouter Deconinck
Browse files

feat: use official docker image

parent d4f7c42c
Branches
Tags
1 merge request!706feat: use official docker image
......@@ -2,9 +2,7 @@ image: alpine
variables:
## External images
# FIXME:
#DOCKER_IMAGE: docker.io/docker:24.0.5
DOCKER_IMAGE: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
DOCKER_IMAGE: docker.io/docker:24.0.5
SINGULARITY_IMAGE: quay.io/singularity/singularity:v3.11.4
## Application versions used for the main release
......@@ -133,23 +131,19 @@ version:
## base job settings for all docker interactions
.docker:
image: ${DOCKER_IMAGE}
services:
- docker:dind
before_script:
- # Login to registries
if [[ -n "${EXPORT_TAG}" ]] ; then
if [[ -n "${DH_REGISTRY}" ]] ; then
./gitlab-ci/docker_login.sh -u ${DH_REGISTRY_USER} -p ${DH_EICWEB_TOKEN}
-n ${DOCKER_NTRIES} -t ${DOCKER_WAIT_TIME}
${DH_REGISTRY} ;
docker login -u ${DH_REGISTRY_USER} -p ${DH_EICWEB_TOKEN} ${DH_REGISTRY} ;
fi ;
if [[ -n "${GH_REGISTRY}" ]] ; then
./gitlab-ci/docker_login.sh -u ${GITHUB_REGISTRY_USER} -p ${GITHUB_REGISTRY_TOKEN}
-n ${DOCKER_NTRIES} -t ${DOCKER_WAIT_TIME}
${GH_REGISTRY} ;
docker login -u ${GITHUB_REGISTRY_USER} -p ${GITHUB_REGISTRY_TOKEN} ${GH_REGISTRY} ;
fi ;
if [[ -n "${CI_REGISTRY}" ]] ; then
./gitlab-ci/docker_login.sh -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD}
-n ${DOCKER_NTRIES} -t ${DOCKER_WAIT_TIME}
${CI_REGISTRY} ;
docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} ;
fi ;
fi
after_script:
......@@ -273,11 +267,12 @@ jug_dev:
script:
- if [ "${BUILD_TYPE}" == "nightly" ] ; then IF_BUILD_NIGHTLY=1 ; fi
- if [ "${BUILD_TYPE}" == "default" ] ; then IF_BUILD_DEFAULT=1 ; fi
- apk add envsubst git
- source spack.sh ;
source eic-spack.sh ;
export SPACK_VERSION ;
cat mirrors.yaml.in | envsubst > mirrors.yaml ;
docker buildx build --push ${BUILD_OPTIONS}
cat mirrors.yaml.in | envsubst > mirrors.yaml
- docker buildx build --push ${BUILD_OPTIONS}
--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE}
${EXPORT_TAG:+
${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
......@@ -334,6 +329,7 @@ jug_xl:default:
- version
- jug_dev
script:
- apk add git
- docker buildx build --push ${BUILD_OPTIONS}
--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}-default
${EXPORT_TAG:+
......@@ -377,6 +373,7 @@ jug_xl:nightly:
if [ "$CI_COMMIT_BRANCH" == "master" ]; then
PUSH_NIGHTLY_WITH_DATE="1"
fi
- apk add git
- docker buildx build --push ${BUILD_OPTIONS}
--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}-nightly
${EXPORT_TAG:+
......@@ -503,93 +500,48 @@ benchmarks:physics:nightly:
strategy: depend
allow_failure: false
clean:
extends: .docker
clean_internal_tag:
image: alpine/curl
stage: finalize
dependencies:
- version
tags:
- docker-new
when: always
script:
## remove the pipeline specific export from eicweb if needed
- echo "Cleaning up pipeline specific docker tags if needed"
- ./gitlab-ci/cleanup_registry.sh -i debian_base -r 66 ${INTERNAL_TAG}
- ./gitlab-ci/cleanup_registry.sh -i ubuntu_base -r 80 ${INTERNAL_TAG}
- ./gitlab-ci/cleanup_registry.sh -i debian_testing_base -r 83 ${INTERNAL_TAG}
- ./gitlab-ci/cleanup_registry.sh -i debian_stable_base -r 92 ${INTERNAL_TAG}
- ./gitlab-ci/cleanup_registry.sh -i jug_prod -r 91 ${INTERNAL_TAG}
- ./gitlab-ci/cleanup_registry.sh -i jug_dev -r 68 ${INTERNAL_TAG}
- ./gitlab-ci/cleanup_registry.sh -i jug_dev -r 68 ${INTERNAL_TAG}-nightly
- ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${INTERNAL_TAG}
- ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${INTERNAL_TAG}-nightly
- |
if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${NIGHTLY_TAG}
fi
for id in 66 68 69 80 83 91 92 ; do
for tag in ${INTERNAL_TAG} ${INTERNAL_TAG}-default ${INTERNAL_TAG}-nightly ; do
curl --request DELETE --header "PRIVATE-TOKEN: ${REG_CLEANUP_TOKEN}" ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/registry/repositories/${id}/tags/${tag} ;
done ;
done
clean_unstable_mr:
extends: .docker
stage: finalize
dependencies:
- clean
- version
tags:
- docker-new
when: always
variables:
API: "https://eicweb.phy.anl.gov/api/v4"
script:
- apt update -q
- apt install -yq jq
- apk add curl jq
- for registry in
eicweb
eicweb.phy.anl.gov:4567/eic/juggler
eicweb.phy.anl.gov:4567/containers/eic_container
; do
docker images
--filter=reference=${registry}/*:unstable-mr*
--format "{{.ID}} {{.Repository}} {{.Tag}}" ;
done | while read id repository tag ; do
if [[ ${tag} =~ ^unstable-mr-([0-9]+)-([0-9]+)(-nightly){0,1}$ ]] ; then
project_id=${BASH_REMATCH[1]} ;
merge_request_iid=${BASH_REMATCH[2]} ;
if [[ $(curl --silent -L
"${API}/projects/${project_id}/merge_requests?scope=all&state=merged&per_page=100"
"${API}/projects/${project_id}/merge_requests?scope=all&state=closed&per_page=100"
| jq -r ".[]|select(.iid == ${merge_request_iid})|.state") =~ ^(merged|closed)$ ]] ; then
docker rmi $repository:$tag ;
else
echo "Image $repository:$tag still in use (project ${project_id}, idd ${merge_request_iid})." ;
fi ;
fi ;
if [[ ${tag} =~ ^unstable-mr-([0-9]+)(-nightly){0,1}$ ]] ; then
merge_request_id=${BASH_REMATCH[1]} ;
suffix=${BASH_REMATCH[2]} ;
if [[ $(curl --silent -L
"${API}/projects/290/merge_requests?scope=all&state=merged&per_page=100"
"${API}/projects/369/merge_requests?scope=all&state=merged&per_page=100"
"${API}/projects/290/merge_requests?scope=all&state=closed&per_page=100"
"${API}/projects/369/merge_requests?scope=all&state=closed&per_page=100"
| jq -r ".[]|select(.id == ${merge_request_id})|.state") =~ ^(merged|closed)$ ]] ; then
docker rmi $repository:$tag ;
else
project_id=$(curl --silent -L
"${API}/projects/290/merge_requests?scope=all&state=opened&per_page=100"
"${API}/projects/369/merge_requests?scope=all&state=opened&per_page=100"
| jq -r ".[]|select(.id == ${merge_request_id})|.project_id") ;
merge_request_iid=$(curl --silent -L
"${API}/projects/290/merge_requests?scope=all&state=opened&per_page=100"
"${API}/projects/369/merge_requests?scope=all&state=opened&per_page=100"
| jq -r ".[]|select(.id == ${merge_request_id})|.iid") ;
echo "Image $repository:$tag still in use (project ${project_id}, idd ${merge_request_iid})." ;
if [[ ${project_id} == "" || ${merge_request_iid} == "" ]] ; then
echo "$tag not found" ;
else
docker tag $repository:$tag $repository:unstable-mr-${project_id}-${merge_request_iid}${suffix} ;
docker rmi $repository:$tag ;
fi ;
fi ;
fi ;
done
| sed -n -E 's/unstable-mr-([0-9]+)-([0-9]+)(-nightly|-default){0,1}$/& \1 \2/p'
| while read id repository tag project_id merge_request_iid ; do
state=$(curl --silent -L
"${CI_API_V4_URL}/projects/${project_id}/merge_requests?scope=all&state=opened&per_page=100"
"${CI_API_V4_URL}/projects/${project_id}/merge_requests?scope=all&state=merged&per_page=100"
"${CI_API_V4_URL}/projects/${project_id}/merge_requests?scope=all&state=closed&per_page=100"
| jq -r ".[]|select(.iid == ${merge_request_iid})|.state") ;
echo "Image $repository:$tag is ${state:-undetermined} (project ${project_id}, iid ${merge_request_iid})." ;
if [ "$state" == "merged" ] ; then docker rmi $repository:$tag ; fi ;
if [ "$state" == "closed" ] ; then docker rmi $repository:$tag ; fi ;
done
- docker system prune --filter until=24h --force
- docker images
base:
debian:
develop: amd64/debian:testing-20211220-slim
runtime: amd64/debian:testing-20211220-slim
Dockerfile: containers/base/Dockerfile.in
oneapi:
develop: intel/oneapi-hpckit:2022.1.2-devel-ubuntu18.04
runtime: intel/oneapi-runtime:2022.1.2-devel-ubuntu18.04
Dockerfile: containers/base/Dockerfile.in
cuda:
develop: nvidia/cuda:11.6.0-devel-ubuntu18.04
runtime: nvidia/cuda:11.6.0-runtime-ubuntu18.04
Dockerfile: containers/base/Dockerfile.in
jug:
jug_dev:
Dockerfile: containers/jug/dev.Dockerfile
jug_xl:
Dockerfile: containers/jug/xl.Dockerfile
#!/bin/bash
## Remove a specific docker tag from the eicweb registry
function print_the_help {
echo "USAGE: -i image -r reg_id tag "
echo "ARGUMENTS:"
echo " -i,--image Registry image name"
echo " -r,--reg-id Registry image ID (integer number)"
echo " -h,--help Print this message"
echo " positional Tag to remove from registry"
echo ""
echo " Remove a specific docker tag from the eicweb registry, if present on the
registry."
echo ""
echo "EXAMPLE: ./cleanup_registry -i debian_base -r 66 unstable"
exit
}
IMAGE=
REG_ID=
TAG=
while [ $# -gt 0 ]; do
key=$1
case $key in
-i|--image)
IMAGE=$2
shift
shift
;;
-r|--reg-id)
REG_ID=$2
shift
shift
;;
-h|--help)
print_the_help
exit 0
;;
-*)
echo "ERROR: unknown flag: $key"
echo "use --help for more info"
exit 1
;;
*)
if [ ! -z ${TAG} ]; then
echo "ERROR: multiple positional arguments specified"
echo "use --help for more info"
exit 1
fi
TAG=$1
shift
;;
esac
done
if [ -z $IMAGE ]; then
echo "ERROR: no image name given, please use -i <IMAGE>"
print_the_help
exit 1
fi
if [ -z $REG_ID ]; then
echo "ERROR: no image id given, please use -r <REG_ID>"
print_the_help
exit 1
fi
if [ -z $TAG ]; then
echo "ERROR: no tag given, please specify a single tag (positional argument)"
print_the_help
exit 1
fi
echo "Cleaning up eicweb registry for ${IMAGE}:${TAG}"
IMAGE_EXISTS=
docker manifest inspect \
eicweb.phy.anl.gov:4567/containers/eic_container/${IMAGE}:${TAG} \
> /dev/null \
&& IMAGE_EXISTS=1 \
|| echo "Image not found, no cleanup needed"
if [ ! -z ${IMAGE_EXISTS} ]; then
curl --request DELETE --header "PRIVATE-TOKEN: ${REG_CLEANUP_TOKEN}" \
${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/registry/repositories/${REG_ID}/tags/${TAG} \
&& echo "Image removed" \
|| echo "Error trying to remove image"
fi
#!/bin/bash
## Resilient docker login with multiple progressive retries in case of TLS issues
function print_the_help {
echo "USAGE: -u user -p -password [...] [registry]"
echo "ARGUMENTS:"
echo " -u,--user Registry user"
echo " -p,--password Registry password"
echo " --ci Login to internal CI registry; -u, -p and positional are ignored"
echo " -t,--time Time interval (in seconds) between attempts"
echo " (doubled each time), default: 5"
echo " -n,--n-attempts Number of attempts, default: 5"
echo " -h,--help Print this message"
echo " positional Registry name (will use dockerhub if none given)"
echo ""
echo " Login to registry."
echo ""
echo "EXAMPLE: ./docker_login.sh -u \$CI_REGISTRY_USER -p \$CI_REGISTRY_PASSWORD \$CI_REGISTRY"
exit
}
REGISTRY=
REG_USER=
REG_PASSWORD=
NTRIES=5
TIME=5
CI=
while [ $# -gt 0 ]; do
key=$1
case $key in
-u|--user)
REG_USER=$2
shift
shift
;;
-p|--password)
REG_PASSWORD=$2
shift
shift
;;
--ci)
CI=1
shift
;;
-t|--time)
TIME=$2
shift
shift
;;
-n|--n-attempts)
NTRIES=$2
shift
shift
;;
-h|--help)
print_the_help
exit 0
;;
-*)
echo "ERROR: unknown flag: $key"
echo "use --help for more info"
exit 1
;;
*)
REGISTRY+="$1"
shift
;;
esac
done
## only sanizize input if not in CI registry mode
if [ -z $CI ]; then
if [ -z $REG_USER ]; then
echo "ERROR: no username given, please use -u <USER>"
print_the_help
exit 1
fi
if [ -z $REG_PASSWORD ]; then
echo "ERROR: no password given, please use -p <PASSWORD>"
print_the_help
exit 1
fi
else
REGISTRY=$CI_REGISTRY
REG_USER=$CI_REGISTRY_USER
fi
while [ $NTRIES != 0 ]; do
if [ -z $CI ]; then
echo $REG_PASSWORD | docker login -u $REG_USER \
--password-stdin $REGISTRY \
&& break \
|| echo "Login failed, retrying in $TIME seconds..."
else
echo $CI_REGISTRY_PASSWORD | docker login -u $REG_USER \
--password-stdin $REGISTRY \
&& break \
|| echo "Login failed, retrying in $TIME seconds..."
fi
sleep ${TIME}s
TIME=$((TIME * 2))
NTRIES=$((NTRIES - 1))
done
if [ $NTRIES = 0 ]; then
echo "Failed to login to registry $REGISTRY"
exit 1
fi
echo "Docker login to $REGISTRY successful"
#!/bin/bash
## Generic script to execute docker push from the input tag to all export tags
## on both dockerhub and eicweb if set
function print_the_help {
echo "USAGE: -i image -l input_tag export_tag [export_tag2 ...]"
echo "ARGUMENTS:"
echo " -i,--image Input image name (e.g., eic_base)"
echo " -l,--label Input tag (eg., v3.0.0)"
echo " -t,--time Time interval (in seconds) between attempts"
echo " (doubled each time), default: 5"
echo " -n,--n-attempts Number of attempts, default: 5"
echo " -h,--help Print this message"
echo " --ghcr Publish to ghcr.io only"
echo " --eicweb Publish to $CI_REGISTRY only"
echo " --dockerhub Publish to DH only"
echo " positional At least one export tag (e.g., v3.0-stable)"
echo ""
echo " Execute docker push from image:label to REGISTRY/image:export_tag for"
echo " all export tags. Will push to $CI_REGISTRY, and optionally also to"
echo " Dockerhub if the DH_PUSH environment variable is set"
echo ""
echo "EXAMPLE: ./docker_push.sh -i eic_base -l 3.0.0 3.0.0 3.0-stable"
exit
}
IMAGE=
INPUT_TAG=
EXPORT_TAGS=()
NTRIES=5
TIME=5
DO_DH=${DH_PUSH}
DO_GH=${GH_PUSH}
DO_EICWEB=1
while [ $# -gt 0 ]; do
key=$1
case $key in
-i|--image)
IMAGE=$2
shift
shift
;;
-l|--label)
INPUT_TAG=$2
shift
shift
;;
-t|--time)
TIME=$2
shift
shift
;;
-n|--n-attempts)
NTRIES=$2
shift
shift
;;
--ghcr)
DO_EICWEB=-
DO_DH=0
DO_GH=1
shift
;;
--eicweb)
DO_EICWEB=1
DO_DH=0
DO_GH=0
shift
;;
--dockerhub)
DO_EICWEB=0
DO_DH=1
DO_GH=0
shift
;;
-h|--help)
print_the_help
exit 0
;;
-*)
echo "ERROR: unknown flag: $key"
echo "use --help for more info"
exit 1
;;
*)
EXPORT_TAGS+=("$1")
shift
;;
esac
done
if [ -z $IMAGE ]; then
echo "ERROR: no image name given, please use -i <IMAGE>"
print_the_help
exit 1
fi
if [ -z $INPUT_TAG ]; then
echo "ERROR: no input_tag given, please use -l <INPUT_TAG>"
print_the_help
exit 1
fi
if [ ${#EXPORT_TAGS[@]} -eq 0 ]; then
echo "ERROR: need at least one export tag (positional argument)"
print_the_help
exit 1
fi
function retry_push () {
INPUT=$1
DESTINATION=$2
time=$TIME
ntries=$NTRIES
echo "Pushing ${INPUT} to ${DESTINATION}"
if [ ${INPUT} != ${DESTINATION} ]; then
echo docker tag ${INPUT} ${DESTINATION}
docker tag ${INPUT} ${DESTINATION}
fi
while [ $ntries != 0 ]; do
echo docker push ${DESTINATION}
docker push ${DESTINATION} && break \
|| echo "Docker push failed, retrying in $time seconds..."
sleep ${time}s
time=$((time * 2))
ntries=$((ntries - 1))
done
if [ ${INPUT} != ${DESTINATION} ]; then
echo docker rmi ${DESTINATION}
docker rmi ${DESTINATION}
fi
if [ $ntries = 0 ]; then
echo "Failed to push $INPUT to $DESTINATION"
exit 1
fi
}
#echo "IMAGE: ${IMAGE}"
#echo "INPUT_TAG: ${INPUT_TAG}"
#echo "EXPORT_TAGS: ${EXPORT_TAGS}"
#echo "DH_PUSH: ${DH_PUSH}"
#echo "DO_DH: ${DO_DH}"
#echo "DO_GH: ${DO_GH}"
#echo "DO_EICWEB: ${DO_EICWEB}"
export INPUT=$CI_REGISTRY_IMAGE/${IMAGE}:${INPUT_TAG}
for TAG in ${EXPORT_TAGS[@]}; do
if [ ${DO_EICWEB} != 0 ]; then
retry_push $INPUT ${CI_REGISTRY_IMAGE}/${IMAGE}:${TAG}
fi
if [ ${DO_DH} != 0 ]; then
retry_push $INPUT ${DH_REGISTRY_USER}/${IMAGE}:${TAG}
fi
if [ ${DO_GH} != 0 ]; then
retry_push $INPUT ${GH_REGISTRY}/eic/${IMAGE}:${TAG}
fi
done
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment