Skip to content
Snippets Groups Projects
Commit ce01b8f8 authored by Sylvester Joosten's avatar Sylvester Joosten Committed by Sylvester Joosten
Browse files

Resolve "Simplify and consolidate container build" 

New container names are jug_dev and jug_xl (dressed container)
parent 983ac450
Branches
Tags
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 805 additions and 526 deletions
.gitlab-ci.yml
+ 262
− 62
View file @ ce01b8f8
image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
variables:
## Application versions used for the main release ## note: nightly builds will always use the master branch
JUGGLER_VERSION: "v1.5.0"
NPDET_VERSION: "master"
EICD_VERSION: "master"
DETECTOR_VERSION: "master"
IP6_VERSION: "master"
ACCELERATOR_VERSION: "master"
## Spack github version
SPACK_VERSION: "63915de99b6d4bc7c8b8a22dc45e5c3dd7ee4f26"
## We need to enable Docker Buildkit to use cache mounts and better
## build performance overal
DOCKER_BUILDKIT: 1
## Dockerhub registry
DH_REGISTRY: eicweb
DH_PUSH: 1
## TLS error resiliency: number of retries and second wait between tries
## (wait time is doubled with each attempt)
DOCKER_NTRIES: 5
DOCKER_WAIT_TIME: 5
## By default this is not a nightly build, unless the CI says so
NIGHTLY: 0
stages: stages:
- config - config
- build - build:base ## base OS image
- deploy - build:dev ## naked dev container image
- build:release ## dressed release container image
- deploy ## build/deploy singularity images
- finalize
## make note if we cannot use caching for one of the stages
## by touching files in .ci_env
default: default:
tags: tags:
- silicon - silicon
artifacts: before_script:
paths: - ./gitlab-ci/docker_login.sh -u $DH_REGISTRY -p $DH_EICWEB_TOKEN
- .ci-env -n $DOCKER_NTRIES -t $DOCKER_WAIT_TIME
detect_changes:builder: - ./gitlab-ci/docker_login.sh --ci -n $DOCKER_NTRIES -t $DOCKER_WAIT_TIME
stage: .pre after_script:
- docker logout
- docker logout ${CI_REGISTRY}
## only run CI for in the following cases:
## master, stable branch, release tag, MR event and nightly builds
## not that nightly builds got from the master branch, but with "NIGHTLY" set to
## 1 which triggers a slightly different workflow
workflow:
rules: rules:
- changes: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- containers/builder/spack.yaml - if: '$CI_COMMIT_BRANCH == "master"'
- spack/packages/**/* - if: '$CI_COMMIT_BRANCH =~ /v[0-9]+\.[0-9]+-stable/'
- if: '$CI_COMMIT_TAG =~ /v[0-9]+\.[0-9]+\.[0-9]+/'
## plan:
## Workflows:
## - master --> config + all build stages + singularity
## - <nightly> --> config + build:release only + singularity
## - v3.0-stable --> config + all build stages + singularity
## - v3.0.0 --> config + all build stages + singularity
## - MR --> config + all build stages
##
## Container images tags
## - master --> testing
## - <nightly> --> nightly
## - v3.0-stable --> 3.0-stable
## - v3.0.0 --> 3.0-stable, 3.0.0
## - MR --> unstable (on all registries)
## --> unstable-mr-XXX (on eicweb only, untag at end of pipeline)
## - all other --> do nothing
##
## caching strategy for dispatch to multiple nodes and to avoid
## --> try this strategy: https://medium.com/titansoft-engineering/docker-build-cache-sharing-on-multi-hosts-with-buildkit-and-buildx-eb8f7005918e
## (first try with buildx didn't pan out, let's try again later)
version:
stage: config
script: script:
- mkdir -p .ci-env - |
- touch .ci-env/builder-nc VERSION=`head -n1 VERSION`
detect_changes:release: STABLE=${VERSION%.*}-stable
stage: .pre TESTING="testing"
UNSTABLE="unstable"
## determine appropriate major docker tag for this scenario
- |
## internal tag used for the CI. Also temporarily tagged
## on eicweb to communicate between jobs (removed in cleanup job)
INTERNAL_TAG="testing-$VERSION"
## main export tag, optional secondary export tag,
EXPORT_TAG=${TESTING}
EXPORT_TAG2=
if [ "x${CI_PIPELINE_SOURCE}" == "xmerge_request_event" ]; then
INTERNAL_TAG="unstable-mr-${CI_MERGE_REQUEST_ID}"
EXPORT_TAG=$UNSTABLE
EXPORT_TAG2=
elif [ "$CI_COMMIT_TAG" = "v${VERSION}" ]; then
INTERNAL_TAG="stable-br-${VERSION}"
EXPORT_TAG=${STABLE}
EXPORT_TAG2=${VERSION}
elif [ "$CI_COMMIT_BRANCH" == "v${STABLE}" ]; then
INTERNAL_TAG="stable-tag-${VERSION}"
EXPORT_TAG=${STABLE}
EXPORT_TAG2=
elif [ "$NIGHTLY" != "0" ]; then
INTERNAL_TAG="nightly-${VERSION}"
EXPORT_TAG="nightly"
EXPORT_TAG2=
fi
echo "INTERNAL_TAG=$INTERNAL_TAG" >> build.env
echo "EXPORT_TAG=$EXPORT_TAG" >> build.env
echo "EXPORT_TAG2=$EXPORT_TAG2" >> build.env
echo "PIPELINE_TMP_TAG=$PIPELINE_TMP_TAG" >> build.env
cat build.env
artifacts:
reports:
dotenv: build.env
## base job for all build jobs. Dependent jobs are expected to set
## the BUILD_IMAGE environment variable, and take care of the actual
## docker build during the "script" step
## note that the nightly builds use a different pipeline
.build:
rules: rules:
- changes: - if: '$NIGHTLY != "0"'
- containers/release/configure_release.sh when: never
- when: on_success
## cookie-cutter docker push code, to be included at the
## end of the regular job scripts
script: script:
- mkdir -p .ci-env - ./gitlab-ci/docker_push.sh -i ${BUILD_IMAGE} -l ${INTERNAL_TAG}
- touch .ci-env/release-nc -n $DOCKER_NTRIES -t $DOCKER_WAIT_TIME
${EXPORT_TAG} ${EXPORT_TAG2}
- ./gitlab-ci/docker_push.sh -i ${BUILD_IMAGE} -l ${INTERNAL_TAG}
-n $DOCKER_NTRIES -t $DOCKER_WAIT_TIME
${INTERNAL_TAG} --eicweb
## Init our job for our desired branches/tags/events ## Images:
init: ## debian_base --> jug_dev --> jug_xl
stage: config ## ---------------> jug_sim
## ---------------> jug_ml
debian_base:default:
extends: .build
stage: build:base
needs:
- version
variables:
BUILD_IMAGE: "debian_base"
script: script:
- ./gitlab-ci/configure.sh containers/build_and_deploy.yml.in - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
artifacts: containers/debian
paths: - !reference [.build, script]
- build_and_deploy.yml
jug_dev:default:
## Quick debug build for new spack packages extends: .build
#run:quick_build: stage: build:dev
# image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:latest needs:
# stage: build - version
# script: - debian_base:default
# - rm -r /opt/spack/np-spack && cp -r spack /opt/spack/np-spack variables:
# - source /opt/spack/share/spack/setup-env.sh BUILD_IMAGE: "jug_dev"
# - spack env activate /opt/spack-environment script:
# - spack add madx && spack install ## calculate a hash based on the spack.yaml file and the spack directory
## and use this spack as a docker variable to force a rebuild when there
## Dispatch if we ran the previous stage ## is a change (versus rerun from cache)
run:default: - PACKAGE_HASH=$(tar cf - spack* | sha1sum | head -c40)
stage: build - echo "PACKAGE_HASH= ${PACKAGE_HASH}"
trigger: ## move spacke directory and spack.yaml into the container build directory
include: - cp -r spack containers/jug
- artifact: build_and_deploy.yml - cp -r spack.yaml containers/jug/spack/spack.yaml
job: init ## now build our image
strategy: depend - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
-f containers/jug/Dockerfile.dev
singularity: --build-arg SPACK_VERSION=${SPACK_VERSION}
stage: deploy --build-arg CACHE_BUST=${PACKAGE_HASH}
needs: ["run:default"] --build-arg INTERNAL_TAG=${INTERNAL_TAG}
containers/jug
- !reference [.build, script]
jug_xl:default:
extends: .build
stage: build:release
needs:
- version
- jug_dev:default
variables:
BUILD_IMAGE: "jug_xl"
script:
- docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
-f containers/jug/Dockerfile.xl
--build-arg INTERNAL_TAG=${INTERNAL_TAG}
--build-arg JUGGLER_VERSION=${JUGGLER_VERSION}
--build-arg NPDET_VERSION=${NPDET_VERSION}
--build-arg EICD_VERSION=${EICD_VERSION}
--build-arg DETECTOR_VERSION=${DETECTOR_VERSION}
--build-arg IP6_VERSION=${IP6_VERSION}
--build-arg ACCELERATOR_VERSION=${ACCELERATOR_VERSION}
containers/jug
- !reference [.build, script]
jug_xl:nightly:
extends: .build
stage: build:release
rules: rules:
- if: '$CI_COMMIT_BRANCH == "master"' - if: '$NIGHTLY != "0"'
when: on_success when: always
- if: '$CI_COMMIT_BRANCH =~ /v[0-9]+\.[0-9]+-stable/' - when: never
when: on_success needs:
- if: '$CI_COMMIT_TAG =~ /v[0-9]+\.[0-9]+\.[0-9]+/' - version
when: on_success variables:
BUILD_IMAGE: "jug_xl"
script: script:
- mkdir -p build - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
- cd build -f containers/jug/Dockerfile.xl
- ../gitlab-ci/configure.sh ../containers/release/eic.def.in --build-arg INTERNAL_TAG="testing"
- /bin/bash ../gitlab-ci/singularity/build.sh eic.def containers/jug
- !reference [.build, script]
.singularity:
rules:
- if: '$NIGHTLY != "0"'
when: never
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- when: on_success
artifacts: artifacts:
expire_in: 90 days expire_in: 90 days
paths: paths:
- build/eic.sif - build/${BUILD_IMAGE}.sif
- build/eic.def script:
- mkdir build
- singularity pull build/${BUILD_IMAGE}.sif docker://${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
jug_dev:singularity:default:
extends: .singularity
stage: deploy
needs:
- version
- jug_dev:default
variables:
BUILD_IMAGE: "jug_dev"
jug_xl:singularity:default:
extends: .singularity
stage: deploy
needs:
- version
- jug_xl:default
variables:
BUILD_IMAGE: "jug_xl"
jug_xl:singularity:nightly:
extends: .singularity
stage: deploy
rules:
- if: '$NIGHTLY != "0"'
when: always
- when: never
needs:
- version
- jug_xl:nightly
variables:
BUILD_IMAGE: "jug_xl"
cleanup:
stage: finalize
dependencies:
- version
script:
## remove the pipeline specific export from eicweb if needed
- echo "Cleaning up pipeline specific docker tags if needed"
- ./gitlab-ci/cleanup_registry.sh -i debian_base -r 66 ${INTERNAL_TAG}
- ./gitlab-ci/cleanup_registry.sh -i jug_dev -r 68 ${INTERNAL_TAG}
- ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${INTERNAL_TAG}
README.md
+ 4
− 3
View file @ ce01b8f8
...@@ -15,9 +15,9 @@ cd eic_container ...@@ -15,9 +15,9 @@ cd eic_container
modeuefile will be installed to `$PREFIX/../../etc/modulefiles`. modeuefile will be installed to `$PREFIX/../../etc/modulefiles`.
You can use the `-v` flag to select the version you want to install, or omit the You can use the `-v` flag to select the version you want to install, or omit the
flag if you want to install the master build. The recommended stable flag if you want to install the master build. The recommended stable
release version is `v2.8.0`. release version is `v2.9.2`.
```bash ```bash
./install.py -v 2.8.0 <PREFIX> ./install.py -v 2.9.2 <PREFIX>
``` ```
Available flags: Available flags:
...@@ -118,11 +118,12 @@ Included software: ...@@ -118,11 +118,12 @@ Included software:
- podio@0.13 - podio@0.13
- geant4@10.7.1 - geant4@10.7.1
- dd4hep@1.16.1 - dd4hep@1.16.1
- acts@5.00.0 - acts@8.01.0
- gaudi@34.0 - gaudi@34.0
- dawn@3.91a - dawn@3.91a
- dawncut@1.54a - dawncut@1.54a
- opencascade - opencascade
- madx@5.06.1
- The singularity build exports the following applications: - The singularity build exports the following applications:
- eic_shell: a development shell in the image - eic_shell: a development shell in the image
- container_dev: same as EIC shell - container_dev: same as EIC shell
......
containers/build_and_deploy.yml.in deleted 100644 → 0
+ 0
− 62
View file @ 983ac450
image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
stages:
- build
- config
- package
## variables:
## - TARGET_XXX: docker build target (including cache modifier)
## (stable, stable-cached, unstable, unstable-cached)
## - TAG: main docker tag to be used internally
## (e.g. 2.5-stable/unstable/<version>)
## - PUBLISH: docker publish directives
variables:
TARGET_BUILDER: @TARGET_BUILDER@
TARGET_RELEASE: @TARGET_RELEASE@
TAG: @TAG@
PUBLISH: "@PUBLISH@"
default:
tags:
- silicon
builder:
stage: build
script:
- cp -r gitlab-ci/docker/* containers/builder
- cp -r spack containers/builder/np-spack
- cd containers/builder
- head Dockerfile
- make login
- echo "Creating builder image for $TARGET_BUILDER"
- make $TARGET_BUILDER
- echo "Publishing image $PUBLISH"
- make $PUBLISH
config:
image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:$TAG
stage: config
needs: ["builder"]
script:
- bash containers/release/configure_dockerfile.sh $TAG
artifacts:
paths:
- config
release:
stage: package
needs: ["config"]
script:
- cp -r gitlab-ci/docker/* containers/release
- cp config/Dockerfile containers/release/Dockerfile
- cp config/eic-env.sh containers/release/eic-env.sh
- cd containers/release
- make login
- echo "Creating release image for $TARGET_RELEASE"
- make $TARGET_RELEASE
- echo "Publishing image $PUBLISH"
- make $PUBLISH
artifacts:
paths:
- config
containers/builder/Dockerfile deleted 100644 → 0
+ 0
− 82
View file @ 983ac450
#syntax=docker/dockerfile:1.2
# Builder with Argonne EIC software
#
FROM eicweb.phy.anl.gov:4567/containers/image_recipes/debian_spack:snapshot-20210408
LABEL maintainer="Sylvester Joosten <sjoosten@anl.gov>" \
name="eic_builder" \
group="eic_builder" \
march="native" \
basedist="debian" \
base="debian"
ENV DOCKERFILE_BASE=debian \
DOCKERFILE_DISTRO=debian \
DOCKERFILE_DISTRO_VERSION=20210408-testing \
SPACK_ROOT=/opt/spack \
DEBIAN_FRONTEND=noninteractive \
CURRENTLY_BUILDING_DOCKER_IMAGE=1 \
container=docker
## install ghostview/ghostscript needed by some of the tools
#RUN apt-get -yqq update \
# && apt-get -yqq install --no-install-recommends \
# ghostscript \
# gv \
# && rm -rf /var/lib/apt/lists/*
## Setup our environment definition
COPY spack.yaml /opt/spack-environment/spack.yaml
## Ensure an up-to-date custom package list
## TODO: We should just remove this from the upstream container
## and only initialize the custom packages here for more
## transparency
RUN rm -rf $SPACK_ROOT/np-spack \
&& echo "repos:" > $SPACK_ROOT/etc/spack/repos.yaml \
&& echo " - $SPACK_ROOT/np-spack" >> $SPACK_ROOT/etc/spack/repos.yaml
COPY np-spack $SPACK_ROOT/np-spack
## Install the software, no garbage collection at this stage
## as this is a raw builder image
RUN \
cd /opt/spack-environment \
&& spack env activate . \
&& spack install -j64 \
&& spack clean -a
## extra post-spack steps
## Including some small fixes:
## - Somehow PODIO env isn't automatically set,
## - and Gaudi likes BINARY_TAG to be set
RUN cd /opt/spack-environment \
&& echo -n "" \
&& echo "Grabbing environment info" \
&& spack env activate --sh -d . > /etc/profile.d/z10_spack_environment.sh \
&& sed -i "s?LD_LIBRARY_PATH=?&/lib/x86_64-linux-gnu:?" /etc/profile.d/z10_spack_environment.sh \
&& cd /opt/spack-environment \
&& echo -n "" \
&& echo "Add extra environment variables for Podio and Gaudi" \
&& spack env activate . \
&& export PODIO=`spack find -p podio | grep software | awk '{print $2}'` \
&& echo "export PODIO=${PODIO};" >> /etc/profile.d/z10_spack_environment.sh \
&& echo "export BINARY_TAG=x86_64-linux-gcc9-opt" >> /etc/profile.d/z10_spack_environment.sh \
&& cd /opt/spack-environment && spack env activate . \
&& echo -n "" \
&& echo "Installing additional python packages" \
&& pip install --trusted-host pypi.org \
--trusted-host files.pythonhosted.org \
--no-cache-dir \
ipython matplotlib scipy yapf pandas \
jupyter jupyterlab uproot pyunfold seaborn \
&& echo -n "" \
&& echo "Executing cmake patch for dd4hep 16.1" \
&& sed -i "s/FIND_PACKAGE(Python/#&/" /usr/local/cmake/DD4hepBuild.cmake
## make sure we have the entrypoints setup correctly
ENTRYPOINT []
CMD ["bash", "--rcfile", "/etc/profile", "-l"]
USER 0
WORKDIR /
containers/builder/config.env deleted 100644 → 0
+ 0
− 11
View file @ 983ac450
REG_HOST ?= eicweb.phy.anl.gov
REG_PORT ?= 4567
REG_NAME ?= $(REG_HOST):$(REG_PORT)
REG_URL ?= https://$(REG_HOST)
APP_NAME = eic_builder
REPO_NAME = eic_builder
GL_GROUP = eic_container
GL_REG_GROUP = containers/eic_container
GL_REG_NAME = $(REG_NAME)
containers/debian/Dockerfile 0 → 100644
+ 66
− 0
View file @ ce01b8f8
#syntax=docker/dockerfile:1.2
# Minimal container based on Debian Testing for up-to-date packages.
# Very lightweight container with a minimal build environment
FROM amd64/debian:testing-20210408-slim
LABEL maintainer="Sylvester Joosten <sjoosten@anl.gov>" \
name="debian_base" \
march="amd64"
COPY bashrc /root/.bashrc
ENV CLICOLOR_FORCE=1 \
LANGUAGE=en_US.UTF-8 \
LANG=en_US.UTF-8 \
LC_ALL=en_US.UTF-8
## Install additional packages. Remove the auto-cleanup functionality
## for docker, as we're using the new buildkit cache instead.
## We also install gitlab-runner, from the buster package (as bullseye is not available atm)
RUN --mount=type=cache,target=/var/cache/apt \
rm -f /etc/apt/apt.conf.d/docker-clean \
&& ls /var/cache/apt/* \
&& apt-get -yqq update \
&& apt-get -yqq install --no-install-recommends \
ca-certificates \
curl \
file \
build-essential \
g++-10 \
gcc-10 \
gfortran-10 \
git \
gnupg2 \
iproute2 \
locales \
lua-posix \
make \
unzip \
nano \
vim-nox \
less \
clang-format \
openssh-client \
wget \
ghostscript \
gv \
poppler-utils \
parallel \
&& localedef -i en_US -f UTF-8 en_US.UTF-8 \
&& update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 \
&& update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 \
&& update-alternatives --install /usr/bin/gfortran gfortran \
/usr/bin/gfortran-10 100 \
&& cc --version \
&& curl -L \
"https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" \
| bash \
&& sed -i "s/bullseye/buster/" \
/etc/apt/sources.list.d/runner_gitlab-runner.list \
&& apt-get -yqq update \
&& apt-get -yqq install --no-install-recommends \
gitlab-runner \
&& apt-get -yqq autoremove \
&& rm -rf /var/lib/apt/lists/*
containers/release/bashrc → containers/debian/bashrc
+ 0
− 0
View file @ ce01b8f8
File moved
containers/jug/Dockerfile.dev 0 → 100644
+ 208
− 0
View file @ ce01b8f8
#syntax=docker/dockerfile:1.2
ARG INTERNAL_TAG="testing"
## ========================================================================================
## STAGE1: spack builder image
## EIC builder image with spack
## ========================================================================================
FROM eicweb.phy.anl.gov:4567/containers/eic_container/debian_base:${INTERNAL_TAG} as builder
## instal some extra spack dependencies
RUN --mount=type=cache,target=/var/cache/apt \
rm -f /etc/apt/apt.conf.d/docker-clean \
&& apt-get -yqq update \
&& apt-get -yqq install --no-install-recommends \
python3 \
python3-pip \
python3-setuptools \
tcl \
uuid-dev \
libfcgi-dev \
x11proto-xext-dev \
&& pip3 install boto3 \
&& rm -rf /var/lib/apt/lists/*
## Setup spack
## parts:
ENV SPACK_ROOT=/opt/spack
ARG SPACK_VERSION="develop"
RUN echo "Part 1: regular spack install (as in containerize)" \
&& git clone https://github.com/spack/spack.git /tmp/spack-staging \
&& cd /tmp/spack-staging && git checkout $SPACK_VERSION && cd - \
&& mkdir -p $SPACK_ROOT/opt/spack \
&& cp -r /tmp/spack-staging/bin $SPACK_ROOT/bin \
&& cp -r /tmp/spack-staging/etc $SPACK_ROOT/etc \
&& cp -r /tmp/spack-staging/lib $SPACK_ROOT/lib \
&& cp -r /tmp/spack-staging/share $SPACK_ROOT/share \
&& cp -r /tmp/spack-staging/var $SPACK_ROOT/var \
&& cp -r /tmp/spack-staging/.git $SPACK_ROOT/.git \
&& rm -rf /tmp/spack-staging \
&& echo 'export LD_LIBRARY_PATH=/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH'\
>> $SPACK_ROOT/share/setup-env.sh \
&& ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/sbin/docker-shell \
&& ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/sbin/interactive-shell \
&& ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/sbin/spack-env \
&& echo "Part 2: Set target to generic x86_64" \
&& echo "packages:" > $SPACK_ROOT/etc/spack/packages.yaml \
&& echo " all:" >> $SPACK_ROOT/etc/spack/packages.yaml \
&& echo " target: [x86_64]" >> $SPACK_ROOT/etc/spack/packages.yaml \
&& cat $SPACK_ROOT/etc/spack/packages.yaml \
&& echo "Part 3: Set config to allow use of more cores for builds" \
&& echo "(and some other settings)" \
&& echo "config:" > $SPACK_ROOT/etc/spack/config.yaml \
&& echo " suppress_gpg_warnings: true" \
>> $SPACK_ROOT/etc/spack/config.yaml \
&& echo " build_jobs: 64" >> $SPACK_ROOT/etc/spack/config.yaml \
&& echo " install_tree:" >> $SPACK_ROOT/etc/spack/config.yaml \
&& echo " root: /opt/software" >> $SPACK_ROOT/etc/spack/config.yaml \
&& cat $SPACK_ROOT/etc/spack/config.yaml
SHELL ["docker-shell"]
## Setup spack buildcache mirrors, including an internal
## spack mirror using the docker build cache, and
## a backup mirror on the internal B010 network
RUN --mount=type=cache,target=/var/cache/spack-mirror \
export PATH=$PATH:$SPACK_ROOT/bin \
&& wget 10.10.241.24/spack-mirror/sodium.pub --no-check-certificate \
&& spack gpg trust sodium.pub \
&& spack mirror add silicon http://10.10.241.24/spack-mirror \
&& spack mirror add docker /var/cache/spack-mirror \
&& spack mirror list
## Setup our custom environment and package overrides
COPY spack $SPACK_ROOT/eic-spack
RUN echo "repos:" > $SPACK_ROOT/etc/spack/repos.yaml \
&& echo " - $SPACK_ROOT/eic-spack" >> $SPACK_ROOT/etc/spack/repos.yaml \
&& mkdir /opt/spack-environment \
&& mv $SPACK_ROOT/eic-spack/spack.yaml /opt/spack-environment/spack.yaml
## This variable will change whenevery either spack.yaml or our spack package
## overrides change, triggering a rebuild
ARG CACHE_BUST="hash"
## Now execute the main build (or fetch from cache if possible)
## note, no-check-signature is needed to allow the quicker signature-less
## packages from the internal (docker) buildcache
RUN --mount=type=cache,target=/var/cache/spack-mirror \
cd /opt/spack-environment \
&& ls /var/cache/spack-mirror \
&& spack env activate . \
&& spack install -j64 --no-check-signature \
&& spack clean -a
## Update the local build cache if needed. Consists of 3 steps:
## 1. Remove the B010 network buildcache
## 2. Get a list of all packages, and compare with what is already on
## the buildcache (using package hash)
## 3. Add packages that need to be added to buildcache if any
RUN --mount=type=cache,target=/var/cache/spack-mirror \
spack buildcache list --allarch --long \
| grep -v -e '---' \
| sed "s/@.\+//" \
| sort > tmp.buildcache.txt \
&& spack find --no-groups --long \
| tail -n +2 \
| grep -v "==>" \
| sed "s/@.\+//" \
| sort > tmp.manifest.txt \
&& comm -23 tmp.manifest.txt tmp.buildcache.txt \
> tmp.needsupdating.txt \
&& if [ $(wc -l < tmp.needsupdating.txt) -ge 1 ]; then \
cat tmp.needsupdating.txt \
| awk '{print($2);}' \
| tr '\n' ' ' \
| xargs spack buildcache create -uaf -d /var/cache/spack-mirror \
&& spack buildcache update-index -d /var/cache/spack-mirror; \
fi \
&& rm tmp.manifest.txt \
&& rm tmp.buildcache.txt \
&& rm tmp.needsupdating.txt
## extra post-spack steps
## Including some small fixes:
## - Somehow PODIO env isn't automatically set,
## - and Gaudi likes BINARY_TAG to be set
RUN cd /opt/spack-environment \
&& echo -n "" \
&& echo "Grabbing environment info" \
&& spack env activate --sh -d . \
> /etc/profile.d/z10_spack_environment.sh \
&& sed -i "s?LD_LIBRARY_PATH=?&/lib/x86_64-linux-gnu:?" \
/etc/profile.d/z10_spack_environment.sh \
&& cd /opt/spack-environment \
&& echo -n "" \
&& echo "Add extra environment variables for Podio and Gaudi" \
&& spack env activate . \
&& export PODIO=`spack find -p podio \
| grep software \
| awk '{print $2}'` \
&& echo "export PODIO=${PODIO};" \
>> /etc/profile.d/z10_spack_environment.sh \
&& echo "export BINARY_TAG=x86_64-linux-gcc9-opt" \
>> /etc/profile.d/z10_spack_environment.sh \
&& cd /opt/spack-environment && spack env activate . \
&& echo -n "" \
&& echo "Installing additional python packages" \
&& pip install --trusted-host pypi.org \
--trusted-host files.pythonhosted.org \
--no-cache-dir \
ipython matplotlib scipy yapf pandas \
jupyter jupyterlab uproot pyunfold seaborn \
&& echo -n "" \
&& echo "Executing cmake patch for dd4hep 16.1" \
&& sed -i "s/FIND_PACKAGE(Python/#&/" /usr/local/cmake/DD4hepBuild.cmake
## make sure we have the entrypoints setup correctly
ENTRYPOINT []
CMD ["bash", "--rcfile", "/etc/profile", "-l"]
USER 0
WORKDIR /
## ========================================================================================
## STAGE 2: staging image with unnecessariy packages removed and stripped binaries
## ========================================================================================
FROM builder as staging
RUN cd /opt/spack-environment && spack env activate . && spack gc -y
# Strip all the binaries
# This reduces the image by factor of x2, so worth the effort
# note that we do not strip python libraries as can cause issues in some cases
RUN find -L /usr/local/* \
-type d -name site-packages -prune -false -o \
-type f -not -name "zdll.lib" \
-exec readlink -f '{}' \; \
| xargs file -i \
| grep 'charset=binary' \
| grep 'x-executable\|x-archive\|x-sharedlib' \
| awk -F: '{print $1}' | xargs strip -s
## ========================================================================================
## STAGE 3
## Lean target image
## ========================================================================================
FROM eicweb.phy.anl.gov:4567/containers/eic_container/debian_base:${INTERNAL_TAG}
LABEL maintainer="Sylvester Joosten <sjoosten@anl.gov>" \
name="jug_xl" \
march="amd64"
## copy over everything we need from staging in a single layer :-)
RUN --mount=from=staging,target=/staging \
rm -rf /usr/local \
&& cp -r /staging/opt/software /opt/software \
&& cp -r /staging/usr/local /usr/local \
&& cp /staging/etc/profile.d/z10_spack_environment.sh /etc/eic-env.sh \
&& sed -i '/MANPATH/ s/;$/:;/' /etc/eic-env.sh \
&& cp /etc/eic-env.sh /etc/profile.d/z10_eic-env.sh
COPY eic-shell /usr/local/bin/eic-shell
## make sure we have the entrypoints setup correctly
ENTRYPOINT []
CMD ["bash", "--rcfile", "/etc/profile", "-l"]
USER 0
WORKDIR /
SHELL ["/usr/local/bin/eic-shell"]
containers/jug/Dockerfile.xl 0 → 100644
+ 68
− 0
View file @ ce01b8f8
#syntax=docker/dockerfile:1.2
ARG INTERNAL_TAG="testing"
## ========================================================================================
## STAGE1: spack builder image
## EIC builder image with spack
## ========================================================================================
FROM eicweb.phy.anl.gov:4567/containers/eic_container/jug_dev:${INTERNAL_TAG}
ARG JUGGLER_VERSION="master"
ARG NPDET_VERSION="master"
ARG EICD_VERSION="master"
## cachebust that can change values to trigger a rebuild, useful for nightly builds
## off the master
ARG CACHEBUST=1
RUN cd /tmp \
&& echo "INSTALLING NPDET" \
&& git clone -b ${NPDET_VERSION} https://eicweb.phy.anl.gov/EIC/NPDet.git \
&& cmake -B build -S NPDet -DCMAKE_CXX_STANDARD=17 \
&& cmake --build build -j12 -- install \
&& rm -rf build NPDet \
&& echo "INSTALLING EICD" \
&& git clone -b ${EICD_VERSION} https://eicweb.phy.anl.gov/EIC/eicd.git \
&& cmake -B build -S eicd -DCMAKE_CXX_STANDARD=17 \
&& cmake --build build -j12 -- install \
&& rm -rf build eicd \
&& echo "INSTALLING JUGGLER" \
&& git clone -b ${JUGGLER_VERSION} https://eicweb.phy.anl.gov/EIC/juggler.git \
&& cmake -B build -S juggler -DCMAKE_CXX_STANDARD=17 \
&& cmake --build build -j12 -- install \
&& rm -rf build juggler
## also install detector/ip geometries into opt
## FIXME: need to add proper compact file install directly to the reference detector
## build
ARG DETECTOR_VERSION="master"
ARG IP6_VERSION="master"
ARG ACCELERATOR_VERSION="master"
RUN cd /tmp \
&& DETECTOR_PREFIX=/opt/detector \
&& DETECTOR_DATA=$DETECTOR_PREFIX/share/reference_detector \
&& mkdir -p /opt/detector/share/reference_detector \
&& echo "INSTALLING REFERENCE DETECTOR" \
&& git clone -b ${DETECTOR_VERSION} \
https://eicweb.phy.anl.gov/EIC/detectors/reference_detector.git \
&& cmake -B build -S reference_detector -DCMAKE_CXX_STANDARD=17 \
-DCMAKE_INSTALL_PREFIX=${DETECTOR_PREFIX} \
&& cmake --build build -j12 -- install \
&& cp -r reference_detector/compact \
reference_detector/reference_detector.xml \
${DETECTOR_DATA} \
&& rm -rf build reference_detector \
&& echo "INSTALLING ACCELERATOR GEOMETRY" \
&& git clone -b ${ACCELERATOR_VERSION} \
https://eicweb.phy.anl.gov/EIC/detectors/accelerator.git \
&& cp -r accelerator/eic \
${DETECTOR_DATA} \
&& rm -rf accelerator \
&& echo "INSTALLING IP6 GEOMETRY" \
&& git clone -b ${IP6_VERSION} \
https://eicweb.phy.anl.gov/EIC/detectors/ip6.git \
&& cmake -B build -S ip6 -DCMAKE_CXX_STANDARD=17 \
-DCMAKE_INSTALL_PREFIX=${DETECTOR_PREFIX} \
&& cmake --build build -j12 -- install \
&& cp -r ip6/ip6 \
${DETECTOR_DATA} \
&& rm -rf build ip6
containers/release/eic-shell → containers/jug/eic-shell
+ 0
− 0
View file @ ce01b8f8
File moved
containers/release/Dockerfile.in deleted 100644 → 0
+ 0
− 82
View file @ 983ac450
# Release container for Argonne EIC software
# Copy over relevant files from builder
#
FROM eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:@TAG@ as builder
RUN cd /opt/spack-environment && spack env activate . && spack gc -y
# Strip all the binaries
# This reduces the image size from 3.2GB --> 1.7GB so worth the time
RUN find -L /usr/local/* \
-type d -name site-packages -prune -false -o \
-type f -not -name "zdll.lib" \
-exec readlink -f '{}' \; | \
xargs file -i | \
grep 'charset=binary' | \
grep 'x-executable\|x-archive\|x-sharedlib' | \
awk -F: '{print $1}' | xargs strip -s
FROM eicweb.phy.anl.gov:4567/containers/image_recipes/debian_minimal:testing-20210408
LABEL maintainer="Sylvester Joosten <sjoosten@anl.gov>" \
name="eic" \
group="eic" \
march="native" \
basedist="debian" \
base="debian"
## @ENV@ will be automatically expanded to auto-load the
## runtime environment
ENV DOCKERFILE_DISTRO=debian \
DOCKERFILE_DISTRO_VERSION=20210408-testing \
DEBIAN_FRONTEND=noninteractive \
@ENV@
## poppler-utils for pdftoppm, is this is right now not supported by the
## Spack version
RUN apt-get -yqq update \
&& apt-get -yqq install --no-install-recommends \
ghostscript \
gv \
poppler-utils \
parallel \
&& apt-get -yqq autoremove \
&& rm -rf /var/lib/apt/lists/*
## Copy over files from builder
COPY --from=builder /opt/spack-environment /opt/spack-environment
COPY --from=builder /opt/software /opt/software
RUN rm -rf /usr/local
COPY --from=builder /usr/local /usr/local
## install gitlab-runner FIXME: needs to be consolidated with other installs
## installing package from buster (stable) instead of bullseye (testing)
RUN curl -L \
"https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" | \
bash \
&& sed -i "s/bullseye/buster/" /etc/apt/sources.list.d/runner_gitlab-runner.list \
&& apt-get update \
&& apt-get -yqq install --no-install-recommends gitlab-runner \
&& apt-get -yqq autoremove \
&& rm -rf /var/lib/apt/lists/*
## eic-shell and environment scripts. Not strictly needed anymore now we auto-load
## the environment but still provided for backward compatibility (and documentation).
COPY eic-shell /usr/local/bin/eic-shell
COPY eic-env.sh /etc/eic-env.sh
COPY eic-env.sh /etc/profile.d/z10_eic-env.s
## also make sure we have the older container_dev command available
COPY eic-shell /usr/local/bin/container_dev
## also make sure we have the older container_dev command available
COPY eic-shell /usr/local/bin/container_dev
## Setup fresh bashrc, useful for singularity
COPY bashrc /etc/bash.bashrc
COPY bashrc /root/.bashrc
## make sure we have the entrypoints setup correctly
ENTRYPOINT []
CMD ["bash", "--rcfile", "/etc/profile", "-l"]
USER 0
WORKDIR /
SHELL ["/usr/local/bin/eic-shell"]
containers/release/config.env deleted 100644 → 0
+ 0
− 11
View file @ 983ac450
REG_HOST ?= eicweb.phy.anl.gov
REG_PORT ?= 4567
REG_NAME ?= $(REG_HOST):$(REG_PORT)
REG_URL ?= https://$(REG_HOST)
APP_NAME = eic
REPO_NAME = eic
GL_GROUP = eic_container
GL_REG_GROUP = containers/eic_container
GL_REG_NAME = $(REG_NAME)
containers/release/configure_dockerfile.sh deleted 100644 → 0
+ 0
− 26
View file @ 983ac450
#!/usr/bin/env bash
## Extract a list of the variable definitions needed
## for this environment from the builder image, and
## then configure the release Dockerfile to set these
## variables
mkdir -p config
cp /etc/profile.d/z10_spack_environment.sh config/spack-env.sh
export TAG=$1
## Spack sets the man-path, which stops bash from using the default man-path
## We can fix this by appending a trailing colon to MANPATH
sed -i '/MANPATH/ s/;$/:;/' config/spack-env.sh
## Extract the desired environment variables as defined by spack
grep export config/spack-env.sh | \
sed 's/export / /' | \
sed 's/;$/ \\/' > config/eic-env.sh
## create our release Dockerfile
sed '/^@ENV@/r config/eic-env.sh' containers/release/Dockerfile.in | \
sed '/^@ENV@/d' | \
sed "s/@TAG@/$TAG/" > config/Dockerfile
containers/release/eic.def.in deleted 100644 → 0
+ 0
− 18
View file @ 983ac450
Bootstrap: docker
From: eicweb.phy.anl.gov:4567/containers/eic_container/eic:@TAG@
%help
EIC software container.
Tools:
- eic_shell : Bash shell in this container
- root : Root shell in the container
- ipython : Python shell in the container
%labels
Maintainer "Whitney Armstrong, Sylvester Joosten"
Version v2.x
%post -c /bin/bash
echo " -------------------------------------------------"
echo " ===> Image setup complete"
echo " -------------------------------------------------"
gitlab-ci/cleanup_registry.sh 0 → 100755
+ 88
− 0
View file @ ce01b8f8
#!/bin/bash
## Remove a specific docker tag from the eicweb registry
function print_the_help {
echo "USAGE: -i image -r reg_id tag "
echo "ARGUMENTS:"
echo " -i,--image Registry image name"
echo " -r,--reg-id Registry image ID (integer number)"
echo " -h,--help Print this message"
echo " positional Tag to remove from registry"
echo ""
echo " Remove a specific docker tag from the eicweb registry, if present on the
registry."
echo ""
echo "EXAMPLE: ./cleanup_registry -i debian_base -r 66 unstable"
exit
}
IMAGE=
REG_ID=
TAG=
while [ $# -gt 0 ]; do
key=$1
case $key in
-i|--image)
IMAGE=$2
shift
shift
;;
-r|--reg-id)
REG_ID=$2
shift
shift
;;
-h|--help)
print_the_help
exit 0
;;
-*)
echo "ERROR: unknown flag: $key"
echo "use --help for more info"
exit 1
;;
*)
if [ ! -z ${TAG} ]; then
echo "ERROR: multiple positional arguments specified"
echo "use --help for more info"
exit 1
fi
TAG=$1
shift
;;
esac
done
if [ -z $IMAGE ]; then
echo "ERROR: no image name given, please use -i <IMAGE>"
print_the_help
exit 1
fi
if [ -z $REG_ID ]; then
echo "ERROR: no image id given, please use -r <REG_ID>"
print_the_help
exit 1
fi
if [ -z $TAG ]; then
echo "ERROR: no tag given, please specify a single tag (positional argument)"
print_the_help
exit 1
fi
echo "Cleaning up eicweb registry for ${IMAGE}:${TAG}"
IMAGE_EXISTS=
docker manifest inspect \
eicweb.phy.anl.gov:4567/containers/eic_container/${IMAGE}:${TAG} \
> /dev/null \
&& IMAGE_EXISTS=1 \
|| echo "Image not found, no cleanup needed"
if [ ! -z ${IMAGE_EXISTS} ]; then
curl --request DELETE --header "PRIVATE-TOKEN: ${REG_CLEANUP_TOKEN}" \
${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/registry/repositories/${REG_ID}/tags/${TAG} \
&& echo "Image removed" \
|| echo "Error trying to remove image"
fi
gitlab-ci/configure.sh deleted 100755 → 0
+ 0
− 68
View file @ 983ac450
#!/bin/bash
## Configure a CI file based on a template file (first and only
## argument to this script).
## Known variables that will be substituted:
## @TAG@ - docker tag used for this build
## @TARGET_BUILDER@ - docker Makefile target for the builder image
## @TARGET_RELEASE@ - docker Makefile target for the release image
## @PUBLISH@ - docker push Makefile target
TEMPLATE_FILE=$1
OUTPUT_FILE=`basename ${TEMPLATE_FILE} .in`
echo "Configuring CI file: ${TEMPLATE_FILE}"
echo "Output will be written to: ${OUTPUT_FILE}"
VERSION=`head -n1 VERSION`
STABLE="${VERSION%.*}-stable"
## Figure out which scenario we are running:
## - master
## - stable
## - tag
## - unstable (default)
TARGET=""
TAG=""
PUBLISH=""
if [ "$CI_COMMIT_BRANCH" = "master" ]; then
TARGET="stable"
TAG="latest"
PUBLISH="publish-latest publish-stable"
elif [ "$CI_COMMIT_TAG" = "v${VERSION}" ]; then
TARGET="stable"
TAG=$VERSION
PUBLISH="publish-version"
elif [ "$CI_COMMIT_BRANCH" = "v${STABLE}" ]; then
TARGET="stable"
TAG=${STABLE}
PUBLISH="publish-stable"
else
TARGET="unstable"
TAG="unstable"
PUBLISH="publish-unstable"
fi
TARGET_BUILDER=$TARGET
TARGET_RELEASE=$TARGET
if [ ! -f .ci-env/builder-nc ]; then
echo "Can use cached build directive for builder"
TARGET_BUILDER="${TARGET_BUILDER}-cached"
else
echo "No-cache required for builder"
fi
if [ ! -f .ci-env/release-nc ]; then
echo "Can use cached build directive for release"
TARGET_RELEASE="${TARGET_RELEASE}-cached"
else
echo "No-cache required for release"
fi
sed "s/@TAG@/$TAG/g" $TEMPLATE_FILE | \
sed "s/@TARGET_BUILDER@/$TARGET_BUILDER/g" | \
sed "s/@TARGET_RELEASE@/$TARGET_RELEASE/g" | \
sed "s/@PUBLISH@/$PUBLISH/g" > ${OUTPUT_FILE}
echo "Done"
gitlab-ci/docker/Makefile deleted 100644 → 0
+ 0
− 90
View file @ 983ac450
# import config.
# You can change the default config with `make cnf="config_special.env" build`
cnf ?= config.env
include $(cnf)
# exports variables in config.env as environment variables
export $(shell sed 's/=.*//' $(cnf))
# import deploy config
# You can change the default deploy config with `make cnf="deploy_special.env" release`
SHELL = bash
## Get our version tag and stable version tag
VERSION=$(shell bash version.sh)
STABLE=$(shell bash version-stable.sh)
# help will output the help for each task
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.PHONY: help
help: ## This help.
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
.DEFAULT_GOAL := help
# ==========================================================================
#
build: ## build the image
DOCKER_BUILDKIT=1 docker build -t $(APP_NAME):$(VERSION) .
build-unstable: ## build the image
DOCKER_BUILDKIT=1 docker build -t $(APP_NAME):$(VERSION)-unstable .
build-nc: ## Build the container without caching (from scratch)
DOCKER_BUILDKIT=1 docker build --no-cache -t $(APP_NAME):$(VERSION) .
build-unstable-nc: ## Build the container without caching (from scratch)
DOCKER_BUILDKIT=1 docker build --no-cache -t $(APP_NAME):$(VERSION)-unstable .
# ==========================================================================
#
login: ## Auto login to AWS-ECR unsing aws-cli
docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
echo "Login COMPLETE"
stable: build-nc
stable-cached: build
unstable: build-unstable-nc
unstable-cached: build-unstable
publish: login publish-latest publish-version #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
@echo "Publishing done"
publish-stable: login
@echo 'publish $(STABLE) to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):$(VERSION) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(STABLE)
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(STABLE)
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(STABLE) || true
publish-latest: login
echo 'publish latest to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):$(VERSION) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest || true
publish-version: login
@echo 'publish $(STABLE) to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):$(VERSION) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION) || true
publish-unstable: login
@echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):$(VERSION)-unstable $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable || true
version: ## Output the current version
@echo $(VERSION)
# ==========================================================================
#
info: ## Output the current version
@echo 'VERSION = $(VERSION) '
@echo 'STABLE = $(STABLE) '
@echo 'REG_NAME = $(REG_NAME) '
@echo 'APP_NAME = $(APP_NAME) '
@echo 'GL_GROUP = $(GL_GROUP) '
@echo 'GL_REG_GROUP = $(GL_REG_GROUP)'
@echo 'GL_REG_NAME = $(GL_REG_NAME) '
gitlab-ci/docker/version-stable.sh deleted 100644 → 0
+ 0
− 6
View file @ 983ac450
#!/bin/bash
## print the stable version tag for this version to the console
VERSION=`head -n1 ../../VERSION`
echo "${VERSION%.*}-stable"
gitlab-ci/docker/version.sh deleted 100644 → 0
+ 0
− 5
View file @ 983ac450
#!/bin/bash
## print the full version number to the console
head -n1 ../../VERSION
gitlab-ci/docker_login.sh 0 → 100755
+ 109
− 0
View file @ ce01b8f8
#!/bin/bash
## Resilient docker login with multiple progressive retries in case of TLS issues
function print_the_help {
echo "USAGE: -u user -p -password [...] [registry]"
echo "ARGUMENTS:"
echo " -u,--user Input image name (e.g., eic_base)"
echo " -p,--password Input tag (eg., v3.0.0)"
echo " --ci Login to internal CI registry; -u, -p and positional are ignored"
echo " -t,--time Time interval (in seconds) between attempts"
echo " (doubled each time), default: 5"
echo " -n,--n-attempts Number of attempts, default: 5"
echo " -h,--help Print this message"
echo " positional Registry name (will use dockerhub if none given)"
echo ""
echo " Login to registry."
echo ""
echo "EXAMPLE: ./docker_login.sh -u \$CI_REGISTRY_USER -p \$CI_REGISTRY_PASSWORD \$CI_REGISTRY"
exit
}
REGISTRY=
REG_USER=
REG_PASSWORD=
NTRIES=5
TIME=5
CI=
while [ $# -gt 0 ]; do
key=$1
case $key in
-u|--user)
REG_USER=$2
shift
shift
;;
-p|--password)
REG_PASSWORD=$2
shift
shift
;;
--ci)
CI=1
shift
;;
-t|--time)
TIME=$2
shift
shift
;;
-n|--n-attempts)
NTRIES=$2
shift
shift
;;
-h|--help)
print_the_help
exit 0
;;
-*)
echo "ERROR: unknown flag: $key"
echo "use --help for more info"
exit 1
;;
*)
REGISTRY+="$1"
shift
;;
esac
done
## only sanizize input if not in CI registry mode
if [ -z $CI ]; then
if [ -z $REG_USER ]; then
echo "ERROR: no username given, please use -u <USER>"
print_the_help
exit 1
fi
if [ -z $REG_PASSWORD ]; then
echo "ERROR: no password given, please use -p <PASSWORD>"
print_the_help
exit 1
fi
else
REGISTRY=$CI_REGISTRY
REG_USER=$CI_REGISTRY_USER
fi
while [ $NTRIES != 0 ]; do
if [ -z $CI ]; then
echo $REG_PASSWORD | docker login -u $REG_USER --password-stdin \
&& break \
|| echo "Login failed, retrying in $TIME seconds..."
else
echo $CI_REGISTRY_PASSWORD | docker login -u $REG_USER \
--password-stdin $REGISTRY \
&& break \
|| echo "Login failed, retrying in $TIME seconds..."
fi
sleep ${TIME}s
TIME=$((TIME * 2))
NTRIES=$((NTRIES - 1))
done
if [ $NTRIES = 0 ]; then
echo "Failed to login to registry $REGISTRY"
exit 1
fi
echo "Docker login to $REGISTRY successful"
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment