Skip to content
Snippets Groups Projects
Commit d24ab146 authored by Sylvester Joosten's avatar Sylvester Joosten
Browse files

Re-factor repository workflow according to #13.

Also rolled back ACTS/Gaudi/python to their v2.3.x versions as this is the last version that works with our downstream software. We can re-upgrade after we set this stable baseline with the new CI implementation.
parent 05244bad
No related branches found
No related tags found
No related merge requests found
Showing
with 276 additions and 671 deletions
image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
stages: stages:
- builder
- config - config
- slim - trigger
- singularity
## We use minimal file artifacts to transport variables between job ## make note if we cannot use caching for one of the stages
## stages, such as the branch we are working on and the need for ## by touching files in .ci_env
## caching.
## By evaluating these files in the before_script, we can set
## relevant environment variables before our pipelines run
default: default:
tags:
- silicon
artifacts: artifacts:
paths: paths:
- .ci-env - .ci-env
before_script: detect_changes:builder:
- mkdir -p .ci-env
- |
if [ -f .ci-env/release ]; then
export BRANCH='release'
elif [ -f .ci-env/develop ]; then
export BRANCH='develop'
else export BRANCH='UNKNOWN'
fi
- |
if [ -f .ci-env/builder-nc ]; then
export BUILDER_TARGET="${BRANCH}"
else
export BUILDER_TARGET="${BRANCH}-cached"
fi
- |
if [ -f .ci-env/release-nc ]; then
export RELEASE_TARGET="${BRANCH}"
else
export RELEASE_TARGET="${BRANCH}-cached"
fi
tags:
- silicon
## Stable or unstable branch?
init:stable:
stage: .pre
rules:
- if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == "master"'
when: always
script:
- touch .ci-env/release
init:unstable:
stage: .pre
rules:
- if: '$CI_COMMIT_TAG == null && $CI_COMMIT_BRANCH != "master"'
when: always
script:
- touch .ci-env/develop
init:builder-nc:
stage: .pre stage: .pre
rules: rules:
- changes: - changes:
- gitlab-ci/docker/*
- containers/builder/Dockerfile - containers/builder/Dockerfile
- containers/builder/spack.yaml - containers/builder/spack.yaml
- spack/packages/**/* - spack/packages/**/*
script: script:
- mkdir -p .ci-env
- touch .ci-env/builder-nc - touch .ci-env/builder-nc
init:release-nc: detect_changes:release:
stage: .pre stage: .pre
rules: rules:
- changes: - changes:
- gitlab-ci/docker/*
- gitlab-ci/singularity/*
- containers/release/Dockerfile.in - containers/release/Dockerfile.in
- containers/release/configure_release.sh - containers/release/configure_release.sh
script: script:
- mkdir -p .ci-env
- touch .ci-env/release-nc - touch .ci-env/release-nc
builder: ## Init our job for our desired branches/tags/events
stage: builder init:
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
when: on_success
- if: '$CI_COMMIT_BRANCH == "develop"'
when: on_success
- if: '$CI_COMMIT_TAG'
when: on_success
- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH != "develop"'
when: on_success
script:
- cp -r spack containers/builder/np-spack
- cd containers/builder
- head Dockerfile
- make login
- echo "Creating builder image for ${BUILDER_TARGET}"
- make ${BUILDER_TARGET}
.config:
stage: config stage: config
script: script:
- bash containers/release/configure_release.sh ${BRANCH} - ./gitlab-ci/configure_pipeline.sh gitlab-ci/build_and_deploy.yml.in
artifacts: artifacts:
paths: paths:
- config - build_and_deploy.yml
config:stable:
extends: .config
image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:latest
only:
- tags
- master
config:unstable:
extends: .config
image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:unstable
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH != "develop"'
when: manual
- if: '$CI_COMMIT_BRANCH == "develop"'
when: on_success
release:
stage: slim
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
when: on_success
- if: '$CI_COMMIT_BRANCH == "develop"'
when: on_success
- if: '$CI_COMMIT_TAG'
when: on_success
- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH != "develop"'
when: manual
script:
- cp config/Dockerfile containers/release/Dockerfile
- cp config/eic-env.sh containers/release/eic-env.sh
- cd containers/release
- make login
- make ${RELEASE_TARGET}
release:singularity: ## Dispatch if we ran the previous stage
stage: singularity run:default:
rules: stage: trigger
- if: '$CI_COMMIT_BRANCH == "master"' trigger:
when: manual include:
- if: '$CI_COMMIT_BRANCH == "develop"' - artifact: build_and_deploy.yml
when: manual job: init
- if: '$CI_COMMIT_TAG' strategy: depend
when: on_success
- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH != "develop"'
when: manual
script:
- cp config/eic.def eic.def
- /bin/bash .gitlabci/setup.sh
- /bin/bash .gitlabci/build.sh eic.def
- mkdir -p build
- cp eic.sif build/.
- cp eic.def build/.
artifacts:
expire_in: 90 days
paths:
- build/eic.sif
- build/eic.def
#!/bin/bash
#apt-get update && apt-get install -y wget git \
# build-essential \
# squashfs-tools \
# libtool \
# autotools-dev \
# libarchive-dev \
# automake \
# autoconf \
# uuid-dev \
# libssl-dev \
# python-pip
#
#
### Somehow sed cant find /etc/sudoers
##sed -i -e 's/^Defaults\tsecure_path.*$//' /etc/sudoers
#
## Check Python
#
#echo "Python Version:"
#python --version
#pip install sregistry[all]
#sregistry version
#
#echo "sregistry Version:"
#
## Install Singularity
#
#cd /tmp && \
# rm -rf singularity && \
# git clone -b vault/release-2.5 https://www.github.com/sylabs/singularity.git
# cd singularity && \
# ./autogen.sh && \
# ./configure --prefix=/usr/local && \
# make -j8 && make install
.gitlabci/sregistry-gitlab.png

37.4 KiB

File deleted
...@@ -10,14 +10,14 @@ git clone https://eicweb.phy.anl.gov/containers/eic_container.git ...@@ -10,14 +10,14 @@ git clone https://eicweb.phy.anl.gov/containers/eic_container.git
cd eic_container cd eic_container
``` ```
2. Run the deploy script `deploy.py` to install to your `<PREFIX>` of choice 2. Run the install script `install.py` to install to your `<PREFIX>` of choice
(e.g. $HOME/local/opt/eic_container_1.0.4). By default the (e.g. $HOME/local/opt/eic_container_1.0.4). By default the
modeuefile will be installed to `$PREFIX/../../etc/modulefiles`. modeuefile will be installed to `$PREFIX/../../etc/modulefiles`.
You can use the `-v` flag to select the version you want to deploy, or omit the You can use the `-v` flag to select the version you want to install, or omit the
flag if you want to install the master build. The recommended stable flag if you want to install the master build. The recommended stable
release version is `v2.5.0`. release version is `v2.5.0`.
```bash ```bash
./deploy.py -v 2.5.0 <PREFIX> ./install.py -v 2.5.0 <PREFIX>
``` ```
Available flags: Available flags:
...@@ -29,7 +29,7 @@ cd eic_container ...@@ -29,7 +29,7 @@ cd eic_container
-m MODULE_PATH, --module-path MODULE_PATH -m MODULE_PATH, --module-path MODULE_PATH
(opt.) Root module path where you want to install a (opt.) Root module path where you want to install a
modulefile. D: <prefix>/../../etc/modulefiles modulefile. D: <prefix>/../../etc/modulefiles
-l, --local Local deploy, will not install the modulefiles (you will have -l, --local Local install, will not install the modulefiles (you will have
to run the launcher scripts from their relative paths). to run the launcher scripts from their relative paths).
-f, --force Force-overwrite already downloaded container with the same name. -f, --force Force-overwrite already downloaded container with the same name.
``` ```
...@@ -41,16 +41,16 @@ cd eic_container ...@@ -41,16 +41,16 @@ cd eic_container
module load eic_container module load eic_container
``` ```
4. To use the container in local mode, you can deploy the container with the `-l` flag, 4. To use the container in local mode, you can install the container with the `-l` flag,
and then use the runscripts (under `$PREFIX/bin`) manually. and then use the runscripts (under `$PREFIX/bin`) manually.
```bash ```bash
./deploy.py $PREFIX -l ./install.py $PREFIX -l
... ...
$PREFIX/bin/eic-shell $PREFIX/bin/eic-shell
``` ```
4. (Advanced) If you need to add additional bind directives for the internal singularity container, 4. (Advanced) If you need to add additional bind directives for the internal singularity container,
you can add them with the `-b` flag. Run `./deploy.py -h` to see a list of all you can add them with the `-b` flag. Run `./install.py -h` to see a list of all
supported options. supported options.
Usage Usage
...@@ -75,9 +75,9 @@ eic-shell ...@@ -75,9 +75,9 @@ eic-shell
### B. Running the singularity development locally (without modulefiles) ### B. Running the singularity development locally (without modulefiles)
1. This is assuming you deployed with the `-l` flag to a prefix `$PREFIX`: 1. This is assuming you installed with the `-l` flag to a prefix `$PREFIX`:
```bash ```bash
./deploy.py $PREFIX ./install.py $PREFIX
``` ```
2. To start a shell in the container environment, do 2. To start a shell in the container environment, do
...@@ -109,7 +109,7 @@ Included software: ...@@ -109,7 +109,7 @@ Included software:
- heppdt@3.04.01 - heppdt@3.04.01
- clhep@2.4.1.3 - clhep@2.4.1.3
- eigen@3.3.8 - eigen@3.3.8
- python@3.8.6 with pip, numpy, pyyaml, pyafp, matplotlib, ipython, scipy - python@3.7.8 with pip, numpy, pyyaml, pyafp, matplotlib, ipython, scipy
- xrootd@4.12.3 - xrootd@4.12.3
- root@6.22.00 - root@6.22.00
- pythia8@8303 - pythia8@8303
...@@ -118,8 +118,8 @@ Included software: ...@@ -118,8 +118,8 @@ Included software:
- podio@master - podio@master
- geant4@10.6.2 - geant4@10.6.2
- dd4hep@1.14.1 - dd4hep@1.14.1
- acts@3.00.0 - acts@1.00.0
- gaudi@35.0 - gaudi@34.0
- dawn@3.91a - dawn@3.91a
- dawncut@1.54a - dawncut@1.54a
- The singularity build exports the following applications: - The singularity build exports the following applications:
......
# import config.
# You can change the default config with `make cnf="config_special.env" build`
cnf ?= config.env
include $(cnf)
# exports variables in config.env as environment variables
export $(shell sed 's/=.*//' $(cnf))
# import deploy config
# You can change the default deploy config with `make cnf="deploy_special.env" release`
SHELL = bash
# grep the version from the mix file
VERSION=$(shell bash version.sh)
# help will output the help for each task
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.PHONY: help
help: ## This help.
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
.DEFAULT_GOAL := help
# ==========================================================================
#
build: ## build the image
docker build -t $(APP_NAME) .
build-nc: ## Build the container without caching (from scratch)
docker build --no-cache -t $(APP_NAME) .
build-alt: ## build the container for various machine architectures (broadwell, haswell, knl)
@echo 'building for architecture: $(ALT_NAME)'
docker build -t $(APP_NAME)_$(ALT_NAME) -f Dockerfile.$(ALT_NAME) .
build-alt-nc: ## build the container for various machine architectures (broadwell, haswell, knl)
@echo 'build-alt-nc: building for architecture: $(ALT_NAME)'
docker build --no-cache -t $(APP_NAME)_$(ALT_NAME) -f Dockerfile.$(ALT_NAME) .
# ==========================================================================
#
run: ## Run container on port configured in `config.env`
docker run -i -t --rm --env-file=./config.env -p=$(PORT):$(PORT) --name="$(APP_NAME)" $(REPO)/$(APP_NAME):$(TAG_VERSION)
up: build run ## Run container on port configured in `config.env` (Alias to run)
stop: ## Stop and remove a running container
docker stop $(APP_NAME); docker rm $(APP_NAME)
# ==========================================================================
#
tag: tag-latest tag-version #tag-version ## Generate container tags for the `{version}` and `latest` tags
tag-latest: ## Generate container `{version}` tag
@echo 'create tag latest'
#docker tag $(APP_NAME) $(REPO)/$(APP_NAME):latest
tag-version: ## Generate container `latest` tag
@echo 'creating tag $(APP_NAME):$(VERSION)'
docker tag $(APP_NAME):latest $(APP_NAME):$(VERSION)
tag-alt: tag-alt-latest # tag-alt-version ## Generate container tags for the `{version}` ans `latest` tags
tag-alt-latest: ## Generate container `{version}` tag
@echo 'create tag latest'
#docker tag $(APP_NAME)_$(ALT_NAME) $(REPO)/$(APP_NAME)_$(ALT_NAME):latest
tag-alt-version: ## Generate container `{version}` tag
@echo 'create tag latest'
docker tag $(APP_NAME)_$(ALT_NAME) $(REG_NAME)/$(REPO)/$(APP_NAME)_$(ALT_NAME):$(VERSION)
# ==========================================================================
#
login: ## Auto login to AWS-ECR unsing aws-cli
docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
echo "Login COMPLETE"
release: build-nc publish ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
@echo "Cleaning up"
docker rmi $(APP_NAME):latest
develop: build-nc publish-dev ## Develop build, do not publish latest
@echo "Cleaning up"
#docker rmi $(APP_NAME):latest
staging: build-nc publish-staging ## Develop build, do not publish latest
@echo "Cleaning up"
#docker rmi $(APP_NAME):latest
release-cached: build publish ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
develop-cached: build login publish-dev ## develop release
staging-cached: build login publish-staging ## develop release
publish: login publish-latest publish-version #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
@echo "Publishing done"
publish-latest: ## Publish the `latest` taged container to ECR
@echo 'publish latest to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
#docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
publish-staging: ## Publish the `unstable` taged container to ECR
@echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):testing
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):testing
publish-dev: ## Publish the `unstable` taged container to ECR
@echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable
publish-version: tag-version ## Publish the `{version}` taged container to ECR
@echo 'publish latest to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
#docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
push: login ## push after login @echo 'push latest to $(REG_NAME)/$(REPO)/$(APP_NAME):latest'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
# ==========================================================================
#
release-alt: build-alt publish-alt ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
publish-alt: login publish-alt-latest #publish-alt-version ## Publish the `{version}` ans `latest` tagged containers to ECR
push-alt: login tag-alt-latest ## push after login
#docker push $(REG_NAME)/$(REPO)/$(APP_NAME)_$(ALT_NAME):latest
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):latest
publish-alt-latest: #tag-alt-latest ## Publish the `latest` taged container to ECR
#docker push $(REG_NAME)/$(REPO)/$(APP_NAME)_$(ALT_NAME):latest
docker tag $(APP_NAME)_$(ALT_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):latest
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):latest
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):latest
publish-alt-version: tag-alt-version ## Publish the `latest` taged container to ECR
#docker push $(REG_NAME)/$(REPO)/$(APP_NAME)_$(ALT_NAME):$(VERSION)
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):$(VERSION)
# ==========================================================================
#
login-dockerhub: ## login to hub.docker.com
@echo 'docker hub login :'
docker login
push-dockerhub: build #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
@echo '$(DH_ORG)/$(APP_NAME):$(TAG_VERSION)'
docker push $(DH_ORG)/$(APP_NAME):latest
#docker push $(DH_ORG)/$(APP_NAME):$(VERSION)
publish-dockerhub: build tag #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
@echo '$(DH_ORG)/$(APP_NAME):$(TAG_VERSION)'
docker push $(DH_ORG)/$(APP_NAME):latest
#docker push $(DH_ORG)/$(APP_NAME):$(VERSION)
# ==========================================================================
#
clean-tags:
docker rmi $(REPO)/$(APP_NAME):latest || true
docker rmi $(GL_GROUP)/$(APP_NAME):latest || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME):latest || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME):latest || true
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest || true
docker rmi $(REPO)/$(APP_NAME)_$(ALT_NAME):latest || true
docker rmi $(GL_GROUP)/$(APP_NAME)_$(ALT_NAME):latest || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):latest || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME)_$(ALT_NAME):latest || true
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):latest || true
clean:
docker rmi $(REPO_NAME)/$(APP_NAME) || true
docker rmi $(REPO_NAME)/$(APP_NAME):$(TAG_VERSION) || true
docker rmi $(REPO_NAME)/$(APP_NAME):$(VERSION) || true
docker rmi $(REPO)/$(APP_NAME) || true
docker rmi $(REPO)/$(APP_NAME):$(TAG_VERSION) || true
docker rmi $(REPO)/$(APP_NAME):$(VERSION) || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME):latest || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME):$(TAG_VERSION) || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME) || true
docker rmi $(DH_ORG)/$(APP_NAME) || true
docker rmi $(DH_ORG)/$(APP_NAME):$(TAG_VERSION) || true
docker rmi $(DH_ORG)/$(APP_NAME):$(VERSION) || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION) || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME) || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):latest || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):$(VERSION) || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME) || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME) || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME):$(VERSION) || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME) || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):$(VERSION) || true
docker rmi $(GL_REG_NAME)/$(REG_USER)/$(APP_NAME) || true
docker rmi $(GL_REG_NAME)/$(REG_USER)/$(APP_NAME):$(VERSION) || true
#docker rmi $(APP_NAME) || true
version: ## Output the current version
@echo $(VERSION)
# ==========================================================================
#
info: ## Output the current version
@echo 'VERSION = $(VERSION) '
@echo 'REG_USER = $(REG_USER) '
@echo 'REG_NAME = $(REG_NAME) '
@echo 'ALT_NAME = $(ALT_NAME) '
@echo 'APP_NAME = $(APP_NAME) '
@echo 'REPO_NAME = $(REPO_NAME) '
@echo 'DH_ORG = $(DH_ORG) '
@echo 'GL_GROUP = $(GL_GROUP) '
@echo 'GL_REG_GROUP = $(GL_REG_GROUP)'
@echo 'GL_REG_NAME = $(GL_REG_NAME) '
@echo 'REPO = $(REPO) '
@echo 'TAG_VERSION = $(TAG_VERSION) '
ls: ## list all docker images
docker images
REG_TOKEN ?= ${CI_IMAGE_BUILD_PAT}
REG_USER ?= whit
REG_HOST ?= eicweb.phy.anl.gov REG_HOST ?= eicweb.phy.anl.gov
REG_NAME ?= eicweb.phy.anl.gov:4567
REG_PORT ?= 4567 REG_PORT ?= 4567
REG_NAME ?= $(REG_HOST):$(REG_PORT)
REG_URL ?= https://$(REG_HOST) REG_URL ?= https://$(REG_HOST)
# name of alternate build:
# Dockerfile.$(ALT_NAME) --> $(APP_NAME)_${ALT_NAME}
ALT_NAME ?= broadwell
APP_NAME = eic_builder APP_NAME = eic_builder
REPO_NAME = eic_builder REPO_NAME = eic_builder
DH_ORG = eic
GL_GROUP = eic_container GL_GROUP = eic_container
GL_REG_GROUP = containers/eic_container GL_REG_GROUP = containers/eic_container
GL_REG_NAME = $(REG_NAME) GL_REG_NAME = $(REG_NAME)
REPO = $(REG_USER)
TAG_VERSION = latest
Bootstrap: docker
From: eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:latest
%help
EIC software container.
Tools:
Tools:
- eic_shell : Bash shell in this container
- root : Root shell in the container
- ipython : Python shell in the container
%labels
Maintainer "Whitney Armstrong, Sylvester Joosten"
Version v2.0
%post -c /bin/bash
echo " -------------------------------------------------"
echo " ===> Image setup complete"
echo " -------------------------------------------------"
...@@ -7,7 +7,7 @@ spack: ...@@ -7,7 +7,7 @@ spack:
- heppdt@3.04.01 - heppdt@3.04.01
- clhep@2.4.1.3 cxxstd=17 - clhep@2.4.1.3 cxxstd=17
- eigen@3.3.8 - eigen@3.3.8
- python@3.8.6 - python@3.7.8
- py-pip - py-pip
- py-numpy - py-numpy
- py-pyyaml - py-pyyaml
...@@ -23,8 +23,8 @@ spack: ...@@ -23,8 +23,8 @@ spack:
- podio@master - podio@master
- geant4@10.6.2 cxxstd=17 +opengl +vecgeom +x11 +qt ^qt +opengl - geant4@10.6.2 cxxstd=17 +opengl +vecgeom +x11 +qt ^qt +opengl
- dd4hep@1.14.1 +geant4 +assimp +hepmc3 +ipo - dd4hep@1.14.1 +geant4 +assimp +hepmc3 +ipo
- acts@3.00.0 +dd4hep +digitization +identification +json +tgeo +ipo - acts@1.02.0 +dd4hep +digitization +identification +json +tgeo +ipo
- gaudi@35.0 - gaudi@34.0
- dawn@3_91a - dawn@3_91a
- dawncut@1_54a - dawncut@1_54a
concretization: together concretization: together
......
# import config.
# You can change the default config with `make cnf="config_special.env" build`
cnf ?= config.env
include $(cnf)
# exports variables in config.env as environment variables
export $(shell sed 's/=.*//' $(cnf))
# import deploy config
# You can change the default deploy config with `make cnf="deploy_special.env" release`
SHELL = bash
# grep the version from the mix file
VERSION=$(shell bash version.sh)
# help will output the help for each task
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.PHONY: help
help: ## This help.
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
.DEFAULT_GOAL := help
# ==========================================================================
#
build: ## build the image
docker build -t $(APP_NAME) .
build-nc: ## Build the container without caching (from scratch)
docker build --no-cache -t $(APP_NAME) .
build-alt: ## build the container for various machine architectures (broadwell, haswell, knl)
@echo 'building for architecture: $(ALT_NAME)'
docker build -t $(APP_NAME)_$(ALT_NAME) -f Dockerfile.$(ALT_NAME) .
build-alt-nc: ## build the container for various machine architectures (broadwell, haswell, knl)
@echo 'build-alt-nc: building for architecture: $(ALT_NAME)'
docker build --no-cache -t $(APP_NAME)_$(ALT_NAME) -f Dockerfile.$(ALT_NAME) .
# ==========================================================================
#
run: ## Run container on port configured in `config.env`
docker run -i -t --rm --env-file=./config.env -p=$(PORT):$(PORT) --name="$(APP_NAME)" $(REPO)/$(APP_NAME):$(TAG_VERSION)
up: build run ## Run container on port configured in `config.env` (Alias to run)
stop: ## Stop and remove a running container
docker stop $(APP_NAME); docker rm $(APP_NAME)
# ==========================================================================
#
tag: tag-latest tag-version #tag-version ## Generate container tags for the `{version}` and `latest` tags
tag-latest: ## Generate container `{version}` tag
@echo 'create tag latest'
#docker tag $(APP_NAME) $(REPO)/$(APP_NAME):latest
tag-version: ## Generate container `latest` tag
@echo 'creating tag $(APP_NAME):$(VERSION)'
docker tag $(APP_NAME):latest $(APP_NAME):$(VERSION)
tag-alt: tag-alt-latest # tag-alt-version ## Generate container tags for the `{version}` ans `latest` tags
tag-alt-latest: ## Generate container `{version}` tag
@echo 'create tag latest'
#docker tag $(APP_NAME)_$(ALT_NAME) $(REPO)/$(APP_NAME)_$(ALT_NAME):latest
tag-alt-version: ## Generate container `{version}` tag
@echo 'create tag latest'
docker tag $(APP_NAME)_$(ALT_NAME) $(REG_NAME)/$(REPO)/$(APP_NAME)_$(ALT_NAME):$(VERSION)
# ==========================================================================
#
login: ## Auto login to AWS-ECR unsing aws-cli
docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
echo "Login COMPLETE"
release: build-nc publish ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
@echo "Cleaning up"
#docker rmi $(APP_NAME):latest
staging: build-nc publish-staging ## Develop build, do not publish latest
@echo "Cleaning up"
#docker rmi $(APP_NAME):latest
develop: build-nc publish-dev ## Develop build, do not publish latest
@echo "Cleaning up"
#docker rmi $(APP_NAME):latest
release-cached: build publish ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
staging-cached: build login publish-staging ## staging release
develop-cached: build login publish-dev ## develop release
publish: login publish-latest publish-version #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
@echo "Publishing done"
publish-latest: ## Publish the `latest` taged container to ECR
@echo 'publish latest to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
#docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
publish-staging: ## Publish the `unstable` taged container to ECR
@echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):testing
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):testing
publish-dev: ## Publish the `unstable` taged container to ECR
@echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable
publish-version: tag-version ## Publish the `{version}` taged container to ECR
@echo 'publish latest to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
#docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
push: login ## push after login @echo 'push latest to $(REG_NAME)/$(REPO)/$(APP_NAME):latest'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
# ==========================================================================
#
release-alt: build-alt publish-alt ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
publish-alt: login publish-alt-latest #publish-alt-version ## Publish the `{version}` ans `latest` tagged containers to ECR
push-alt: login tag-alt-latest ## push after login
#docker push $(REG_NAME)/$(REPO)/$(APP_NAME)_$(ALT_NAME):latest
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):latest
publish-alt-latest: #tag-alt-latest ## Publish the `latest` taged container to ECR
#docker push $(REG_NAME)/$(REPO)/$(APP_NAME)_$(ALT_NAME):latest
docker tag $(APP_NAME)_$(ALT_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):latest
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):latest
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):latest
publish-alt-version: tag-alt-version ## Publish the `latest` taged container to ECR
#docker push $(REG_NAME)/$(REPO)/$(APP_NAME)_$(ALT_NAME):$(VERSION)
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):$(VERSION)
# ==========================================================================
#
login-dockerhub: ## login to hub.docker.com
@echo 'docker hub login :'
docker login
push-dockerhub: build #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
@echo '$(DH_ORG)/$(APP_NAME):$(TAG_VERSION)'
docker push $(DH_ORG)/$(APP_NAME):latest
#docker push $(DH_ORG)/$(APP_NAME):$(VERSION)
publish-dockerhub: build tag #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
@echo '$(DH_ORG)/$(APP_NAME):$(TAG_VERSION)'
docker push $(DH_ORG)/$(APP_NAME):latest
#docker push $(DH_ORG)/$(APP_NAME):$(VERSION)
# ==========================================================================
#
clean-tags:
docker rmi $(REPO)/$(APP_NAME):latest || true
docker rmi $(GL_GROUP)/$(APP_NAME):latest || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME):latest || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME):latest || true
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest || true
docker rmi $(REPO)/$(APP_NAME)_$(ALT_NAME):latest || true
docker rmi $(GL_GROUP)/$(APP_NAME)_$(ALT_NAME):latest || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):latest || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME)_$(ALT_NAME):latest || true
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):latest || true
clean:
docker rmi $(REPO_NAME)/$(APP_NAME) || true
docker rmi $(REPO_NAME)/$(APP_NAME):$(TAG_VERSION) || true
docker rmi $(REPO_NAME)/$(APP_NAME):$(VERSION) || true
docker rmi $(REPO)/$(APP_NAME) || true
docker rmi $(REPO)/$(APP_NAME):$(TAG_VERSION) || true
docker rmi $(REPO)/$(APP_NAME):$(VERSION) || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME):latest || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME):$(TAG_VERSION) || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME) || true
docker rmi $(DH_ORG)/$(APP_NAME) || true
docker rmi $(DH_ORG)/$(APP_NAME):$(TAG_VERSION) || true
docker rmi $(DH_ORG)/$(APP_NAME):$(VERSION) || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION) || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME) || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):latest || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):$(VERSION) || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME) || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME) || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME):$(VERSION) || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME) || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME)_$(ALT_NAME):$(VERSION) || true
docker rmi $(GL_REG_NAME)/$(REG_USER)/$(APP_NAME) || true
docker rmi $(GL_REG_NAME)/$(REG_USER)/$(APP_NAME):$(VERSION) || true
#docker rmi $(APP_NAME) || true
version: ## Output the current version
@echo $(VERSION)
# ==========================================================================
#
info: ## Output the current version
@echo 'VERSION = $(VERSION) '
@echo 'REG_USER = $(REG_USER) '
@echo 'REG_NAME = $(REG_NAME) '
@echo 'ALT_NAME = $(ALT_NAME) '
@echo 'APP_NAME = $(APP_NAME) '
@echo 'REPO_NAME = $(REPO_NAME) '
@echo 'DH_ORG = $(DH_ORG) '
@echo 'GL_GROUP = $(GL_GROUP) '
@echo 'GL_REG_GROUP = $(GL_REG_GROUP)'
@echo 'GL_REG_NAME = $(GL_REG_NAME) '
@echo 'REPO = $(REPO) '
@echo 'TAG_VERSION = $(TAG_VERSION) '
ls: ## list all docker images
docker images
REG_TOKEN ?= ${CI_IMAGE_BUILD_PAT}
REG_USER ?= whit
REG_HOST ?= eicweb.phy.anl.gov REG_HOST ?= eicweb.phy.anl.gov
REG_NAME ?= eicweb.phy.anl.gov:4567
REG_PORT ?= 4567 REG_PORT ?= 4567
REG_NAME ?= $(REG_HOST):$(REG_PORT)
REG_URL ?= https://$(REG_HOST) REG_URL ?= https://$(REG_HOST)
# name of alternate build:
# Dockerfile.$(ALT_NAME) --> $(APP_NAME)_${ALT_NAME}
ALT_NAME ?= broadwell
APP_NAME = eic APP_NAME = eic
REPO_NAME = eic REPO_NAME = eic
DH_ORG = eic
GL_GROUP = eic_container GL_GROUP = eic_container
GL_REG_GROUP = containers/eic_container GL_REG_GROUP = containers/eic_container
GL_REG_NAME = $(REG_NAME) GL_REG_NAME = $(REG_NAME)
REPO = $(REG_USER)
TAG_VERSION = latest
#!/bin/bash
head -n1 ../../VERSION
image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
stages:
- build
- config
- package
- singularity
## variables:
## - TARGET_XXX: docker build target (including cache modifier)
## (stable, stable-cached, unstable, unstable-cached)
## - TAG: main docker tag to be used internally
## (e.g. 2.5-stable/unstable/<version>)
## - PUBLISH: docker publish directives
variables:
TARGET_BUILDER: @TARGET_BUILDER@
TARGET_RELEASE: @TARGET_RELEASE@
TAG: @TAG@
PUBLISH: "@PUBLISH@"
default:
tags:
- silicon
builder:
stage: build
script:
- cp -r gitlab-ci/docker/* containers/builder
- cp -r spack containers/builder/np-spack
- cd containers/builder
- head Dockerfile
- make login
- echo "Creating builder image for $TARGET_BUILDER"
- make $TARGET_BUILDER
- echo "Publishing image $PUBLISH"
- make $PUBLISH
config:
image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:$TAG
stage: config
needs: ["builder"]
script:
- bash containers/release/configure_release.sh $TAG
artifacts:
paths:
- config
release:docker:
stage: package
needs: ["config"]
script:
- cp -r gitlab-ci/docker/* containers/release
- cp config/Dockerfile containers/release/Dockerfile
- cp config/eic-env.sh containers/release/eic-env.sh
- cd containers/release
- make login
- echo "Creating release image for $TARGET_RELEASE"
- make $TARGET_RELEASE
- echo "Publishing image $PUBLISH"
- make $PUBLISH
release:singularity:
stage: singularity
needs: ["release:docker"]
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
when: on_success
- if: '$CI_COMMIT_BRANCH == "v$TAG"'
when: on_success
- if: '$CI_COMMIT_TAG == "v$TAG"'
when: on_success
script:
- cp config/eic.def eic.def
- /bin/bash .gitlabci/setup.sh
- /bin/bash .gitlabci/build.sh eic.def
- mkdir -p build
- cp eic.sif build/.
- cp eic.def build/.
artifacts:
expire_in: 90 days
paths:
- build/eic.sif
- build/eic.def
#!/bin/bash
## Configure a CI pipeline based on a template file (first and only
## argument to this script).
TEMPLATE_FILE=$1
OUTPUT_FILE=`basename ${TEMPLATE_FILE} .in`
echo "Configuring pipeline script: ${TEMPLATE_FILE}"
echo "Output will be written to: ${OUTPUT_FILE}"
VERSION=`head -n1 VERSION`
STABLE="${VERSION%.*}-stable"
## Figure out which scenario we are running:
## - master
## - stable
## - tag
## - unstable (default)
TARGET=""
TAG=""
PUBLISH=""
if [ "$CI_COMMIT_BRANCH" = "master" ]; then
TARGET="stable"
TAG="latest"
PUBLISH="publish-latest publish-stable"
elif [ "$CI_COMMIT_TAG" = "v${VERSION}" ]; then
TARGET=$VERSION
TAG=$VERSION
PUBLISH="publish-version"
elif [ "$CI_COMMIT_BRANCH" = "v${STABLE}" ]; then
TARGET="stable"
TAG=${STABLE}
PUBLISH="publish-stable"
else
TARGET="unstable"
TAG="unstable"
PUBLISH="publish-unstable"
fi
TARGET_BUILDER=$TARGET
TARGET_RELEASE=$TARGET
if [ ! -f .ci_env/buider-nc ]; then
TARGET_BUILDER="${TARGET_BUILDER}-cached"
fi
if [ ! -f .ci_env/release-nc ]; then
TARGET_BUILDER="${TARGET_RELEASE}-cached"
fi
sed "s/@TAG@/$TAG/g" $TEMPLATE_FILE | \
sed "s/@TARGET_BUILDER@/$TARGET_BUILDER/g" | \
sed "s/@TARGET_RELEASE@/$TARGET_RELEASE/g" | \
sed "s/@PUBLISH@/$PUBLISH/g" > ${OUTPUT_FILE}
echo "Done"
# import config.
# You can change the default config with `make cnf="config_special.env" build`
cnf ?= config.env
include $(cnf)
# exports variables in config.env as environment variables
export $(shell sed 's/=.*//' $(cnf))
# import deploy config
# You can change the default deploy config with `make cnf="deploy_special.env" release`
SHELL = bash
## Get our version tag and stable version tag
VERSION=$(shell head -n1 ../../VERSION)
STABLE=$(shell echo ${$(head -n1 ../../VERSION)%.*}-stable)
# help will output the help for each task
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.PHONY: help
help: ## This help.
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
.DEFAULT_GOAL := help
# ==========================================================================
#
build: ## build the image
docker build -t $(APP_NAME):$(VERSION) .
build-unstable: ## build the image
docker build -t $(APP_NAME):$(VERSION)-unstable .
build-nc: ## Build the container without caching (from scratch)
docker build --no-cache -t $(APP_NAME):$(VERSION) .
build-unstable-nc: ## Build the container without caching (from scratch)
docker build --no-cache -t $(APP_NAME):$(VERSION)-unstable .
# ==========================================================================
#
login: ## Auto login to AWS-ECR unsing aws-cli
docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}/${CI_PROJECT_PATH}
echo "Login COMPLETE"
stable: build-nc
stable-cached: build
unstable: build-unstable-nc
unstable-cached: build-unstable
publish: login publish-latest publish-version #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
@echo "Publishing done"
publish-stable: login
@echo 'publish $(STABLE) to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):$(VERSION) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(STABLE)
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(STABLE)
publish-latest: login
@echo 'publish latest to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):$(VERSION) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
publish-version: login
@echo 'publish $(STABLE) to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):$(VERSION) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
publish-unstable: login
@echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):$(VERSION)-unstable $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable
version: ## Output the current version
@echo $(VERSION)
# ==========================================================================
#
info: ## Output the current version
@echo 'VERSION = $(VERSION) '
@echo 'STABLE = $(STABLE) '
@echo 'REG_NAME = $(REG_NAME) '
@echo 'APP_NAME = $(APP_NAME) '
@echo 'GL_GROUP = $(GL_GROUP) '
@echo 'GL_REG_GROUP = $(GL_REG_GROUP)'
@echo 'GL_REG_NAME = $(GL_REG_NAME) '
#!/bin/bash
## print the stable version tag for this version to the console
version=head -n1 ../../VERSION
echo "${VERSION%.*}-stable"
#!/bin/bash #!/bin/bash
## print the full version number to the console
head -n1 ../../VERSION head -n1 ../../VERSION
File moved
File moved
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment