Commit b8d06008 authored by Sylvester Joosten's avatar Sylvester Joosten

Redid the CI to handle different upstream branches, and to ensure the MR images run the benchmarks.

parent 7fb3075c
Pipeline #5985 passed with stages
in 5 minutes and 5 seconds
FROM eicweb.phy.anl.gov:4567/containers/eic_container/eic:latest
FROM eicweb.phy.anl.gov:4567/containers/eic_container/eic:@EIC_TAG@
LABEL maintainer="Whitney Armstrong <warmstrong@anl.gov>" \
name="juggler" \
group="EIC/juggler" \
march="native" \
basedist="debian" \
base="eic_container/eic" \
version="2.5.0"
base="eic_container/eic"
RUN cd /tmp \
&& git clone https://eicweb.phy.anl.gov/EIC/NPDet.git \
......
# import config.
# You can change the default config with `make cnf="config_special.env" build`
cnf ?= config.env
include $(cnf)
# exports variables in config.env as environment variables
export $(shell sed 's/=.*//' $(cnf))
SHELL = bash
# grep the version from the mix file
VERSION=$(shell bash version.sh)
TAG_VERSION=$(VERSION)
LONG_TAG=$(VERSION)-$(PUBLISH_TAG)
# help will output the help for each task
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.PHONY: help
help: ## This help.
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
.DEFAULT_GOAL := help
# ==========================================================================
#
build: ## build the image
docker build -t $(APP_NAME):$(LONG_TAG) .
build-nc: ## Build the container without caching (from scratch)
docker build --no-cache -t $(APP_NAME):$(LONG_TAG) .
# ==========================================================================
#
login: ## Auto login to AWS-ECR unsing aws-cli
@docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
echo "Login COMPLETE"
# ==========================================================================
#
publish: login ## Publish a tagged container to ECR
@echo 'publish $(PUBLISH_TAG) to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):$(LONG_TAG) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(PUBLISH_TAG)
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(PUBLISH_TAG)
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(PUBLISH_TAG)
publish-version: login ## Publish the `{version}` taged container to ECR
@echo 'publish $(VERSION) to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)'
docker tag $(APP_NAME):$(LONG_TAG) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
# ==========================================================================
# remove container from registry
unpublish: login
@echo 'removing $(PUBLISH_TAG)'
curl --request DELETE --header "PRIVATE-TOKEN: $(CI_JOB_TOKEN)" "$(REG_API_URL)/$(PUBLISH_TAG)"
# ==========================================================================
# cleanup docker registry on system used by runner
cleanup:
@echo 'removing $(REG_NAME):$(LONG_TAG)'
docker rmi $(REG_NAME):$(LONG_TAG)
......@@ -8,13 +8,10 @@ REG_URL ?= https://$(REG_HOST)
APP_NAME = juggler
REPO_NAME = juggler
DH_ORG = argonneeic
GL_GROUP = eic
GL_REG_GROUP = eic/juggler
GL_REG_NAME = $(REG_NAME)
PUBLISH_TAG = $(JUGGLER_TAG)
REG_API_URL = $(REG_URL)/api/v4/projects/$(CI_PROJECT_ID)/registry/repositories/50/tags
# import config.
# You can change the default config with `make cnf="config_special.env" build`
cnf ?= config.env
include $(cnf)
# exports variables in config.env as environment variables
export $(shell sed 's/=.*//' $(cnf))
SHELL = bash
# grep the version from the mix file
VERSION=$(shell bash version.sh)
TAG_VERSION=$(VERSION)
# help will output the help for each task
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.PHONY: help
help: ## This help.
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
.DEFAULT_GOAL := help
# ==========================================================================
#
build: ## build the image
docker build -t $(APP_NAME) .
build-nc: ## Build the container without caching (from scratch)
docker build --no-cache -t $(APP_NAME) .
# ==========================================================================
#
run: ## Run container on port configured in `config.env`
docker run -i -t --rm --env-file=./config.env -p=$(PORT):$(PORT) --name="$(APP_NAME)" $(REPO)/$(APP_NAME):$(TAG_VERSION)
up: build run ## Run container on port configured in `config.env` (Alias to run)
stop: ## Stop and remove a running container
docker stop $(APP_NAME); docker rm $(APP_NAME)
# ==========================================================================
#
tag: tag-latest tag-version #tag-version ## Generate container tags for the `{version}` and `latest` tags
tag-latest: ## Generate container `{version}` tag
@echo 'create tag latest'
#docker tag $(APP_NAME) $(REPO)/$(APP_NAME):latest
tag-version: ## Generate container `latest` tag
@echo 'creating tag $(APP_NAME):$(VERSION)'
docker tag $(APP_NAME):latest $(APP_NAME):$(VERSION)
# ==========================================================================
#
login: ## Auto login to AWS-ECR unsing aws-cli
@docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
echo "Login COMPLETE"
release-latest: build-nc login publish-latest ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
release: build-nc publish clean-all## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
release-cached: build publish ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
publish: login publish-latest publish-version #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
@echo "Publishing done"
publish-latest: ## Publish the `latest` taged container to ECR
@echo 'publish latest to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
publish-version: ## Publish the `{version}` taged container to ECR
@echo 'publish latest to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
push: login ## push after login @echo 'push latest to $(REG_NAME)/$(REPO)/$(APP_NAME):latest'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
# ==========================================================================
#
login-dockerhub: ## login to hub.docker.com
@echo 'docker hub login :'
docker login
push-dockerhub: build #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
@echo '$(DH_ORG)/$(APP_NAME):$(TAG_VERSION)'
docker push $(DH_ORG)/$(APP_NAME):latest
#docker push $(DH_ORG)/$(APP_NAME):$(VERSION)
publish-dockerhub: build tag #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
@echo '$(DH_ORG)/$(APP_NAME):$(TAG_VERSION)'
docker push $(DH_ORG)/$(APP_NAME):latest
#docker push $(DH_ORG)/$(APP_NAME):$(VERSION)
# ==========================================================================
#
clean-tags:
docker rmi $(REPO)/$(APP_NAME):latest || true
docker rmi $(GL_GROUP)/$(APP_NAME):latest || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME):latest || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME):latest || true
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest || true
clean:
docker rmi $(REPO_NAME)/$(APP_NAME) || true
docker rmi $(REPO_NAME)/$(APP_NAME):$(TAG_VERSION) || true
docker rmi $(REPO_NAME)/$(APP_NAME):$(VERSION) || true
docker rmi $(REPO)/$(APP_NAME) || true
docker rmi $(REPO)/$(APP_NAME):$(TAG_VERSION) || true
docker rmi $(REPO)/$(APP_NAME):$(VERSION) || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME):latest || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME):$(TAG_VERSION) || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME) || true
docker rmi $(DH_ORG)/$(APP_NAME) || true
docker rmi $(DH_ORG)/$(APP_NAME):$(TAG_VERSION) || true
docker rmi $(DH_ORG)/$(APP_NAME):$(VERSION) || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION) || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME) || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME) || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME):$(VERSION) || true
#docker rmi $(APP_NAME) || true
clean-all: clean
docker rmi $(REPO_NAME)/$(APP_NAME) || true
version: ## Output the current version
@echo $(VERSION)
# ==========================================================================
#
info: ## Output the current version
@echo 'VERSION = $(VERSION) '
@echo 'REG_NAME = $(REG_NAME) '
@echo 'APP_NAME = $(APP_NAME) '
@echo 'REPO_NAME = $(REPO_NAME) '
@echo 'DH_ORG = $(DH_ORG) '
@echo 'GL_GROUP = $(GL_GROUP) '
@echo 'GL_REG_GROUP = $(GL_REG_GROUP)'
@echo 'GL_REG_NAME = $(GL_REG_NAME) '
@echo 'REPO_NAME = $(REPO) '
@echo 'TAG_VERSION = $(TAG_VERSION) '
ls: ## list all docker images
docker images
# INSTALL
# - copy the files deploy.env, config.env, version.sh and Makefile to your repo
# - replace the vars in deploy.env
# - define the version script
# Build the container
make build
# Build and publish the container
make release
# Publish a container to AWS-ECR.
# This includes the login to the repo
make publish
# Run the container
make run
# Build an run the container
make up
# Stop the running container
make stop
# Build the container with differnt config and deploy file
make cnf=another_config.env dpl=another_deploy.env build
\ No newline at end of file
Bootstrap: docker
From: eicweb.phy.anl.gov:4567/eic/juggler/juggler:latest
From: eicweb.phy.anl.gov:4567/eic/juggler/juggler:@JUGGLER_TAG@
%help
singularity container for Juggler development
......
#!/bin/bash
cat ../../VERSION | sed 's/)//'
cat ../VERSION | sed 's/)//'
......@@ -22,7 +22,7 @@ TEST*/*
# cmake
CMakeCache.txt
CMakeFiles
Makefile
#Makefile
cmake_install.cmake
install_manifest.txt
......
image: eicweb.phy.anl.gov:4567/containers/eic_container/eic:2.5-stable
variables:
EIC_TAG: "2.5-stable"
#default:
# artifacts:
# paths:
# - build/
default:
image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
tags:
- silicon
stages:
- config
- build
- docker_build
- docker_push
- push
- deploy
compile:
stage: build
rules:
- if: '$CI_COMMIT_BRANCH != "master" && $CI_PIPELINE_SOURCE == "merge_request_event"'
tags:
- silicon
script:
- apt-get update && apt-get install -y rsync
- export homedir=$(pwd) && pwd && cd /tmp && git clone --depth=1 https://eicweb.phy.anl.gov/EIC/NPDet.git && mkdir build && cd build && cmake ../NPDet/. && make -j20 install
- cd /tmp && git clone --depth=1 https://eicweb.phy.anl.gov/EIC/eicd.git && mkdir eicd_build && cd eicd_build && cmake ../eicd/. && make -j20 install
- cd $homedir && ls -lrth && mkdir build && cd build && cmake .. && make -j10
- cleanup
docker:build:
image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
stage: docker_build
workflow:
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_BRANCH != "master"'
when: never
- when: on_success
tags:
- silicon
- if: '$CI_COMMIT_BRANCH == "master"'
- if: '$CI_COMMIT_TAG'
env:
stage: config
script:
- cd .containers/docker
- make build-nc
- export JUGGLER_TAG="latest"
- |
if [ "x${CI_PIPELINE_SOURCE}" == "xmerge_request_event" ]; then
export JUGGLER_TAG="testing-mr-${CI_MERGE_REQUEST_IID}"
fi
- echo "CI JUGGLER_TAG for this pipeline set to ${JUGGLER_TAG}"
- echo "JUGGLER_TAG=$JUGGLER_TAG" >> juggler.env
artifacts:
reports:
dotenv: juggler.env
docker:
stage: build
needs:
- env
script:
- echo $JUGGLER_TAG
- ./.gitlabci/configure.sh .containers/Dockerfile.in
- cd .containers
- make build
#- make build-nc
docker:push:
image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
stage: docker_push
needs: ["docker:build"]
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_BRANCH != "master"'
when: never
- when: on_success
publish:
stage: push
needs:
- env
- docker
script:
- cd .containers
- |
if [ "x$CI_COMMIT_TAG" != "x" ]; then
make publish publish-version
else
make publish
fi
retry:
max: 2
when:
- runner_system_failure
- stuck_or_timeout_failure
tags:
- silicon
.singularity:
stage: deploy
needs:
- env
- publish
script:
- cd .containers/docker
- make push
- ./.gitlabci/configure.sh .containers/juggler.def.in
- mkdir -p build
- mv .containers/juggler.def build
- singularity build build/juggler.sif build/juggler.def
retry:
max: 2
when:
- runner_system_failure
- stuck_or_timeout_failure
docker:release:
image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
stage: docker_build
singularity:latest:
extends: .singularity
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
when: on_success
- if: '$CI_COMMIT_TAG'
tags:
- silicon
script:
- cd .containers/docker && make release
when: on_success
artifacts:
expire_in: 90 days
paths:
- build/juggler.sif
- build/juggler.def
docker:singularity:
image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
stage: docker_push
needs: ["docker:build"]
singularity:testing:
extends: .singularity
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_BRANCH != "master"'
when: never
- when: on_success
tags:
- silicon
script:
- cp .containers/singularity/juggler.def .
- /bin/bash .gitlabci/build.sh juggler.def
- mkdir -p build
- cp juggler.sif build/.
- cp juggler.def build/.
when: manual
artifacts:
expire_in: 90 days
expire_in: 1 days
paths:
- build/juggler.sif
- build/juggler.def
#singularity:push:
# stage: deploy
# when: manual
benchmarks:reconstruction:
stage: deploy
needs:
- env
- publish
variables:
JUGGLER_TAG: "$JUGGLER_TAG"
trigger:
project: EIC/benchmarks/reconstruction_benchmarks
needs: ["docker:push"]
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_BRANCH != "master"'
when: never
- when: on_success
strategy: depend
benchmarks:physics:
stage: deploy
needs:
- env
- publish
variables:
JUGGLER_TAG: "$JUGGLER_TAG"
trigger:
project: EIC/benchmarks/physics_benchmarks
needs: ["docker:push"]
strategy: depend
purge_image:
stage: cleanup
dependencies:
- env
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_BRANCH != "master"'
when: never
- when: on_success
script:
- cd .containers
- make cleanup unpublish
#- make unpublish
#!/bin/bash
## Configure a CI file based on a template file (first and only
## argument to this script).
## Known variables that will be substituted:
## @EIC_TAG@ - docker tag used for the EIC container
## @JUGGLER_TAG@ - output tag for the Juggler version
TEMPLATE_FILE=$1
OUTPUT_FILE=${TEMPLATE_FILE%.*}
echo "Configuring CI file: ${TEMPLATE_FILE}"
echo "Output will be written to: ${OUTPUT_FILE}"
TAG=$EIC_VERSION
sed "s/@EIC_TAG@/$EIC_TAG/g" $TEMPLATE_FILE | \
sed "s/@JUGGLER_TAG@/$JUGGLER_TAG/g" > ${OUTPUT_FILE}
echo "Done"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment