Commit b8d06008 authored by Sylvester Joosten's avatar Sylvester Joosten
Browse files

Redid the CI to handle different upstream branches, and to ensure the MR images run the benchmarks.

parent 7fb3075c
FROM eicweb.phy.anl.gov:4567/containers/eic_container/eic:latest
FROM eicweb.phy.anl.gov:4567/containers/eic_container/eic:@EIC_TAG@
LABEL maintainer="Whitney Armstrong <warmstrong@anl.gov>" \
name="juggler" \
group="EIC/juggler" \
march="native" \
basedist="debian" \
base="eic_container/eic" \
version="2.5.0"
base="eic_container/eic"
RUN cd /tmp \
&& git clone https://eicweb.phy.anl.gov/EIC/NPDet.git \
......
# import config.
# You can change the default config with `make cnf="config_special.env" build`
cnf ?= config.env
include $(cnf)
# exports variables in config.env as environment variables
export $(shell sed 's/=.*//' $(cnf))
SHELL = bash
# grep the version from the mix file
VERSION=$(shell bash version.sh)
TAG_VERSION=$(VERSION)
LONG_TAG=$(VERSION)-$(PUBLISH_TAG)
# help will output the help for each task
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.PHONY: help
help: ## This help.
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
.DEFAULT_GOAL := help
# ==========================================================================
#
build: ## build the image
docker build -t $(APP_NAME):$(LONG_TAG) .
build-nc: ## Build the container without caching (from scratch)
docker build --no-cache -t $(APP_NAME):$(LONG_TAG) .
# ==========================================================================
#
login: ## Auto login to AWS-ECR unsing aws-cli
@docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
echo "Login COMPLETE"
# ==========================================================================
#
publish: login ## Publish a tagged container to ECR
@echo 'publish $(PUBLISH_TAG) to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):$(LONG_TAG) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(PUBLISH_TAG)
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(PUBLISH_TAG)
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(PUBLISH_TAG)
publish-version: login ## Publish the `{version}` taged container to ECR
@echo 'publish $(VERSION) to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)'
docker tag $(APP_NAME):$(LONG_TAG) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
# ==========================================================================
# remove container from registry
unpublish: login
@echo 'removing $(PUBLISH_TAG)'
curl --request DELETE --header "PRIVATE-TOKEN: $(CI_JOB_TOKEN)" "$(REG_API_URL)/$(PUBLISH_TAG)"
# ==========================================================================
# cleanup docker registry on system used by runner
cleanup:
@echo 'removing $(REG_NAME):$(LONG_TAG)'
docker rmi $(REG_NAME):$(LONG_TAG)
......@@ -8,13 +8,10 @@ REG_URL ?= https://$(REG_HOST)
APP_NAME = juggler
REPO_NAME = juggler
DH_ORG = argonneeic
GL_GROUP = eic
GL_REG_GROUP = eic/juggler
GL_REG_NAME = $(REG_NAME)
PUBLISH_TAG = $(JUGGLER_TAG)
REG_API_URL = $(REG_URL)/api/v4/projects/$(CI_PROJECT_ID)/registry/repositories/50/tags
# import config.
# You can change the default config with `make cnf="config_special.env" build`
cnf ?= config.env
include $(cnf)
# exports variables in config.env as environment variables
export $(shell sed 's/=.*//' $(cnf))
SHELL = bash
# grep the version from the mix file
VERSION=$(shell bash version.sh)
TAG_VERSION=$(VERSION)
# help will output the help for each task
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.PHONY: help
help: ## This help.
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
.DEFAULT_GOAL := help
# ==========================================================================
#
build: ## build the image
docker build -t $(APP_NAME) .
build-nc: ## Build the container without caching (from scratch)
docker build --no-cache -t $(APP_NAME) .
# ==========================================================================
#
run: ## Run container on port configured in `config.env`
docker run -i -t --rm --env-file=./config.env -p=$(PORT):$(PORT) --name="$(APP_NAME)" $(REPO)/$(APP_NAME):$(TAG_VERSION)
up: build run ## Run container on port configured in `config.env` (Alias to run)
stop: ## Stop and remove a running container
docker stop $(APP_NAME); docker rm $(APP_NAME)
# ==========================================================================
#
tag: tag-latest tag-version #tag-version ## Generate container tags for the `{version}` and `latest` tags
tag-latest: ## Generate container `{version}` tag
@echo 'create tag latest'
#docker tag $(APP_NAME) $(REPO)/$(APP_NAME):latest
tag-version: ## Generate container `latest` tag
@echo 'creating tag $(APP_NAME):$(VERSION)'
docker tag $(APP_NAME):latest $(APP_NAME):$(VERSION)
# ==========================================================================
#
login: ## Auto login to AWS-ECR unsing aws-cli
@docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
echo "Login COMPLETE"
release-latest: build-nc login publish-latest ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
release: build-nc publish clean-all## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
release-cached: build publish ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
publish: login publish-latest publish-version #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
@echo "Publishing done"
publish-latest: ## Publish the `latest` taged container to ECR
@echo 'publish latest to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
publish-version: ## Publish the `{version}` taged container to ECR
@echo 'publish latest to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)
push: login ## push after login @echo 'push latest to $(REG_NAME)/$(REPO)/$(APP_NAME):latest'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
# ==========================================================================
#
login-dockerhub: ## login to hub.docker.com
@echo 'docker hub login :'
docker login
push-dockerhub: build #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
@echo '$(DH_ORG)/$(APP_NAME):$(TAG_VERSION)'
docker push $(DH_ORG)/$(APP_NAME):latest
#docker push $(DH_ORG)/$(APP_NAME):$(VERSION)
publish-dockerhub: build tag #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
@echo '$(DH_ORG)/$(APP_NAME):$(TAG_VERSION)'
docker push $(DH_ORG)/$(APP_NAME):latest
#docker push $(DH_ORG)/$(APP_NAME):$(VERSION)
# ==========================================================================
#
clean-tags:
docker rmi $(REPO)/$(APP_NAME):latest || true
docker rmi $(GL_GROUP)/$(APP_NAME):latest || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME):latest || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME):latest || true
docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest || true
clean:
docker rmi $(REPO_NAME)/$(APP_NAME) || true
docker rmi $(REPO_NAME)/$(APP_NAME):$(TAG_VERSION) || true
docker rmi $(REPO_NAME)/$(APP_NAME):$(VERSION) || true
docker rmi $(REPO)/$(APP_NAME) || true
docker rmi $(REPO)/$(APP_NAME):$(TAG_VERSION) || true
docker rmi $(REPO)/$(APP_NAME):$(VERSION) || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME):latest || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME):$(TAG_VERSION) || true
docker rmi $(REG_NAME)/$(REPO)/$(APP_NAME) || true
docker rmi $(DH_ORG)/$(APP_NAME) || true
docker rmi $(DH_ORG)/$(APP_NAME):$(TAG_VERSION) || true
docker rmi $(DH_ORG)/$(APP_NAME):$(VERSION) || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION) || true
docker rmi $(GL_REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME) || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME) || true
docker rmi $(GL_REG_GROUP)/$(APP_NAME):$(VERSION) || true
#docker rmi $(APP_NAME) || true
clean-all: clean
docker rmi $(REPO_NAME)/$(APP_NAME) || true
version: ## Output the current version
@echo $(VERSION)
# ==========================================================================
#
info: ## Output the current version
@echo 'VERSION = $(VERSION) '
@echo 'REG_NAME = $(REG_NAME) '
@echo 'APP_NAME = $(APP_NAME) '
@echo 'REPO_NAME = $(REPO_NAME) '
@echo 'DH_ORG = $(DH_ORG) '
@echo 'GL_GROUP = $(GL_GROUP) '
@echo 'GL_REG_GROUP = $(GL_REG_GROUP)'
@echo 'GL_REG_NAME = $(GL_REG_NAME) '
@echo 'REPO_NAME = $(REPO) '
@echo 'TAG_VERSION = $(TAG_VERSION) '
ls: ## list all docker images
docker images
# INSTALL
# - copy the files deploy.env, config.env, version.sh and Makefile to your repo
# - replace the vars in deploy.env
# - define the version script
# Build the container
make build
# Build and publish the container
make release
# Publish a container to AWS-ECR.
# This includes the login to the repo
make publish
# Run the container
make run
# Build an run the container
make up
# Stop the running container
make stop
# Build the container with differnt config and deploy file
make cnf=another_config.env dpl=another_deploy.env build
\ No newline at end of file
Bootstrap: docker
From: eicweb.phy.anl.gov:4567/eic/juggler/juggler:latest
From: eicweb.phy.anl.gov:4567/eic/juggler/juggler:@JUGGLER_TAG@
%help
singularity container for Juggler development
......
#!/bin/bash
cat ../../VERSION | sed 's/)//'
cat ../VERSION | sed 's/)//'
......@@ -22,7 +22,7 @@ TEST*/*
# cmake
CMakeCache.txt
CMakeFiles
Makefile
#Makefile
cmake_install.cmake
install_manifest.txt
......
image: eicweb.phy.anl.gov:4567/containers/eic_container/eic:2.5-stable
variables:
EIC_TAG: "2.5-stable"
#default:
# artifacts:
# paths:
# - build/
default:
image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
tags:
- silicon
stages:
- config
- build
- docker_build
- docker_push
- push
- deploy
compile:
stage: build
rules:
- if: '$CI_COMMIT_BRANCH != "master" && $CI_PIPELINE_SOURCE == "merge_request_event"'
tags:
- silicon
script:
- apt-get update && apt-get install -y rsync
- export homedir=$(pwd) && pwd && cd /tmp && git clone --depth=1 https://eicweb.phy.anl.gov/EIC/NPDet.git && mkdir build && cd build && cmake ../NPDet/. && make -j20 install
- cd /tmp && git clone --depth=1 https://eicweb.phy.anl.gov/EIC/eicd.git && mkdir eicd_build && cd eicd_build && cmake ../eicd/. && make -j20 install
- cd $homedir && ls -lrth && mkdir build && cd build && cmake .. && make -j10
- cleanup
docker:build:
image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
stage: docker_build
workflow:
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_BRANCH != "master"'
when: never
- when: on_success
tags:
- silicon
- if: '$CI_COMMIT_BRANCH == "master"'
- if: '$CI_COMMIT_TAG'
env:
stage: config
script:
- cd .containers/docker
- make build-nc
- export JUGGLER_TAG="latest"
- |
if [ "x${CI_PIPELINE_SOURCE}" == "xmerge_request_event" ]; then
export JUGGLER_TAG="testing-mr-${CI_MERGE_REQUEST_IID}"
fi
- echo "CI JUGGLER_TAG for this pipeline set to ${JUGGLER_TAG}"
- echo "JUGGLER_TAG=$JUGGLER_TAG" >> juggler.env
artifacts:
reports:
dotenv: juggler.env
docker:
stage: build
needs:
- env
script:
- echo $JUGGLER_TAG
- ./.gitlabci/configure.sh .containers/Dockerfile.in
- cd .containers
- make build
#- make build-nc
docker:push:
image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
stage: docker_push
needs: ["docker:build"]
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_BRANCH != "master"'
when: never
- when: on_success
publish:
stage: push
needs:
- env
- docker
script:
- cd .containers
- |
if [ "x$CI_COMMIT_TAG" != "x" ]; then
make publish publish-version
else
make publish
fi
retry:
max: 2
when:
- runner_system_failure
- stuck_or_timeout_failure
tags:
- silicon
.singularity:
stage: deploy
needs:
- env
- publish
script:
- cd .containers/docker
- make push
- ./.gitlabci/configure.sh .containers/juggler.def.in
- mkdir -p build
- mv .containers/juggler.def build
- singularity build build/juggler.sif build/juggler.def
retry:
max: 2
when:
- runner_system_failure
- stuck_or_timeout_failure
docker:release:
image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
stage: docker_build
singularity:latest:
extends: .singularity
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
when: on_success
- if: '$CI_COMMIT_TAG'
tags:
- silicon
script:
- cd .containers/docker && make release
when: on_success
artifacts:
expire_in: 90 days
paths:
- build/juggler.sif
- build/juggler.def
docker:singularity:
image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
stage: docker_push
needs: ["docker:build"]
singularity:testing:
extends: .singularity
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_BRANCH != "master"'
when: never
- when: on_success
tags:
- silicon
script:
- cp .containers/singularity/juggler.def .
- /bin/bash .gitlabci/build.sh juggler.def
- mkdir -p build
- cp juggler.sif build/.
- cp juggler.def build/.
when: manual
artifacts:
expire_in: 90 days
expire_in: 1 days
paths:
- build/juggler.sif
- build/juggler.def
#singularity:push:
# stage: deploy
# when: manual
benchmarks:reconstruction:
stage: deploy
needs:
- env
- publish
variables:
JUGGLER_TAG: "$JUGGLER_TAG"
trigger:
project: EIC/benchmarks/reconstruction_benchmarks
needs: ["docker:push"]
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_BRANCH != "master"'
when: never
- when: on_success
strategy: depend
benchmarks:physics:
stage: deploy
needs:
- env
- publish
variables:
JUGGLER_TAG: "$JUGGLER_TAG"
trigger:
project: EIC/benchmarks/physics_benchmarks
needs: ["docker:push"]
strategy: depend
purge_image:
stage: cleanup
dependencies:
- env
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: never
- if: '$CI_COMMIT_BRANCH != "master"'
when: never
- when: on_success
script:
- cd .containers
- make cleanup unpublish
#- make unpublish
#!/bin/bash
## Configure a CI file based on a template file (first and only
## argument to this script).
## Known variables that will be substituted:
## @EIC_TAG@ - docker tag used for the EIC container
## @JUGGLER_TAG@ - output tag for the Juggler version
TEMPLATE_FILE=$1
OUTPUT_FILE=${TEMPLATE_FILE%.*}
echo "Configuring CI file: ${TEMPLATE_FILE}"
echo "Output will be written to: ${OUTPUT_FILE}"
TAG=$EIC_VERSION
sed "s/@EIC_TAG@/$EIC_TAG/g" $TEMPLATE_FILE | \
sed "s/@JUGGLER_TAG@/$JUGGLER_TAG/g" > ${OUTPUT_FILE}
echo "Done"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment