Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
EIC
Project Juggler
Commits
b8d06008
Commit
b8d06008
authored
Jan 14, 2021
by
Sylvester Joosten
Browse files
Redid the CI to handle different upstream branches, and to ensure the MR images run the benchmarks.
parent
7fb3075c
Changes
10
Hide whitespace changes
Inline
Side-by-side
.containers/
docker/
Dockerfile
→
.containers/Dockerfile
.in
View file @
b8d06008
FROM
eicweb.phy.anl.gov:4567/containers/eic_container/eic:
latest
FROM eicweb.phy.anl.gov:4567/containers/eic_container/eic:
@EIC_TAG@
LABEL maintainer="Whitney Armstrong <warmstrong@anl.gov>" \
name="juggler" \
group="EIC/juggler" \
march="native" \
basedist="debian" \
base="eic_container/eic" \
version="2.5.0"
base="eic_container/eic"
RUN cd /tmp \
&& git clone https://eicweb.phy.anl.gov/EIC/NPDet.git \
...
...
.containers/Makefile
0 → 100644
View file @
b8d06008
# import config.
# You can change the default config with `make cnf="config_special.env" build`
cnf
?=
config.env
include
$(cnf)
# exports variables in config.env as environment variables
export
$(shell
sed
's/=.*//'
$(cnf))
SHELL
=
bash
# grep the version from the mix file
VERSION
=
$(
shell
bash version.sh
)
TAG_VERSION
=
$(VERSION)
LONG_TAG
=
$(VERSION)
-
$(PUBLISH_TAG)
# help will output the help for each task
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.PHONY
:
help
help
:
##
This help.
@
awk
'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
$(MAKEFILE_LIST)
.DEFAULT_GOAL
:=
help
# ==========================================================================
#
build
:
##
build the image
docker build
-t
$(APP_NAME)
:
$(LONG_TAG)
.
build-nc
:
##
Build the container without caching (from scratch)
docker build
--no-cache
-t
$(APP_NAME)
:
$(LONG_TAG)
.
# ==========================================================================
#
login
:
##
Auto login to AWS-ECR unsing aws-cli
@
docker login
-u
${CI_REGISTRY_USER}
-p
${CI_REGISTRY_PASSWORD}
${CI_REGISTRY}
echo
"Login COMPLETE"
# ==========================================================================
#
publish
:
login
##
Publish a tagged container to ECR
@
echo
'publish
$(PUBLISH_TAG)
to
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
'
docker tag
$(APP_NAME)
:
$(LONG_TAG)
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:
$(PUBLISH_TAG)
docker push
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:
$(PUBLISH_TAG)
docker rmi
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:
$(PUBLISH_TAG)
publish-version
:
login
##
Publish the `{version}` taged container to ECR
@
echo
'publish
$(VERSION)
to
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:
$(VERSION)
'
docker tag
$(APP_NAME)
:
$(LONG_TAG)
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:
$(VERSION)
docker push
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:
$(VERSION)
docker rmi
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:
$(VERSION)
# ==========================================================================
# remove container from registry
unpublish
:
login
@
echo
'removing
$(PUBLISH_TAG)
'
curl
--request
DELETE
--header
"PRIVATE-TOKEN:
$(CI_JOB_TOKEN)
"
"
$(REG_API_URL)
/
$(PUBLISH_TAG)
"
# ==========================================================================
# cleanup docker registry on system used by runner
cleanup
:
@
echo
'removing
$(REG_NAME)
:
$(LONG_TAG)
'
docker rmi
$(REG_NAME)
:
$(LONG_TAG)
.containers/
docker/
config.env
→
.containers/config.env
View file @
b8d06008
...
...
@@ -8,13 +8,10 @@ REG_URL ?= https://$(REG_HOST)
APP_NAME = juggler
REPO_NAME = juggler
DH_ORG = argonneeic
GL_GROUP = eic
GL_REG_GROUP = eic/juggler
GL_REG_NAME = $(REG_NAME)
PUBLISH_TAG = $(JUGGLER_TAG)
REG_API_URL = $(REG_URL)/api/v4/projects/$(CI_PROJECT_ID)/registry/repositories/50/tags
.containers/docker/Makefile
deleted
100644 → 0
View file @
7fb3075c
# import config.
# You can change the default config with `make cnf="config_special.env" build`
cnf
?=
config.env
include
$(cnf)
# exports variables in config.env as environment variables
export
$(shell
sed
's/=.*//'
$(cnf))
SHELL
=
bash
# grep the version from the mix file
VERSION
=
$(
shell
bash version.sh
)
TAG_VERSION
=
$(VERSION)
# help will output the help for each task
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.PHONY
:
help
help
:
##
This help.
@
awk
'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
$(MAKEFILE_LIST)
.DEFAULT_GOAL
:=
help
# ==========================================================================
#
build
:
##
build the image
docker build
-t
$(APP_NAME)
.
build-nc
:
##
Build the container without caching (from scratch)
docker build
--no-cache
-t
$(APP_NAME)
.
# ==========================================================================
#
run
:
##
Run container on port configured in `config.env`
docker run
-i
-t
--rm
--env-file
=
./config.env
-p
=
$(PORT)
:
$(PORT)
--name
=
"
$(APP_NAME)
"
$(REPO)
/
$(APP_NAME)
:
$(TAG_VERSION)
up
:
build run
##
Run container on port configured in `config.env` (Alias to run)
stop
:
##
Stop and remove a running container
docker stop
$(APP_NAME)
;
docker
rm
$(APP_NAME)
# ==========================================================================
#
tag
:
tag-latest tag-version
#
tag-version
##
Generate container tags for the `{version}` and `latest` tags
tag-latest
:
##
Generate container `{version}` tag
@
echo
'create tag latest'
#docker tag
$(APP_NAME)
$(REPO)
/
$(APP_NAME)
:latest
tag-version
:
##
Generate container `latest` tag
@
echo
'creating tag
$(APP_NAME)
:
$(VERSION)
'
docker tag
$(APP_NAME)
:latest
$(APP_NAME)
:
$(VERSION)
# ==========================================================================
#
login
:
##
Auto login to AWS-ECR unsing aws-cli
@
docker login
-u
${CI_REGISTRY_USER}
-p
${CI_REGISTRY_PASSWORD}
${CI_REGISTRY}
echo
"Login COMPLETE"
release-latest
:
build-nc login publish-latest
##
Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
release
:
build-nc publish clean-all
##
Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
release-cached
:
build publish
##
Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
publish
:
login publish-latest publish-version
#
publish-version
##
Publish the `{version}` ans `latest` tagged containers to ECR
@
echo
"Publishing done"
publish-latest
:
##
Publish the `latest` taged container to ECR
@
echo
'publish latest to
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
'
docker tag
$(APP_NAME)
:latest
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:latest
docker push
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:latest
docker rmi
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:latest
publish-version
:
##
Publish the `{version}` taged container to ECR
@
echo
'publish latest to
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:
$(VERSION)
'
docker tag
$(APP_NAME)
:latest
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:
$(VERSION)
docker push
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:
$(VERSION)
docker rmi
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:
$(VERSION)
push
:
login
##
push after login @echo 'push latest to $(REG_NAME)/$(REPO)/$(APP_NAME):latest'
docker tag
$(APP_NAME)
:latest
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:latest
docker push
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:latest
docker rmi
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:latest
# ==========================================================================
#
login-dockerhub
:
##
login to hub.docker.com
@
echo
'docker hub login :'
docker login
push-dockerhub
:
build
#
publish-version
##
Publish the `{version}` ans `latest` tagged containers to ECR
@
echo
'
$(DH_ORG)
/
$(APP_NAME)
:
$(TAG_VERSION)
'
docker push
$(DH_ORG)
/
$(APP_NAME)
:latest
#docker push
$(DH_ORG)
/
$(APP_NAME)
:
$(VERSION)
publish-dockerhub
:
build tag
#
publish-version
##
Publish the `{version}` ans `latest` tagged containers to ECR
@
echo
'
$(DH_ORG)
/
$(APP_NAME)
:
$(TAG_VERSION)
'
docker push
$(DH_ORG)
/
$(APP_NAME)
:latest
#docker push
$(DH_ORG)
/
$(APP_NAME)
:
$(VERSION)
# ==========================================================================
#
clean-tags
:
docker rmi
$(REPO)
/
$(APP_NAME)
:latest
||
true
docker rmi
$(GL_GROUP)
/
$(APP_NAME)
:latest
||
true
docker rmi
$(GL_REG_GROUP)
/
$(APP_NAME)
:latest
||
true
docker rmi
$(REG_NAME)
/
$(REPO)
/
$(APP_NAME)
:latest
||
true
docker rmi
$(REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:latest
||
true
clean
:
docker rmi
$(REPO_NAME)
/
$(APP_NAME)
||
true
docker rmi
$(REPO_NAME)
/
$(APP_NAME)
:
$(TAG_VERSION)
||
true
docker rmi
$(REPO_NAME)
/
$(APP_NAME)
:
$(VERSION)
||
true
docker rmi
$(REPO)
/
$(APP_NAME)
||
true
docker rmi
$(REPO)
/
$(APP_NAME)
:
$(TAG_VERSION)
||
true
docker rmi
$(REPO)
/
$(APP_NAME)
:
$(VERSION)
||
true
docker rmi
$(REG_NAME)
/
$(REPO)
/
$(APP_NAME)
:latest
||
true
docker rmi
$(REG_NAME)
/
$(REPO)
/
$(APP_NAME)
:
$(TAG_VERSION)
||
true
docker rmi
$(REG_NAME)
/
$(REPO)
/
$(APP_NAME)
||
true
docker rmi
$(DH_ORG)
/
$(APP_NAME)
||
true
docker rmi
$(DH_ORG)
/
$(APP_NAME)
:
$(TAG_VERSION)
||
true
docker rmi
$(DH_ORG)
/
$(APP_NAME)
:
$(VERSION)
||
true
docker rmi
$(GL_REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:latest
||
true
docker rmi
$(GL_REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
:
$(VERSION)
||
true
docker rmi
$(GL_REG_NAME)
/
$(GL_REG_GROUP)
/
$(APP_NAME)
||
true
docker rmi
$(GL_REG_GROUP)
/
$(APP_NAME)
||
true
docker rmi
$(GL_REG_GROUP)
/
$(APP_NAME)
:
$(VERSION)
||
true
#docker rmi
$(APP_NAME)
||
true
clean-all
:
clean
docker rmi
$(REPO_NAME)
/
$(APP_NAME)
||
true
version
:
##
Output the current version
@
echo
$(VERSION)
# ==========================================================================
#
info
:
##
Output the current version
@
echo
'VERSION =
$(VERSION)
'
@
echo
'REG_NAME =
$(REG_NAME)
'
@
echo
'APP_NAME =
$(APP_NAME)
'
@
echo
'REPO_NAME =
$(REPO_NAME)
'
@
echo
'DH_ORG =
$(DH_ORG)
'
@
echo
'GL_GROUP =
$(GL_GROUP)
'
@
echo
'GL_REG_GROUP =
$(GL_REG_GROUP)
'
@
echo
'GL_REG_NAME =
$(GL_REG_NAME)
'
@
echo
'REPO_NAME =
$(REPO)
'
@
echo
'TAG_VERSION =
$(TAG_VERSION)
'
ls
:
##
list all docker images
docker images
.containers/docker/usage.sh
deleted
100644 → 0
View file @
7fb3075c
# INSTALL
# - copy the files deploy.env, config.env, version.sh and Makefile to your repo
# - replace the vars in deploy.env
# - define the version script
# Build the container
make build
# Build and publish the container
make release
# Publish a container to AWS-ECR.
# This includes the login to the repo
make publish
# Run the container
make run
# Build an run the container
make up
# Stop the running container
make stop
# Build the container with differnt config and deploy file
make
cnf
=
another_config.env
dpl
=
another_deploy.env build
\ No newline at end of file
.containers/
singularity/
juggler.def
→
.containers/juggler.def
.in
View file @
b8d06008
Bootstrap: docker
From: eicweb.phy.anl.gov:4567/eic/juggler/juggler:
latest
From: eicweb.phy.anl.gov:4567/eic/juggler/juggler:
@JUGGLER_TAG@
%help
singularity container for Juggler development
...
...
.containers/
docker/
version.sh
→
.containers/version.sh
View file @
b8d06008
#!/bin/bash
cat
../
../VERSION |
sed
's/)//'
cat
../VERSION |
sed
's/)//'
.gitignore
View file @
b8d06008
...
...
@@ -22,7 +22,7 @@ TEST*/*
# cmake
CMakeCache.txt
CMakeFiles
Makefile
#
Makefile
cmake_install.cmake
install_manifest.txt
...
...
.gitlab-ci.yml
View file @
b8d06008
image
:
eicweb.phy.anl.gov:4567/containers/eic_container/eic:2.5-stable
variables
:
EIC_TAG
:
"
2.5-stable"
#
default:
#
artifacts:
# path
s:
#
-
build/
default
:
image
:
eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
tag
s
:
-
silicon
stages
:
-
config
-
build
-
docker_build
-
docker_push
-
push
-
deploy
compile
:
stage
:
build
rules
:
-
if
:
'
$CI_COMMIT_BRANCH
!=
"master"
&&
$CI_PIPELINE_SOURCE
==
"merge_request_event"'
tags
:
-
silicon
script
:
-
apt-get update && apt-get install -y rsync
-
export homedir=$(pwd) && pwd && cd /tmp && git clone --depth=1 https://eicweb.phy.anl.gov/EIC/NPDet.git && mkdir build && cd build && cmake ../NPDet/. && make -j20 install
-
cd /tmp && git clone --depth=1 https://eicweb.phy.anl.gov/EIC/eicd.git && mkdir eicd_build && cd eicd_build && cmake ../eicd/. && make -j20 install
-
cd $homedir && ls -lrth && mkdir build && cd build && cmake .. && make -j10
-
cleanup
docker:build:
image
:
eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
stage
:
docker_build
workflow
:
rules
:
-
if
:
'
$CI_PIPELINE_SOURCE
==
"merge_request_event"'
when
:
never
-
if
:
'
$CI_COMMIT_BRANCH
!=
"master"'
when
:
never
-
when
:
on_success
tags
:
-
silicon
-
if
:
'
$CI_COMMIT_BRANCH
==
"master"'
-
if
:
'
$CI_COMMIT_TAG'
env
:
stage
:
config
script
:
-
cd .containers/docker
-
make build-nc
-
export JUGGLER_TAG="latest"
-
|
if [ "x${CI_PIPELINE_SOURCE}" == "xmerge_request_event" ]; then
export JUGGLER_TAG="testing-mr-${CI_MERGE_REQUEST_IID}"
fi
-
echo "CI JUGGLER_TAG for this pipeline set to ${JUGGLER_TAG}"
-
echo "JUGGLER_TAG=$JUGGLER_TAG" >> juggler.env
artifacts
:
reports
:
dotenv
:
juggler.env
docker
:
stage
:
build
needs
:
-
env
script
:
-
echo $JUGGLER_TAG
-
./.gitlabci/configure.sh .containers/Dockerfile.in
-
cd .containers
-
make build
#- make build-nc
docker:push:
image
:
eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
stage
:
docker_push
needs
:
[
"
docker:build"
]
rules
:
-
if
:
'
$CI_PIPELINE_SOURCE
==
"merge_request_event"'
when
:
never
-
if
:
'
$CI_COMMIT_BRANCH
!=
"master"'
when
:
never
-
when
:
on_success
publish
:
stage
:
push
needs
:
-
env
-
docker
script
:
-
cd .containers
-
|
if [ "x$CI_COMMIT_TAG" != "x" ]; then
make publish publish-version
else
make publish
fi
retry
:
max
:
2
when
:
-
runner_system_failure
-
stuck_or_timeout_failure
tags
:
-
silicon
.singularity
:
stage
:
deploy
needs
:
-
env
-
publish
script
:
-
cd .containers/docker
-
make push
-
./.gitlabci/configure.sh .containers/juggler.def.in
-
mkdir -p build
-
mv .containers/juggler.def build
-
singularity build build/juggler.sif build/juggler.def
retry
:
max
:
2
when
:
-
runner_system_failure
-
stuck_or_timeout_failure
docker:release:
image
:
eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
stage
:
docker_build
singularity:latest:
extends
:
.singularity
rules
:
-
if
:
'
$CI_COMMIT_BRANCH
==
"master"'
when
:
on_success
-
if
:
'
$CI_COMMIT_TAG'
tags
:
-
silicon
script
:
-
cd .containers/docker && make release
when
:
on_success
artifacts
:
expire_in
:
90 days
paths
:
-
build/juggler.sif
-
build/juggler.def
docker:singularity:
image
:
eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
stage
:
docker_push
needs
:
[
"
docker:build"
]
singularity:testing:
extends
:
.singularity
rules
:
-
if
:
'
$CI_PIPELINE_SOURCE
==
"merge_request_event"'
when
:
never
-
if
:
'
$CI_COMMIT_BRANCH
!=
"master"'
when
:
never
-
when
:
on_success
tags
:
-
silicon
script
:
-
cp .containers/singularity/juggler.def .
-
/bin/bash .gitlabci/build.sh juggler.def
-
mkdir -p build
-
cp juggler.sif build/.
-
cp juggler.def build/.
when
:
manual
artifacts
:
expire_in
:
90
days
expire_in
:
1
days
paths
:
-
build/juggler.sif
-
build/juggler.def
#singularity:push:
# stage
:
deploy
# when
:
manual
benchmarks:reconstruction:
stage
:
deploy
needs
:
-
env
-
publish
variables
:
JUGGLER_TAG
:
"
$JUGGLER_TAG"
trigger
:
project
:
EIC/benchmarks/reconstruction_benchmarks
needs
:
[
"
docker:push"
]
rules
:
-
if
:
'
$CI_PIPELINE_SOURCE
==
"merge_request_event"'
when
:
never
-
if
:
'
$CI_COMMIT_BRANCH
!=
"master"'
when
:
never
-
when
:
on_success
strategy
:
depend
benchmarks:physics:
stage
:
deploy
needs
:
-
env
-
publish
variables
:
JUGGLER_TAG
:
"
$JUGGLER_TAG"
trigger
:
project
:
EIC/benchmarks/physics_benchmarks
needs
:
[
"
docker:push"
]
strategy
:
depend
purge_image
:
stage
:
cleanup
dependencies
:
-
env
rules
:
-
if
:
'
$CI_PIPELINE_SOURCE
==
"merge_request_event"'
when
:
never
-
if
:
'
$CI_COMMIT_BRANCH
!=
"master"'
when
:
never
-
when
:
on_success
script
:
-
cd .containers
-
make cleanup unpublish
#- make unpublish
.gitlabci/configure.sh
0 → 100755
View file @
b8d06008
#!/bin/bash
## Configure a CI file based on a template file (first and only
## argument to this script).
## Known variables that will be substituted:
## @EIC_TAG@ - docker tag used for the EIC container
## @JUGGLER_TAG@ - output tag for the Juggler version
TEMPLATE_FILE
=
$1
OUTPUT_FILE
=
${
TEMPLATE_FILE
%.*
}
echo
"Configuring CI file:
${
TEMPLATE_FILE
}
"
echo
"Output will be written to:
${
OUTPUT_FILE
}
"
TAG
=
$EIC_VERSION
sed
"s/@EIC_TAG@/
$EIC_TAG
/g"
$TEMPLATE_FILE
|
\
sed
"s/@JUGGLER_TAG@/
$JUGGLER_TAG
/g"
>
${
OUTPUT_FILE
}
echo
"Done"
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment