Skip to content
Snippets Groups Projects
Commit 24c960fc authored by Sylvester Joosten's avatar Sylvester Joosten
Browse files

Merge branch 'staging' into 'master' 

Optimize CI setup, add libpoppler, and some minor polish

See merge request !11
parents ada0b9b7 bc5fe556
No related branches found
Tags v2.2.2
1 merge request!11Optimize CI setup, add libpoppler, and some minor polish
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 45
− 11
View file @ 24c960fc
...@@ -3,7 +3,7 @@ image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest ...@@ -3,7 +3,7 @@ image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
stages: stages:
- builder - builder
- config - config
- main - slim
- singularity - singularity
builder:stable: builder:stable:
...@@ -13,12 +13,22 @@ builder:stable: ...@@ -13,12 +13,22 @@ builder:stable:
only: only:
- tags - tags
- master - master
- staging
script: script:
- cd containers/builder - cd containers/builder
- head Dockerfile - head Dockerfile
- make login - make login
- make release-cached - make release-cached
builder:testing:
stage: builder
tags:
- silicon
only:
- staging
script:
- cd containers/builder
- head Dockerfile
- make login
- make staging-cached
builder:unstable: builder:unstable:
stage: builder stage: builder
tags: tags:
...@@ -31,7 +41,7 @@ builder:unstable: ...@@ -31,7 +41,7 @@ builder:unstable:
- cd containers/builder - cd containers/builder
- head Dockerfile - head Dockerfile
- make login - make login
- make develop - make develop-cached
## I wish there were a simple way to transfer the "latest/unstable" strings ## I wish there were a simple way to transfer the "latest/unstable" strings
## between jobs... ## between jobs...
...@@ -43,10 +53,24 @@ config:stable: ...@@ -43,10 +53,24 @@ config:stable:
only: only:
- tags - tags
- master - master
- staging
needs: ["builder:stable"] needs: ["builder:stable"]
script: script:
- bash containers/release/configure_release.sh - bash containers/release/configure_release.sh latest
artifacts:
paths:
- config/spack-env.sh
- config/eic-env.sh
- config/Dockerfile
config:testing:
image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:testing
stage: config
tags:
- silicon
only:
- staging
needs: ["builder:testing"]
script:
- bash containers/release/configure_release.sh testing
artifacts: artifacts:
paths: paths:
- config/spack-env.sh - config/spack-env.sh
...@@ -63,7 +87,7 @@ config:unstable: ...@@ -63,7 +87,7 @@ config:unstable:
- staging - staging
needs: ["builder:unstable"] needs: ["builder:unstable"]
script: script:
- bash containers/release/configure_release.sh - bash containers/release/configure_release.sh unstable
artifacts: artifacts:
paths: paths:
- config/spack-env.sh - config/spack-env.sh
...@@ -71,13 +95,12 @@ config:unstable: ...@@ -71,13 +95,12 @@ config:unstable:
- config/Dockerfile - config/Dockerfile
release:stable: release:stable:
stage: main stage: slim
tags: tags:
- silicon - silicon
only: only:
- tags - tags
- master - master
- staging
needs: ["config:stable"] needs: ["config:stable"]
script: script:
- cp config/Dockerfile containers/release/Dockerfile - cp config/Dockerfile containers/release/Dockerfile
...@@ -85,8 +108,21 @@ release:stable: ...@@ -85,8 +108,21 @@ release:stable:
- cd containers/release - cd containers/release
- make login - make login
- make release-cached - make release-cached
release:testing:
stage: slim
tags:
- silicon
only:
- staging
needs: ["config:testing"]
script:
- cp config/Dockerfile containers/release/Dockerfile
- cp config/eic-env.sh containers/release/eic-env.sh
- cd containers/release
- make login
- make staging-cached
release:unstable: release:unstable:
stage: main stage: slim
tags: tags:
- silicon - silicon
except: except:
...@@ -106,7 +142,6 @@ builder:singularity: ...@@ -106,7 +142,6 @@ builder:singularity:
tags: tags:
- singularity - singularity
only: only:
- staging
- tags - tags
needs: ["builder:stable"] needs: ["builder:stable"]
when: manual when: manual
...@@ -127,7 +162,6 @@ release:singularity: ...@@ -127,7 +162,6 @@ release:singularity:
tags: tags:
- singularity - singularity
only: only:
- staging
- tags - tags
needs: ["release:stable"] needs: ["release:stable"]
script: script:
......
VERSION
+ 1
− 1
View file @ 24c960fc
2.2.1 2.2.2
containers/builder/Makefile
+ 10
− 0
View file @ 24c960fc
...@@ -85,8 +85,13 @@ develop: build-nc publish-dev ## Develop build, do not publish latest ...@@ -85,8 +85,13 @@ develop: build-nc publish-dev ## Develop build, do not publish latest
@echo "Cleaning up" @echo "Cleaning up"
#docker rmi $(APP_NAME):latest #docker rmi $(APP_NAME):latest
staging: build-nc publish-staging ## Develop build, do not publish latest
@echo "Cleaning up"
#docker rmi $(APP_NAME):latest
release-cached: build publish ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR release-cached: build publish ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
develop-cached: build login publish-dev ## develop release develop-cached: build login publish-dev ## develop release
staging-cached: build login publish-staging ## develop release
publish: login publish-latest publish-version #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR publish: login publish-latest publish-version #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
@echo "Publishing done" @echo "Publishing done"
...@@ -98,6 +103,11 @@ publish-latest: ## Publish the `latest` taged container to ECR ...@@ -98,6 +103,11 @@ publish-latest: ## Publish the `latest` taged container to ECR
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
#docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest #docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
publish-staging: ## Publish the `unstable` taged container to ECR
@echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):testing
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):testing
publish-dev: ## Publish the `unstable` taged container to ECR publish-dev: ## Publish the `unstable` taged container to ECR
@echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)' @echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable
......
containers/release/Dockerfile.in
+ 6
− 3
View file @ 24c960fc
# Release container for Argonne EIC software # Release container for Argonne EIC software
# Copy over relevant files from builder # Copy over relevant files from builder
# #
FROM eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:latest as builder FROM eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:@TAG@ as builder
RUN cd /opt/spack-environment && spack env activate . && spack gc -y RUN cd /opt/spack-environment && spack env activate . && spack gc -y
# Strip all the binaries # Strip all the binaries
...@@ -28,12 +28,15 @@ ENV DOCKERFILE_DISTRO=ubuntu \ ...@@ -28,12 +28,15 @@ ENV DOCKERFILE_DISTRO=ubuntu \
## TODO: move this to upstream, as we should have ssh available in ## TODO: move this to upstream, as we should have ssh available in
## any dev container ## any dev container
## poppler-utils for pdftoppm, is this is right now now supported by the
## Spack version
RUN apt-get -yqq update \ RUN apt-get -yqq update \
&& apt-get -yqq install --no-install-recommends \ && apt-get -yqq install --no-install-recommends \
openssh-client \
ghostscript \ ghostscript \
gv \ gv \
&& locale-gen en_US.UTF-8 \ poppler-utils \
&& apt-get -yqq purge lmod \
&& apt-get -yqq autoremove \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
## Copy over files from builder ## Copy over files from builder
......
containers/release/Makefile
+ 10
− 0
View file @ 24c960fc
...@@ -81,11 +81,16 @@ release: build-nc publish ## Make a release by building and publishing the `{ver ...@@ -81,11 +81,16 @@ release: build-nc publish ## Make a release by building and publishing the `{ver
@echo "Cleaning up" @echo "Cleaning up"
#docker rmi $(APP_NAME):latest #docker rmi $(APP_NAME):latest
staging: build-nc publish-staging ## Develop build, do not publish latest
@echo "Cleaning up"
#docker rmi $(APP_NAME):latest
develop: build-nc publish-dev ## Develop build, do not publish latest develop: build-nc publish-dev ## Develop build, do not publish latest
@echo "Cleaning up" @echo "Cleaning up"
#docker rmi $(APP_NAME):latest #docker rmi $(APP_NAME):latest
release-cached: build publish ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR release-cached: build publish ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
staging-cached: build login publish-staging ## staging release
develop-cached: build login publish-dev ## develop release develop-cached: build login publish-dev ## develop release
publish: login publish-latest publish-version #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR publish: login publish-latest publish-version #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
...@@ -98,6 +103,11 @@ publish-latest: ## Publish the `latest` taged container to ECR ...@@ -98,6 +103,11 @@ publish-latest: ## Publish the `latest` taged container to ECR
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
#docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest #docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
publish-staging: ## Publish the `unstable` taged container to ECR
@echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):testing
docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):testing
publish-dev: ## Publish the `unstable` taged container to ECR publish-dev: ## Publish the `unstable` taged container to ECR
@echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)' @echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable
......
containers/release/configure_release.sh
+ 4
− 1
View file @ 24c960fc
...@@ -9,6 +9,8 @@ mkdir -p config ...@@ -9,6 +9,8 @@ mkdir -p config
cp /etc/profile.d/z10_spack_environment.sh config/spack-env.sh cp /etc/profile.d/z10_spack_environment.sh config/spack-env.sh
export TAG=$1
## Spack sets the man-path, which stops bash from using the default man-path ## Spack sets the man-path, which stops bash from using the default man-path
## We can fix this by appending a trailing colon to MANPATH ## We can fix this by appending a trailing colon to MANPATH
sed -i '/MANPATH/ s/;$/:;/' config/spack-env.sh sed -i '/MANPATH/ s/;$/:;/' config/spack-env.sh
...@@ -20,4 +22,5 @@ grep export config/spack-env.sh | \ ...@@ -20,4 +22,5 @@ grep export config/spack-env.sh | \
## create our release Dockerfile ## create our release Dockerfile
sed '/^@ENV@/r config/eic-env.sh' containers/release/Dockerfile.in | \ sed '/^@ENV@/r config/eic-env.sh' containers/release/Dockerfile.in | \
sed '/^@ENV@/d' > config/Dockerfile sed '/^@ENV@/d' | \
sed "s/@TAG@/$TAG/" > config/Dockerfile
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment