Merge branch 'staging' into 'master'

Optimize CI setup, add libpoppler, and some minor polish See merge request !11

Merge branch 'staging' into 'master'
24c960fc · Sylvester Joosten · ada0b9b7 · bc5fe556 · 24c960fc · 24c960fc
Commit 24c960fc authored 4 years ago by Sylvester Joosten
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,7 +3,7 @@ image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
 stages:
  - builder
  - config
-  - main
+  - slim
  - singularity

 builder:stable:
@@ -13,12 +13,22 @@ builder:stable:
  only:
     - tags
     - master
-     - staging
  script:
     - cd containers/builder
     - head Dockerfile
     - make login
     - make release-cached
+builder:testing:
+  stage: builder
+  tags:
+     - silicon
+  only:
+     - staging
+  script:
+     - cd containers/builder
+     - head Dockerfile
+     - make login
+     - make staging-cached
 builder:unstable:
  stage: builder
  tags:
@@ -31,7 +41,7 @@ builder:unstable:
     - cd containers/builder
     - head Dockerfile
     - make login
-     - make develop
+     - make develop-cached

 ## I wish there were a simple way to transfer the "latest/unstable" strings
 ## between jobs...
@@ -43,10 +53,24 @@ config:stable:
  only:
     - tags
     - master
-     - staging
  needs: ["builder:stable"]
  script:
-    - bash containers/release/configure_release.sh
+    - bash containers/release/configure_release.sh latest
+  artifacts:
+    paths:
+      - config/spack-env.sh
+      - config/eic-env.sh
+      - config/Dockerfile
+config:testing:
+  image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:testing
+  stage: config
+  tags:
+    - silicon
+  only:
+     - staging
+  needs: ["builder:testing"]
+  script:
+    - bash containers/release/configure_release.sh testing
  artifacts:
    paths:
      - config/spack-env.sh
@@ -63,7 +87,7 @@ config:unstable:
     - staging
  needs: ["builder:unstable"]
  script:
-    - bash containers/release/configure_release.sh
+    - bash containers/release/configure_release.sh unstable
  artifacts:
    paths:
      - config/spack-env.sh
@@ -71,13 +95,12 @@ config:unstable:
      - config/Dockerfile
      
 release:stable:
-  stage: main
+  stage: slim
  tags:
     - silicon
  only:
     - tags
     - master
-     - staging
  needs: ["config:stable"]
  script:
     - cp config/Dockerfile containers/release/Dockerfile
@@ -85,8 +108,21 @@ release:stable:
     - cd containers/release
     - make login
     - make release-cached
+release:testing:
+  stage: slim
+  tags:
+     - silicon
+  only:
+     - staging
+  needs: ["config:testing"]
+  script:
+     - cp config/Dockerfile containers/release/Dockerfile
+     - cp config/eic-env.sh containers/release/eic-env.sh
+     - cd containers/release
+     - make login
+     - make staging-cached
 release:unstable:
-  stage: main
+  stage: slim
  tags:
     - silicon
  except:
@@ -106,7 +142,6 @@ builder:singularity:
  tags:
     - singularity
  only:
-     - staging
     - tags
  needs: ["builder:stable"]
  when: manual
@@ -127,7 +162,6 @@ release:singularity:
  tags:
     - singularity
  only:
-     - staging
     - tags
  needs: ["release:stable"]
  script:

--- a/VERSION
+++ b/VERSION
-2.2.1
+2.2.2
--- a/containers/builder/Makefile
+++ b/containers/builder/Makefile
@@ -85,8 +85,13 @@ develop: build-nc publish-dev ## Develop build, do not publish latest
 	@echo "Cleaning up"
 	#docker rmi $(APP_NAME):latest

+staging: build-nc publish-staging ## Develop build, do not publish latest
+	@echo "Cleaning up"
+	#docker rmi $(APP_NAME):latest
+
 release-cached: build publish ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
 develop-cached: build login publish-dev ## develop release
+staging-cached: build login publish-staging ## develop release

 publish: login publish-latest publish-version #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
 	@echo "Publishing done"
@@ -98,6 +103,11 @@ publish-latest: ## Publish the `latest` taged container to ECR
 	docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
 	#docker rmi  $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest

+publish-staging: ## Publish the `unstable` taged container to ECR
+	@echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
+	docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):testing
+	docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):testing
+
 publish-dev: ## Publish the `unstable` taged container to ECR
 	@echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
 	docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable

--- a/containers/release/Dockerfile.in
+++ b/containers/release/Dockerfile.in
 # Release container for Argonne EIC software
 # Copy over relevant files from builder
 #
-FROM eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:latest as builder
+FROM eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:@TAG@ as builder
 RUN cd /opt/spack-environment && spack env activate . && spack gc -y

 # Strip all the binaries
@@ -28,12 +28,15 @@ ENV DOCKERFILE_DISTRO=ubuntu          \

 ## TODO: move this to upstream, as we should have ssh available in
 ## any dev container
+## poppler-utils for pdftoppm, is this is right now now supported by the
+## Spack version
 RUN apt-get -yqq update \
 && apt-get -yqq install --no-install-recommends \
-        openssh-client \
        ghostscript \
        gv \
- && locale-gen en_US.UTF-8 \
+        poppler-utils \
+ && apt-get -yqq purge lmod \
+ && apt-get -yqq autoremove \
 && rm -rf /var/lib/apt/lists/*

 ## Copy over files from builder

--- a/containers/release/Makefile
+++ b/containers/release/Makefile
@@ -81,11 +81,16 @@ release: build-nc publish ## Make a release by building and publishing the `{ver
 	@echo "Cleaning up"
 	#docker rmi $(APP_NAME):latest

+staging: build-nc publish-staging ## Develop build, do not publish latest
+	@echo "Cleaning up"
+	#docker rmi $(APP_NAME):latest
+
 develop: build-nc publish-dev ## Develop build, do not publish latest
 	@echo "Cleaning up"
 	#docker rmi $(APP_NAME):latest

 release-cached: build publish ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
+staging-cached: build login publish-staging ## staging release
 develop-cached: build login publish-dev ## develop release

 publish: login publish-latest publish-version #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
@@ -98,6 +103,11 @@ publish-latest: ## Publish the `latest` taged container to ECR
 	docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
 	#docker rmi  $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest

+publish-staging: ## Publish the `unstable` taged container to ECR
+	@echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
+	docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):testing
+	docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):testing
+
 publish-dev: ## Publish the `unstable` taged container to ECR
 	@echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
 	docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable

--- a/containers/release/configure_release.sh
+++ b/containers/release/configure_release.sh
@@ -9,6 +9,8 @@ mkdir -p config

 cp /etc/profile.d/z10_spack_environment.sh config/spack-env.sh

+export TAG=$1
+
 ## Spack sets the man-path, which stops bash from using the default man-path
 ## We can fix this by appending a trailing colon to MANPATH
 sed -i '/MANPATH/ s/;$/:;/' config/spack-env.sh
@@ -20,4 +22,5 @@ grep export config/spack-env.sh | \

 ## create our release Dockerfile
 sed '/^@ENV@/r config/eic-env.sh' containers/release/Dockerfile.in | \
-  sed '/^@ENV@/d' > config/Dockerfile
+  sed '/^@ENV@/d' | \
+  sed "s/@TAG@/$TAG/" > config/Dockerfile