Finished refactoring CI, need testing

61ad8351 · Sylvester Joosten · a30f1ccb · 61ad8351 · 61ad8351 · 61ad8351
Commit 61ad8351 authored 4 years ago by Sylvester Joosten
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
 image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest

 stages:
-  - builder
  - config
-  - slim
-  - singularity
+  - build_and_deploy

-## We use minimal file artifacts to transport variables between job
-## stages, such as the branch we are working on and the need for 
-## caching.
-## By evaluating these files in the before_script, we can set
-## relevant environment variables before our pipelines run
+## make note if we cannot use caching for one of the stages
+## by touching files in .ci_env
 default:
  artifacts:
    paths: 
      - .ci-env
-  before_script:
-    - mkdir -p .ci-env
-    - |
-      if [ -f .ci-env/release ]; then 
-        export BRANCH='release'
-      elif [ -f .ci-env/develop ]; then
-        export BRANCH='develop'
-      else export BRANCH='UNKNOWN'
-      fi
-    - |
-      if [ -f .ci-env/builder-nc ]; then 
-        export BUILDER_TARGET="${BRANCH}"
-      else
-        export BUILDER_TARGET="${BRANCH}-cached"
-      fi
-    - |
-      if [ -f .ci-env/release-nc ]; then 
-        export RELEASE_TARGET="${BRANCH}"
-      else
-        export RELEASE_TARGET="${BRANCH}-cached"
-      fi
-  tags:
-    - silicon
-
-## Stable or unstable branch?
-init:stable:
-  stage: .pre
-  rules:
-    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == "master"'
-      when: always
-  script:
-    - touch .ci-env/release
-init:unstable:
-  stage: .pre
-  rules:
-    - if: '$CI_COMMIT_TAG == null && $CI_COMMIT_BRANCH != "master"'
-      when: always
-  script:
-    - touch .ci-env/develop
-init:builder-nc:
+detect_changes:builder:
  stage: .pre
  rules:
    - changes:
+        - containers/Makefile
        - containers/builder/Dockerfile
        - containers/builder/spack.yaml
        - spack/packages/**/*
  script:
+    - mkdir -p .ci_env
    - touch .ci-env/builder-nc
-init:release-nc:
+detect_changes:release:
  stage: .pre
  rules:
    - changes:
+        - containers/Makefile
        - containers/release/Dockerfile.in
        - containers/release/configure_release.sh
  script:
+    - mkdir -p .ci_env
    - touch .ci-env/release-nc

-builder:
-  stage: builder
+## Init our job for our desired branches/tags/events
+init:
+  stage: config
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
-      when: on_success
-    - if: '$CI_COMMIT_BRANCH == "develop"'
-      when: on_success
-    - if: '$CI_COMMIT_TAG'
-      when: on_success
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH != "develop"'
-      when: on_success
-  script:
-    - cp -r spack containers/builder/np-spack
-    - cd containers/builder
-    - head Dockerfile
-    - make login
-    - echo "Creating builder image for ${BUILDER_TARGET}"
-    - make ${BUILDER_TARGET}
-
-.config:
-  stage: config
+      when: always
+    - if: '$CI_COMMIT_BRANCH ~= "/^v[0-9]+\.[0-9]-stable/"'
+      when: always
+    - if: '$CI_COMMIT_TAG ~= "/^v[0-9]+\.[0-9]+\.[0-9]+/"'
+      when: always
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: always
  script:
-    - bash containers/release/configure_release.sh ${BRANCH}
+    - ./ci/configure_pipeline.sh ci/build_and_deploy.yml.in
  artifacts:
    paths:
-      - config
-config:stable:
-  extends: .config
-  image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:latest
-  only:
-    - tags
-    - master
-config:unstable:
-  extends: .config
-  image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:unstable
-  rules:
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH != "develop"'
-      when: manual
-    - if: '$CI_COMMIT_BRANCH == "develop"'
-      when: on_success
-      
-release:
-  stage: slim
-  rules:
-    - if: '$CI_COMMIT_BRANCH == "master"'
-      when: on_success
-    - if: '$CI_COMMIT_BRANCH == "develop"'
-      when: on_success
-    - if: '$CI_COMMIT_TAG'
-      when: on_success
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH != "develop"'
-      when: manual
-  script:
-     - cp config/Dockerfile containers/release/Dockerfile
-     - cp config/eic-env.sh containers/release/eic-env.sh
-     - cd containers/release
-     - make login
-     - make ${RELEASE_TARGET}
+      - build_and_deploy.yml

-release:singularity:
-  stage: singularity
-  rules:
-    - if: '$CI_COMMIT_BRANCH == "master"'
-      when: manual
-    - if: '$CI_COMMIT_BRANCH == "develop"'
-      when: manual
-    - if: '$CI_COMMIT_TAG'
-      when: on_success
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH != "develop"'
-      when: manual
-  script:
-     - cp config/eic.def eic.def
-     - /bin/bash .gitlabci/setup.sh
-     - /bin/bash .gitlabci/build.sh eic.def
-     - mkdir -p build 
-     - cp eic.sif build/.
-     - cp eic.def build/.
-  artifacts:
-      expire_in: 90 days
-      paths:
-        - build/eic.sif
-        - build/eic.def
+## Dispatch if we ran the previous stage
+run:default:
+  stage: build_and_deploy
+  needs: ["init"]
+  trigger:
+    include: 
+      - artifact: build_and_deploy.yml
+      - job: build_and_deploy
--- a/ci/build_and_deploy.yml
+++ b/ci/build_and_deploy.yml
+image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
+
+stages:
+  - build
+  - config
+  - package
+  - singularity
+
+## variables:
+##   - TARGET_XXX: docker build target (including cache modifier) 
+##     (stable, stable-cached, unstable, unstable-cached)
+##   - TAG: main docker tag to be used internally 
+##     (e.g. 2.5-stable/unstable/<version>)
+##   - PUBLISH: docker publish directives
+variables:
+  TARGET_BUILDER: @TARGET_BUILDER@
+  TARGET_RELEASE: @TARGET_RELEASE@
+  TAG: @TAG@
+  PUBLISH: @PUBLISH@
+
+builder:
+  stage: build
+  script:
+    - cp -r spack containers/builder/np-spack
+    - cd containers/builder
+    - head Dockerfile
+    - make login
+    - echo "Creating builder image for: $TARGET_BUILDER"
+    - make $TARGET_BUILDER
+    - echo "Publishing image: $PUBLISH"
+    - make $PUBLISH
+
+config:
+  image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:$TAG
+  stage: config
+  needs: ["builder"]
+  script:
+    - bash containers/release/configure_release.sh $TAG
+  artifacts:
+    paths:
+      - config
+      
+release:docker:
+  stage: package
+  needs: ["config"]
+  script:
+    - cp config/Dockerfile containers/release/Dockerfile
+    - cp config/eic-env.sh containers/release/eic-env.sh
+    - cd containers/release
+    - make login
+    - echo "Creating release image for: $TARGET_RELEASE"
+    - make $TARGET_RELEASE
+    - echo "Publishing image: $PUBLISH"
+    - make $PUBLISH
+
+release:singularity:
+  stage: singularity
+  needs: ["release:docker"]
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master"'
+      when: on_success
+    - if: '$CI_COMMIT_BRANCH == "v$TAG"'
+      when: on_success
+    - if: '$CI_COMMIT_TAG == "v$TAG"'
+      when: on_success
+  script:
+     - cp config/eic.def eic.def
+     - /bin/bash .gitlabci/setup.sh
+     - /bin/bash .gitlabci/build.sh eic.def
+     - mkdir -p build 
+     - cp eic.sif build/.
+     - cp eic.def build/.
+  artifacts:
+      expire_in: 90 days
+      paths:
+        - build/eic.sif
+        - build/eic.def
--- a/ci/configure_pipeline.sh
+++ b/ci/configure_pipeline.sh
+#!/bin/bash
+
+## Configure a CI pipeline based on a template file (first and only
+## argument to this script).
+TEMPLATE_FILE=$1
+OUTPUT_FILE=`basename ${TEMPLATE_FILE} .in`
+
+echo "Configuring pipeline script: ${TEMPLATE_FILE}"
+echo "Output will be written to: ${OUTPUT_FILE}"
+
+VERSION=`head -n1 VERSION`
+STABLE="${VERSION%.*}-stable"
+
+## Figure out which scenario we are running:
+##  - master
+##  - stable
+##  - tag
+##  - unstable (default)
+TARGET=""
+TAG=""
+PUBLISH=""
+if [ $CI_COMMIT_BRANCH = "master" ]; then
+  TARGET="stable"
+	TAG="latest"
+	PUBLISH="publish-latest publish-stable"
+elif [ $CI_COMMIT_TAG = "v${VERSION}" ]; then
+  TARGET=$VERSION
+	TAG=$VERSION
+	PUBLISH="publish-version"
+elif [ $CI_COMMIT_BRANCH = "v${STABLE}" ]; then
+  TARGET="stable"
+	TAG=${STABLE}
+	PUBLISH="publish-stable"
+else
+  TARGET="unstable"
+	TAG="unstable"
+	PUBLISH="publish-unstable"
+fi
+
+TARGET_BUILDER=$TARGET
+TARGET_RELEASE=$TARGET
+
+if [ ! -f .ci_env/buider-nc ]; then
+  TARGET_BUILDER="${TARGET_BUILDER}-cached"
+fi
+if [ ! -f .ci_env/release-nc ]; then
+  TARGET_BUILDER="${TARGET_RELEASE}-cached"
+fi
+
+sed "/@TAG@/$TAG/g" $TEMPLATE_FILE | \
+  sed "/@TARGET_BUILDER@/$TARGET_BUILDER/g" | \
+	sed "/@TARGET_RELEASE@/$TARGET_RELEASE/g" | \
+	sed "/@PUBLISH@/$PUBLISH/g" > ${OUTPUT_FILE}
+
+echo "Done"
--- a/containers/Makefile
+++ b/containers/Makefile
@@ -13,7 +13,7 @@ SHELL = bash

 ## Get our version tag and stable version tag
 VERSION=$(shell head -n1 ../../VERSION)
-STABLE=$(shell echo ${$(head -n1 ../../VERSION)%.*})
+STABLE=$(shell echo ${$(head -n1 ../../VERSION)%.*}-stable)

 # help will output the help for each task
 # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
@@ -42,12 +42,8 @@ login: ## Auto login to AWS-ECR unsing aws-cli
 	docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
 	echo "Login COMPLETE"

-master: build-nc publish-latest publish-stable
-master-cached: build publish-latest publish-stable
-stable: build-nc publish-stable 
-stable-cached: build publish-stable
-version: build-nc publish-version 
-version-cached: build publish-version
+stable: build-nc
+stable-cached: build
 unstable: build-unstable-nc publish-unstable 
 version-unstable: build-unstable publish-unstable

@@ -55,16 +51,16 @@ publish: login publish-latest publish-version #publish-version ## Publish the `{
 	@echo "Publishing done"


-publish-latest: login 
-	@echo 'publish latest to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
-	docker tag $(APP_NAME):$(VERSION) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
-	docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
-
 publish-stable: login 
 	@echo 'publish $(STABLE) to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
 	docker tag $(APP_NAME):$(VERSION) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(STABLE)
 	docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(STABLE)

+publish-latest: login 
+	@echo 'publish latest to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
+	docker tag $(APP_NAME):$(VERSION) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
+	docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest
+
 publish-version: login 
 	@echo 'publish $(STABLE) to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)'
 	docker tag $(APP_NAME):$(VERSION) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)