diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5175a1a97ad8aeef1e8a304407302cbed17d049f..1de0061cbd9b2a82d990a2e4f61ac1f46a36d373 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,156 +1,60 @@ image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest stages: - - builder - config - - slim - - singularity + - build_and_deploy -## We use minimal file artifacts to transport variables between job -## stages, such as the branch we are working on and the need for -## caching. -## By evaluating these files in the before_script, we can set -## relevant environment variables before our pipelines run +## make note if we cannot use caching for one of the stages +## by touching files in .ci_env default: artifacts: paths: - .ci-env - before_script: - - mkdir -p .ci-env - - | - if [ -f .ci-env/release ]; then - export BRANCH='release' - elif [ -f .ci-env/develop ]; then - export BRANCH='develop' - else export BRANCH='UNKNOWN' - fi - - | - if [ -f .ci-env/builder-nc ]; then - export BUILDER_TARGET="${BRANCH}" - else - export BUILDER_TARGET="${BRANCH}-cached" - fi - - | - if [ -f .ci-env/release-nc ]; then - export RELEASE_TARGET="${BRANCH}" - else - export RELEASE_TARGET="${BRANCH}-cached" - fi - tags: - - silicon - -## Stable or unstable branch? -init:stable: - stage: .pre - rules: - - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == "master"' - when: always - script: - - touch .ci-env/release -init:unstable: - stage: .pre - rules: - - if: '$CI_COMMIT_TAG == null && $CI_COMMIT_BRANCH != "master"' - when: always - script: - - touch .ci-env/develop -init:builder-nc: +detect_changes:builder: stage: .pre rules: - changes: + - containers/Makefile - containers/builder/Dockerfile - containers/builder/spack.yaml - spack/packages/**/* script: + - mkdir -p .ci_env - touch .ci-env/builder-nc -init:release-nc: +detect_changes:release: stage: .pre rules: - changes: + - containers/Makefile - containers/release/Dockerfile.in - containers/release/configure_release.sh script: + - mkdir -p .ci_env - touch .ci-env/release-nc -builder: - stage: builder +## Init our job for our desired branches/tags/events +init: + stage: config rules: - if: '$CI_COMMIT_BRANCH == "master"' - when: on_success - - if: '$CI_COMMIT_BRANCH == "develop"' - when: on_success - - if: '$CI_COMMIT_TAG' - when: on_success - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH != "develop"' - when: on_success - script: - - cp -r spack containers/builder/np-spack - - cd containers/builder - - head Dockerfile - - make login - - echo "Creating builder image for ${BUILDER_TARGET}" - - make ${BUILDER_TARGET} - -.config: - stage: config + when: always + - if: '$CI_COMMIT_BRANCH ~= "/^v[0-9]+\.[0-9]-stable/"' + when: always + - if: '$CI_COMMIT_TAG ~= "/^v[0-9]+\.[0-9]+\.[0-9]+/"' + when: always + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always script: - - bash containers/release/configure_release.sh ${BRANCH} + - ./ci/configure_pipeline.sh ci/build_and_deploy.yml.in artifacts: paths: - - config -config:stable: - extends: .config - image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:latest - only: - - tags - - master -config:unstable: - extends: .config - image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:unstable - rules: - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH != "develop"' - when: manual - - if: '$CI_COMMIT_BRANCH == "develop"' - when: on_success - -release: - stage: slim - rules: - - if: '$CI_COMMIT_BRANCH == "master"' - when: on_success - - if: '$CI_COMMIT_BRANCH == "develop"' - when: on_success - - if: '$CI_COMMIT_TAG' - when: on_success - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH != "develop"' - when: manual - script: - - cp config/Dockerfile containers/release/Dockerfile - - cp config/eic-env.sh containers/release/eic-env.sh - - cd containers/release - - make login - - make ${RELEASE_TARGET} + - build_and_deploy.yml -release:singularity: - stage: singularity - rules: - - if: '$CI_COMMIT_BRANCH == "master"' - when: manual - - if: '$CI_COMMIT_BRANCH == "develop"' - when: manual - - if: '$CI_COMMIT_TAG' - when: on_success - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH != "develop"' - when: manual - script: - - cp config/eic.def eic.def - - /bin/bash .gitlabci/setup.sh - - /bin/bash .gitlabci/build.sh eic.def - - mkdir -p build - - cp eic.sif build/. - - cp eic.def build/. - artifacts: - expire_in: 90 days - paths: - - build/eic.sif - - build/eic.def +## Dispatch if we ran the previous stage +run:default: + stage: build_and_deploy + needs: ["init"] + trigger: + include: + - artifact: build_and_deploy.yml + - job: build_and_deploy diff --git a/ci/build_and_deploy.yml b/ci/build_and_deploy.yml new file mode 100644 index 0000000000000000000000000000000000000000..ca43feb850a653603bc228793cb17b418b799206 --- /dev/null +++ b/ci/build_and_deploy.yml @@ -0,0 +1,77 @@ +image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest + +stages: + - build + - config + - package + - singularity + +## variables: +## - TARGET_XXX: docker build target (including cache modifier) +## (stable, stable-cached, unstable, unstable-cached) +## - TAG: main docker tag to be used internally +## (e.g. 2.5-stable/unstable/<version>) +## - PUBLISH: docker publish directives +variables: + TARGET_BUILDER: @TARGET_BUILDER@ + TARGET_RELEASE: @TARGET_RELEASE@ + TAG: @TAG@ + PUBLISH: @PUBLISH@ + +builder: + stage: build + script: + - cp -r spack containers/builder/np-spack + - cd containers/builder + - head Dockerfile + - make login + - echo "Creating builder image for: $TARGET_BUILDER" + - make $TARGET_BUILDER + - echo "Publishing image: $PUBLISH" + - make $PUBLISH + +config: + image: eicweb.phy.anl.gov:4567/containers/eic_container/eic_builder:$TAG + stage: config + needs: ["builder"] + script: + - bash containers/release/configure_release.sh $TAG + artifacts: + paths: + - config + +release:docker: + stage: package + needs: ["config"] + script: + - cp config/Dockerfile containers/release/Dockerfile + - cp config/eic-env.sh containers/release/eic-env.sh + - cd containers/release + - make login + - echo "Creating release image for: $TARGET_RELEASE" + - make $TARGET_RELEASE + - echo "Publishing image: $PUBLISH" + - make $PUBLISH + +release:singularity: + stage: singularity + needs: ["release:docker"] + rules: + - if: '$CI_COMMIT_BRANCH == "master"' + when: on_success + - if: '$CI_COMMIT_BRANCH == "v$TAG"' + when: on_success + - if: '$CI_COMMIT_TAG == "v$TAG"' + when: on_success + script: + - cp config/eic.def eic.def + - /bin/bash .gitlabci/setup.sh + - /bin/bash .gitlabci/build.sh eic.def + - mkdir -p build + - cp eic.sif build/. + - cp eic.def build/. + artifacts: + expire_in: 90 days + paths: + - build/eic.sif + - build/eic.def diff --git a/ci/configure_pipeline.sh b/ci/configure_pipeline.sh new file mode 100644 index 0000000000000000000000000000000000000000..298089934d229ebd62ef1aaa33fdc4908f241ed8 --- /dev/null +++ b/ci/configure_pipeline.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +## Configure a CI pipeline based on a template file (first and only +## argument to this script). +TEMPLATE_FILE=$1 +OUTPUT_FILE=`basename ${TEMPLATE_FILE} .in` + +echo "Configuring pipeline script: ${TEMPLATE_FILE}" +echo "Output will be written to: ${OUTPUT_FILE}" + +VERSION=`head -n1 VERSION` +STABLE="${VERSION%.*}-stable" + +## Figure out which scenario we are running: +## - master +## - stable +## - tag +## - unstable (default) +TARGET="" +TAG="" +PUBLISH="" +if [ $CI_COMMIT_BRANCH = "master" ]; then + TARGET="stable" + TAG="latest" + PUBLISH="publish-latest publish-stable" +elif [ $CI_COMMIT_TAG = "v${VERSION}" ]; then + TARGET=$VERSION + TAG=$VERSION + PUBLISH="publish-version" +elif [ $CI_COMMIT_BRANCH = "v${STABLE}" ]; then + TARGET="stable" + TAG=${STABLE} + PUBLISH="publish-stable" +else + TARGET="unstable" + TAG="unstable" + PUBLISH="publish-unstable" +fi + +TARGET_BUILDER=$TARGET +TARGET_RELEASE=$TARGET + +if [ ! -f .ci_env/buider-nc ]; then + TARGET_BUILDER="${TARGET_BUILDER}-cached" +fi +if [ ! -f .ci_env/release-nc ]; then + TARGET_BUILDER="${TARGET_RELEASE}-cached" +fi + +sed "/@TAG@/$TAG/g" $TEMPLATE_FILE | \ + sed "/@TARGET_BUILDER@/$TARGET_BUILDER/g" | \ + sed "/@TARGET_RELEASE@/$TARGET_RELEASE/g" | \ + sed "/@PUBLISH@/$PUBLISH/g" > ${OUTPUT_FILE} + +echo "Done" diff --git a/containers/Makefile b/containers/Makefile index 25b919cbfe6d23914367a10df78a8080f9a744c2..30522dd6821dac5b9b993715c6981c412b5456a6 100644 --- a/containers/Makefile +++ b/containers/Makefile @@ -13,7 +13,7 @@ SHELL = bash ## Get our version tag and stable version tag VERSION=$(shell head -n1 ../../VERSION) -STABLE=$(shell echo ${$(head -n1 ../../VERSION)%.*}) +STABLE=$(shell echo ${$(head -n1 ../../VERSION)%.*}-stable) # help will output the help for each task # thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html @@ -42,12 +42,8 @@ login: ## Auto login to AWS-ECR unsing aws-cli docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} echo "Login COMPLETE" -master: build-nc publish-latest publish-stable -master-cached: build publish-latest publish-stable -stable: build-nc publish-stable -stable-cached: build publish-stable -version: build-nc publish-version -version-cached: build publish-version +stable: build-nc +stable-cached: build unstable: build-unstable-nc publish-unstable version-unstable: build-unstable publish-unstable @@ -55,16 +51,16 @@ publish: login publish-latest publish-version #publish-version ## Publish the `{ @echo "Publishing done" -publish-latest: login - @echo 'publish latest to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)' - docker tag $(APP_NAME):$(VERSION) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest - docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest - publish-stable: login @echo 'publish $(STABLE) to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)' docker tag $(APP_NAME):$(VERSION) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(STABLE) docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(STABLE) +publish-latest: login + @echo 'publish latest to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)' + docker tag $(APP_NAME):$(VERSION) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest + docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest + publish-version: login @echo 'publish $(STABLE) to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)' docker tag $(APP_NAME):$(VERSION) $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)