Prep files for merge

Modify .gitlab-ci.yml to add new containers.

.gitlab-ci.yml

@@ -39,9 +39,11 @@ variables:
 
 stages:
   - config
-  - build:base            ## build stable Debian
-  - build:oneapi_jug      ## OneAPI + Jug_Dev Image 
-  - deploy                ## build/deploy singularity images            
+  - build:base            ## base OS image
+  - build:jug             ## jug container images
+  - build:oneapi_jug      ## jug containers with Intel oneAPI support 
+  - deploy                ## build/deploy singularity images
+  - test            
   - finalize
 
 default:
@@ -85,7 +87,10 @@ workflow:
 ##
 ## caching strategy for dispatch to multiple nodes and to avoid
 ##  --> try this strategy: https://medium.com/titansoft-engineering/docker-build-cache-sharing-on-multi-hosts-with-buildkit-and-buildx-eb8f7005918e
+<<<<<<< HEAD
 
+=======
+>>>>>>> Prep files for merge
 
 version:
   stage: config
@@ -172,6 +177,7 @@ version:
 ## TODO
 ## oneapi_runtime + jug_xl --> oneapi_prod 
 
+
 debian_base:default:
   extends: .build
   stage: build:base
@@ -181,6 +187,20 @@ debian_base:default:
     BUILD_IMAGE: "debian_base"
   script:
     - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG} 
+                   -f containers/debian/base.Dockerfile
+                   containers/debian
+    - !reference [.build, script]
+
+debian_stable_base:default:
+  extends: .build
+  stage: build:base
+  needs:
+    - version
+  variables:
+    BUILD_IMAGE: "debian_stable_base"
+  script:
+    - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG} 
+                   -f containers/debian/stable.Dockerfile
                    containers/debian
     - !reference [.build, script]
 
@@ -193,9 +213,29 @@ oneapi_base:default:
     BUILD_IMAGE: "oneapi_base"
   script:
     - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG} 
+                   -f containers/oneapi/base.Dockerfile
                    containers/oneapi
     - !reference [.build, script]
 
+oneapi_jug_dev:default:
+  extends: .build
+  stage: build:oneapi_jug
+  needs:
+    - version
+    - debian_stable_base:default
+  variables:
+    BUILD_IMAGE: "oneapi_jug_dev"
+  script:
+    ## Copy spack directory and spack.yaml into build context
+    - cp -r spack containers/oneapi/
+    - cp spack.yaml containers/oneapi/spack/spack.yaml
+    ## Copy jug_dev files into build context
+    - cp -r containers/jug/* containers/oneapi
+    - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG} 
+                   -f containers/oneapi/onedev.Dockerfile
+                    containers/oneapi
+    - !reference [.build, script]
+
 cuda_base:default:
   extends: .build
   stage: build:base
@@ -205,42 +245,154 @@ cuda_base:default:
     BUILD_IMAGE: "cuda_base"
   script:
     - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG} 
-                   containers/cuda
+                   containers/oneapi
     - !reference [.build, script]
 
-debian_stable_base:default:
+jug_dev:default:
   extends: .build
-  stage: build:base
+  stage: build:jug
   needs:
     - version
+    - debian_base:default
   variables:
-    BUILD_IMAGE: "debian_stable_base"
+    BUILD_IMAGE: "jug_dev"
   script:
-    - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG} 
-                   -f containers/debian/stable.Dockerfile
-                   containers/debian
+    ## calculate a hash based on the spack.yaml file and the spack directory
+    ## and use this spack as a docker variable to force a rebuild when there
+    ## is a change (versus rerun from cache)
+    - PACKAGE_HASH=$(tar cf - spack* | sha1sum | head -c40)
+    - echo "PACKAGE_HASH= ${PACKAGE_HASH}"
+    ## move spacke directory and spack.yaml into the container build directory
+    - cp -r spack containers/jug
+    - cp -r spack.yaml containers/jug/spack/spack.yaml
+    - CACHE_FLAG=""
+    - |
+      if [ $FORCE_NOCACHE = 1 ]; then
+        echo "FORCE_NOCACHE set"
+        export CACHE_FLAG="--no-cache"
+      fi   
+    ## Optionally build the raw builder image
+    - test ${EXPORT_BUILDER} = 1 && docker build ${CACHE_FLAG}
+                   -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:builder-${INTERNAL_TAG} 
+                   --target=builder
+                   -f containers/jug/dev.Dockerfile
+                   --build-arg SPACK_VERSION="${SPACK_VERSION}"
+                   --build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
+                   --build-arg CACHE_BUST=${PACKAGE_HASH}
+                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
+                   --build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD)
+                   containers/jug
+    ## now build our image
+    - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG} ${CACHE_FLAG}
+                   -f containers/jug/dev.Dockerfile
+                   --build-arg SPACK_VERSION="${SPACK_VERSION}"
+                   --build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
+                   --build-arg CACHE_BUST=${PACKAGE_HASH}
+                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
+                   --build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD)
+                   containers/jug
+    ## push builder image do DH if desired
+    - test ${EXPORT_BUILDER} = 1 && ./gitlab-ci/docker_push.sh 
+                                 -i ${BUILD_IMAGE} -l builder-${INTERNAL_TAG}
+                                 -n ${DOCKER_NTRIES} -t ${DOCKER_WAIT_TIME}
+                                 builder-${EXPORT_TAG} --dockerhub
+    ## standard exports
     - !reference [.build, script]
 
-oneapi_jug_dev:default:
+jug_xl:default:
   extends: .build
-  stage: build:oneapi_jug
+  stage: build:jug
+  resource_group: build
   needs:
     - version
-    - debian_stable_base:default
+    - jug_dev:default
   variables:
-    BUILD_IMAGE: "oneapi_jug_dev"
+    BUILD_IMAGE: "jug_xl"
   script:
-    ## Copy spack directory and spack.yaml into build context
-    - cp -r spack containers/oneapi/
-    - cp spack.yaml containers/oneapi/spack/spack.yaml
-    ## Copy jug_dev files into build context
-    - cp -r containers/jug/* containers/oneapi
+    - cp detectors.yaml containers/jug
     - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG} 
-                      -f containers/oneapi/onedev.Dockerfile
-                      containers/oneapi
+                   -f containers/jug/xl.Dockerfile
+                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
+                   --build-arg JUGGLER_VERSION=${JUGGLER_VERSION}
+                   --build-arg NPDET_VERSION=${NPDET_VERSION}
+                   --build-arg EICD_VERSION=${EICD_VERSION}
+                   --build-arg AFTERBURNER_VERSION=${AFTERBURNER_VERSION}
+                   --build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD)
+                   containers/jug
     - !reference [.build, script]
 
+jug_xl:nightly:
+  extends: .build
+  stage: build:jug
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master"'
+      when: on_success
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: on_success
+    - when: never
+  needs:
+    - version
+    - jug_dev:default
+  variables:
+    BUILD_IMAGE: "jug_xl"
+  script:
+    - cp detectors.yaml containers/jug
+    - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${NIGHTLY_TAG} 
+                   -f containers/jug/xl.Dockerfile
+                   --build-arg INTERNAL_TAG=${INTERNAL_TAG}
+                   --build-arg JUG_VERSION=nightly-$(date +%Y-%m-%d_%H-%M-%S)-${INTERNAL_TAG}-$(git rev-parse HEAD)
+                   --build-arg NIGHTLY=1
+                   containers/jug
+    - |
+      PUSH_FLAG=""
+      if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
+        PUSH_FLAG="--eicweb"
+      fi
+    - ./gitlab-ci/docker_push.sh -i ${BUILD_IMAGE} -l ${NIGHTLY_TAG} 
+                                 -n $DOCKER_NTRIES -t $DOCKER_WAIT_TIME
+                                 ${NIGHTLY_TAG} ${PUSH_FLAG}
+    - if [ -z "${PUSH_FLAG}" ] ; then 
+      ./gitlab-ci/docker_push.sh -i ${BUILD_IMAGE} -l ${NIGHTLY_TAG} 
+                                 -n $DOCKER_NTRIES -t $DOCKER_WAIT_TIME
+                                 ${NIGHTLY_TAG}-$(date +%Y-%m-%d) --dockerhub ;
+      fi
 
+jug_xl:feature:
+  extends: .build
+  stage: build:jug
+  rules: 
+    - if: '$CI_COMMIT_BRANCH == "master"'
+      when: on_success
+    - when: never
+  needs: 
+    - version
+    #- jug_xl:default
+    - jug_xl:nightly
+  variables:
+    BUILD_IMAGE: "jug_xl"
+  parallel:
+    matrix:
+      - DETECTOR: athena
+        DETECTOR_BRANCH:
+          - canyonlands
+          - deathvalley
+          - deathvalley-1.5T
+  script:
+    - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}-${DETECTOR}-${DETECTOR_BRANCH}
+                   -f containers/jug/feature.Dockerfile
+                   --build-arg INTERNAL_TAG=${NIGHTLY_TAG}
+                   --build-arg DETECTOR=${DETECTOR}
+                   --build-arg DETECTOR_BRANCH=${DETECTOR_BRANCH}
+                   containers/jug
+    - |
+      PUSH_FLAG=""
+      if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
+        PUSH_FLAG="--eicweb"
+      fi
+      FEATURE_EXPORT_TAG="${VERSION_SHORT}-${DETECTOR_BRANCH}-stable"
+    - ./gitlab-ci/docker_push.sh -i ${BUILD_IMAGE} -l ${INTERNAL_TAG}-${DETECTOR}-${DETECTOR_BRANCH}
+                                 -n ${DOCKER_NTRIES} -t ${DOCKER_WAIT_TIME}
+                                 ${FEATURE_EXPORT_TAG} ${PUSH_FLAG}
 
 .singularity:
   stage: deploy
@@ -261,11 +413,78 @@ oneapi_jug_dev:singularity:default:
   extends: .singularity
   needs:
     - version
-    - oneapi_jug_dev:default
+    - debian_stable_base:default
   variables:
     BUILD_IMAGE: "oneapi_jug_dev"
 
+jug_dev:singularity:default:
+  extends: .singularity
+  needs: 
+    - version
+    - jug_dev:default
+  variables:
+    BUILD_IMAGE: "jug_dev"
+
+jug_xl:singularity:default:
+  extends: .singularity
+  needs: 
+    - version
+    - jug_xl:default
+  variables:
+    BUILD_IMAGE: "jug_xl"
 
+jug_xl:singularity:nightly:
+  stage: deploy
+  extends: .singularity
+  needs: 
+    - version
+    - jug_xl:nightly
+  variables:
+    BUILD_IMAGE: "jug_xl"
+  script:
+    - mkdir build
+    - singularity pull build/${BUILD_IMAGE}.sif docker://${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${NIGHTLY_TAG}
+
+## trigger juggler rebuild on nightly schedule to ensure both images remain in sync
+juggler:master:
+  stage: deploy
+  rules:
+    - if: '$NIGHTLY != "0" && $CI_COMMIT_BRANCH == "master"'
+      when: on_success
+    - when: never
+  needs:
+    - version
+    - jug_xl:nightly
+  variables:
+    TRIGGERED_BY_NIGHTLY: 1
+    JUGGLER_VERSION: master
+    EICD_VERSION: master
+    NPDET_VERSION: master
+    DETECTOR_VERSION: master
+    IP6_VERSION: master
+  trigger:
+    project: EIC/juggler
+  allow_failure: true
+
+.test:
+  image: eicweb.phy.anl.gov:4567/containers/eic_container/jug_xl:${NIGHTLY_TAG}
+  stage: test
+  needs:
+    - version
+    - jug_xl:nightly
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master"'
+      when: on_success
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: on_success
+    - when: never
+  before_script:
+    - echo "Testing the new container setup"
+  after_script:
+    - echo "Testing completed"
+
+include:
+  - local: 'tests/tutorial/config.yml'
 
 cleanup:
   stage: finalize
@@ -281,3 +500,4 @@ cleanup:
       if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
         ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${NIGHTLY_TAG}
       fi
+