.gitlab-ci.yml

@@ -4,14 +4,13 @@ variables:
   ## Application versions used for the main release 
   ## note: nightly builds will always use the master/main branch
   JUGGLER_VERSION: "v9.3.0"
-  EICRECON_VERSION: "v0.6.2"
+  EICRECON_VERSION: "v0.6.4"
 
   ## Spack organization and repository, e.g. spack/spack
   SPACK_ORGREPO: "spack/spack"
   ## Spack github version, e.g. v0.18.1 or commit hash
   SPACK_VERSION: "v0.19.1"
   ## Space-separated list of spack cherry-picks
-## Space-separated list of spack cherry-picks
   SPACK_CHERRYPICKS: >
     b5ef5c2eb5145020f9de1bcb964626ce6ac2d02e
     99056e03bd3e903b222e300636ec484d85d4b3fb
@@ -30,6 +29,9 @@ variables:
     3afe6f1adcc24335cbca9a9c03ffea188f802766
     559c3de213707b5d52d899fd0382495f2cc8508d
     8e84dcd7ef999e2659822b34372515175f1723c4
+    65bd9b9ac556480b4a9dcc60f7539492af195d4a
+    1a32cea11495cbdd699fea4fe622babab83e630d
+    6edc4807369a05786e36f63b5d959588ae94a1fa
   ## Ref: https://github.com/spack/spack/commit/[hash]
   ## [hash]: [description]
   ## b5ef5c2eb5145020f9de1bcb964626ce6ac2d02e: geant4: version bumps for Geant4 11.1.0
@@ -49,6 +51,9 @@ variables:
   ## 3afe6f1adcc24335cbca9a9c03ffea188f802766: ROOT: add math/gsl conflict and change version-dependent features to conditional variants
   ## 559c3de213707b5d52d899fd0382495f2cc8508d: ROOT: new versions and associated dependency constraints
   ## 8e84dcd7ef999e2659822b34372515175f1723c4: root: new version 6.28.00
+  ## 65bd9b9ac556480b4a9dcc60f7539492af195d4a: podio, edm4hep: add v0.7.2 and v0.16.1 respectively
+  ## 1a32cea11495cbdd699fea4fe622babab83e630d: podio: add v0.16.2
+  ## 6edc4807369a05786e36f63b5d959588ae94a1fa: podio: Add version 0.16.3
 
   ## EIC spack organization and repository, e.g. eic/eic-spack
   EICSPACK_ORGREPO: "eic/eic-spack"
@@ -189,6 +194,10 @@ version:
   resource_group: ${CI_COMMIT_REF_NAME}
   tags:
     - docker-new
+  before_script:
+    - !reference [default, before_script]
+    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+    - docker buildx create --name builder --driver docker-container --bootstrap --use
 
 ## Images:
 ## debian_testing_base --> jug_dev  --> jug_xl
@@ -220,8 +229,6 @@ base:
   needs:
     - version
   script:
-    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-    - docker buildx create --name builder --driver docker-container --bootstrap --use
     - docker buildx build --push
                    -t ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
                    ${EXPORT_TAG:+
@@ -253,8 +260,6 @@ jug_dev:default:
     - version
     - base
   script:
-    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-    - docker buildx create --name builder --driver docker-container --bootstrap --use
     ## Which targets to build
     - TARGETS="export"
     - |
@@ -316,8 +321,6 @@ jug_xl:default:
     - version
     - jug_dev:default
   script:
-    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-    - docker buildx create --name builder --driver docker-container --bootstrap --use
     - docker buildx build --push
                    -t ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
                    ${EXPORT_TAG:+
@@ -356,8 +359,6 @@ jug_xl:nightly:
     - version
     - jug_dev:default
   script:
-    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-    - docker buildx create --name builder --driver docker-container --bootstrap --use
     - |
       PUSH_NIGHTLY_WITH_DATE=""
       if [ "$CI_COMMIT_BRANCH" == "master" ]; then
@@ -401,7 +402,6 @@ jug_xl:feature:
         DETECTOR_BRANCH:
           - main
   script:
-    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
     - docker buildx build --load -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}-${DETECTOR}-${DETECTOR_BRANCH}
                    -f containers/jug/feature.Dockerfile
                    --build-arg INTERNAL_TAG=${NIGHTLY_TAG}
@@ -535,15 +535,68 @@ clean:
       if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
         ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${NIGHTLY_TAG}
       fi
-    - docker images
 
-deep_clean:
+clean_unstable_mr:
   stage: finalize
   dependencies:
     - clean
   tags:
     - docker-new
-  when: manual
+  when: always
+  variables:
+    API: "https://eicweb.phy.anl.gov/api/v4"
   script:
+    - apt update -q
+    - apt install -yq jq
+    - for registry in
+        eicweb
+        eicweb.phy.anl.gov:4567/eic/juggler
+        eicweb.phy.anl.gov:4567/containers/eic_container
+      ; do 
+        docker images 
+          --filter=reference=${registry}/*:unstable-mr*
+          --format "{{.ID}} {{.Repository}} {{.Tag}}" ;
+      done | while read id repository tag ; do
+        if [[ ${tag} =~ ^unstable-mr-([0-9]+)-([0-9]+)(-nightly){0,1}$ ]] ; then
+          project_id=${BASH_REMATCH[1]} ;
+          merge_request_iid=${BASH_REMATCH[2]} ;
+          if [[ $(curl --silent -L
+                  "${API}/projects/${project_id}/merge_requests?scope=all&state=merged&per_page=100"
+                  "${API}/projects/${project_id}/merge_requests?scope=all&state=closed&per_page=100"
+                  | jq -r ".[]|select(.iid == ${merge_request_iid})|.state") =~ ^(merged|closed)$ ]] ; then
+            docker rmi $repository:$tag ;
+          else
+            echo "Image $repository:$tag still in use (project ${project_id}, idd ${merge_request_iid})." ;
+          fi ;
+        fi ;
+        if [[ ${tag} =~ ^unstable-mr-([0-9]+)(-nightly){0,1}$ ]] ; then
+          merge_request_id=${BASH_REMATCH[1]} ;
+          suffix=${BASH_REMATCH[2]} ;
+          if [[ $(curl --silent -L
+                  "${API}/projects/290/merge_requests?scope=all&state=merged&per_page=100"
+                  "${API}/projects/369/merge_requests?scope=all&state=merged&per_page=100"
+                  "${API}/projects/290/merge_requests?scope=all&state=closed&per_page=100"
+                  "${API}/projects/369/merge_requests?scope=all&state=closed&per_page=100"
+                  | jq -r ".[]|select(.id == ${merge_request_id})|.state") =~ ^(merged|closed)$ ]] ; then
+            docker rmi $repository:$tag ;
+          else
+            project_id=$(curl --silent -L
+                        "${API}/projects/290/merge_requests?scope=all&state=opened&per_page=100"
+                        "${API}/projects/369/merge_requests?scope=all&state=opened&per_page=100"
+                        | jq -r ".[]|select(.id == ${merge_request_id})|.project_id") ;
+            merge_request_iid=$(curl --silent -L 
+                        "${API}/projects/290/merge_requests?scope=all&state=opened&per_page=100"
+                        "${API}/projects/369/merge_requests?scope=all&state=opened&per_page=100"
+                        | jq -r ".[]|select(.id == ${merge_request_id})|.iid") ;
+            echo "Image $repository:$tag still in use (project ${project_id}, idd ${merge_request_iid})." ;
+            if [[ ${project_id} == "" || ${merge_request_iid} == "" ]] ; then
+              echo "$tag not found" ;
+            else
+              docker tag $repository:$tag $repository:unstable-mr-${project_id}-${merge_request_iid}${suffix} ;
+              docker rmi $repository:$tag ;
+            fi ;
+          fi ;
+        fi ;
+      done
     - docker system prune --force
     - docker images

containers/jug/dev.Dockerfile

@@ -11,7 +11,7 @@ FROM ${DOCKER_REGISTRY}${BASE_IMAGE}:${INTERNAL_TAG} as builder
 ARG TARGETPLATFORM
 
 ## install some extra spack dependencies
-RUN --mount=type=cache,target=/var/cache/apt                            \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked             \
     rm -f /etc/apt/apt.conf.d/docker-clean                              \
  && apt-get -yqq update                                                 \
  && apt-get -yqq install --no-install-recommends                        \
@@ -58,7 +58,7 @@ RUN declare -A arch=(                                                   \
 ## Setup spack buildcache mirrors, including an internal
 ## spack mirror using the docker build cache, and
 ## a backup mirror on the internal B010 network
-RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
+RUN --mount=type=cache,target=/var/cache/spack-mirror,sharing=locked    \
     export PATH=$PATH:$SPACK_ROOT/bin                                   \
  && spack mirror add docker /var/cache/spack-mirror                     \
  && spack buildcache update-index -d /var/cache/spack-mirror            \
@@ -67,7 +67,7 @@ RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
 ## Setup eic-spack buildcache mirrors (FIXME: leaks credentials into layer)
 ARG S3_ACCESS_KEY=""
 ARG S3_SECRET_KEY=""
-RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
+RUN --mount=type=cache,target=/var/cache/spack-mirror,sharing=locked    \
     export PATH=$PATH:$SPACK_ROOT/bin                                   \
  && if [ -n "${S3_ACCESS_KEY}" ] ; then                                 \
     spack mirror add --scope site                                       \
@@ -117,7 +117,7 @@ RUN rm -r /usr/local                                                    \
 ## 2. Get a list of all packages, and compare with what is already on
 ##    the buildcache (using package hash)
 ## 3. Add packages that need to be added to buildcache if any
-RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
+RUN --mount=type=cache,target=/var/cache/spack-mirror,sharing=locked    \
     cd /opt/spack-environment                                           \
  && source $SPACK_ROOT/share/spack/setup-env.sh                         \
  && spack env activate .                                                \

containers/jug/setup_detectors.py

 #!/usr/bin/env python3
 
 import os
+import sys
 import yaml
 import argparse
+import subprocess
+from datetime import datetime
 
 DETECTOR_REPO_GROUP = 'https://github.com/eic'
 DETECTOR_BEAMLINE_ENV ='''
@@ -97,6 +100,7 @@ if __name__ == '__main__':
                         default_found = True
                 print('    - {}: {}{}'.format(det, branch, default_str))
     print(' --> Building and installing detector/ip libraries')
+    process_list = []
     for det in detectors:
         if not args.nightly and 'nightly' in detectors[det]:
             del detectors[det]['nightly']    
@@ -128,7 +132,7 @@ if __name__ == '__main__':
                        '&&',
                        'cmake --build /tmp/build -j$(($(($(nproc)/4))+1)) -- install']
                 print(' '.join(cmd))
-                os.system(' '.join(cmd))
+                subprocess.check_call(' '.join(cmd), shell=True)
                 ## write version info to jug_info if available
                 if os.path.exists('/etc/jug_info'):
                     cmd = ['cd /tmp/det',
@@ -141,17 +145,17 @@ if __name__ == '__main__':
                            '&&',
                            'cd -']
                     print(' '.join(cmd))
-                    os.system(' '.join(cmd))
+                    subprocess.check_call(' '.join(cmd), shell=True)
                 ## also copy over IP configuration to the detector
                 if 'ip' in cfg and os.path.exists('/tmp/det/{ip}'.format(ip=cfg['ip']['config'])):
                     cmd = 'cp -r /tmp/det/{ip} {data_dir}'.format(
                                     ip=ip['config'], data_dir=data_dir)
                     print(cmd)
-                    os.system(cmd)
+                    subprocess.check_call(cmd, shell=True)
                 ## cleanup
                 cmd = 'rm -rf /tmp/det /tmp/build'
                 print(cmd)
-                os.system(cmd)
+                subprocess.check_call(cmd, shell=True)
             # be resilient against failures
             if os.path.exists(prefix):
                 ## create a shortcut for the prefix if desired
@@ -160,7 +164,7 @@ if __name__ == '__main__':
                                 prefix=prefix,
                                 shortcut='{}/{}-{}'.format(args.prefix, det, branch))
                     print(cmd)
-                    os.system(cmd)
+                    subprocess.check_call(cmd, shell=True)
                 ## write an environment file for this detector
                 with open('{prefix}/setup.sh'.format(prefix=prefix), 'w') as f:
                     if 'ip' in cfg:
@@ -183,15 +187,31 @@ if __name__ == '__main__':
                             file=f)
                 ## run once inside global prefix to initialize artifacts in /opt/detectors
                 os.environ['DETECTOR_PATH'] = args.prefix
-                cmd = f'bash -c \'cd {args.prefix} && source {prefix}/setup.sh && checkGeometry -c {prefix}/share/{det}/{det}.xml\''
+                cmd = f'cd {args.prefix} && source {prefix}/setup.sh && checkGeometry -c {prefix}/share/{det}/{det}.xml'
                 print(cmd)
-                os.system(cmd)
+                process_list.append(subprocess.Popen(cmd, shell=True, executable='/bin/bash', stdout=subprocess.PIPE, stderr=subprocess.STDOUT))
                 ## run once inside specific prefix to initialize artifacts in $DETECTOR_PATH
                 os.environ['DETECTOR_PATH'] = args.prefix
-                cmd = f'bash -c \'cd {prefix}/share/{det} && source {prefix}/setup.sh && checkGeometry -c {prefix}/share/{det}/{det}.xml\''
+                cmd = f'cd {prefix}/share/{det} && source {prefix}/setup.sh && checkGeometry -c {prefix}/share/{det}/{det}.xml'
                 print(cmd)
-                os.system(cmd)
-    
+                process_list.append(subprocess.Popen(cmd, shell=True, executable='/bin/bash', stdout=subprocess.PIPE, stderr=subprocess.STDOUT))
+
+    while len(process_list) > 0:
+        now = datetime.now()
+        current_time = now.strftime("%H:%M:%S")
+        print("{} processes running... ({})".format(len(process_list), current_time))
+        (out, err) = process_list[-1].communicate()
+        if process_list[-1].wait() != 0:
+            print(process_list[-1].args)
+            if out is not None:
+                print("stdout:")
+                print(out.decode())
+            if err is not None:
+                print("stderr:")
+                print(err.decode())
+            sys.exit(1)
+        process_list.pop()
+
     if not default_found and not args.nightly:
         # Skip symlinking if no defaults present and its not a nightly build
         pass
@@ -204,6 +224,6 @@ if __name__ == '__main__':
             '&&',
             'ln -sf {full_prefix}/setup.sh {short_prefix}']
         print(' '.join(cmd))
-        os.system(' '.join(cmd).format(full_prefix=full_prefix, short_prefix=args.prefix))
+        subprocess.check_call(' '.join(cmd).format(full_prefix=full_prefix, short_prefix=args.prefix), shell=True)
 
     print('All done!')

containers/jug/xl.Dockerfile

@@ -22,12 +22,12 @@ RUN cd /tmp
  && echo " - jug_xl: ${JUG_VERSION}" >> /etc/jug_info
 
 ADD ${EICWEB}/369/repository/tree?ref=${JUGGLER_VERSION} /tmp/369.json
-RUN --mount=type=cache,target=/ccache/                                          \
+RUN --mount=type=cache,target=/ccache/,sharing=locked,id=${TARGETPLATFORM}      \
     cd /tmp                                                                     \
  && echo "INSTALLING JUGGLER"                                                   \
  && git clone -b ${JUGGLER_VERSION} --depth 1                                   \
         https://eicweb.phy.anl.gov/EIC/juggler.git                              \
- && export CCACHE_DIR=/ccache/$TARGETPLATFORM                                   \     
+ && export CCACHE_DIR=/ccache                                                   \     
  && cmake -B build -S juggler                                                   \
           -DCMAKE_CXX_FLAGS="-Wno-psabi"                                        \
           -DCMAKE_CXX_STANDARD=17                                               \
@@ -42,12 +42,12 @@ RUN --mount=type=cache,target=/ccache/
  && rm -rf build juggler
 
 ADD https://api.github.com/repos/eic/eicrecon/commits/${EICRECON_VERSION} /tmp/eicrecon.json
-RUN --mount=type=cache,target=/ccache/                                          \
+RUN --mount=type=cache,target=/ccache/,sharing=locked,id=${TARGETPLATFORM}      \
     cd /tmp                                                                     \
  && echo "INSTALLING EICRECON"                                                  \
  && git clone -b ${EICRECON_VERSION} --depth 1                                  \
         https://github.com/eic/eicrecon.git                                     \
- && export CCACHE_DIR=/ccache/$TARGETPLATFORM                                   \
+ && export CCACHE_DIR=/ccache                                                   \
  && cmake -B build -S eicrecon                                                  \
           -DCMAKE_CXX_FLAGS="-Wno-psabi"                                        \
           -DCMAKE_CXX_STANDARD=17                                               \
@@ -130,9 +130,9 @@ ADD https://api.github.com/repos/eic/ip6 /tmp/ip6.json
 ADD https://api.github.com/repos/eic/epic /tmp/epic.json
 COPY setup_detectors.py /tmp
 COPY --from=detectors detectors.yaml /tmp
-RUN --mount=type=cache,target=/ccache/                                          \
+RUN --mount=type=cache,target=/ccache/,sharing=locked,id=${TARGETPLATFORM}      \
     cd /tmp                                                                     \
- && export CCACHE_DIR=/ccache/$TARGETPLATFORM                                   \
+ && export CCACHE_DIR=/ccache                                                   \
  && [ "z$NIGHTLY" = "z1" ] && NIGHTLY_FLAG="--nightly" || NIGHTLY_FLAG=""       \
  && /tmp/setup_detectors.py --prefix /opt/detector --config /tmp/detectors.yaml \
                          $NIGHTLY_FLAG                                          \

detectors.yaml

@@ -41,3 +41,5 @@ detectors:
       version: 23.01.0
     23.03.0:
       version: 23.03.0
+    23.03.0-alt-img:
+      version: alternative-geometries-img-ecal

spack.yaml

@@ -19,24 +19,27 @@ spack:
     - eigen@3.4.0
     - emacs@28.2
     - fastjet@3.4.0
+    - fjcontrib@1.045
     - fmt@8.1.1 +shared cxxstd=17
     - gaudi@36.7 +aida
-    - geant4@11.1.1 cxxstd=17 +opengl +vecgeom +x11 +qt +threads
+    - geant4@11.1.1 cxxstd=17 +opengl -vecgeom +x11 +qt +threads
     - hepmc3@3.2.5 +python +rootio
     - heppdt@2.06.01
     - imagemagick@7.0.8-7
     - intel-tbb@2020.3
-    - irt@1.0.0
-    - jana2@2.0.8 -ipo +root +zmq
+    - irt@1.0.5
+    - jana2@2.1.0 -ipo +podio +root +zmq
     - lcov@1.15
     - madx@5.08.01
     - mesa@22.1.2 +glx -llvm +opengl +opengles +osmesa
     - nlohmann-json@3.11.2
     - npsim@1.1.0 +geocad +http
+    - onnx@1.12.0
     - opencascade@7.6.3
     - osg-ca-certs@1.109.igtf.1.117
+    - phonebook-cli@1.0.0
     - pkg-config@0.29.2
-    - podio@0.16.2
+    - podio@0.16.3
     - prmon@3.0.2 +plot
     - pythia8@8.306 +fastjet
     - python@3.10.8