.github/workflows/mirror.yaml

@@ -22,6 +22,7 @@ jobs:
       with:
         fetch-depth: 0
     - name: Push to EICweb
+      if: github.event_name == 'push'
       uses: eic/gitlab-sync@master
       with:
         url: https://eicweb.phy.anl.gov/containers/eic_container.git/
@@ -29,7 +30,7 @@ jobs:
         username: ${{ secrets.GITLAB_USERNAME }}
         ciskip: true
     - name: Trigger EICweb
-      if: github.event_name == 'pull_request'
+      if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
       uses: eic/trigger-gitlab-ci@v2
       with:
         url: https://eicweb.phy.anl.gov

.gitlab-ci.yml

@@ -159,7 +159,9 @@ status:pending:
   services:
     - docker:dind
   before_script:
-    - # Login to registries
+    - # Login to local registry
+      echo ${CI_REGISTRY_PASSWORD} | docker login -u ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY}
+    - # Login to remote registries
       if [[ -n "${EXPORT_TAG}" ]] ; then
         if [[ -n "${DH_REGISTRY}" ]] ; then
           echo ${DH_EICWEB_TOKEN} | docker login -u ${DH_REGISTRY_USER} --password-stdin ${DH_REGISTRY} ;
@@ -167,12 +169,11 @@ status:pending:
         if [[ -n "${GH_REGISTRY}" ]] ; then
           echo ${GITHUB_REGISTRY_TOKEN} | docker login -u ${GITHUB_REGISTRY_USER} --password-stdin ${GH_REGISTRY} ;
         fi ;
-        if [[ -n "${CI_REGISTRY}" ]] ; then
-          echo ${CI_REGISTRY_PASSWORD} | docker login -u ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY} ;
-        fi ;
       fi
   after_script:
-    - # Logout from registries
+    - # Logout from local registry
+      docker logout ${CI_REGISTRY}
+    - # Logout from remote registries
       if [[ -n "${EXPORT_TAG}" ]] ; then
         if [[ -n "${DH_REGISTRY}" ]] ; then
           docker logout ${DH_REGISTRY} ;
@@ -180,9 +181,6 @@ status:pending:
         if [[ -n "${GH_REGISTRY}" ]] ; then
           docker logout ${GH_REGISTRY} ;
         fi ;
-        if [[ -n "${CI_REGISTRY}" ]] ; then
-          docker logout ${CI_REGISTRY} ;
-        fi ;
       fi
 
 ## base job settings for all docker build jobs
@@ -270,6 +268,7 @@ jug_dev:
         BUILD_IMAGE: jug_
         PLATFORM: linux/amd64
         ENV:
+        - dbg
         - dev
         - prod
 #      - BUILD_TYPE:
@@ -319,6 +318,22 @@ jug_dev:
                        }
                      }
                    }
+                   ${EXPORT_TAG:+
+                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE/jug/eic}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
+                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/jug/eic}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
+                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/jug/eic}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}}
+                   }
+                   ${IF_BUILD_NIGHTLY:+
+                     ${NIGHLTY:+
+                       ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE/jug/eic}${ENV}:${NIGHTLY_TAG}}
+                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/jug/eic}${ENV}:${NIGHTLY_TAG}}
+                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/jug/eic}${ENV}:${NIGHTLY_TAG}}
+                       ${PUSH_NIGHTLY_WITH_DATE:+
+                         ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/jug/eic}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
+                         ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/jug/eic}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
+                       }
+                     }
+                   }
                    --file containers/jug/dev.Dockerfile
                    --platform ${PLATFORM}
                    --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
@@ -327,6 +342,7 @@ jug_dev:
                    --build-arg SPACK_ORGREPO=${SPACK_ORGREPO}
                    --build-arg SPACK_VERSION=${SPACK_VERSION}
                    --build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
+                   --build-arg SPACK_CHERRYPICKS_FILES="${SPACK_CHERRYPICKS_FILES}"
                    --build-arg EICSPACK_ORGREPO=${EICSPACK_ORGREPO}
                    --build-arg EICSPACK_VERSION=${EICSPACK_VERSION}
                    --build-arg EICSPACK_CHERRYPICKS="${EICSPACK_CHERRYPICKS}"
@@ -381,6 +397,11 @@ jug_xl:default:
                      ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
                      ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
                    }
+                   ${EXPORT_TAG:+
+                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE/jug/eic}:${EXPORT_TAG}}
+                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/jug/eic}:${EXPORT_TAG}}
+                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/jug/eic}:${EXPORT_TAG}}
+                   }
                    --file containers/jug/xl.Dockerfile
                    --platform ${PLATFORM}
                    --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
@@ -444,6 +465,20 @@ jug_xl:nightly:
                        ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
                      }
                    }
+                   ${EXPORT_TAG:+
+                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE/jug/eic}:${EXPORT_TAG}-nightly}
+                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/jug/eic}:${EXPORT_TAG}-nightly}
+                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/jug/eic}:${EXPORT_TAG}-nightly}
+                   }
+                   ${NIGHTLY:+
+                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE/jug/eic}:${NIGHTLY_TAG}}
+                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/jug/eic}:${NIGHTLY_TAG}}
+                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/jug/eic}:${NIGHTLY_TAG}}
+                     ${PUSH_NIGHTLY_WITH_DATE:+
+                       ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/jug/eic}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
+                       ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/jug/eic}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)}
+                     }
+                   }
                    --file containers/jug/xl.Dockerfile
                    --platform ${PLATFORM}
                    --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/
@@ -451,6 +486,7 @@ jug_xl:nightly:
                    --build-arg INTERNAL_TAG=${INTERNAL_TAG}-nightly
                    --build-arg JUG_VERSION=${EXPORT_TAG}-nightly-$(git rev-parse HEAD)-$(date +%Y-%m-%d_%H-%M-%S)
                    --build-arg NIGHTLY=1
+                   --build-arg jobs=${JOBS}
                    --build-context detectors=.
                    --provenance false
                    containers/jug

containers/debian/base.Dockerfile

@@ -44,9 +44,10 @@ apt-get -yqq install --no-install-recommends                            \
         iputils-ping                                                    \
         iputils-tracepath                                               \
         less                                                            \
+        libc6-dbg                                                       \
         libcbor-xs-perl                                                 \
         libjson-xs-perl                                                 \
-        libgl-dev                                                      \
+        libgl-dev                                                       \
         libglx-dev                                                      \
         libopengl-dev                                                   \
         locales                                                         \
@@ -100,7 +101,7 @@ else
 fi
 # Install packages
 apt-get -yqq update
-apt-get -yqq install gcc${GCC} g++${GCC} gfortran${GCC}
+apt-get -yqq install cpp${GCC} gcc${GCC} g++${GCC} gfortran${GCC}
 apt-get -yqq install clang${CLANG} clang-tidy${CLANG} clang-format${CLANG} libclang${CLANG}-dev
 apt-get -yqq autoremove
 # Ensure alternatives without version tags

containers/jug/dev.Dockerfile

@@ -33,14 +33,25 @@ ENV SPACK_ROOT=/opt/spack
 ARG SPACK_ORGREPO="spack/spack"
 ARG SPACK_VERSION="releases/v0.20"
 ARG SPACK_CHERRYPICKS=""
+ARG SPACK_CHERRYPICKS_FILES=""
 ADD https://api.github.com/repos/${SPACK_ORGREPO}/commits/${SPACK_VERSION} /tmp/spack.json
 RUN <<EOF
+git config --global user.email "gitlab@eicweb.phy.anl.gov"
+git config --global user.name "EIC Container Build Service"
 git config --global advice.detachedHead false
 git clone --filter=tree:0 https://github.com/${SPACK_ORGREPO}.git ${SPACK_ROOT}
 git -C ${SPACK_ROOT} checkout ${SPACK_VERSION}
 if [ -n "${SPACK_CHERRYPICKS}" ] ; then
   SPACK_CHERRYPICKS=$(git -C ${SPACK_ROOT} rev-list --topo-order ${SPACK_CHERRYPICKS} | grep -m $(echo ${SPACK_CHERRYPICKS} | wc -w)  "${SPACK_CHERRYPICKS}" | tac)
-  git -C ${SPACK_ROOT} cherry-pick -n ${SPACK_CHERRYPICKS}
+  eval "declare -A SPACK_CHERRYPICKS_FILES_ARRAY=(${SPACK_CHERRYPICKS_FILES})"
+  for hash in ${SPACK_CHERRYPICKS} ; do
+    if [ -n "${SPACK_CHERRYPICKS_FILES_ARRAY[${hash}]+found}" ] ; then
+      git -C ${SPACK_ROOT} show ${hash} -- ${SPACK_CHERRYPICKS_FILES_ARRAY[${hash}]//,/ } | patch -p1 -d ${SPACK_ROOT}
+      git -C ${SPACK_ROOT} commit --all --message "$(git -C ${SPACK_ROOT} show --no-patch --pretty=format:%s ${hash})"
+    else
+      git -C ${SPACK_ROOT} cherry-pick ${hash}
+    fi
+  done
 fi
 ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash /usr/bin/docker-shell
 ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash /usr/bin/interactive-shell
@@ -51,7 +62,7 @@ EOF
 SHELL ["docker-shell"]
 
 ## Setup build configuration
-ARG jobs=64
+ARG jobs=1
 RUN <<EOF
 set -e
 declare -A target=(["linux/amd64"]="x86_64_v2" ["linux/arm64"]="aarch64")

containers/jug/setup_detectors.py

@@ -85,22 +85,23 @@ if __name__ == '__main__':
             ## build and install
             print('    - {}-{}'.format(det, cfg['version']))
             ## cleanup
-            cmd = ['rm -rf /tmp/build /tmp/det']
+            cmd = [f'rm -rf /tmp/build-{version} /tmp/det-{version}']
             print(' '.join(cmd))
             subprocess.check_call(' '.join(cmd), shell=True)
             ## clone
             cmd = [
-                'git clone --depth 1 -b {version} {repo_grp}/{detector}.git /tmp/det'.format(
-                    version=cfg['version'],
+                'git clone --depth 1 -b {branch} {repo_grp}/{detector}.git /tmp/det-{version}'.format(
+                    branch=cfg['version'],
                     repo_grp=DETECTOR_REPO_GROUP,
-                    detector=det)
+                    detector=det,
+                    version=version)
             ]
             print(' '.join(cmd))
             subprocess.check_call(' '.join(cmd), shell=True)
             ## patches
             if cfg.get('patches'):
                 for patch in cfg['patches']:
-                    cmd = [f'curl -L {patch} | patch -p1 -d/tmp/det']
+                    cmd = [f'curl -L {patch} | patch -p1 -d/tmp/det-{version}']
                     print(' '.join(cmd))
                     subprocess.check_call(' '.join(cmd), shell=True)
             ## build
@@ -110,7 +111,7 @@ if __name__ == '__main__':
             if cfg.get('cxxflags'):
                 cxxflags = cfg['cxxflags']
             cmd = [
-                f'cmake -B /tmp/build -S /tmp/det -DCMAKE_CXX_STANDARD=17',
+                f'cmake -B /tmp/build-{version} -S /tmp/det-{version} -DCMAKE_CXX_STANDARD=17',
                 f'-DCMAKE_CXX_FLAGS="-Wno-psabi {cxxflags}"',
                 f'-DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache',
                 f'-DCMAKE_INSTALL_PREFIX={prefix}'
@@ -119,13 +120,13 @@ if __name__ == '__main__':
             subprocess.check_call(' '.join(cmd), shell=True)
             ## install
             cmd = [
-                'cmake --build /tmp/build -j$(($(($(nproc)/4))+1)) -- install'
+                f'cmake --build /tmp/build-{version} -j$(($(($(nproc)/4))+1)) -- install'
             ]
             print(' '.join(cmd))
             subprocess.check_output(' '.join(cmd), shell=True)
             ## write version info to jug_info if available
             if os.path.exists('/etc/jug_info'):
-                cmd = ['cd /tmp/det',
+                cmd = [f'cd /tmp/det-{version}',
                         '&&',
                         'echo " - {detector}/{branch}: {version}-$(git rev-parse HEAD)"'.format(
                             detector=det,
@@ -137,7 +138,7 @@ if __name__ == '__main__':
                 print(' '.join(cmd))
                 subprocess.check_call(' '.join(cmd), shell=True)
             ## cleanup
-            cmd = 'rm -rf /tmp/det /tmp/build'
+            cmd = f'rm -rf /tmp/det-{version} /tmp/build-{version}'
             print(cmd)
             subprocess.check_call(cmd, shell=True)
             # be resilient against failures

containers/jug/xl.Dockerfile

@@ -11,7 +11,7 @@ FROM ${DOCKER_REGISTRY}${BASE_IMAGE}:${INTERNAL_TAG}
 ARG TARGETPLATFORM
 
 ARG EICWEB="https://eicweb.phy.anl.gov/api/v4/projects"
-ARG jobs=8
+ARG jobs=1
 
 ## version will automatically bust cache for nightly, as it includes
 ## the date

detectors.yaml

+# Only retain last 6 months of epic geometry versions
 detectors:
   epic:
     nightly:
@@ -5,23 +6,6 @@ detectors:
       version: main
     main:
       version: main
-    23.06.1:
-      version: 23.06.1
-      patches:
-      - https://github.com/eic/epic/pull/449.patch
-      - https://github.com/eic/epic/pull/520.patch
-    23.07.0:
-      version: 23.07.0
-      patches:
-      - https://github.com/eic/epic/pull/520.patch
-    23.07.1:
-      version: 23.07.1
-      patches:
-      - https://github.com/eic/epic/pull/520.patch
-    23.07.2:
-      version: 23.07.2
-      patches:
-      - https://github.com/eic/epic/pull/520.patch
     23.08.0:
       version: 23.08.0
       patches:
@@ -40,3 +24,5 @@ detectors:
       version: 23.11.0
     23.12.0:
       version: 23.12.0
+    24.02.0:
+      version: 24.02.0
\ No newline at end of file

eic-spack.sh

@@ -3,7 +3,7 @@ EICSPACK_ORGREPO="eic/eic-spack"
 
 ## EIC spack commit hash or github version, e.g. v0.19.7
 ## note: nightly builds could use a branch e.g. releases/v0.19
-EICSPACK_VERSION="bc6a4382636dd8c8bcd8c665163840815d48a25e"
+EICSPACK_VERSION="acde63153d3e650275d85334523e118e45edb414"
 
 ## Space-separated list of eic-spack cherry-picks
 read -r -d '' EICSPACK_CHERRYPICKS <<- \

spack-environment/dbg/spack.yaml

+spack:
+  include:
+  - ../concretizer.yaml
+  - ../packages.yaml
+  config:
+    install_missing_compilers: false
+  specs:
+  - acts build_type=Debug
+  - algorithms build_type=Debug
+  - cmake
+  - dd4hep build_type=Debug
+  - edm4eic build_type=Debug
+  - edm4hep build_type=Debug
+  - eicrecon build_type=Debug
+  - gdb
+  - irt build_type=Debug
+  - jana2 build_type=Debug
+  - valgrind
+  view: false

spack-environment/dev/spack.yaml

@@ -40,12 +40,13 @@ spack:
   - irt
   - iwyu
   - jana2
-  - k4actstracking
   - juggler
+  - k4actstracking
   - k4fwcore
   - lcov
   - lhapdf
   - madx
+  - nano
   - npsim
   - onnx
   - opencascade

spack-environment/packages.yaml

@@ -32,7 +32,7 @@ packages:
     - +root +zlib
   algorithms:
     require:
-    - '@git.e42f75e031e46e3425e3223c8b6af361a618c3ed'
+    - '@git.6a0e78e3463004bacd78fcb234019f6db5257a9a'
   blas:
     require:
     - openblas
@@ -42,7 +42,7 @@ packages:
     - +fc+ft+X+pdf+gobject
   catch2:
     require:
-    - '@3.0.1'
+    - '@3.4.0'
     - -ipo
   cernlib:
     require:
@@ -83,18 +83,18 @@ packages:
     - '@656aa3192b097a631ddd1e0380e80c26fd6644a7'
   edm4eic:
     require:
-    - '@4.0.0'
+    - '@5.0.0'
     - cxxstd=20
   edm4hep:
     require:
-    - '@0.10.2'
+    - '@0.10.3'
     - cxxstd=20
   eic-smear:
     require:
     - '@1.1.12'
   eicrecon:
     require:
-    - '@1.9.1'
+    - '@1.10.0'
   eigen:
     require:
     - '@3.4.0'
@@ -137,10 +137,10 @@ packages:
   graphviz:
     require:
     - '@8.0.1'
-    - +expat
+    - +expat +pangocairo
   hepmc3:
     require:
-    - '@3.2.6'
+    - '@3.2.7'
     - +python +rootio
   heppdt:
     require:
@@ -163,7 +163,7 @@ packages:
     - -ipo +podio +root +zmq
   juggler:
     require:
-    - '@13.0.0'
+    - '@14.0.0'
     - cxxstd=20
   k4actstracking:
     require:
@@ -194,6 +194,9 @@ packages:
   nlohmann-json:
     require:
     - '@3.11.2'
+  nano:
+    require:
+    - '@7.2'
   npsim:
     require:
     - '@1.3.0'
@@ -230,7 +233,7 @@ packages:
     - +plot
   pythia8:
     require:
-    - '@8.309'
+    - '@8.310'
     - +fastjet +root
   python:
     require:
@@ -252,7 +255,7 @@ packages:
     - '@6.3.0'
   py-epic-capybara:
     require:
-    - '@git.3a6e6a9c75630fd5e7375a8bd7f9b5c20eb227cd'
+    - '@git.190a2e87b9d089092a0b582e65a9be38055809e1'
   py-ipython:
     require:
     - '@8.11.0'
@@ -335,8 +338,8 @@ packages:
     - +opengl
   root:
     require:
-    - '@6.28.06'
-    - cxxstd=20 +fftw +fortran +gdml +http -ipo +mlp +python +root7 +tmva +vc +xrootd +ssl
+    - '@6.30.02'
+    - cxxstd=20 +fftw +fortran +gdml +http -ipo +mlp +python +root7 +tmva +tmva-sofie +vc +xrootd +ssl
     - any_of: [+opengl +webgui +x, -opengl -webgui -x]
   snakemake:
     require:

spack.sh

@@ -28,6 +28,17 @@ c07ddf83c32b7129247fe90eed486dd844047087
 8c29e90fa9962f4a44f39f47217b46c85176af28
 1255620a14afa3ad4aad681a847a3a1704141976
 0fed2d66bf0eec799707dd1b88ac9419f6ae14e1
+963e2ca82883cdc1287f1035c15d1a7e9a6fe612
+d3c796f2ce1da2dda198707def297aeab702d33c
+19c20563cc86140aaf352d72079bd9de292be0ac
+d50f8d7b19e07f25a3ce8de28ff9b352fd926d7f
+9e4fab277b8a5b9ec5587c8a8b6514f12c8042a4
+8f4f691e2b2a6263f661fb0a455bcaf73e90036a
+---
+## Optional hash table with comma-separated file list
+read -r -d '' SPACK_CHERRYPICKS_FILES <<- \
+--- || true
+[19c20563cc86140aaf352d72079bd9de292be0ac]=var/spack/repos/builtin/packages/hepmc3/package.py,var/spack/repos/builtin/packages/pythia8/package.py
 ---
 ## Ref: https://github.com/spack/spack/commit/[hash]
 ## [hash]: [description]
@@ -51,3 +62,9 @@ c07ddf83c32b7129247fe90eed486dd844047087
 ## 8c29e90fa9962f4a44f39f47217b46c85176af28: Build cache: make signed/unsigned a mirror property
 ## 1255620a14afa3ad4aad681a847a3a1704141976: Fix infinite recursion when computing concretization errors
 ## 0fed2d66bf0eec799707dd1b88ac9419f6ae14e1: (py-)onnx: new version 1.14.{0,1}, 1.15.0
+## 963e2ca82883cdc1287f1035c15d1a7e9a6fe612: edm4hep: add latest tag 0.10.3
+## d3c796f2ce1da2dda198707def297aeab702d33c: pythia8: new version 8.310
+## 19c20563cc86140aaf352d72079bd9de292be0ac: Initial License Checkin
+## d50f8d7b19e07f25a3ce8de28ff9b352fd926d7f: root: sha256 change on latest versions
+## 9e4fab277b8a5b9ec5587c8a8b6514f12c8042a4: [root] New variants, patches
+## 8f4f691e2b2a6263f661fb0a455bcaf73e90036a: hepmc3: add v3.2.7