feat: install epic with spack

.gitlab-ci.yml

@@ -102,7 +102,6 @@ version:
         NIGHTLY_TAG="${VERSION}-nightly"
         EXPORT_TAG="${VERSION}"
         DH_PUSH=""
-        GH_PUSH=""
       elif [ "x${CI_PIPELINE_SOURCE}" = "xtrigger" ]; then
         VERSION="${CI_COMMIT_BRANCH}"
         DH_PUSH=""

containers/jug/dev.Dockerfile

@@ -32,6 +32,7 @@ EOF
 ENV SPACK_ROOT=/opt/spack
 ARG SPACK_ORGREPO="spack/spack"
 ARG SPACK_VERSION="releases/v0.20"
+ENV SPACK_PYTHON=/usr/bin/python3
 ARG SPACK_CHERRYPICKS=""
 ARG SPACK_CHERRYPICKS_FILES=""
 ADD https://api.github.com/repos/${SPACK_ORGREPO}/commits/${SPACK_VERSION} /tmp/spack.json
@@ -150,13 +151,19 @@ source ${SPACK_ROOT}/share/spack/setup-env.sh
 mkdir -p /var/cache/spack/blobs/sha256/
 find /var/cache/spack/blobs/sha256/ -ignore_readdir_race -atime +7 -delete
 spack buildcache update-index eics3rw
+echo -e "\n  view: false" >> ${SPACK_ENV}/spack.yaml
 spack env activate --dir ${SPACK_ENV}
-spack concretize --fresh --force --quiet
+spack concretize --fresh --force
 make --jobs ${jobs} --keep-going --directory /opt/spack-environment \
   SPACK_ENV=${SPACK_ENV} \
   BUILDCACHE_OCI_PROMPT="eicweb" \
   BUILDCACHE_OCI_FINAL="ghcr" \
   BUILDCACHE_S3_FINAL="eics3rw"
+spack find --implicit --no-groups \
+| sed -e '1,/Installed packages/d;s/\([^@]*\).*/\1/g' \
+| uniq -d | grep -v py-pip | grep -v py-cython \
+| tee /tmp/duplicates.txt
+test -s /tmp/duplicates.txt && exit 1
 ccache --show-stats
 ccache --zero-stats
 EOF
@@ -173,40 +180,48 @@ RUN --mount=type=cache,target=/ccache,id=${TARGETPLATFORM}              \
     --mount=type=cache,target=/var/cache/spack                          \
     --mount=type=secret,id=mirrors,target=/opt/spack/etc/spack/mirrors.yaml \
     <<EOF
-source ${SPACK_ROOT}/share/spack/setup-env.sh
+set -e
 export CCACHE_DIR=/ccache
 spack buildcache update-index eics3rw
 spack env activate --dir ${SPACK_ENV}
 if [ "${EDM4EIC_VERSION}" != "8aeb507f93a93257c99985efbce0ec1371e0b331" ] ; then
   export EDM4EIC_VERSION=$(jq -r .sha /tmp/edm4eic.json)
-  spack config add "packages:edm4eic::require:['@git.${EDM4EIC_VERSION}=main']"
+  sed -i "/# EDM4EIC_VERSION$/ s/@[^\s']*/@git.${EDM4EIC_VERSION}=main/" /opt/spack-environment/packages.yaml
   spack deconcretize -y --all edm4eic
 fi
 if [ "${EICRECON_VERSION}" != "28108da4a1e8919a05dfdb5f11e114800a2cbe96" ] ; then
   export EICRECON_VERSION=$(jq -r .sha /tmp/eicrecon.json)
-  spack config add "packages:eicrecon::require:['@git.${EICRECON_VERSION}=main']"
+  sed -i "/# EICRECON_VERSION$/ s/@[^\s']*/@git.${EICRECON_VERSION}=main/" /opt/spack-environment/packages.yaml
   spack deconcretize -y --all eicrecon
 fi
 if [ "${JUGGLER_VERSION}" != "df87bf1f8643afa8e80bece9d36d6dc26dfe8132" ] ; then
   export JUGGLER_VERSION=$(jq -r .sha /tmp/juggler.json)
-  spack config add "packages:juggler::require:['@git.${JUGGLER_VERSION}=main']"
+  sed -i "/# JUGGLER_VERSION$/ s/@[^\s']*/@git.${JUGGLER_VERSION}=main/" /opt/spack-environment/packages.yaml
   spack deconcretize -y --all juggler
 fi
-spack concretize --fresh --force --quiet
+spack concretize --fresh --force
 make --jobs ${jobs} --keep-going --directory /opt/spack-environment \
   SPACK_ENV=${SPACK_ENV} \
   BUILDCACHE_OCI_PROMPT="eicweb" \
   BUILDCACHE_OCI_FINAL="ghcr" \
   BUILDCACHE_S3_PROMPT="eics3rw"
+spack find --implicit --no-groups \
+| sed -e '1,/Installed packages/d;s/\([^@]*\).*/\1/g' \
+| uniq -d | grep -v py-pip | grep -v py-cython \
+| tee /tmp/duplicates.txt
+test -s /tmp/duplicates.txt && exit 1
 ccache --show-stats
 ccache --zero-stats
 EOF
 
-## Create view at /usr/local
+## Create views at /usr/local and /opt/detectors
 RUN <<EOF
 set -e
 rm -r /usr/local
-spack -e ${SPACK_ENV} env view enable /usr/local
+sed -i -e '/view: false/d' ${SPACK_ENV}/spack.yaml
+cat /opt/spack-environment/view.yaml >> ${SPACK_ENV}/spack.yaml
+spack -e ${SPACK_ENV} env view regenerate /usr/local
+spack -e ${SPACK_ENV} env view regenerate /opt/detectors
 EOF
 
 ## Place cvmfs catalogs
@@ -298,6 +313,7 @@ COPY --from=staging /opt/spack /opt/spack
 COPY --from=staging /opt/spack-environment /opt/spack-environment
 COPY --from=staging /opt/software /opt/software
 COPY --from=staging /usr/._local /usr/._local
+COPY --from=staging /opt/._detectors /opt/._detectors
 COPY --from=staging /etc/profile.d /etc/profile.d
 COPY --from=staging /etc/jug_info /etc/jug_info
 COPY --from=staging /etc/eic-env.sh /etc/eic-env.sh
@@ -311,11 +327,13 @@ SHELL ["docker-shell"]
 ## ensure /usr/local is the view, not a symlink
 RUN <<EOF
 set -ex
-rm -rf /usr/local
-PREFIX_PATH=$(realpath $(ls /usr/._local/ | tail -n1))
-echo "Found spack true prefix path to be $PREFIX_PATH"
-mv /usr/._local/${PREFIX_PATH} /usr/local
-ln -s /usr/local /usr/._local/${PREFIX_PATH}
+rm -rf /usr/local /opt/detectors
+LOCAL_PREFIX_PATH=$(realpath $(ls /usr/._local/ | tail -n1))
+mv /usr/._local/${LOCAL_PREFIX_PATH} /usr/local
+ln -s /usr/local /usr/._local/${LOCAL_PREFIX_PATH}
+DETECTORS_PREFIX_PATH=$(realpath $(ls /opt/._detectors/ | tail -n1))
+mv /opt/._detectors/${DETECTORS_PREFIX_PATH} /opt/detectors
+ln -s /opt/detectors /opt/._detectors/${DETECTORS_PREFIX_PATH}
 EOF
 
 ## set ROOT TFile forward compatibility

eic-spack.sh

@@ -3,7 +3,7 @@ EICSPACK_ORGREPO="eic/eic-spack"
 
 ## EIC spack commit hash or github version, e.g. v0.19.7
 ## note: nightly builds could use a branch e.g. releases/v0.19
-EICSPACK_VERSION="3913c7fc5f45f719d6f74c86b4f08191ad6a5513"
+EICSPACK_VERSION="c0da90347ee66d2744fcee905d8e49e58a639782"
 
 ## Space-separated list of eic-spack cherry-picks
 read -r -d '' EICSPACK_CHERRYPICKS <<- \

spack-environment/concretizer.yaml

-# This file contains the preferences for the concretizer
-# - we disable reuse of build cache products during concretization
-#   (but build cache products will get used if available during install)
-# - we want unified environments that can be installed in a simple view
+# This file contains the preferences for the concretizer:
+# - We disable reuse of build cache products during concretization
+#   (but build cache products will get used if available during install).
+# - We want unified environments that can be installed in a simple view,
+#   but unfortunately that prevents multiple geometry versions. We use
+#   when_possible and consider duplicate dependencies an error.
 #
 concretizer:
   reuse: false
-  unify: true
+  unify: when_possible

spack-environment/dbg/spack.yaml

@@ -12,8 +12,8 @@ spack:
   - edm4eic build_type=Debug
   - edm4hep build_type=Debug
   - eicrecon build_type=Debug
+  - epic build_type=Debug
   - gdb
   - irt build_type=Debug
   - jana2 build_type=Debug
-  - valgrind
-  view: false
+  - valgrind
\ No newline at end of file

spack-environment/dev/spack.yaml

@@ -27,6 +27,15 @@ spack:
   - eicrecon
   - eigen
   - emacs
+  - epic@main
+  - epic@23.10.0
+  - epic@23.11.0
+  - epic@23.12.0
+  - epic@24.02.0
+  - epic@24.02.1
+  - epic@24.03.0
+  - epic@24.03.1
+  - epic@24.04.0
   - fastjet
   - fjcontrib
   - fmt
@@ -92,4 +101,3 @@ spack:
   - valgrind
   - xrootd
   - xeyes
-  view: false

spack-environment/packages.yaml

@@ -11,10 +11,11 @@
 #
 packages:
   all:
-    compiler: [gcc]
     require:
+    - '%gcc'
     - any_of: [+ipo, '@:']
     - any_of: [build_type=Release, '@:']
+    - any_of: [^py-pip@23.1.2, '@:']
   acts:
     require:
     - '@31.2.0'
@@ -83,7 +84,7 @@ packages:
     - '@656aa3192b097a631ddd1e0380e80c26fd6644a7'
   edm4eic:
     require:
-    - '@5.0.0'
+    - '@5.0.0' # EDM4EIC_VERSION
     - cxxstd=20
   edm4hep:
     require:
@@ -94,13 +95,16 @@ packages:
     - '@1.1.12'
   eicrecon:
     require:
-    - '@1.12.0'
+    - '@1.12.0' # EICRECON_VERSION
   eigen:
     require:
     - '@3.4.0'
   emacs:
     require:
     - '@28.2'
+  epic:
+    require:
+    - 'artifacts=epic_craterlake'
   fastjet:
     require:
     - '@3.4.0'
@@ -163,7 +167,7 @@ packages:
     - -ipo +podio +root +zmq
   juggler:
     require:
-    - '@14.0.0'
+    - '@14.0.0' # JUGGLER_VERSION
     - cxxstd=20
   k4actstracking:
     require:

spack-environment/prod/spack.yaml

@@ -10,6 +10,7 @@ spack:
   - dd4hep -ddeve
   - edm4eic
   - eicrecon
+  - epic
   - fastjet
   - fjcontrib
   - gaudi
@@ -25,5 +26,4 @@ spack:
   - py-jinja2-cli
   - py-pip
   - root -opengl
-  - xrootd
-  view: false
+  - xrootd
\ No newline at end of file

spack-environment/view.yaml

+  view:
+    default:
+      root: /usr/local
+      exclude: [epic]
+      link_type: symlink
+    detectors:
+      root: /opt/detectors
+      select: [epic]
+      projections:
+        all: '{name}-{version}'
+      link: roots
+      link_type: symlink

spack.sh

@@ -64,6 +64,9 @@ e8ae9a403ca7db7738d36bf41bf99977b9c88a84
 62132919e1465a40e3522e40278e4bbc8ec32978
 946c539dbd95cd78ababda66e58a1c3f8953c95a
 519dd0a6ff9f76384b3f19fa884fd6397cf79ac9
+f1ec4859c8bfd47ff1565d1eb74bcaab22a7ea16
+bbcd4224fa1edfa1fb91e101b58596f30b79b85f
+c2eef8bab26adb00b250992e29d697b4706356a0
 ---
 ## Optional hash table with comma-separated file list
 read -r -d '' SPACK_CHERRYPICKS_FILES <<- \
@@ -71,7 +74,9 @@ read -r -d '' SPACK_CHERRYPICKS_FILES <<- \
 [70fb0b35e55f8f9b3182b091bc8c40b8770041b7]=var/spack/repos/builtin/packages/py-fsspec/package.py
 [5140a9b6a3588d9e44a98a6a1f3993e7687ef2fe]=var/spack/repos/builtin/packages/py-jax/package.py,var/spack/repos/builtin/packages/py-jaxlib/package.py
 [9c47ecaeb25300ac2a6a2609628ecd4c928fcf49]=lib/spack/spack/directives.py
-[19c20563cc86140aaf352d72079bd9de292be0ac]=var/spack/repos/builtin/packages/abseil-cpp/package.py,var/spack/repos/builtin/packages/acts/package.py,var/spack/repos/builtin/packages/actsvg/package.py,var/spack/repos/builtin/packages/dd4hep/package.py,var/spack/repos/builtin/packages/hepmc3/package.py,var/spack/repos/builtin/packages/protobuf/package.py,var/spack/repos/builtin/packages/pythia8/package.py,var/spack/repos/builtin/packages/py-aiobotocore/package.py,var/spack/repos/builtin/packages/py-cdsapi/package.py,var/spack/repos/builtin/packages/py-cfgrib/package.py,var/spack/repos/builtin/packages/py-chex/package.py,var/spack/repos/builtin/packages/py-dm-haiku/package.py,var/spack/repos/builtin/packages/py-etils/package.py,var/spack/repos/builtin/packages/py-flit-core/package.py,var/spack/repos/builtin/packages/py-fsspec/package.py,var/spack/repos/builtin/packages/py-gcsfs/package.py,var/spack/repos/builtin/packages/py-h5netcdf/package.py,var/spack/repos/builtin/packages/py-importlib-metadata/package.py,var/spack/repos/builtin/packages/py-jax/package.py,var/spack/repos/builtin/packages/py-jaxlib/package.py,var/spack/repos/builtin/packages/py-netcdf4/package.py,var/spack/repos/builtin/packages/py-onnx/package.py,var/spack/repos/builtin/packages/py-onnxruntime/package.py,var/spack/repos/builtin/packages/py-pytest-asyncio/package.py,var/spack/repos/builtin/packages/py-pytest-timeout/package.py,var/spack/repos/builtin/packages/py-python-dotenv/package.py,var/spack/repos/builtin/packages/py-s3fs/package.py,var/spack/repos/builtin/packages/py-zarr/package.py,var/spack/repos/builtin/packages/xrootd/package.py
+[19c20563cc86140aaf352d72079bd9de292be0ac]=var/spack/repos/builtin/packages/abseil-cpp/package.py,var/spack/repos/builtin/packages/acts/package.py,var/spack/repos/builtin/packages/actsvg/package.py,var/spack/repos/builtin/packages/dd4hep/package.py,var/spack/repos/builtin/packages/fontconfig/package.py,var/spack/repos/builtin/packages/hepmc3/package.py,var/spack/repos/builtin/packages/protobuf/package.py,var/spack/repos/builtin/packages/pythia8/package.py,var/spack/repos/builtin/packages/py-aiobotocore/package.py,var/spack/repos/builtin/packages/py-cdsapi/package.py,var/spack/repos/builtin/packages/py-cfgrib/package.py,var/spack/repos/builtin/packages/py-chex/package.py,var/spack/repos/builtin/packages/py-dm-haiku/package.py,var/spack/repos/builtin/packages/py-etils/package.py,var/spack/repos/builtin/packages/py-flit-core/package.py,var/spack/repos/builtin/packages/py-fsspec/package.py,var/spack/repos/builtin/packages/py-gcsfs/package.py,var/spack/repos/builtin/packages/py-h5netcdf/package.py,var/spack/repos/builtin/packages/py-importlib-metadata/package.py,var/spack/repos/builtin/packages/py-jax/package.py,var/spack/repos/builtin/packages/py-jaxlib/package.py,var/spack/repos/builtin/packages/py-netcdf4/package.py,var/spack/repos/builtin/packages/py-onnx/package.py,var/spack/repos/builtin/packages/py-onnxruntime/package.py,var/spack/repos/builtin/packages/py-pytest-asyncio/package.py,var/spack/repos/builtin/packages/py-pytest-timeout/package.py,var/spack/repos/builtin/packages/py-python-dotenv/package.py,var/spack/repos/builtin/packages/py-s3fs/package.py,var/spack/repos/builtin/packages/py-zarr/package.py,var/spack/repos/builtin/packages/xrootd/package.py
+[f1ec4859c8bfd47ff1565d1eb74bcaab22a7ea16]=var/spack/repos/builtin/packages/fontconfig/package.py
+[bbcd4224fa1edfa1fb91e101b58596f30b79b85f]=var/spack/repos/builtin/packages/fontconfig/package.py
 ---
 ## Ref: https://github.com/spack/spack/commit/[hash]
 ## [hash]: [description]
@@ -131,3 +136,6 @@ read -r -d '' SPACK_CHERRYPICKS_FILES <<- \
 ## 62132919e1465a40e3522e40278e4bbc8ec32978: xrootd: new version 5.6.9
 ## 946c539dbd95cd78ababda66e58a1c3f8953c95a: osg-ca-certs: new version osg-1.119 igtf-1.128
 ## 519dd0a6ff9f76384b3f19fa884fd6397cf79ac9: pythia8: patch latest 8.311 for upstream bug
+## f1ec4859c8bfd47ff1565d1eb74bcaab22a7ea16: unmaintained packages: add new versions
+## bbcd4224fa1edfa1fb91e101b58596f30b79b85f: py-matplotlib: add v3.8.4
+## c2eef8bab26adb00b250992e29d697b4706356a0: fontconfig: depends_on gperf when 2.11.1: