Merge branch 'master' into 'root-6.28.04'

# Conflicts:
#   spack-environment/dev/spack.yaml

.gitlab-ci.yml

@@ -536,7 +536,9 @@ clean:
     - ./gitlab-ci/cleanup_registry.sh -i debian_testing_base -r 83 ${INTERNAL_TAG}
     - ./gitlab-ci/cleanup_registry.sh -i debian_stable_base -r 92 ${INTERNAL_TAG}
     - ./gitlab-ci/cleanup_registry.sh -i jug_dev -r 68 ${INTERNAL_TAG}
+    - ./gitlab-ci/cleanup_registry.sh -i jug_dev -r 68 ${INTERNAL_TAG}-nightly
     - ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${INTERNAL_TAG}
+    - ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${INTERNAL_TAG}-nightly
     - |
       if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
         ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${NIGHTLY_TAG}

containers/jug/dev.Dockerfile

@@ -4,10 +4,10 @@ ARG BASE_IMAGE="debian_stable_base"
 ARG INTERNAL_TAG="testing"
 
 ## ========================================================================================
-## STAGE1: spack builder image
-## EIC builder image with spack
+## STAGE0: spack image
+## EIC spack image with spack and eic-spack repositories
 ## ========================================================================================
-FROM ${DOCKER_REGISTRY}${BASE_IMAGE}:${INTERNAL_TAG} as builder
+FROM ${DOCKER_REGISTRY}${BASE_IMAGE}:${INTERNAL_TAG} as spack
 ARG TARGETPLATFORM
 
 ## install some extra spack dependencies
@@ -20,8 +20,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${TARGETPLATFORM}
         python3-dev                                                     \
         python3-distutils                                               \
         python3-boto3                                                   \
-        python-is-python3                                               \
- && rm -rf /var/lib/apt/lists/*
+        python-is-python3
 
 ## Setup spack
 ENV SPACK_ROOT=/opt/spack
@@ -43,6 +42,7 @@ RUN git clone https://github.com/${SPACK_ORGREPO}.git ${SPACK_ROOT}     \
 
 SHELL ["docker-shell"]
 
+## Setup build configuration
 ARG jobs=64
 RUN declare -A target=(                                                 \
       ["linux/amd64"]="x86_64_v2"                                       \
@@ -62,8 +62,8 @@ RUN declare -A target=(                                                 \
  && spack config blame compilers
 
 ## Setup local buildcache mirrors
-RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
-    spack mirror add local /var/cache/spack-mirror/${SPACK_VERSION}     \
+RUN --mount=type=cache,target=/var/cache/spack                          \
+    spack mirror add local /var/cache/spack/mirror/${SPACK_VERSION}     \
  && spack buildcache update-index local                                 \
  && spack mirror list
 
@@ -72,7 +72,7 @@ RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
 ## - the write-enabled mirror is provided later as a secret mount
 ARG S3_ACCESS_KEY=""
 ARG S3_SECRET_KEY=""
-RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
+RUN --mount=type=cache,target=/var/cache/spack                          \
     if [ -n "${S3_ACCESS_KEY}" ] ; then                                 \
     spack mirror add --scope site                                       \
       --s3-endpoint-url https://eics3.sdcc.bnl.gov:9000                 \
@@ -82,12 +82,7 @@ RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
     ; fi                                                                \
  && spack mirror list
 
-## This variable will change whenever either spack.yaml or our spack package
-## overrides change, triggering a rebuild
-ARG CACHE_BUST="hash"
-ARG CACHE_NUKE=""
-
-## Setup our custom package overrides
+## Setup eic-spack
 ENV EICSPACK_ROOT=${SPACK_ROOT}/var/spack/repos/eic-spack
 ARG EICSPACK_ORGREPO="eic/eic-spack"
 ARG EICSPACK_VERSION="$SPACK_VERSION"
@@ -100,12 +95,18 @@ RUN git clone https://github.com/${EICSPACK_ORGREPO}.git ${EICSPACK_ROOT} \
     fi                                                                  \
  && spack repo add --scope site "${EICSPACK_ROOT}"
 
+## ========================================================================================
+## STAGE1: builder
+## EIC builder image with spack environment
+## ========================================================================================
+FROM spack as builder
+
 ## Setup our custom environment (secret mount for write-enabled mirror)
 COPY --from=spack-environment . /opt/spack-environment/
 ARG ENV=dev
 ENV SPACK_ENV=/opt/spack-environment/${ENV}
 RUN --mount=type=cache,target=/ccache,id=${TARGETPLATFORM}              \
-    --mount=type=cache,target=/var/cache/spack-mirror                   \
+    --mount=type=cache,target=/var/cache/spack                          \
     --mount=type=secret,id=mirrors,target=/opt/spack/etc/spack/mirrors.yaml \
     source $SPACK_ROOT/share/spack/setup-env.sh                         \
  && export CCACHE_DIR=/ccache                                           \
@@ -116,7 +117,7 @@ RUN --mount=type=cache,target=/ccache,id=${TARGETPLATFORM}              \
  && ccache --show-stats
 
 ## Create view at /usr/local
-RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
+RUN --mount=type=cache,target=/var/cache/spack                          \
     source $SPACK_ROOT/share/spack/setup-env.sh                         \
  && spack env activate --dir ${SPACK_ENV}                               \
  && rm -r /usr/local                                                    \
@@ -125,9 +126,10 @@ RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
 ## Optional, nuke the buildcache after install, before (re)caching
 ## This is useful when going to completely different containers,
 ## or intermittently to keep the buildcache step from taking too much time
-RUN --mount=type=cache,target=/var/cache/spack-mirror,sharing=locked    \
+ARG CACHE_NUKE=""
+RUN --mount=type=cache,target=/var/cache/spack,sharing=locked           \
     [ -z "${CACHE_NUKE}" ]                                              \
-    || rm -rf /var/cache/spack-mirror/${SPACK_VERSION}/build_cache/*
+    || rm -rf /var/cache/spack/mirror/${SPACK_VERSION}/build_cache/*
 
 ## Extra post-spack steps:
 ##   - Python packages

spack-environment/concretizer.yaml

-concretizer: 
+# This file contains the preferences for the concretizer
+# - we disable reuse of build cache products during concretization
+#   (but build cache products will get used if available during install)
+# - we want unified environments that can be installed in a simple view
+#
+concretizer:
+  reuse: false
   unify: true

spack-environment/dev/spack.yaml

@@ -3,85 +3,76 @@ spack:
   - ../concretizer.yaml
   - ../packages.yaml
   specs:
-    - acts@21.1.1 +dd4hep +examples +fatras +geant4 +identification +json +python +tgeo
-    - actsvg@0.4.26 +examples
-    - acts-dd4hep@1.0.1
-    - afterburner@0.1.2 +root +zlib
-    - cairo@1.16.0 +fc+ft+X+pdf+gobject
-    - catch2@3.0.1 -ipo
-    - cernlib@2022.11.08.0-free
-    - clhep@2.4.6.4 cxxstd=17
-    - cli11@2.3.2
-    - cmake@3.26.3
-    - cnpy@master
-    #- cppcoro@git.10bbcdbf2be3ad3aa56febcf4c7662d771460a99
-    - dawn@3_91a
-    - dawncut@1_54a
-    - dd4hep@1.25.1 +ddg4 +ddcad -frames +hepmc3 +lcio +xercesc
-    - dpmjet@19.3.5
-    - edm4eic@2.0.0 cxxstd=17
-    - edm4hep@0.7.2 cxxstd=17
-    - eic-smear@1.1.10
-    - eigen@3.4.0
-    - emacs@28.2
-    - fastjet@3.4.0
-    - fjcontrib@1.051
-    - fmt@9.1.0 +shared cxxstd=17
-    - gaudi@36.10 +aida
-    - gdb@12.1 -debuginfod
-    - geant4@11.1.1 cxxstd=17 +opengl -vecgeom +x11 +qt +threads
-    - graphviz@8.0.1
-    - hepmc3@3.2.6 +python +rootio
-    - heppdt@2.06.01
-    - imagemagick@7.0.8-7
-    - intel-tbb@2020.3
-    - irt@1.0.6
-    - jana2@2.1.0 -ipo +podio +root +zmq
-    - lcov@1.16
-    - lhapdf@6.5.1 +python
-    - madx@5.08.01
-    - mesa@22.1.6 +glx -llvm +opengl +opengles +osmesa
-    - nlohmann-json@3.11.2
-    - npsim@1.1.0 +geocad +http
-    - onnx@1.13.1
-    - opencascade@7.7.1
-    - osg-ca-certs@1.109.igtf.1.117
-    - phonebook-cli@1.0.0
-    - podio@0.16.5
-    - prmon@3.0.2 +plot
-    - pythia8@8.309 +fastjet
-    - python@3.10.10
-    - py-awkward@2.1.1
-    - py-boto3@1.26.26
-    - py-dask@2021.6.2
-    - py-deepdiff@6.3.0
-    - py-ipython@8.11.0
-    - py-jinja2@3.1.2
-    - py-jinja2-cli@0.8.2
-    - py-jupyter-console@6.4.4
-    - py-jupyterlab@3.4.8
-    - py-lmfit@1.0.2
-    - py-lxml@4.9.1
-    - py-matplotlib@3.7.1
-    - py-mplhep@0.3.26
-    - py-htgettoken@1.16-33-g3788bb4
-    - py-numpy@1.23.4
-    - py-pandas@1.5.3
-    - py-pip@23.0
-    - py-pre-commit@2.20.0
-    - py-pycairo@1.20.0
-    - py-pyyaml@6.0
-    - py-scipy@1.10.1
-    - py-seaborn@0.12.2
-    - py-toml@0.10.2
-    - py-uproot@5.0.5
-    - py-yapf@0.30.0
-    - qt@5.15.9 +opengl
-    - root@6.28.04 cxxstd=17 
-          +fftw +fortran +gdml +http -ipo +mlp +pythia8 
-          +root7 +tmva +vc -webgui +xrootd +ssl 
-    - spdlog@1.11.0 +fmt_external
-    - stow@2.3.1
-    - tensorflow-lite@2.8.0 -xnnpack
-    - xrootd@5.5.5 cxxstd=17 -davix +python +scitokens-cpp
+    - acts
+    - actsvg
+    - acts-dd4hep
+    - afterburner
+    - cairo
+    - catch2
+    - cernlib
+    - cli11
+    - cmake
+    - cnpy
+    #- cppcoro
+    - dawn
+    - dawncut
+    - dd4hep
+    - dpmjet
+    - edm4eic
+    - edm4hep
+    - eic-smear
+    - eigen
+    - emacs
+    - fastjet
+    - fjcontrib
+    - fmt
+    - gaudi
+    - gdb
+    - geant4
+    - graphviz
+    - hepmc3
+    - heppdt
+    - imagemagick
+    - irt
+    - jana2
+    - lcov
+    - lhapdf
+    - madx
+    - npsim
+    - onnx
+    - opencascade
+    - osg-ca-certs
+    - phonebook-cli
+    - prmon
+    - pythia8
+    - py-awkward
+    - py-boto3
+    - py-dask
+    - py-deepdiff
+    - py-ipython
+    - py-jinja2
+    - py-jinja2-cli
+    - py-jupyter-console
+    - py-jupyterlab
+    - py-lmfit
+    - py-lxml
+    - py-matplotlib
+    - py-mplhep
+    - py-htgettoken
+    - py-numpy
+    - py-pandas
+    - py-pip
+    - py-pre-commit
+    - py-pycairo
+    - py-pyyaml
+    - py-scipy
+    - py-seaborn
+    - py-toml
+    - py-uproot
+    - py-yapf
+    - root
+    - spdlog
+    - stow
+    - tensorflow-lite
+    - xrootd
   view: false

spack-environment/packages.yaml

+# This file contains the preferences for versions and variants.
+# Note that spack can deviate from these preferences for various reasons:
+# - if explicitly requested in spack.yaml
+# - if required to satisfy explicit requests
+# - if reuse == true and build caches contain other versions/variants
+#   (this is disabled by concretize:reuse:false in concretizer.yaml)
+#
+# The variants in packages:all:variants are overridden by package-specific 
+# variants. Therefore, we use the pattern require:any_of:[+variant,@:]
+# which sets the variant or doesn't do anything if not supported.
+#
 packages:
   all:
     compiler: [gcc]
-    variants:
-      +ipo build_type=Release
+    require:
+    - any_of: [+ipo, '@:']
+    - any_of: [build_type=Release, '@:']
+  acts:
+    require:
+    - '@21.1.1'
+    - +dd4hep +examples +fatras +geant4 +identification +json +python +tgeo
+  actsvg:
+    require:
+    - '@0.4.26'
+    - +examples
+  acts-dd4hep:
+    require:
+    - '@1.0.1'
+  afterburner:
+    require:
+    - '@0.1.2'
+    - +root +zlib
+  cairo:
+    require:
+    - '@1.16.0'
+    - +fc+ft+X+pdf+gobject
+  catch2:
+    require:
+    - '@3.0.1'
+    - -ipo
+  cernlib:
+    require:
+    - '@2022.11.08.0-free'
+  clhep:
+    require:
+    - '@2.4.6.4'
+    - cxxstd=17
+  cli11:
+    require:
+    - '@2.3.2'
+  cmake:
+    require:
+    - '@3.26.3'
+  cnpy:
+    require:
+    - '@master'
+  cppcoro:
+    require:
+    - '@git.10bbcdbf2be3ad3aa56febcf4c7662d771460a99'
+  dawn:
+    require:
+    - '@3_91a'
+  dawncut:
+    require:
+    - '@1_54a'
+  dd4hep:
+    require:
+    - '@1.25.1'
+    - +ddg4 +ddcad -frames +hepmc3 +lcio +xercesc
+  dpmjet:
+    require:
+    - '@19.3.5'
+  edm4eic:
+    require:
+    - '@2.0.0'
+    - cxxstd=17
+  edm4hep:
+    require:
+    - '@0.7.2'
+    - cxxstd=17
+  eic-smear:
+    require:
+    - '@1.1.10'
+  eigen:
+    require:
+    - '@3.4.0'
+  emacs:
+    require:
+    - '@28.2'
+  fastjet:
+    require:
+    - '@3.4.0'
+  fjcontrib:
+    require:
+    - '@1.051'
+  fmt:
+    require:
+    - '@9.1.0'
+    - +shared cxxstd=17
+  gaudi:
+    require:
+    - '@36.10'
+    - +aida
+  gdb:
+    require:
+    - '@12.1'
+    - -debuginfod
+  geant4:
+    require:
+    - '@11.1.1'
+    - cxxstd=17 +opengl -vecgeom +x11 +qt +threads
+  graphviz:
+    require:
+    - '@8.0.1'
+  hepmc3:
+    require:
+    - '@3.2.6'
+    - +python +rootio
+  heppdt:
+    require:
+    - '@2.06.01'
+  imagemagick:
+    require:
+    - '@7.0.8-7'
+  intel-tbb:
+    require:
+    - '@2020.3'
+  irt:
+    require:
+    - '@1.0.6'
+  jana2:
+    require:
+    - '@2.1.0'
+    - -ipo +podio +root +zmq
+  lcov:
+    require:
+    - '@1.16'
+  lhapdf:
+    require:
+    - '@6.5.1'
+    - +python
+  madx:
+    require:
+    - '@5.08.01'
+  mesa:
+    require:
+    - '@22.1.6'
+    - +glx -llvm +opengl +opengles +osmesa
+  nlohmann-json:
+    require:
+    - '@3.11.2'
+  npsim:
+    require:
+    - '@1.1.0'
+    - +geocad +http
+  onnx:
+    require:
+    - '@1.13.1'
+  opencascade:
+    require:
+    - '@7.7.1'
+  osg-ca-certs:
+    require:
+    - '@1.109.igtf.1.117'
+  phonebook-cli:
+    require:
+    - '@1.0.0'
+  podio:
+    require:
+    - '@0.16.5'
+  prmon:
+    require:
+    - '@3.0.2'
+    - +plot
+  pythia8:
+    require:
+    - '@8.309'
+    - +fastjet
+  python:
+    require:
+    - '@3.10.10'
+  py-awkward:
+    require:
+    - '@2.1.1'
+  py-boto3:
+    require:
+    - '@1.26.26'
+  py-dask:
+    require:
+    - '@2021.6.2'
+  py-deepdiff:
+    require:
+    - '@6.3.0'
+  py-ipython:
+    require:
+    - '@8.11.0'
+  py-jinja2:
+    require:
+    - '@3.1.2'
+  py-jinja2-cli:
+    require:
+    - '@0.8.2'
+  py-jupyter-console:
+    require:
+    - '@6.4.4'
+  py-jupyterlab:
+    require:
+    - '@3.4.8'
+  py-lmfit:
+    require:
+    - '@1.0.2'
+  py-lxml:
+    require:
+    - '@4.9.1'
+  py-matplotlib:
+    require:
+    - '@3.7.1'
+  py-mplhep:
+    require:
+    - '@0.3.26'
+  py-htgettoken:
+    require:
+    - '@1.16-33-g3788bb4'
+  py-numpy:
+    require:
+    - '@1.23.4'
+  py-pandas:
+    require:
+    - '@1.5.3'
+  py-pip:
+    require:
+    - '@23.0'
+  py-pre-commit:
+    require:
+    - '@2.20.0'
+  py-pycairo:
+    require:
+    - '@1.20.0'
+  py-pyyaml:
+    require:
+    - '@6.0'
+  py-scipy:
+    require:
+    - '@1.10.1'
+  py-seaborn:
+    require:
+    - '@0.12.2'
+  py-toml:
+    require:
+    - '@0.10.2'
+  py-uproot:
+    require:
+    - '@5.0.5'
+  py-yapf:
+    require:
+    - '@0.30.0'
+  qt:
+    require:
+    - '@5.15.9'
+    - +opengl
+  root:
+    require:
+    - '@6.26.10'
+    - cxxstd=17 +fftw +fortran +gdml +http -ipo +mlp +pythia8 +root7 +tmva +vc -webgui +xrootd +ssl 
+  spdlog:
+    require:
+    - '@1.11.0'
+    - +fmt_external
+  stow:
+    require:
+    - '@2.3.1'
+  tensorflow-lite:
+    require:
+    - '@2.8.0'
+    - -xnnpack
+  xrootd:
+    require:
+    - '@5.5.5'
+    - cxxstd=17 -davix +python +scitokens-cpp

spack.sh

@@ -11,9 +11,11 @@ read -r -d '' SPACK_CHERRYPICKS <<- \
 9ee2d79de172de14477a78e5d407548a63eea33a
 776ab132760d63eab0703b7c0ebebc72a8443f5b
 188168c476eabe99764634db8d78eb3f9ea2a927
+b3268c2703b84f4e4961c1e2cdf43e8998f283e6
 ---
 ## Ref: https://github.com/spack/spack/commit/[hash]
 ## [hash]: [description]
 ## 9ee2d79de172de14477a78e5d407548a63eea33a: libxpm package: fix RHEL8 build with libintl
 ## 776ab132760d63eab0703b7c0ebebc72a8443f5b: [xrootd] New variants, new version, improve build config
 ## 188168c476eabe99764634db8d78eb3f9ea2a927: podio: Add 0.16.5 tag
+## b3268c2703b84f4e4961c1e2cdf43e8998f283e6: freetype: add pic and shared variants