Compare revisions

Dmitry Kalinkin · Dmitry Kalinkin · Dmitry Kalinkin · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck
--- a/.github/workflows/mirror.yaml
+++ b/.github/workflows/mirror.yaml
 name: Mirror and Trigger EICweb

 on:
-  pull_request:
+  delete:
  push:
  workflow_dispatch:

@@ -22,7 +22,6 @@ jobs:
      with:
        fetch-depth: 0
    - name: Push to EICweb
-      if: github.event_name == 'push'
      uses: eic/gitlab-sync@master
      with:
        url: https://eicweb.phy.anl.gov/containers/eic_container.git/
@@ -30,7 +29,7 @@ jobs:
        username: ${{ secrets.GITLAB_USERNAME }}
        ciskip: true
    - name: Trigger EICweb
-      if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
+      if: ${{ github.event_name != 'delete' }}
      uses: eic/trigger-gitlab-ci@v3
      with:
        url: https://eicweb.phy.anl.gov

--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -96,6 +96,7 @@ nvidia-smi:
  script:
    - |
      nvidia-smi
+  allow_failure: true

 ## Default version and tags
 ## - internal tag used for the CI. Also temporarily tagged
@@ -511,6 +512,8 @@ jug_xl:singularity:nightly:

 benchmarks:detector:default:
  extends: .benchmarks
+  rules:
+   - if: '$CI_PIPELINE_SOURCE != "schedule"'
  variables:
    BENCHMARKS_TAG: "${INTERNAL_TAG}-default"
  trigger:
@@ -520,6 +523,7 @@ benchmarks:detector:default:
 benchmarks:detector:nightly:
  extends: .benchmarks
  rules:
+    - if: '$CI_PIPELINE_SOURCE != "schedule"'
    - !reference ['.nightly', rules]
  variables:
    BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly"
@@ -529,6 +533,8 @@ benchmarks:detector:nightly:

 benchmarks:phyiscs:default:
  extends: .benchmarks
+  rules:
+   - if: '$CI_PIPELINE_SOURCE != "schedule"'
  variables:
    BENCHMARKS_TAG: "${INTERNAL_TAG}-default"
  trigger:
@@ -538,6 +544,7 @@ benchmarks:phyiscs:default:
 benchmarks:physics:nightly:
  extends: .benchmarks
  rules:
+    - if: '$CI_PIPELINE_SOURCE != "schedule"'
    - !reference ['.nightly', rules]
  variables:
    BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly"
@@ -547,6 +554,8 @@ benchmarks:physics:nightly:

 benchmarks:reconstruction:default:
  extends: .benchmarks
+  rules:
+   - if: '$CI_PIPELINE_SOURCE != "schedule"'
  variables:
    BENCHMARKS_TAG: "${INTERNAL_TAG}-default"
  trigger:
@@ -556,6 +565,7 @@ benchmarks:reconstruction:default:
 benchmarks:reconstruction:nightly:
  extends: .benchmarks
  rules:
+    - if: '$CI_PIPELINE_SOURCE != "schedule"'
    - !reference ['.nightly', rules]
  variables:
    BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly"
@@ -570,6 +580,8 @@ df:
    - docker-new
  needs: 
    - version
+  rules:
+    - when: manual
  script:
    - docker images
    - docker ps --size
@@ -588,6 +600,7 @@ df:
      - docker-stats-all.txt
      - docker-buildx-du-verbose.txt
      - docker-system-df-verbose.txt
+  allow_failure: true

 prune: 
  extends: .build

--- a/containers/debian/base.Dockerfile
+++ b/containers/debian/base.Dockerfile
-#syntax=docker/dockerfile:1.4
+#syntax=docker/dockerfile:1.8
+#check
 ARG BASE_IMAGE="amd64/debian:stable-slim"
 ARG BUILD_IMAGE="debian_stable_base"


--- a/containers/jug/dev.Dockerfile
+++ b/containers/jug/dev.Dockerfile
-#syntax=docker/dockerfile:1.4
+#syntax=docker/dockerfile:1.8
+#check
 ARG DOCKER_REGISTRY="eicweb/"
 ARG BUILDER_IMAGE="debian_stable_base"
 ARG RUNTIME_IMAGE="debian_stable_base"
@@ -155,13 +156,12 @@ spack concretize --fresh --force
 make --jobs ${jobs} --keep-going --directory /opt/spack-environment \
  SPACK_ENV=${SPACK_ENV} \
  BUILDCACHE_OCI_PROMPT="eicweb" \
-  BUILDCACHE_OCI_FINAL="ghcr" \
-  BUILDCACHE_S3_FINAL="eics3rw"
-spack find --implicit --no-groups \
+  BUILDCACHE_OCI_FINAL="ghcr"
+spack find --long --no-groups \
 | sed -e '1,/Installed packages/d;s/\([^@]*\).*/\1/g' \
-| uniq -d | grep -v py-pip | grep -v py-cython \
+| uniq -D -f1 | grep -v -w -e "\(epic\|py-pip\|py-cython\)" \
 | tee /tmp/duplicates.txt
-test -s /tmp/duplicates.txt && exit 1
+test -s /tmp/duplicates.txt && ( cat /tmp/duplicates.txt | while read hash spec ; do spack spec --long /${hash} ; done ) && exit 1
 ccache --show-stats
 ccache --zero-stats
 EOF
@@ -207,13 +207,12 @@ fi
 spack concretize --fresh --force
 make --jobs ${jobs} --keep-going --directory /opt/spack-environment \
  SPACK_ENV=${SPACK_ENV} \
-  BUILDCACHE_OCI_FINAL="eicweb" \
-  BUILDCACHE_S3_FINAL="eics3rw"
-spack find --implicit --no-groups \
+  BUILDCACHE_OCI_FINAL="eicweb"
+spack find --long --no-groups \
 | sed -e '1,/Installed packages/d;s/\([^@]*\).*/\1/g' \
-| uniq -d | grep -v py-pip | grep -v py-cython \
+| uniq -D -f1 | grep -v -w -e "\(epic\|py-pip\|py-cython\)" \
 | tee /tmp/duplicates.txt
-test -s /tmp/duplicates.txt && exit 1
+test -s /tmp/duplicates.txt && ( cat /tmp/duplicates.txt | while read hash spec ; do spack spec --long /${hash} ; done ) && exit 1
 ccache --show-stats
 ccache --zero-stats
 EOF

--- a/eic-spack.sh
+++ b/eic-spack.sh
@@ -3,4 +3,4 @@ EICSPACK_ORGREPO="eic/eic-spack"

 ## EIC spack commit hash or github version, e.g. v0.19.7
 ## note: nightly builds could use a branch e.g. releases/v0.19
-EICSPACK_VERSION="5823ee8e20515529539f3e1a792913b90bf21202"
+EICSPACK_VERSION="ddb9df26338cae470a696a1f1a56eaa192ffb4c4"
--- a/mirrors.yaml.in
+++ b/mirrors.yaml.in
@@ -2,6 +2,10 @@ mirrors:
  eicweb:
    url: oci://${CI_REGISTRY}/${CI_PROJECT_PATH}/spack-${SPACK_VERSION}
    signed: false
+    fetch:
+      access_pair:
+      - ${CI_REGISTRY_USER}
+      - ${CI_REGISTRY_PASSWORD}
    push:
      access_pair:
      - ${CI_REGISTRY_USER}
@@ -9,11 +13,16 @@ mirrors:
  ghcr:
    url: oci://ghcr.io/eic/spack-${SPACK_VERSION}
    signed: false
+    fetch:
+      access_pair:
+      - ${GITHUB_REGISTRY_USER}
+      - ${GITHUB_REGISTRY_TOKEN}
    push:
      access_pair:
      - ${GITHUB_REGISTRY_USER}
      - ${GITHUB_REGISTRY_TOKEN}
  eics3rw:
+    autopush: true
    signed: false
    fetch:
      url: s3://eictest/EPIC/spack/${SPACK_VERSION}

--- a/spack-environment/Makefile
+++ b/spack-environment/Makefile
 MAKEFLAGS += -Orecurse

-SPACK ?= spack
+SPACK ?= spack --backtrace
 SPACK_INSTALL_FLAGS += --no-check-signature --show-log-on-error

 export SPACK_COLOR = always
@@ -22,15 +22,15 @@ include $(SPACK_ENV)/spack.mk
 endif

 $(SPACK_ENV)/push: $(addprefix $(SPACK_ENV)/push/,$($(SPACK_ENV)/SPACK_PACKAGE_IDS))
-	$(foreach buildcache, $(BUILDCACHE_S3_FINAL), $(SPACK) buildcache push --unsigned $(buildcache) ;)
+	$(foreach buildcache, $(BUILDCACHE_S3_FINAL), $(SPACK) buildcache push --private --unsigned $(buildcache) ;)
 	$(foreach buildcache, $(BUILDCACHE_S3_FINAL), $(SPACK) buildcache update-index $(buildcache) ;)
 	$(foreach buildcache, $(BUILDCACHE_S3_PROMPT), $(SPACK) buildcache update-index $(buildcache) ;)
-	$(foreach buildcache, $(BUILDCACHE_OCI_FINAL), $(SPACK) buildcache push --unsigned --base-image $(BUILDCACHE_OCI_BASE_IMAGE) $(buildcache) | grep -v "Using cached archive" ;)
+	$(foreach buildcache, $(BUILDCACHE_OCI_FINAL), $(SPACK) buildcache push --private --unsigned --base-image $(BUILDCACHE_OCI_BASE_IMAGE) $(buildcache) | grep -v "Using cached archive" ;)

 $(SPACK_ENV)/push/%: $(SPACK_ENV)/install/%
 	@mkdir -p $(dir $@)
-	$(foreach buildcache, $(BUILDCACHE_S3_PROMPT), $(SPACK) buildcache push --unsigned --only=package $(buildcache) /$(HASH) ;) # push $(SPEC)
-	$(foreach buildcache, $(BUILDCACHE_OCI_PROMPT), $(SPACK) buildcache push --unsigned --base-image $(BUILDCACHE_OCI_BASE_IMAGE) $(buildcache) /$(HASH) | grep -v "Using cached archive" ;) # push $(SPEC)
+	$(foreach buildcache, $(BUILDCACHE_S3_PROMPT), $(SPACK) buildcache push --private --unsigned --only=package $(buildcache) /$(HASH) ;) # push $(SPEC)
+	$(foreach buildcache, $(BUILDCACHE_OCI_PROMPT), $(SPACK) buildcache push --private --unsigned --base-image $(BUILDCACHE_OCI_BASE_IMAGE) $(buildcache) /$(HASH) | grep -v "Using cached archive" ;) # push $(SPEC)
 	@touch $@

 $(SPACK_ENV)/spack.lock: $(SPACK_ENV)/spack.yaml Makefile

--- a/spack-environment/cuda/spack.yaml
+++ b/spack-environment/cuda/spack.yaml
@@ -28,17 +28,11 @@ spack:
  - eigen
  - emacs
  - epic@main # EPIC_VERSION
-  - epic@23.10.0
-  - epic@23.11.0
-  - epic@23.12.0
-  - epic@24.02.0
-  - epic@24.02.1
-  - epic@24.03.0
-  - epic@24.03.1
-  - epic@24.04.0
  - epic@24.05.0
  - epic@24.05.2
  - epic@24.06.0
+  - epic@24.07.0
+  - epic@24.08.0
  - fastjet
  - fjcontrib
  - fmt

--- a/spack-environment/packages.yaml
+++ b/spack-environment/packages.yaml
@@ -14,8 +14,8 @@ packages:
    require:
    - '%gcc'
    - any_of: [+ipo, '@:']
+    - any_of: [build_system=cmake, '@:']
    - any_of: [build_type=Release, '@:']
-    - any_of: [^py-pip@23.1.2, '@:']
  acts:
    require:
    - '@31.2.0'
@@ -87,7 +87,7 @@ packages:
    - '@656aa3192b097a631ddd1e0380e80c26fd6644a7'
  edm4eic:
    require:
-    - '@6.0.1' # EDM4EIC_VERSION
+    - '@7.0.0' # EDM4EIC_VERSION
    - cxxstd=20
  edm4hep:
    require:
@@ -103,7 +103,7 @@ packages:
    - '@1.1.12'
  eicrecon:
    require:
-    - '@1.14.1' # EICRECON_VERSION
+    - '@1.16.1' # EICRECON_VERSION
  eigen:
    require:
    - '@3.4.0'
@@ -127,6 +127,9 @@ packages:
    require:
    - '@9.1.0'
    - +shared cxxstd=17
+  freetype:
+    require:
+    - build_system=autotools
  gaudi:
    require:
    - '@38.1'
@@ -141,12 +144,15 @@ packages:
      prefix: /opt/software/externals/g4ndl
  geant4:
    require:
-    - '@11.2.1.east'
+    - '@11.2.2.east'
    - cxxstd=20 -vecgeom +threads -vtk
    - any_of: [+opengl +qt +x11, -opengl -qt -x11]
-  glew:
+  gettext: 
+    require:
+    - +libxml2
+  gl:
    require:
-    - gl=glx
+    - glx
  gloo:
    require:
    - '@2023-05-19'
@@ -156,7 +162,7 @@ packages:
    - +expat +pangocairo
  hepmc3:
    require:
-    - '@3.2.7'
+    - '@3.3.0'
    - +python +rootio
  heppdt:
    require:
@@ -175,11 +181,11 @@ packages:
    - '@0.20'
  jana2:
    require:
-    - '@2.2.1-rc1'
+    - '@2.3.1'
    - -ipo +podio +root +zmq
  juggler:
    require:
-    - '@14.0.3' # JUGGLER_VERSION
+    - '@14.2.0' # JUGGLER_VERSION
    - cxxstd=20
  k4actstracking:
    require:
@@ -251,6 +257,9 @@ packages:
    require:
    - '@3.0.2'
    - +plot
+  protobuf:
+    require:
+    - '@3.21.12'
  pythia8:
    require:
    - '@8.311'
@@ -273,6 +282,15 @@ packages:
  py-epic-capybara:
    require:
    - '@git.c388cecce90d06f2db75e45bb0427816db04c836'
+  py-hepunits:
+    require:
+    - '@2.3.2'
+  py-hist:
+    require:
+    - '@2.6.1'
+  py-histoprint:
+    require:
+    - '@2.4.0'
  py-ipython:
    require:
    - '@8.11.0'
@@ -315,12 +333,18 @@ packages:
  py-pandas:
    require:
    - '@1.5.3'
+  py-particle:
+    require:
+    - '@0.23.0'
  py-pip:
    require:
    - any_of: ['@23.1.2', '@23.0']
  py-pre-commit:
    require:
    - '@3.3.3'
+  py-protobuf:
+    require:
+    - '@4.21.9'
  py-pygithub:
    require:
    - '@2.1.1'
@@ -346,21 +370,34 @@ packages:
  py-uproot:
    require:
    - '@5.0.5'
+  py-vector:
+    require:
+    - '@1.4.1'
  py-wurlitzer:
    require:
    - '@3.0.2'
  py-yapf:
    require:
    - '@0.30.0'
+  qmake:
+    require:
+    - qt-base
  qt:
    require:
-    - '@5.15.11'
+    - '@5.15.12'
+    - +opengl
+  qt-base:
+    require:
+    - '@6.7.2'
    - +opengl
  root:
    require:
    - '@6.30.02'
    - cxxstd=20 +fftw +fortran +gdml +http -ipo +mlp +python +root7 +tmva +tmva-sofie +vc +xrootd +ssl
    - any_of: [+opengl +webgui +x, -opengl -webgui -x]
+  simsipm:
+    require:
+    - '@2.0.2'
  snakemake:
    require:
    - '@7.22.0'
@@ -380,7 +417,7 @@ packages:
    - '@3.20.0'
  xrootd:
    require:
-    - '@5.6.9'
+    - '@5.7.0'
    - cxxstd=20 -davix +python +scitokens-cpp
  zlib-api:
    require:

--- a/spack-environment/view.yaml
+++ b/spack-environment/view.yaml
  view:
    default:
      root: /opt/local
-      exclude: [epic]
+      exclude: 
+      - epic
+      - py-pip@23.0
      link_type: symlink
    detectors:
      root: /opt/detector
-      select: [epic]
+      select:
+      - epic
      projections:
        all: '{name}-{version}'
      link: roots

--- a/spack-environment/xl/spack.yaml
+++ b/spack-environment/xl/spack.yaml
@@ -28,23 +28,18 @@ spack:
  - eigen
  - emacs
  - epic@main # EPIC_VERSION
-  - epic@23.10.0
-  - epic@23.11.0
-  - epic@23.12.0
-  - epic@24.02.0
-  - epic@24.02.1
-  - epic@24.03.0
-  - epic@24.03.1
-  - epic@24.04.0
  - epic@24.05.0
  - epic@24.05.2
  - epic@24.06.0
+  - epic@24.07.0
+  - epic@24.08.0
+  - epic@24.08.1
  - fastjet
  - fjcontrib
  - fmt
  - gaudi
  - gdb
-  - geant4
+  - geant4 +opengl
  - graphviz
  - hepmc3
  - heppdt
@@ -74,6 +69,9 @@ spack:
  - py-dask
  - py-deepdiff
  - py-epic-capybara
+  - py-hepunits
+  - py-hist
+  - py-histoprint
  - py-ipython
  - py-jinja2
  - py-jinja2-cli
@@ -88,6 +86,7 @@ spack:
  - py-onnx
  - py-onnxruntime
  - py-pandas
+  - py-particle
  - py-pip
  - py-pre-commit
  - py-pycairo
@@ -97,9 +96,11 @@ spack:
  - py-toml
  - py-torch
  - py-uproot
+  - py-vector
  - py-wurlitzer
  - py-yapf
  - root +opengl +webgui +x
+  - simsipm
  - snakemake
  - spdlog
  - stow

--- a/spack.sh
+++ b/spack.sh
@@ -3,7 +3,7 @@ SPACK_ORGREPO="spack/spack"

 ## Spack github version, e.g. v0.18.1 or commit hash
 ## note: nightly builds will use e.g. releases/v0.19
-SPACK_VERSION="v0.22.0"
+SPACK_VERSION="v0.22.1"

 ## Space-separated list of spack cherry-picks
 read -r -d '' SPACK_CHERRYPICKS <<- \
@@ -11,11 +11,18 @@ read -r -d '' SPACK_CHERRYPICKS <<- \
 09f75ee426a2e05e0543570821582480ff823ba5
 f6d50f790ee8b123f7775429f6ca6394170e6de9
 63f6e6079aacc99078386e5c8ff06173841b9595
-9bcc43c4c158639fa6cb575c6106595a34682081
 9f3e45ddbee24aaa7993e575297827e0aed2e6fe
 85f13442d2a7486daba81fdd9a3b6a1182ba11f6
 f73d7d2dce226857cbc774e942454bad2992969e
 cbab451c1a342523ed75e9be1098615a597a9b59
+4fe5f35c2fff6288e4c05e5946798ad2191a2c40
+4c60deb9921ff2cbbfa6771f4f63ff812a8a5840
+b894acf1fca8eb5cc52d2267b0c4c187065998c0
+8b45fa089e24c6ab7de2eaf614977369e69daa54
+2d8ca8af6932dfd50204d1e4f6fe587dec7beef5
+d3bf1e04fca844abb7c1eeac38dda4e126c81b67
+81125c3bd80e71a2e57c7fcff8e02c4b3add5b90
+8b2fec61f23a7b8230c0ed0378d90d04d8f590d8
 ---
 ## Optional hash table with comma-separated file list
 read -r -d '' SPACK_CHERRYPICKS_FILES <<- \
@@ -26,8 +33,15 @@ read -r -d '' SPACK_CHERRYPICKS_FILES <<- \
 ## 09f75ee426a2e05e0543570821582480ff823ba5: setup-env.sh: if exe contains qemu, use /proc/$$/comm instead
 ## f6d50f790ee8b123f7775429f6ca6394170e6de9: gaudi: Add version 38.1
 ## 63f6e6079aacc99078386e5c8ff06173841b9595: gaudi: upstream patch when @38.1 for missing #include <list>
-## 9bcc43c4c158639fa6cb575c6106595a34682081: protobuf: update hash for patch needed when="@3.4:3.21"
 ## 9f3e45ddbee24aaa7993e575297827e0aed2e6fe: acts: pass cuda_arch to CMAKE_CUDA_ARCHITECTURES
 ## 85f13442d2a7486daba81fdd9a3b6a1182ba11f6: Consolidate concretization output for environments
 ## f73d7d2dce226857cbc774e942454bad2992969e: dd4hep: cleanup recipe, remove deprecated versions and patches
 ## cbab451c1a342523ed75e9be1098615a597a9b59: dd4hep: Add version 1.29
+## 4fe5f35c2fff6288e4c05e5946798ad2191a2c40: xrootd: add v5.7.0
+## 4c60deb9921ff2cbbfa6771f4f63ff812a8a5840: xrootd: add github as secondary url to avoid SSL issues
+## b894acf1fca8eb5cc52d2267b0c4c187065998c0: geant4: add v11.2.2, incl g4ndl v4.7.1
+## 8b45fa089e24c6ab7de2eaf614977369e69daa54: geant4: support Qt5 and Qt6
+## 2d8ca8af6932dfd50204d1e4f6fe587dec7beef5: qt-*: add v6.7.1, v6.7.2
+## d3bf1e04fca844abb7c1eeac38dda4e126c81b67: py-vector: add through v1.4.1 (switch to hatchling)
+## 81125c3bd80e71a2e57c7fcff8e02c4b3add5b90: hepmc3: pass root variant cxxstd as HEPMC3_CXX_STANDARD
+## 8b2fec61f23a7b8230c0ed0378d90d04d8f590d8: hepmc3: add v3.3.0
No results found