diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 12664433619e3bd93b36076c82ac520cb2609f57..91caecd1df5e4c88becef1d98fafdbc1afe1338d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -248,8 +248,11 @@ base:
# - BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04
# BUILD_IMAGE: oneapi_base
# PLATFORM: linux/amd64
- - BASE_IMAGE: nvidia/cuda:12.4.1-devel-ubuntu22.04
- BUILD_IMAGE: cuda_base
+ - BASE_IMAGE: nvidia/cuda:12.3.2-devel-ubuntu22.04
+ BUILD_IMAGE: cuda_devel
+ PLATFORM: linux/amd64
+ - BASE_IMAGE: nvidia/cuda:12.3.2-runtime-ubuntu22.04
+ BUILD_IMAGE: cuda_runtime
PLATFORM: linux/amd64
extends: .build
stage: base
@@ -305,13 +308,22 @@ eic:
# - prod
- BUILD_TYPE:
- default
- BUILDER_IMAGE: cuda_base
- RUNTIME_IMAGE: cuda_base
+ BUILDER_IMAGE: cuda_devel
+ RUNTIME_IMAGE: cuda_runtime
BASE_IMAGE: cuda_base
- BUILD_IMAGE: cuda_
+ BUILD_IMAGE: eic_
PLATFORM: linux/amd64
ENV:
- - xl
+ - cuda
+ - BUILD_TYPE:
+ - default
+ BUILDER_IMAGE: cuda_devel
+ RUNTIME_IMAGE: cuda_devel
+ BASE_IMAGE: cuda_base
+ BUILD_IMAGE: eic_dev_
+ PLATFORM: linux/amd64
+ ENV:
+ - cuda
extends: .build
stage: eic
needs:
diff --git a/containers/jug/dev.Dockerfile b/containers/jug/dev.Dockerfile
index ee44670d24c9a588d9bd1141241b1a03eeabc938..45d68e3f50e23e6f0ec110b030243d20a9ab9596 100644
--- a/containers/jug/dev.Dockerfile
+++ b/containers/jug/dev.Dockerfile
@@ -72,6 +72,7 @@ declare -A target=(["linux/amd64"]="x86_64_v2" ["linux/arm64"]="aarch64")
target=${target[${TARGETPLATFORM}]}
spack config --scope site add "packages:all:require:[target=${target}]"
spack config --scope site add "packages:all:target:[${target}]"
+spack external find --not-buildable --scope site --path /usr/local/cuda/bin cuda
spack external find --not-buildable --scope site llvm
spack config blame packages
spack config --scope user add "config:suppress_gpg_warnings:true"
@@ -217,13 +218,13 @@ ccache --show-stats
ccache --zero-stats
EOF
-## Create views at /usr/local and /opt/detector
+## Create views at /opt/local and /opt/detector
RUN <<EOF
set -e
-rm -r /usr/local
+rm -rf /opt/local
sed -i -e '/view: false/d' ${SPACK_ENV}/spack.yaml
cat /opt/spack-environment/view.yaml >> ${SPACK_ENV}/spack.yaml
-spack -e ${SPACK_ENV} env view regenerate /usr/local
+spack -e ${SPACK_ENV} env view regenerate /opt/local
spack -e ${SPACK_ENV} env view regenerate /opt/detector
EOF
@@ -232,7 +233,7 @@ RUN <<EOF
touch ${SPACK_ROOT}/.cvmfscatalog
touch /opt/software/.cvmfscatalog
find /opt/software -mindepth 2 -maxdepth 3 -type d -exec touch {}/.cvmfscatalog \;
-touch /usr/local/.cvmfscatalog
+touch /opt/local/.cvmfscatalog
EOF
## Store environment
@@ -265,8 +266,8 @@ RUN spack -e ${SPACK_ENV} gc -y
## and links therin for more info
RUN <<EOF
set -ex
-if [ -f /usr/local/lib/libQt5Core.so ] ; then
- strip --remove-section=.note.ABI-tag /usr/local/lib/libQt5Core.so
+if [ -f /opt/local/lib/libQt5Core.so ] ; then
+ strip --remove-section=.note.ABI-tag /opt/local/lib/libQt5Core.so
fi
EOF
@@ -278,9 +279,9 @@ spack graph --dot > /opt/spack-environment/env.dot
EOF
## Copy custom content
-COPY eic-shell /usr/local/bin/eic-shell
-COPY eic-info /usr/local/bin/eic-info
-COPY entrypoint.sh /usr/local/sbin/entrypoint.sh
+COPY eic-shell /opt/local/bin/eic-shell
+COPY eic-info /opt/local/bin/eic-info
+COPY entrypoint.sh /opt/local/sbin/entrypoint.sh
COPY eic-env.sh /etc/eic-env.sh
COPY profile.d/a00_cleanup.sh /etc/profile.d
COPY profile.d/z11_jug_env.sh /etc/profile.d
@@ -294,16 +295,16 @@ for detector in /opt/detector/epic-git.*_* ; do
done
EOF
-## Add minio client into /usr/local/bin
-ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-amd64/mc /usr/local/bin/mc-amd64
-ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-arm64/mc /usr/local/bin/mc-arm64
+## Add minio client into /opt/local/bin
+ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-amd64/mc /opt/local/bin/mc-amd64
+ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-arm64/mc /opt/local/bin/mc-arm64
RUN <<EOF
set -ex
declare -A target=(["linux/amd64"]="amd64" ["linux/arm64"]="arm64")
-mv /usr/local/bin/mc-${target[${TARGETPLATFORM}]} /usr/local/bin/mc
+mv /opt/local/bin/mc-${target[${TARGETPLATFORM}]} /opt/local/bin/mc
unset target[${TARGETPLATFORM}]
for t in ${target[*]} ; do
- rm /usr/local/bin/mc-${t}
+ rm /opt/local/bin/mc-${t}
done
EOF
@@ -323,7 +324,7 @@ LABEL maintainer="Sylvester Joosten <sjoosten@anl.gov>" \
COPY --from=staging /opt/spack /opt/spack
COPY --from=staging /opt/spack-environment /opt/spack-environment
COPY --from=staging /opt/software /opt/software
-COPY --from=staging /usr/._local /usr/._local
+COPY --from=staging /opt/._local /opt/._local
COPY --from=staging /opt/._detector /opt/._detector
COPY --from=staging /etc/profile.d /etc/profile.d
COPY --from=staging /etc/jug_info /etc/jug_info
@@ -335,25 +336,25 @@ COPY --from=staging /usr/bin/docker-shell /usr/bin/docker-shell
ENV SPACK_ROOT=/opt/spack
SHELL ["docker-shell"]
-## ensure /usr/local is the view, not a symlink
+## ensure /opt/local is the view, not a symlink
RUN <<EOF
set -ex
-rm -rf /usr/local /opt/detector
-LOCAL_PREFIX_PATH=$(realpath $(ls /usr/._local/ | tail -n1))
-mv /usr/._local/${LOCAL_PREFIX_PATH} /usr/local
-ln -s /usr/local /usr/._local/${LOCAL_PREFIX_PATH}
+rm -rf /opt/local /opt/detector
+LOCAL_PREFIX_PATH=$(realpath $(ls /opt/._local/ | tail -n1))
+mv /opt/._local/${LOCAL_PREFIX_PATH} /opt/local
+ln -s /opt/local /opt/._local/${LOCAL_PREFIX_PATH}
DETECTOR_PREFIX_PATH=$(realpath $(ls /opt/._detector/ | tail -n1))
mv /opt/._detector/${DETECTOR_PREFIX_PATH} /opt/detector
ln -s /opt/detector /opt/._detector/${DETECTOR_PREFIX_PATH}
EOF
## set ROOT TFile forward compatibility
-RUN sed --in-place --follow-symlinks 's/# \(TFile.v630forwardCompatibility:\) no/\1 yes/' /usr/local/etc/root/system.rootrc
+RUN sed --in-place --follow-symlinks 's/# \(TFile.v630forwardCompatibility:\) no/\1 yes/' /opt/local/etc/root/system.rootrc
## Setup ld.so.conf with what could go in LD_LIBRARY_PATH (but lower priority)
## Ref: https://man7.org/linux/man-pages/man8/ld.so.8.html
COPY <<EOF /etc/ld.so.conf.d/eic-shell.conf
-/usr/local/lib/root
+/opt/local/lib/root
EOF
RUN ldconfig
@@ -422,15 +423,15 @@ git clone --filter=tree:0 -b ${CAMPAIGNS_SLURM_VERSION} --depth 1 https://github
EOF
## make sure we have the entrypoints setup correctly
-ENTRYPOINT ["/usr/local/sbin/entrypoint.sh"]
+ENTRYPOINT ["/opt/local/sbin/entrypoint.sh"]
CMD ["bash", "--rcfile", "/etc/profile", "-l"]
USER 0
WORKDIR /
-SHELL ["/usr/local/bin/eic-shell"]
+SHELL ["/opt/local/bin/eic-shell"]
## eic-news
-COPY --chmod=0755 eic-news /usr/local/bin/eic-news
-RUN echo "test -f $HOME/.eic-news && source /usr/local/bin/eic-news" > /etc/profile.d/z13_eic-news.sh
+COPY --chmod=0755 eic-news /opt/local/bin/eic-news
+RUN echo "test -f $HOME/.eic-news && source /opt/local/bin/eic-news" > /etc/profile.d/z13_eic-news.sh
## Hotfix for misbehaving OSG nodes
RUN mkdir /hadoop /localscratch
diff --git a/spack-environment/cuda/spack.yaml b/spack-environment/cuda/spack.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3ff983317a94c561b4957410917beec8deef89af
--- /dev/null
+++ b/spack-environment/cuda/spack.yaml
@@ -0,0 +1,107 @@
+spack:
+ include:
+ - ../concretizer.yaml
+ - ../packages.yaml
+ config:
+ install_missing_compilers: false
+ specs:
+ - acts +cuda cuda_arch=75
+ - actsvg
+ - afterburner
+ - algorithms
+ - cairo
+ - catch2
+ - cernlib
+ - cli11
+ - cmake
+ - cnpy
+ - cppcoro
+ - dawn
+ - dawncut
+ - dd4hep +ddeve
+ - dpmjet
+ - east
+ - edm4eic
+ - edm4hep
+ - eic-smear
+ - eicrecon
+ - eigen
+ - emacs
+ - epic@main # EPIC_VERSION
+ - epic@23.10.0
+ - epic@23.11.0
+ - epic@23.12.0
+ - epic@24.02.0
+ - epic@24.02.1
+ - epic@24.03.0
+ - epic@24.03.1
+ - epic@24.04.0
+ - epic@24.05.0
+ - epic@24.05.2
+ - fastjet
+ - fjcontrib
+ - fmt
+ - gaudi
+ - gdb
+ - geant4
+ - graphviz
+ - hepmc3
+ - heppdt
+ - imagemagick
+ - irt
+ - iwyu
+ - jana2
+ - juggler
+ - k4actstracking
+ - k4fwcore
+ - lcov
+ - lhapdf
+ - madx
+ - nano
+ - nopayloadclient
+ - npsim
+ - onnx
+ - opencascade
+ - osg-ca-certs
+ - phonebook-cli
+ - podio
+ - prmon
+ - pythia8
+ - py-awkward
+ - py-bokeh
+ - py-boto3
+ - py-dask
+ - py-deepdiff
+ - py-epic-capybara
+ - py-ipython
+ - py-jinja2
+ - py-jinja2-cli
+ - py-jupyter-console
+ - py-jupyterlab
+ - py-lmfit
+ - py-lxml
+ - py-matplotlib
+ - py-mplhep
+ - py-htgettoken
+ - py-numpy
+ - py-onnx
+ - py-onnxruntime
+ - py-pandas
+ - py-pip
+ - py-pre-commit
+ - py-pycairo
+ - py-pyyaml
+ - py-scipy
+ - py-seaborn
+ - py-toml
+ - py-torch +cuda cuda_arch=75
+ - py-uproot
+ - py-wurlitzer
+ - py-yapf
+ - root +opengl +webgui +x
+ - snakemake
+ - spdlog
+ - stow
+ - valgrind
+ - xrootd
+ - xeyes
diff --git a/spack-environment/view.yaml b/spack-environment/view.yaml
index fb5d3ea5ff0d23d7e5574974ac53878fd67b1ecc..889bfaea85873416af15d4c13e0a31b485e2d4a7 100644
--- a/spack-environment/view.yaml
+++ b/spack-environment/view.yaml
@@ -1,6 +1,6 @@
view:
default:
- root: /usr/local
+ root: /opt/local
exclude: [epic]
link_type: symlink
detectors:
diff --git a/spack.sh b/spack.sh
index db79671c94197f31f9017597ac8a64b09a9538bb..ca1a57906286c46769c958a95224e841db6527c3 100644
--- a/spack.sh
+++ b/spack.sh
@@ -12,6 +12,7 @@ read -r -d '' SPACK_CHERRYPICKS <<- \
f6d50f790ee8b123f7775429f6ca6394170e6de9
63f6e6079aacc99078386e5c8ff06173841b9595
9bcc43c4c158639fa6cb575c6106595a34682081
+9f3e45ddbee24aaa7993e575297827e0aed2e6fe
---
## Optional hash table with comma-separated file list
read -r -d '' SPACK_CHERRYPICKS_FILES <<- \
@@ -23,3 +24,4 @@ read -r -d '' SPACK_CHERRYPICKS_FILES <<- \
## f6d50f790ee8b123f7775429f6ca6394170e6de9: gaudi: Add version 38.1
## 63f6e6079aacc99078386e5c8ff06173841b9595: gaudi: upstream patch when @38.1 for missing #include <list>
## 9bcc43c4c158639fa6cb575c6106595a34682081: protobuf: update hash for patch needed when="@3.4:3.21"
+## 9f3e45ddbee24aaa7993e575297827e0aed2e6fe: acts: pass cuda_arch to CMAKE_CUDA_ARCHITECTURES