Skip to content
Snippets Groups Projects
Commit 58b0546f authored by Wouter Deconinck's avatar Wouter Deconinck
Browse files

feat: cp xl to cuda env, find external cuda

parent 3b11e079
No related branches found
No related tags found
1 merge request!930feat: cp xl to cuda env, find external cuda
Pipeline #94268 passed with warnings
Stage: status-pending
Stage: config
Stage: base
Stage: eic
Stage: deploy
Stage: benchmarks
Stage: finalize
Stage: status-report
Pipeline: physics_benchmarks

#94273

    Pipeline: physics_benchmarks

    #94272

      Pipeline: detector_benchmarks

      #94271

        +1
        Hide whitespace changes
        Inline Side-by-side
        .gitlab-ci.yml
        + 18
        − 6
        View file @ 58b0546f
        ...@@ -248,8 +248,11 @@ base: ...@@ -248,8 +248,11 @@ base:
        # - BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04 # - BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04
        # BUILD_IMAGE: oneapi_base # BUILD_IMAGE: oneapi_base
        # PLATFORM: linux/amd64 # PLATFORM: linux/amd64
        - BASE_IMAGE: nvidia/cuda:12.4.1-devel-ubuntu22.04 - BASE_IMAGE: nvidia/cuda:12.3.2-devel-ubuntu22.04
        BUILD_IMAGE: cuda_base BUILD_IMAGE: cuda_devel
        PLATFORM: linux/amd64
        - BASE_IMAGE: nvidia/cuda:12.3.2-runtime-ubuntu22.04
        BUILD_IMAGE: cuda_runtime
        PLATFORM: linux/amd64 PLATFORM: linux/amd64
        extends: .build extends: .build
        stage: base stage: base
        ...@@ -305,13 +308,22 @@ eic: ...@@ -305,13 +308,22 @@ eic:
        # - prod # - prod
        - BUILD_TYPE: - BUILD_TYPE:
        - default - default
        BUILDER_IMAGE: cuda_base BUILDER_IMAGE: cuda_devel
        RUNTIME_IMAGE: cuda_base RUNTIME_IMAGE: cuda_runtime
        BASE_IMAGE: cuda_base BASE_IMAGE: cuda_base
        BUILD_IMAGE: cuda_ BUILD_IMAGE: eic_
        PLATFORM: linux/amd64 PLATFORM: linux/amd64
        ENV: ENV:
        - xl - cuda
        - BUILD_TYPE:
        - default
        BUILDER_IMAGE: cuda_devel
        RUNTIME_IMAGE: cuda_devel
        BASE_IMAGE: cuda_base
        BUILD_IMAGE: eic_dev_
        PLATFORM: linux/amd64
        ENV:
        - cuda
        extends: .build extends: .build
        stage: eic stage: eic
        needs: needs:
        ......
        containers/jug/dev.Dockerfile
        + 27
        − 26
        View file @ 58b0546f
        ...@@ -72,6 +72,7 @@ declare -A target=(["linux/amd64"]="x86_64_v2" ["linux/arm64"]="aarch64") ...@@ -72,6 +72,7 @@ declare -A target=(["linux/amd64"]="x86_64_v2" ["linux/arm64"]="aarch64")
        target=${target[${TARGETPLATFORM}]} target=${target[${TARGETPLATFORM}]}
        spack config --scope site add "packages:all:require:[target=${target}]" spack config --scope site add "packages:all:require:[target=${target}]"
        spack config --scope site add "packages:all:target:[${target}]" spack config --scope site add "packages:all:target:[${target}]"
        spack external find --not-buildable --scope site --path /usr/local/cuda/bin cuda
        spack external find --not-buildable --scope site llvm spack external find --not-buildable --scope site llvm
        spack config blame packages spack config blame packages
        spack config --scope user add "config:suppress_gpg_warnings:true" spack config --scope user add "config:suppress_gpg_warnings:true"
        ...@@ -217,13 +218,13 @@ ccache --show-stats ...@@ -217,13 +218,13 @@ ccache --show-stats
        ccache --zero-stats ccache --zero-stats
        EOF EOF
        ## Create views at /usr/local and /opt/detector ## Create views at /opt/local and /opt/detector
        RUN <<EOF RUN <<EOF
        set -e set -e
        rm -r /usr/local rm -rf /opt/local
        sed -i -e '/view: false/d' ${SPACK_ENV}/spack.yaml sed -i -e '/view: false/d' ${SPACK_ENV}/spack.yaml
        cat /opt/spack-environment/view.yaml >> ${SPACK_ENV}/spack.yaml cat /opt/spack-environment/view.yaml >> ${SPACK_ENV}/spack.yaml
        spack -e ${SPACK_ENV} env view regenerate /usr/local spack -e ${SPACK_ENV} env view regenerate /opt/local
        spack -e ${SPACK_ENV} env view regenerate /opt/detector spack -e ${SPACK_ENV} env view regenerate /opt/detector
        EOF EOF
        ...@@ -232,7 +233,7 @@ RUN <<EOF ...@@ -232,7 +233,7 @@ RUN <<EOF
        touch ${SPACK_ROOT}/.cvmfscatalog touch ${SPACK_ROOT}/.cvmfscatalog
        touch /opt/software/.cvmfscatalog touch /opt/software/.cvmfscatalog
        find /opt/software -mindepth 2 -maxdepth 3 -type d -exec touch {}/.cvmfscatalog \; find /opt/software -mindepth 2 -maxdepth 3 -type d -exec touch {}/.cvmfscatalog \;
        touch /usr/local/.cvmfscatalog touch /opt/local/.cvmfscatalog
        EOF EOF
        ## Store environment ## Store environment
        ...@@ -265,8 +266,8 @@ RUN spack -e ${SPACK_ENV} gc -y ...@@ -265,8 +266,8 @@ RUN spack -e ${SPACK_ENV} gc -y
        ## and links therin for more info ## and links therin for more info
        RUN <<EOF RUN <<EOF
        set -ex set -ex
        if [ -f /usr/local/lib/libQt5Core.so ] ; then if [ -f /opt/local/lib/libQt5Core.so ] ; then
        strip --remove-section=.note.ABI-tag /usr/local/lib/libQt5Core.so strip --remove-section=.note.ABI-tag /opt/local/lib/libQt5Core.so
        fi fi
        EOF EOF
        ...@@ -278,9 +279,9 @@ spack graph --dot > /opt/spack-environment/env.dot ...@@ -278,9 +279,9 @@ spack graph --dot > /opt/spack-environment/env.dot
        EOF EOF
        ## Copy custom content ## Copy custom content
        COPY eic-shell /usr/local/bin/eic-shell COPY eic-shell /opt/local/bin/eic-shell
        COPY eic-info /usr/local/bin/eic-info COPY eic-info /opt/local/bin/eic-info
        COPY entrypoint.sh /usr/local/sbin/entrypoint.sh COPY entrypoint.sh /opt/local/sbin/entrypoint.sh
        COPY eic-env.sh /etc/eic-env.sh COPY eic-env.sh /etc/eic-env.sh
        COPY profile.d/a00_cleanup.sh /etc/profile.d COPY profile.d/a00_cleanup.sh /etc/profile.d
        COPY profile.d/z11_jug_env.sh /etc/profile.d COPY profile.d/z11_jug_env.sh /etc/profile.d
        ...@@ -294,16 +295,16 @@ for detector in /opt/detector/epic-git.*_* ; do ...@@ -294,16 +295,16 @@ for detector in /opt/detector/epic-git.*_* ; do
        done done
        EOF EOF
        ## Add minio client into /usr/local/bin ## Add minio client into /opt/local/bin
        ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-amd64/mc /usr/local/bin/mc-amd64 ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-amd64/mc /opt/local/bin/mc-amd64
        ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-arm64/mc /usr/local/bin/mc-arm64 ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-arm64/mc /opt/local/bin/mc-arm64
        RUN <<EOF RUN <<EOF
        set -ex set -ex
        declare -A target=(["linux/amd64"]="amd64" ["linux/arm64"]="arm64") declare -A target=(["linux/amd64"]="amd64" ["linux/arm64"]="arm64")
        mv /usr/local/bin/mc-${target[${TARGETPLATFORM}]} /usr/local/bin/mc mv /opt/local/bin/mc-${target[${TARGETPLATFORM}]} /opt/local/bin/mc
        unset target[${TARGETPLATFORM}] unset target[${TARGETPLATFORM}]
        for t in ${target[*]} ; do for t in ${target[*]} ; do
        rm /usr/local/bin/mc-${t} rm /opt/local/bin/mc-${t}
        done done
        EOF EOF
        ...@@ -323,7 +324,7 @@ LABEL maintainer="Sylvester Joosten <sjoosten@anl.gov>" \ ...@@ -323,7 +324,7 @@ LABEL maintainer="Sylvester Joosten <sjoosten@anl.gov>" \
        COPY --from=staging /opt/spack /opt/spack COPY --from=staging /opt/spack /opt/spack
        COPY --from=staging /opt/spack-environment /opt/spack-environment COPY --from=staging /opt/spack-environment /opt/spack-environment
        COPY --from=staging /opt/software /opt/software COPY --from=staging /opt/software /opt/software
        COPY --from=staging /usr/._local /usr/._local COPY --from=staging /opt/._local /opt/._local
        COPY --from=staging /opt/._detector /opt/._detector COPY --from=staging /opt/._detector /opt/._detector
        COPY --from=staging /etc/profile.d /etc/profile.d COPY --from=staging /etc/profile.d /etc/profile.d
        COPY --from=staging /etc/jug_info /etc/jug_info COPY --from=staging /etc/jug_info /etc/jug_info
        ...@@ -335,25 +336,25 @@ COPY --from=staging /usr/bin/docker-shell /usr/bin/docker-shell ...@@ -335,25 +336,25 @@ COPY --from=staging /usr/bin/docker-shell /usr/bin/docker-shell
        ENV SPACK_ROOT=/opt/spack ENV SPACK_ROOT=/opt/spack
        SHELL ["docker-shell"] SHELL ["docker-shell"]
        ## ensure /usr/local is the view, not a symlink ## ensure /opt/local is the view, not a symlink
        RUN <<EOF RUN <<EOF
        set -ex set -ex
        rm -rf /usr/local /opt/detector rm -rf /opt/local /opt/detector
        LOCAL_PREFIX_PATH=$(realpath $(ls /usr/._local/ | tail -n1)) LOCAL_PREFIX_PATH=$(realpath $(ls /opt/._local/ | tail -n1))
        mv /usr/._local/${LOCAL_PREFIX_PATH} /usr/local mv /opt/._local/${LOCAL_PREFIX_PATH} /opt/local
        ln -s /usr/local /usr/._local/${LOCAL_PREFIX_PATH} ln -s /opt/local /opt/._local/${LOCAL_PREFIX_PATH}
        DETECTOR_PREFIX_PATH=$(realpath $(ls /opt/._detector/ | tail -n1)) DETECTOR_PREFIX_PATH=$(realpath $(ls /opt/._detector/ | tail -n1))
        mv /opt/._detector/${DETECTOR_PREFIX_PATH} /opt/detector mv /opt/._detector/${DETECTOR_PREFIX_PATH} /opt/detector
        ln -s /opt/detector /opt/._detector/${DETECTOR_PREFIX_PATH} ln -s /opt/detector /opt/._detector/${DETECTOR_PREFIX_PATH}
        EOF EOF
        ## set ROOT TFile forward compatibility ## set ROOT TFile forward compatibility
        RUN sed --in-place --follow-symlinks 's/# \(TFile.v630forwardCompatibility:\) no/\1 yes/' /usr/local/etc/root/system.rootrc RUN sed --in-place --follow-symlinks 's/# \(TFile.v630forwardCompatibility:\) no/\1 yes/' /opt/local/etc/root/system.rootrc
        ## Setup ld.so.conf with what could go in LD_LIBRARY_PATH (but lower priority) ## Setup ld.so.conf with what could go in LD_LIBRARY_PATH (but lower priority)
        ## Ref: https://man7.org/linux/man-pages/man8/ld.so.8.html ## Ref: https://man7.org/linux/man-pages/man8/ld.so.8.html
        COPY <<EOF /etc/ld.so.conf.d/eic-shell.conf COPY <<EOF /etc/ld.so.conf.d/eic-shell.conf
        /usr/local/lib/root /opt/local/lib/root
        EOF EOF
        RUN ldconfig RUN ldconfig
        ...@@ -422,15 +423,15 @@ git clone --filter=tree:0 -b ${CAMPAIGNS_SLURM_VERSION} --depth 1 https://github ...@@ -422,15 +423,15 @@ git clone --filter=tree:0 -b ${CAMPAIGNS_SLURM_VERSION} --depth 1 https://github
        EOF EOF
        ## make sure we have the entrypoints setup correctly ## make sure we have the entrypoints setup correctly
        ENTRYPOINT ["/usr/local/sbin/entrypoint.sh"] ENTRYPOINT ["/opt/local/sbin/entrypoint.sh"]
        CMD ["bash", "--rcfile", "/etc/profile", "-l"] CMD ["bash", "--rcfile", "/etc/profile", "-l"]
        USER 0 USER 0
        WORKDIR / WORKDIR /
        SHELL ["/usr/local/bin/eic-shell"] SHELL ["/opt/local/bin/eic-shell"]
        ## eic-news ## eic-news
        COPY --chmod=0755 eic-news /usr/local/bin/eic-news COPY --chmod=0755 eic-news /opt/local/bin/eic-news
        RUN echo "test -f $HOME/.eic-news && source /usr/local/bin/eic-news" > /etc/profile.d/z13_eic-news.sh RUN echo "test -f $HOME/.eic-news && source /opt/local/bin/eic-news" > /etc/profile.d/z13_eic-news.sh
        ## Hotfix for misbehaving OSG nodes ## Hotfix for misbehaving OSG nodes
        RUN mkdir /hadoop /localscratch RUN mkdir /hadoop /localscratch
        spack-environment/cuda/spack.yaml 0 → 100644
        + 107
        − 0
        View file @ 58b0546f
        spack:
        include:
        - ../concretizer.yaml
        - ../packages.yaml
        config:
        install_missing_compilers: false
        specs:
        - acts +cuda cuda_arch=75
        - actsvg
        - afterburner
        - algorithms
        - cairo
        - catch2
        - cernlib
        - cli11
        - cmake
        - cnpy
        - cppcoro
        - dawn
        - dawncut
        - dd4hep +ddeve
        - dpmjet
        - east
        - edm4eic
        - edm4hep
        - eic-smear
        - eicrecon
        - eigen
        - emacs
        - epic@main # EPIC_VERSION
        - epic@23.10.0
        - epic@23.11.0
        - epic@23.12.0
        - epic@24.02.0
        - epic@24.02.1
        - epic@24.03.0
        - epic@24.03.1
        - epic@24.04.0
        - epic@24.05.0
        - epic@24.05.2
        - fastjet
        - fjcontrib
        - fmt
        - gaudi
        - gdb
        - geant4
        - graphviz
        - hepmc3
        - heppdt
        - imagemagick
        - irt
        - iwyu
        - jana2
        - juggler
        - k4actstracking
        - k4fwcore
        - lcov
        - lhapdf
        - madx
        - nano
        - nopayloadclient
        - npsim
        - onnx
        - opencascade
        - osg-ca-certs
        - phonebook-cli
        - podio
        - prmon
        - pythia8
        - py-awkward
        - py-bokeh
        - py-boto3
        - py-dask
        - py-deepdiff
        - py-epic-capybara
        - py-ipython
        - py-jinja2
        - py-jinja2-cli
        - py-jupyter-console
        - py-jupyterlab
        - py-lmfit
        - py-lxml
        - py-matplotlib
        - py-mplhep
        - py-htgettoken
        - py-numpy
        - py-onnx
        - py-onnxruntime
        - py-pandas
        - py-pip
        - py-pre-commit
        - py-pycairo
        - py-pyyaml
        - py-scipy
        - py-seaborn
        - py-toml
        - py-torch +cuda cuda_arch=75
        - py-uproot
        - py-wurlitzer
        - py-yapf
        - root +opengl +webgui +x
        - snakemake
        - spdlog
        - stow
        - valgrind
        - xrootd
        - xeyes
        spack-environment/view.yaml
        + 1
        − 1
        View file @ 58b0546f
        view: view:
        default: default:
        root: /usr/local root: /opt/local
        exclude: [epic] exclude: [epic]
        link_type: symlink link_type: symlink
        detectors: detectors:
        ......
        spack.sh
        + 2
        − 0
        View file @ 58b0546f
        ...@@ -12,6 +12,7 @@ read -r -d '' SPACK_CHERRYPICKS <<- \ ...@@ -12,6 +12,7 @@ read -r -d '' SPACK_CHERRYPICKS <<- \
        f6d50f790ee8b123f7775429f6ca6394170e6de9 f6d50f790ee8b123f7775429f6ca6394170e6de9
        63f6e6079aacc99078386e5c8ff06173841b9595 63f6e6079aacc99078386e5c8ff06173841b9595
        9bcc43c4c158639fa6cb575c6106595a34682081 9bcc43c4c158639fa6cb575c6106595a34682081
        9f3e45ddbee24aaa7993e575297827e0aed2e6fe
        --- ---
        ## Optional hash table with comma-separated file list ## Optional hash table with comma-separated file list
        read -r -d '' SPACK_CHERRYPICKS_FILES <<- \ read -r -d '' SPACK_CHERRYPICKS_FILES <<- \
        ...@@ -23,3 +24,4 @@ read -r -d '' SPACK_CHERRYPICKS_FILES <<- \ ...@@ -23,3 +24,4 @@ read -r -d '' SPACK_CHERRYPICKS_FILES <<- \
        ## f6d50f790ee8b123f7775429f6ca6394170e6de9: gaudi: Add version 38.1 ## f6d50f790ee8b123f7775429f6ca6394170e6de9: gaudi: Add version 38.1
        ## 63f6e6079aacc99078386e5c8ff06173841b9595: gaudi: upstream patch when @38.1 for missing #include <list> ## 63f6e6079aacc99078386e5c8ff06173841b9595: gaudi: upstream patch when @38.1 for missing #include <list>
        ## 9bcc43c4c158639fa6cb575c6106595a34682081: protobuf: update hash for patch needed when="@3.4:3.21" ## 9bcc43c4c158639fa6cb575c6106595a34682081: protobuf: update hash for patch needed when="@3.4:3.21"
        ## 9f3e45ddbee24aaa7993e575297827e0aed2e6fe: acts: pass cuda_arch to CMAKE_CUDA_ARCHITECTURES
        0% or .
        You are about to add 0 people to the discussion. Proceed with caution.
        Please register or to comment