Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • containers/eic_container
  • wdconinc/eic_container
  • tooba/eic_container
3 results
Show changes
Commits on Source (14)
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 78b71668
......@@ -3,7 +3,7 @@ image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
variables:
## Application versions used for the main release
## note: nightly builds will always use the master/main branch
JUGGLER_VERSION: "v9.4.0"
JUGGLER_VERSION: "v10.0.0"
EICRECON_VERSION: "v1.4.0"
## Local registry
......@@ -382,41 +382,6 @@ jug_xl:nightly:
--build-context detectors=.
containers/jug
jug_xl:feature:
extends: .build
stage: jug
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
when: on_success
- when: never
needs:
- version
- jug_xl:nightly
variables:
BUILD_IMAGE: "jug_xl"
parallel:
matrix:
- DETECTOR: epic
DETECTOR_BRANCH:
- main
script:
- docker buildx build --load ${BUILD_OPTIONS}
--tag ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}-${DETECTOR}-${DETECTOR_BRANCH}
--file containers/jug/feature.Dockerfile
--build-arg INTERNAL_TAG=${NIGHTLY_TAG}
--build-arg DETECTOR=${DETECTOR}
--build-arg DETECTOR_BRANCH=${DETECTOR_BRANCH}
containers/jug
- |
PUSH_FLAG=""
if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
PUSH_FLAG="--eicweb"
fi
FEATURE_EXPORT_TAG="${VERSION}-${DETECTOR}-${DETECTOR_BRANCH}-stable"
- ./gitlab-ci/docker_push.sh -i ${BUILD_IMAGE} -l ${INTERNAL_TAG}-${DETECTOR}-${DETECTOR_BRANCH}
-n ${DOCKER_NTRIES} -t ${DOCKER_WAIT_TIME}
${FEATURE_EXPORT_TAG} ${PUSH_FLAG}
.singularity:
stage: deploy
interruptible: true
......@@ -539,6 +504,7 @@ clean:
- ./gitlab-ci/cleanup_registry.sh -i ubuntu_base -r 80 ${INTERNAL_TAG}
- ./gitlab-ci/cleanup_registry.sh -i debian_testing_base -r 83 ${INTERNAL_TAG}
- ./gitlab-ci/cleanup_registry.sh -i debian_stable_base -r 92 ${INTERNAL_TAG}
- ./gitlab-ci/cleanup_registry.sh -i jug_prod -r 91 ${INTERNAL_TAG}
- ./gitlab-ci/cleanup_registry.sh -i jug_dev -r 68 ${INTERNAL_TAG}
- ./gitlab-ci/cleanup_registry.sh -i jug_dev -r 68 ${INTERNAL_TAG}-nightly
- ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${INTERNAL_TAG}
......
containers/jug/dev.Dockerfile
View file @ 78b71668
......@@ -50,6 +50,7 @@ RUN declare -A target=( \
) \
&& target=${target[${TARGETPLATFORM}]} \
&& spack config --scope site add "packages:all:require:[target=${target}]" \
&& spack config --scope site add "packages:all:target:[${target}]" \
&& spack config blame packages \
&& spack config --scope user add "config:suppress_gpg_warnings:true" \
&& spack config --scope user add "config:build_jobs:${jobs}" \
......@@ -218,22 +219,21 @@ LABEL maintainer="Sylvester Joosten <sjoosten@anl.gov>" \
name="jug_xl" \
march="$TARGETPLATFORM"
## copy over everything we need from staging in a single layer :-)
RUN --mount=from=staging,target=/staging \
rm -rf /usr/local \
&& cp -r /staging/opt/spack /opt/spack \
&& cp -r /staging/opt/spack-environment /opt/spack-environment \
&& cp -r /staging/opt/software /opt/software \
&& cp -r /staging/usr/._local /usr/._local \
&& cd /usr/._local \
&& PREFIX_PATH=$(realpath $(ls | tail -n1)) \
## copy over everything we need from staging
COPY --from=staging /opt/spack /opt/spack
COPY --from=staging /opt/spack-environment /opt/spack-environment
COPY --from=staging /opt/software /opt/software
COPY --from=staging /usr/._local /usr/._local
COPY --from=staging /etc/profile.d /etc/profile.d
COPY --from=staging /etc/jug_info /etc/jug_info
COPY --from=staging /etc/eic-env.sh /etc/eic-env.sh
COPY --from=staging /.singularity.d /.singularity.d
## ensure /usr/local link is pointing to the right view
RUN rm -rf /usr/local \
&& PREFIX_PATH=$(realpath $(ls /usr/._local/ | tail -n1)) \
&& echo "Found spack true prefix path to be $PREFIX_PATH" \
&& cd - \
&& ln -s ${PREFIX_PATH} /usr/local \
&& cp /staging/etc/profile.d/*.sh /etc/profile.d/ \
&& cp /staging/etc/eic-env.sh /etc/eic-env.sh \
&& cp /staging/etc/jug_info /etc/jug_info \
&& cp -r /staging/.singularity.d /.singularity.d
&& ln -s /usr/._local/${PREFIX_PATH} /usr/local
## set the local spack configuration
ENV SPACK_DISABLE_LOCAL_CONFIG="true"
......@@ -241,6 +241,8 @@ RUN . /opt/spack/share/spack/setup-env.sh \
&& spack config --scope site add "config:install_tree:root:~/spack" \
&& spack config --scope site add "config:source_cache:~/.spack/cache" \
&& spack config --scope site add "config:binary_index_root:~/.spack" \
&& spack config --scope site add "config:environments_root:~/.spack/env" \
&& spack config --scope site add "config:suppress_gpg_warnings:true" \
&& spack config blame config \
&& spack config --scope site add "upstreams:eic-shell:install_tree:/opt/software" \
&& spack config blame upstreams
......@@ -250,9 +252,77 @@ ARG JUG_VERSION=1
RUN echo "" >> /etc/jug_info \
&& echo " - jug_dev: ${JUG_VERSION}" >> /etc/jug_info
## eicweb shortcut
ARG EICWEB="https://eicweb.phy.anl.gov/api/v4/projects"
## Install benchmarks into the container
ARG BENCHMARK_COM_VERSION="master"
ARG BENCHMARK_DET_VERSION="master"
ARG BENCHMARK_REC_VERSION="master"
ARG BENCHMARK_PHY_VERSION="master"
## cache bust when updated repositories
ADD ${EICWEB}/458/repository/tree?ref=${BENCHMARK_COM_VERSION} /tmp/485.json
ADD ${EICWEB}/399/repository/tree?ref=${BENCHMARK_DET_VERSION} /tmp/399.json
ADD ${EICWEB}/408/repository/tree?ref=${BENCHMARK_REC_VERSION} /tmp/408.json
ADD ${EICWEB}/400/repository/tree?ref=${BENCHMARK_PHY_VERSION} /tmp/400.json
RUN mkdir -p /opt/benchmarks \
&& cd /opt/benchmarks \
&& git clone -b ${BENCHMARK_COM_VERSION} --depth 1 \
https://eicweb.phy.anl.gov/EIC/benchmarks/common_bench.git \
&& mkdir -p /opt/benchmarks \
&& cd /opt/benchmarks \
&& git clone -b ${BENCHMARK_DET_VERSION} --depth 1 \
https://eicweb.phy.anl.gov/EIC/benchmarks/detector_benchmarks.git \
&& ln -sf ../common_bench detector_benchmarks/.local \
&& mkdir -p /opt/benchmarks \
&& cd /opt/benchmarks \
&& git clone -b ${BENCHMARK_REC_VERSION} --depth 1 \
https://eicweb.phy.anl.gov/EIC/benchmarks/reconstruction_benchmarks.git \
&& ln -sf ../common_bench reconstruction_benchmarks/.local \
&& mkdir -p /opt/benchmarks \
&& cd /opt/benchmarks \
&& git clone -b ${BENCHMARK_PHY_VERSION} --depth 1 \
https://eicweb.phy.anl.gov/EIC/benchmarks/physics_benchmarks.git \
&& ln -sf ../common_bench physics_benchmarks/.local
## Install campaigns into the container
ARG CAMPAIGNS_SINGLE_VERSION="main"
ARG CAMPAIGNS_HEPMC3_VERSION="main"
ARG CAMPAIGNS_CONDOR_VERSION="main"
ARG CAMPAIGNS_SLURM_VERSION="main"
## cache bust when updated repositories
ADD https://api.github.com/repos/eic/simulation_campaign_single/commits/${CAMPAIGNS_SINGLE_VERSION} /tmp/simulation_campaign_single.json
ADD https://api.github.com/repos/eic/simulation_campaign_hepmc3/commits/${CAMPAIGNS_HEPMC3_VERSION} /tmp/simulation_campaign_hepmc3.json
ADD https://api.github.com/repos/eic/job_submission_condor/commits/${CAMPAIGNS_CONDOR_VERSION} /tmp/job_submission_condor.json
ADD https://api.github.com/repos/eic/job_submission_slurm/commits/${CAMPAIGNS_SLURM_VERSION} /tmp/job_submission_slurm.json
RUN mkdir -p /opt/campaigns \
&& cd /opt/campaigns \
&& git clone -b ${CAMPAIGNS_SINGLE_VERSION} --depth 1 \
https://github.com/eic/simulation_campaign_single.git single \
&& mkdir -p /opt/campaigns \
&& cd /opt/campaigns \
&& git clone -b ${CAMPAIGNS_HEPMC3_VERSION} --depth 1 \
https://github.com/eic/simulation_campaign_hepmc3.git hepmc3 \
&& mkdir -p /opt/campaigns \
&& cd /opt/campaigns \
&& git clone -b ${CAMPAIGNS_CONDOR_VERSION} --depth 1 \
https://github.com/eic/job_submission_condor.git condor \
&& mkdir -p /opt/campaigns \
&& cd /opt/campaigns \
&& git clone -b ${CAMPAIGNS_SLURM_VERSION} --depth 1 \
https://github.com/eic/job_submission_slurm.git slurm
## make sure we have the entrypoints setup correctly
ENTRYPOINT ["/usr/local/sbin/entrypoint.sh"]
CMD ["bash", "--rcfile", "/etc/profile", "-l"]
USER 0
WORKDIR /
SHELL ["/usr/local/bin/eic-shell"]
## eic-news
COPY --chmod=0755 eic-news /usr/local/bin/eic-news
RUN echo "test -f $HOME/.eic-news && source /usr/local/bin/eic-news" \
> /etc/profile.d/z13_eic-news.sh
## Hotfix for misbehaving OSG nodes
RUN mkdir /hadoop
containers/jug/feature.Dockerfile deleted 100644 → 0
View file @ 8ab81405
#syntax=docker/dockerfile:1.2
ARG DOCKER_REGISTRY="eicweb.phy.anl.gov:4567/containers/eic_container/"
ARG INTERNAL_TAG="testing"
## Make a specific "feature" image, identical to jug_xl except for the detector
## symlinked as default in /opt/detector. Not that these images will be removed
## in the future once we move the detectors already installed in the main
## image.
FROM ${DOCKER_REGISTRY}jug_xl:${INTERNAL_TAG}
## also install detector/ip geometries into opt
ARG DETECTOR=athena
ARG DETECTOR_BRANCH=canyonlands
RUN rm -rf /opt/detector/{setup.sh,lib,share} \
&& ln -sf /opt/detector/${DETECTOR}-${DETECTOR_BRANCH}/setup.sh \
/opt/detector/setup.sh \
&& ln -sf /opt/detector/${DETECTOR}-${DETECTOR_BRANCH}/lib \
/opt/detector/lib \
&& ln -sf /opt/detector/${DETECTOR}-${DETECTOR_BRANCH}/share \
/opt/detector/share
containers/jug/setup_detectors.py
View file @ 78b71668
......@@ -105,7 +105,7 @@ if __name__ == '__main__':
subprocess.check_call(' '.join(cmd), shell=True)
## build
cxxflags = ''
if os.environ['CXXFLAGS']:
if 'CXXFLAGS' in os.environ:
cxxflags = os.environ['CXXFLAGS']
if cfg.get('cxxflags'):
cxxflags = cfg['cxxflags']
......
containers/jug/xl.Dockerfile
View file @ 78b71668
......@@ -64,62 +64,6 @@ RUN --mount=type=cache,target=/ccache/,sharing=locked,id=${TARGETPLATFORM}
&& popd \
&& rm -rf build eicrecon
## Install benchmarks into the container
ARG BENCHMARK_COM_VERSION="master"
ARG BENCHMARK_DET_VERSION="master"
ARG BENCHMARK_REC_VERSION="master"
ARG BENCHMARK_PHY_VERSION="master"
## cache bust when updated repositories
ADD ${EICWEB}/458/repository/tree?ref=${BENCHMARK_COM_VERSION} /tmp/485.json
ADD ${EICWEB}/399/repository/tree?ref=${BENCHMARK_DET_VERSION} /tmp/399.json
ADD ${EICWEB}/408/repository/tree?ref=${BENCHMARK_REC_VERSION} /tmp/408.json
ADD ${EICWEB}/400/repository/tree?ref=${BENCHMARK_PHY_VERSION} /tmp/400.json
RUN mkdir -p /opt/benchmarks \
&& cd /opt/benchmarks \
&& git clone -b ${BENCHMARK_COM_VERSION} --depth 1 \
https://eicweb.phy.anl.gov/EIC/benchmarks/common_bench.git \
&& mkdir -p /opt/benchmarks \
&& cd /opt/benchmarks \
&& git clone -b ${BENCHMARK_DET_VERSION} --depth 1 \
https://eicweb.phy.anl.gov/EIC/benchmarks/detector_benchmarks.git \
&& ln -sf ../common_bench detector_benchmarks/.local \
&& mkdir -p /opt/benchmarks \
&& cd /opt/benchmarks \
&& git clone -b ${BENCHMARK_REC_VERSION} --depth 1 \
https://eicweb.phy.anl.gov/EIC/benchmarks/reconstruction_benchmarks.git \
&& ln -sf ../common_bench reconstruction_benchmarks/.local \
&& mkdir -p /opt/benchmarks \
&& cd /opt/benchmarks \
&& git clone -b ${BENCHMARK_PHY_VERSION} --depth 1 \
https://eicweb.phy.anl.gov/EIC/benchmarks/physics_benchmarks.git \
&& ln -sf ../common_bench physics_benchmarks/.local
## Install campaigns into the container
ARG CAMPAIGNS_SINGLE_VERSION="main"
ARG CAMPAIGNS_HEPMC3_VERSION="main"
ARG CAMPAIGNS_CONDOR_VERSION="main"
ARG CAMPAIGNS_SLURM_VERSION="main"
## cache bust when updated repositories
ADD https://api.github.com/repos/eic/simulation_campaign_single/commits/${CAMPAIGNS_SINGLE_VERSION} /tmp/simulation_campaign_single.json
ADD https://api.github.com/repos/eic/simulation_campaign_hepmc3/commits/${CAMPAIGNS_HEPMC3_VERSION} /tmp/simulation_campaign_hepmc3.json
ADD https://api.github.com/repos/eic/job_submission_condor/commits/${CAMPAIGNS_CONDOR_VERSION} /tmp/job_submission_condor.json
ADD https://api.github.com/repos/eic/job_submission_slurm/commits/${CAMPAIGNS_SLURM_VERSION} /tmp/job_submission_slurm.json
RUN mkdir -p /opt/campaigns \
&& cd /opt/campaigns \
&& git clone -b ${CAMPAIGNS_SINGLE_VERSION} --depth 1 \
https://github.com/eic/simulation_campaign_single.git single \
&& mkdir -p /opt/campaigns \
&& cd /opt/campaigns \
&& git clone -b ${CAMPAIGNS_HEPMC3_VERSION} --depth 1 \
https://github.com/eic/simulation_campaign_hepmc3.git hepmc3 \
&& mkdir -p /opt/campaigns \
&& cd /opt/campaigns \
&& git clone -b ${CAMPAIGNS_CONDOR_VERSION} --depth 1 \
https://github.com/eic/job_submission_condor.git condor \
&& mkdir -p /opt/campaigns \
&& cd /opt/campaigns \
&& git clone -b ${CAMPAIGNS_SLURM_VERSION} --depth 1 \
https://github.com/eic/job_submission_slurm.git slurm
## also install detector/ip geometries into opt
ARG NIGHTLY=''
## cache bust when updated repositories
......@@ -139,11 +83,3 @@ RUN --mount=type=cache,target=/ccache/,sharing=locked,id=${TARGETPLATFORM}
$NIGHTLY_FLAG \
&& ccache --show-stats \
&& rm /tmp/setup_detectors.py
## eic-news
COPY --chmod=0755 eic-news /usr/local/bin/eic-news
RUN echo "test -f $HOME/.eic-news && source /usr/local/bin/eic-news" \
> /etc/profile.d/z13_eic-news.sh
## Hotfix for misbehaving OSG nodes
RUN mkdir /hadoop
eic-spack.sh
View file @ 78b71668
......@@ -3,7 +3,7 @@ EICSPACK_ORGREPO="eic/eic-spack"
## EIC spack github version, e.g. v0.19.7 or commit hash
## note: nightly builds will use e.g. releases/v0.19
EICSPACK_VERSION="v0.20.11"
EICSPACK_VERSION="v0.20.13"
## Space-separated list of eic-spack cherry-picks
read -r -d '' EICSPACK_CHERRYPICKS <<- \
......
spack-environment/packages.yaml
View file @ 78b71668
......@@ -44,6 +44,7 @@ packages:
cernlib:
require:
- '@2022.11.08.0-free'
- -shared
clhep:
require:
- '@2.4.6.4'
......@@ -68,8 +69,8 @@ packages:
- '@1_54a'
dd4hep:
require:
- '@1.25.1'
- +ddg4 +ddcad -frames +hepmc3 +lcio +xercesc
- '@1.26'
- +ddg4 +ddcad +hepmc3 +lcio +xercesc
dpmjet:
require:
- '@19.3.5'
......@@ -113,7 +114,7 @@ packages:
- -debuginfod
geant4:
require:
- '@11.1.1'
- '@11.1.2.east'
- cxxstd=17 -vecgeom +threads -vtk
- any_of: [+opengl +qt +x11, -opengl -qt -x11]
graphviz:
......
spack.sh
View file @ 78b71668
......@@ -21,6 +21,9 @@ ea1439dfa11a3996c9927ed792dc9fe4b7efc1b8
31431f967a59c07585021cba40683c2ca6ff2c47
316bfd8b7d0d1bc63a7bccf030845775a442a317
6e47f1645f31ce598d7f1f9770e24b483fb117d9
63bb2c9bad8acb018f220630e5ce58e4a039d8a2
ed76eab6943221f17776fd8d128ade6ba69e492c
6c5d125cb06a86ce05bec27ae9fb9b07103bc1c5
---
## Ref: https://github.com/spack/spack/commit/[hash]
## [hash]: [description]
......@@ -37,3 +40,6 @@ ea1439dfa11a3996c9927ed792dc9fe4b7efc1b8
## 31431f967a59c07585021cba40683c2ca6ff2c47: Environment/depfile: fix bug with Git hash versions
## 316bfd8b7d0d1bc63a7bccf030845775a442a317: opencascade: new variants
## 6e47f1645f31ce598d7f1f9770e24b483fb117d9: opencascade: typo in True
## 63bb2c9bad8acb018f220630e5ce58e4a039d8a2: py-cryptography: does not run-depend on py-setuptools-rust
## ed76eab6943221f17776fd8d128ade6ba69e492c: geant4: new version 11.1.2
## 6c5d125cb06a86ce05bec27ae9fb9b07103bc1c5: cernlib: new variant shared