diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 91499251717ac339cfdf786fdd842fa7e41a9c42..18c0942832fd41f7c68a9d1bdda8554fc94e11ae 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,7 +16,7 @@ variables:
## adc9f887eac78a81bb8189d603f4dc45ed3509c1: acts-dd4hep: new package; acts: new version
## We need to enable Docker Buildkit to use cache mounts and better
- ## build performance overal
+ ## build performance overall
DOCKER_BUILDKIT: 1
## Dockerhub registry
@@ -39,9 +39,10 @@ variables:
stages:
- config
- - build:base ## base OS image
- - build:jug ## jug container images
- - deploy ## build/deploy singularity images
+ - build:base ## base OS image
+ - build:jug ## jug container images
+ - build:oneapi_jug ## jug containers with Intel oneAPI support
+ - deploy ## build/deploy singularity images
- test
- finalize
@@ -87,7 +88,6 @@ workflow:
## caching strategy for dispatch to multiple nodes and to avoid
## --> try this strategy: https://medium.com/titansoft-engineering/docker-build-cache-sharing-on-multi-hosts-with-buildkit-and-buildx-eb8f7005918e
-
version:
stage: config
script:
@@ -167,6 +167,11 @@ version:
## debian_base --> jug_dev --> jug_xl
## ---------------> jug_sim
## ---------------> jug_ml
+## debian_stable_base --> oneapi_jug_dev
+## oneapi_jug_dev + jug_xl --> oneapi_jug_xl
+
+## TODO
+## oneapi_runtime + jug_xl --> oneapi_prod
debian_base:default:
extends: .build
@@ -177,6 +182,20 @@ debian_base:default:
BUILD_IMAGE: "debian_base"
script:
- docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
+ -f containers/debian/base.Dockerfile
+ containers/debian
+ - !reference [.build, script]
+
+debian_stable_base:default:
+ extends: .build
+ stage: build:base
+ needs:
+ - version
+ variables:
+ BUILD_IMAGE: "debian_stable_base"
+ script:
+ - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
+ -f containers/debian/stable.Dockerfile
containers/debian
- !reference [.build, script]
@@ -189,9 +208,99 @@ oneapi_base:default:
BUILD_IMAGE: "oneapi_base"
script:
- docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
+ -f containers/oneapi/base.Dockerfile
containers/oneapi
- !reference [.build, script]
+oneapi_jug_dev:default:
+ extends: .build
+ stage: build:oneapi_jug
+ needs:
+ - version
+ - debian_stable_base:default
+ variables:
+ BUILD_IMAGE: "oneapi_jug_dev"
+ script:
+ ## calculate a hash based on the spack.yaml file and the spack directory
+ ## and use this spack as a docker variable to force a rebuild when there
+ ## is a change (versus rerun from cache)
+ - PACKAGE_HASH=$(tar cf - spack* | sha1sum | head -c40)
+ - echo "PACKAGE_HASH= ${PACKAGE_HASH}"
+ ## Copy spack directory and spack.yaml into build context
+ - cp -r spack containers/oneapi/
+ - cp spack.yaml containers/oneapi/spack/spack.yaml
+ - CACHE_FLAG=""
+ - |
+ if [ $FORCE_NOCACHE = 1 ]; then
+ echo "FORCE_NOCACHE set"
+ export CACHE_FLAG="--no-cache"
+ fi
+ ## Copy jug_dev files into build context
+ - cp -r containers/jug/* containers/oneapi
+ - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
+ -f containers/oneapi/onedev.Dockerfile
+ --build-arg SPACK_VERSION="${SPACK_VERSION}"
+ --build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
+ --build-arg CACHE_BUST=${PACKAGE_HASH}
+ --build-arg INTERNAL_TAG=${INTERNAL_TAG}
+ --build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD)
+ containers/oneapi
+ - !reference [.build, script]
+
+oneapi_jug_xl:default:
+ extends: .build
+ stage: build:oneapi_jug
+ needs:
+ - version
+ - oneapi_jug_dev:default
+ variables:
+ BUILD_IMAGE: "oneapi_jug_xl"
+ script:
+ - cp containers/jug/setup_detectors.py containers/oneapi
+ - cp detectors.yaml containers/oneapi
+ - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
+ -f containers/oneapi/onexl.Dockerfile
+ --build-arg INTERNAL_TAG=${INTERNAL_TAG}
+ --build-arg JUGGLER_VERSION=${JUGGLER_VERSION}
+ --build-arg NPDET_VERSION=${NPDET_VERSION}
+ --build-arg EICD_VERSION=${EICD_VERSION}
+ --build-arg AFTERBURNER_VERSION=${AFTERBURNER_VERSION}
+ --build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD)
+ containers/oneapi
+ - !reference [.build, script]
+
+oneapi_jug_xl:nightly:
+ extends: .build
+ stage: build:oneapi_jug
+ needs:
+ - version
+ - oneapi_jug_dev:default
+ variables:
+ BUILD_IMAGE: "oneapi_jug_xl"
+ script:
+ - cp containers/jug/setup_detectors.py containers/oneapi
+ - cp detectors.yaml containers/oneapi
+ - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${NIGHTLY_TAG}
+ -f containers/oneapi/onexl.Dockerfile
+ --build-arg INTERNAL_TAG=${INTERNAL_TAG}
+ --build-arg JUG_VERSION=nightly-$(date +%Y-%m-%d_%H-%M-%S)-${INTERNAL_TAG}-$(git rev-parse HEAD)
+ --build-arg NIGHTLY=1
+ containers/oneapi
+ - |
+ PUSH_FLAG=""
+ if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
+ PUSH_FLAG="--eicweb"
+ fi
+ - ./gitlab-ci/docker_push.sh -i ${BUILD_IMAGE} -l ${NIGHTLY_TAG}
+ -n $DOCKER_NTRIES -t $DOCKER_WAIT_TIME
+ ${NIGHTLY_TAG} ${PUSH_FLAG}
+ - if [ -z "${PUSH_FLAG}" ] ; then
+ ./gitlab-ci/docker_push.sh -i ${BUILD_IMAGE} -l ${NIGHTLY_TAG}
+ -n $DOCKER_NTRIES -t $DOCKER_WAIT_TIME
+ ${NIGHTLY_TAG}-$(date +%Y-%m-%d) --dockerhub ;
+ fi
+ - !reference [.build, script]
+
cuda_base:default:
extends: .build
stage: build:base
@@ -218,7 +327,7 @@ jug_dev:default:
## is a change (versus rerun from cache)
- PACKAGE_HASH=$(tar cf - spack* | sha1sum | head -c40)
- echo "PACKAGE_HASH= ${PACKAGE_HASH}"
- ## move spacke directory and spack.yaml into the container build directory
+ ## move spack directory and spack.yaml into the container build directory
- cp -r spack containers/jug
- cp -r spack.yaml containers/jug/spack/spack.yaml
- CACHE_FLAG=""
@@ -365,6 +474,34 @@ jug_xl:feature:
- mkdir build
- singularity pull build/${BUILD_IMAGE}.sif docker://${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
+oneapi_jug_dev:singularity:default:
+ extends: .singularity
+ needs:
+ - version
+ - oneapi_jug_dev:default
+ variables:
+ BUILD_IMAGE: "oneapi_jug_dev"
+
+oneapi_jug_xl:singularity:default:
+ extends: .singularity
+ needs:
+ - version
+ - oneapi_jug_xl:default
+ variables:
+ BUILD_IMAGE: "oneapi_jug_xl"
+
+oneapi_jug_xl:singularity:nightly:
+ stage: deploy
+ extends: .singularity
+ needs:
+ - version
+ - oneapi_jug_xl:nightly
+ variables:
+ BUILD_IMAGE: "oneapi_jug_xl"
+ script:
+ - mkdir build
+ - singularity pull build/${BUILD_IMAGE}.sif docker://${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${NIGHTLY_TAG}
+
jug_dev:singularity:default:
extends: .singularity
needs:
@@ -448,3 +585,4 @@ cleanup:
if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then
./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${NIGHTLY_TAG}
fi
+
diff --git a/containers/debian/Dockerfile b/containers/debian/base.Dockerfile
similarity index 100%
rename from containers/debian/Dockerfile
rename to containers/debian/base.Dockerfile
diff --git a/containers/debian/stable.Dockerfile b/containers/debian/stable.Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..20941582372a497f56370fea255f909873cc388d
--- /dev/null
+++ b/containers/debian/stable.Dockerfile
@@ -0,0 +1,76 @@
+# syntax=docker/dockerfile:1.2
+
+# Minimal container based on Debian Stable for LTS packages,
+# required for oneAPI containers
+# Very lightweight container with a minimal build environment.
+
+FROM amd64/debian:stable-20220527-slim
+
+
+COPY bashrc /root/.bashrc
+
+ENV CLICOLOR_FORCE=1 \
+ LANGUAGE=en_US.UTF-8 \
+ LANG=en_US.UTF-8 \
+ LC_ALL=en_US.UTF-8
+
+## Install additional packages. Remove the auto-cleanup functionality
+## for docker, as we're using the new buildkit cache instead.
+## TODO: libyaml-cpp-dev is a dependency for afterburner. We can probably remove
+## this once afterburner is added to spack
+RUN --mount=type=cache,target=/var/cache/apt \
+ rm -f /etc/apt/apt.conf.d/docker-clean \
+ && ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime \
+ && echo "US/Eastern" > /etc/timezone \
+ && apt-get -yqq update \
+ && apt-get -yqq upgrade \
+ && apt-get -yqq install --no-install-recommends \
+ bc \
+ ca-certificates \
+ clang-format \
+ clang-tidy \
+ curl \
+ file \
+ build-essential \
+ g++-10 \
+ gcc-10 \
+ gdb \
+ gfortran-10 \
+ ghostscript \
+ git \
+ gnupg2 \
+ gv \
+ iproute2 \
+ iputils-ping \
+ iputils-tracepath \
+ less \
+ libcbor-xs-perl \
+ libjson-xs-perl \
+ libyaml-cpp-dev \
+ locales \
+ lua-posix \
+ make \
+ nano \
+ openssh-client \
+ parallel \
+ poppler-utils \
+ time \
+ unzip \
+ valgrind \
+ vim-nox \
+ wget \
+ && localedef -i en_US -f UTF-8 en_US.UTF-8 \
+ && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 \
+ && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 \
+ && update-alternatives --install /usr/bin/gfortran gfortran \
+ /usr/bin/gfortran-10 100 \
+ && gcc --version \
+ && curl -L \
+ "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" \
+ | bash \
+ && apt-get -yqq update \
+ && apt-get -yqq install --no-install-recommends \
+ gitlab-runner \
+ && apt-get -yqq autoremove \
+ && rm -rf /var/lib/apt/lists/*
+
diff --git a/containers/jug/setup_detectors.py b/containers/jug/setup_detectors.py
index 4495b25d34bf695c696e095fd3e598d772cf0f42..5a6eb726c024e3237e4d4d07c052ca4a2aeece35 100755
--- a/containers/jug/setup_detectors.py
+++ b/containers/jug/setup_detectors.py
@@ -104,7 +104,7 @@ if __name__ == '__main__':
'cmake -B /tmp/build -S /tmp/det -DCMAKE_CXX_STANDARD=17',
'-DCMAKE_INSTALL_PREFIX={prefix}'.format(prefix=prefix),
'&&',
- 'cmake --build /tmp/build -j$((($(nproc)/4)+1)) -- install']
+ 'cmake --build /tmp/build -j$(($(($(nproc)/4))+1)) -- install']
print(' '.join(cmd))
os.system(' '.join(cmd))
## write version info to jug_info if available
@@ -160,25 +160,19 @@ if __name__ == '__main__':
cmd = f'bash -c \'cd {prefix} && source {prefix}/setup.sh && npdet_info print world_z {prefix}/share/{det}/{det}.xml\''
print(cmd)
os.system(cmd)
- print(' --> Symlinking default detector for backward compatibility')
- full_prefix='{}/{}-{}'.format(args.prefix, default_detector, default_version)
- cmd = ['ln -sf {full_prefix}/share {short_prefix}',
- '&&',
- 'ln -sf {full_prefix}/lib {short_prefix}',
- '&&',
- 'ln -sf {full_prefix}/setup.sh {short_prefix}']
- print(' '.join(cmd))
- os.system(' '.join(cmd).format(full_prefix=full_prefix, short_prefix=args.prefix))
+
+ if not default_found and not args.nightly:
+ # Skip symlinking if no defaults present and its not a nightly build
+ pass
+ else:
+ print(' --> Symlinking default detector for backward compatibility')
+ full_prefix='{}/{}-{}'.format(args.prefix, default_detector, default_version)
+ cmd = ['ln -sf {full_prefix}/share {short_prefix}',
+ '&&',
+ 'ln -sf {full_prefix}/lib {short_prefix}',
+ '&&',
+ 'ln -sf {full_prefix}/setup.sh {short_prefix}']
+ print(' '.join(cmd))
+ os.system(' '.join(cmd).format(full_prefix=full_prefix, short_prefix=args.prefix))
print('All done!')
-
-
-
-
-
-
-
-
-
-
-
diff --git a/containers/oneapi/Dockerfile b/containers/oneapi/base.Dockerfile
similarity index 100%
rename from containers/oneapi/Dockerfile
rename to containers/oneapi/base.Dockerfile
diff --git a/containers/oneapi/onedev.Dockerfile b/containers/oneapi/onedev.Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..bdc29037e80b9605776e7217d25f3a8e5f252a31
--- /dev/null
+++ b/containers/oneapi/onedev.Dockerfile
@@ -0,0 +1,342 @@
+# syntax=docker/dockerfile:1.2
+
+# Container based on Jug_dev with Intel oneAPI support.
+
+# Start with Debian-stable and layer oneAPI on top, prevents spack compiler errors.
+
+## ========================================================================================
+## STAGE 1: spack builder image
+## EIC builder image with spack
+## ========================================================================================
+
+ARG DOCKER_REGISTRY="eicweb.phy.anl.gov:4567/containers/eic_container/"
+# Internal Tag will be set by GitLab CI
+ARG INTERNAL_TAG="testing"
+
+FROM ${DOCKER_REGISTRY}debian_stable_base:${INTERNAL_TAG} as builder
+
+## Install some extra spack dependencies
+## Do not use Cache mount as it conflicts with oneapi stage
+RUN \
+ rm -f /etc/apt/apt.conf.d/docker-clean \
+ && apt-get -yqq update \
+ && apt-get -yqq install --no-install-recommends \
+ python3 \
+ python3-dev \
+ python3-distutils \
+ python-is-python3 \
+ && rm -rf /var/lib/apt/lists/*
+
+
+## Setup spack
+## parts:
+ARG SPACK_ROOT=/opt/spack
+ARG SPACK_VERSION="develop"
+ARG SPACK_CHERRYPICKS=""
+RUN echo "Part 1: regular spack install (as in containerize)" \
+ && git clone https://github.com/spack/spack.git /tmp/spack-staging \
+ && cd /tmp/spack-staging \
+ && git checkout $SPACK_VERSION \
+ && if [ -n "$SPACK_CHERRYPICKS" ] ; then \
+ git cherry-pick -n $SPACK_CHERRYPICKS ; \
+ fi \
+ && cd - \
+ && mkdir -p $SPACK_ROOT/opt/spack \
+ && cp -r /tmp/spack-staging/bin $SPACK_ROOT/bin \
+ && cp -r /tmp/spack-staging/etc $SPACK_ROOT/etc \
+ && cp -r /tmp/spack-staging/lib $SPACK_ROOT/lib \
+ && cp -r /tmp/spack-staging/share $SPACK_ROOT/share \
+ && cp -r /tmp/spack-staging/var $SPACK_ROOT/var \
+ && cp -r /tmp/spack-staging/.git $SPACK_ROOT/.git \
+ && rm -rf /tmp/spack-staging \
+ && echo 'export LD_LIBRARY_PATH=/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH'\
+ >> $SPACK_ROOT/share/setup-env.sh \
+ && ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
+ /usr/sbin/docker-shell \
+ && ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
+ /usr/sbin/interactive-shell \
+ && ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
+ /usr/sbin/spack-env \
+ && echo "Part 2: Set target to generic x86_64" \
+ && echo "packages:" > $SPACK_ROOT/etc/spack/packages.yaml \
+ && echo " all:" >> $SPACK_ROOT/etc/spack/packages.yaml \
+ && echo " target: [x86_64]" >> $SPACK_ROOT/etc/spack/packages.yaml \
+ && cat $SPACK_ROOT/etc/spack/packages.yaml \
+ && echo "Part 3: Set config to allow use of more cores for builds" \
+ && echo "(and some other settings)" \
+ && echo "config:" > $SPACK_ROOT/etc/spack/config.yaml \
+ && echo " suppress_gpg_warnings: true" \
+ >> $SPACK_ROOT/etc/spack/config.yaml \
+ && echo " build_jobs: 64" >> $SPACK_ROOT/etc/spack/config.yaml \
+ && echo " install_tree:" >> $SPACK_ROOT/etc/spack/config.yaml \
+ && echo " root: /opt/software" >> $SPACK_ROOT/etc/spack/config.yaml \
+ && cat $SPACK_ROOT/etc/spack/config.yaml
+
+SHELL ["docker-shell"]
+
+## Setup spack buildcache mirrors, including an internal
+## spack mirror using the docker build cache, and
+## a backup mirror on the internal B010 network
+RUN --mount=type=cache,target=/var/cache/spack-mirror \
+ export OLD_PATH=$PATH \
+ && export PATH=$PATH:$SPACK_ROOT/bin \
+ && spack mirror add docker /var/cache/spack-mirror \
+ && spack mirror list
+
+## This variable will change whenevery either spack.yaml or our spack package
+## overrides change, triggering a rebuild
+ARG CACHE_BUST="hash"
+ARG CACHE_NUKE=""
+
+## Setup our custom environment and package overrides
+COPY spack $SPACK_ROOT/eic-spack
+RUN spack repo add --scope site "$SPACK_ROOT/eic-spack" \
+ && mkdir /opt/spack-environment \
+ && cd /opt/spack-environment \
+ && mv $SPACK_ROOT/eic-spack/spack.yaml . \
+ && rm -r /usr/local \
+ && spack env activate . \
+ && spack concretize
+
+
+## Now execute the main build (or fetch from cache if possible)
+## note, no-check-signature is needed to allow the quicker signature-less
+## packages from the internal (docker) buildcache
+##
+## Optional, nuke the buildcache after install, before (re)caching
+## This is useful when going to completely different containers,
+## or intermittently to keep the buildcache step from taking too much time
+##
+## Update the local build cache if needed. Consists of 3 steps:
+## 1. Remove the B010 network buildcache (silicon)
+## 2. Get a list of all packages, and compare with what is already on
+## the buildcache (using package hash)
+## 3. Add packages that need to be added to buildcache if any
+RUN --mount=type=cache,target=/var/cache/spack-mirror \
+ cd /opt/spack-environment \
+ && ls /var/cache/spack-mirror \
+ && spack env activate . \
+ && status=0 \
+ && spack install -j64 --no-check-signature \
+ || spack install -j64 --no-check-signature \
+ || spack install -j64 --no-check-signature \
+ || status=$? \
+ && [ -z "${CACHE_NUKE}" ] \
+ || rm -rf /var/cache/spack-mirror/build_cache/* \
+ && mkdir -p /var/cache/spack-mirror/build_cache \
+ && spack buildcache update-index -d /var/cache/spack-mirror \
+ && spack buildcache list --allarch --very-long \
+ | sed '/^$/d;/^--/d;s/@.\+//;s/\([a-z0-9]*\) \(.*\)/\2\/\1/' \
+ | sort > tmp.buildcache.txt \
+ && spack find --format {name}/{hash} | sort \
+ | comm -23 - tmp.buildcache.txt \
+ | xargs --no-run-if-empty \
+ spack buildcache create --allow-root --only package --unsigned \
+ --directory /var/cache/spack-mirror \
+ --rebuild-index \
+ && spack clean -a \
+ && exit $status
+
+## Extra post-spack steps:
+## - Python packages
+COPY requirements.txt /usr/local/etc/requirements.txt
+RUN --mount=type=cache,target=/var/cache/pip \
+ echo "Installing additional python packages" \
+ && cd /opt/spack-environment && spack env activate . \
+ && python -m pip install \
+ --trusted-host pypi.org \
+ --trusted-host files.pythonhosted.org \
+ --cache-dir /var/cache/pip \
+ --requirement /usr/local/etc/requirements.txt \
+ --no-warn-script-location
+ # ^ Supress not on PATH Warnings
+
+## Including some small fixes:
+## - Somehow PODIO env isn't automatically set,
+## - and Gaudi likes BINARY_TAG to be set
+RUN cd /opt/spack-environment \
+ && echo -n "" \
+ && echo "Grabbing environment info" \
+ && spack env activate --sh -d . \
+ | sed "s?LD_LIBRARY_PATH=?&/lib/x86_64-linux-gnu:?" \
+ | sed '/MANPATH/ s/;$/:;/' \
+ > /etc/profile.d/z10_spack_environment.sh \
+ && cd /opt/spack-environment && spack env activate . \
+ && echo -n "" \
+ && echo "Add extra environment variables for Jug, Podio and Gaudi" \
+ && echo "export PODIO=$(spack location -i podio);" \
+ >> /etc/profile.d/z10_spack_environment.sh \
+ && echo -n "" \
+ && echo "Executing cmake patch for dd4hep 16.1" \
+ && sed -i "s/FIND_PACKAGE(Python/#&/" /usr/local/cmake/DD4hepBuild.cmake
+
+## make sure we have the entrypoints setup correctly
+ENTRYPOINT []
+CMD ["bash", "--rcfile", "/etc/profile", "-l"]
+USER 0
+WORKDIR /
+
+## ========================================================================================
+## STAGE 2: staging image with unnecessariy packages removed and stripped binaries
+## ========================================================================================
+FROM builder AS staging
+
+RUN cd /opt/spack-environment && spack env activate . && spack gc -y
+# Strip all the binaries
+# This reduces the image by factor of x2, so worth the effort
+# note that we do not strip python libraries as can cause issues in some cases
+RUN find -L /usr/local/* \
+ -type d -name site-packages -prune -false -o \
+ -type f -not -name "zdll.lib" -not -name libtensorflow-lite.a \
+ -exec realpath '{}' \; \
+ | xargs file -i \
+ | grep 'charset=binary' \
+ | grep 'x-executable\|x-archive\|x-sharedlib' \
+ | awk -F: '{print $1}' | xargs strip -s
+
+## Bugfix to address issues loading the Qt5 libraries on Linux kernels prior to 3.15
+## See
+#https://askubuntu.com/questions/1034313/ubuntu-18-4-libqt5core-so-5-cannot-open-shared-object-file-no-such-file-or-dir
+## and links therin for more info
+RUN strip --remove-section=.note.ABI-tag /usr/local/lib/libQt5Core.so
+
+## Address Issue #72
+## missing precompiled headers for cppyy due to missing symlink in root
+## install (should really be addressed by ROOT spack package)
+RUN cd /opt/spack-environment && spack env activate . \
+ && if [ ! -e $(spack location -i root)/lib/cppyy_backend/etc ]; then \
+ ln -sf $(spack location -i root)/etc \
+ $(spack location -i root)/lib/cppyy_backend/etc; \
+ fi
+
+RUN spack debug report \
+ | sed "s/^/ - /" | sed "s/\* \*\*//" | sed "s/\*\*//" \
+ >> /etc/jug_info \
+ && spack find --no-groups --long --variants | sed "s/^/ - /" >> /etc/jug_info
+
+COPY eic-shell /usr/local/bin/eic-shell
+COPY eic-info /usr/local/bin/eic-info
+COPY entrypoint.sh /usr/local/sbin/entrypoint.sh
+COPY eic-env.sh /etc/eic-env.sh
+COPY profile.d/a00_cleanup.sh /etc/profile.d
+COPY profile.d/z11_jug_env.sh /etc/profile.d
+COPY singularity.d /.singularity.d
+
+## Add minio client into /usr/local/bin
+ADD https://dl.min.io/client/mc/release/linux-amd64/mc /usr/local/bin
+RUN chmod a+x /usr/local/bin/mc
+
+## ========================================================================================
+## STAGE 3: Set up Intel OneAPI Toolkit
+## Use Intel's HPC-kit and Create Lean image from staging
+## ========================================================================================
+
+FROM intel/oneapi-basekit:2022.2-devel-ubuntu20.04 AS oneapi
+
+
+COPY bashrc /root/.bashrc
+
+ENV CLICOLOR_FORCE=1 \
+ LANGUAGE=en_US.UTF-8 \
+ LANG=en_US.UTF-8 \
+ LC_ALL=en_US.UTF-8
+
+## Install additional packages. Remove the auto-cleanup functionality
+## for docker, as we're using the new buildkit cache instead.
+## We install gcc-10 and g++-10 as they are not installed by default in
+## Ubuntu 20.04
+## TODO: libyaml-cpp-dev is a dependency for afterburner. We can probably remove
+## this once afterburner is added to spack
+RUN --mount=type=cache,target=/var/cache/apt \
+ rm -f /etc/apt/apt.conf.d/docker-clean \
+ && ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime \
+ && echo "US/Eastern" > /etc/timezone \
+ && apt-get -yqq update \
+ && apt-get -yqq upgrade \
+ && apt-get -yqq install --no-install-recommends \
+ bc \
+ ca-certificates \
+ clang-format \
+ clang-tidy \
+ curl \
+ file \
+ build-essential \
+ g++-10 \
+ gcc-10 \
+ gdb \
+ ghostscript \
+ gfortran-10 \
+ git \
+ gnupg2 \
+ gv \
+ iproute2 \
+ iputils-ping \
+ iputils-tracepath \
+ less \
+ libcbor-xs-perl \
+ libjson-xs-perl \
+ libyaml-cpp-dev \
+ locales \
+ lua-posix \
+ make \
+ nano \
+ openssh-client \
+ parallel \
+ poppler-utils \
+ python3 \
+ python3-dev \
+ python3-distutils \
+ time \
+ unzip \
+ valgrind \
+ vim-nox \
+ wget \
+ && localedef -i en_US -f UTF-8 en_US.UTF-8 \
+ && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 \
+ && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 \
+ && update-alternatives --install /usr/bin/gfortran gfortran \
+ /usr/bin/gfortran-10 100 \
+ && gcc --version \
+ && curl -L \
+ "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" \
+ | bash \
+ && apt-get -yqq update \
+ && apt-get -yqq install --no-install-recommends \
+ gitlab-runner \
+ && apt-get -yqq autoremove \
+ && rm -rf /var/lib/apt/lists/*
+
+## copy over everything we need from staging in a single layer
+RUN --mount=from=staging,target=/staging \
+ rm -rf /usr/local \
+ && cp -r /staging/opt/spack-environment /opt/spack-environment \
+ && cp -r /staging/opt/software /opt/software \
+ && cp -r /staging/usr/._local /usr/._local \
+ && cd /usr/._local \
+ && PREFIX_PATH=$(realpath $(ls | tail -n1)) \
+ && echo "Found spack true prefix path to be $PREFIX_PATH" \
+ && cd - \
+ && ln -s ${PREFIX_PATH} /usr/local \
+ && cp /staging/etc/profile.d/*.sh /etc/profile.d/ \
+ && cp /staging/etc/eic-env.sh /etc/eic-env.sh \
+ && cp /staging/etc/jug_info /etc/jug_info \
+ && cp -r /staging/.singularity.d /.singularity.d
+
+## set the jug_dev version and add the afterburner
+## TODO: move afterburner to spack when possible
+ARG JUG_VERSION=1
+ARG AFTERBURNER_VERSION=main
+RUN echo "" >> /etc/jug_info \
+ && echo " - jug_dev: ${JUG_VERSION}" >> /etc/jug_info
+
+## make entrypoint executable and set up oneAPI environment
+RUN chmod +x /usr/local/sbin/entrypoint.sh \
+ && echo "source /opt/intel/oneapi/setvars.sh --force 2> /dev/null" \
+ >> /etc/profile
+
+## make sure we have the entrypoints setup correctly
+ENTRYPOINT ["/usr/local/sbin/entrypoint.sh"]
+CMD ["bash", "--rcfile", "/etc/profile", "-l"]
+USER 0
+WORKDIR /
+SHELL ["/usr/local/bin/eic-shell"]
diff --git a/containers/oneapi/onexl.Dockerfile b/containers/oneapi/onexl.Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..dfb12e0fa84227059f3245f2d86756382f1e0024
--- /dev/null
+++ b/containers/oneapi/onexl.Dockerfile
@@ -0,0 +1,144 @@
+#syntax=docker/dockerfile:1.2
+ARG DOCKER_REGISTRY="eicweb.phy.anl.gov:4567/containers/eic_container/"
+# Internal Tag will be set by GitLab CI
+ARG INTERNAL_TAG="testing"
+
+## ========================================================================================
+## STAGE 1: Base XL Image off jug_dev
+## Clone repos and build using cmake
+## ========================================================================================
+FROM ${DOCKER_REGISTRY}oneapi_jug_dev:${INTERNAL_TAG}
+
+ARG EICWEB="https://eicweb.phy.anl.gov/api/v4/projects"
+ARG JUGGLER_VERSION="master"
+ARG NPDET_VERSION="master"
+ARG EICD_VERSION="master"
+## afterburner
+## TODO move to spack build
+ARG AFTERBURNER_VERSION=main
+
+## version will automatically bust cache for nightly, as it includes
+## the date
+ARG JUG_VERSION=1
+
+ADD ${EICWEB}/18/repository/tree?ref=${NPDET_VERSION} /tmp/18.json
+ADD ${EICWEB}/373/repository/tree?ref=${EICD_VERSION} /tmp/373.json
+ADD ${EICWEB}/492/repository/tree?ref=${AFTERBURNER_VERSION} /tmp/492.json
+RUN cd /tmp \
+ && echo " - jug_xl: ${JUG_VERSION}" >> /etc/jug_info \
+ && echo "INSTALLING NPDET" \
+ && git clone -b ${NPDET_VERSION} --depth 1 \
+ https://eicweb.phy.anl.gov/EIC/NPDet.git \
+ && cmake -B build -S NPDet -DCMAKE_CXX_STANDARD=17 \
+ && cmake --build build -j12 -- install \
+ && pushd NPDet \
+ && echo " - NPDet: ${NPDET_VERSION}-$(git rev-parse HEAD)">> /etc/jug_info \
+ && popd \
+ && rm -rf build NPDet \
+ && cd /tmp \
+ && echo "INSTALLING EICD" \
+ && git clone -b ${EICD_VERSION} --depth 1 \
+ https://eicweb.phy.anl.gov/EIC/eicd.git \
+ && cmake -B build -S eicd -DCMAKE_CXX_STANDARD=17 \
+ && cmake --build build -j12 -- install \
+ && pushd eicd \
+ && echo " - EICD: ${EICD_VERSION}-$(git rev-parse HEAD)">> /etc/jug_info \
+ && popd \
+ && rm -rf build eicd \
+ && cd /tmp \
+ && echo "INSTALLING AFTERBURNER" \
+ && git clone -b ${AFTERBURNER_VERSION} --depth 1 \
+ https://eicweb.phy.anl.gov/monte_carlo/afterburner.git \
+ && cmake -B build -S afterburner/cpp -DCMAKE_INSTALL_PREFIX=/usr/local \
+ -DCMAKE_CXX_STANDARD=17 \
+ && cmake --build build -j12 --target all -- install \
+ && pushd afterburner \
+ && echo " - afterburner: ${AFTERBURNER_VERSION}-$(git rev-parse HEAD)" \
+ >> jug_info \
+ && popd \
+ && rm -rf build afterburner
+
+ADD ${EICWEB}/369/repository/tree?ref=${JUGGLER_VERSION} /tmp/369.json
+RUN cd /tmp \
+ && echo "INSTALLING JUGGLER" \
+ && git clone -b ${JUGGLER_VERSION} --depth 1 \
+ https://eicweb.phy.anl.gov/EIC/juggler.git \
+ && cmake -B build -S juggler \
+ -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=/usr/local \
+ && cmake --build build -j12 -- install \
+ && pushd juggler \
+ && echo " - Juggler: ${JUGGLER_VERSION}-$(git rev-parse HEAD)" \
+ >> /etc/jug_info \
+ && popd \
+ && rm -rf build juggler
+
+## also install detector/ip geometries into opt
+ARG NIGHTLY=''
+ADD ${EICWEB}/473/repository/tree?ref=master /tmp/473.json
+ADD ${EICWEB}/452/repository/tree?ref=master /tmp/452.json
+COPY setup_detectors.py /tmp
+COPY detectors.yaml /tmp
+RUN cd /tmp \
+ && [ "z$NIGHTLY" = "z1" ] && NIGHTLY_FLAG="--nightly" || NIGHTLY_FLAG="" \
+ && /tmp/setup_detectors.py --prefix /opt/detector --config /tmp/detectors.yaml \
+ $NIGHTLY_FLAG \
+ && rm /tmp/setup_detectors.py
+
+## Install benchmarks into the container
+
+ARG BENCHMARK_COM_VERSION="master"
+ARG BENCHMARK_DET_VERSION="master"
+ARG BENCHMARK_REC_VERSION="master"
+ARG BENCHMARK_PHY_VERSION="master"
+
+ADD ${EICWEB}/458/repository/tree?ref=${BENCHMARK_COM_VERSION} /tmp/485.json
+ADD ${EICWEB}/399/repository/tree?ref=${BENCHMARK_DET_VERSION} /tmp/399.json
+ADD ${EICWEB}/408/repository/tree?ref=${BENCHMARK_REC_VERSION} /tmp/408.json
+ADD ${EICWEB}/400/repository/tree?ref=${BENCHMARK_PHY_VERSION} /tmp/400.json
+RUN mkdir -p /opt/benchmarks \
+ && cd /opt/benchmarks \
+ && git clone -b ${BENCHMARK_COM_VERSION} --depth 1 \
+ https://eicweb.phy.anl.gov/EIC/benchmarks/common_bench.git \
+ && mkdir -p /opt/benchmarks \
+ && cd /opt/benchmarks \
+ && git clone -b ${BENCHMARK_DET_VERSION} --depth 1 \
+ https://eicweb.phy.anl.gov/EIC/benchmarks/detector_benchmarks.git \
+ && ln -sf ../common_bench detector_benchmarks/.local \
+ && mkdir -p /opt/benchmarks \
+ && cd /opt/benchmarks \
+ && git clone -b ${BENCHMARK_REC_VERSION} --depth 1 \
+ https://eicweb.phy.anl.gov/EIC/benchmarks/reconstruction_benchmarks.git \
+ && ln -sf ../common_bench reconstruction_benchmarks/.local \
+ && mkdir -p /opt/benchmarks \
+ && cd /opt/benchmarks \
+ && git clone -b ${BENCHMARK_PHY_VERSION} --depth 1 \
+ https://eicweb.phy.anl.gov/EIC/benchmarks/physics_benchmarks.git \
+ && ln -sf ../common_bench physics_benchmarks/.local
+
+## Install campaigns into the container
+
+ARG CAMPAIGNS_SINGLE_VERSION="main"
+ARG CAMPAIGNS_HEPMC3_VERSION="main"
+ARG CAMPAIGNS_CONDOR_VERSION="main"
+ARG CAMPAIGNS_SLURM_VERSION="main"
+
+ADD ${EICWEB}/482/repository/tree?ref=${CAMPAIGNS_SINGLE_VERSION} /tmp/482.json
+ADD ${EICWEB}/483/repository/tree?ref=${CAMPAIGNS_HEPMC3_VERSION} /tmp/483.json
+ADD ${EICWEB}/484/repository/tree?ref=${CAMPAIGNS_CONDOR_VERSION} /tmp/484.json
+ADD ${EICWEB}/485/repository/tree?ref=${CAMPAIGNS_SLURM_VERSION} /tmp/485.json
+RUN mkdir -p /opt/campaigns \
+ && cd /opt/campaigns \
+ && git clone -b ${CAMPAIGNS_SINGLE_VERSION} --depth 1 \
+ https://eicweb.phy.anl.gov/EIC/campaigns/single.git \
+ && mkdir -p /opt/campaigns \
+ && cd /opt/campaigns \
+ && git clone -b ${CAMPAIGNS_HEPMC3_VERSION} --depth 1 \
+ https://eicweb.phy.anl.gov/EIC/campaigns/hepmc3.git \
+ && mkdir -p /opt/campaigns \
+ && cd /opt/campaigns \
+ && git clone -b ${CAMPAIGNS_CONDOR_VERSION} --depth 1 \
+ https://eicweb.phy.anl.gov/EIC/campaigns/condor.git \
+ && mkdir -p /opt/campaigns \
+ && cd /opt/campaigns \
+ && git clone -b ${CAMPAIGNS_SLURM_VERSION} --depth 1 \
+ https://eicweb.phy.anl.gov/EIC/campaigns/slurm.git