diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index af5316914b5385da60b0ac7716476dac02925d5b..43e365497ce0cff5898acf223017b75450c05a82 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -193,22 +193,28 @@ base:
parallel:
matrix:
- BASE_IMAGE: amd64/debian:testing-20220822-slim
- BUILD_IMAGE: debian_testing_base
- - BASE_IMAGE: amd64/debian:stable-20220527-slim
- BUILD_IMAGE: debian_stable_base_new
- - BASE_IMAGE: ubuntu:20.04
- BUILD_IMAGE: ubuntu_base
+ BUILD_IMAGE: amd64_debian_testing_base
+ - BASE_IMAGE: amd64/ubuntu:20.04
+ BUILD_IMAGE: amd64_ubuntu_base
- BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04
BUILD_IMAGE: oneapi_base
- BASE_IMAGE: nvidia/cuda:11.8.0-devel-ubuntu20.04
BUILD_IMAGE: cuda_base
+ - BASE_IMAGE: arm64v8/debian:testing-20220822-slim
+ BUILD_IMAGE: arm64v8_debian_testing_base
+# - BASE_IMAGE: debian:testing-20220822-slim
+# BUILD_IMAGE: debian_testing_base_multiarch
+# PLATFORM: linux/amd64,linux/arm64/v8
extends: .build
stage: base
needs:
- version
script:
- - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
+ - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+ - docker buildx create --name builder --driver docker-container --bootstrap --use
+ - docker buildx build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
-f containers/debian/base.Dockerfile
+ ${PLATFORM:+--platform ${PLATFORM}}
--build-arg BASE_IMAGE=${BASE_IMAGE}
--build-arg BUILD_IMAGE=${BUILD_IMAGE}
containers/debian
@@ -217,18 +223,25 @@ base:
jug_dev:default:
parallel:
matrix:
- - BASE_IMAGE: debian_testing_base
+ - BASE_IMAGE: amd64_debian_testing_base
BUILD_IMAGE: jug_dev
+ SPACK_ARCH: x86_64
+ - BASE_IMAGE: arm64v8_debian_testing_base
+ BUILD_IMAGE: jug_dev_arm64v8
+ SPACK_ARCH: aarch64
# - BASE_IMAGE: oneapi_base
# BUILD_IMAGE: jug_dev_oneapi
+# SPACK_ARCH: x86_64
# - BASE_IMAGE: cuda_base_new
# BUILD_IMAGE: jug_dev_cuda
+# SPACK_ARCH: x86_64
extends: .build
stage: jug
needs:
- version
- base
script:
+ - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
## calculate a hash based on the spack.yaml file and the spack directory
## and use this spack as a docker variable to force a rebuild when there
## is a change (versus rerun from cache)
@@ -243,11 +256,12 @@ jug_dev:default:
export CACHE_FLAG="--no-cache"
fi
## Optionally build the raw builder image
- - test ${EXPORT_BUILDER} = 1 && docker build ${CACHE_FLAG}
+ - test ${EXPORT_BUILDER} = 1 && docker buildx build ${CACHE_FLAG}
-t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:builder-${INTERNAL_TAG}
--target=builder
-f containers/jug/dev.Dockerfile
--build-arg BASE_IMAGE="${BASE_IMAGE}"
+ --build-arg SPACK_ARCH="${SPACK_ARCH}"
--build-arg SPACK_ORGREPO="${SPACK_ORGREPO}"
--build-arg SPACK_VERSION="${SPACK_VERSION}"
--build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
@@ -261,9 +275,10 @@ jug_dev:default:
--build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD)
containers/jug
## now build our image
- - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG} ${CACHE_FLAG}
+ - docker buildx build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG} ${CACHE_FLAG}
-f containers/jug/dev.Dockerfile
--build-arg BASE_IMAGE="${BASE_IMAGE}"
+ --build-arg SPACK_ARCH="${SPACK_ARCH}"
--build-arg SPACK_ORGREPO="${SPACK_ORGREPO}"
--build-arg SPACK_VERSION="${SPACK_VERSION}"
--build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
@@ -284,19 +299,37 @@ jug_dev:default:
## standard exports
- !reference [.build, script]
+jug_dev:multiarch:
+ parallel:
+ matrix:
+ - IMAGE: jug_dev
+ ARCHS: "amd64 arm64v8"
+ stage: jug
+ needs:
+ - version
+ - jug_dev:default
+ script:
+ - echo docker manifest create ${IMAGE}:${EXPORT_TAG} $(echo ${ARCHS} | xargs -d " " -I "{}" echo -n "--amend ${IMAGE}:${EXPORT_TAG}-{}")
+
jug_xl:default:
parallel:
matrix:
- BASE_IMAGE: jug_dev
BUILD_IMAGE: jug_xl
+ DOCKER_ARCH: amd64
+ - BASE_IMAGE: jug_dev_arm64v8
+ BUILD_IMAGE: jug_xl_arm64v8
+ DOCKER_ARCH: arm64v8
# - BASE_IMAGE: jug_dev_oneapi
# BUILD_IMAGE: jug_xl_oneapi
+# DOCKER_ARCH: amd64
extends: .build
stage: jug
needs:
- version
- jug_dev:default
script:
+ - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
- cp detectors.yaml containers/jug
- docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
-f containers/jug/xl.Dockerfile
@@ -312,8 +345,13 @@ jug_xl:nightly:
matrix:
- BASE_IMAGE: jug_dev
BUILD_IMAGE: jug_xl
+ DOCKER_ARCH: amd64
+ - BASE_IMAGE: jug_dev_arm64v8
+ BUILD_IMAGE: jug_xl_arm64v8
+ DOCKER_ARCH: arm64v8
# - BASE_IMAGE: jug_dev_oneapi
# BUILD_IMAGE: jug_xl_oneapi
+# DOCKER_ARCH: amd64
extends: .build
stage: jug
rules:
@@ -326,6 +364,7 @@ jug_xl:nightly:
- version
- jug_dev:default
script:
+ - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
- cp detectors.yaml containers/jug
- docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${NIGHTLY_TAG}
-f containers/jug/xl.Dockerfile
diff --git a/containers/debian/base.Dockerfile b/containers/debian/base.Dockerfile
index bb2c26d65348ebd92e1b8ed40de4ecf606d476f6..6791c2ba85367479b1731d0847693b75cbbd4b9c 100644
--- a/containers/debian/base.Dockerfile
+++ b/containers/debian/base.Dockerfile
@@ -17,8 +17,8 @@ ENV CLICOLOR_FORCE=1 \
## Install additional packages. Remove the auto-cleanup functionality
## for docker, as we're using the new buildkit cache instead.
-RUN --mount=type=cache,target=/var/cache/apt \
- --mount=type=cache,target=/var/lib/apt/lists \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
rm -f /etc/apt/apt.conf.d/docker-clean \
&& ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime \
&& echo "US/Eastern" > /etc/timezone \
@@ -60,8 +60,8 @@ RUN --mount=type=cache,target=/var/cache/apt \
# Install updated compilers, with support for multiple base images
## Ubuntu: latest gcc from toolchain ppa, latest stable clang
## Debian: default gcc with distribution, latest stable clang
-RUN --mount=type=cache,target=/var/cache/apt \
- --mount=type=cache,target=/var/lib/apt/lists \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
. /etc/os-release \
&& mkdir -p /etc/apt/source.list.d \
&& if [ "${ID}" = "ubuntu" ] ; then \
diff --git a/containers/jug/dev.Dockerfile b/containers/jug/dev.Dockerfile
index ffae11e3a7ff0a3fbaf05dfb4d8ce1df5ab4de00..c31dc6b0694b0e0fa1143896e6bf2319cdb3dd54 100644
--- a/containers/jug/dev.Dockerfile
+++ b/containers/jug/dev.Dockerfile
@@ -23,6 +23,7 @@ RUN --mount=type=cache,target=/var/cache/apt \
## Setup spack
## parts:
+ARG SPACK_ARCH="x86_64"
ENV SPACK_ROOT=/opt/spack
ARG SPACK_ORGREPO="spack/spack"
ARG SPACK_VERSION="develop"
@@ -30,8 +31,8 @@ ARG SPACK_CHERRYPICKS=""
ADD https://api.github.com/repos/${SPACK_ORGREPO}/commits/${SPACK_VERSION} /tmp/spack.json
RUN git clone https://github.com/${SPACK_ORGREPO}.git ${SPACK_ROOT} \
&& git -C ${SPACK_ROOT} checkout ${SPACK_VERSION} \
- && if [ -n "$SPACK_CHERRYPICKS" ] ; then \
- git -C ${SPACK_ROOT} cherry-pick -n $SPACK_CHERRYPICKS ; \
+ && if [ -n "${SPACK_CHERRYPICKS}" ] ; then \
+ git -C ${SPACK_ROOT} cherry-pick -n ${SPACK_CHERRYPICKS} ; \
fi \
&& ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/sbin/docker-shell \
@@ -40,7 +41,7 @@ RUN git clone https://github.com/${SPACK_ORGREPO}.git ${SPACK_ROOT} \
&& ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/sbin/spack-env \
&& export PATH=${PATH}:${SPACK_ROOT}/bin \
- && spack config --scope site add "packages:all:require:arch=x86_64" \
+ && spack config --scope site add "packages:all:require:arch=${SPACK_ARCH}" \
&& spack config blame packages \
&& spack config --scope site add "config:suppress_gpg_warnings:true" \
&& spack config --scope site add "config:build_jobs:64" \
@@ -92,6 +93,7 @@ RUN git clone https://github.com/${EICSPACK_ORGREPO}.git ${EICSPACK_ROOT} \
## Setup our custom environment
COPY spack.yaml /opt/spack-environment/
RUN rm -r /usr/local \
+ && source $SPACK_ROOT/share/spack/setup-env.sh \
&& spack env activate /opt/spack-environment/ \
&& spack concretize --fresh
@@ -111,7 +113,7 @@ RUN rm -r /usr/local \
## 3. Add packages that need to be added to buildcache if any
RUN --mount=type=cache,target=/var/cache/spack-mirror \
cd /opt/spack-environment \
- && ls /var/cache/spack-mirror \
+ && source $SPACK_ROOT/share/spack/setup-env.sh \
&& spack env activate . \
&& status=0 \
&& spack install -j64 --no-check-signature \
@@ -139,7 +141,9 @@ RUN --mount=type=cache,target=/var/cache/spack-mirror \
COPY requirements.txt /usr/local/etc/requirements.txt
RUN --mount=type=cache,target=/var/cache/pip \
echo "Installing additional python packages" \
- && cd /opt/spack-environment && spack env activate . \
+ && cd /opt/spack-environment \
+ && source $SPACK_ROOT/share/spack/setup-env.sh \
+ && spack env activate . \
&& python -m pip install \
--trusted-host pypi.org \
--trusted-host files.pythonhosted.org \
@@ -152,6 +156,7 @@ RUN --mount=type=cache,target=/var/cache/pip \
## - Somehow PODIO env isn't automatically set,
## - and Gaudi likes BINARY_TAG to be set
RUN cd /opt/spack-environment \
+ && source $SPACK_ROOT/share/spack/setup-env.sh \
&& echo -n "" \
&& echo "Grabbing environment info" \
&& spack env activate --sh -d . \
@@ -179,7 +184,10 @@ WORKDIR /
FROM builder as staging
# Garbage collect in environment
-RUN cd /opt/spack-environment && spack env activate . && spack gc -y
+RUN cd /opt/spack-environment \
+ && source $SPACK_ROOT/share/spack/setup-env.sh \
+ && spack env activate . \
+ && spack gc -y
## Bugfix to address issues loading the Qt5 libraries on Linux kernels prior to 3.15
## See
diff --git a/spack.yaml b/spack.yaml
index eeda5fefa642758c84e12228a83e6e903d2c29dd..e95ab0538362200e9c93cd954d106e5ba31db21e 100644
--- a/spack.yaml
+++ b/spack.yaml
@@ -25,7 +25,6 @@ spack:
- geant4@11.1.0 cxxstd=17 +opengl +vecgeom +x11 +qt +threads
- hepmc3@3.2.5 +python +rootio
- heppdt@2.06.01
- - igprof@5.9.16
- imagemagick@7.0.8-7
- intel-tbb@2020.3
- irt@1.0.0