Skip to content
Snippets Groups Projects
Commit 4b852f86 authored by Sylvester Joosten's avatar Sylvester Joosten Committed by Wouter Deconinck
Browse files

arm64/qemu build

parent e6f8f724
No related branches found
No related tags found
1 merge request!444arm64/qemu build
...@@ -193,22 +193,28 @@ base: ...@@ -193,22 +193,28 @@ base:
parallel: parallel:
matrix: matrix:
- BASE_IMAGE: amd64/debian:testing-20220822-slim - BASE_IMAGE: amd64/debian:testing-20220822-slim
BUILD_IMAGE: debian_testing_base BUILD_IMAGE: amd64_debian_testing_base
- BASE_IMAGE: amd64/debian:stable-20220527-slim - BASE_IMAGE: amd64/ubuntu:20.04
BUILD_IMAGE: debian_stable_base_new BUILD_IMAGE: amd64_ubuntu_base
- BASE_IMAGE: ubuntu:20.04
BUILD_IMAGE: ubuntu_base
- BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04 - BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04
BUILD_IMAGE: oneapi_base BUILD_IMAGE: oneapi_base
- BASE_IMAGE: nvidia/cuda:11.8.0-devel-ubuntu20.04 - BASE_IMAGE: nvidia/cuda:11.8.0-devel-ubuntu20.04
BUILD_IMAGE: cuda_base BUILD_IMAGE: cuda_base
- BASE_IMAGE: arm64v8/debian:testing-20220822-slim
BUILD_IMAGE: arm64v8_debian_testing_base
# - BASE_IMAGE: debian:testing-20220822-slim
# BUILD_IMAGE: debian_testing_base_multiarch
# PLATFORM: linux/amd64,linux/arm64/v8
extends: .build extends: .build
stage: base stage: base
needs: needs:
- version - version
script: script:
- docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG} - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
- docker buildx create --name builder --driver docker-container --bootstrap --use
- docker buildx build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
-f containers/debian/base.Dockerfile -f containers/debian/base.Dockerfile
${PLATFORM:+--platform ${PLATFORM}}
--build-arg BASE_IMAGE=${BASE_IMAGE} --build-arg BASE_IMAGE=${BASE_IMAGE}
--build-arg BUILD_IMAGE=${BUILD_IMAGE} --build-arg BUILD_IMAGE=${BUILD_IMAGE}
containers/debian containers/debian
...@@ -217,18 +223,25 @@ base: ...@@ -217,18 +223,25 @@ base:
jug_dev:default: jug_dev:default:
parallel: parallel:
matrix: matrix:
- BASE_IMAGE: debian_testing_base - BASE_IMAGE: amd64_debian_testing_base
BUILD_IMAGE: jug_dev BUILD_IMAGE: jug_dev
SPACK_ARCH: x86_64
- BASE_IMAGE: arm64v8_debian_testing_base
BUILD_IMAGE: jug_dev_arm64v8
SPACK_ARCH: aarch64
# - BASE_IMAGE: oneapi_base # - BASE_IMAGE: oneapi_base
# BUILD_IMAGE: jug_dev_oneapi # BUILD_IMAGE: jug_dev_oneapi
# SPACK_ARCH: x86_64
# - BASE_IMAGE: cuda_base_new # - BASE_IMAGE: cuda_base_new
# BUILD_IMAGE: jug_dev_cuda # BUILD_IMAGE: jug_dev_cuda
# SPACK_ARCH: x86_64
extends: .build extends: .build
stage: jug stage: jug
needs: needs:
- version - version
- base - base
script: script:
- docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
## calculate a hash based on the spack.yaml file and the spack directory ## calculate a hash based on the spack.yaml file and the spack directory
## and use this spack as a docker variable to force a rebuild when there ## and use this spack as a docker variable to force a rebuild when there
## is a change (versus rerun from cache) ## is a change (versus rerun from cache)
...@@ -243,11 +256,12 @@ jug_dev:default: ...@@ -243,11 +256,12 @@ jug_dev:default:
export CACHE_FLAG="--no-cache" export CACHE_FLAG="--no-cache"
fi fi
## Optionally build the raw builder image ## Optionally build the raw builder image
- test ${EXPORT_BUILDER} = 1 && docker build ${CACHE_FLAG} - test ${EXPORT_BUILDER} = 1 && docker buildx build ${CACHE_FLAG}
-t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:builder-${INTERNAL_TAG} -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:builder-${INTERNAL_TAG}
--target=builder --target=builder
-f containers/jug/dev.Dockerfile -f containers/jug/dev.Dockerfile
--build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg BASE_IMAGE="${BASE_IMAGE}"
--build-arg SPACK_ARCH="${SPACK_ARCH}"
--build-arg SPACK_ORGREPO="${SPACK_ORGREPO}" --build-arg SPACK_ORGREPO="${SPACK_ORGREPO}"
--build-arg SPACK_VERSION="${SPACK_VERSION}" --build-arg SPACK_VERSION="${SPACK_VERSION}"
--build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}" --build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
...@@ -261,9 +275,10 @@ jug_dev:default: ...@@ -261,9 +275,10 @@ jug_dev:default:
--build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD) --build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD)
containers/jug containers/jug
## now build our image ## now build our image
- docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG} ${CACHE_FLAG} - docker buildx build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG} ${CACHE_FLAG}
-f containers/jug/dev.Dockerfile -f containers/jug/dev.Dockerfile
--build-arg BASE_IMAGE="${BASE_IMAGE}" --build-arg BASE_IMAGE="${BASE_IMAGE}"
--build-arg SPACK_ARCH="${SPACK_ARCH}"
--build-arg SPACK_ORGREPO="${SPACK_ORGREPO}" --build-arg SPACK_ORGREPO="${SPACK_ORGREPO}"
--build-arg SPACK_VERSION="${SPACK_VERSION}" --build-arg SPACK_VERSION="${SPACK_VERSION}"
--build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}" --build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
...@@ -284,19 +299,37 @@ jug_dev:default: ...@@ -284,19 +299,37 @@ jug_dev:default:
## standard exports ## standard exports
- !reference [.build, script] - !reference [.build, script]
jug_dev:multiarch:
parallel:
matrix:
- IMAGE: jug_dev
ARCHS: "amd64 arm64v8"
stage: jug
needs:
- version
- jug_dev:default
script:
- echo docker manifest create ${IMAGE}:${EXPORT_TAG} $(echo ${ARCHS} | xargs -d " " -I "{}" echo -n "--amend ${IMAGE}:${EXPORT_TAG}-{}")
jug_xl:default: jug_xl:default:
parallel: parallel:
matrix: matrix:
- BASE_IMAGE: jug_dev - BASE_IMAGE: jug_dev
BUILD_IMAGE: jug_xl BUILD_IMAGE: jug_xl
DOCKER_ARCH: amd64
- BASE_IMAGE: jug_dev_arm64v8
BUILD_IMAGE: jug_xl_arm64v8
DOCKER_ARCH: arm64v8
# - BASE_IMAGE: jug_dev_oneapi # - BASE_IMAGE: jug_dev_oneapi
# BUILD_IMAGE: jug_xl_oneapi # BUILD_IMAGE: jug_xl_oneapi
# DOCKER_ARCH: amd64
extends: .build extends: .build
stage: jug stage: jug
needs: needs:
- version - version
- jug_dev:default - jug_dev:default
script: script:
- docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
- cp detectors.yaml containers/jug - cp detectors.yaml containers/jug
- docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG} - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
-f containers/jug/xl.Dockerfile -f containers/jug/xl.Dockerfile
...@@ -312,8 +345,13 @@ jug_xl:nightly: ...@@ -312,8 +345,13 @@ jug_xl:nightly:
matrix: matrix:
- BASE_IMAGE: jug_dev - BASE_IMAGE: jug_dev
BUILD_IMAGE: jug_xl BUILD_IMAGE: jug_xl
DOCKER_ARCH: amd64
- BASE_IMAGE: jug_dev_arm64v8
BUILD_IMAGE: jug_xl_arm64v8
DOCKER_ARCH: arm64v8
# - BASE_IMAGE: jug_dev_oneapi # - BASE_IMAGE: jug_dev_oneapi
# BUILD_IMAGE: jug_xl_oneapi # BUILD_IMAGE: jug_xl_oneapi
# DOCKER_ARCH: amd64
extends: .build extends: .build
stage: jug stage: jug
rules: rules:
...@@ -326,6 +364,7 @@ jug_xl:nightly: ...@@ -326,6 +364,7 @@ jug_xl:nightly:
- version - version
- jug_dev:default - jug_dev:default
script: script:
- docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
- cp detectors.yaml containers/jug - cp detectors.yaml containers/jug
- docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${NIGHTLY_TAG} - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${NIGHTLY_TAG}
-f containers/jug/xl.Dockerfile -f containers/jug/xl.Dockerfile
......
...@@ -17,8 +17,8 @@ ENV CLICOLOR_FORCE=1 \ ...@@ -17,8 +17,8 @@ ENV CLICOLOR_FORCE=1 \
## Install additional packages. Remove the auto-cleanup functionality ## Install additional packages. Remove the auto-cleanup functionality
## for docker, as we're using the new buildkit cache instead. ## for docker, as we're using the new buildkit cache instead.
RUN --mount=type=cache,target=/var/cache/apt \ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt/lists \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
rm -f /etc/apt/apt.conf.d/docker-clean \ rm -f /etc/apt/apt.conf.d/docker-clean \
&& ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime \ && ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime \
&& echo "US/Eastern" > /etc/timezone \ && echo "US/Eastern" > /etc/timezone \
...@@ -60,8 +60,8 @@ RUN --mount=type=cache,target=/var/cache/apt \ ...@@ -60,8 +60,8 @@ RUN --mount=type=cache,target=/var/cache/apt \
# Install updated compilers, with support for multiple base images # Install updated compilers, with support for multiple base images
## Ubuntu: latest gcc from toolchain ppa, latest stable clang ## Ubuntu: latest gcc from toolchain ppa, latest stable clang
## Debian: default gcc with distribution, latest stable clang ## Debian: default gcc with distribution, latest stable clang
RUN --mount=type=cache,target=/var/cache/apt \ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt/lists \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
. /etc/os-release \ . /etc/os-release \
&& mkdir -p /etc/apt/source.list.d \ && mkdir -p /etc/apt/source.list.d \
&& if [ "${ID}" = "ubuntu" ] ; then \ && if [ "${ID}" = "ubuntu" ] ; then \
......
...@@ -23,6 +23,7 @@ RUN --mount=type=cache,target=/var/cache/apt \ ...@@ -23,6 +23,7 @@ RUN --mount=type=cache,target=/var/cache/apt \
## Setup spack ## Setup spack
## parts: ## parts:
ARG SPACK_ARCH="x86_64"
ENV SPACK_ROOT=/opt/spack ENV SPACK_ROOT=/opt/spack
ARG SPACK_ORGREPO="spack/spack" ARG SPACK_ORGREPO="spack/spack"
ARG SPACK_VERSION="develop" ARG SPACK_VERSION="develop"
...@@ -30,8 +31,8 @@ ARG SPACK_CHERRYPICKS="" ...@@ -30,8 +31,8 @@ ARG SPACK_CHERRYPICKS=""
ADD https://api.github.com/repos/${SPACK_ORGREPO}/commits/${SPACK_VERSION} /tmp/spack.json ADD https://api.github.com/repos/${SPACK_ORGREPO}/commits/${SPACK_VERSION} /tmp/spack.json
RUN git clone https://github.com/${SPACK_ORGREPO}.git ${SPACK_ROOT} \ RUN git clone https://github.com/${SPACK_ORGREPO}.git ${SPACK_ROOT} \
&& git -C ${SPACK_ROOT} checkout ${SPACK_VERSION} \ && git -C ${SPACK_ROOT} checkout ${SPACK_VERSION} \
&& if [ -n "$SPACK_CHERRYPICKS" ] ; then \ && if [ -n "${SPACK_CHERRYPICKS}" ] ; then \
git -C ${SPACK_ROOT} cherry-pick -n $SPACK_CHERRYPICKS ; \ git -C ${SPACK_ROOT} cherry-pick -n ${SPACK_CHERRYPICKS} ; \
fi \ fi \
&& ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \ && ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/sbin/docker-shell \ /usr/sbin/docker-shell \
...@@ -40,7 +41,7 @@ RUN git clone https://github.com/${SPACK_ORGREPO}.git ${SPACK_ROOT} \ ...@@ -40,7 +41,7 @@ RUN git clone https://github.com/${SPACK_ORGREPO}.git ${SPACK_ROOT} \
&& ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \ && ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash \
/usr/sbin/spack-env \ /usr/sbin/spack-env \
&& export PATH=${PATH}:${SPACK_ROOT}/bin \ && export PATH=${PATH}:${SPACK_ROOT}/bin \
&& spack config --scope site add "packages:all:require:arch=x86_64" \ && spack config --scope site add "packages:all:require:arch=${SPACK_ARCH}" \
&& spack config blame packages \ && spack config blame packages \
&& spack config --scope site add "config:suppress_gpg_warnings:true" \ && spack config --scope site add "config:suppress_gpg_warnings:true" \
&& spack config --scope site add "config:build_jobs:64" \ && spack config --scope site add "config:build_jobs:64" \
...@@ -92,6 +93,7 @@ RUN git clone https://github.com/${EICSPACK_ORGREPO}.git ${EICSPACK_ROOT} \ ...@@ -92,6 +93,7 @@ RUN git clone https://github.com/${EICSPACK_ORGREPO}.git ${EICSPACK_ROOT} \
## Setup our custom environment ## Setup our custom environment
COPY spack.yaml /opt/spack-environment/ COPY spack.yaml /opt/spack-environment/
RUN rm -r /usr/local \ RUN rm -r /usr/local \
&& source $SPACK_ROOT/share/spack/setup-env.sh \
&& spack env activate /opt/spack-environment/ \ && spack env activate /opt/spack-environment/ \
&& spack concretize --fresh && spack concretize --fresh
...@@ -111,7 +113,7 @@ RUN rm -r /usr/local \ ...@@ -111,7 +113,7 @@ RUN rm -r /usr/local \
## 3. Add packages that need to be added to buildcache if any ## 3. Add packages that need to be added to buildcache if any
RUN --mount=type=cache,target=/var/cache/spack-mirror \ RUN --mount=type=cache,target=/var/cache/spack-mirror \
cd /opt/spack-environment \ cd /opt/spack-environment \
&& ls /var/cache/spack-mirror \ && source $SPACK_ROOT/share/spack/setup-env.sh \
&& spack env activate . \ && spack env activate . \
&& status=0 \ && status=0 \
&& spack install -j64 --no-check-signature \ && spack install -j64 --no-check-signature \
...@@ -139,7 +141,9 @@ RUN --mount=type=cache,target=/var/cache/spack-mirror \ ...@@ -139,7 +141,9 @@ RUN --mount=type=cache,target=/var/cache/spack-mirror \
COPY requirements.txt /usr/local/etc/requirements.txt COPY requirements.txt /usr/local/etc/requirements.txt
RUN --mount=type=cache,target=/var/cache/pip \ RUN --mount=type=cache,target=/var/cache/pip \
echo "Installing additional python packages" \ echo "Installing additional python packages" \
&& cd /opt/spack-environment && spack env activate . \ && cd /opt/spack-environment \
&& source $SPACK_ROOT/share/spack/setup-env.sh \
&& spack env activate . \
&& python -m pip install \ && python -m pip install \
--trusted-host pypi.org \ --trusted-host pypi.org \
--trusted-host files.pythonhosted.org \ --trusted-host files.pythonhosted.org \
...@@ -152,6 +156,7 @@ RUN --mount=type=cache,target=/var/cache/pip \ ...@@ -152,6 +156,7 @@ RUN --mount=type=cache,target=/var/cache/pip \
## - Somehow PODIO env isn't automatically set, ## - Somehow PODIO env isn't automatically set,
## - and Gaudi likes BINARY_TAG to be set ## - and Gaudi likes BINARY_TAG to be set
RUN cd /opt/spack-environment \ RUN cd /opt/spack-environment \
&& source $SPACK_ROOT/share/spack/setup-env.sh \
&& echo -n "" \ && echo -n "" \
&& echo "Grabbing environment info" \ && echo "Grabbing environment info" \
&& spack env activate --sh -d . \ && spack env activate --sh -d . \
...@@ -179,7 +184,10 @@ WORKDIR / ...@@ -179,7 +184,10 @@ WORKDIR /
FROM builder as staging FROM builder as staging
# Garbage collect in environment # Garbage collect in environment
RUN cd /opt/spack-environment && spack env activate . && spack gc -y RUN cd /opt/spack-environment \
&& source $SPACK_ROOT/share/spack/setup-env.sh \
&& spack env activate . \
&& spack gc -y
## Bugfix to address issues loading the Qt5 libraries on Linux kernels prior to 3.15 ## Bugfix to address issues loading the Qt5 libraries on Linux kernels prior to 3.15
## See ## See
......
...@@ -25,7 +25,6 @@ spack: ...@@ -25,7 +25,6 @@ spack:
- geant4@11.1.0 cxxstd=17 +opengl +vecgeom +x11 +qt +threads - geant4@11.1.0 cxxstd=17 +opengl +vecgeom +x11 +qt +threads
- hepmc3@3.2.5 +python +rootio - hepmc3@3.2.5 +python +rootio
- heppdt@2.06.01 - heppdt@2.06.01
- igprof@5.9.16
- imagemagick@7.0.8-7 - imagemagick@7.0.8-7
- intel-tbb@2020.3 - intel-tbb@2020.3
- irt@1.0.0 - irt@1.0.0
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment