Skip to content
Snippets Groups Projects
Commit 0853423c authored by Wouter Deconinck's avatar Wouter Deconinck
Browse files

feat: avoid large copy into runtime image

parent f6fb1bc3
Branches
Tags
1 merge request!994feat: avoid large copy into runtime image
Pipeline #103036 canceled
Stage: status-pending
Stage: config
Stage: base
Stage: eic
Stage: deploy
Stage: benchmarks
Stage: finalize
Stage: status-report
Pipeline: reconstruction_benchmarks

#103052

    Pipeline: eic_container: feat: avoid large copy into runtime image

    #103051

      Pipeline: eic_container: feat: avoid large copy into runtime image

      #103050

        Hide whitespace changes
        Inline Side-by-side
        .gitlab-ci.yml
        + 13
        − 10
        View file @ 0853423c
        ......@@ -270,6 +270,9 @@ base:
        needs:
        - version
        script:
        - source spack.sh ;
        source key4hep-spack.sh ;
        source eic-spack.sh ;
        - attempts=0
        - nocache=""
        - while !
        ......@@ -284,6 +287,16 @@ base:
        --platform ${PLATFORM}
        --build-arg BASE_IMAGE=${BASE_IMAGE}
        --build-arg BUILD_IMAGE=${BUILD_IMAGE}
        --build-arg SPACK_ORGREPO=${SPACK_ORGREPO}
        --build-arg SPACK_VERSION=${SPACK_VERSION}
        --build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
        --build-arg SPACK_CHERRYPICKS_FILES="${SPACK_CHERRYPICKS_FILES}"
        --build-arg KEY4HEPSPACK_ORGREPO=${KEY4HEPSPACK_ORGREPO}
        --build-arg KEY4HEPSPACK_VERSION=${KEY4HEPSPACK_VERSION}
        --build-arg EICSPACK_ORGREPO=${EICSPACK_ORGREPO}
        --build-arg EICSPACK_VERSION=${EICSPACK_VERSION}
        --build-arg S3_ACCESS_KEY=${S3_ACCESS_KEY}
        --build-arg S3_SECRET_KEY=${S3_SECRET_KEY}
        --provenance false
        containers/debian
        2>&1 | tee build.log
        ......@@ -419,16 +432,6 @@ eic:
        --build-arg BUILDER_IMAGE=${BUILDER_IMAGE}
        --build-arg RUNTIME_IMAGE=${RUNTIME_IMAGE}
        --build-arg INTERNAL_TAG=${INTERNAL_TAG}
        --build-arg SPACK_ORGREPO=${SPACK_ORGREPO}
        --build-arg SPACK_VERSION=${SPACK_VERSION}
        --build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}"
        --build-arg SPACK_CHERRYPICKS_FILES="${SPACK_CHERRYPICKS_FILES}"
        --build-arg KEY4HEPSPACK_ORGREPO=${KEY4HEPSPACK_ORGREPO}
        --build-arg KEY4HEPSPACK_VERSION=${KEY4HEPSPACK_VERSION}
        --build-arg EICSPACK_ORGREPO=${EICSPACK_ORGREPO}
        --build-arg EICSPACK_VERSION=${EICSPACK_VERSION}
        --build-arg S3_ACCESS_KEY=${S3_ACCESS_KEY}
        --build-arg S3_SECRET_KEY=${S3_SECRET_KEY}
        --build-arg JUG_VERSION=${EXPORT_TAG}-${BUILD_TYPE}-$(git rev-parse HEAD)
        ${IF_BUILD_DEFAULT+
        ${EDM4EIC_VERSION:+--build-arg EDM4EIC_VERSION=${EDM4EIC_VERSION}}
        ......
        containers/debian/base.Dockerfile
        + 115
        − 0
        View file @ 0853423c
        ......@@ -5,6 +5,8 @@ ARG BUILD_IMAGE="debian_stable_base"
        # Minimal container based on Debian base systems for up-to-date packages.
        FROM ${BASE_IMAGE}
        ARG TARGETPLATFORM
        LABEL maintainer="Sylvester Joosten <sjoosten@anl.gov>" \
        name="${BUILD_IMAGE}" \
        march="amd64"
        ......@@ -125,3 +127,116 @@ update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 100
        gcc --version
        clang --version
        EOF
        ## Install some extra spack dependencies
        RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${TARGETPLATFORM} \
        --mount=type=cache,target=/var/lib/apt/lists,sharing=locked,id=${TARGETPLATFORM} <<EOF
        rm -f /etc/apt/apt.conf.d/docker-clean
        apt-get -yqq update
        apt-get -yqq install --no-install-recommends \
        jq \
        python3 \
        python3-dev \
        python3-distutils \
        python3-boto3 \
        python-is-python3
        EOF
        ## Setup spack
        ENV SPACK_ROOT=/opt/spack
        ARG SPACK_ORGREPO="spack/spack"
        ARG SPACK_VERSION="releases/v0.20"
        ENV SPACK_PYTHON=/usr/bin/python3
        ARG SPACK_CHERRYPICKS=""
        ARG SPACK_CHERRYPICKS_FILES=""
        ADD https://api.github.com/repos/${SPACK_ORGREPO}/commits/${SPACK_VERSION} /tmp/spack.json
        RUN <<EOF
        git config --global user.email "gitlab@eicweb.phy.anl.gov"
        git config --global user.name "EIC Container Build Service"
        git config --global advice.detachedHead false
        git clone --filter=tree:0 https://github.com/${SPACK_ORGREPO}.git ${SPACK_ROOT}
        git -C ${SPACK_ROOT} checkout ${SPACK_VERSION}
        if [ -n "${SPACK_CHERRYPICKS}" ] ; then
        SPACK_CHERRYPICKS=$(git -C ${SPACK_ROOT} rev-list --topo-order ${SPACK_CHERRYPICKS} | grep -m $(echo ${SPACK_CHERRYPICKS} | wc -w) "${SPACK_CHERRYPICKS}" | tac)
        eval "declare -A SPACK_CHERRYPICKS_FILES_ARRAY=(${SPACK_CHERRYPICKS_FILES})"
        for hash in ${SPACK_CHERRYPICKS} ; do
        if [ -n "${SPACK_CHERRYPICKS_FILES_ARRAY[${hash}]+found}" ] ; then
        git -C ${SPACK_ROOT} show ${hash} -- ${SPACK_CHERRYPICKS_FILES_ARRAY[${hash}]//,/ } | patch -p1 -d ${SPACK_ROOT}
        git -C ${SPACK_ROOT} commit --all --message "$(git -C ${SPACK_ROOT} show --no-patch --pretty=format:%s ${hash})"
        else
        git -C ${SPACK_ROOT} cherry-pick ${hash}
        fi
        done
        fi
        git -C $SPACK_ROOT gc --prune=all --aggressive
        sed -i 's/timeout=60/timeout=None/' $SPACK_ROOT/lib/spack/spack/stage.py
        ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash /usr/bin/docker-shell
        ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash /usr/bin/interactive-shell
        ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash /usr/bin/spack-env
        EOF
        ## Use spack entrypoint. NOTE: Requires `set -ex` in all multi-line scripts!
        SHELL ["docker-shell"]
        ## Setup build configuration
        ARG jobs=1
        RUN <<EOF
        set -e
        declare -A target=(["linux/amd64"]="x86_64_v2" ["linux/arm64"]="aarch64")
        target=${target[${TARGETPLATFORM}]}
        spack config --scope site add "packages:all:require:[target=${target}]"
        spack config --scope site add "packages:all:target:[${target}]"
        spack config blame packages
        spack config --scope user add "config:suppress_gpg_warnings:true"
        spack config --scope user add "config:build_jobs:${jobs}"
        spack config --scope user add "config:db_lock_timeout:${jobs}00"
        spack config --scope user add "config:source_cache:/var/cache/spack"
        spack config --scope user add "config:install_tree:root:/opt/software"
        spack config --scope user add "config:ccache:true"
        spack config blame config
        spack compiler find --scope site
        spack config blame compilers
        EOF
        ## Setup buildcache mirrors
        ## - this always adds the read-only mirror to the container
        ## - the write-enabled mirror is provided later as a secret mount
        ARG S3_ACCESS_KEY=""
        ARG S3_SECRET_KEY=""
        RUN --mount=type=cache,target=/var/cache/spack <<EOF
        set -e
        if [ -n "${S3_ACCESS_KEY}" ] ; then
        spack mirror add --scope site --unsigned \
        --s3-endpoint-url https://eics3.sdcc.bnl.gov:9000 \
        --s3-access-key-id "${S3_ACCESS_KEY}" \
        --s3-access-key-secret "${S3_SECRET_KEY}" \
        eics3 s3://eictest/EPIC/spack/${SPACK_VERSION}
        fi
        spack mirror add --scope site --signed spack-${SPACK_VERSION} https://binaries.spack.io/${SPACK_VERSION}
        spack mirror add --scope site --unsigned ghcr-${SPACK_VERSION} oci://ghcr.io/eic/spack-${SPACK_VERSION}
        spack mirror list
        EOF
        ## Setup eic-spack
        ENV EICSPACK_ROOT=${SPACK_ROOT}/var/spack/repos/eic-spack
        ARG EICSPACK_ORGREPO="eic/eic-spack"
        ARG EICSPACK_VERSION="$SPACK_VERSION"
        ADD https://api.github.com/repos/${EICSPACK_ORGREPO}/commits/${EICSPACK_VERSION} /tmp/eic-spack.json
        RUN <<EOF
        set -e
        git clone --filter=tree:0 https://github.com/${EICSPACK_ORGREPO}.git ${EICSPACK_ROOT}
        git -C ${EICSPACK_ROOT} checkout ${EICSPACK_VERSION}
        spack repo add --scope site "${EICSPACK_ROOT}"
        EOF
        ## Setup key4hep-spack
        ENV KEY4HEPSPACK_ROOT=${SPACK_ROOT}/var/spack/repos/key4hep-spack
        ARG KEY4HEPSPACK_ORGREPO="key4hep/key4hep-spack"
        ARG KEY4HEPSPACK_VERSION="main"
        ADD https://api.github.com/repos/${KEY4HEPSPACK_ORGREPO}/commits/${KEY4HEPSPACK_VERSION} /tmp/key4hep-spack.json
        RUN <<EOF
        set -e
        git clone --filter=tree:0 https://github.com/${KEY4HEPSPACK_ORGREPO}.git ${KEY4HEPSPACK_ROOT}
        git -C ${KEY4HEPSPACK_ROOT} checkout ${KEY4HEPSPACK_VERSION}
        spack repo add --scope site "${KEY4HEPSPACK_ROOT}"
        EOF
        containers/jug/dev.Dockerfile
        + 198
        − 201
        View file @ 0853423c
        ......@@ -5,155 +5,69 @@ ARG BUILDER_IMAGE="debian_stable_base"
        ARG RUNTIME_IMAGE="debian_stable_base"
        ARG INTERNAL_TAG="testing"
        ##
        ## This docker build follows two tracks, in order to ensure that we build all packages
        ## in a builder image, but install them in a runtime image, while at the same time
        ## avoiding a expensive filesystem copy operation at the end that breaks layering.
        ##
        ## The build is split in an infrequently-changing default environment, upon which
        ## an environment with custom versions (e.g. individual commits) is layered. The
        ## custom environment will change frequently but layers will be smaller, allowing
        ## for easier deployment with smaller delta layers.
        ##
        ## The separation in a builder and runtime image is particularly relevant to end up with
        ## lightweight images for expensive build dependencies, such as for example CUDA.
        ##
        ## builder track runtime track
        ## ----------------------------------------------------------------------
        ## builder_image runtime_image
        ## builder_concretization_default
        ## builder_installation_default -> runtime_concretization_default (copy spack.lock)
        ## \-> runtime_installation_default (from buildcache)
        ## builder_concretization_custom
        ## builder_installation_custom -> runtime_concretization_custom (copy spack.lock)
        ## \-> runtime_installation_custom (from buildcache)
        ##
        ## ========================================================================================
        ## STAGE 0: spack image
        ## EIC spack image with spack and eic-spack repositories
        ## builder_concretization_default
        ## - builder base with concretization of default versions
        ## ========================================================================================
        FROM ${DOCKER_REGISTRY}${BUILDER_IMAGE}:${INTERNAL_TAG} as spack
        FROM ${DOCKER_REGISTRY}${BUILDER_IMAGE}:${INTERNAL_TAG} as builder_concretization_default
        ARG TARGETPLATFORM
        ## With heredocs for multi-line scripts, we want to fail on error and the print failing line.
        ## Ref: https://docs.docker.com/engine/reference/builder/#example-running-a-multi-line-script
        SHELL ["bash", "-ex", "-c"]
        ## install some extra spack dependencies
        RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${TARGETPLATFORM} \
        --mount=type=cache,target=/var/lib/apt/lists,sharing=locked,id=${TARGETPLATFORM} <<EOF
        rm -f /etc/apt/apt.conf.d/docker-clean
        apt-get -yqq update
        apt-get -yqq install --no-install-recommends \
        jq \
        python3 \
        python3-dev \
        python3-distutils \
        python3-boto3 \
        python-is-python3
        EOF
        ## Setup spack
        ENV SPACK_ROOT=/opt/spack
        ARG SPACK_ORGREPO="spack/spack"
        ARG SPACK_VERSION="releases/v0.20"
        ENV SPACK_PYTHON=/usr/bin/python3
        ARG SPACK_CHERRYPICKS=""
        ARG SPACK_CHERRYPICKS_FILES=""
        ADD https://api.github.com/repos/${SPACK_ORGREPO}/commits/${SPACK_VERSION} /tmp/spack.json
        RUN <<EOF
        git config --global user.email "gitlab@eicweb.phy.anl.gov"
        git config --global user.name "EIC Container Build Service"
        git config --global advice.detachedHead false
        git clone --filter=tree:0 https://github.com/${SPACK_ORGREPO}.git ${SPACK_ROOT}
        git -C ${SPACK_ROOT} checkout ${SPACK_VERSION}
        if [ -n "${SPACK_CHERRYPICKS}" ] ; then
        SPACK_CHERRYPICKS=$(git -C ${SPACK_ROOT} rev-list --topo-order ${SPACK_CHERRYPICKS} | grep -m $(echo ${SPACK_CHERRYPICKS} | wc -w) "${SPACK_CHERRYPICKS}" | tac)
        eval "declare -A SPACK_CHERRYPICKS_FILES_ARRAY=(${SPACK_CHERRYPICKS_FILES})"
        for hash in ${SPACK_CHERRYPICKS} ; do
        if [ -n "${SPACK_CHERRYPICKS_FILES_ARRAY[${hash}]+found}" ] ; then
        git -C ${SPACK_ROOT} show ${hash} -- ${SPACK_CHERRYPICKS_FILES_ARRAY[${hash}]//,/ } | patch -p1 -d ${SPACK_ROOT}
        git -C ${SPACK_ROOT} commit --all --message "$(git -C ${SPACK_ROOT} show --no-patch --pretty=format:%s ${hash})"
        else
        git -C ${SPACK_ROOT} cherry-pick ${hash}
        fi
        done
        fi
        git -C $SPACK_ROOT gc --prune=all --aggressive
        sed -i 's/timeout=60/timeout=None/' $SPACK_ROOT/lib/spack/spack/stage.py
        ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash /usr/bin/docker-shell
        ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash /usr/bin/interactive-shell
        ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash /usr/bin/spack-env
        EOF
        ## Use spack entrypoint. NOTE: Requires `set -ex` in all multi-line scripts!
        SHELL ["docker-shell"]
        ## Setup build configuration
        ARG jobs=1
        RUN <<EOF
        set -e
        declare -A target=(["linux/amd64"]="x86_64_v2" ["linux/arm64"]="aarch64")
        target=${target[${TARGETPLATFORM}]}
        spack config --scope site add "packages:all:require:[target=${target}]"
        spack config --scope site add "packages:all:target:[${target}]"
        spack external find --not-buildable --scope site --path /usr/local/cuda/bin cuda
        spack external find --not-buildable --scope site llvm
        spack config blame packages
        spack config --scope user add "config:suppress_gpg_warnings:true"
        spack config --scope user add "config:build_jobs:${jobs}"
        spack config --scope user add "config:db_lock_timeout:${jobs}00"
        spack config --scope user add "config:source_cache:/var/cache/spack"
        spack config --scope user add "config:install_tree:root:/opt/software"
        spack config --scope user add "config:ccache:true"
        spack config blame config
        spack compiler find --scope site
        spack config blame compilers
        EOF
        ## Setup buildcache mirrors
        ## - this always adds the read-only mirror to the container
        ## - the write-enabled mirror is provided later as a secret mount
        ARG S3_ACCESS_KEY=""
        ARG S3_SECRET_KEY=""
        RUN --mount=type=cache,target=/var/cache/spack <<EOF
        set -e
        if [ -n "${S3_ACCESS_KEY}" ] ; then
        spack mirror add --scope site --unsigned \
        --s3-endpoint-url https://eics3.sdcc.bnl.gov:9000 \
        --s3-access-key-id "${S3_ACCESS_KEY}" \
        --s3-access-key-secret "${S3_SECRET_KEY}" \
        eics3 s3://eictest/EPIC/spack/${SPACK_VERSION}
        fi
        spack mirror add --scope site --signed spack-${SPACK_VERSION} https://binaries.spack.io/${SPACK_VERSION}
        spack mirror add --scope site --unsigned ghcr-${SPACK_VERSION} oci://ghcr.io/eic/spack-${SPACK_VERSION}
        spack mirror list
        EOF
        ## Setup eic-spack
        ENV EICSPACK_ROOT=${SPACK_ROOT}/var/spack/repos/eic-spack
        ARG EICSPACK_ORGREPO="eic/eic-spack"
        ARG EICSPACK_VERSION="$SPACK_VERSION"
        ADD https://api.github.com/repos/${EICSPACK_ORGREPO}/commits/${EICSPACK_VERSION} /tmp/eic-spack.json
        RUN <<EOF
        set -e
        git clone --filter=tree:0 https://github.com/${EICSPACK_ORGREPO}.git ${EICSPACK_ROOT}
        git -C ${EICSPACK_ROOT} checkout ${EICSPACK_VERSION}
        spack repo add --scope site "${EICSPACK_ROOT}"
        EOF
        ## Copy our default environment
        COPY --from=spack-environment . /opt/spack-environment/
        ARG ENV=dev
        ENV SPACK_ENV=/opt/spack-environment/${ENV}
        ## Setup key4hep-spack
        ENV KEY4HEPSPACK_ROOT=${SPACK_ROOT}/var/spack/repos/key4hep-spack
        ARG KEY4HEPSPACK_ORGREPO="key4hep/key4hep-spack"
        ARG KEY4HEPSPACK_VERSION="main"
        ADD https://api.github.com/repos/${KEY4HEPSPACK_ORGREPO}/commits/${KEY4HEPSPACK_VERSION} /tmp/key4hep-spack.json
        # Concretization (default environment)
        RUN <<EOF
        set -e
        git clone --filter=tree:0 https://github.com/${KEY4HEPSPACK_ORGREPO}.git ${KEY4HEPSPACK_ROOT}
        git -C ${KEY4HEPSPACK_ROOT} checkout ${KEY4HEPSPACK_VERSION}
        spack repo add --scope site "${KEY4HEPSPACK_ROOT}"
        echo -e "\n view: false" >> ${SPACK_ENV}/spack.yaml
        spack env activate --dir ${SPACK_ENV}
        spack external find --not-buildable --scope env:${SPACK_ENV} --path /usr/local/cuda/bin cuda
        spack external find --not-buildable --scope env:${SPACK_ENV} llvm
        spack concretize --fresh --force
        EOF
        ## ========================================================================================
        ## STAGE 1: builder
        ## EIC builder image with spack environment
        ## builder_installation_default
        ## - builder base with installation of default versions
        ## ========================================================================================
        FROM spack as builder
        FROM builder_concretization_default as builder_installation_default
        ARG TARGETPLATFORM
        ## 1. Setup our default environment (secret mount for write-enabled mirror)
        COPY --from=spack-environment . /opt/spack-environment/
        ARG ENV=dev
        ENV SPACK_ENV=/opt/spack-environment/${ENV}
        # Installation (default environment)
        RUN --mount=type=cache,target=/ccache,id=${TARGETPLATFORM} \
        --mount=type=cache,target=/var/cache/spack \
        --mount=type=secret,id=mirrors,target=/opt/spack/etc/spack/mirrors.yaml \
        <<EOF
        set -e
        export CCACHE_DIR=/ccache
        source ${SPACK_ROOT}/share/spack/setup-env.sh
        mkdir -p /var/cache/spack/blobs/sha256/
        find /var/cache/spack/blobs/sha256/ -ignore_readdir_race -atime +7 -delete
        echo -e "\n view: false" >> ${SPACK_ENV}/spack.yaml
        spack env activate --dir ${SPACK_ENV}
        spack concretize --fresh --force
        make --jobs ${jobs} --keep-going --directory /opt/spack-environment \
        SPACK_ENV=${SPACK_ENV} \
        BUILDCACHE_OCI_PROMPT="eicweb" \
        ......@@ -163,11 +77,72 @@ spack find --long --no-groups \
        | sed -e '1,/Installed packages/d;s/\([^@]*\).*/\1/g' \
        | uniq -D -f1 | grep -v -w -e "\(epic\|py-pip\|py-cython\)" \
        | tee /tmp/duplicates.txt
        test -s /tmp/duplicates.txt && ( cat /tmp/duplicates.txt | while read hash spec ; do spack spec --long /${hash} ; done ) && exit 1
        if [ -s /tmp/duplicates.txt ] ; then
        echo "Duplicate packages found"
        cat /tmp/duplicates.txt | while read hash spec ; do spack spec --long /${hash} ; done
        exit 1
        fi
        ccache --show-stats
        ccache --zero-stats
        EOF
        ## ========================================================================================
        ## runtime_concretization_default
        ## - runtime base with concretization of default versions (taken from equivalent builder)
        ## ========================================================================================
        FROM ${DOCKER_REGISTRY}${RUNTIME_IMAGE}:${INTERNAL_TAG} as runtime_concretization_default
        ARG TARGETPLATFORM
        ## Copy our default environment
        COPY --from=spack-environment . /opt/spack-environment/
        ARG ENV=dev
        ENV SPACK_ENV=/opt/spack-environment/${ENV}
        RUN echo -e "\n view: false" >> ${SPACK_ENV}/spack.yaml
        COPY --from=builder_installation_default \
        /opt/spack-environment/${ENV}/spack.* \
        /opt/spack-environment/${ENV}/
        ## ========================================================================================
        ## runtime_installation_default
        ## - runtime base with installation of default versions (buildcache populated by builder)
        ## ========================================================================================
        FROM runtime_concretization_default as runtime_installation_default
        ARG TARGETPLATFORM
        # Installation (default environment, from buildcache)
        RUN --mount=type=cache,target=/var/cache/spack \
        --mount=type=secret,id=mirrors,target=/opt/spack/etc/spack/mirrors.yaml \
        <<EOF
        make --jobs ${jobs} --keep-going --directory /opt/spack-environment \
        SPACK_ENV=${SPACK_ENV} SPACK_INSTALL_FLAGS="--use-buildcache only"
        spack gc --yes-to-all
        EOF
        ## Add minio client into /opt/mc/bin/mc
        ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-amd64/mc /opt/mc/bin/mc-amd64
        ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-arm64/mc /opt/mc/bin/mc-arm64
        RUN <<EOF
        set -e
        declare -A target=(["linux/amd64"]="amd64" ["linux/arm64"]="arm64")
        mv /opt/mc/bin/mc-${target[${TARGETPLATFORM}]} /opt/mc/bin/mc
        unset target[${TARGETPLATFORM}]
        for t in ${target[*]} ; do
        rm /opt/mc/bin/mc-${t}
        done
        EOF
        ## ========================================================================================
        ## builder_concretization_custom
        ## - builder base with concretization of custom versions
        ## ========================================================================================
        FROM builder_installation_default as builder_concretization_custom
        ARG TARGETPLATFORM
        ## 2. Setup our environment with custom versions (on top of cached layer)
        ## Note: these default versions are just the very first commit.
        ARG EDM4EIC_VERSION="8aeb507f93a93257c99985efbce0ec1371e0b331"
        ......@@ -178,12 +153,9 @@ ADD https://api.github.com/repos/eic/edm4eic/commits/${EDM4EIC_VERSION} /tmp/edm
        ADD https://api.github.com/repos/eic/eicrecon/commits/${EICRECON_VERSION} /tmp/eicrecon.json
        ADD https://api.github.com/repos/eic/epic/commits/${EPIC_VERSION} /tmp/epic.json
        ADD https://api.github.com/repos/eic/juggler/commits/${JUGGLER_VERSION} /tmp/juggler.json
        RUN --mount=type=cache,target=/ccache,id=${TARGETPLATFORM} \
        --mount=type=cache,target=/var/cache/spack \
        --mount=type=secret,id=mirrors,target=/opt/spack/etc/spack/mirrors.yaml \
        <<EOF
        set -e
        export CCACHE_DIR=/ccache
        # Concretization (custom environment)
        RUN <<EOF
        spack env activate --dir ${SPACK_ENV}
        if [ "${EDM4EIC_VERSION}" != "8aeb507f93a93257c99985efbce0ec1371e0b331" ] ; then
        export EDM4EIC_VERSION=$(jq -r .sha /tmp/edm4eic.json)
        ......@@ -207,19 +179,88 @@ if [ "${JUGGLER_VERSION}" != "df87bf1f8643afa8e80bece9d36d6dc26dfe8132" ] ; then
        spack deconcretize -y --all juggler
        fi
        spack concretize --fresh --force
        EOF
        ## ========================================================================================
        ## builder_installation_custom
        ## - builder base with installation of custom versions
        ## ========================================================================================
        FROM builder_concretization_custom as builder_installation_custom
        ARG TARGETPLATFORM
        # Installation (custom environment)
        RUN --mount=type=cache,target=/ccache,id=${TARGETPLATFORM} \
        --mount=type=cache,target=/var/cache/spack \
        --mount=type=secret,id=mirrors,target=/opt/spack/etc/spack/mirrors.yaml \
        <<EOF
        set -e
        export CCACHE_DIR=/ccache
        make --jobs ${jobs} --keep-going --directory /opt/spack-environment \
        SPACK_ENV=${SPACK_ENV} \
        BUILDCACHE_OCI_FINAL="eicweb"
        BUILDCACHE_OCI_PROMPT="eicweb" \
        BUILDCACHE_OCI_FINAL="ghcr"
        spack gc --yes-to-all
        spack find --long --no-groups \
        | sed -e '1,/Installed packages/d;s/\([^@]*\).*/\1/g' \
        | uniq -D -f1 | grep -v -w -e "\(epic\|py-pip\|py-cython\)" \
        | tee /tmp/duplicates.txt
        test -s /tmp/duplicates.txt && ( cat /tmp/duplicates.txt | while read hash spec ; do spack spec --long /${hash} ; done ) && exit 1
        if [ -s /tmp/duplicates.txt ] ; then
        echo "Duplicate packages found"
        cat /tmp/duplicates.txt | while read hash spec ; do spack spec --long /${hash} ; done
        exit 1
        fi
        ccache --show-stats
        ccache --zero-stats
        EOF
        ## ========================================================================================
        ## runtime_concretization_custom
        ## - runtime base with concretization of custom versions (taken from equivalent builder)
        ## ========================================================================================
        FROM runtime_installation_default as runtime_concretization_custom
        COPY --from=builder_installation_custom \
        /opt/spack-environment/${ENV}/spack.* \
        /opt/spack-environment/${ENV}/
        COPY --from=builder_installation_custom \
        /opt/spack-environment/packages.yaml \
        /opt/spack-environment/
        ## ========================================================================================
        ## runtime_installation_custom
        ## - runtime base with installation of custom versions (buildcache populated by builder)
        ## ========================================================================================
        FROM runtime_concretization_custom as runtime_installation_custom
        ARG TARGETPLATFORM
        # Installation (default environment, from buildcache)
        RUN --mount=type=cache,target=/var/cache/spack \
        --mount=type=secret,id=mirrors,target=/opt/spack/etc/spack/mirrors.yaml \
        <<EOF
        set -e
        make --jobs ${jobs} --keep-going --directory /opt/spack-environment \
        SPACK_ENV=${SPACK_ENV} SPACK_INSTALL_FLAGS="--use-buildcache only"
        spack gc --yes-to-all
        spack find --long --no-groups \
        | sed -e '1,/Installed packages/d;s/\([^@]*\).*/\1/g' \
        | uniq -D -f1 | grep -v -w -e "\(epic\|py-pip\|py-cython\)" \
        | tee /tmp/duplicates.txt
        if [ -s /tmp/duplicates.txt ] ; then
        echo "Duplicate packages found"
        cat /tmp/duplicates.txt | while read hash spec ; do spack spec --long /${hash} ; done
        exit 1
        fi
        EOF
        ## ========================================================================================
        ## final image, based on runtime_installation_custom
        ## ========================================================================================
        FROM runtime_installation_custom
        ARG TARGETPLATFORM
        ## Create views at /opt/local and /opt/detector
        RUN <<EOF
        set -e
        ......@@ -228,10 +269,24 @@ sed -i -e '/view: false/d' ${SPACK_ENV}/spack.yaml
        cat /opt/spack-environment/view.yaml >> ${SPACK_ENV}/spack.yaml
        spack -e ${SPACK_ENV} env view regenerate /opt/local
        spack -e ${SPACK_ENV} env view regenerate /opt/detector
        # ensure /opt/local is the view, not a symlink
        rm -rf /opt/local /opt/detector
        LOCAL_PREFIX_PATH=$(realpath $(ls /opt/._local/ | tail -n1))
        mv /opt/._local/${LOCAL_PREFIX_PATH} /opt/local
        ln -s /opt/local /opt/._local/${LOCAL_PREFIX_PATH}
        DETECTOR_PREFIX_PATH=$(realpath $(ls /opt/._detector/ | tail -n1))
        mv /opt/._detector/${DETECTOR_PREFIX_PATH} /opt/detector
        ln -s /opt/detector /opt/._detector/${DETECTOR_PREFIX_PATH}
        EOF
        ## Link minio client into /opt/local/bin/mc
        RUN <<EOF
        ln -sf /opt/mc/bin/mc /opt/local/bin/mc
        EOF
        ## Place cvmfs catalogs
        RUN <<EOF
        set -e
        touch ${SPACK_ROOT}/.cvmfscatalog
        touch /opt/software/.cvmfscatalog
        find /opt/software -mindepth 2 -maxdepth 3 -type d -exec touch {}/.cvmfscatalog \;
        ......@@ -246,6 +301,7 @@ EOF
        ## Fixup /opt/detector/epic-git.fcf90937193c983c0af2acf1251e01f2e2c3a259_main
        RUN <<EOF
        set -e
        shopt -s nullglob
        cd /opt/detector
        for detector in epic-git.*_* ; do
        ......@@ -255,7 +311,7 @@ EOF
        ## Fill jug_info
        RUN <<EOF
        set -ex
        set -e
        spack debug report | sed "s/^/ - /" | sed "s/\* \*\*//" | sed "s/\*\*//" >> /etc/jug_info
        spack find --no-groups --long --variants | sed "s/^/ - /" >> /etc/jug_info
        spack graph --dot > /opt/spack-environment/env.dot
        ......@@ -270,65 +326,6 @@ COPY profile.d/a00_cleanup.sh /etc/profile.d
        COPY profile.d/z11_jug_env.sh /etc/profile.d
        COPY singularity.d /.singularity.d
        ## Add minio client into /opt/local/bin
        ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-amd64/mc /opt/local/bin/mc-amd64
        ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-arm64/mc /opt/local/bin/mc-arm64
        RUN <<EOF
        set -ex
        declare -A target=(["linux/amd64"]="amd64" ["linux/arm64"]="arm64")
        mv /opt/local/bin/mc-${target[${TARGETPLATFORM}]} /opt/local/bin/mc
        unset target[${TARGETPLATFORM}]
        for t in ${target[*]} ; do
        rm /opt/local/bin/mc-${t}
        done
        EOF
        ## make sure we have the entrypoints setup correctly
        ENTRYPOINT []
        CMD ["bash", "--rcfile", "/etc/profile", "-l"]
        USER 0
        WORKDIR /
        ## ========================================================================================
        ## STAGE 2
        ## Lean target image
        ## ========================================================================================
        FROM ${DOCKER_REGISTRY}${RUNTIME_IMAGE}:${INTERNAL_TAG} as runtime
        ARG TARGETPLATFORM
        LABEL maintainer="Sylvester Joosten <sjoosten@anl.gov>" \
        name="jug_xl" \
        march="$TARGETPLATFORM"
        ## copy over everything we need from builder
        COPY --from=builder /opt/spack /opt/spack
        COPY --from=builder /opt/spack-environment /opt/spack-environment
        COPY --from=builder /opt/software /opt/software
        COPY --from=builder /opt/._local /opt/._local
        COPY --from=builder /opt/._detector /opt/._detector
        COPY --from=builder /etc/profile.d /etc/profile.d
        COPY --from=builder /etc/jug_info /etc/jug_info
        COPY --from=builder /etc/eic-env.sh /etc/eic-env.sh
        COPY --from=builder /.singularity.d /.singularity.d
        COPY --from=builder /usr/bin/docker-shell /usr/bin/docker-shell
        ## Use spack entrypoint. NOTE: Requires `set -ex` in all multi-line scripts!
        ENV SPACK_ROOT=/opt/spack
        SHELL ["docker-shell"]
        ## ensure /opt/local is the view, not a symlink
        RUN <<EOF
        set -ex
        rm -rf /opt/local /opt/detector
        LOCAL_PREFIX_PATH=$(realpath $(ls /opt/._local/ | tail -n1))
        mv /opt/._local/${LOCAL_PREFIX_PATH} /opt/local
        ln -s /opt/local /opt/._local/${LOCAL_PREFIX_PATH}
        DETECTOR_PREFIX_PATH=$(realpath $(ls /opt/._detector/ | tail -n1))
        mv /opt/._detector/${DETECTOR_PREFIX_PATH} /opt/detector
        ln -s /opt/detector /opt/._detector/${DETECTOR_PREFIX_PATH}
        EOF
        ## set ROOT TFile forward compatibility
        RUN sed --in-place --follow-symlinks 's/# \(TFile.v630forwardCompatibility:\) no/\1 yes/' /opt/local/etc/root/system.rootrc
        ......@@ -342,7 +339,7 @@ RUN ldconfig
        ## set the local spack configuration
        ENV SPACK_DISABLE_LOCAL_CONFIG="true"
        RUN <<EOF
        set -ex
        set -e
        spack config --scope site add "config:install_tree:root:~/spack"
        spack config --scope site add "config:source_cache:~/.spack/cache"
        spack config --scope site add "config:binary_index_root:~/.spack"
        ......@@ -371,7 +368,7 @@ ADD ${EICWEB}/399/repository/commits/${BENCHMARK_DET_VERSION} /tmp/399.json
        ADD ${EICWEB}/408/repository/commits/${BENCHMARK_REC_VERSION} /tmp/408.json
        ADD ${EICWEB}/400/repository/commits/${BENCHMARK_PHY_VERSION} /tmp/400.json
        RUN <<EOF
        set -ex
        set -e
        mkdir -p /opt/benchmarks
        cd /opt/benchmarks
        git clone --filter=tree:0 -b ${BENCHMARK_COM_VERSION} --depth 1 https://eicweb.phy.anl.gov/EIC/benchmarks/common_bench.git
        ......@@ -394,7 +391,7 @@ ADD https://api.github.com/repos/eic/simulation_campaign_hepmc3/commits/${CAMPAI
        ADD https://api.github.com/repos/eic/job_submission_condor/commits/${CAMPAIGNS_CONDOR_VERSION} /tmp/job_submission_condor.json
        ADD https://api.github.com/repos/eic/job_submission_slurm/commits/${CAMPAIGNS_SLURM_VERSION} /tmp/job_submission_slurm.json
        RUN <<EOF
        set -ex
        set -e
        mkdir -p /opt/campaigns
        cd /opt/campaigns
        git clone --filter=tree:0 -b ${CAMPAIGNS_SINGLE_VERSION} --depth 1 https://github.com/eic/simulation_campaign_single.git single
        ......
        0% or .
        You are about to add 0 people to the discussion. Proceed with caution.
        Please register or to comment