dev.Dockerfile

#syntax=docker/dockerfile:1.4
ARG DOCKER_REGISTRY="eicweb/"
ARG BASE_IMAGE="debian_stable_base"
ARG INTERNAL_TAG="testing"

## ========================================================================================
## STAGE0: spack image
## EIC spack image with spack and eic-spack repositories
## ========================================================================================
FROM ${DOCKER_REGISTRY}${BASE_IMAGE}:${INTERNAL_TAG} as spack
ARG TARGETPLATFORM

## install some extra spack dependencies
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${TARGETPLATFORM} \
    --mount=type=cache,target=/var/lib/apt/lists,sharing=locked,id=${TARGETPLATFORM} \
    rm -f /etc/apt/apt.conf.d/docker-clean                              \
 && apt-get -yqq update                                                 \
 && apt-get -yqq install --no-install-recommends                        \
        python3                                                         \
        python3-dev                                                     \
        python3-distutils                                               \
        python3-boto3                                                   \
        python-is-python3

## Setup spack
ENV SPACK_ROOT=/opt/spack
ARG SPACK_ORGREPO="spack/spack"
ARG SPACK_VERSION="releases/v0.20"
ARG SPACK_CHERRYPICKS=""
ADD https://api.github.com/repos/${SPACK_ORGREPO}/commits/${SPACK_VERSION} /tmp/spack.json
RUN git clone --filter=tree:0 https://github.com/${SPACK_ORGREPO}.git ${SPACK_ROOT}     \
 && git -C ${SPACK_ROOT} checkout ${SPACK_VERSION}                      \
 && if [ -n "${SPACK_CHERRYPICKS}" ] ; then                             \
      git -C ${SPACK_ROOT} cherry-pick -n ${SPACK_CHERRYPICKS} ;        \
    fi                                                                  \
 && ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash                \
          /usr/bin/docker-shell                                         \
 && ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash                \
          /usr/bin/interactive-shell                                    \
 && ln -s $SPACK_ROOT/share/spack/docker/entrypoint.bash                \
          /usr/bin/spack-env

SHELL ["docker-shell"]

## Setup build configuration
ARG jobs=64
RUN declare -A target=(                                                 \
      ["linux/amd64"]="x86_64_v2"                                       \
      ["linux/arm64"]="aarch64"                                         \
    )                                                                   \
 && target=${target[${TARGETPLATFORM}]}                                 \
 && spack config --scope site add "packages:all:require:[target=${target}]" \
 && spack config blame packages                                         \
 && spack config --scope user add "config:suppress_gpg_warnings:true"   \
 && spack config --scope user add "config:build_jobs:${jobs}"           \
 && spack config --scope user add "config:db_lock_timeout:${jobs}0"     \
 && spack config --scope user add "config:source_cache:/var/cache/spack" \
 && spack config --scope user add "config:install_tree:root:/opt/software" \
 && spack config --scope user add "config:ccache:true"                  \
 && spack config blame config                                           \
 && spack compiler find --scope site                                    \
 && spack config blame compilers

## Setup local buildcache mirrors
RUN --mount=type=cache,target=/var/cache/spack                          \
    spack mirror add local /var/cache/spack/mirror/${SPACK_VERSION}     \
 && spack buildcache update-index local                                 \
 && spack mirror list

## Setup eics3 buildcache mirrors
## - this always adds the read-only mirror to the container
## - the write-enabled mirror is provided later as a secret mount
ARG S3_ACCESS_KEY=""
ARG S3_SECRET_KEY=""
RUN --mount=type=cache,target=/var/cache/spack                          \
    if [ -n "${S3_ACCESS_KEY}" ] ; then                                 \
    spack mirror add --scope site                                       \
      --s3-endpoint-url https://eics3.sdcc.bnl.gov:9000                 \
      --s3-access-key-id "${S3_ACCESS_KEY}"                             \
      --s3-access-key-secret "${S3_SECRET_KEY}"                         \
      eics3 s3://eictest/EPIC/spack/${SPACK_VERSION}                    \
    ; fi                                                                \
 && spack mirror list

## Setup eic-spack
ENV EICSPACK_ROOT=${SPACK_ROOT}/var/spack/repos/eic-spack
ARG EICSPACK_ORGREPO="eic/eic-spack"
ARG EICSPACK_VERSION="$SPACK_VERSION"
ARG EICSPACK_CHERRYPICKS=""
ADD https://api.github.com/repos/${EICSPACK_ORGREPO}/commits/${EICSPACK_VERSION} /tmp/eic-spack.json
RUN git clone --filter=tree:0 https://github.com/${EICSPACK_ORGREPO}.git ${EICSPACK_ROOT} \
 && git -C ${EICSPACK_ROOT} checkout ${EICSPACK_VERSION}                \
 && if [ -n "${EICSPACK_CHERRYPICKS}" ] ; then                          \
      git -C ${EICSPACK_ROOT} cherry-pick -n ${EICSPACK_CHERRYPICKS} ;  \
    fi                                                                  \
 && spack repo add --scope site "${EICSPACK_ROOT}"

## ========================================================================================
## STAGE1: builder
## EIC builder image with spack environment
## ========================================================================================
FROM spack as builder

## Setup our custom environment (secret mount for write-enabled mirror)
COPY --from=spack-environment . /opt/spack-environment/
ARG ENV=dev
ENV SPACK_ENV=/opt/spack-environment/${ENV}
RUN --mount=type=cache,target=/ccache,id=${TARGETPLATFORM}              \
    --mount=type=cache,target=/var/cache/spack                          \
    --mount=type=secret,id=mirrors,target=/opt/spack/etc/spack/mirrors.yaml \
    source $SPACK_ROOT/share/spack/setup-env.sh                         \
 && export CCACHE_DIR=/ccache                                           \
 && spack env activate --dir ${SPACK_ENV}                               \
 && make --jobs ${jobs} --keep-going --directory /opt/spack-environment \
    SPACK_ENV=${SPACK_ENV}                                              \
    BUILDCACHE_MIRROR="local eics3rw"                                   \
 && ccache --show-stats

## Create view at /usr/local
RUN --mount=type=cache,target=/var/cache/spack                          \
    source $SPACK_ROOT/share/spack/setup-env.sh                         \
 && spack env activate --dir ${SPACK_ENV}                               \
 && rm -r /usr/local                                                    \
 && spack env view enable /usr/local

## Optional, nuke the buildcache after install, before (re)caching
## This is useful when going to completely different containers,
## or intermittently to keep the buildcache step from taking too much time
ARG CACHE_NUKE=""
RUN --mount=type=cache,target=/var/cache/spack,sharing=locked           \
    [ -z "${CACHE_NUKE}" ]                                              \
    || rm -rf /var/cache/spack/mirror/${SPACK_VERSION}/build_cache/*

## Extra post-spack steps:
##   - Python packages
COPY requirements.txt /usr/local/etc/requirements.txt
RUN --mount=type=cache,target=/var/cache/pip,sharing=locked,id=${TARGETPLATFORM} \
    echo "Installing additional python packages"                        \
 && source $SPACK_ROOT/share/spack/setup-env.sh                         \
 && spack env activate --dir ${SPACK_ENV}                               \
 && python -m pip install                                               \
    --trusted-host pypi.org                                             \
    --trusted-host files.pythonhosted.org                               \
    --cache-dir /var/cache/pip                                          \
    --requirement /usr/local/etc/requirements.txt                       \
    --no-warn-script-location
    # ^ Supress not on PATH Warnings

## Including some small fixes
RUN echo "Grabbing environment info"                                    \
 && spack env activate --sh --dir ${SPACK_ENV}                          \
    > /etc/profile.d/z10_spack_environment.sh

## make sure we have the entrypoints setup correctly
ENTRYPOINT []
CMD ["bash", "--rcfile", "/etc/profile", "-l"]
USER 0
WORKDIR /

## ========================================================================================
## STAGE 2: staging image with unnecessariy packages removed and stripped binaries
## ========================================================================================
FROM builder as staging

# Garbage collect in environment
RUN spack -e ${SPACK_ENV} gc -y

# Garbage collect in git
RUN du -sh $SPACK_ROOT                                                  \
 && git -C $SPACK_ROOT fetch --depth=1                                  \
 && git -C $SPACK_ROOT gc --prune=all --aggressive                      \
 && du -sh $SPACK_ROOT

## Bugfix to address issues loading the Qt5 libraries on Linux kernels prior to 3.15
## See
#https://askubuntu.com/questions/1034313/ubuntu-18-4-libqt5core-so-5-cannot-open-shared-object-file-no-such-file-or-dir
## and links therin for more info
RUN if [ -f /usr/local/lib/libQt5Core.so ] ; then                       \
      strip --remove-section=.note.ABI-tag /usr/local/lib/libQt5Core.so;\
    fi

RUN spack debug report                                                  \
      | sed "s/^/ - /" | sed "s/\* \*\*//" | sed "s/\*\*//"             \
    >> /etc/jug_info                                                    \
 && spack find --no-groups --long --variants | sed "s/^/ - /" >> /etc/jug_info \
 && spack graph --dot --installed > /opt/spack-environment/env.dot


COPY eic-shell /usr/local/bin/eic-shell
COPY eic-info /usr/local/bin/eic-info
COPY entrypoint.sh /usr/local/sbin/entrypoint.sh
COPY eic-env.sh /etc/eic-env.sh
COPY profile.d/a00_cleanup.sh /etc/profile.d
COPY profile.d/z11_jug_env.sh /etc/profile.d
COPY singularity.d /.singularity.d

## Add minio client into /usr/local/bin
ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-amd64/mc /usr/local/bin/mc-amd64
ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-arm64/mc /usr/local/bin/mc-arm64
RUN declare -A target=(                                                 \
      ["linux/amd64"]="amd64"                                           \
      ["linux/arm64"]="arm64"                                           \
    )                                                                   \
 && mv /usr/local/bin/mc-${target[${TARGETPLATFORM}]} /usr/local/bin/mc \
 && unset target[${TARGETPLATFORM}]                                     \
 && for t in ${target[*]} ; do                                          \
      rm /usr/local/bin/mc-${t} ;                                       \
    done

## ========================================================================================
## STAGE 3
## Lean target image
## ========================================================================================
FROM ${DOCKER_REGISTRY}${BASE_IMAGE}:${INTERNAL_TAG} as export
ARG TARGETPLATFORM

LABEL maintainer="Sylvester Joosten <sjoosten@anl.gov>" \
      name="jug_xl" \
      march="$TARGETPLATFORM"

## copy over everything we need from staging
COPY --from=staging /opt/spack /opt/spack
COPY --from=staging /opt/spack-environment /opt/spack-environment
COPY --from=staging /opt/software /opt/software
COPY --from=staging /usr/._local /usr/._local
COPY --from=staging /etc/profile.d /etc/profile.d
COPY --from=staging /etc/jug_info /etc/jug_info
COPY --from=staging /etc/eic-env.sh /etc/eic-env.sh
COPY --from=staging /.singularity.d /.singularity.d

## ensure /usr/local link is pointing to the right view
RUN rm -rf /usr/local                                                   \
 && PREFIX_PATH=$(realpath $(ls /usr/._local/ | tail -n1))              \
 && echo "Found spack true prefix path to be $PREFIX_PATH"              \
 && ln -s /usr/._local/${PREFIX_PATH} /usr/local

## set the local spack configuration
ENV SPACK_DISABLE_LOCAL_CONFIG="true"
RUN . /opt/spack/share/spack/setup-env.sh                               \
 && spack config --scope site add "config:install_tree:root:~/spack"    \
 && spack config --scope site add "config:source_cache:~/.spack/cache"  \
 && spack config --scope site add "config:binary_index_root:~/.spack"   \
 && spack config --scope site add "config:environments_root:~/.spack/env" \
 && spack config --scope site add "config:suppress_gpg_warnings:true"   \
 && spack config blame config                                           \
 && spack config --scope site add "upstreams:eic-shell:install_tree:/opt/software" \
 && spack config blame upstreams

## set the jug_dev version and add the afterburner
ARG JUG_VERSION=1
RUN echo "" >> /etc/jug_info                                            \
 && echo " - jug_dev: ${JUG_VERSION}" >> /etc/jug_info

## make sure we have the entrypoints setup correctly
ENTRYPOINT ["/usr/local/sbin/entrypoint.sh"]
CMD ["bash", "--rcfile", "/etc/profile", "-l"]
USER 0
WORKDIR /
SHELL ["/usr/local/bin/eic-shell"]