Skip to content
Snippets Groups Projects
Commit ff2205a2 authored by Ujwal Kundur's avatar Ujwal Kundur Committed by Wouter Deconinck
Browse files

Repo Cleanup

parent b7217126
Branches
No related tags found
1 merge request!309Repo Cleanup
...@@ -70,23 +70,23 @@ workflow: ...@@ -70,23 +70,23 @@ workflow:
## Workflows: ## Workflows:
## - master --> config + all build stages + singularity ## - master --> config + all build stages + singularity
# + nightly build:jug + nightly singularity # + nightly build:jug + nightly singularity
## - v3.0-stable --> config + all build stages + singularity ## - v4.0-stable --> config + all build stages + singularity
## - v3.0.0 --> config + all build stages + singularity ## - v4.0.0 --> config + all build stages + singularity
## - MR --> config + all build stages ## - MR --> config + all build stages
## ##
## Container images tags ## Container images tags
## - master --> testing ## - master --> testing
## - <nightly> --> nightly (run as part of master) ## - <nightly> --> nightly (run as part of master)
## - v3.0-stable --> 3.0-stable ## - v4.0-stable --> 4.0-stable
## - v3.0.0 --> 3.0-stable, 3.0.0 ## - v4.0.0 --> 4.0-stable, 4.0.0
## - acadia ## - acadia
## - MR --> 3.0-unstable (on all registries) ## - MR --> 4.0-unstable (on all registries)
## --> unstable-mr-XXX (on eicweb only, untag at end of pipeline) ## --> unstable-mr-XXX (on eicweb only, untag at end of pipeline)
## - all other --> do nothing ## - all other --> do nothing
## ##
## caching strategy for dispatch to multiple nodes and to avoid ## caching strategy for dispatch to multiple nodes and to avoid
## --> try this strategy: https://medium.com/titansoft-engineering/docker-build-cache-sharing-on-multi-hosts-with-buildkit-and-buildx-eb8f7005918e ## --> try this strategy: https://medium.com/titansoft-engineering/docker-build-cache-sharing-on-multi-hosts-with-buildkit-and-buildx-eb8f7005918e
## (first try with buildx didn't pan out, let's try again later)
version: version:
stage: config stage: config
...@@ -201,7 +201,7 @@ cuda_base:default: ...@@ -201,7 +201,7 @@ cuda_base:default:
BUILD_IMAGE: "cuda_base" BUILD_IMAGE: "cuda_base"
script: script:
- docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG} - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}
containers/oneapi containers/cuda
- !reference [.build, script] - !reference [.build, script]
jug_dev:default: jug_dev:default:
......
...@@ -17,7 +17,7 @@ ENV CLICOLOR_FORCE=1 \ ...@@ -17,7 +17,7 @@ ENV CLICOLOR_FORCE=1 \
## Install additional packages. Remove the auto-cleanup functionality ## Install additional packages. Remove the auto-cleanup functionality
## for docker, as we're using the new buildkit cache instead. ## for docker, as we're using the new buildkit cache instead.
## We also install gitlab-runner, from the buster package (as bullseye is not available atm) ## We also install gitlab-runner, from the bullseye package (as bookworm is not available atm)
## TODO: libyaml-cpp-dev is a dependency for afterburner. We can probably remove ## TODO: libyaml-cpp-dev is a dependency for afterburner. We can probably remove
## this once afterburner is added to spack ## this once afterburner is added to spack
RUN --mount=type=cache,target=/var/cache/apt \ RUN --mount=type=cache,target=/var/cache/apt \
...@@ -70,7 +70,7 @@ RUN --mount=type=cache,target=/var/cache/apt \ ...@@ -70,7 +70,7 @@ RUN --mount=type=cache,target=/var/cache/apt \
&& curl -L \ && curl -L \
"https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" \ "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" \
| bash \ | bash \
&& sed -i "s/bookworm/buster/" \ && sed -i "s/bookworm/bullseye/" \
/etc/apt/sources.list.d/runner_gitlab-runner.list \ /etc/apt/sources.list.d/runner_gitlab-runner.list \
&& apt-get -yqq update \ && apt-get -yqq update \
&& apt-get -yqq install --no-install-recommends \ && apt-get -yqq install --no-install-recommends \
......
...@@ -8,7 +8,7 @@ ARG INTERNAL_TAG="testing" ...@@ -8,7 +8,7 @@ ARG INTERNAL_TAG="testing"
## ======================================================================================== ## ========================================================================================
FROM ${DOCKER_REGISTRY}debian_base:${INTERNAL_TAG} as builder FROM ${DOCKER_REGISTRY}debian_base:${INTERNAL_TAG} as builder
## instal some extra spack dependencies ## install some extra spack dependencies
RUN --mount=type=cache,target=/var/cache/apt \ RUN --mount=type=cache,target=/var/cache/apt \
rm -f /etc/apt/apt.conf.d/docker-clean \ rm -f /etc/apt/apt.conf.d/docker-clean \
&& apt-get -yqq update \ && apt-get -yqq update \
...@@ -32,7 +32,7 @@ RUN echo "Part 1: regular spack install (as in containerize)" \ ...@@ -32,7 +32,7 @@ RUN echo "Part 1: regular spack install (as in containerize)" \
git cherry-pick -n $SPACK_CHERRYPICKS ; \ git cherry-pick -n $SPACK_CHERRYPICKS ; \
fi \ fi \
&& cd - \ && cd - \
&& mkdir -p $SPACK_ROOT/opt/spack \ && mkdir -p $SPACK_ROOT \
&& cp -r /tmp/spack-staging/bin $SPACK_ROOT/bin \ && cp -r /tmp/spack-staging/bin $SPACK_ROOT/bin \
&& cp -r /tmp/spack-staging/etc $SPACK_ROOT/etc \ && cp -r /tmp/spack-staging/etc $SPACK_ROOT/etc \
&& cp -r /tmp/spack-staging/lib $SPACK_ROOT/lib \ && cp -r /tmp/spack-staging/lib $SPACK_ROOT/lib \
...@@ -73,6 +73,11 @@ RUN --mount=type=cache,target=/var/cache/spack-mirror \ ...@@ -73,6 +73,11 @@ RUN --mount=type=cache,target=/var/cache/spack-mirror \
&& spack mirror add docker /var/cache/spack-mirror \ && spack mirror add docker /var/cache/spack-mirror \
&& spack mirror list && spack mirror list
## This variable will change whenevery either spack.yaml or our spack package
## overrides change, triggering a rebuild
ARG CACHE_BUST="hash"
ARG CACHE_NUKE=""
## Setup our custom environment and package overrides ## Setup our custom environment and package overrides
COPY spack $SPACK_ROOT/eic-spack COPY spack $SPACK_ROOT/eic-spack
RUN spack repo add --scope site "$SPACK_ROOT/eic-spack" \ RUN spack repo add --scope site "$SPACK_ROOT/eic-spack" \
...@@ -83,10 +88,6 @@ RUN spack repo add --scope site "$SPACK_ROOT/eic-spack" \ ...@@ -83,10 +88,6 @@ RUN spack repo add --scope site "$SPACK_ROOT/eic-spack" \
&& spack env activate . \ && spack env activate . \
&& spack concretize && spack concretize
## This variable will change whenevery either spack.yaml or our spack package
## overrides change, triggering a rebuild
ARG CACHE_BUST="hash"
ARG CACHE_NUKE=""
## Now execute the main build (or fetch from cache if possible) ## Now execute the main build (or fetch from cache if possible)
## note, no-check-signature is needed to allow the quicker signature-less ## note, no-check-signature is needed to allow the quicker signature-less
...@@ -136,7 +137,9 @@ RUN --mount=type=cache,target=/var/cache/pip \ ...@@ -136,7 +137,9 @@ RUN --mount=type=cache,target=/var/cache/pip \
--trusted-host pypi.org \ --trusted-host pypi.org \
--trusted-host files.pythonhosted.org \ --trusted-host files.pythonhosted.org \
--cache-dir /var/cache/pip \ --cache-dir /var/cache/pip \
--requirement /usr/local/etc/requirements.txt --requirement /usr/local/etc/requirements.txt \
--no-warn-script-location
# ^ Supress not on PATH Warnings
## Including some small fixes: ## Including some small fixes:
## - Somehow PODIO env isn't automatically set, ## - Somehow PODIO env isn't automatically set,
...@@ -171,7 +174,7 @@ FROM builder as staging ...@@ -171,7 +174,7 @@ FROM builder as staging
RUN cd /opt/spack-environment && spack env activate . && spack gc -y RUN cd /opt/spack-environment && spack env activate . && spack gc -y
# Strip all the binaries # Strip all the binaries
# This reduces the image by factor of x2, so worth the effort # This reduces the image by factor of x2, so worth the effort
# note that we do not strip python libraries as can cause issues in some cases # note that we do not strip python libraries as it can cause issues in some cases
RUN find -L /usr/local/* \ RUN find -L /usr/local/* \
-type d -name site-packages -prune -false -o \ -type d -name site-packages -prune -false -o \
-type f -not -name "zdll.lib" -not -name libtensorflow-lite.a \ -type f -not -name "zdll.lib" -not -name libtensorflow-lite.a \
......
...@@ -27,7 +27,7 @@ if [ -f /etc/jug_info ]; then ...@@ -27,7 +27,7 @@ if [ -f /etc/jug_info ]; then
esac esac
ps1_preamble="${container}${sigil}> " ps1_preamble="${container}${sigil}> "
export PS1_SIGIL=${sigil} export PS1_SIGIL=${sigil}
unset ${sigil} unset sigil
fi fi
unset version unset version
unset container unset container
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment