Compare revisions

Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck
--- a/.github/workflows/mirror.yaml
+++ b/.github/workflows/mirror.yaml
+name: Mirror and Trigger EICweb
+
+on:
+  pull_request:
+  push:
+  workflow_dispatch:
+
+concurrency:
+  group: mirror
+  cancel-in-progress: false
+
+jobs:
+  build:
+    name: Mirror and Trigger EICweb
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      contents: read
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Push to EICweb
+      uses: eic/gitlab-sync@master
+      with:
+        url: https://eicweb.phy.anl.gov/containers/eic_container.git/
+        token: ${{ secrets.GITLAB_TOKEN }}
+        username: ${{ secrets.GITLAB_USERNAME }}
+        ciskip: true
+    - name: Trigger EICweb
+      if: github.event_name == 'pull_request'
+      uses: eic/trigger-gitlab-ci@v2
+      with:
+        url: https://eicweb.phy.anl.gov
+        project_id: 290
+        token: ${{ secrets.EICWEB_EIC_CONTAINER_TRIGGER }}
+        ref_name: ${{ github.event.pull_request.head.ref || github.ref }}
+        variables:
+          GITHUB_REPOSITORY=${{ github.repository }}
+          GITHUB_SHA=${{ github.event.pull_request.head.sha || github.sha }}
+          GITHUB_PR=${{ github.event.pull_request.number }}
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,13 +2,13 @@ image: alpine

 variables:
  ## External images
-  DOCKER_IMAGE: docker.io/docker:24.0.5
-  SINGULARITY_IMAGE: quay.io/singularity/singularity:v3.11.4
+  DOCKER_IMAGE: docker.io/docker:24.0.7
+  SINGULARITY_IMAGE: quay.io/singularity/singularity:v3.11.5

-  ## Application versions used for the main release 
+  ## Default versions are specified in packages.yaml but can be overridden
  ## note: nightly builds will always use the master/main branch
-  JUGGLER_VERSION: "v11.0.0"
-  EICRECON_VERSION: "v1.9.0"
+  JUGGLER_VERSION: ""
+  EICRECON_VERSION: ""

  ## Local registry
  CI_PUSH: 1
@@ -21,11 +21,6 @@ variables:
  GH_REGISTRY_USER: eic
  GH_PUSH: 1

-  ## TLS error resiliency: number of retries and second wait between tries 
-  ## (wait time is doubled with each attempt)
-  DOCKER_NTRIES: 5
-  DOCKER_WAIT_TIME: 5
-
  ## Number of jobs to start during container builds
  JOBS: 32

@@ -62,6 +57,8 @@ workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_PIPELINE_SOURCE == "web"'
+    - if: '$CI_PIPELINE_SOURCE == "webide"'
+    - if: '$CI_PIPELINE_SOURCE == "trigger"'
    - if: '$CI_COMMIT_BRANCH == "master"'
    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-(alpha|beta|stable)/'          ## main stable branch: vX.Y-stable
    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+-[a-z]+-(alpha|beta|stable)/'   ## special stable branch: vX.Y-acadia-stable (etc)
@@ -159,19 +156,19 @@ status:pending:
 ## base job settings for all docker interactions
 .docker:
  image: ${DOCKER_IMAGE}
-  services: 
+  services:
    - docker:dind
  before_script:
    - # Login to registries
      if [[ -n "${EXPORT_TAG}" ]] ; then
        if [[ -n "${DH_REGISTRY}" ]] ; then
-          docker login -u ${DH_REGISTRY_USER} -p ${DH_EICWEB_TOKEN} ${DH_REGISTRY} ;
+          echo ${DH_EICWEB_TOKEN} | docker login -u ${DH_REGISTRY_USER} --password-stdin ${DH_REGISTRY} ;
        fi ;
        if [[ -n "${GH_REGISTRY}" ]] ; then
-          docker login -u ${GITHUB_REGISTRY_USER} -p ${GITHUB_REGISTRY_TOKEN} ${GH_REGISTRY} ;
+          echo ${GITHUB_REGISTRY_TOKEN} | docker login -u ${GITHUB_REGISTRY_USER} --password-stdin ${GH_REGISTRY} ;
        fi ;
        if [[ -n "${CI_REGISTRY}" ]] ; then
-          docker login -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY} ;
+          echo ${CI_REGISTRY_PASSWORD} | docker login -u ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY} ;
        fi ;
      fi
  after_script:
@@ -271,7 +268,7 @@ jug_dev:
        - nightly
        BASE_IMAGE: debian_stable_base
        BUILD_IMAGE: jug_
-        PLATFORM: linux/amd64,linux/arm64/v8
+        PLATFORM: linux/amd64
        ENV:
        - dev
        - prod
@@ -337,8 +334,12 @@ jug_dev:
                   --build-arg S3_SECRET_KEY=${S3_SECRET_KEY}
                   --build-arg JUG_VERSION=${EXPORT_TAG}-${BUILD_TYPE}-$(git rev-parse HEAD)
                   ${IF_BUILD_DEFAULT:+
-                     --build-arg JUGGLER_VERSION=${JUGGLER_VERSION}
-                     --build-arg EICRECON_VERSION=${EICRECON_VERSION}
+                     ${JUGGLER_VERSION:+--build-arg JUGGLER_VERSION=${JUGGLER_VERSION}}
+                     ${EICRECON_VERSION:+--build-arg EICRECON_VERSION=${EICRECON_VERSION}}
+                   }
+                   ${IF_BUILD_NIGHTLY:+
+                     --build-arg JUGGLER_VERSION=main
+                     --build-arg EICRECON_VERSION=main
                   }
                   --build-arg ENV=${ENV}
                   --build-arg jobs=${JOBS}
@@ -368,7 +369,7 @@ jug_xl:default:
          - default
          BASE_IMAGE: debian_stable_base
          BUILD_IMAGE: jug_
-          PLATFORM: linux/amd64,linux/arm64/v8
+          PLATFORM: linux/amd64
          ENV:
          - dev
  script:
@@ -417,7 +418,7 @@ jug_xl:nightly:
          - nightly
          BASE_IMAGE: debian_stable_base
          BUILD_IMAGE: jug_
-          PLATFORM: linux/amd64,linux/arm64/v8
+          PLATFORM: linux/amd64
          ENV:
          - dev
  script:
@@ -527,7 +528,50 @@ benchmarks:detector:nightly:
    project: EIC/benchmarks/detector_benchmarks
    strategy: depend
  allow_failure: false
-  
+
+df:
+  extends: .build
+  stage: config
+  tags:
+    - docker-new
+  needs: 
+    - version
+  script:
+    - docker images
+    - docker ps --size
+    - docker stats --all --no-stream --no-trunc > docker-stats-all.txt
+    - docker context ls
+    - docker volume ls
+    - docker volume ls --filter=name=buildx_buildkit_builder
+    - docker buildx ls
+    - docker buildx inspect
+    - docker buildx du
+    - docker buildx du --verbose > docker-buildx-du-verbose.txt
+    - docker system df
+    - docker system df --verbose > docker-system-df-verbose.txt
+  artifacts:
+    paths:
+      - docker-stats-all.txt
+      - docker-buildx-du-verbose.txt
+      - docker-system-df-verbose.txt
+
+prune: 
+  extends: .build
+  stage: finalize
+  tags:
+    - docker-new
+  needs:
+    - clean_internal_tag
+    - clean_unstable_mr
+    - clean_pipeline
+  rules:
+    - when: manual
+  script:
+    - docker buildx prune --all --force --filter=until=24h --verbose
+    - docker system prune --all --force --filter=until=24h
+    - docker system prune --volumes --force
+  allow_failure: true
+
 clean_internal_tag:
  image: alpine/curl
  stage: finalize
@@ -567,9 +611,6 @@ clean_unstable_mr:
        if [ "$state" == "merged" ] ; then docker rmi $repository:$tag ; fi ;
        if [ "$state" == "closed" ] ; then docker rmi $repository:$tag ; fi ;
      done
-    - docker system prune --filter until=24h --force
-    - docker system df
-    - docker images

 clean_pipeline:
  extends: .docker
@@ -600,9 +641,6 @@ clean_pipeline:
        if [ "$status" == "failed" ] ; then docker rmi $repository:$tag ; fi ;
        if [ "$status" == "canceled" ] ; then docker rmi $repository:$tag ; fi ;
      done
-    - docker system prune --filter until=24h --force
-    - docker system df
-    - docker images

 status:success:
  stage: status-report

--- a/containers/debian/base.Dockerfile
+++ b/containers/debian/base.Dockerfile
@@ -34,6 +34,7 @@ apt-get -yqq install --no-install-recommends                            \
        ccache                                                          \
        curl                                                            \
        file                                                            \
+        gawk                                                            \
        gdb                                                             \
        ghostscript                                                     \
        git                                                             \

--- a/containers/jug/dev.Dockerfile
+++ b/containers/jug/dev.Dockerfile
@@ -71,7 +71,7 @@ spack compiler find --scope site
 spack config blame compilers
 EOF

-## Setup eics3 buildcache mirrors
+## Setup buildcache mirrors
 ## - this always adds the read-only mirror to the container
 ## - the write-enabled mirror is provided later as a secret mount
 ARG S3_ACCESS_KEY=""
@@ -79,12 +79,14 @@ ARG S3_SECRET_KEY=""
 RUN --mount=type=cache,target=/var/cache/spack <<EOF
 set -e
 if [ -n "${S3_ACCESS_KEY}" ] ; then
-  spack mirror add --scope site                                         \
+  spack mirror add --scope site --unsigned                              \
      --s3-endpoint-url https://eics3.sdcc.bnl.gov:9000                 \
      --s3-access-key-id "${S3_ACCESS_KEY}"                             \
      --s3-access-key-secret "${S3_SECRET_KEY}"                         \
      eics3 s3://eictest/EPIC/spack/${SPACK_VERSION}
 fi
+spack mirror add --scope site --signed spack-${SPACK_VERSION} https://binaries.spack.io/${SPACK_VERSION}
+spack mirror add --scope site --unsigned ghcr-${SPACK_VERSION} oci://ghcr.io/eic/spack-${SPACK_VERSION}
 spack mirror list
 EOF

@@ -123,13 +125,9 @@ EOF
 ## ========================================================================================
 FROM spack as builder

-## Setup our custom environment (secret mount for write-enabled mirror)
+## 1. Setup our default environment (secret mount for write-enabled mirror)
 COPY --from=spack-environment . /opt/spack-environment/
 ARG ENV=dev
-ARG JUGGLER_VERSION="main"
-ADD https://eicweb.phy.anl.gov/api/v4/projects/EIC%2Fjuggler/repository/commits/${JUGGLER_VERSION} /tmp/juggler.json
-ARG EICRECON_VERSION="main"
-ADD https://api.github.com/repos/eic/eicrecon/commits/${EICRECON_VERSION} /tmp/eicrecon.json
 ENV SPACK_ENV=/opt/spack-environment/${ENV}
 RUN --mount=type=cache,target=/ccache,id=${TARGETPLATFORM}              \
    --mount=type=cache,target=/var/cache/spack                          \
@@ -137,34 +135,62 @@ RUN --mount=type=cache,target=/ccache,id=${TARGETPLATFORM}              \
    <<EOF
 set -e
 export CCACHE_DIR=/ccache
+source ${SPACK_ROOT}/share/spack/setup-env.sh
+mkdir -p /var/cache/spack/blobs/sha256/
 find /var/cache/spack/blobs/sha256/ -atime +7 -delete
-JUGGLER_VERSION=$(jq -r .id /tmp/juggler.json)
-EICRECON_VERSION=$(jq -r .sha /tmp/eicrecon.json)
 spack buildcache update-index eics3rw
 spack env activate --dir ${SPACK_ENV}
-spack add juggler@git.${JUGGLER_VERSION}
-spack add eicrecon@git.${EICRECON_VERSION}
 spack concretize --fresh --force --quiet
-make --jobs ${jobs} --keep-going --directory /opt/spack-environment SPACK_ENV=${SPACK_ENV} BUILDCACHE_MIRROR_ONLY_PACKAGE="eics3rw" BUILDCACHE_MIRROR_DEPENDENCIES="eicweb ghcr"
+make --jobs ${jobs} --keep-going --directory /opt/spack-environment \
+  SPACK_ENV=${SPACK_ENV} \
+  BUILDCACHE_OCI_PROMPT="eicweb" \
+  BUILDCACHE_OCI_FINAL="ghcr" \
+  BUILDCACHE_S3_FINAL="eics3rw"
+ccache --show-stats
+ccache --zero-stats
+EOF
+
+## 2. Setup our environment with custom versions (on top of cached layer)
+## Note: these default versions are just the very first commit.
+ARG JUGGLER_VERSION="df87bf1f8643afa8e80bece9d36d6dc26dfe8132"
+ARG EICRECON_VERSION="28108da4a1e8919a05dfdb5f11e114800a2cbe96"
+ADD https://api.github.com/repos/eic/juggler/commits/${JUGGLER_VERSION} /tmp/juggler.json
+ADD https://api.github.com/repos/eic/eicrecon/commits/${EICRECON_VERSION} /tmp/eicrecon.json
+RUN --mount=type=cache,target=/ccache,id=${TARGETPLATFORM}              \
+    --mount=type=cache,target=/var/cache/spack                          \
+    --mount=type=secret,id=mirrors,target=/opt/spack/etc/spack/mirrors.yaml \
+    <<EOF
+source ${SPACK_ROOT}/share/spack/setup-env.sh
+export CCACHE_DIR=/ccache
+spack buildcache update-index eics3rw
+spack env activate --dir ${SPACK_ENV}
+if [ "${JUGGLER_VERSION}" != "df87bf1f8643afa8e80bece9d36d6dc26dfe8132" ] ; then
+  export JUGGLER_VERSION=$(jq -r .sha /tmp/juggler.json)
+  spack config add "packages:juggler::require:['@git.${JUGGLER_VERSION}=main']"
+  spack deconcretize -y juggler
+fi
+if [ "${EICRECON_VERSION}" != "28108da4a1e8919a05dfdb5f11e114800a2cbe96" ] ; then
+  export EICRECON_VERSION=$(jq -r .sha /tmp/eicrecon.json)
+  spack config add "packages:eicrecon::require:['@git.${EICRECON_VERSION}=main']"
+  spack deconcretize -y eicrecon
+fi
+spack concretize --fresh --force --quiet
+make --jobs ${jobs} --keep-going --directory /opt/spack-environment \
+  SPACK_ENV=${SPACK_ENV} \
+  BUILDCACHE_OCI_PROMPT="eicweb" \
+  BUILDCACHE_OCI_FINAL="ghcr" \
+  BUILDCACHE_S3_FINAL="eics3rw"
 ccache --show-stats
 ccache --zero-stats
 EOF

 ## Create view at /usr/local
-RUN --mount=type=cache,target=/var/cache/spack <<EOF
+RUN <<EOF
 set -e
 rm -r /usr/local
 spack -e ${SPACK_ENV} env view enable /usr/local
 EOF

-## Optional, nuke the buildcache after install, before (re)caching
-## This is useful when going to completely different containers,
-## or intermittently to keep the buildcache step from taking too much time
-ARG CACHE_NUKE=""
-RUN --mount=type=cache,target=/var/cache/spack,sharing=locked <<EOF
-[ -z "${CACHE_NUKE}" ] || rm -rf /var/cache/spack/mirror/${SPACK_VERSION}/build_cache/*
-EOF
-
 ## Store environment
 RUN <<EOF
 set -e

--- a/eic-spack.sh
+++ b/eic-spack.sh
@@ -3,7 +3,7 @@ EICSPACK_ORGREPO="eic/eic-spack"

 ## EIC spack commit hash or github version, e.g. v0.19.7
 ## note: nightly builds could use a branch e.g. releases/v0.19
-EICSPACK_VERSION="f3e18d6a2273029d16462575fe6cc4f4e1efeee0"
+EICSPACK_VERSION="bc6a4382636dd8c8bcd8c665163840815d48a25e"

 ## Space-separated list of eic-spack cherry-picks
 read -r -d '' EICSPACK_CHERRYPICKS <<- \

--- a/mirrors.yaml.in
+++ b/mirrors.yaml.in
 mirrors:
  eicweb:
    url: oci://${CI_REGISTRY}/${CI_PROJECT_PATH}/spack-${SPACK_VERSION}
+    signed: false
    push:
      access_pair:
      - ${CI_REGISTRY_USER}
      - ${CI_REGISTRY_PASSWORD}
  ghcr:
    url: oci://ghcr.io/eic/spack-${SPACK_VERSION}
+    signed: false
    push:
      access_pair:
      - ${GITHUB_REGISTRY_USER}
      - ${GITHUB_REGISTRY_TOKEN}
  eics3rw:
+    signed: false
    fetch:
      url: s3://eictest/EPIC/spack/${SPACK_VERSION}
      access_pair:

--- a/spack-environment/Makefile
+++ b/spack-environment/Makefile
@@ -7,8 +7,10 @@ export SPACK_COLOR = always

 SPACK_ENV ?= dev

-BUILDCACHE_MIRROR_ONLY_PACKAGE :=
-BUILDCACHE_MIRROR_DEPENDENCIES :=
+BUILDCACHE_S3_PROMPT :=
+BUILDCACHE_S3_FINAL :=
+BUILDCACHE_OCI_PROMPT :=
+BUILDCACHE_OCI_FINAL :=
 BUILDCACHE_OCI_BASE_IMAGE := ghcr.io/eic/debian:stable-slim

 .PHONY: all clean
@@ -20,13 +22,15 @@ include $(SPACK_ENV)/spack.mk
 endif

 $(SPACK_ENV)/push: $(addprefix $(SPACK_ENV)/push/,$($(SPACK_ENV)/SPACK_PACKAGE_IDS))
-	$(foreach buildcache, $(BUILDCACHE_MIRROR_ONLY_PACKAGE), $(SPACK) buildcache update-index $(buildcache) ;)
-	$(foreach buildcache, $(BUILDCACHE_MIRROR_DEPENDENCIES), $(SPACK) buildcache update-index $(buildcache) ;)
+	$(foreach buildcache, $(BUILDCACHE_S3_FINAL), $(SPACK) buildcache push --unsigned $(buildcache) ;)
+	$(foreach buildcache, $(BUILDCACHE_S3_FINAL), $(SPACK) buildcache update-index $(buildcache) ;)
+	$(foreach buildcache, $(BUILDCACHE_S3_PROMPT), $(SPACK) buildcache update-index $(buildcache) ;)
+	$(foreach buildcache, $(BUILDCACHE_OCI_FINAL), $(SPACK) buildcache push --unsigned --base-image $(BUILDCACHE_OCI_BASE_IMAGE) $(buildcache) ;)

 $(SPACK_ENV)/push/%: $(SPACK_ENV)/install/%
 	@mkdir -p $(dir $@)
-	$(foreach buildcache, $(BUILDCACHE_MIRROR_ONLY_PACKAGE), $(SPACK) buildcache push --only=package --unsigned $(buildcache) /$(HASH) ;) # push $(SPEC)
-	$(foreach buildcache, $(BUILDCACHE_MIRROR_DEPENDENCIES), $(SPACK) buildcache push --unsigned --base-image $(BUILDCACHE_OCI_BASE_IMAGE) $(buildcache) /$(HASH) | grep -v "Using cached archive" ;) # push $(SPEC)
+	$(foreach buildcache, $(BUILDCACHE_S3_PROMPT), $(SPACK) buildcache push --unsigned --only=package $(buildcache) /$(HASH) ;) # push $(SPEC)
+	$(foreach buildcache, $(BUILDCACHE_OCI_PROMPT), $(SPACK) buildcache push --unsigned --base-image $(BUILDCACHE_OCI_BASE_IMAGE) $(buildcache) /$(HASH) | grep -v "Using cached archive" ;) # push $(SPEC)
 	@touch $@

 $(SPACK_ENV)/spack.lock: $(SPACK_ENV)/spack.yaml Makefile

--- a/spack-environment/dev/spack.yaml
+++ b/spack-environment/dev/spack.yaml
@@ -24,6 +24,7 @@ spack:
  - edm4eic
  - edm4hep
  - eic-smear
+  - eicrecon
  - eigen
  - emacs
  - fastjet
@@ -40,6 +41,7 @@ spack:
  - iwyu
  - jana2
  - k4actstracking
+  - juggler
  - k4fwcore
  - lcov
  - lhapdf

--- a/spack-environment/packages.yaml
+++ b/spack-environment/packages.yaml
@@ -17,7 +17,7 @@ packages:
    - any_of: [build_type=Release, '@:']
  acts:
    require:
-    - '@30.3.2'
+    - '@31.2.0'
    - cxxstd=20 +binaries +dd4hep +examples +fatras +geant4 +identification +json +python +tgeo
  actsvg:
    require:
@@ -92,6 +92,9 @@ packages:
  eic-smear:
    require:
    - '@1.1.12'
+  eicrecon:
+    require:
+    - '@1.9.1'
  eigen:
    require:
    - '@3.4.0'
@@ -160,6 +163,7 @@ packages:
    - -ipo +podio +root +zmq
  juggler:
    require:
+    - '@13.0.0'
    - cxxstd=20
  k4actstracking:
    require:
@@ -242,7 +246,7 @@ packages:
    - '@1.26.26'
  py-dask:
    require:
-    - '@2021.6.2'
+    - '@2023.4.1'
  py-deepdiff:
    require:
    - '@6.3.0'
@@ -312,6 +316,7 @@ packages:
  py-torch:
    require:
    - '@2.1.0'
+    - -mkldnn
  py-uproot:
    require:
    - '@5.0.5'

--- a/spack-environment/prod/spack.yaml
+++ b/spack-environment/prod/spack.yaml
 spack:
-  include: 
+  include:
  - ../concretizer.yaml
  - ../packages.yaml
  config:
@@ -9,6 +9,7 @@ spack:
  - cmake
  - dd4hep -ddeve
  - edm4eic
+  - eicrecon
  - fastjet
  - fjcontrib
  - gaudi
@@ -17,6 +18,7 @@ spack:
  - intel-tbb
  - irt
  - jana2
+  - juggler
  - npsim -geocad
  - py-boto3
  - py-jinja2

--- a/spack.sh
+++ b/spack.sh
@@ -23,6 +23,9 @@ b111064e221aae83e62226672cd8bf9a7524423d
 c07ddf83c32b7129247fe90eed486dd844047087
 00875fbc749a7e4e979c35d59c0a8d60d32d4cd7
 64cf67822471ec2d3df2625a6d713f80cbe7ff43
+7a0c4e8017033430e5f15ed628be6b539e935ba9
+48fcfda1e7c1781cab4cada6d099823b263ab0cc
+8c29e90fa9962f4a44f39f47217b46c85176af28
 ---
 ## Ref: https://github.com/spack/spack/commit/[hash]
 ## [hash]: [description]
@@ -41,3 +44,6 @@ c07ddf83c32b7129247fe90eed486dd844047087
 ## c07ddf83c32b7129247fe90eed486dd844047087: py-torch: set env OpenBLAS_HOME
 ## 00875fbc749a7e4e979c35d59c0a8d60d32d4cd7: py-torch: patch for ${OpenBLAS_HOME}/include/openblas
 ## 64cf67822471ec2d3df2625a6d713f80cbe7ff43: py-torch: move patch before def patch
+## 7a0c4e8017033430e5f15ed628be6b539e935ba9: acts: new versions 31.*
+## 48fcfda1e7c1781cab4cada6d099823b263ab0cc: setup-env.sh: if exe contains qemu, use /proc/$$/comm instead
+## 8c29e90fa9962f4a44f39f47217b46c85176af28: Build cache: make signed/unsigned a mirror property
No results found