Compare revisions

Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck · Wouter Deconinck
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,9 +17,6 @@ variables:
  GH_REGISTRY_USER: eic
  GH_PUSH: 1

-  ## Also export the raw builder image, will be stored on DH only
-  EXPORT_BUILDER: 0
-
  ## TLS error resiliency: number of retries and second wait between tries 
  ## (wait time is doubled with each attempt)
  DOCKER_NTRIES: 5
@@ -31,8 +28,8 @@ variables:
  ## is this nightly or not?
  NIGHTLY:

-  ## Force a rebuild without using cache
-  FORCE_NOCACHE: 0
+  ## Additional options for docker build (e.g. --no-cache)
+  BUILD_OPTIONS:

  ## Default version and tags
  ## - internal tag used for the CI. Also temporarily tagged
@@ -160,7 +157,22 @@ version:
 ## oneapi_runtime + jug_xl --> oneapi_prod 

 ## Docker-in-docker image
-dind:
+dind-build:
+  parallel:
+    matrix:
+      - BUILD_IMAGE: dind
+  extends: .build
+  stage: dind
+  needs:
+    - version
+  script:
+    - docker buildx build --push
+        --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}     
+        --file containers/dind/Dockerfile
+        containers/dind
+
+dind-publish:
+  image: ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
  parallel:
    matrix:
      - BUILD_IMAGE: dind
@@ -168,9 +180,15 @@ dind:
  stage: dind
  needs:
    - version
+    - dind-build
  script:
    - docker buildx build --push
        --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
+        ${EXPORT_TAG:+
+          ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
+          ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
+          ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
+        }        
        --file containers/dind/Dockerfile
        containers/dind

@@ -186,9 +204,9 @@ base:
      - BASE_IMAGE: ubuntu:22.04
        BUILD_IMAGE: ubuntu_base
        PLATFORM: linux/amd64,linux/arm64/v8
-      - BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04
-        BUILD_IMAGE: oneapi_base
-        PLATFORM: linux/amd64
+#      - BASE_IMAGE: intel/oneapi-hpckit:2022.3.0-devel-ubuntu20.04
+#        BUILD_IMAGE: oneapi_base
+#        PLATFORM: linux/amd64
 #      - BASE_IMAGE: nvidia/cuda:11.8.0-devel-ubuntu20.04
 #        BUILD_IMAGE: cuda_base
 #        PLATFORM: linux/amd64
@@ -197,7 +215,7 @@ base:
  needs:
    - version
  script:
-    - docker buildx build --push
+    - docker buildx build --push ${BUILD_OPTIONS}
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
                   ${EXPORT_TAG:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
@@ -228,36 +246,17 @@ jug_dev:default:
    - version
    - base
  script:
-    ## Which targets to build
-    - TARGETS="export"
-    - |
-      if [ ${EXPORT_BUILDER} = 1 ]; then
-        TARGETS="builder ${TARGETS}"
-      fi
-    ## Disable caching
-    - CACHE_FLAG=""
-    - |
-      if [ $FORCE_NOCACHE = 1 ]; then
-        echo "FORCE_NOCACHE set"
-        export CACHE_FLAG="--no-cache"
-      fi
    ## Setup mirrors.yaml with rw mirror secrets
    - cat mirrors.yaml.in | envsubst > mirrors.yaml
    ## Build targets
    - source spack.sh ; 
-      source eic-spack.sh ; 
-      for target in ${TARGETS} ; do
-        if [ ! ${target} = export ] ; then
-          suffix=-${target} ;
-        fi ;
-        docker buildx build --push
-                   --target ${target}
-                   ${CACHE_FLAG}
-                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}${suffix}
+      source eic-spack.sh ;
+      docker buildx build --push ${BUILD_OPTIONS}
+                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
                   ${EXPORT_TAG:+
-                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}${suffix}}
-                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}${suffix}}
-                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}${suffix}}
+                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
+                     ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
+                     ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}}
                   }
                   --file containers/jug/dev.Dockerfile
                   --platform ${PLATFORM}
@@ -277,7 +276,6 @@ jug_dev:default:
                   --build-context spack-environment=spack-environment
                   --secret id=mirrors,src=mirrors.yaml
                   containers/jug
-      ; done

 jug_xl:default:
  parallel:
@@ -294,7 +292,7 @@ jug_xl:default:
    - version
    - jug_dev:default
  script:
-    - docker buildx build --push
+    - docker buildx build --push ${BUILD_OPTIONS}
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
                   ${EXPORT_TAG:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}}
@@ -339,7 +337,7 @@ jug_xl:nightly:
      if [ "$CI_COMMIT_BRANCH" == "master" ]; then
        PUSH_NIGHTLY_WITH_DATE="1"
      fi
-    - docker buildx build --push
+    - docker buildx build --push ${BUILD_OPTIONS}
                   --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}
                   ${NIGHTLY:+
                     ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${NIGHTLY_TAG}}
@@ -378,7 +376,8 @@ jug_xl:feature:
        DETECTOR_BRANCH:
          - main
  script:
-    - docker buildx build --load --tag ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}-${DETECTOR}-${DETECTOR_BRANCH}
+    - docker buildx build --load ${BUILD_OPTIONS}
+                   --tag ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${INTERNAL_TAG}-${DETECTOR}-${DETECTOR_BRANCH}
                   --file containers/jug/feature.Dockerfile
                   --build-arg INTERNAL_TAG=${NIGHTLY_TAG}
                   --build-arg DETECTOR=${DETECTOR}
@@ -553,5 +552,5 @@ clean_unstable_mr:
          fi ;
        fi ;
      done
-    - docker system prune --filter until=144h --force
+    - docker system prune --filter until=24h --force
    - docker images
--- a/containers/dind/Dockerfile
+++ b/containers/dind/Dockerfile
@@ -17,6 +17,7 @@ RUN apt-get update && \
    gettext \
    git \
    iptables \
+    libglib2.0-dev \
    libgpgme11-dev \
    libseccomp-dev \
    libssl-dev \
@@ -29,28 +30,33 @@ RUN apt-get update && \
    squashfs-tools \
    wget

-RUN cd /tmp \
-    && export VERSION=1.15.6 OS=linux ARCH=amd64 && \
-    wget https://dl.google.com/go/go$VERSION.$OS-$ARCH.tar.gz && \
-    tar -C /usr/local -xzvf go$VERSION.$OS-$ARCH.tar.gz && \
-    rm go$VERSION.$OS-$ARCH.tar.gz 
+# Install Go
+ARG GO_OS=linux
+ARG GO_ARCH=amd64
+ARG GO_VERSION=1.20.5
+ARG GO_URL=https://dl.google.com/go/go${GO_VERSION}.${GO_OS}-${GO_ARCH}.tar.gz
+RUN curl -L ${GO_URL} | tar -C /usr/local -xzvf -

-RUN cd /tmp \
-    && export PATH=/usr/local/go/bin:$PATH \
-    && export VERSION=3.7.4 && \
-    wget https://github.com/sylabs/singularity/releases/download/v${VERSION}/singularity-${VERSION}.tar.gz && \ 
-    tar -xzf singularity-${VERSION}.tar.gz && \
-    cd singularity \
-    && ./mconfig && \
-    make -C builddir && \
-    make -C builddir install
+# Install Singularity CE
+ARG SINGULARITY_VERSION=3.11.4
+ARG SINGULARITY_URL=https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_VERSION}/singularity-ce-${SINGULARITY_VERSION}.tar.gz
+RUN curl -L ${SINGULARITY_URL} | tar -C /tmp -xzf - \
+ && cd /tmp/singularity-ce-${SINGULARITY_VERSION} \
+ && export PATH=/usr/local/go/bin:$PATH \
+ && ./mconfig \
+ && make -C builddir \
+ && make -C builddir install \
+ && rm -rf /tmp/singularity-ce-${SINGULARITY_VERSION}

-# Install Docker from Docker Inc. repositories. 
-RUN export VERSION=20.10.17 \
- && curl -sSL https://get.docker.com/ | sh
-# Install the magic wrapper. 
+# Install Docker
+ARG DOCKER_VERSION=24
+ADD https://get.docker.com/ /tmp/get-docker.sh
+RUN bash /tmp/get-docker.sh --version ${DOCKER_VERSION}
+
+# Install the magic wrapper.
 ADD ./wrapdocker /usr/local/bin/wrapdocker
 RUN chmod +x /usr/local/bin/wrapdocker
-# Define additional metadata for our image. 
+
+# Define additional metadata for our image.
 VOLUME /var/lib/docker
 CMD ["wrapdocker"]
--- a/containers/jug/dev.Dockerfile
+++ b/containers/jug/dev.Dockerfile
@@ -173,7 +173,9 @@ RUN du -sh $SPACK_ROOT                                                  \
 ## See
 #https://askubuntu.com/questions/1034313/ubuntu-18-4-libqt5core-so-5-cannot-open-shared-object-file-no-such-file-or-dir
 ## and links therin for more info
-RUN strip --remove-section=.note.ABI-tag /usr/local/lib/libQt5Core.so
+RUN if [ -f /usr/local/lib/libQt5Core.so ] ; then                       \
+      strip --remove-section=.note.ABI-tag /usr/local/lib/libQt5Core.so;\
+    fi

 RUN spack debug report                                                  \
      | sed "s/^/ - /" | sed "s/\* \*\*//" | sed "s/\*\*//"             \
No results found