diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4f5098fe513d348b928c9ded83bfee884d5741ee..8acc9629e7a710069dff3b4319ab5ef2323bdd9d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,19 +27,14 @@ variables: ## is this nightly or not? NIGHTLY: + ## Add to tag + NIGHTLY_TAG: "nightly" ## Additional options for docker build (e.g. --no-cache) BUILD_OPTIONS: - ## Default version and tags - ## - internal tag used for the CI. Also temporarily tagged - ## on eicweb to communicate between jobs (removed in cleanup job) - ## - export tag to public registries, optional secondary export tag - VERSION: "${CI_COMMIT_REF_NAME}" - NIGHTLY_TAG: "nightly" - TESTING_TAG: "testing" - INTERNAL_TAG: "${TESTING_TAG}-${VERSION}" - EXPORT_TAG: "" + ## Internal tag used for the CI + INTERNAL_TAG: "pipeline-${CI_PIPELINE_ID}" stages: - config @@ -98,30 +93,36 @@ workflow: ## caching strategy for dispatch to multiple nodes and to avoid ## --> try this strategy: https://medium.com/titansoft-engineering/docker-build-cache-sharing-on-multi-hosts-with-buildkit-and-buildx-eb8f7005918e +## Default version and tags +## - internal tag used for the CI. Also temporarily tagged +## on eicweb to communicate between jobs (removed in cleanup job) +## - export tag to public registries, optional secondary export tag version: stage: config script: - | if [ "x${CI_PIPELINE_SOURCE}" = "xmerge_request_event" ]; then VERSION="unstable-mr-${CI_MERGE_REQUEST_PROJECT_ID}-${CI_MERGE_REQUEST_IID}" - INTERNAL_TAG="${VERSION}" NIGHTLY_TAG="${VERSION}-nightly" EXPORT_TAG="${VERSION}" elif [[ "$CI_COMMIT_TAG" =~ ^v[0-9.]+\.[0-9]+\.[0-9]+ ]]; then VERSION="${CI_COMMIT_TAG:1}" - INTERNAL_TAG="stable-tag-${VERSION}" EXPORT_TAG="${VERSION}" - elif [[ "$CI_COMMIT_BRANCH" = ^v[0-9.]+\.[0-9]+-stable ]]; then + elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+\.[0-9]+-stable ]]; then VERSION="${CI_COMMIT_BRANCH:1}" - INTERNAL_TAG="stable-br-${VERSION}" EXPORT_TAG="${VERSION}" elif [[ "$CI_COMMIT_BRANCH" =~ ^v[0-9.]+\.[0-9]+-[a-z]+-stable ]]; then VERSION="${CI_COMMIT_BRANCH:1}" - INTERNAL_TAG="stable-br-${VERSION}" EXPORT_TAG="${VERSION}" + elif [[ "$CI_COMMIT_BRANCH" =~ ^master$ ]]; then + VERSION="${CI_COMMIT_BRANCH}" + EXPORT_TAG="${VERSION}" + else + VERSION="${CI_COMMIT_REF_NAME}-${CI_PIPELINE_ID}" + EXPORT_TAG="" fi - echo "VERSION=$VERSION" >> build.env echo "INTERNAL_TAG=$INTERNAL_TAG" >> build.env + echo "VERSION=$VERSION" >> build.env echo "NIGHTLY_TAG=$NIGHTLY_TAG" >> build.env echo "EXPORT_TAG=$EXPORT_TAG" >> build.env cat build.env @@ -159,10 +160,19 @@ version: ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}} ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}} ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}} - } + } --file containers/dind/Dockerfile containers/dind +## rules for nightly jobs +.nightly: + rules: + - if: '$CI_COMMIT_BRANCH == "master"' + when: on_success + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: on_success + - when: never + ## Images: ## debian_testing_base --> jug_dev --> jug_xl ## ---------------> jug_sim @@ -340,7 +350,12 @@ jug_xl:nightly: PUSH_NIGHTLY_WITH_DATE="1" fi - docker buildx build --push ${BUILD_OPTIONS} - --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} + --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG}-nightly + ${EXPORT_TAG:+ + ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}-nightly} + ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}-nightly} + ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}-nightly} + } ${NIGHTLY:+ ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${NIGHTLY_TAG}} ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${NIGHTLY_TAG}} @@ -441,10 +456,8 @@ jug_xl:singularity:nightly: - mkdir build - singularity pull build/${BUILD_IMAGE}.sif docker://${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${NIGHTLY_TAG} -benchmarks:reconstruction: +benchmarks:reconstruction:default: stage: benchmarks - rules: - - if: '$CI_SERVER_HOST == "eicweb.phy.anl.gov"' needs: - version - jug_xl:default @@ -457,10 +470,8 @@ benchmarks:reconstruction: strategy: depend allow_failure: true -benchmarks:physics: +benchmarks:physics:default: stage: benchmarks - rules: - - if: '$CI_SERVER_HOST == "eicweb.phy.anl.gov"' needs: - version - jug_xl:default @@ -474,6 +485,39 @@ benchmarks:physics: strategy: depend allow_failure: false +benchmarks:reconstruction:nightly: + stage: benchmarks + rules: + - !reference ['.nightly', rules] + needs: + - version + - jug_xl:nightly + variables: + BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly" + BENCHMARKS_CONTAINER: "jug_xl" + BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE" + trigger: + project: EIC/benchmarks/reconstruction_benchmarks + strategy: depend + allow_failure: true + +benchmarks:physics:nightly: + stage: benchmarks + rules: + - !reference ['.nightly', rules] + needs: + - version + - jug_xl:nightly + variables: + RECO: "eicrecon" + BENCHMARKS_TAG: "${INTERNAL_TAG}-nightly" + BENCHMARKS_CONTAINER: "jug_xl" + BENCHMARKS_REGISTRY: "$CI_REGISTRY_IMAGE" + trigger: + project: EIC/benchmarks/physics_benchmarks + strategy: depend + allow_failure: false + clean: stage: finalize dependencies: @@ -484,13 +528,17 @@ clean: script: ## remove the pipeline specific export from eicweb if needed - echo "Cleaning up pipeline specific docker tags if needed" - - ./gitlab-ci/cleanup_registry.sh -i debian_testing_base -r 66 ${INTERNAL_TAG} + - ./gitlab-ci/cleanup_registry.sh -i debian_base -r 66 ${INTERNAL_TAG} + - ./gitlab-ci/cleanup_registry.sh -i ubuntu_base -r 80 ${INTERNAL_TAG} + - ./gitlab-ci/cleanup_registry.sh -i debian_testing_base -r 83 ${INTERNAL_TAG} + - ./gitlab-ci/cleanup_registry.sh -i debian_stable_base -r 92 ${INTERNAL_TAG} - ./gitlab-ci/cleanup_registry.sh -i jug_dev -r 68 ${INTERNAL_TAG} - ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${INTERNAL_TAG} - | if [ "$CI_PIPELINE_SOURCE" == "merge_request_event" ]; then ./gitlab-ci/cleanup_registry.sh -i jug_xl -r 69 ${NIGHTLY_TAG} fi + - ./gitlab-ci/cleanup_registry.sh -i dind -r 93 ${INTERNAL_TAG} clean_unstable_mr: stage: finalize diff --git a/containers/dind/Dockerfile b/containers/dind/Dockerfile index bfd03f61f0f6ce37439f02568a1dc24e99bf3710..e6a7a97a8ba3b8d41ee3d7e609c0fdd402939e74 100644 --- a/containers/dind/Dockerfile +++ b/containers/dind/Dockerfile @@ -54,8 +54,7 @@ ADD https://get.docker.com/ /tmp/get-docker.sh RUN bash /tmp/get-docker.sh --version ${DOCKER_VERSION} # Install the magic wrapper. -ADD ./wrapdocker /usr/local/bin/wrapdocker -RUN chmod +x /usr/local/bin/wrapdocker +ADD --chmod=0755 wrapdocker /usr/local/bin/wrapdocker # Define additional metadata for our image. VOLUME /var/lib/docker diff --git a/containers/jug/dev.Dockerfile b/containers/jug/dev.Dockerfile index 4b0c8f8ee9f5eef4f4b2bf966092396773d419c8..3cfd615aec28bf9a2565241469774fc38f069b39 100644 --- a/containers/jug/dev.Dockerfile +++ b/containers/jug/dev.Dockerfile @@ -193,14 +193,13 @@ COPY profile.d/z11_jug_env.sh /etc/profile.d COPY singularity.d /.singularity.d ## Add minio client into /usr/local/bin -ADD https://dl.min.io/client/mc/release/linux-amd64/mc /usr/local/bin/mc-amd64 -ADD https://dl.min.io/client/mc/release/linux-arm64/mc /usr/local/bin/mc-arm64 +ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-amd64/mc /usr/local/bin/mc-amd64 +ADD --chmod=0755 https://dl.min.io/client/mc/release/linux-arm64/mc /usr/local/bin/mc-arm64 RUN declare -A target=( \ ["linux/amd64"]="amd64" \ ["linux/arm64"]="arm64" \ ) \ && mv /usr/local/bin/mc-${target[${TARGETPLATFORM}]} /usr/local/bin/mc \ - && chmod a+x /usr/local/bin/mc \ && unset target[${TARGETPLATFORM}] \ && for t in ${target[*]} ; do \ rm /usr/local/bin/mc-${t} ; \ diff --git a/containers/jug/eic-news b/containers/jug/eic-news new file mode 100644 index 0000000000000000000000000000000000000000..e01795173660881d1ce6c15536ea6ea1d91f504d --- /dev/null +++ b/containers/jug/eic-news @@ -0,0 +1,22 @@ +#!/bin/bash + +news=$HOME/.eic-news + +# run only if stdin and sterr +if [[ -t 0 && -t 2 && ! -f $HOME/.hushlogin && -f $news ]] ; then + + if [[ $- == *i* ]] ; then + # sourced: print once a day + if [[ -z $(find $HOME/.eic-news -atime -1) ]] ; then + cat $news + fi + else + # else: print always + cat $news + fi + + # update once a day + if [[ -z $(find $HOME/.eic-news -mtime -1) ]] ; then + (&>/dev/null curl --silent --connect-timeout 5 --location https://eic.github.io/motd --output $news &) + fi +fi diff --git a/containers/jug/setup_detectors.py b/containers/jug/setup_detectors.py index c9389cc0de789eb4d751478c030c5e935bc9810c..f7ba8717b263bde253ce1e0bbcbe4252cc448920 100755 --- a/containers/jug/setup_detectors.py +++ b/containers/jug/setup_detectors.py @@ -105,6 +105,8 @@ if __name__ == '__main__': subprocess.check_call(' '.join(cmd), shell=True) ## build cxxflags = '' + if os.environ['CXXFLAGS']: + cxxflags = os.environ['CXXFLAGS'] if cfg.get('cxxflags'): cxxflags = cfg['cxxflags'] cmd = [ diff --git a/containers/jug/xl.Dockerfile b/containers/jug/xl.Dockerfile index 31a0b7c75bb4e6b470bb229d6d7d90f0e84612c3..ff5d2c9a38d619224e2839fa7e354e3c61d742b5 100644 --- a/containers/jug/xl.Dockerfile +++ b/containers/jug/xl.Dockerfile @@ -140,5 +140,10 @@ RUN --mount=type=cache,target=/ccache/,sharing=locked,id=${TARGETPLATFORM} && ccache --show-stats \ && rm /tmp/setup_detectors.py +## eic-news +COPY --chmod=0755 eic-news /usr/local/bin/eic-news +RUN echo "test -f $HOME/.eic-news && source /usr/local/bin/eic-news" \ + > /etc/profile.d/z13_eic-news.sh + ## Hotfix for misbehaving OSG nodes RUN mkdir /hadoop diff --git a/detectors.yaml b/detectors.yaml index ca6506a13cb9fb0ec0a4451ece5522032f9b1e9b..a38549536b5be200e67db2c6e3ed8acd78ff2331 100644 --- a/detectors.yaml +++ b/detectors.yaml @@ -3,48 +3,37 @@ detectors: nightly: default: true version: main - cxxflags: -DDD4HEP_FIELD_TYPE_OVERRIDE=field_type main: version: main - cxxflags: -DDD4HEP_FIELD_TYPE_OVERRIDE=field_type 23.01.0: version: 23.01.0 patches: - https://github.com/eic/epic/pull/449.patch - cxxflags: -DDD4HEP_FIELD_TYPE_OVERRIDE=field_type 23.03.0: version: 23.03.0 patches: - https://github.com/eic/epic/pull/449.patch - cxxflags: -DDD4HEP_FIELD_TYPE_OVERRIDE=field_type 23.05.0: version: 23.05.0 patches: - https://github.com/eic/epic/pull/449.patch - cxxflags: -DDD4HEP_FIELD_TYPE_OVERRIDE=field_type 23.05.1: version: 23.05.1 patches: - https://github.com/eic/epic/pull/449.patch - cxxflags: -DDD4HEP_FIELD_TYPE_OVERRIDE=field_type 23.05.2: version: 23.05.2 patches: - https://github.com/eic/epic/pull/449.patch - cxxflags: -DDD4HEP_FIELD_TYPE_OVERRIDE=field_type 23.06.0: version: 23.06.0 patches: - https://github.com/eic/epic/pull/449.patch - cxxflags: -DDD4HEP_FIELD_TYPE_OVERRIDE=field_type 23.06.1: version: 23.06.1 patches: - https://github.com/eic/epic/pull/449.patch - cxxflags: -DDD4HEP_FIELD_TYPE_OVERRIDE=field_type 23.07.0: version: 23.07.0 - cxxflags: -DDD4HEP_FIELD_TYPE_OVERRIDE=field_type 23.07.1: version: 23.07.1 - cxxflags: -DDD4HEP_FIELD_TYPE_OVERRIDE=field_type diff --git a/eic-spack.sh b/eic-spack.sh index 8878fb169b01cd58611d5b6e5f3a08ba278dc506..06f5e3df4b317f3f713d88f0a39ad21a19f8adfa 100644 --- a/eic-spack.sh +++ b/eic-spack.sh @@ -3,7 +3,7 @@ EICSPACK_ORGREPO="eic/eic-spack" ## EIC spack github version, e.g. v0.19.7 or commit hash ## note: nightly builds will use e.g. releases/v0.19 -EICSPACK_VERSION="v0.20.7" +EICSPACK_VERSION="v0.20.8" ## Space-separated list of eic-spack cherry-picks read -r -d '' EICSPACK_CHERRYPICKS <<- \ diff --git a/spack-environment/dev/spack.yaml b/spack-environment/dev/spack.yaml index 8f5ca7a4bec406fb99c74897b97172a1b78ca714..4eccde66d3b230d02a1f499f8cac6b8623d3bdea 100644 --- a/spack-environment/dev/spack.yaml +++ b/spack-environment/dev/spack.yaml @@ -20,7 +20,7 @@ spack: - dd4hep@1.25.1 +ddg4 +ddcad -frames +hepmc3 +lcio +xercesc - dpmjet@19.3.5 - edm4eic@2.0.0 cxxstd=17 - - edm4hep@=0.7 cxxstd=17 + - edm4hep@0.7.2 cxxstd=17 - eic-smear@1.1.10 - eigen@3.4.0 - emacs@28.2 @@ -30,6 +30,7 @@ spack: - gaudi@36.10 +aida - gdb@12.1 -debuginfod - geant4@11.1.1 cxxstd=17 +opengl -vecgeom +x11 +qt +threads + - graphviz@8.0.1 - hepmc3@3.2.6 +python +rootio - heppdt@2.06.01 - imagemagick@7.0.8-7 diff --git a/spack.sh b/spack.sh index dffdb2d6d8485625e188570edd3ca6ae26a086ef..a5b29f5246fe0b167f7a48ed6dfe0cc20dc04ab2 100644 --- a/spack.sh +++ b/spack.sh @@ -3,7 +3,7 @@ SPACK_ORGREPO="spack/spack" ## Spack github version, e.g. v0.18.1 or commit hash ## note: nightly builds will use e.g. releases/v0.19 -SPACK_VERSION="v0.20.0" +SPACK_VERSION="v0.20.1" ## Space-separated list of spack cherry-picks read -r -d '' SPACK_CHERRYPICKS <<- \