Resolve "Tweak singularity container deploy"

.gitlab-ci.yml

@@ -177,6 +177,7 @@ jug_dev:default:
                    --build-arg SPACK_VERSION=${SPACK_VERSION}
                    --build-arg CACHE_BUST=${PACKAGE_HASH}
                    --build-arg INTERNAL_TAG=${INTERNAL_TAG}
+                   --build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD)
                    containers/jug
     - !reference [.build, script]
 
@@ -197,6 +198,7 @@ jug_xl:default:
                    --build-arg EICD_VERSION=${EICD_VERSION}
                    --build-arg DETECTOR_VERSION=${DETECTOR_VERSION}
                    --build-arg IP6_VERSION=${IP6_VERSION}
+                   --build-arg JUG_VERSION=${INTERNAL_TAG}-$(git rev-parse HEAD)
                    containers/jug
     - !reference [.build, script]
 
@@ -213,11 +215,10 @@ jug_xl:nightly:
   variables:
     BUILD_IMAGE: "jug_xl"
   script:
-    - CACHEBUST=`date +%s`
     - docker build -t ${CI_REGISTRY_IMAGE}/${BUILD_IMAGE}:${NIGHTLY_TAG} 
                    -f containers/jug/Dockerfile.xl
                    --build-arg INTERNAL_TAG=${INTERNAL_TAG}
-                   --build-arg CACHEBUST=${CACHEBUST}
+                   --build-arg JUG_VERSION=nightly-$(date +%Y-%m-%d_%H-%M-%S)-$(git rev-parse HEAD)
                    containers/jug
     - ./gitlab-ci/docker_push.sh -i ${BUILD_IMAGE} -l ${NIGHTLY_TAG} 
                                  -n $DOCKER_NTRIES -t $DOCKER_WAIT_TIME

README.md

@@ -25,6 +25,20 @@ eic-shell
 4. Within your development environment (`eic-shell`), you can install software to the
    internal `$ATHENA_PREFIX`
 
+Singularity Container Dowload for Development Usage
+-------------
+**Note: this container download script is meant for expert usage. If it is unclear to you
+why you would want to do this, you are probably looking for the single installation
+above.**
+To download the `jug_dev:testing` base image, do
+```bash
+curl https://eicweb.phy.anl.gov/containers/eic_container/-/raw/master/download_dev.sh | bash
+```
+To download the `jug_xl:nightly` image, do
+```bash
+curl https://eicweb.phy.anl.gov/containers/eic_container/-/raw/master/download_dev.sh | bash -s -- -c jug_xl -v nightly
+```
+
 Using the docker container for your CI purposes
 -----------------------------------------------
 

containers/jug/Dockerfile.dev

@@ -94,12 +94,13 @@ RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
  && spack clean -a
 
 ## Update the local build cache if needed. Consists of 3 steps:
-## 1. Remove the B010 network buildcache
+## 1. Remove the B010 network buildcache (silicon)
 ## 2. Get a list of all packages, and compare with what is already on
 ##    the buildcache (using package hash)
 ## 3. Add packages that need to be added to buildcache if any
 RUN --mount=type=cache,target=/var/cache/spack-mirror                   \
-    spack buildcache list --allarch --long                              \
+    spack mirror remove silicon                                         \
+ && spack buildcache list --allarch --long                              \
      | grep -v -e '---'                                                 \
      | sed "s/@.\+//"                                                   \
      | sort > tmp.buildcache.txt                                        \
@@ -136,8 +137,6 @@ RUN cd /opt/spack-environment                                           \
  && echo -n ""                                                          \
  && echo "Add extra environment variables for Jug, Podio and Gaudi"     \
  && spack env activate .                                                \
- && echo "export JUG_DEV_VERSION=${INTERNAL_TAG}-$(date +%Y-%m-%d)"     \
-        >> /etc/profile.d/z11_jug_env.sh                                \
  && export PODIO=`spack find -p podio                                   \
         | grep software                                                 \
         | awk '{print $2}'`                                             \
@@ -198,6 +197,14 @@ RUN strip --remove-section=.note.ABI-tag /usr/local/lib/libQt5Core.so
 COPY eic-shell /usr/local/bin/eic-shell
 COPY entrypoint.sh /usr/local/sbin/entrypoint.sh
 
+RUN spack debug report                                                  \
+      | sed "s/^/ - /" | sed "s/\* \*\*//" | sed "s/\*\*//"             \
+    >> /etc/jug_info                                                    \
+ && spack find --no-groups | sed "s/^/ - /" >> /etc/jug_info            \
+ && echo "#!/bin/bash" > /usr/local/bin/eic-info                        \
+ && echo "cat /etc/jug_info" >> /usr/local/bin/eic-info                 \
+ && chmod +x /usr/local/bin/eic-info
+
 ## ========================================================================================
 ## STAGE 3
 ## Lean target image
@@ -218,8 +225,12 @@ RUN --mount=from=staging,target=/staging                                \
  && cp /staging/etc/profile.d/z11_jug_env.sh                            \
        /etc/profile.d/z11_jug_env.sh                                    \
  && cat /etc/profile.d/z11_jug_env.sh >> /etc/eic-env.sh                \
- && cp /etc/eic-env.sh /etc/profile.d/z10_eic-env.sh
+ && cp /etc/eic-env.sh /etc/profile.d/z10_eic-env.sh                    \
+ && cp /staging/etc/jug_info /etc/jug_info
 
+ARG JUG_VERSION=1
+RUN echo "" >> /etc/jug_info                                            \
+ && echo " - jug_dev: ${JUG_VERSION}" >> /etc/jug_info
 
 ## make sure we have the entrypoints setup correctly
 ENTRYPOINT ["/usr/local/sbin/entrypoint.sh"]

containers/jug/Dockerfile.xl

@@ -11,25 +11,36 @@ ARG JUGGLER_VERSION="master"
 ARG NPDET_VERSION="master"
 ARG EICD_VERSION="master"
 
-## cachebust that can change values to trigger a rebuild, useful for nightly builds
-## off the master
-ARG CACHEBUST=1
+## version will automatically bust cache for nightly, as it includes
+## the date
+ARG JUG_VERSION=1
 RUN cd /tmp                                                                     \
+ && echo " - jug_xl: ${JUG_VERSION}" >> /etc/jug_info                           \
  && echo "INSTALLING NPDET"                                                     \
  && git clone -b ${NPDET_VERSION}  https://eicweb.phy.anl.gov/EIC/NPDet.git     \
  && cmake -B build -S NPDet -DCMAKE_CXX_STANDARD=17                             \
  && cmake --build build -j12 -- install                                         \
+ && pushd NPDet                                                                 \
+ && echo " - NPDet: ${NPDET_VERSION}-$(git rev-parse HEAD)">> /etc/jug_info     \
+ && popd                                                                        \
  && rm -rf build NPDet                                                          \
  && echo "INSTALLING EICD"                                                      \
  && git clone -b ${EICD_VERSION}  https://eicweb.phy.anl.gov/EIC/eicd.git       \
  && cmake -B build -S eicd -DCMAKE_CXX_STANDARD=17                              \
  && cmake --build build -j12 -- install                                         \
+ && pushd eicd                                                                  \
+ && echo " - EICD: ${EICD_VERSION}-$(git rev-parse HEAD)">> /etc/jug_info       \
+ && popd                                                                        \
  && rm -rf build eicd                                                           \
  && echo "INSTALLING JUGGLER"                                                   \
  && git clone -b ${JUGGLER_VERSION} https://eicweb.phy.anl.gov/EIC/juggler.git  \
  && cmake -B build -S juggler                                                   \
           -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=/usr/local             \
  && cmake --build build -j12 -- install                                         \
+ && pushd juggler                                                               \
+ && echo " - Juggler: ${JUGGLER_VERSION}-$(git rev-parse HEAD)"                 \
+          >> /etc/jug_info                                                      \
+ && popd                                                                        \
  && rm -rf build juggler
 
 ## also install detector/ip geometries into opt
@@ -37,12 +48,11 @@ RUN cd /tmp
 ##        build
 ARG DETECTOR_VERSION="master"
 ARG IP6_VERSION="master"
-ARG ACCELERATOR_VERSION="master"
 RUN cd /tmp                                                                     \
  && DETECTOR_PREFIX=/opt/detector                                               \
  && DETECTOR_DATA=$DETECTOR_PREFIX/share/athena                                 \
  && mkdir -p /opt/detector/share/athena                                         \
- && echo "INSTALLING REFERENCE DETECTOR"                                        \
+ && echo "INSTALLING ATHENA"                                                    \
  && git clone -b ${DETECTOR_VERSION}                                            \
                 https://eicweb.phy.anl.gov/EIC/detectors/athena.git             \
  && cmake -B build -S athena -DCMAKE_CXX_STANDARD=17                            \
@@ -51,6 +61,10 @@ RUN cd /tmp
  && cp -r athena/compact                                                        \
           athena/athena.xml                                                     \
           ${DETECTOR_DATA}                                                      \
+ && pushd athena                                                                \
+ && echo " - Athena: ${DETECTOR_VERSION}-$(git rev-parse HEAD)"                 \
+          >> /etc/jug_info                                                      \
+ && popd                                                                        \
  && rm -rf build athena                                                         \
  && echo "INSTALLING IP6 GEOMETRY"                                              \
  && git clone -b ${IP6_VERSION}                                                 \
@@ -60,6 +74,10 @@ RUN cd /tmp
  && cmake --build build -j12 -- install                                         \
  && cp -r ip6/ip6                                                               \
           ${DETECTOR_DATA}                                                      \
+ && pushd ip6                                                                   \
+ && echo " - IP6: ${IP6_VERSION}-$(git rev-parse HEAD)"                         \
+          >> /etc/jug_info                                                      \
+ && popd                                                                        \
  && rm -rf build ip6                                                            \
  && echo "ADDING SETUP SCRIPT"                                                  \
  && echo "export LD_LIBRARY_PATH=/opt/detector/lib:\$LD_LIBRARY_PATH"           \

install_dev.sh → download_dev.sh

@@ -5,18 +5,23 @@ VERSION="testing"
 ODIR="$PWD"
 
 function print_the_help {
-  echo "USAGE:  ./install_dev.sh [-o DIR] [-v VERSION]"
+  echo "USAGE:  ./download_dev.sh [-o DIR] [-v VERSION]"
   echo "OPTIONAL ARGUMENTS:"
   echo "          -o,--outdir     Directory to download the container to (D: $ODIR)"
-  echo "          -v,--version    Version to install (D: $VERSION)"
+  echo "          -c,--container  Container to download (D: $CONTAINER)"
+  echo "          -v,--version    Version to download (D: $VERSION)"
   echo "          -h,--help       Print this message"
   echo ""
   echo "  Download development container into an output directory"
   echo ""
-  echo "EXAMPLE: ./install.sh" 
+  echo "EXAMPLE: ./download_dev.sh" 
   exit
 }
 
+echo "WARNING: this container download script is meant for expert usage"
+echo "         if you don't know what this script does, you are probably"
+echo "         looking for install.sh"
+
 while [ $# -gt 0 ]; do
   key=$1
   case $key in
@@ -25,6 +30,11 @@ while [ $# -gt 0 ]; do
       shift
       shift
       ;;
+    -c|--container)
+      CONTAINER=$2
+      shift
+      shift
+      ;;
     -v|--version)
       VERSION=$2
       shift
@@ -72,7 +82,8 @@ wget https://eicweb.phy.anl.gov/containers/eic_container/-/raw/master/install.py
 chmod +x install.py
 ./install.py -f -c $CONTAINER -v $VERSION .
 
-SIF=`ls lib/$CONTAINER.sif.* | head -n1`
+SIF=lib/$CONTAINER-$VERSION.sif
+chmod +x ${SIF}
 
 ## That's all
 if [ -z $SIF -o ! -f $SIF ]; then

install.py

@@ -287,7 +287,7 @@ if __name__ == "__main__":
     ## Builder SIF is not built anymore, deprecated
     #if args.builder:
         #img += "_builder"
-    container = '{}/{}.sif.{}'.format(libdir, img, version_docker)
+    container = '{}/{}-{}.sif'.format(libdir, img, version_docker)
     if not os.path.exists(container) or args.force:
         url = CONTAINER_URL.format(ref=version_gitlab, img=img, job=build_job)
         print('Downloading container from:', url)

install.sh

@@ -96,30 +96,34 @@ echo " - Found singularity at $SINGULARITY"
 SINGULARITY_VERSION=`$SINGULARITY --version`
 SIF=
 if [ ${SINGULARITY_VERSION:0:1} = 2 ]; then
+  SIF="$PREFIX/local/lib/${CONTAINER}-${VERSION}.simg"
+
   echo "WARNING: your singularity version $SINGULARITY_VERSION is ancient, we strongly recommend using version 3.x"
   echo "We will attempt to use a fall-back SIMG image to be used with this singularity version"
   if [ -f /gpfs02/eic/athena/${CONTAINER}-${VERSION}.simg ]; then
-    ln -sf /gpfs02/eic/athena/${CONTAINER}-${VERSION}.simg local/lib
-    SIF="$PREFIX/local/lib/${CONTAINER}-${VERSION}.simg"
+    ln -sf /gpfs02/eic/athena/${CONTAINER}-${VERSION}.simg ${SIF}
   else
     echo "Attempting last-resort singularity pull for old image"
     echo "This may take a few minutes..."
-    SIF="$PREFIX/local/lib/${CONTAINER}-${VERSION}.simg"
-    singularity pull --name "$SIF" docker://eicweb/$CONTAINER:$VERSION
-    mv `basename $SIF` $SIF
+    INSIF=`basename ${SIF}`
+    singularity pull --name "${INSIF}" docker://eicweb/$CONTAINER:$VERSION
+    mv ${INSIF} $SIF
+    chmod +x ${SIF}
+    unset INSIF
   fi
 ## we are in sane territory, yay!
 else
   ## check if we can just use cvmfs for the image
+  SIF="$PREFIX/local/lib/${CONTAINER}-${VERSION}.sif"
   if [ -d /cvmfs/singularity.opensciencegrid.org/eicweb/${CONTAINER}:${VERSION} ]; then
-    ln -sf /cvmfs/singularity.opensciencegrid.org/eicweb/${CONTAINER}:${VERSION} local/lib
-    SIF="$PREFIX/local/lib/${CONTAINER}:${VERSION}"
-  elif [ -f /gpfs02/cvmfst0/eic.opensciencegrid.org/singularity/athena/${CONTAINER}_${VERSION}.sif ]; then
-    ln -sf /gpfs02/cvmfst0/eic.opensciencegrid.org/singularity/athena/${CONTAINER}_${VERSION}.sif local/lib
-    SIF="$PREFIX/local/lib/${CONTAINER}-${VERSION}.sif"
+    SIF="$PREFIX/local/lib/${CONTAINER}-${VERSION}"
+    ## need to cleanup in this case, else it will try to make a subdirectory
+    rm -rf ${SIF}
+    ln -sf /cvmfs/singularity.opensciencegrid.org/eicweb/${CONTAINER}:${VERSION} ${SIF}
+  elif [ -f /cvmfs/eic.opensciencegrid.org/singularity/athena/${CONTAINER}_v${VERSION}.sif ]; then
+    ln -sf /cvmfs/eic.opensciencegrid.org/singularity/athena/${CONTAINER}_v${VERSION}.sif ${SIF}
   elif [ -f /gpfs02/cvmfst0/eic.opensciencegrid.org/singularity/athena/${CONTAINER}_v${VERSION}.sif ]; then
-    ln -sf /gpfs02/cvmfst0/eic.opensciencegrid.org/singularity/athena/${CONTAINER}_v${VERSION}.sif local/lib
-    SIF="$PREFIX/local/lib/${CONTAINER}-v${VERSION}.sif"
+    ln -sf /gpfs02/cvmfst0/eic.opensciencegrid.org/singularity/athena/${CONTAINER}_v${VERSION}.sif ${SIF}
   ## if not, download the container to the system
   else
     ## get the python installer and run the old-style install
@@ -129,17 +133,19 @@ else
     wget https://eicweb.phy.anl.gov/containers/eic_container/-/raw/master/install.py
     chmod +x install.py
     ./install.py -f -c $CONTAINER -v $VERSION .
-    INSIF=`ls lib/$CONTAINER.sif.* | head -n1`
-    SIF="$PREFIX/local/lib/${CONTAINER}-${VERSION}.sif"
+    INSIF=lib/`basename ${SIF}`
     mv $INSIF $SIF
+    chmod +x ${SIF}
     ## cleanup
     popd
     rm -rf $tmp_dir
+    unset INSIF
   fi
 fi
 
 echo $SIF
-if [ -z $SIF ]; then
+ls $SIF 2>&1 > /dev/null && GOOD_SIF=1 
+if [ -z "$SIF" -o -z "$GOOD_SIF" ]; then
   echo "ERROR: no singularity image found"
   exit 1
 else