Resolve "Singularity download approach doesn't work in child pipeline"

Added singularity to main pipeline

updated install.py script consistent with new repo structure.

.gitlab-ci.yml

@@ -2,7 +2,8 @@ image: eicweb.phy.anl.gov:4567/containers/image_recipes/ubuntu_dind:latest
 
 stages:
   - config
-  - trigger
+  - build
+  - deploy
 
 ## make note if we cannot use caching for one of the stages
 ## by touching files in .ci_env
@@ -36,16 +37,37 @@ detect_changes:release:
 init:
   stage: config
   script:
-    - ./gitlab-ci/configure_pipeline.sh gitlab-ci/build_and_deploy.yml.in
+    - ./gitlab-ci/configure.sh gitlab-ci/build_and_deploy.yml.in
   artifacts:
     paths:
       - build_and_deploy.yml
 
 ## Dispatch if we ran the previous stage
 run:default:
-  stage: trigger
+  stage: build
   trigger:
     include: 
       - artifact: build_and_deploy.yml
         job: init
     strategy: depend
+
+singularity:
+  stage: deploy
+  needs: ["run:default"]
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master"'
+      when: on_success
+    - if: '$CI_COMMIT_BRANCH =~ /v[0-9]+\.[0-9]+-stable/'
+      when: on_success
+    - if: '$CI_COMMIT_TAG =~ /v[0-9]+\.[0-9]+\.[0-9]+/'
+      when: on_success
+  script:
+     - mkdir -p build 
+     - cd build
+     - ../gitlab-ci/configure.sh ../containers/release/eic.def.in
+     - /bin/bash ../gitlab-ci/singularity/build.sh eic.def
+  artifacts:
+      expire_in: 90 days
+      paths:
+        - build/eic.sif
+        - build/eic.def

containers/release/configure_release.sh → containers/release/configure_dockerfile.sh

@@ -24,8 +24,3 @@ grep export config/spack-env.sh | \
 sed '/^@ENV@/r config/eic-env.sh' containers/release/Dockerfile.in | \
   sed '/^@ENV@/d' | \
   sed "s/@TAG@/$TAG/" > config/Dockerfile
-
-## And release singularity definition
-sed '/^@ENV@/r config/eic-env.sh' containers/release/eic.def.in | \
-  sed '/^@ENV@/d' | \
-  sed "s/@TAG@/$TAG/" > config/eic.def

gitlab-ci/build_and_deploy.yml.in

@@ -4,7 +4,6 @@ stages:
   - build
   - config
   - package
-  - singularity
 
 ## variables:
 ##   - TARGET_XXX: docker build target (including cache modifier) 
@@ -40,12 +39,12 @@ config:
   stage: config
   needs: ["builder"]
   script:
-    - bash containers/release/configure_release.sh $TAG
+    - bash containers/release/configure_dockerfile.sh $TAG
   artifacts:
     paths:
       - config
       
-release:docker:
+release:
   stage: package
   needs: ["config"]
   script:
@@ -61,25 +60,3 @@ release:docker:
   artifacts:
     paths:
       - config
-
-release:singularity:
-  stage: singularity
-  needs: ["release:docker"]
-  rules:
-    - if: '$CI_COMMIT_BRANCH == "master"'
-      when: on_success
-    - if: '$CI_COMMIT_BRANCH == "v@TAG@"'
-      when: on_success
-    - if: '$CI_COMMIT_TAG == "v@TAG@"'
-      when: on_success
-  script:
-     - cp config/eic.def eic.def
-     - /bin/bash gitlab-ci/singularity/build.sh eic.def
-     - mkdir -p build 
-     - cp eic.sif build/.
-     - cp eic.def build/.
-  artifacts:
-      expire_in: 90 days
-      paths:
-        - build/eic.sif
-        - build/eic.def

gitlab-ci/configure_pipeline.sh → gitlab-ci/configure.sh

 #!/bin/bash
 
-## Configure a CI pipeline based on a template file (first and only
+## Configure a CI file based on a template file (first and only
 ## argument to this script).
+
+## Known variables that will be substituted:
+## @TAG@             - docker tag used for this build
+## @TARGET_BUILDER@  - docker Makefile target for the builder image
+## @TARGET_RELEASE@  - docker Makefile target for the release image
+## @PUBLISH@         - docker push Makefile target
+
 TEMPLATE_FILE=$1
 OUTPUT_FILE=`basename ${TEMPLATE_FILE} .in`
 
-echo "Configuring pipeline script: ${TEMPLATE_FILE}"
+echo "Configuring CI file: ${TEMPLATE_FILE}"
 echo "Output will be written to: ${OUTPUT_FILE}"
 
 VERSION=`head -n1 VERSION`

install.py

@@ -15,6 +15,7 @@ Authors:
 
 import os
 import argparse
+import re
 import urllib.request
 from install import make_launcher, make_modulefile
 from install.util import smart_mkdir, project_version, InvalidArgumentError
@@ -30,7 +31,7 @@ PROGRAMS = ['eic-shell',
             'ipython']
 
 ## URL for the current container (git tag will be filled in by the script)
-CONTAINER_URL = r'https://eicweb.phy.anl.gov/api/v4/projects/290/jobs/artifacts/{version}/raw/build/{img}.sif?job=release:singularity'
+CONTAINER_URL = r'https://eicweb.phy.anl.gov/api/v4/projects/290/jobs/artifacts/{version}/raw/build/{img}.sif?job=singularity'
 #api/v4/projects/1/jobs/artifacts/master/raw/some/release/file.pdf
 
 ## Singularity bind directive
@@ -89,14 +90,34 @@ if __name__ == "__main__":
                 raise InvalidArgumentError()
         bind_directive = ' '.join([BIND_DIRECTIVE.format(path) for path in args.bind_paths])
 
-    ## We want to slightly modify our version specifier: if it leads with a 'v' drop the v
-    ## for everything installed, but ensure we have the leading v as well where needed
-    version = '{}'.format(args.version)
-    vversion = '{}'.format(args.version)
-    if version[0] == 'v':
-        version = version[1:]
-    if vversion[0].isdigit():
-        vversion= 'v{}'.format(args.version)
+    ## Naming schemes:
+    ## We need to deduce both the correct git branch and an appropriate
+    ## local version number from the desired version number
+    ## by default we use whatever version number is given in VERSION, but we want
+    ## to allow users to specify either X.Y.Z or vX.Y.Z for versions (same for stable
+    ## branches).
+    ## 
+    ## Policy:
+    ## numbered releases: (v)X.Y.Z --> git vX.Y.Z and local X.Y.Z
+    ## stable branches: (v)X.Y-stable --> git vX.Y-stable and local X.Y-stable
+    ## master branch: latest/master --> git master and local stable
+    ## for other branches --> git <BRANCH> and local unstable
+
+    version_local = None
+    version_repo = None
+    if args.version in ('master', 'latest'):
+        version_local = 'latest'
+        version_repo = 'master'
+    elif re.search('[0-9]+\.[0-9]+\.[0-9]|[0-9]+\.[0-9]-stable', args.version) is not None:
+        version_local = args.version
+        version_repo = args.version
+        if version_local[0] == 'v':
+            version_local = version_local[1:]
+        if version_repo[0].isdigit():
+            version_repo = 'v{}'.format(args.version)
+    else:
+        version_local = 'unstable'
+        version_repo = args.version
 
     ## Create our install prefix if needed and ensure it is writable
     args.prefix = os.path.abspath(args.prefix)
@@ -125,10 +146,10 @@ if __name__ == "__main__":
     ## Builder SIF is not built anymore, deprecated
     #if args.builder:
         #img += "_builder"
-    container = '{}/{}.sif.{}'.format(libdir, img, version)
+    container = '{}/{}.sif.{}'.format(libdir, img, version_local)
     if not os.path.exists(container) or args.force:
         url = CONTAINER_URL.format(group=GROUP_NAME, project=PROJECT_NAME,
-                version=vversion, img=img)
+                version=version_repo, img=img)
         print('Downloading container from:', url)
         print('Destination:', container)
         urllib.request.urlretrieve(url, container)
@@ -137,7 +158,7 @@ if __name__ == "__main__":
         print(' ---> run with -f to force a re-download')
 
     if not args.local:
-        make_modulefile(PROJECT_NAME, version, moduledir, bindir)
+        make_modulefile(PROJECT_NAME, version_local, moduledir, bindir)
 
     ## configure the application launchers
     print('Configuring applications launchers: ')