Added python-based deploy scripts, and updated Readme.

README.md

@@ -2,7 +2,40 @@ EIC software container
 ============================================
 
 Installation
------------
+------------
+
+1. Clone the repository and go into the directory
+```bash
+git clone https://eicweb.phy.anl.gov/containers/eic_container.git
+cd eic_container
+```
+
+2. Run the deploy script `deploy.py` to install to your `<PREFIX>` of choice 
+   (e.g. $HOME/local/opt/eic_container_1.0.4). By default the
+   modelefile will be installed to `$PREFIX/../../etc/modulefiles`. 
+   You can use the `-v` flag to select the version you want to deploy, or omit the 
+   flag if you want to install the master build. The recommended stable 
+   release version is `v1.0.4`.
+```bash
+./deploy.py -v 1.0.4 <PREFIX>
+```
+
+3. To use the container: load the modulefile, and then use the included apps as if
+   they are native apps on your system!
+```
+module load eic_container
+```
+
+4. (Advanced) If you need to add additional bind directives for the internal singularity container,
+   you can add them with the `-b` flag. Run `./deploy.py -h` to see a list of all
+   supported options.
+
+
+Installation (throug cmake)
+---------------------------
+
+*Use of the cmake-based deploy is deprecated, We recommend to use the `deploy.py` method
+instead.*
 
 1. Checkout the repository and create a build directory
 ```

deploy.py

+#!/usr/bin/env python3
+
+## eic_container: Argonne Universal EIC Container
+
+'''
+Deploy the singularity container built by the CI for this version of the software.
+
+The current version is determined from the currently loaded git branch or tag,
+unless it is explicitly set on the command line.
+
+Authors:
+    - Whitney Armstrong <warmstrong@anl.gov>
+    - Sylvester Joosten <sjoosten@anl.gov>
+'''
+
+import os
+import argparse
+import urllib.request
+from install import make_launcher, make_modulefile
+from install.util import smart_mkdir, project_version, InvalidArgumentError
+
+## Gitlab group and project/program name. 
+GROUP_NAME='containers'
+PROJECT_NAME='eic_container'
+PROGRAMS = [('container_dev', '/usr/bin/bash'),
+            'ddsim', 
+            'geoConverter',
+            'materialScan',
+            'geoDisplay',
+            'geoPluginRun',
+            'teveDisplay',
+            'ddeve',
+            'g4FromXML'
+            'geoDisplay',
+            'listcomponents',
+            'print_materials',
+            'dumpBfield',
+            'g4gdmlDisplay',
+            'geoPluginRun',
+            'materialBudget',
+            'pyddg4',
+            'dumpdetector',
+            'graphicalScan',
+            'root',
+            'root-config',
+            'rootbrowse',
+            'rootls',
+            'mongo',
+            'mongod',
+            'mongodump',
+            'mongoexport',
+            'mongoimport',
+            'mongostat']
+
+## URL for the current container (git tag will be filled in by the script)
+CONTAINER_URL = r'https://eicweb.phy.anl.gov/{group}/{project}/-/jobs/artifacts/{version}/raw/build/eic.sif?job=eic_singularity'
+
+CONTAINER_ENV=r'''source /usr/local/bin/thisdd4hep.sh
+ROOT_INCLUDE_PATH=/usr/local/include:/usr/include/eigen3:$ROOT_INCLUDE_PATH
+'''
+
+## Singularity bind directive
+BIND_DIRECTIVE= '-B {0}:{0}'
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+            'prefix',
+            help='Install prefix. This is where the container will be deployed.')
+    parser.add_argument(
+            '-v', '--version',
+            dest='version',
+            default=project_version(),
+            help='(opt.) project version. Default: current git branch/tag.')
+    parser.add_argument(
+            '-f', '--force',
+            action='store_true',
+            help='Force-overwrite already downloaded container',
+            default=False)
+    parser.add_argument(
+            '-b', '--bind-path',
+            dest='bind_paths',
+            action='append',
+            help='(opt.) extra bind paths for singularity.')
+    parser.add_argument(
+            '-m', '--module-path',
+            dest='module_path',
+            help='(opt.) Root module path where you want to install a modulefile. D: <prefix>/../../etc/modulefiles')
+
+    args = parser.parse_args()
+
+    print('Deploying', PROJECT_NAME, 'version', args.version)
+
+    ## Check if our bind paths are valid
+    bind_directive = ''
+    if args.bind_paths and len(args.bind_paths):
+        print('Singularity bind paths:')
+        for path in args.bind_paths:
+            print(' -', path)
+            if not os.path.exists(path):
+                print('ERROR: path', path, 'does not exist.')
+                raise InvalidArgumentError()
+        bind_directive = ' '.join([BIND_DIRECTIVE.format(path) for path in args.bind_paths])
+
+    ## We want to slightly modify our version specifier: if it leads with a 'v' drop the v
+    ## for everything installed, but ensure we have the leading v as well where needed
+    version = '{}'.format(args.version)
+    vversion = '{}'.format(args.version)
+    if version[0] is 'v':
+        version = version[1:]
+    if vversion[0].isdigit():
+        vversion= 'v{}'.format(args.version)
+
+    ## Create our install prefix if needed and ensure it is writable
+    args.prefix = os.path.abspath(args.prefix)
+    if not args.module_path:
+        args.module_path = '{}/etc/modulefiles'.format(args.prefix)
+    print('Install prefix:', args.prefix)
+    print('Creating install prefix if needed...')
+    bindir = '{}/bin'.format(args.prefix)
+    libdir = '{}/lib'.format(args.prefix)
+    libexecdir = '{}/libexec'.format(args.prefix)
+    root_prefix = os.path.abspath('{}/..'.format(args.prefix))
+    moduledir = '{}/etc/modulefiles/{}'.format(root_prefix, PROJECT_NAME)
+    for dir in [bindir, libdir, libexecdir, moduledir]:
+        print(' -', dir)
+        smart_mkdir(dir)
+
+    ## At this point we know we can write to our desired prefix and that we have a set of
+    ## valid bind paths
+
+    ## Get the container
+    ## We want to slightly modify our version specifier: if it leads with a 'v' drop the v
+    container = '{}/{}.sif.{}'.format(libdir, PROJECT_NAME, version)
+    if not os.path.exists(container) or args.force:
+        url = CONTAINER_URL.format(group=GROUP_NAME, project=PROJECT_NAME, version=vversion)
+        print('Downloading container from:', url)
+        print('Destination:', container)
+        urllib.request.urlretrieve(url, container)
+    else:
+        print('WARNING: Container found at', container)
+        print(' ---> run with -f to force a re-download')
+
+    make_modulefile(PROJECT_NAME, version, moduledir, bindir)
+
+    ## configure the application launchers
+    print('Configuring applications launchers: ')
+    for prog in PROGRAMS:
+        app = prog
+        exe = prog
+        if type(prog) == tuple:
+            app = prog[0]
+            exe = prog[1]
+        make_launcher(app, container, bindir,
+                      bind=bind_directive,
+                      libexecdir=libexecdir,
+                      exe=exe,
+                      env=CONTAINER_ENV)
+
+    print('Container deployment successful!')

install/__init__.py

+#!/usr/bin/env python3
+
+## eic_container: Argonne Universal EIC Container
+
+from install.util import smart_mkdir, project_version
+from install.launcher import make_launcher
+from install.modulefile import make_modulefile

install/launcher.py

+#!/usr/bin/env python3
+
+'''
+Generic launcher script to launch applications in this container.
+
+The launcher script fires off an auxilary wrapper script in the container,
+responsible to correctly setup the environment and then launch the application
+of choice.
+
+Authors:
+    - Whitney Armstrong <warmstrong@anl.gov>
+    - Sylvester Joosten <sjoosten@anl.gov>
+'''
+
+import os
+
+## generic launcher bash script to launch the application
+_LAUNCHER='''#!/usr/bin/env bash
+
+## Boilerplate to make pipes work
+piped_args=
+if [ -p /dev/stdin ]; then
+  # If we want to read the input line by line
+  while IFS= read line; do
+    if [ -z "$piped_args" ]; then
+      piped_args="${{line}}"
+    else 
+      piped_args="${{piped_args}}\n${{line}}"
+    fi
+  done
+fi
+
+## Fire off the application wrapper
+if [ ${{piped_args}} ]  ; then
+    echo -e ${{piped_args}} | singularity exec {bind} {container} {wrapper} $@
+else
+    singularity exec {bind} {container} {wrapper} $@
+fi
+'''
+
+## Wrapper script called from within the container that loads the propper environment and
+## to then actually call our app
+_WRAPPER='''#!/usr/bin/env bash
+
+## setup container environment
+{env}
+
+## Boilerplate to make pipes work
+piped_args=
+if [ -p /dev/stdin ]; then
+  # If we want to read the input line by line
+  while IFS= read line; do
+    if [ -z "$piped_args" ]; then
+      piped_args="${{line}}"
+    else 
+      piped_args="${{piped_args}}\n${{line}}"
+    fi
+  done
+fi
+
+## Launch the exe
+if [ ${{piped_args}} ]  ; then
+    echo -e ${{piped_args}} | {exe} $@
+else
+    {exe} $@
+fi
+'''
+
+def _write_script(path, content):
+    print(' - creating', path)
+    with open(path, 'w') as file:
+        file.write(content)
+    os.system('chmod +x {}'.format(path))
+    
+def make_launcher(app, container, bindir, 
+                  bind='', libexecdir=None, exe=None, env=''):
+    '''Configure and install a launcher/wrapper pair.
+
+    Arguments:
+        - app: our application
+        - container: absolute path to container
+        - bindir: absolute launcher install path
+    Optional:
+        - bind: singularity bind directives
+        - libexecdir: absolute wrapper install path. 
+                      Default is bindir.
+        - exe: executable to be associated with app. 
+               Default is app.
+        - env: environment directives to be added to the wrapper. 
+               Multiline string. Default is nothing
+    '''
+    ## assume bindir and libexecdir exist, are absolute, and are writable
+    if libexecdir is None:
+        libexecdir = bindir
+
+    ## actual exe we want to run, default: same as app
+    exe=app
+    
+    ## paths
+    launcher_path = '{}/{}'.format(bindir, app)
+    wrapper_path = '{}/{}_wrap'.format(libexecdir, app)
+
+    ## scripts --> use absolute path for wrapper path inside launcher
+    launcher = _LAUNCHER.format(container=container, 
+                                bind=bind,
+                                wrapper=wrapper_path)
+    wrapper = _WRAPPER.format(env=env, exe=exe)
+
+    ## write our scripts
+    _write_script(launcher_path, launcher)
+    _write_script(wrapper_path, wrapper)

install/modulefile.py

+#!/usr/bin/env python3
+
+## eic_container: Argonne Universal EIC Container
+
+'''
+Install modulefile for this container.
+
+Authors:
+    - Whitney Armstrong <warmstrong@anl.gov>
+    - Sylvester Joosten <sjoosten@anl.gov>
+'''
+
+import os
+
+## Generic module file
+_MODULEFILE='''#%Module1.0#####################################################################
+##
+## for {name} {version}
+##
+proc ModulesHelp {{ }} {{
+    puts stderr "This module sets up the environment for the {name} container"
+}}
+module-whatis "{name} {version}"
+
+# For Tcl script use only
+set version 4.1.4
+
+prepend-path    PATH    {bindir}
+'''
+
+def make_modulefile(project, version, moduledir, bindir):
+    '''Configure and install a modulefile for this project.
+
+    Arguments:
+        - project: project name
+        - version: project version
+        - moduledir: root modulefile directory
+        - bindir: where executables for this project are located
+    '''
+
+    ## create our modulefile
+    content = _MODULEFILE.format(name=project, version=version, bindir=bindir)
+    fname = '{}/{}'.format(moduledir, version)
+    print(' - creating', fname)
+    with open(fname, 'w') as file:
+        file.write(content)

install/util.py

+#!/usr/bin/env python3
+
+## eic_container: Argonne Universal EIC Container
+
+'''
+Utility functions for this container
+
+Authors:
+    - Whitney Armstrong <warmstrong@anl.gov>
+    - Sylvester Joosten <sjoosten@anl.gov>
+'''
+
+import os
+
+class InvalidArgumentError(Exception):
+    pass
+
+def smart_mkdir(dir):
+    '''functions as mkdir -p, with a write-check.
+    
+    Raises an exception if the directory is not writeable.
+    '''
+    if not os.path.exists(dir):
+        try:
+            os.makedirs(dir)
+        except Exception as e:
+            print('ERROR: unable to create directory', dir)
+            raise e
+    if not os.access(dir, os.W_OK):
+        print('ERROR: We do not have the write privileges to', dir)
+        raise InvalidArgumentError()
+
+def project_version():
+    '''Return the project version based on the current git branch/tag.'''
+    ## Shell command to get the current git version
+    git_version_cmd = 'git symbolic-ref -q --short HEAD || git describe --tags --exact-match'
+    ## Strip will remove the leading \n character
+    return os.popen(git_version_cmd).read().strip()