Skip to content
Snippets Groups Projects
Commit ce01b8f8 authored by Sylvester Joosten's avatar Sylvester Joosten Committed by Sylvester Joosten
Browse files

Resolve "Simplify and consolidate container build" 

New container names are jug_dev and jug_xl (dressed container)
parent 983ac450
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
gitlab-ci/docker_push.sh 0 → 100755
+ 132
− 0
View file @ ce01b8f8
#!/bin/bash
## Generic script to execute docker push from the input tag to all export tags
## on both dockerhub and eicweb if set
function print_the_help {
echo "USAGE: -i image -l input_tag export_tag [export_tag2 ...]"
echo "ARGUMENTS:"
echo " -i,--image Input image name (e.g., eic_base)"
echo " -l,--label Input tag (eg., v3.0.0)"
echo " -t,--time Time interval (in seconds) between attempts"
echo " (doubled each time), default: 5"
echo " -n,--n-attempts Number of attempts, default: 5"
echo " -h,--help Print this message"
echo " --eicweb Publish to $CI_REGISTRY only"
echo " positional At least one export tag (e.g., v3.0-stable)"
echo ""
echo " Execute docker push from image:input_tag to REGISTRY/image:export_tag for"
echo " for all export tags. Will push to $CI_REGISTRY, and optionally also to"
echo " Dockerhub if the DH_PUSH environment variable is set"
echo ""
echo "EXAMPLE: ./docker_push.sh -i eic_base -l 3.0.0 3.0.0 3.0-stable"
exit
}
IMAGE=
INPUT_TAG=
EXPORT_TAGS=()
NTRIES=5
TIME=5
DO_DH=${DH_PUSH}
while [ $# -gt 0 ]; do
key=$1
case $key in
-i|--image)
IMAGE=$2
shift
shift
;;
-l|--label)
INPUT_TAG=$2
shift
shift
;;
-t|--time)
TIME=$2
shift
shift
;;
-n|--n-attempts)
NTRIES=$2
shift
shift
;;
--eicweb)
DO_DH=0
shift
;;
-h|--help)
print_the_help
exit 0
;;
-*)
echo "ERROR: unknown flag: $key"
echo "use --help for more info"
exit 1
;;
*)
EXPORT_TAGS+=("$1")
shift
;;
esac
done
if [ -z $IMAGE ]; then
echo "ERROR: no image name given, please use -i <IMAGE>"
print_the_help
exit 1
fi
if [ -z $INPUT_TAG ]; then
echo "ERROR: no input_Tag given, please use -t <INPUT_TAG>"
print_the_help
exit 1
fi
if [ ${#EXPORT_TAGS[@]} -eq 0 ]; then
echo "ERROR: need at least one export tag (positional argument)"
print_the_help
exit 1
fi
function retry_push () {
INPUT=$1
DESTINATION=$2
time=$TIME
ntries=$NTRIES
echo "Pushing ${INPUT} to ${DESTINATION}"
if [ ${INPUT} != ${DESTINATION} ]; then
echo docker tag ${INPUT} ${DESTINATION}
docker tag ${INPUT} ${DESTINATION}
fi
while [ $ntries != 0 ]; do
echo docker push ${DESTINATION}
docker push ${DESTINATION} && break \
|| echo "Docker push failed, retrying in $time seconds..."
sleep ${time}s
time=$((time * 2))
ntries=$((ntries - 1))
done
if [ ${INPUT} != ${DESTINATION} ]; then
echo docker rmi ${DESTINATION}
docker rmi ${DESTINATION}
fi
if [ $ntries = 0 ]; then
echo "Failed to push $INPUT to $DESTINATION"
exit 1
fi
}
#echo "IMAGE: ${IMAGE}"
#echo "INPUT_TAG: ${INPUT_TAG}"
#echo "EXPORT_TAGS: ${EXPORT_TAGS}"
#echo "DH_PUSH: ${DH_PUSH}"
#echo "DO_DH: ${DO_DH}"
export INPUT=$CI_REGISTRY_IMAGE/${IMAGE}:${INPUT_TAG}
for TAG in ${EXPORT_TAGS[@]}; do
retry_push $INPUT $CI_REGISTRY_IMAGE/${IMAGE}:${TAG}
if [ ${DO_DH} != 0 ]; then
retry_push $INPUT $DH_REGISTRY/${IMAGE}:${TAG}
fi
done
gitlab-ci/singularity/build.sh deleted 100644 → 0
+ 0
− 176
View file @ 983ac450
#!/bin/bash
# build.sh will build a Singularity container. It's not overly complicated.
#
# USAGE: build.sh --uri collection-name/container-name --cli registry Singularity
# build.sh --uri collection-name/container-name --cli registry
# build.sh Singularity
# Copyright (C) 2017-2018 Vanessa Sochat.
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
set -o errexit
set -o nounset
function usage() {
echo "USAGE: build [recipe] [options]"
echo ""
echo "OPTIONS:
Image Format
--uri -u if uploading, a uri to give to sregistry
--cli -c the sregistry client to use (if uploading)
--help -h show this help and exit
"
}
# --- Option processing --------------------------------------------------------
uri=""
cli=""
tag=""
while true; do
case ${1:-} in
-h|--help|help)
usage
exit 0
;;
-u|-uri)
shift
uri="${1:-}"
shift
;;
-t|--tag)
shift
tag="${1:-}"
shift
;;
-c|--cli)
shift
cli="${1:-}"
shift
;;
\?) printf "illegal option: -%s\n" "${1:-}" >&2
usage
exit 1
;;
-*)
printf "illegal option: -%s\n" "${1:-}" >&2
usage
exit 1
;;
*)
break;
;;
esac
done
################################################################################
### Recipe File ################################################################
################################################################################
if [ $# == 0 ] ; then
recipe="Singularity"
else
recipe=$1
fi
echo ""
echo "Image Recipe: ${recipe}"
################################################################################
### Storage Client #############################################################
################################################################################
is_valid_client () {
local e match="$1"
shift
for e; do [[ "$e" == "$match" ]] && return 0; done
return 1
}
# Test if client is valid
clients=("google-storage" "registry" "globus" "dropbox" "google-drive")
if [ "${cli}" != "" ]; then
is_valid_client "${cli}" "${clients[@]}"
if [ $? -ne 0 ]; then
echo "${cli} is not a valid choice! Choose from ${clients[@]}";
exit 1
fi
echo "Storage Client: ${cli}"
else
echo "Storage Client: none"
fi
################################################################################
### Build! #####################################################################
################################################################################
# Continue if the image recipe is found
if [ -f "$recipe" ]; then
imagefile="`basename ${recipe} .def`.sif"
echo "Creating $imagefile using $recipe..."
singularity build $imagefile $recipe
# If the image is successfully built, test it and upload (examples)
if [ -f "${imagefile}" ]; then
# Example testing using run (you could also use test command)
echo "Testing the image... Marco!"
singularity exec $imagefile echo "Polo!"
# Example sregistry commands to push to endpoints
if [ "${cli}" != "" ]; then
# If the uri isn't provided, he gets a robot name
if [ "${uri}" == "" ]; then
uri=$(python -c "from sregistry.logger.namer import RobotNamer; bot=RobotNamer(); print(bot.generate())")
fi
# If a tag is provided, add to uri
if [ "${tag}" != "" ]; then
uri="${uri}:${tag}"
fi
echo "Pushing ${uri} to ${cli}://"
echo "SREGISTRY_CLIENT=${cli} sregistry push --name ${uri} ${imagefile}"
SREGISTRY_CLIENT="${cli}" sregistry push --name "${uri}" "${imagefile}"
else
echo "Skipping upload. Image $imagefile is finished!"
fi
fi
else
echo "Singularity recipe ${recipe} not found!"
exit 1
fi
install.py
+ 87
− 50
View file @ ce01b8f8
......@@ -21,32 +21,57 @@ from install import make_launcher, make_modulefile
from install.util import smart_mkdir, project_version, InvalidArgumentError
## Gitlab group and project/program name.
GROUP_NAME='containers'
PROJECT_NAME='eic_container'
IMAGE_ROOT='eic'
DEFAULT_IMG='eic'
DEFAULT_VERSION='2.9.2'
PROGRAMS = ['eic-shell',
'container_dev',
#'root',
'ipython']
SHORTCUTS = ['eic-shell']
## URL for the current container (git tag will be filled in by the script)
CONTAINER_URL = r'https://eicweb.phy.anl.gov/api/v4/projects/290/jobs/artifacts/{version}/raw/build/{img}.sif?job=singularity'
#api/v4/projects/1/jobs/artifacts/master/raw/some/release/file.pdf
## components:
## - {ref}:
## - branch/tag --> git branch or tag
## - MR XX --> refs/merge-requests/XX/head
## - nightly --> just use fallback singularity pull
## - {img}: image name
## - {job}: the CI job that built the artifact
CONTAINER_URL = r'hhttps://eicweb.phy.anl.gov/api/v4/projects/290/jobs/artifacts/{ref}/raw/build/{img}.sif?job={job}'
## Docker ref is used as fallback in case regular artifact download fails
## The components are:
## - {img}: image name
## - {tag}: docker tag associated with image
## - master --> testing
## - branch/tag --> branch/tag without leading v
## - MR XX --> unstable (may be incorrect if multiple MRs active)
## - nightly --> nightly
DOCKER_REF = r'docker://eicweb/{img}:{tag}'
## Singularity bind directive
BIND_DIRECTIVE= '-B {0}:{0}'
class UnknownVersionError(Exception):
pass
class ContainerDownloadError(Exception):
pass
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
'prefix',
help='Install prefix. This is where the container will be deployed.')
parser.add_argument(
'-c', '--container',
dest='container',
default=DEFAULT_IMG,
help='(opt.) Container to install. '
'D: {} (will migrate to jug_xl for v3.0).'.format(DEFAULT_IMG))
parser.add_argument(
'-v', '--version',
dest='version',
default=project_version(),
help='(opt.) project version. Default: current version (in repo).')
# default=project_version(),
default=DEFAULT_VERSION,
help='(opt.) project version. '
'D: {}. For MRs, use mr-XXX.'.format(DEFAULT_VERSION))
parser.add_argument(
'-f', '--force',
action='store_true',
......@@ -60,24 +85,12 @@ if __name__ == "__main__":
parser.add_argument(
'-m', '--module-path',
dest='module_path',
help='(opt.) Root module path where you want to install a modulefile. D: <prefix>/../../etc/modulefiles')
parser.add_argument(
'-l', '--local',
action='store_true',
dest='local',
help='Local deploy, will not install the modulefiles (you will have to run'
'the launchers scripts from their relative paths).')
## deprecated, we should just make sure the release image is good enough
## builder singularity image will most likely be removed from the CI
## in a future release
#parser.add_argument(
#'--install-builder',
#dest='builder',
#help='(opt.) Install fat builder image, instead of normal slim image')
help='(opt.) Root module path to install a modulefile. '
'D: Do not install a modulefile')
args = parser.parse_args()
print('Deploying', PROJECT_NAME, 'version', args.version)
print('Deploying', args.container, 'version', args.version)
## Check if our bind paths are valid
bind_directive = ''
......@@ -103,35 +116,50 @@ if __name__ == "__main__":
## master branch: latest/master --> git master and local stable
## for other branches --> git <BRANCH> and local unstable
version_local = None
version_repo = None
if args.version in ('master', 'latest'):
version_local = 'latest'
version_repo = 'master'
version_docker = None
version_gitlab = None
build_job = '{}:singularity:default'.format(args.container)
if args.version in ('master', 'testing'):
version_docker = 'testing'
version_gitlab = 'master'
elif re.search('[0-9]+\.[0-9]+\.[0-9]|[0-9]+\.[0-9]-stable', args.version) is not None:
version_local = args.version
version_repo = args.version
if version_local[0] == 'v':
version_local = version_local[1:]
if version_repo[0].isdigit():
version_repo = 'v{}'.format(args.version)
version_docker = args.version
version_gitlab = args.version
if version_docker[0] == 'v':
version_docker = version_docker[1:]
if version_gitlab[0].isdigit():
version_gitlab = 'v{}'.format(args.version)
elif args.version[:3] == 'mr-':
version_docker = 'unstable'
version_gitlab = 'refs/merge-requests/{}/head'.format(args.version[3:])
elif args.version == 'nightly':
version_docker = 'nightly'
version_gitlab = 'master'
build_job = '{}:singularity:nightly'.format(args.container)
else:
version_local = 'unstable'
version_repo = args.version
## fixme add proper error handling
print('Unknown requested version:', args.version)
raise UnknownVersionError()
## when working with the old container, the build job is just 'singularity'
if args.container == 'eic':
build_job = 'singularity'
## Create our install prefix if needed and ensure it is writable
args.prefix = os.path.abspath(args.prefix)
if not args.module_path:
args.module_path = os.path.abspath('{}/../../etc/modulefiles'.format(args.prefix))
deploy_local=True
else:
deploy_local=False
print('Install prefix:', args.prefix)
print('Creating install prefix if needed...')
bindir = '{}/bin'.format(args.prefix)
libdir = '{}/lib'.format(args.prefix)
libexecdir = '{}/libexec'.format(args.prefix)
root_prefix = os.path.abspath('{}/..'.format(args.prefix))
moduledir = '{}/{}'.format(args.module_path, PROJECT_NAME)
dirs = [bindir, libdir, libexecdir]
if not args.local:
if not deploy_local:
moduledir = '{}/{}'.format(args.module_path, args.container)
dirs.append(moduledir)
for dir in dirs:
print(' -', dir)
......@@ -142,27 +170,36 @@ if __name__ == "__main__":
## Get the container
## We want to slightly modify our version specifier: if it leads with a 'v' drop the v
img = IMAGE_ROOT
img = args.container
## Builder SIF is not built anymore, deprecated
#if args.builder:
#img += "_builder"
container = '{}/{}.sif.{}'.format(libdir, img, version_local)
container = '{}/{}.sif.{}'.format(libdir, img, version_docker)
if not os.path.exists(container) or args.force:
url = CONTAINER_URL.format(group=GROUP_NAME, project=PROJECT_NAME,
version=version_repo, img=img)
url = CONTAINER_URL.format(ref=version_gitlab, img=img, job=build_job)
print('Downloading container from:', url)
print('Destination:', container)
try:
urllib.request.urlretrieve(url, container)
except:
print('WARNING: failed to retrieve container artifact')
print('Attempting alternative download from docker registry')
cmd = ['singularity pull', '--force', container, DOCKER_REF.format(img=img, tag=version_docker)]
cmd = ' '.join(cmd)
print('Executing:', cmd)
err = os.system(cmd)
if err:
raise ContainerDownloadError()
else:
print('WARNING: Container found at', container)
print(' ---> run with -f to force a re-download')
if not args.local:
make_modulefile(PROJECT_NAME, version_local, moduledir, bindir)
if not deploy_local:
make_modulefile(args.container, version_docker, moduledir, bindir)
## configure the application launchers
print('Configuring applications launchers: ')
for prog in PROGRAMS:
for prog in SHORTCUTS:
app = prog
exe = prog
if type(prog) == tuple:
......
containers/builder/spack.yaml → spack.yaml
+ 0
− 0
View file @ ce01b8f8
File moved
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment