Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
E
eic_container
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container registry
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
containers
eic_container
Commits
a1da8057
Commit
a1da8057
authored
4 years ago
by
Sylvester Joosten
Browse files
Options
Downloads
Patches
Plain Diff
simplify deploy as wrappers are no longer needed
parent
ec7c448d
Branches
Branches containing commit
Tags
Tags containing commit
1 merge request
!4
Container now includes proper environment setup
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
deploy.py
+12
-12
12 additions, 12 deletions
deploy.py
install/launcher.py
+11
-47
11 additions, 47 deletions
install/launcher.py
with
23 additions
and
59 deletions
deploy.py
+
12
−
12
View file @
a1da8057
...
@@ -29,9 +29,7 @@ PROGRAMS = ['eic-shell',
...
@@ -29,9 +29,7 @@ PROGRAMS = ['eic-shell',
'
ipython
'
]
'
ipython
'
]
## URL for the current container (git tag will be filled in by the script)
## URL for the current container (git tag will be filled in by the script)
CONTAINER_URL
=
r
'
https://eicweb.phy.anl.gov/{group}/{project}/-/jobs/artifacts/{version}/raw/build/{img}.sif?job={img}_singularity
'
CONTAINER_URL
=
r
'
https://eicweb.phy.anl.gov/{group}/{project}/-/jobs/artifacts/{version}/raw/build/{img}.sif?job=release_singularity
'
CONTAINER_ENV
=
r
'''
source /etc/profile
'''
## Singularity bind directive
## Singularity bind directive
BIND_DIRECTIVE
=
'
-B {0}:{0}
'
BIND_DIRECTIVE
=
'
-B {0}:{0}
'
...
@@ -66,10 +64,13 @@ if __name__ == "__main__":
...
@@ -66,10 +64,13 @@ if __name__ == "__main__":
dest
=
'
local
'
,
dest
=
'
local
'
,
help
=
'
Local deploy, will not install the modulefiles (you will have to run
'
help
=
'
Local deploy, will not install the modulefiles (you will have to run
'
'
the launchers scripts from their relative paths).
'
)
'
the launchers scripts from their relative paths).
'
)
parser
.
add_argument
(
## deprecated, we should just make sure the release image is good enough
'
--install-builder
'
,
## builder singularity image will most likely be removed from the CI
dest
=
'
builder
'
,
## in a future release
help
=
'
(opt.) Install fat builder image, instead of normal slim image
'
)
#parser.add_argument(
#'--install-builder',
#dest='builder',
#help='(opt.) Install fat builder image, instead of normal slim image')
args
=
parser
.
parse_args
()
args
=
parser
.
parse_args
()
...
@@ -119,8 +120,9 @@ if __name__ == "__main__":
...
@@ -119,8 +120,9 @@ if __name__ == "__main__":
## Get the container
## Get the container
## We want to slightly modify our version specifier: if it leads with a 'v' drop the v
## We want to slightly modify our version specifier: if it leads with a 'v' drop the v
img
=
IMAGE_ROOT
img
=
IMAGE_ROOT
if
args
.
builder
:
## Builder SIF is not built anymore, deprecated
img
+=
"
_builder
"
#if args.builder:
#img += "_builder"
container
=
'
{}/{}.sif.{}
'
.
format
(
libdir
,
img
,
version
)
container
=
'
{}/{}.sif.{}
'
.
format
(
libdir
,
img
,
version
)
if
not
os
.
path
.
exists
(
container
)
or
args
.
force
:
if
not
os
.
path
.
exists
(
container
)
or
args
.
force
:
url
=
CONTAINER_URL
.
format
(
group
=
GROUP_NAME
,
project
=
PROJECT_NAME
,
url
=
CONTAINER_URL
.
format
(
group
=
GROUP_NAME
,
project
=
PROJECT_NAME
,
...
@@ -145,8 +147,6 @@ if __name__ == "__main__":
...
@@ -145,8 +147,6 @@ if __name__ == "__main__":
exe
=
prog
[
1
]
exe
=
prog
[
1
]
make_launcher
(
app
,
container
,
bindir
,
make_launcher
(
app
,
container
,
bindir
,
bind
=
bind_directive
,
bind
=
bind_directive
,
libexecdir
=
libexecdir
,
exe
=
exe
)
exe
=
exe
,
env
=
CONTAINER_ENV
)
print
(
'
Container deployment successful!
'
)
print
(
'
Container deployment successful!
'
)
This diff is collapsed.
Click to expand it.
install/launcher.py
+
11
−
47
View file @
a1da8057
...
@@ -3,9 +3,9 @@
...
@@ -3,9 +3,9 @@
'''
'''
Generic launcher script to launch applications in this container.
Generic launcher script to launch applications in this container.
The launcher script
fires off an auxilary wrapper script in the container,
The launcher script
calls the desired executable from the singularity image.
responsible to correctly setup the environment and then launch the application
As the new images have the environment properly setup, we can accomplish this
of choice
.
without using any wrapper scripts
.
Authors:
Authors:
- Whitney Armstrong <warmstrong@anl.gov>
- Whitney Armstrong <warmstrong@anl.gov>
...
@@ -32,37 +32,9 @@ fi
...
@@ -32,37 +32,9 @@ fi
## Fire off the application wrapper
## Fire off the application wrapper
if [ ${{piped_args}} ] ; then
if [ ${{piped_args}} ] ; then
echo -e ${{piped_args}} | singularity exec {bind} {container} {
wrapper
} $@
echo -e ${{piped_args}} | singularity exec {bind} {container} {
exe
} $@
else
else
singularity exec {bind} {container} {wrapper} $@
singularity exec {bind} {container} {exe} $@
fi
'''
## Wrapper script called from within the container that loads the propper environment and
## to then actually call our app
_WRAPPER
=
'''
#!/usr/bin/env bash
## setup container environment
{env}
## Boilerplate to make pipes work
piped_args=
if [ -p /dev/stdin ]; then
# If we want to read the input line by line
while IFS= read line; do
if [ -z
"
$piped_args
"
]; then
piped_args=
"
${{line}}
"
else
piped_args=
"
${{piped_args}}
\n
${{line}}
"
fi
done
fi
## Launch the exe
if [ ${{piped_args}} ] ; then
echo -e ${{piped_args}} | {exe} $@
else
{exe} $@
fi
fi
'''
'''
...
@@ -73,8 +45,8 @@ def _write_script(path, content):
...
@@ -73,8 +45,8 @@ def _write_script(path, content):
os
.
system
(
'
chmod +x {}
'
.
format
(
path
))
os
.
system
(
'
chmod +x {}
'
.
format
(
path
))
def
make_launcher
(
app
,
container
,
bindir
,
def
make_launcher
(
app
,
container
,
bindir
,
bind
=
''
,
libexecdir
=
None
,
exe
=
None
,
env
=
''
):
bind
=
''
,
exe
=
None
):
'''
Configure and install a launcher
/wrapper pair
.
'''
Configure and install a launcher.
Arguments:
Arguments:
- app: our application
- app: our application
...
@@ -82,30 +54,22 @@ def make_launcher(app, container, bindir,
...
@@ -82,30 +54,22 @@ def make_launcher(app, container, bindir,
- bindir: absolute launcher install path
- bindir: absolute launcher install path
Optional:
Optional:
- bind: singularity bind directives
- bind: singularity bind directives
- libexecdir: absolute wrapper install path.
Default is bindir.
- exe: executable to be associated with app.
- exe: executable to be associated with app.
Default is app.
Default is app.
- env: environment directives to be added to the wrapper.
- env: environment directives to be added to the wrapper.
Multiline string. Default is nothing
Multiline string. Default is nothing
'''
'''
## assume bindir and libexecdir exist, are absolute, and are writable
if
not
exe
:
if
libexecdir
is
None
:
exe
=
app
libexecdir
=
bindir
## actual exe we want to run, default: same as app
exe
=
exe
## paths
## paths
launcher_path
=
'
{}/{}
'
.
format
(
bindir
,
app
)
launcher_path
=
'
{}/{}
'
.
format
(
bindir
,
app
)
wrapper_path
=
'
{}/{}_wrap
'
.
format
(
libexecdir
,
app
)
## scripts --> use absolute path for wrapper path inside launcher
## scripts --> use absolute path for wrapper path inside launcher
launcher
=
_LAUNCHER
.
format
(
container
=
container
,
launcher
=
_LAUNCHER
.
format
(
container
=
container
,
bind
=
bind
,
bind
=
bind
,
wrapper
=
wrapper_path
)
exe
=
exe
)
wrapper
=
_WRAPPER
.
format
(
env
=
env
,
exe
=
exe
)
## write our scripts
## write our scripts
_write_script
(
launcher_path
,
launcher
)
_write_script
(
launcher_path
,
launcher
)
_write_script
(
wrapper_path
,
wrapper
)
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment