diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 21f142d400294a0cc78be2ab03b5f8c363f1b5d0..789f9ccee3e518d6f848ad6cc26cc52318f19de6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,30 +5,60 @@ stages: - phase2 - phase3 -eic_builder_docker: +eic_builder_release: stage: phase1 tags: - sodium dind + only: + - tags + - master + - staging script: - cd containers/builder - head Dockerfile - make login - make release-cached +eic_builder_develop: + stage: phase1 + tags: + - sodium dind + only: + - develop + script: + - cd containers/builder + - head Dockerfile + - make login + - make develop-cached -eic_docker: +eic_release: stage: phase2 tags: - sodium dind + only: + - tags + - master + - staging script: - cd containers/release - make login - make release-cached +eic_develop: + stage: phase2 + tags: + - sodium dind + only: + - develop + script: + - cd containers/release + - make login + - make develop-cached eic_builder_singularity: stage: phase2 tags: - sodium dind only: + - staging - tags when: manual script: @@ -48,6 +78,7 @@ eic_singularity: tags: - sodium dind only: + - staging - tags script: - cp containers/release/eic.def . diff --git a/README.md b/README.md index 280b54ef1ba3d415a07a077472fa54363d1d257a..b47cdf83731179ca5d2063600a6937bafade0b3f 100644 --- a/README.md +++ b/README.md @@ -15,33 +15,80 @@ cd eic_container modelefile will be installed to `$PREFIX/../../etc/modulefiles`. You can use the `-v` flag to select the version you want to deploy, or omit the flag if you want to install the master build. The recommended stable - release version is `v2.0.0`. + release version is `v2.0.1`. ```bash -./deploy.py -v 2.0.0 <PREFIX> +./deploy.py -v 2.0.1 <PREFIX> ``` Available flags: ```bash -v VERSION, --version VERSION - (opt.) project version. Default: current git branch/tag. + (opt.) project version. Default: current version (in repo). -b BIND_PATHS, --bind-path BIND_PATHS (opt.) extra bind paths for singularity. -m MODULE_PATH, --module-path MODULE_PATH (opt.) Root module path where you want to install a modulefile. D: <prefix>/../../etc/modulefiles + -l, --local Local deploy, will not install the modulefiles (you will have + to run the launcher scripts from their relative paths). -f, --force Force-overwrite already downloaded container with the same name. - --install-builder BUILDER - (opt.) Install fat builder image, instead of normal - slim image ``` -3. To use the container: load the modulefile, and then use the included apps as if - they are native apps on your system! +3. To use the container in installed mode, you can load the modulefile, + and then use the included apps as if they are native apps on your system! ```bash module load eic_container ``` +4. To use the container in local mode, you can run the runscripts (under `$PREFIX/bin`) + manually. + 4. (Advanced) If you need to add additional bind directives for the internal singularity container, you can add them with the `-b` flag. Run `./deploy.py -h` to see a list of all supported options. + +Usage +----- + +### A. Running the singularity development environment with modulefiles + +1. Add the installed modulefile to your module path, e.g., +```bash +module use <prefix>/../../etc/modulefiles +``` + +2. Load the eic container +```bash +module load eic_container +``` + +3. To start a shell in the container environment, do +```bash +eic-shell +``` + +### B. Running the singularity development locally (without modulefiles) + +1. This is assuming you deployed with the `-l` flag to a prefix `$PREFIX`: +```bash +./deploy.py $PREFIX +``` + +2. To start a shell in the container environment, do +```bash +$PREFIX/bin/eic-shell +``` + +### C. Using the docker container for your CI purposes + +1. To load the container environment in your run scripts, you can + - launch the script or program using `eic-shell`, + - or `source /etc/eic-env.sh` at the start of your commands. + +2. If using this container as a basis for a new container, you can direction access + the full container environment from a docker `RUN` shell command with no further + action needed. For the most optimal experience, you can install your software to + `/opt/view` to fully integrate with the existing environment. Depending on your + use case, installation to `/usr/local` may also work, but this might require you + to write and run additional environment scripts. diff --git a/VERSION b/VERSION index 227cea215648b1af34a87c9acf5b707fe02d2072..38f77a65b3015cb4dc42eebe91514e49b47b8597 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.0 +2.0.1 diff --git a/containers/builder/Makefile b/containers/builder/Makefile index e66f3dffb0ee5060610f84bc99a3c51d35f215c8..53a827dbc53508a83b8d236d1cfb7000502bcc74 100644 --- a/containers/builder/Makefile +++ b/containers/builder/Makefile @@ -81,7 +81,12 @@ release: build-nc publish ## Make a release by building and publishing the `{ver @echo "Cleaning up" docker rmi $(APP_NAME):latest +develop: build-nc publish-dev ## Develop build, do not publish latest + @echo "Cleaning up" + docker rmi $(APP_NAME):unstable + release-cached: build publish ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR +develop-cached: build login publish-dev ## develop release publish: login publish-latest publish-version #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR @echo "Publishing done" @@ -93,6 +98,11 @@ publish-latest: ## Publish the `latest` taged container to ECR docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest #docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest +publish-dev: ## Publish the `unstable` taged container to ECR + @echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)' + docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable + docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable + publish-version: tag-version ## Publish the `{version}` taged container to ECR @echo 'publish latest to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)' docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION) diff --git a/containers/release/Dockerfile b/containers/release/Dockerfile index ab677c9c14908c2490df2698b0228a94e0fa7319..d37df77b794524cf80b7e46d320d73ac0fe7caf8 100644 --- a/containers/release/Dockerfile +++ b/containers/release/Dockerfile @@ -26,9 +26,12 @@ COPY --from=builder /opt/view /opt/view COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/profile.d/z10_spack_environment.sh ## Setup global bashrc, useful for singularity COPY --from=builder /root/.bashrc /etc/bash.bashrc +COPY eic-shell /usr/local/bin/eic-shell +COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/eic-env.sh ## make sure we have the entrypoints setup correctly ENTRYPOINT [] CMD ["bash", "--rcfile", "/etc/profile", "-l"] USER 0 WORKDIR / +SHELL ["/usr/local/bin/eic-shell"] diff --git a/containers/release/Makefile b/containers/release/Makefile index e66f3dffb0ee5060610f84bc99a3c51d35f215c8..53a827dbc53508a83b8d236d1cfb7000502bcc74 100644 --- a/containers/release/Makefile +++ b/containers/release/Makefile @@ -81,7 +81,12 @@ release: build-nc publish ## Make a release by building and publishing the `{ver @echo "Cleaning up" docker rmi $(APP_NAME):latest +develop: build-nc publish-dev ## Develop build, do not publish latest + @echo "Cleaning up" + docker rmi $(APP_NAME):unstable + release-cached: build publish ## Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR +develop-cached: build login publish-dev ## develop release publish: login publish-latest publish-version #publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR @echo "Publishing done" @@ -93,6 +98,11 @@ publish-latest: ## Publish the `latest` taged container to ECR docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest #docker rmi $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):latest +publish-dev: ## Publish the `unstable` taged container to ECR + @echo 'publish unstable to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME)' + docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable + docker push $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):unstable + publish-version: tag-version ## Publish the `{version}` taged container to ECR @echo 'publish latest to $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION)' docker tag $(APP_NAME):latest $(REG_NAME)/$(GL_REG_GROUP)/$(APP_NAME):$(VERSION) diff --git a/containers/release/eic-shell b/containers/release/eic-shell new file mode 100755 index 0000000000000000000000000000000000000000..070f3859cb268372f5d6a7696f034497817ab5c6 --- /dev/null +++ b/containers/release/eic-shell @@ -0,0 +1,23 @@ +#! /usr/bin/env bash + +source /etc/profile + +if [ -p /dev/stdin ]; then + # If we want to read the input line by line + while IFS= read line; do + if [ -z "$piped_args" ]; then + piped_args="${line}" + else + piped_args="${piped_args}\n${line}" + fi + done +fi + +## Fire off bash +if [ ${piped_args} ] ; then + echo -e ${piped_args} | exec bash +elif [ $# -gt 0 ]; then + exec bash -c "$@" +else + exec bash +fi diff --git a/deploy.py b/deploy.py index 43b998050fa23bee1b485d7f1e812541191b49de..c19229310a28df0b637da10cb0548250ee61ba62 100755 --- a/deploy.py +++ b/deploy.py @@ -24,7 +24,7 @@ GROUP_NAME='containers' PROJECT_NAME='eic_container' IMAGE_ROOT='eic' -PROGRAMS = [('eic_shell', '/usr/bin/bash'), +PROGRAMS = ['eic-shell', 'root', 'ipython'] @@ -45,7 +45,7 @@ if __name__ == "__main__": '-v', '--version', dest='version', default=project_version(), - help='(opt.) project version. Default: current git branch/tag.') + help='(opt.) project version. Default: current version (in repo).') parser.add_argument( '-f', '--force', action='store_true', @@ -60,6 +60,12 @@ if __name__ == "__main__": '-m', '--module-path', dest='module_path', help='(opt.) Root module path where you want to install a modulefile. D: <prefix>/../../etc/modulefiles') + parser.add_argument( + '-l', '--local', + action='store_true', + dest='local', + help='Local deploy, will not install the modulefiles (you will have to run' + 'the launchers scripts from their relative paths).') parser.add_argument( '--install-builder', dest='builder', @@ -100,7 +106,10 @@ if __name__ == "__main__": libexecdir = '{}/libexec'.format(args.prefix) root_prefix = os.path.abspath('{}/..'.format(args.prefix)) moduledir = '{}/{}'.format(args.module_path, PROJECT_NAME) - for dir in [bindir, libdir, libexecdir, moduledir]: + dirs = [bindir, libdir, libexecdir] + if not args.local: + dirs.append(moduledir) + for dir in dirs: print(' -', dir) smart_mkdir(dir) @@ -123,7 +132,8 @@ if __name__ == "__main__": print('WARNING: Container found at', container) print(' ---> run with -f to force a re-download') - make_modulefile(PROJECT_NAME, version, moduledir, bindir) + if not args.local: + make_modulefile(PROJECT_NAME, version, moduledir, bindir) ## configure the application launchers print('Configuring applications launchers: ') diff --git a/install/util.py b/install/util.py index 47cb594a662dc0d0d121b73653bbf3c0e8746063..e3b2a86b5e86b459ef048b643fe2d3f0d442e798 100644 --- a/install/util.py +++ b/install/util.py @@ -32,7 +32,7 @@ def smart_mkdir(dir): def project_version(): '''Return the project version based on the current git branch/tag.''' - ## Shell command to get the current git version - git_version_cmd = 'git symbolic-ref -q --short HEAD || git describe --tags --exact-match' + ## Shell command to get the current project version + version_cmd = 'cat VERSION' ## Strip will remove the leading \n character - return os.popen(git_version_cmd).read().strip() + return os.popen(version_cmd).read().strip()