release workflow: Add build scripts for jobs and means to upload pkgs

bin/rebuild-index.sh

+#!/bin/bash
+
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+set -x
+
+SPACK_BIN_DIR="${CI_PROJECT_DIR}/bin"
+export PATH="${SPACK_BIN_DIR}:${PATH}"
+
+spack upload-s3 index

bin/rebuild-package.sh

+#!/bin/bash
+
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+###
+### This script represents a gitlab-ci job, corresponding to a single release
+### spec.  As such this script must first decide whether or not the spec it
+### has been assigned is up to date on the remote binary mirror.  If it is
+### not (i.e. the source code has changed in a way that caused a change in the
+### full_hash of the spec), this script will build the package, create a
+### binary cache for it, and then push all related files to the remote binary
+### mirror.  This script also communicates with a remote CDash instance to
+### share status on the package build process.
+###
+### The following environment variables are expected to be set in order for
+### the various elements in this script to function properly.  Listed first
+### are two defaults we rely on from gitlab, then three we set up in the
+### variables section of gitlab ourselves, and finally four variables
+### written into the .gitlab-ci.yml file.
+###
+### CI_PROJECT_DIR
+### CI_JOB_NAME
+###
+### AWS_ACCESS_KEY_ID
+### AWS_SECRET_ACCESS_KEY
+### SPACK_SIGNING_KEY
+###
+### CDASH_BASE_URL
+### ROOT_SPEC
+### DEPENDENCIES
+### MIRROR_URL
+###
+
+shopt -s expand_aliases
+
+export FORCE_UNSAFE_CONFIGURE=1
+
+TEMP_DIR="${CI_PROJECT_DIR}/jobs_scratch_dir"
+
+JOB_LOG_DIR="${TEMP_DIR}/logs"
+SPEC_DIR="${TEMP_DIR}/specs"
+LOCAL_MIRROR="${CI_PROJECT_DIR}/local_mirror"
+BUILD_CACHE_DIR="${LOCAL_MIRROR}/build_cache"
+SPACK_BIN_DIR="${CI_PROJECT_DIR}/bin"
+CDASH_UPLOAD_URL="${CDASH_BASE_URL}/submit.php?project=Spack"
+DEP_JOB_RELATEBUILDS_URL="${CDASH_BASE_URL}/api/v1/relateBuilds.php"
+declare -a JOB_DEPS_PKG_NAMES
+
+export SPACK_ROOT=${CI_PROJECT_DIR}
+export PATH="${SPACK_BIN_DIR}:${PATH}"
+export GNUPGHOME="${CI_PROJECT_DIR}/opt/spack/gpg"
+
+mkdir -p ${JOB_LOG_DIR}
+mkdir -p ${SPEC_DIR}
+
+cleanup() {
+    set +x
+
+    if [ -z "$exit_code" ] ; then
+
+        exit_code=$1
+        if [ -z "$exit_code" ] ; then
+            exit_code=0
+        fi
+
+        restore_io
+
+        if [ "$( type -t finalize )" '=' 'function' ] ; then
+            finalize "$JOB_LOG_DIR/cdash_log.txt"
+        fi
+
+        # We can clean these out later on, once we have a good sense for
+        # how the logging infrastructure is working
+        # rm -rf "$JOB_LOG_DIR"
+    fi
+
+    \exit $exit_code
+}
+
+alias exit='cleanup'
+
+begin_logging() {
+    trap "cleanup 1; \\exit \$exit_code" INT TERM QUIT
+    trap "cleanup 0; \\exit \$exit_code" EXIT
+
+    rm -rf "$JOB_LOG_DIR/cdash_log.txt"
+
+    # NOTE: Here, some redirects are set up
+    exec 3>&1 # fd 3 is now a dup of stdout
+    exec 4>&2 # fd 4 is now a dup of stderr
+
+    # stdout and stderr are joined and redirected to the log
+    exec &> "$JOB_LOG_DIR/cdash_log.txt"
+
+    set -x
+}
+
+restore_io() {
+    exec  >&-
+    exec 2>&-
+
+    exec  >&3
+    exec 2>&4
+
+    exec 3>&-
+    exec 4>&-
+}
+
+finalize() {
+    # If you define a finalize function:
+    #  - it will always be called at the very end of the script
+    #  - the log file will be passed in as the first argument, and
+    #  - the code in this function will not be logged.
+    echo "The full log file is located at $1"
+    # TODO: send this log data to cdash!
+}
+
+check_error()
+{
+    local last_exit_code=$1
+    local last_cmd=$2
+    if [[ ${last_exit_code} -ne 0 ]]; then
+        echo "${last_cmd} exited with code ${last_exit_code}"
+        echo "TERMINATING JOB"
+        exit 1
+    else
+        echo "${last_cmd} completed successfully"
+    fi
+}
+
+extract_build_id()
+{
+    LINES_TO_SEARCH=$1
+    regex="buildSummary\.php\?buildid=([[:digit:]]+)"
+    SINGLE_LINE_OUTPUT=$(echo ${LINES_TO_SEARCH} | tr -d '\n')
+
+    if [[ ${SINGLE_LINE_OUTPUT} =~ ${regex} ]]; then
+        echo "${BASH_REMATCH[1]}"
+    else
+        echo "NONE"
+    fi
+}
+
+get_relate_builds_post_data()
+{
+  cat <<EOF
+{
+  "project": "${1}",
+  "buildid": ${2},
+  "relatedid": ${3},
+  "relationship": "depends on"
+}
+EOF
+}
+
+gen_full_specs_for_job_and_deps() {
+
+    read -ra PARTSARRAY <<< "${CI_JOB_NAME}"
+    local pkgName="${PARTSARRAY[0]}"
+    local pkgVersion="${PARTSARRAY[1]}"
+    local compiler="${PARTSARRAY[2]}"
+    local osarch="${PARTSARRAY[3]}"
+
+    JOB_SPEC_NAME="${pkgName}@${pkgVersion}%${compiler} arch=${osarch}"
+    JOB_PKG_NAME="${pkgName}"
+    SPEC_YAML_PATH="${SPEC_DIR}/${pkgName}.yaml"
+    local root_spec_name="${ROOT_SPEC}"
+    local spec_names_to_save="${pkgName}"
+
+    IFS=';' read -ra DEPS <<< "${DEPENDENCIES}"
+    for i in "${DEPS[@]}"; do
+        read -ra PARTSARRAY <<< "${i}"
+        pkgName="${PARTSARRAY[0]}"
+        spec_names_to_save="${spec_names_to_save} ${pkgName}"
+        JOB_DEPS_PKG_NAMES+=("${pkgName}")
+    done
+
+    spack -d buildcache save-yaml --specs "${spec_names_to_save}" --root-spec "${root_spec_name}" --yaml-dir "${SPEC_DIR}"
+}
+
+begin_logging
+
+gen_full_specs_for_job_and_deps
+
+echo "Building package ${JOB_SPEC_NAME}, ${HASH}, ${MIRROR_URL}"
+
+# Finally, list the compilers spack knows about
+echo "Compiler Configurations:"
+spack config get compilers
+
+# Make the build_cache directory if it doesn't exist
+mkdir -p "${BUILD_CACHE_DIR}"
+
+# Get buildcache name so we can write a CDash build id file in the right place.
+# If we're unable to get the buildcache name, we may have encountered a problem
+# concretizing the spec, or some other issue that will eventually cause the job
+# to fail.
+JOB_BUILD_CACHE_ENTRY_NAME=`spack -d buildcache get-buildcache-name --spec-yaml "${SPEC_YAML_PATH}"`
+if [[ $? -ne 0 ]]; then
+    echo "ERROR, unable to get buildcache entry name for job ${CI_JOB_NAME} (spec: ${JOB_SPEC_NAME})"
+    exit 1
+fi
+
+# This should create the directory we referred to as GNUPGHOME earlier
+spack gpg list
+
+# Importing the secret key using gpg2 directly should allow to
+# sign and verify both
+set +x
+KEY_IMPORT_RESULT=`echo ${SPACK_SIGNING_KEY} | base64 --decode | gpg2 --import`
+check_error $? "gpg2 --import"
+set -x
+
+spack gpg list --trusted
+spack gpg list --signing
+
+# Whether we have to build the spec or download it pre-built, we expect to find
+# the cdash build id file sitting in this location afterwards.
+JOB_CDASH_ID_FILE="${BUILD_CACHE_DIR}/${JOB_BUILD_CACHE_ENTRY_NAME}.cdashid"
+
+# Finally, we can check the spec we have been tasked with build against
+# the built binary on the remote mirror to see if it needs to be rebuilt
+spack -d buildcache check --spec-yaml "${SPEC_YAML_PATH}" --mirror-url "${MIRROR_URL}" --rebuild-on-error
+
+if [[ $? -ne 0 ]]; then
+    # Configure mirror
+    spack mirror add local_artifact_mirror "file://${LOCAL_MIRROR}"
+
+    JOB_CDASH_ID="NONE"
+
+    # Install package, using the buildcache from the local mirror to
+    # satisfy dependencies.
+    BUILD_ID_LINE=`spack -d -k -v install --use-cache --cdash-upload-url "${CDASH_UPLOAD_URL}" --cdash-build "${JOB_SPEC_NAME}" --cdash-site "Spack AWS Gitlab Instance" --cdash-track "Experimental" -f "${SPEC_YAML_PATH}" | grep "buildSummary\\.php"`
+    check_error $? "spack install"
+
+    # By parsing the output of the "spack install" command, we can get the
+    # buildid generated for us by CDash
+    JOB_CDASH_ID=$(extract_build_id "${BUILD_ID_LINE}")
+
+    # Create buildcache entry for this package, reading the spec from the yaml
+    # file.
+    spack -d buildcache create --spec-yaml "${SPEC_YAML_PATH}" -a -f -d "${LOCAL_MIRROR}" --no-rebuild-index
+    check_error $? "spack buildcache create"
+
+    # Write the .cdashid file to the buildcache as well
+    echo "${JOB_CDASH_ID}" >> ${JOB_CDASH_ID_FILE}
+
+    # TODO: The upload-s3 command should eventually be replaced with something
+    # like: "spack buildcache put <mirror> <spec>", when that subcommand is
+    # properly implemented.
+    spack -d upload-s3 spec --base-dir "${LOCAL_MIRROR}" --spec-yaml "${SPEC_YAML_PATH}"
+    check_error $? "spack upload-s3 spec"
+else
+    echo "spec ${JOB_SPEC_NAME} is already up to date on remote mirror, downloading it"
+
+    # Configure remote mirror so we can download buildcache entry
+    spack mirror add remote_binary_mirror ${MIRROR_URL}
+
+    # Now download it
+    spack -d buildcache download --spec-yaml "${SPEC_YAML_PATH}" --path "${BUILD_CACHE_DIR}/" --require-cdashid
+    check_error $? "spack buildcache download"
+fi
+
+# The next step is to relate this job to the jobs it depends on
+if [ -f "${JOB_CDASH_ID_FILE}" ]; then
+    JOB_CDASH_BUILD_ID=$(<${JOB_CDASH_ID_FILE})
+
+    if [ "${JOB_CDASH_BUILD_ID}" == "NONE" ]; then
+        echo "ERROR: unable to read this jobs id from ${JOB_CDASH_ID_FILE}"
+        exit 1
+    fi
+
+    # Now get CDash ids for dependencies and "relate" each dependency build
+    # with this jobs build
+    for DEP_PKG_NAME in "${JOB_DEPS_PKG_NAMES[@]}"; do
+        echo "Getting cdash id for dependency --> ${DEP_PKG_NAME} <--"
+        DEP_SPEC_YAML_PATH="${SPEC_DIR}/${DEP_PKG_NAME}.yaml"
+        DEP_JOB_BUILDCACHE_NAME=`spack -d buildcache get-buildcache-name --spec-yaml "${DEP_SPEC_YAML_PATH}"`
+
+        if [[ $? -eq 0 ]]; then
+            DEP_JOB_ID_FILE="${BUILD_CACHE_DIR}/${DEP_JOB_BUILDCACHE_NAME}.cdashid"
+            echo "DEP_JOB_ID_FILE path = ${DEP_JOB_ID_FILE}"
+
+            if [ -f "${DEP_JOB_ID_FILE}" ]; then
+                DEP_JOB_CDASH_BUILD_ID=$(<${DEP_JOB_ID_FILE})
+                echo "File ${DEP_JOB_ID_FILE} contained value ${DEP_JOB_CDASH_BUILD_ID}"
+                echo "Relating builds -> ${JOB_SPEC_NAME} (buildid=${JOB_CDASH_BUILD_ID}) depends on ${DEP_PKG_NAME} (buildid=${DEP_JOB_CDASH_BUILD_ID})"
+                relateBuildsPostBody="$(get_relate_builds_post_data "Spack" ${JOB_CDASH_BUILD_ID} ${DEP_JOB_CDASH_BUILD_ID})"
+                relateBuildsResult=`curl "${DEP_JOB_RELATEBUILDS_URL}" -H "Content-Type: application/json" -H "Accept: application/json" -d "${relateBuildsPostBody}"`
+                echo "Result of curl request: ${relateBuildsResult}"
+            else
+                echo "ERROR: Did not find expected .cdashid file for dependency: ${DEP_JOB_ID_FILE}"
+                exit 1
+            fi
+        else
+            echo "ERROR: Unable to get buildcache entry name for ${DEP_SPEC_NAME}"
+            exit 1
+        fi
+    done
+else
+    echo "ERROR: Did not find expected .cdashid file ${JOB_CDASH_ID_FILE}"
+    exit 1
+fi
+
+# Show the size of the buildcache and a list of what's in it, directly
+# in the gitlab log output
+(
+    restore_io
+    du -sh ${BUILD_CACHE_DIR}
+    find ${BUILD_CACHE_DIR} -maxdepth 3 -type d -ls
+)
+
+echo "End of rebuild package script"

lib/spack/spack/cmd/upload_s3.py

+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+# TODO: This will be merged into the buildcache command once
+# everything is working.
+
+import os
+import re
+import sys
+
+try:
+    import boto3
+    import botocore
+    have_boto3_support = True
+except ImportError:
+    have_boto3_support = False
+
+import llnl.util.tty as tty
+
+from spack.error import SpackError
+import spack.tengine as template_engine
+from spack.spec import Spec
+
+
+import spack.binary_distribution as bindist
+
+
+description = "temporary command to upload buildcaches to 's3.spack.io'"
+section = "packaging"
+level = "long"
+
+
+def setup_parser(subparser):
+    setup_parser.parser = subparser
+    subparsers = subparser.add_subparsers(help='upload-s3 sub-commands')
+
+    # sub-command to upload a built spec to s3
+    spec = subparsers.add_parser('spec', help=upload_spec.__doc__)
+
+    spec.add_argument('-s', '--spec', default=None,
+                      help='Spec to upload')
+
+    spec.add_argument('-y', '--spec-yaml', default=None,
+                      help='Path to spec yaml file containing spec to upload')
+
+    spec.add_argument('-b', '--base-dir', default=None,
+                      help='Path to root of buildcaches')
+
+    spec.add_argument('-e', '--endpoint-url',
+                      default='https://s3.spack.io', help='URL of mirror')
+
+    spec.set_defaults(func=upload_spec)
+
+    # sub-command to update the index of a buildcache on s3
+    index = subparsers.add_parser('index', help=update_index.__doc__)
+
+    index.add_argument('-e', '--endpoint-url',
+        default='https://s3.spack.io', help='URL of mirror')
+
+    index.set_defaults(func=update_index)
+
+
+def get_s3_session(endpoint_url):
+    if not have_boto3_support:
+        raise SpackError('boto3 module not available')
+
+    session = boto3.Session()
+    s3 = session.resource('s3')
+
+    bucket_names = []
+    for bucket in s3.buckets.all():
+        bucket_names.append(bucket.name)
+
+    if len(bucket_names) > 1:
+        raise SpackError('More than one bucket associated with credentials')
+
+    bucket_name = bucket_names[0]
+
+    return s3, bucket_name
+
+
+def update_index(args):
+    """Update the index of an s3 buildcache"""
+    s3, bucket_name = get_s3_session(args.endpoint_url)
+
+    bucket = s3.Bucket(bucket_name)
+    exists = True
+
+    try:
+        s3.meta.client.head_bucket(Bucket=bucket_name)
+    except botocore.exceptions.ClientError as e:
+        # If a client error is thrown, then check that it was a 404 error.
+        # If it was a 404 error, then the bucket does not exist.
+        error_code = e.response['Error']['Code']
+        if error_code == '404':
+            exists = False
+
+    if not exists:
+        tty.error('S3 bucket "{0}" does not exist'.format(bucket_name))
+        sys.exit(1)
+
+    build_cache_dir = os.path.join(
+        'mirror', bindist.build_cache_relative_path())
+
+    spec_yaml_regex = re.compile('{0}/(.+\\.spec\\.yaml)$'.format(
+        build_cache_dir))
+    spack_regex = re.compile('{0}/([^/]+)/.+\\.spack$'.format(
+        build_cache_dir))
+
+    top_level_keys = set()
+
+    for key in bucket.objects.all():
+        m = spec_yaml_regex.search(key.key)
+        if m:
+            top_level_keys.add(m.group(1))
+            print(m.group(1))
+            continue
+
+        m = spack_regex.search(key.key)
+        if m:
+            top_level_keys.add(m.group(1))
+            print(m.group(1))
+            continue
+
+    index_data = {
+        'top_level_keys': top_level_keys,
+    }
+
+    env = template_engine.make_environment()
+    template_dir = 'misc'
+    index_template = os.path.join(template_dir, 'buildcache_index.html')
+    t = env.get_template(index_template)
+    contents = t.render(index_data)
+
+    index_key = os.path.join(build_cache_dir, 'index.html')
+
+    tty.debug('Generated index:')
+    tty.debug(contents)
+    tty.debug('Pushing it to {0} -> {1}'.format(bucket_name, index_key))
+
+    s3_obj = s3.Object(bucket_name, index_key)
+    s3_obj.put(Body=contents, ACL='public-read')
+
+
+def upload_spec(args):
+    """Upload a spec to s3 bucket"""
+    if not args.spec and not args.spec_yaml:
+        tty.error('Cannot upload spec without spec arg or path to spec yaml')
+        sys.exit(1)
+
+    if not args.base_dir:
+        tty.error('No base directory for buildcache specified')
+        sys.exit(1)
+
+    if args.spec:
+        try:
+            spec = Spec(args.spec)
+            spec.concretize()
+        except Exception:
+            tty.error('Unable to concrectize spec from string {0}'.format(
+                args.spec))
+            sys.exit(1)
+    else:
+        try:
+            with open(args.spec_yaml, 'r') as fd:
+                spec = Spec.from_yaml(fd.read())
+        except Exception:
+            tty.error('Unable to concrectize spec from yaml {0}'.format(
+                args.spec_yaml))
+            sys.exit(1)
+
+    s3, bucket_name = get_s3_session(args.endpoint_url)
+
+    build_cache_dir = bindist.build_cache_relative_path()
+
+    tarball_key = os.path.join(
+        build_cache_dir, bindist.tarball_path_name(spec, '.spack'))
+    tarball_path = os.path.join(args.base_dir, tarball_key)
+
+    specfile_key = os.path.join(
+        build_cache_dir, bindist.tarball_name(spec, '.spec.yaml'))
+    specfile_path = os.path.join(args.base_dir, specfile_key)
+
+    cdashidfile_key = os.path.join(
+        build_cache_dir, bindist.tarball_name(spec, '.cdashid'))
+    cdashidfile_path = os.path.join(args.base_dir, cdashidfile_key)
+
+    tty.msg('Uploading {0}'.format(tarball_key))
+    s3.meta.client.upload_file(
+        tarball_path, bucket_name,
+        os.path.join('mirror', tarball_key),
+        ExtraArgs={'ACL': 'public-read'})
+
+    tty.msg('Uploading {0}'.format(specfile_key))
+    s3.meta.client.upload_file(
+        specfile_path, bucket_name,
+        os.path.join('mirror', specfile_key),
+        ExtraArgs={'ACL': 'public-read'})
+
+    if os.path.exists(cdashidfile_path):
+        tty.msg('Uploading {0}'.format(cdashidfile_key))
+        s3.meta.client.upload_file(
+            cdashidfile_path, bucket_name,
+            os.path.join('mirror', cdashidfile_key),
+            ExtraArgs={'ACL': 'public-read'})
+
+
+def upload_s3(parser, args):
+    if args.func:
+        args.func(args)