release workflow: Add spack command for generating the .gitlab-ci.yml

This spack command adds a new schema for a file which describes the
builder containers available, along with the compilers availabe on
each builder.  The release-jobs command then generates the .gitlab-ci.yml
file by first expanding the release spec set, concretizing each spec
(in an appropriate docker container if --this-machine-only argument is
not provided on command line), and then combining and staging all the
concrete specs as jobs to be run by gitlab.

etc/spack/defaults/release.yaml

+# -------------------------------------------------------------------------
+# This is the default spack release spec set.
+# -------------------------------------------------------------------------
+spec-set:
+    include: []
+    exclude: []
+    matrix:
+        - packages:
+            xsdk:
+                versions: [0.4.0]
+        - compilers:
+            gcc:
+                versions: [5.5.0]
+            clang:
+                versions: [6.0.0, '6.0.0-1ubuntu2']
+    cdash: ["https://spack.io/cdash/submit.php?project=spack"]

lib/spack/spack/schema/os_container_mapping.py

+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+"""Schema for os-container-mapping.yaml configuration file.
+
+.. literalinclude:: ../spack/schema/os_container_mapping.py
+   :lines: 32-
+"""
+
+
+schema = {
+    '$schema': 'http://json-schema.org/schema#',
+    'title': 'Spack release builds os/container mapping config file schema',
+    'type': 'object',
+    'additionalProperties': False,
+    'patternProperties': {
+        r'containers': {
+            'type': 'object',
+            'default': {},
+            'patternProperties': {
+                r'[\w\d\-_\.]+': {
+                    'type': 'object',
+                    'default': {},
+                    'additionalProperties': False,
+                    'required': ['image'],
+                    'properties': {
+                        'image': {'type': 'string'},
+                        'setup_script': {'type': 'string'},
+                        'compilers': {
+                            'type': 'array',
+                            'default': [],
+                            'items': {
+                                'type': 'object',
+                                'default': {},
+                                'additionalProperties': False,
+                                'required': ['name'],
+                                'properties': {
+                                    'name': {'type': 'string'},
+                                    'path': {'type': 'string'},
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    },
+}

lib/spack/spack/schema/specs_deps.py

+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+"""Schema for expressing dependencies of a set of specs in a JSON file
+
+.. literalinclude:: ../spack/schema/specs_deps.py
+   :lines: 32-
+"""
+
+
+schema = {
+    '$schema': 'http://json-schema.org/schema#',
+    'title': 'Spack schema for the dependencies of a set of specs',
+    'type': 'object',
+    'additionalProperties': False,
+    'required': ['specs'],
+    'properties': {
+        r'dependencies': {
+            'type': 'array',
+            'default': [],
+            'items': {
+                'type': 'object',
+                'additionalProperties': False,
+                'required': ['depends', 'spec'],
+                'properties': {
+                    r'depends': {'type': 'string'},
+                    r'spec': {'type': 'string'},
+                },
+            },
+        },
+        r'specs': {
+            'type': 'array',
+            'default': [],
+            'items': {
+                'type': 'object',
+                'additionalProperties': False,
+                'required': ['root_spec', 'spec', 'label'],
+                'properties': {
+                    r'root_spec': {'type': 'string'},
+                    r'spec': {'type': 'string'},
+                    r'label': {'type': 'string'},
+                }
+            },
+        },
+    },
+}

lib/spack/spack/test/cmd/release_jobs.py

+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import json
+
+from jsonschema import validate
+
+from spack import repo
+from spack.architecture import sys_type
+from spack.cmd.release_jobs import stage_spec_jobs, spec_deps_key_label
+from spack.main import SpackCommand
+from spack.schema.specs_deps import schema as specs_deps_schema
+from spack.spec import Spec
+from spack.test.conftest import MockPackage, MockPackageMultiRepo
+
+
+release_jobs = SpackCommand('release-jobs')
+
+
+def test_specs_deps(tmpdir, config):
+    """If we ask for the specs dependencies to be written to disk, then make
+    sure we get a file of the correct format."""
+
+    output_path = str(tmpdir.mkdir('json').join('spec_deps.json'))
+    release_jobs('--specs-deps-output', output_path, 'readline')
+
+    deps_object = None
+
+    with open(output_path, 'r') as fd:
+        deps_object = json.loads(fd.read())
+
+    assert (deps_object is not None)
+
+    validate(deps_object, specs_deps_schema)
+
+
+def test_specs_staging(config):
+    """Make sure we achieve the best possible staging for the following
+spec DAG::
+
+        a
+       /|
+      c b
+        |\
+        e d
+          |\
+          f g
+
+In this case, we would expect 'c', 'e', 'f', and 'g' to be in the first stage,
+and then 'd', 'b', and 'a' to be put in the next three stages, respectively.
+
+"""
+    current_system = sys_type()
+
+    config_compilers = config.get_config('compilers')
+    first_compiler = config_compilers[0]
+    compiler_spec = first_compiler['compiler']['spec']
+
+    # Whatever that first compiler in the configuration was, let's make sure
+    # we mock up an entry like we'd find in os-container-mapping.yaml which
+    # has that compiler.
+    mock_containers = {}
+    mock_containers[current_system] = {
+        "image": "dontcare",
+        "compilers": [
+            {
+                "name": compiler_spec,
+            }
+        ],
+    }
+
+    default = ('build', 'link')
+
+    g = MockPackage('g', [], [])
+    f = MockPackage('f', [], [])
+    e = MockPackage('e', [], [])
+    d = MockPackage('d', [f, g], [default, default])
+    c = MockPackage('c', [], [])
+    b = MockPackage('b', [d, e], [default, default])
+    a = MockPackage('a', [b, c], [default, default])
+
+    mock_repo = MockPackageMultiRepo([a, b, c, d, e, f, g])
+
+    with repo.swap(mock_repo):
+        # Now we'll ask for the root package to be compiled with whatever that
+        # first compiler in the configuration was.
+        spec_a = Spec('a%{0}'.format(compiler_spec))
+        spec_a.concretize()
+
+        spec_a_label = spec_deps_key_label(spec_a)[1]
+        spec_b_label = spec_deps_key_label(spec_a['b'])[1]
+        spec_c_label = spec_deps_key_label(spec_a['c'])[1]
+        spec_d_label = spec_deps_key_label(spec_a['d'])[1]
+        spec_e_label = spec_deps_key_label(spec_a['e'])[1]
+        spec_f_label = spec_deps_key_label(spec_a['f'])[1]
+        spec_g_label = spec_deps_key_label(spec_a['g'])[1]
+
+        spec_labels, dependencies, stages = stage_spec_jobs(
+            [spec_a], mock_containers, current_system)
+
+        assert (len(stages) == 4)
+
+        assert (len(stages[0]) == 4)
+        assert (spec_c_label in stages[0])
+        assert (spec_e_label in stages[0])
+        assert (spec_f_label in stages[0])
+        assert (spec_g_label in stages[0])
+
+        assert (len(stages[1]) == 1)
+        assert (spec_d_label in stages[1])
+
+        assert (len(stages[2]) == 1)
+        assert (spec_b_label in stages[2])
+
+        assert (len(stages[3]) == 1)
+        assert (spec_a_label in stages[3])

share/spack/docker/os-container-mapping.yaml

+containers:
+  linux-ubuntu18.04-x86_64:
+    image: scottwittenburg/spack_builder_ubuntu_18.04
+    compilers:
+      - name: gcc@5.5.0
+      - name: clang@6.0.0-1ubuntu2
+  linux-centos7-x86_64:
+    image: scottwittenburg/spack_builder_centos_7
+    compilers:
+      - name: gcc@5.5.0
+      - name: clang@6.0.0