From 1d152a0740b7b96aa30e8250c1766a94ea0d029a Mon Sep 17 00:00:00 2001
From: Gregory Becker <becker33@llnl.gov>
Date: Wed, 18 Dec 2019 11:27:32 -0800
Subject: [PATCH] WIP infrastructure for Spack test command to test existing
installations
---
lib/spack/spack/build_environment.py | 58 +++--
lib/spack/spack/cmd/test.py | 234 ++++++------------
lib/spack/spack/cmd/unit_test.py | 105 ++++++++
lib/spack/spack/package.py | 22 ++
lib/spack/spack/test/cmd/test.py | 2 +-
lib/spack/spack/util/executable.py | 17 +-
.../repos/builtin/packages/python/package.py | 15 ++
7 files changed, 273 insertions(+), 180 deletions(-)
create mode 100644 lib/spack/spack/cmd/unit_test.py
diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py
index 4a57dde77b..72ff65f587 100644
--- a/lib/spack/spack/build_environment.py
+++ b/lib/spack/spack/build_environment.py
@@ -59,7 +59,7 @@
from spack.util.environment import (
env_flag, filter_system_paths, get_path, is_system_path,
EnvironmentModifications, validate, preserve_environment)
-from spack.util.environment import system_dirs
+from spack.util.environment import system_dirs, inspect_path
from spack.error import NoLibrariesError, NoHeadersError
from spack.util.executable import Executable
from spack.util.module_cmd import load_module, path_from_modules, module
@@ -711,28 +711,40 @@ def load_external_modules(pkg):
load_module(external_module)
-def setup_package(pkg, dirty):
+def setup_package(pkg, dirty, context='build'):
"""Execute all environment setup routines."""
- build_env = EnvironmentModifications()
+ env = EnvironmentModifications()
+ # clean environment
if not dirty:
clean_environment()
- set_compiler_environment_variables(pkg, build_env)
- set_build_environment_variables(pkg, build_env, dirty)
- pkg.architecture.platform.setup_platform_environment(pkg, build_env)
+ # setup compilers and build tools for build contexts
+ if context == 'build':
+ set_compiler_environment_variables(pkg, env)
+ set_build_environment_variables(pkg, env, dirty)
+
+ # architecture specific setup
+ pkg.architecture.platform.setup_platform_environment(pkg, env)
- build_env.extend(
- modifications_from_dependencies(pkg.spec, context='build')
+ # recursive post-order dependency information
+ env.extend(
+ modifications_from_dependencies(pkg.spec, context=context)
)
- if (not dirty) and (not build_env.is_unset('CPATH')):
+ if context == 'build' and (not dirty) and (not env.is_unset('CPATH')):
tty.debug("A dependency has updated CPATH, this may lead pkg-config"
" to assume that the package is part of the system"
" includes and omit it when invoked with '--cflags'.")
+ # setup package itself
set_module_variables_for_package(pkg)
- pkg.setup_build_environment(build_env)
+ if context == 'build':
+ pkg.setup_build_environment(env)
+ elif context == 'test':
+ import spack.user_environment as uenv # avoid circular import
+ env.extend(inspect_path(pkg.spec.prefix,
+ uenv.prefix_inspections(pkg.spec.platform)))
# Loading modules, in particular if they are meant to be used outside
# of Spack, can change environment variables that are relevant to the
@@ -742,15 +754,16 @@ def setup_package(pkg, dirty):
# unnecessary. Modules affecting these variables will be overwritten anyway
with preserve_environment('CC', 'CXX', 'FC', 'F77'):
# All module loads that otherwise would belong in previous
- # functions have to occur after the build_env object has its
+ # functions have to occur after the env object has its
# modifications applied. Otherwise the environment modifications
# could undo module changes, such as unsetting LD_LIBRARY_PATH
# after a module changes it.
- for mod in pkg.compiler.modules:
- # Fixes issue https://github.com/spack/spack/issues/3153
- if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
- load_module("cce")
- load_module(mod)
+ if context == 'build':
+ for mod in pkg.compiler.modules:
+ # Fixes issue https://github.com/spack/spack/issues/3153
+ if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
+ load_module("cce")
+ load_module(mod)
# kludge to handle cray libsci being automatically loaded by PrgEnv
# modules on cray platform. Module unload does no damage when
@@ -768,8 +781,8 @@ def setup_package(pkg, dirty):
':'.join(implicit_rpaths))
# Make sure nothing's strange about the Spack environment.
- validate(build_env, tty.warn)
- build_env.apply_modifications()
+ validate(env, tty.warn)
+ env.apply_modifications()
def modifications_from_dependencies(spec, context):
@@ -789,7 +802,8 @@ def modifications_from_dependencies(spec, context):
deptype_and_method = {
'build': (('build', 'link', 'test'),
'setup_dependent_build_environment'),
- 'run': (('link', 'run'), 'setup_dependent_run_environment')
+ 'run': (('link', 'run'), 'setup_dependent_run_environment'),
+ 'test': (('link', 'run', 'test'), 'setup_dependent_run_environment')
}
deptype, method = deptype_and_method[context]
@@ -803,7 +817,7 @@ def modifications_from_dependencies(spec, context):
return env
-def fork(pkg, function, dirty, fake):
+def fork(pkg, function, dirty, fake, context='build'):
"""Fork a child process to do part of a spack build.
Args:
@@ -815,6 +829,8 @@ def fork(pkg, function, dirty, fake):
dirty (bool): If True, do NOT clean the environment before
building.
fake (bool): If True, skip package setup b/c it's not a real build
+ context (string): If 'build', setup build environment. If 'test', setup
+ test environment.
Usage::
@@ -843,7 +859,7 @@ def child_process(child_pipe, input_stream):
try:
if not fake:
- setup_package(pkg, dirty=dirty)
+ setup_package(pkg, dirty=dirty, context=context)
return_value = function()
child_pipe.send(return_value)
diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py
index 8cbc0fbccf..5dddd8c676 100644
--- a/lib/spack/spack/cmd/test.py
+++ b/lib/spack/spack/cmd/test.py
@@ -4,166 +4,94 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from __future__ import print_function
-from __future__ import division
-
-import collections
-import sys
-import re
+import os
import argparse
-import pytest
-from six import StringIO
-import llnl.util.tty.color as color
-from llnl.util.filesystem import working_dir
-from llnl.util.tty.colify import colify
+import llnl.util.tty as tty
-import spack.paths
+import spack.environment as ev
+import spack.cmd
-description = "run spack's unit tests (wrapper around pytest)"
-section = "developer"
+description = "run spack's tests for an install"
+section = "administrator"
level = "long"
def setup_parser(subparser):
- subparser.add_argument(
- '-H', '--pytest-help', action='store_true', default=False,
- help="show full pytest help, with advanced options")
+# subparser.add_argument(
+# '--log-format',
+# default=None,
+# choices=spack.report.valid_formats,
+# help="format to be used for log files"
+# )
+# subparser.add_argument(
+# '--output-file',
+# default=None,
+# help="filename for the log file. if not passed a default will be used"
+# )
+# subparser.add_argument(
+# '--cdash-upload-url',
+# default=None,
+# help="CDash URL where reports will be uploaded"
+# )
+# subparser.add_argument(
+# '--cdash-build',
+# default=None,
+# help="""The name of the build that will be reported to CDash.
+# Defaults to spec of the package to install."""
+# )
+# subparser.add_argument(
+# '--cdash-site',
+# default=None,
+# help="""The site name that will be reported to CDash.
+# Defaults to current system hostname."""
+# )
+# cdash_subgroup = subparser.add_mutually_exclusive_group()
+# cdash_subgroup.add_argument(
+# '--cdash-track',
+# default='Experimental',
+# help="""Results will be reported to this group on CDash.
+# Defaults to Experimental."""
+# )
+# cdash_subgroup.add_argument(
+# '--cdash-buildstamp',
+# default=None,
+# help="""Instead of letting the CDash reporter prepare the
+# buildstamp which, when combined with build name, site and project,
+# uniquely identifies the build, provide this argument to identify
+# the build yourself. Format: %%Y%%m%%d-%%H%%M-[cdash-track]"""
+# )
+# arguments.add_common_arguments(subparser, ['yes_to_all'])
+ length_group = subparser.add_mutually_exclusive_group()
+ length_group.add_argument(
+ '--smoke', action='store_true', dest='smoke_test', default=True,
+ help='run smoke tests (default)')
+ length_group.add_argument(
+ '--capability', action='store_false', dest='smoke_test', default=True,
+ help='run full capability tests using pavilion')
- # extra spack arguments to list tests
- list_group = subparser.add_argument_group("listing tests")
- list_mutex = list_group.add_mutually_exclusive_group()
- list_mutex.add_argument(
- '-l', '--list', action='store_const', default=None,
- dest='list', const='list', help="list test filenames")
- list_mutex.add_argument(
- '-L', '--list-long', action='store_const', default=None,
- dest='list', const='long', help="list all test functions")
- list_mutex.add_argument(
- '-N', '--list-names', action='store_const', default=None,
- dest='list', const='names', help="list full names of all tests")
-
- # use tests for extension
subparser.add_argument(
- '--extension', default=None,
- help="run test for a given spack extension")
-
- # spell out some common pytest arguments, so they'll show up in help
- pytest_group = subparser.add_argument_group(
- "common pytest arguments (spack test --pytest-help for more details)")
- pytest_group.add_argument(
- "-s", action='append_const', dest='parsed_args', const='-s',
- help="print output while tests run (disable capture)")
- pytest_group.add_argument(
- "-k", action='store', metavar="EXPRESSION", dest='expression',
- help="filter tests by keyword (can also use w/list options)")
- pytest_group.add_argument(
- "--showlocals", action='append_const', dest='parsed_args',
- const='--showlocals', help="show local variable values in tracebacks")
-
- # remainder is just passed to pytest
- subparser.add_argument(
- 'pytest_args', nargs=argparse.REMAINDER, help="arguments for pytest")
-
-
-def do_list(args, extra_args):
- """Print a lists of tests than what pytest offers."""
- # Run test collection and get the tree out.
- old_output = sys.stdout
- try:
- sys.stdout = output = StringIO()
- pytest.main(['--collect-only'] + extra_args)
- finally:
- sys.stdout = old_output
-
- lines = output.getvalue().split('\n')
- tests = collections.defaultdict(lambda: set())
- prefix = []
-
- # collect tests into sections
- for line in lines:
- match = re.match(r"(\s*)<([^ ]*) '([^']*)'", line)
- if not match:
- continue
- indent, nodetype, name = match.groups()
-
- # strip parametrized tests
- if "[" in name:
- name = name[:name.index("[")]
-
- depth = len(indent) // 2
-
- if nodetype.endswith("Function"):
- key = tuple(prefix)
- tests[key].add(name)
- else:
- prefix = prefix[:depth]
- prefix.append(name)
-
- def colorize(c, prefix):
- if isinstance(prefix, tuple):
- return "::".join(
- color.colorize("@%s{%s}" % (c, p))
- for p in prefix if p != "()"
- )
- return color.colorize("@%s{%s}" % (c, prefix))
-
- if args.list == "list":
- files = set(prefix[0] for prefix in tests)
- color_files = [colorize("B", file) for file in sorted(files)]
- colify(color_files)
-
- elif args.list == "long":
- for prefix, functions in sorted(tests.items()):
- path = colorize("*B", prefix) + "::"
- functions = [colorize("c", f) for f in sorted(functions)]
- color.cprint(path)
- colify(functions, indent=4)
- print()
-
- else: # args.list == "names"
- all_functions = [
- colorize("*B", prefix) + "::" + colorize("c", f)
- for prefix, functions in sorted(tests.items())
- for f in sorted(functions)
- ]
- colify(all_functions)
-
-
-def add_back_pytest_args(args, unknown_args):
- """Add parsed pytest args, unknown args, and remainder together.
-
- We add some basic pytest arguments to the Spack parser to ensure that
- they show up in the short help, so we have to reassemble things here.
- """
- result = args.parsed_args or []
- result += unknown_args or []
- result += args.pytest_args or []
- if args.expression:
- result += ["-k", args.expression]
- return result
-
-
-def test(parser, args, unknown_args):
- if args.pytest_help:
- # make the pytest.main help output more accurate
- sys.argv[0] = 'spack test'
- return pytest.main(['-h'])
-
- # add back any parsed pytest args we need to pass to pytest
- pytest_args = add_back_pytest_args(args, unknown_args)
-
- # The default is to test the core of Spack. If the option `--extension`
- # has been used, then test that extension.
- pytest_root = spack.paths.spack_root
- if args.extension:
- target = args.extension
- extensions = spack.config.get('config:extensions')
- pytest_root = spack.extensions.path_for_extension(target, *extensions)
-
- # pytest.ini lives in the root of the spack repository.
- with working_dir(pytest_root):
- if args.list:
- do_list(args, pytest_args)
- return
-
- return pytest.main(pytest_args)
+ 'specs', nargs=argparse.REMAINDER,
+ help="list of specs to test")
+
+
+def test(parser, args):
+ env = ev.get_env(args, 'test')
+ hashes = env.all_hashes() if env else None
+
+ specs = spack.cmd.parse_specs(args.specs) if args.specs else [None]
+ specs_to_test = []
+ for spec in specs:
+ matching = spack.store.db.query_local(spec, hashes=hashes)
+ if spec and not matching:
+ tty.warn("No installed packages match spec %s" % spec)
+ specs_to_test.extend(matching)
+
+ log_dir = os.getcwd()
+
+ if args.smoke_test:
+ for spec in specs_to_test:
+ log_file = os.path.join(log_dir, 'test-%s' % spec.dag_hash())
+ spec.package.do_test(log_file)
+ else:
+ raise NotImplementedError
diff --git a/lib/spack/spack/cmd/unit_test.py b/lib/spack/spack/cmd/unit_test.py
new file mode 100644
index 0000000000..3cc6680b68
--- /dev/null
+++ b/lib/spack/spack/cmd/unit_test.py
@@ -0,0 +1,105 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from __future__ import print_function
+
+import sys
+import os
+import re
+import argparse
+import pytest
+from six import StringIO
+
+from llnl.util.filesystem import working_dir
+from llnl.util.tty.colify import colify
+
+import spack.paths
+
+description = "run spack's unit tests"
+section = "developer"
+level = "long"
+
+
+def setup_parser(subparser):
+ subparser.add_argument(
+ '-H', '--pytest-help', action='store_true', default=False,
+ help="print full pytest help message, showing advanced options")
+
+ list_group = subparser.add_mutually_exclusive_group()
+ list_group.add_argument(
+ '-l', '--list', action='store_true', default=False,
+ help="list basic test names")
+ list_group.add_argument(
+ '-L', '--long-list', action='store_true', default=False,
+ help="list the entire hierarchy of tests")
+ subparser.add_argument(
+ '--extension', default=None,
+ help="run test for a given Spack extension"
+ )
+ subparser.add_argument(
+ 'tests', nargs=argparse.REMAINDER,
+ help="list of tests to run (will be passed to pytest -k)")
+
+
+def do_list(args, unknown_args):
+ """Print a lists of tests than what pytest offers."""
+ # Run test collection and get the tree out.
+ old_output = sys.stdout
+ try:
+ sys.stdout = output = StringIO()
+ pytest.main(['--collect-only'])
+ finally:
+ sys.stdout = old_output
+
+ # put the output in a more readable tree format.
+ lines = output.getvalue().split('\n')
+ output_lines = []
+ for line in lines:
+ match = re.match(r"(\s*)<([^ ]*) '([^']*)'", line)
+ if not match:
+ continue
+ indent, nodetype, name = match.groups()
+
+ # only print top-level for short list
+ if args.list:
+ if not indent:
+ output_lines.append(
+ os.path.basename(name).replace('.py', ''))
+ else:
+ print(indent + name)
+
+ if args.list:
+ colify(output_lines)
+
+
+def unit_test(parser, args, unknown_args):
+ if args.pytest_help:
+ # make the pytest.main help output more accurate
+ sys.argv[0] = 'spack unit-test'
+ pytest.main(['-h'])
+ return
+
+ # The default is to test the core of Spack. If the option `--extension`
+ # has been used, then test that extension.
+ pytest_root = spack.paths.test_path
+ if args.extension:
+ target = args.extension
+ extensions = spack.config.get('config:extensions')
+ pytest_root = spack.extensions.path_for_extension(target, *extensions)
+
+ # pytest.ini lives in the root of the spack repository.
+ with working_dir(pytest_root):
+ # --list and --long-list print the test output better.
+ if args.list or args.long_list:
+ do_list(args, unknown_args)
+ return
+
+ # Allow keyword search without -k if no options are specified
+ if (args.tests and not unknown_args and
+ not any(arg.startswith('-') for arg in args.tests)):
+ return pytest.main(['-k'] + args.tests)
+
+ # Just run the pytest command
+ return pytest.main(unknown_args + args.tests)
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index d5cb3065a8..b571769cf3 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -1576,6 +1576,28 @@ def do_install(self, **kwargs):
do_install.__doc__ += install_args_docstring
+ def do_test(self, log_file, dirty=False):
+ def test_process():
+ with log_output(log_file) as logger:
+ with logger.force_echo():
+ tty.msg('Testing package %s' %
+ self.spec.format('{name}-{hash:7}'))
+ old_debug = tty.is_debug()
+ tty.set_debug(True)
+ self.test()
+ tty.set_debug(old_debug)
+
+ try:
+ spack.build_environment.fork(
+ self, test_process, dirty=dirty, fake=False, context='test')
+ except Exception as e:
+ tty.error('Tests failed. See test log for details\n'
+ ' %s\n' % log_file)
+
+
+ def test(self):
+ pass
+
def unit_test_check(self):
"""Hook for unit tests to assert things about package internals.
diff --git a/lib/spack/spack/test/cmd/test.py b/lib/spack/spack/test/cmd/test.py
index a9ef735afe..31100994e7 100644
--- a/lib/spack/spack/test/cmd/test.py
+++ b/lib/spack/spack/test/cmd/test.py
@@ -5,7 +5,7 @@
from spack.main import SpackCommand
-spack_test = SpackCommand('test')
+spack_test = SpackCommand('unit-test')
cmd_test_py = 'lib/spack/spack/test/cmd/test.py'
diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py
index 097da3337e..7aba677a0c 100644
--- a/lib/spack/spack/util/executable.py
+++ b/lib/spack/spack/util/executable.py
@@ -2,7 +2,7 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
+import sys
import os
import re
import shlex
@@ -98,6 +98,9 @@ def __call__(self, *args, **kwargs):
If both ``output`` and ``error`` are set to ``str``, then one string
is returned containing output concatenated with error. Not valid
for ``input``
+ * ``str.split``, as in the ``split`` method of the Python string type.
+ Behaves the same as ``str``, except that value is also written to
+ ``stdout`` or ``stderr``.
By default, the subprocess inherits the parent's file descriptors.
@@ -132,7 +135,7 @@ def __call__(self, *args, **kwargs):
def streamify(arg, mode):
if isinstance(arg, string_types):
return open(arg, mode), True
- elif arg is str:
+ elif arg in (str, str.split):
return subprocess.PIPE, False
else:
return arg, False
@@ -168,12 +171,16 @@ def streamify(arg, mode):
out, err = proc.communicate()
result = None
- if output is str or error is str:
+ if output in (str, str.split) or error in (str, str.split):
result = ''
- if output is str:
+ if output in (str, str.split):
result += text_type(out.decode('utf-8'))
- if error is str:
+ if output is str.split:
+ sys.stdout.write(out)
+ if error in (str, str.split):
result += text_type(err.decode('utf-8'))
+ if error is str.split:
+ sys.stderr.write(err)
rc = self.returncode = proc.returncode
if fail_on_error and rc != 0 and (rc not in ignore_errors):
diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py
index 4282d8e014..2a6e407824 100644
--- a/var/spack/repos/builtin/packages/python/package.py
+++ b/var/spack/repos/builtin/packages/python/package.py
@@ -1109,3 +1109,18 @@ def remove_files_from_view(self, view, merge_map):
view.remove_file(src, dst)
else:
os.remove(dst)
+
+ def test(self):
+ # contains python executable
+ python = which('python')
+ assert os.path.dirname(python.path) == os.path.dirname(self.command.path)
+
+ # run hello world
+ output = self.command('-c', 'print("hello world!")',
+ output=str.split, error=str.split)
+ assert output == "hello world!\n"
+
+# error = self.command('-c', 'print("Error: failed.")',
+# output=str.split, error=str.split)
+
+# assert error.strip() == 'Error: failed.'
--
GitLab