diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py
index 21eb9ee3baa340b2e7acc4ee4479bb5268c4c44d..70c646bed07e4ec649237406637cdfaee95df9c1 100644
--- a/lib/spack/llnl/util/lang.py
+++ b/lib/spack/llnl/util/lang.py
@@ -12,6 +12,8 @@
import inspect
from datetime import datetime, timedelta
from six import string_types
+import sys
+
# Ignore emacs backups when listing modules
ignore_modules = [r'^\.#', '~$']
@@ -597,3 +599,33 @@ def __str__(self):
def __repr__(self):
return repr(self.ref_function())
+
+
+def load_module_from_file(module_name, module_path):
+ """Loads a python module from the path of the corresponding file.
+
+ Args:
+ module_name (str): namespace where the python module will be loaded,
+ e.g. ``foo.bar``
+ module_path (str): path of the python file containing the module
+
+ Returns:
+ A valid module object
+
+ Raises:
+ ImportError: when the module can't be loaded
+ FileNotFoundError: when module_path doesn't exist
+ """
+ if sys.version_info[0] == 3 and sys.version_info[1] >= 5:
+ import importlib.util
+ spec = importlib.util.spec_from_file_location(module_name, module_path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ elif sys.version_info[0] == 3 and sys.version_info[1] < 5:
+ import importlib.machinery
+ loader = importlib.machinery.SourceFileLoader(module_name, module_path)
+ module = loader.load_module()
+ elif sys.version_info[0] == 2:
+ import imp
+ module = imp.load_source(module_name, module_path)
+ return module
diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py
index 630a6b6e6378d7a526e238aa04c8517b10729316..9d003eb656c2cdab47306245f9d9c8411af639d7 100644
--- a/lib/spack/spack/cmd/__init__.py
+++ b/lib/spack/spack/cmd/__init__.py
@@ -17,6 +17,7 @@
from llnl.util.filesystem import working_dir
import spack.config
+import spack.extensions
import spack.paths
import spack.spec
import spack.store
@@ -32,9 +33,6 @@
SETUP_PARSER = "setup_parser"
DESCRIPTION = "description"
-#: Names of all commands
-all_commands = []
-
def python_name(cmd_name):
"""Convert ``-`` to ``_`` in command name, to make a valid identifier."""
@@ -60,11 +58,16 @@ def all_commands():
global _all_commands
if _all_commands is None:
_all_commands = []
- for file in os.listdir(spack.paths.command_path):
- if file.endswith(".py") and not re.search(ignore_files, file):
- cmd = re.sub(r'.py$', '', file)
- _all_commands.append(cmd_name(cmd))
+ command_paths = [spack.paths.command_path] # Built-in commands
+ command_paths += spack.extensions.get_command_paths() # Extensions
+ for path in command_paths:
+ for file in os.listdir(path):
+ if file.endswith(".py") and not re.search(ignore_files, file):
+ cmd = re.sub(r'.py$', '', file)
+ _all_commands.append(cmd_name(cmd))
+
_all_commands.sort()
+
return _all_commands
@@ -85,10 +88,18 @@ def get_module(cmd_name):
(contains ``-``, not ``_``).
"""
pname = python_name(cmd_name)
- module_name = "%s.%s" % (__name__, pname)
- module = __import__(module_name,
- fromlist=[pname, SETUP_PARSER, DESCRIPTION],
- level=0)
+
+ try:
+ # Try to import the command from the built-in directory
+ module_name = "%s.%s" % (__name__, pname)
+ module = __import__(module_name,
+ fromlist=[pname, SETUP_PARSER, DESCRIPTION],
+ level=0)
+ tty.debug('Imported {0} from built-in commands'.format(pname))
+ except ImportError:
+ module = spack.extensions.get_module(cmd_name)
+ if not module:
+ raise
attr_setdefault(module, SETUP_PARSER, lambda *args: None) # null-op
attr_setdefault(module, DESCRIPTION, "")
diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py
index 1c1b739a621ce119e8f532493eab4fa13640eccf..63e3778a6bec909192a74f5beb7b529eecf45690 100644
--- a/lib/spack/spack/cmd/test.py
+++ b/lib/spack/spack/cmd/test.py
@@ -34,6 +34,10 @@ def setup_parser(subparser):
list_group.add_argument(
'-L', '--long-list', action='store_true', default=False,
help="list the entire hierarchy of tests")
+ subparser.add_argument(
+ '--extension', default=None,
+ help="run test for a given Spack extension"
+ )
subparser.add_argument(
'tests', nargs=argparse.REMAINDER,
help="list of tests to run (will be passed to pytest -k)")
@@ -77,8 +81,16 @@ def test(parser, args, unknown_args):
pytest.main(['-h'])
return
- # pytest.ini lives in lib/spack/spack/test
- with working_dir(spack.paths.test_path):
+ # The default is to test the core of Spack. If the option `--extension`
+ # has been used, then test that extension.
+ pytest_root = spack.paths.test_path
+ if args.extension:
+ target = args.extension
+ extensions = spack.config.get('config:extensions')
+ pytest_root = spack.extensions.path_for_extension(target, *extensions)
+
+ # pytest.ini lives in the root of the spack repository.
+ with working_dir(pytest_root):
# --list and --long-list print the test output better.
if args.list or args.long_list:
do_list(args, unknown_args)
diff --git a/lib/spack/spack/extensions.py b/lib/spack/spack/extensions.py
new file mode 100644
index 0000000000000000000000000000000000000000..449798e94efa49ed079756295e1874fe6aa8d036
--- /dev/null
+++ b/lib/spack/spack/extensions.py
@@ -0,0 +1,123 @@
+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+"""Service functions and classes to implement the hooks
+for Spack's command extensions.
+"""
+import os
+import re
+
+import llnl.util.lang
+import llnl.util.tty as tty
+
+import spack.config
+
+
+extension_regexp = re.compile(r'spack-([\w]*)')
+
+
+def extension_name(path):
+ """Returns the name of the extension in the path passed as argument.
+
+ Args:
+ path (str): path where the extension resides
+
+ Returns:
+ The extension name or None if path doesn't match the format
+ for Spack's extension.
+ """
+ regexp_match = re.search(extension_regexp, os.path.basename(path))
+ if not regexp_match:
+ msg = "[FOLDER NAMING]"
+ msg += " {0} doesn't match the format for Spack's extensions"
+ tty.warn(msg.format(path))
+ return None
+ return regexp_match.group(1)
+
+
+def load_command_extension(command, path):
+ """Loads a command extension from the path passed as argument.
+
+ Args:
+ command (str): name of the command
+ path (str): base path of the command extension
+
+ Returns:
+ A valid module object if the command is found or None
+ """
+ extension = extension_name(path)
+ if not extension:
+ return None
+
+ # Compute the absolute path of the file to be loaded, along with the
+ # name of the python module where it will be stored
+ cmd_path = os.path.join(path, extension, 'cmd', command + '.py')
+ python_name = command.replace('-', '_')
+ module_name = '{0}.{1}'.format(__name__, python_name)
+
+ try:
+ module = llnl.util.lang.load_module_from_file(module_name, cmd_path)
+ except (ImportError, IOError):
+ module = None
+
+ return module
+
+
+def get_command_paths():
+ """Return the list of paths where to search for command files."""
+ command_paths = []
+ extension_paths = spack.config.get('config:extensions') or []
+
+ for path in extension_paths:
+ extension = extension_name(path)
+ if extension:
+ command_paths.append(os.path.join(path, extension, 'cmd'))
+
+ return command_paths
+
+
+def path_for_extension(target_name, *paths):
+ """Return the test root dir for a given extension.
+
+ Args:
+ target_name (str): name of the extension to test
+ *paths: paths where the extensions reside
+
+ Returns:
+ Root directory where tests should reside or None
+ """
+ for path in paths:
+ name = extension_name(path)
+ if name == target_name:
+ return path
+ else:
+ raise IOError('extension "{0}" not found'.format(target_name))
+
+
+def get_module(cmd_name):
+ """Imports the extension module for a particular command name
+ and returns it.
+
+ Args:
+ cmd_name (str): name of the command for which to get a module
+ (contains ``-``, not ``_``).
+ """
+ # If built-in failed the import search the extension
+ # directories in order
+ extensions = spack.config.get('config:extensions') or []
+ for folder in extensions:
+ module = load_command_extension(cmd_name, folder)
+ if module:
+ return module
+ else:
+ return None
+
+
+def get_template_dirs():
+ """Returns the list of directories where to search for templates
+ in extensions.
+ """
+ extension_dirs = spack.config.get('config:extensions') or []
+ extensions = [os.path.join(x, 'templates') for x in extension_dirs]
+ return extensions
diff --git a/lib/spack/spack/schema/config.py b/lib/spack/spack/schema/config.py
index 1aa54bf48e90cff67429884b0e8b159475eb741a..30c0bf0591e61e09d3e631f8a9899f7a5b09c5c3 100644
--- a/lib/spack/spack/schema/config.py
+++ b/lib/spack/spack/schema/config.py
@@ -25,6 +25,10 @@
{'type': 'array',
'items': {'type': 'string'}}],
},
+ 'extensions': {
+ 'type': 'array',
+ 'items': {'type': 'string'}
+ },
'template_dirs': {
'type': 'array',
'items': {'type': 'string'}
diff --git a/lib/spack/spack/tengine.py b/lib/spack/spack/tengine.py
index a48f9a27b78975e268adbdfeeff9aac546edbfb2..6aceb391cd2722ee89b968fc10f887b72c73b216 100644
--- a/lib/spack/spack/tengine.py
+++ b/lib/spack/spack/tengine.py
@@ -2,7 +2,7 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
+import itertools
import textwrap
import jinja2
@@ -72,8 +72,11 @@ def make_environment(dirs=None):
"""Returns an configured environment for template rendering."""
if dirs is None:
# Default directories where to search for templates
+ builtins = spack.config.get('config:template_dirs')
+ extensions = spack.extensions.get_template_dirs()
dirs = [canonicalize_path(d)
- for d in spack.config.get('config:template_dirs')]
+ for d in itertools.chain(builtins, extensions)]
+
# Loader for the templates
loader = jinja2.FileSystemLoader(dirs)
# Environment of the template engine
diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py
index ff95f463bdc1c2680205e3f3fafb7047becdd04f..b2d255c34051acbef1f8ac88882d92be35d42306 100644
--- a/lib/spack/spack/test/conftest.py
+++ b/lib/spack/spack/test/conftest.py
@@ -24,6 +24,7 @@
import spack.database
import spack.directory_layout
import spack.environment as ev
+import spack.package_prefs
import spack.paths
import spack.platforms.test
import spack.repo
@@ -118,7 +119,7 @@ def mock_stage(tmpdir_factory):
@pytest.fixture(scope='session')
-def _ignore_stage_files():
+def ignore_stage_files():
"""Session-scoped helper for check_for_leftover_stage_files.
Used to track which leftover files in the stage have been seen.
@@ -145,7 +146,7 @@ def working_env():
@pytest.fixture(scope='function', autouse=True)
-def check_for_leftover_stage_files(request, mock_stage, _ignore_stage_files):
+def check_for_leftover_stage_files(request, mock_stage, ignore_stage_files):
"""Ensure that each test leaves a clean stage when done.
This can be disabled for tests that are expected to dirty the stage
@@ -160,7 +161,7 @@ def check_for_leftover_stage_files(request, mock_stage, _ignore_stage_files):
files_in_stage = set()
if os.path.exists(spack.paths.stage_path):
files_in_stage = set(
- os.listdir(spack.paths.stage_path)) - _ignore_stage_files
+ os.listdir(spack.paths.stage_path)) - ignore_stage_files
if 'disable_clean_stage_check' in request.keywords:
# clean up after tests that are expected to be dirty
@@ -168,7 +169,7 @@ def check_for_leftover_stage_files(request, mock_stage, _ignore_stage_files):
path = os.path.join(spack.paths.stage_path, f)
remove_whatever_it_is(path)
else:
- _ignore_stage_files |= files_in_stage
+ ignore_stage_files |= files_in_stage
assert not files_in_stage
diff --git a/lib/spack/spack/test/llnl/util/lang.py b/lib/spack/spack/test/llnl/util/lang.py
index edeab72c28bad95cafed41b1a5cec8d6cdae174a..cbc24b51db3eccd4dd1cb5137beb75bba2f6b4e1 100644
--- a/lib/spack/spack/test/llnl/util/lang.py
+++ b/lib/spack/spack/test/llnl/util/lang.py
@@ -5,6 +5,7 @@
import pytest
+import os.path
from datetime import datetime, timedelta
import llnl.util.lang
@@ -16,6 +17,19 @@ def now():
return datetime.now()
+@pytest.fixture()
+def module_path(tmpdir):
+ m = tmpdir.join('foo.py')
+ content = """
+import os.path
+
+value = 1
+path = os.path.join('/usr', 'bin')
+"""
+ m.write(content)
+ return str(m)
+
+
def test_pretty_date():
"""Make sure pretty_date prints the right dates."""
now = datetime.now()
@@ -110,3 +124,9 @@ def test_match_predicate():
with pytest.raises(ValueError):
matcher = match_predicate(object())
matcher('foo')
+
+
+def test_load_modules_from_file(module_path):
+ foo = llnl.util.lang.load_module_from_file('foo', module_path)
+ assert foo.value == 1
+ assert foo.path == os.path.join('/usr', 'bin')