Skip to content
Snippets Groups Projects
Commit 0a006351 authored by Massimiliano Culpo's avatar Massimiliano Culpo Committed by Peter Scheibel
Browse files

Spack can be extended with external commands (#8612)

This provides a mechanism to implement a new Spack command in a
separate directory, and with a small configuration change point Spack
to the new command.

To register the command, the directory must be added to the
"extensions" section of config.yaml. The command directory name must
have the prefix "spack-", and have the following layout:

  spack-X/
    pytest.ini #optional, for testing
    X/
	  cmd/
	    name-of-command1.py
	    name-of-command2.py
	    ...
    tests/ #optional
      conftest.py
	  test_name-of-command1.py
    templates/ #optional jinja templates, if needed

And in config.yaml:

  config:
    extensions:
      - /path/to/spack-X

If the extension includes tests, you can run them via spack by adding
the --extension option, like "spack test --extension=X"
parent b2b91a1f
No related branches found
No related tags found
No related merge requests found
...@@ -12,6 +12,8 @@ ...@@ -12,6 +12,8 @@
import inspect import inspect
from datetime import datetime, timedelta from datetime import datetime, timedelta
from six import string_types from six import string_types
import sys
# Ignore emacs backups when listing modules # Ignore emacs backups when listing modules
ignore_modules = [r'^\.#', '~$'] ignore_modules = [r'^\.#', '~$']
...@@ -597,3 +599,33 @@ def __str__(self): ...@@ -597,3 +599,33 @@ def __str__(self):
def __repr__(self): def __repr__(self):
return repr(self.ref_function()) return repr(self.ref_function())
def load_module_from_file(module_name, module_path):
"""Loads a python module from the path of the corresponding file.
Args:
module_name (str): namespace where the python module will be loaded,
e.g. ``foo.bar``
module_path (str): path of the python file containing the module
Returns:
A valid module object
Raises:
ImportError: when the module can't be loaded
FileNotFoundError: when module_path doesn't exist
"""
if sys.version_info[0] == 3 and sys.version_info[1] >= 5:
import importlib.util
spec = importlib.util.spec_from_file_location(module_name, module_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
elif sys.version_info[0] == 3 and sys.version_info[1] < 5:
import importlib.machinery
loader = importlib.machinery.SourceFileLoader(module_name, module_path)
module = loader.load_module()
elif sys.version_info[0] == 2:
import imp
module = imp.load_source(module_name, module_path)
return module
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
from llnl.util.filesystem import working_dir from llnl.util.filesystem import working_dir
import spack.config import spack.config
import spack.extensions
import spack.paths import spack.paths
import spack.spec import spack.spec
import spack.store import spack.store
...@@ -32,9 +33,6 @@ ...@@ -32,9 +33,6 @@
SETUP_PARSER = "setup_parser" SETUP_PARSER = "setup_parser"
DESCRIPTION = "description" DESCRIPTION = "description"
#: Names of all commands
all_commands = []
def python_name(cmd_name): def python_name(cmd_name):
"""Convert ``-`` to ``_`` in command name, to make a valid identifier.""" """Convert ``-`` to ``_`` in command name, to make a valid identifier."""
...@@ -60,11 +58,16 @@ def all_commands(): ...@@ -60,11 +58,16 @@ def all_commands():
global _all_commands global _all_commands
if _all_commands is None: if _all_commands is None:
_all_commands = [] _all_commands = []
for file in os.listdir(spack.paths.command_path): command_paths = [spack.paths.command_path] # Built-in commands
if file.endswith(".py") and not re.search(ignore_files, file): command_paths += spack.extensions.get_command_paths() # Extensions
cmd = re.sub(r'.py$', '', file) for path in command_paths:
_all_commands.append(cmd_name(cmd)) for file in os.listdir(path):
if file.endswith(".py") and not re.search(ignore_files, file):
cmd = re.sub(r'.py$', '', file)
_all_commands.append(cmd_name(cmd))
_all_commands.sort() _all_commands.sort()
return _all_commands return _all_commands
...@@ -85,10 +88,18 @@ def get_module(cmd_name): ...@@ -85,10 +88,18 @@ def get_module(cmd_name):
(contains ``-``, not ``_``). (contains ``-``, not ``_``).
""" """
pname = python_name(cmd_name) pname = python_name(cmd_name)
module_name = "%s.%s" % (__name__, pname)
module = __import__(module_name, try:
fromlist=[pname, SETUP_PARSER, DESCRIPTION], # Try to import the command from the built-in directory
level=0) module_name = "%s.%s" % (__name__, pname)
module = __import__(module_name,
fromlist=[pname, SETUP_PARSER, DESCRIPTION],
level=0)
tty.debug('Imported {0} from built-in commands'.format(pname))
except ImportError:
module = spack.extensions.get_module(cmd_name)
if not module:
raise
attr_setdefault(module, SETUP_PARSER, lambda *args: None) # null-op attr_setdefault(module, SETUP_PARSER, lambda *args: None) # null-op
attr_setdefault(module, DESCRIPTION, "") attr_setdefault(module, DESCRIPTION, "")
......
...@@ -34,6 +34,10 @@ def setup_parser(subparser): ...@@ -34,6 +34,10 @@ def setup_parser(subparser):
list_group.add_argument( list_group.add_argument(
'-L', '--long-list', action='store_true', default=False, '-L', '--long-list', action='store_true', default=False,
help="list the entire hierarchy of tests") help="list the entire hierarchy of tests")
subparser.add_argument(
'--extension', default=None,
help="run test for a given Spack extension"
)
subparser.add_argument( subparser.add_argument(
'tests', nargs=argparse.REMAINDER, 'tests', nargs=argparse.REMAINDER,
help="list of tests to run (will be passed to pytest -k)") help="list of tests to run (will be passed to pytest -k)")
...@@ -77,8 +81,16 @@ def test(parser, args, unknown_args): ...@@ -77,8 +81,16 @@ def test(parser, args, unknown_args):
pytest.main(['-h']) pytest.main(['-h'])
return return
# pytest.ini lives in lib/spack/spack/test # The default is to test the core of Spack. If the option `--extension`
with working_dir(spack.paths.test_path): # has been used, then test that extension.
pytest_root = spack.paths.test_path
if args.extension:
target = args.extension
extensions = spack.config.get('config:extensions')
pytest_root = spack.extensions.path_for_extension(target, *extensions)
# pytest.ini lives in the root of the spack repository.
with working_dir(pytest_root):
# --list and --long-list print the test output better. # --list and --long-list print the test output better.
if args.list or args.long_list: if args.list or args.long_list:
do_list(args, unknown_args) do_list(args, unknown_args)
......
# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""Service functions and classes to implement the hooks
for Spack's command extensions.
"""
import os
import re
import llnl.util.lang
import llnl.util.tty as tty
import spack.config
extension_regexp = re.compile(r'spack-([\w]*)')
def extension_name(path):
"""Returns the name of the extension in the path passed as argument.
Args:
path (str): path where the extension resides
Returns:
The extension name or None if path doesn't match the format
for Spack's extension.
"""
regexp_match = re.search(extension_regexp, os.path.basename(path))
if not regexp_match:
msg = "[FOLDER NAMING]"
msg += " {0} doesn't match the format for Spack's extensions"
tty.warn(msg.format(path))
return None
return regexp_match.group(1)
def load_command_extension(command, path):
"""Loads a command extension from the path passed as argument.
Args:
command (str): name of the command
path (str): base path of the command extension
Returns:
A valid module object if the command is found or None
"""
extension = extension_name(path)
if not extension:
return None
# Compute the absolute path of the file to be loaded, along with the
# name of the python module where it will be stored
cmd_path = os.path.join(path, extension, 'cmd', command + '.py')
python_name = command.replace('-', '_')
module_name = '{0}.{1}'.format(__name__, python_name)
try:
module = llnl.util.lang.load_module_from_file(module_name, cmd_path)
except (ImportError, IOError):
module = None
return module
def get_command_paths():
"""Return the list of paths where to search for command files."""
command_paths = []
extension_paths = spack.config.get('config:extensions') or []
for path in extension_paths:
extension = extension_name(path)
if extension:
command_paths.append(os.path.join(path, extension, 'cmd'))
return command_paths
def path_for_extension(target_name, *paths):
"""Return the test root dir for a given extension.
Args:
target_name (str): name of the extension to test
*paths: paths where the extensions reside
Returns:
Root directory where tests should reside or None
"""
for path in paths:
name = extension_name(path)
if name == target_name:
return path
else:
raise IOError('extension "{0}" not found'.format(target_name))
def get_module(cmd_name):
"""Imports the extension module for a particular command name
and returns it.
Args:
cmd_name (str): name of the command for which to get a module
(contains ``-``, not ``_``).
"""
# If built-in failed the import search the extension
# directories in order
extensions = spack.config.get('config:extensions') or []
for folder in extensions:
module = load_command_extension(cmd_name, folder)
if module:
return module
else:
return None
def get_template_dirs():
"""Returns the list of directories where to search for templates
in extensions.
"""
extension_dirs = spack.config.get('config:extensions') or []
extensions = [os.path.join(x, 'templates') for x in extension_dirs]
return extensions
...@@ -25,6 +25,10 @@ ...@@ -25,6 +25,10 @@
{'type': 'array', {'type': 'array',
'items': {'type': 'string'}}], 'items': {'type': 'string'}}],
}, },
'extensions': {
'type': 'array',
'items': {'type': 'string'}
},
'template_dirs': { 'template_dirs': {
'type': 'array', 'type': 'array',
'items': {'type': 'string'} 'items': {'type': 'string'}
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details. # Spack Project Developers. See the top-level COPYRIGHT file for details.
# #
# SPDX-License-Identifier: (Apache-2.0 OR MIT) # SPDX-License-Identifier: (Apache-2.0 OR MIT)
import itertools
import textwrap import textwrap
import jinja2 import jinja2
...@@ -72,8 +72,11 @@ def make_environment(dirs=None): ...@@ -72,8 +72,11 @@ def make_environment(dirs=None):
"""Returns an configured environment for template rendering.""" """Returns an configured environment for template rendering."""
if dirs is None: if dirs is None:
# Default directories where to search for templates # Default directories where to search for templates
builtins = spack.config.get('config:template_dirs')
extensions = spack.extensions.get_template_dirs()
dirs = [canonicalize_path(d) dirs = [canonicalize_path(d)
for d in spack.config.get('config:template_dirs')] for d in itertools.chain(builtins, extensions)]
# Loader for the templates # Loader for the templates
loader = jinja2.FileSystemLoader(dirs) loader = jinja2.FileSystemLoader(dirs)
# Environment of the template engine # Environment of the template engine
......
...@@ -24,6 +24,7 @@ ...@@ -24,6 +24,7 @@
import spack.database import spack.database
import spack.directory_layout import spack.directory_layout
import spack.environment as ev import spack.environment as ev
import spack.package_prefs
import spack.paths import spack.paths
import spack.platforms.test import spack.platforms.test
import spack.repo import spack.repo
...@@ -118,7 +119,7 @@ def mock_stage(tmpdir_factory): ...@@ -118,7 +119,7 @@ def mock_stage(tmpdir_factory):
@pytest.fixture(scope='session') @pytest.fixture(scope='session')
def _ignore_stage_files(): def ignore_stage_files():
"""Session-scoped helper for check_for_leftover_stage_files. """Session-scoped helper for check_for_leftover_stage_files.
Used to track which leftover files in the stage have been seen. Used to track which leftover files in the stage have been seen.
...@@ -145,7 +146,7 @@ def working_env(): ...@@ -145,7 +146,7 @@ def working_env():
@pytest.fixture(scope='function', autouse=True) @pytest.fixture(scope='function', autouse=True)
def check_for_leftover_stage_files(request, mock_stage, _ignore_stage_files): def check_for_leftover_stage_files(request, mock_stage, ignore_stage_files):
"""Ensure that each test leaves a clean stage when done. """Ensure that each test leaves a clean stage when done.
This can be disabled for tests that are expected to dirty the stage This can be disabled for tests that are expected to dirty the stage
...@@ -160,7 +161,7 @@ def check_for_leftover_stage_files(request, mock_stage, _ignore_stage_files): ...@@ -160,7 +161,7 @@ def check_for_leftover_stage_files(request, mock_stage, _ignore_stage_files):
files_in_stage = set() files_in_stage = set()
if os.path.exists(spack.paths.stage_path): if os.path.exists(spack.paths.stage_path):
files_in_stage = set( files_in_stage = set(
os.listdir(spack.paths.stage_path)) - _ignore_stage_files os.listdir(spack.paths.stage_path)) - ignore_stage_files
if 'disable_clean_stage_check' in request.keywords: if 'disable_clean_stage_check' in request.keywords:
# clean up after tests that are expected to be dirty # clean up after tests that are expected to be dirty
...@@ -168,7 +169,7 @@ def check_for_leftover_stage_files(request, mock_stage, _ignore_stage_files): ...@@ -168,7 +169,7 @@ def check_for_leftover_stage_files(request, mock_stage, _ignore_stage_files):
path = os.path.join(spack.paths.stage_path, f) path = os.path.join(spack.paths.stage_path, f)
remove_whatever_it_is(path) remove_whatever_it_is(path)
else: else:
_ignore_stage_files |= files_in_stage ignore_stage_files |= files_in_stage
assert not files_in_stage assert not files_in_stage
......
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
import pytest import pytest
import os.path
from datetime import datetime, timedelta from datetime import datetime, timedelta
import llnl.util.lang import llnl.util.lang
...@@ -16,6 +17,19 @@ def now(): ...@@ -16,6 +17,19 @@ def now():
return datetime.now() return datetime.now()
@pytest.fixture()
def module_path(tmpdir):
m = tmpdir.join('foo.py')
content = """
import os.path
value = 1
path = os.path.join('/usr', 'bin')
"""
m.write(content)
return str(m)
def test_pretty_date(): def test_pretty_date():
"""Make sure pretty_date prints the right dates.""" """Make sure pretty_date prints the right dates."""
now = datetime.now() now = datetime.now()
...@@ -110,3 +124,9 @@ def test_match_predicate(): ...@@ -110,3 +124,9 @@ def test_match_predicate():
with pytest.raises(ValueError): with pytest.raises(ValueError):
matcher = match_predicate(object()) matcher = match_predicate(object())
matcher('foo') matcher('foo')
def test_load_modules_from_file(module_path):
foo = llnl.util.lang.load_module_from_file('foo', module_path)
assert foo.value == 1
assert foo.path == os.path.join('/usr', 'bin')
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment