PythonPackage: Let There Be Tests! (#2869)

* Run python setup.py test if --run-tests
* Attempt to import the Python module after installation
* Add testing support to numpy and scipy
* Remove duplicated comments
* Update to new run-tests callback methodology
* Remove unrelated changes for another PR

lib/spack/spack/build_systems/python.py

@@ -24,6 +24,7 @@
 ##############################################################################
 
 import inspect
+import os
 
 from spack.directives import extends
 from spack.package import PackageBase, run_after
@@ -91,10 +92,26 @@ def configure(self, spec, prefix):
     # Default phases
     phases = ['build', 'install']
 
+    # Name of modules that the Python package provides
+    # This is used to test whether or not the installation succeeded
+    # These names generally come from running:
+    #
+    # >>> import setuptools
+    # >>> setuptools.find_packages()
+    #
+    # in the source tarball directory
+    import_modules = []
+
     # To be used in UI queries that require to know which
     # build-system class we are using
     build_system_class = 'PythonPackage'
 
+    #: Callback names for build-time test
+    build_time_test_callbacks = ['test']
+
+    #: Callback names for install-time test
+    install_time_test_callbacks = ['import_module_test']
+
     extends('python')
 
     def setup_file(self):
@@ -106,19 +123,38 @@ def build_directory(self):
         """The directory containing the ``setup.py`` file."""
         return self.stage.source_path
 
-    def python(self, *args):
-        inspect.getmodule(self).python(*args)
+    def python(self, *args, **kwargs):
+        inspect.getmodule(self).python(*args, **kwargs)
 
-    def setup_py(self, *args):
+    def setup_py(self, *args, **kwargs):
         setup = self.setup_file()
 
         with working_dir(self.build_directory):
-            self.python(setup, '--no-user-cfg', *args)
+            self.python(setup, '--no-user-cfg', *args, **kwargs)
+
+    def _setup_command_available(self, command):
+        """Determines whether or not a setup.py command exists.
+
+        :param str command: The command to look for
+        :return: True if the command is found, else False
+        :rtype: bool
+        """
+        kwargs = {
+            'output': os.devnull,
+            'error':  os.devnull,
+            'fail_on_error': False
+        }
+
+        python = inspect.getmodule(self).python
+        setup = self.setup_file()
+
+        python(setup, '--no-user-cfg', command, '--help', **kwargs)
+        return python.returncode == 0
 
     # The following phases and their descriptions come from:
     #   $ python setup.py --help-commands
-    # Only standard commands are included here, but some packages
-    # define extra commands as well
+
+    # Standard commands
 
     def build(self, spec, prefix):
         """Build everything needed to install."""
@@ -306,5 +342,37 @@ def check_args(self, spec, prefix):
         """Arguments to pass to check."""
         return []
 
+    # Testing
+
+    def test(self):
+        """Run unit tests after in-place build.
+
+        These tests are only run if the package actually has a 'test' command.
+        """
+        if self._setup_command_available('test'):
+            args = self.test_args(self.spec, self.prefix)
+
+            self.setup_py('test', *args)
+
+    def test_args(self, spec, prefix):
+        """Arguments to pass to test."""
+        return []
+
+    run_after('build')(PackageBase._run_default_build_time_test_callbacks)
+
+    def import_module_test(self):
+        """Attempts to import the module that was just installed.
+
+        This test is only run if the package overrides
+        :py:attr:`import_modules` with a list of module names."""
+
+        # Make sure we are importing the installed modules,
+        # not the ones in the current directory
+        with working_dir('..'):
+            for module in self.import_modules:
+                self.python('-c', 'import {0}'.format(module))
+
+    run_after('install')(PackageBase._run_default_install_time_test_callbacks)
+
     # Check that self.prefix is there after installation
     run_after('install')(PackageBase.sanity_check_prefix)

lib/spack/spack/util/executable.py

@@ -68,7 +68,7 @@ def __call__(self, *args, **kwargs):
 
             Raise an exception if the subprocess returns an
             error. Default is True.  When not set, the return code is
-            avaiale as `exe.returncode`.
+            available as `exe.returncode`.
 
           ignore_errors
 

var/spack/repos/builtin/packages/py-nose/package.py

@@ -34,6 +34,10 @@ class PyNose(PythonPackage):
     list_url = "https://pypi.python.org/pypi/nose/"
     list_depth = 2
 
+    import_modules = [
+        'nose', 'nose.ext', 'nose.plugins', 'nose.sphinx', 'nose.tools'
+    ]
+
     version('1.3.7', '4d3ad0ff07b61373d2cefc89c5d0b20b')
     version('1.3.6', '0ca546d81ca8309080fc80cb389e7a16')
     version('1.3.4', '6ed7169887580ddc9a8e16048d38274d')

var/spack/repos/builtin/packages/py-numpy/package.py

@@ -36,6 +36,18 @@ class PyNumpy(PythonPackage):
     homepage = "http://www.numpy.org/"
     url      = "https://pypi.io/packages/source/n/numpy/numpy-1.9.1.tar.gz"
 
+    install_time_test_callbacks = ['install_test', 'import_module_test']
+
+    import_modules = [
+        'numpy', 'numpy.compat', 'numpy.core', 'numpy.distutils', 'numpy.doc',
+        'numpy.f2py', 'numpy.fft', 'numpy.lib', 'numpy.linalg', 'numpy.ma',
+        'numpy.matrixlib', 'numpy.polynomial', 'numpy.random', 'numpy.testing',
+        'numpy.distutils.command', 'numpy.distutils.fcompiler'
+    ]
+
+    # FIXME: numpy._build_utils and numpy.core.code_generators failed to import
+    # FIXME: Is this expected?
+
     version('1.12.0', '33e5a84579f31829bbbba084fe0a4300',
             url="https://pypi.io/packages/source/n/numpy/numpy-1.12.0.zip")
     version('1.11.2', '03bd7927c314c43780271bf1ab795ebc')
@@ -53,6 +65,10 @@ class PyNumpy(PythonPackage):
     depends_on('blas',   when='+blas')
     depends_on('lapack', when='+lapack')
 
+    # Tests require:
+    # TODO: Add a 'test' deptype
+    # depends_on('py-nose@1.0.0:', type='test')
+
     def setup_dependent_package(self, module, dependent_spec):
         python_version = self.spec['python'].version.up_to(2)
         arch = '{0}-{1}'.format(platform.system().lower(), platform.machine())
@@ -132,3 +148,22 @@ def build_args(self, spec, prefix):
             args = ['-j', str(make_jobs)]
 
         return args
+
+    def test(self):
+        # `setup.py test` is not supported.  Use one of the following
+        # instead:
+        #
+        # - `python runtests.py`              (to build and test)
+        # - `python runtests.py --no-build`   (to test installed numpy)
+        # - `>>> numpy.test()`           (run tests for installed numpy
+        #                                 from within an interpreter)
+        pass
+
+    def install_test(self):
+        # Change directories due to the following error:
+        #
+        # ImportError: Error importing numpy: you should not try to import
+        #       numpy from its source directory; please exit the numpy
+        #       source tree, and relaunch your python interpreter from there.
+        with working_dir('..'):
+            python('-c', 'import numpy; numpy.test("full", verbose=2)')

var/spack/repos/builtin/packages/py-scipy/package.py

@@ -33,6 +33,22 @@ class PyScipy(PythonPackage):
     homepage = "http://www.scipy.org/"
     url = "https://pypi.io/packages/source/s/scipy/scipy-0.18.1.tar.gz"
 
+    install_time_test_callbacks = ['install_test', 'import_module_test']
+
+    import_modules = [
+        'scipy', 'scipy._build_utils', 'scipy._lib', 'scipy.cluster',
+        'scipy.constants', 'scipy.fftpack', 'scipy.integrate',
+        'scipy.interpolate', 'scipy.io', 'scipy.linalg', 'scipy.misc',
+        'scipy.ndimage', 'scipy.odr', 'scipy.optimize', 'scipy.signal',
+        'scipy.sparse', 'scipy.spatial', 'scipy.special', 'scipy.stats',
+        'scipy.weave', 'scipy.io.arff', 'scipy.io.harwell_boeing',
+        'scipy.io.matlab', 'scipy.optimize._lsq', 'scipy.sparse.csgraph',
+        'scipy.sparse.linalg', 'scipy.sparse.linalg.dsolve',
+        'scipy.sparse.linalg.eigen', 'scipy.sparse.linalg.isolve',
+        'scipy.sparse.linalg.eigen.arpack', 'scipy.sparse.linalg.eigen.lobpcg',
+        'scipy.special._precompute'
+    ]
+
     version('0.19.0', '91b8396231eec780222a57703d3ec550',
             url="https://pypi.io/packages/source/s/scipy/scipy-0.19.0.zip")
     version('0.18.1', '5fb5fb7ccb113ab3a039702b6c2f3327')
@@ -49,6 +65,10 @@ class PyScipy(PythonPackage):
     depends_on('blas')
     depends_on('lapack')
 
+    # Tests require:
+    # TODO: Add a 'test' deptype
+    # depends_on('py-nose', type='test')
+
     def build_args(self, spec, prefix):
         args = []
 
@@ -59,3 +79,22 @@ def build_args(self, spec, prefix):
             args.extend(['-j', str(make_jobs)])
 
         return args
+
+    def test(self):
+        # `setup.py test` is not supported.  Use one of the following
+        # instead:
+        #
+        # - `python runtests.py`              (to build and test)
+        # - `python runtests.py --no-build`   (to test installed scipy)
+        # - `>>> scipy.test()`           (run tests for installed scipy
+        #                                 from within an interpreter)
+        pass
+
+    def install_test(self):
+        # Change directories due to the following error:
+        #
+        # ImportError: Error importing scipy: you should not try to import
+        #       scipy from its source directory; please exit the scipy
+        #       source tree, and relaunch your python interpreter from there.
+        with working_dir('..'):
+            python('-c', 'import scipy; scipy.test("full", verbose=2)')

var/spack/repos/builtin/packages/py-setuptools/package.py

@@ -32,6 +32,12 @@ class PySetuptools(PythonPackage):
     homepage = "https://pypi.python.org/pypi/setuptools"
     url      = "https://pypi.io/packages/source/s/setuptools/setuptools-25.2.0.tar.gz"
 
+    import_modules = [
+        'pkg_resources', 'setuptools', 'pkg_resources.extern',
+        'pkg_resources._vendor', 'pkg_resources._vendor.packaging',
+        'setuptools.extern', 'setuptools.command'
+    ]
+
     version('34.2.0', '41b630da4ea6cfa5894d9eb3142922be',
             url="https://pypi.io/packages/source/s/setuptools/setuptools-34.2.0.zip")
     version('25.2.0', 'a0dbb65889c46214c691f6c516cf959c')
@@ -53,3 +59,11 @@ class PySetuptools(PythonPackage):
     depends_on('py-packaging@16.8:', when='@34.0.0:', type=('build', 'run'))
     depends_on('py-six@1.6.0:',      when='@34.0.0:', type=('build', 'run'))
     depends_on('py-appdirs@1.4.0:',  when='@34.0.0:', type=('build', 'run'))
+
+    # Tests require:
+    # TODO: Add a 'test' deptype
+    # FIXME: All of these depend on setuptools, creating a dependency loop
+    # FIXME: Is there any way around this problem?
+    # depends_on('py-pytest-flake8', type='test')
+    # depends_on('pytest@2.8:', type='test')
+    # depends_on('py-mock', when='^python@:3.2', type='test')