Add PythonPackage base class

- Add a PythonPackage class with build system support.
  - Support build phases in PythonPackage
  - Add a custom sanity check for PythonPackages
  - Get rid of nolink dependencies in python packages

- Update spack create to use new PythonPackage class

- Port most of Python packages to new PythonPackage class

- Conducted a massive install and activate of Python packages.
  - Fixed bugs introduced by install and activate.

- Update API docs on PythonPackage

var/spack/repos/builtin/packages/py-lockfile/package.py

@@ -25,7 +25,7 @@
 from spack import *
 
 
-class PyLockfile(Package):
+class PyLockfile(PythonPackage):
     """The lockfile package exports a LockFile class which provides a
        simple API for locking files. Unlike the Windows msvcrt.locking
        function, the fcntl.lockf and flock functions, and the
@@ -41,8 +41,4 @@ class PyLockfile(Package):
 
     version('0.10.2', '1aa6175a6d57f082cd12e7ac6102ab15')
 
-    extends("python")
     depends_on("py-setuptools", type='build')
-
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix=%s' % prefix)

var/spack/repos/builtin/packages/py-logilab-common/package.py

@@ -22,11 +22,10 @@
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-from spack import depends_on, extends, version
-from spack import Package
+from spack import *
 
 
-class PyLogilabCommon(Package):
+class PyLogilabCommon(PythonPackage):
     """Common modules used by Logilab projects"""
     homepage = "https://www.logilab.org/project/logilab-common"
     url      = "https://pypi.python.org/packages/a7/31/1650d23e44794d46935d82b86e73454cc83b814cbe1365260ccce8a2f4c6/logilab-common-1.2.0.tar.gz"
@@ -36,6 +35,3 @@ class PyLogilabCommon(Package):
     extends('python', ignore=r'bin/pytest')
     depends_on("py-setuptools", type='build')
     depends_on("py-six", type=('build', 'run'))
-
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix=%s' % prefix)

var/spack/repos/builtin/packages/py-macs2/package.py

@@ -26,7 +26,7 @@
 from spack import *
 
 
-class PyMacs2(Package):
+class PyMacs2(PythonPackage):
     """MACS2 Model-based Analysis of ChIP-Seq"""
 
     homepage = "https://github.com/taoliu/MACS"
@@ -34,13 +34,9 @@ class PyMacs2(Package):
 
     version('2.1.1.20160309', '2008ba838f83f34f8e0fddefe2a3a0159f4a740707c68058f815b31ddad53d26')
 
-    extends('python')
     depends_on('python@2.7:2.8')
 
     # Most Python packages only require py-setuptools as a build dependency.
     # However, py-macs2 requires py-setuptools during runtime as well.
     depends_on('py-setuptools', type=('build', 'run'))
     depends_on('py-numpy@1.6:', type=('build', 'run'))
-
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix={0}'.format(prefix))

var/spack/repos/builtin/packages/py-mako/package.py

@@ -25,7 +25,7 @@
 from spack import *
 
 
-class PyMako(Package):
+class PyMako(PythonPackage):
     """A super-fast templating language that borrows the best
        ideas from the existing templating languages."""
 
@@ -35,12 +35,7 @@ class PyMako(Package):
     version('1.0.4', 'c5fc31a323dd4990683d2f2da02d4e20')
     version('1.0.1', '9f0aafd177b039ef67b90ea350497a54')
 
-    extends('python')
-
     depends_on('py-setuptools', type='build')
     # depends_on('py-mock',   type='test')  # TODO: Add test deptype
     # depends_on('py-pytest', type='test')  # TODO: Add test deptype
     depends_on('py-markupsafe@0.9.2:', type=('build', 'run'))
-
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix={0}'.format(prefix))

var/spack/repos/builtin/packages/py-markdown/package.py

@@ -26,7 +26,7 @@
 from spack import *
 
 
-class PyMarkdown(Package):
+class PyMarkdown(PythonPackage):
     """This is a Python implementation of John Gruber's Markdown. It is
     almost completely compliant with the reference implementation, though
     there are a few very minor differences. See John's Syntax
@@ -48,14 +48,9 @@ class PyMarkdown(Package):
     version('2.5.1', 'be6f6ba65a8fb843d2aaf1fcdd68c755')
     version('2.5', '8393ceab9c6e33357fb8a7be063a4849')
 
-    extends('python')
-
     depends_on('py-setuptools', type='build')
     depends_on('python@2.7:2.8,3.2:3.4')
 
     def url_for_version(self, version):
         base_url = "https://github.com/waylan/Python-Markdown/archive"
         return "{0}/{1}-final.tar.gz".format(base_url, version)
-
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix={0}'.format(prefix))

var/spack/repos/builtin/packages/py-markupsafe/package.py

@@ -25,7 +25,7 @@
 from spack import *
 
 
-class PyMarkupsafe(Package):
+class PyMarkupsafe(PythonPackage):
     """MarkupSafe is a library for Python that implements a unicode
     string that is aware of HTML escaping rules and can be used to
     implement automatic string escaping. It is used by Jinja 2, the
@@ -40,9 +40,4 @@ class PyMarkupsafe(Package):
     version('0.20', '7da066d9cb191a70aa85d0a3d43565d1')
     version('0.19', 'ccb3f746c807c5500850987006854a6d')
 
-    extends('python')
-
     depends_on('py-setuptools', type='build')
-
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix={0}'.format(prefix))

var/spack/repos/builtin/packages/py-matplotlib/package.py

@@ -26,7 +26,7 @@
 import os
 
 
-class PyMatplotlib(Package):
+class PyMatplotlib(PythonPackage):
     """matplotlib is a python 2D plotting library which produces publication
     quality figures in a variety of hardcopy formats and interactive
     environments across platforms."""
@@ -80,7 +80,7 @@ class PyMatplotlib(Package):
 
     # --------- Optional dependencies
     depends_on('pkg-config', type='build')    # why not...
-    depends_on('py-pillow', when='+image', type=('build', 'run'))
+    depends_on('pil', when='+image', type=('build', 'run'))
     depends_on('py-ipython', when='+ipython')
     depends_on('ghostscript', when='+latex', type='run')
     depends_on('texlive', when='+latex', type='run')
@@ -95,9 +95,10 @@ class PyMatplotlib(Package):
     # depends_on('ttconv')
     depends_on('py-six@1.9.0:', type=('build', 'run'))
 
-    def install(self, spec, prefix):
-        setup_py('build')
-        setup_py('install', '--prefix={0}'.format(prefix))
+    @PythonPackage.sanity_check('install')
+    def set_backend(self):
+        spec = self.spec
+        prefix = self.prefix
 
         if '+qt' in spec or '+tk' in spec:
             # Set backend in matplotlib configuration file

var/spack/repos/builtin/packages/py-mccabe/package.py

@@ -25,7 +25,7 @@
 from spack import *
 
 
-class PyMccabe(Package):
+class PyMccabe(PythonPackage):
     """Ned's script to check McCabe complexity."""
 
     homepage = "https://github.com/PyCQA/mccabe"
@@ -41,13 +41,9 @@ class PyMccabe(Package):
     version('0.2',   '36d4808c37e187dbb1fe2373a0ac6645')
     version('0.1',   '3c9e8e72612a9c01d865630cc569150a')
 
-    extends('python')
     depends_on('python@2.7:2.8,3.3:')
 
     depends_on('py-setuptools', type='build')
 
     # TODO: Add test dependencies
     # depends_on('py-pytest', type='test')
-
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix={0}'.format(prefix))

var/spack/repos/builtin/packages/py-meep/package.py

@@ -25,7 +25,7 @@
 from spack import *
 
 
-class PyMeep(Package):
+class PyMeep(PythonPackage):
     """Python-meep is a wrapper around libmeep. It allows the scripting of
     Meep-simulations with Python"""
 
@@ -36,7 +36,6 @@ class PyMeep(Package):
 
     variant('mpi', default=True, description='Enable MPI support')
 
-    extends('python')
     depends_on('py-numpy', type=('build', 'run'))
     depends_on('py-scipy', type=('build', 'run'))
     depends_on('py-matplotlib', type=('build', 'run'))
@@ -50,9 +49,12 @@ class PyMeep(Package):
     # or else it can't handle newer C++ compilers and flags.
     depends_on('swig@1.3.39:3.0.2')
 
-    def install(self, spec, prefix):
-        setup = 'setup-mpi.py' if '+mpi' in spec else 'setup.py'
+    phases = ['clean', 'build_ext', 'install', 'bdist']
 
+    def setup_file(self, spec, prefix):
+        return 'setup-mpi.py' if '+mpi' in spec else 'setup.py'
+
+    def common_args(self, spec, prefix):
         include_dirs = [
             spec['meep'].prefix.include,
             spec['py-numpy'].include
@@ -69,7 +71,19 @@ def install(self, spec, prefix):
         include_flags = '-I{0}'.format(','.join(include_dirs))
         library_flags = '-L{0}'.format(','.join(library_dirs))
 
-        python(setup, 'clean', '--all')
-        python(setup, 'build_ext', include_flags, library_flags)
-        python(setup, 'install', '--prefix={0}'.format(prefix))
-        python(setup, 'bdist', include_flags, library_flags)
+        # FIXME: For some reason, this stopped working.
+        # The -I and -L are no longer being properly forwarded to setup.py:
+        # meep_common.i:87: Error: Unable to find 'meep/mympi.hpp'
+        # meep_common.i:88: Error: Unable to find 'meep/vec.hpp'
+        # meep_common.i:89: Error: Unable to find 'meep.hpp'
+
+        return [include_flags, library_flags]
+
+    def clean_args(self, spec, prefix):
+        return ['--all']
+
+    def build_ext_args(self, spec, prefix):
+        return self.common_args(spec, prefix)
+
+    def bdist_args(self, spec, prefix):
+        return self.common_args(spec, prefix)

var/spack/repos/builtin/packages/py-mistune/package.py

@@ -22,11 +22,10 @@
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-from spack import depends_on, extends, version
-from spack import Package
+from spack import *
 
 
-class PyMistune(Package):
+class PyMistune(PythonPackage):
     """
     Python markdown parser
     """
@@ -39,8 +38,4 @@ class PyMistune(Package):
     version('0.5.1', '1c6cfce28a4aa90cf125217cd6c6fe6c')
     version('0.5', '997736554f1f95eea78c66ae339b5722')
 
-    extends('python')
     depends_on('py-setuptools', type='build')
-
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix=%s' % prefix)

var/spack/repos/builtin/packages/py-mock/package.py

@@ -25,7 +25,7 @@
 from spack import *
 
 
-class PyMock(Package):
+class PyMock(PythonPackage):
     """mock is a library for testing in Python. It allows you to replace parts
     of your system under test with mock objects and make assertions about how
     they have been used."""
@@ -35,9 +35,5 @@ class PyMock(Package):
 
     version('1.3.0', '73ee8a4afb3ff4da1b4afa287f39fdeb')
 
-    extends('python')
     depends_on('py-pbr', type=('build', 'run'))
     depends_on('py-setuptools@17.1:', type='build')
-
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix=%s' % prefix)

var/spack/repos/builtin/packages/py-monotonic/package.py

@@ -25,7 +25,7 @@
 from spack import *
 
 
-class PyMonotonic(Package):
+class PyMonotonic(PythonPackage):
     """An implementation of time.monotonic() for Python 2 & < 3.3"""
 
     homepage = "https://pypi.python.org/pypi/monotonic"
@@ -33,9 +33,4 @@ class PyMonotonic(Package):
 
     version('1.2', 'd14c93aabc3d6af25ef086b032b123cf')
 
-    extends('python')
-
     depends_on('py-setuptools', type='build')
-
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix={0}'.format(prefix))

var/spack/repos/builtin/packages/py-mpi4py/package.py

@@ -25,7 +25,7 @@
 from spack import *
 
 
-class PyMpi4py(Package):
+class PyMpi4py(PythonPackage):
     """This package provides Python bindings for the Message Passing
        Interface (MPI) standard. It is implemented on top of the
        MPI-1/MPI-2 specification and exposes an API which grounds on the
@@ -38,9 +38,5 @@ class PyMpi4py(Package):
     version('2.0.0', '4f7d8126d7367c239fd67615680990e3')
     version('1.3.1', 'dbe9d22bdc8ed965c23a7ceb6f32fc3c')
 
-    extends('python')
     depends_on('py-setuptools', type='build')
     depends_on('mpi')
-
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix=%s' % prefix)

var/spack/repos/builtin/packages/py-mpmath/package.py

@@ -25,14 +25,9 @@
 from spack import *
 
 
-class PyMpmath(Package):
+class PyMpmath(PythonPackage):
     """A Python library for arbitrary-precision floating-point arithmetic."""
     homepage = "http://mpmath.org"
     url      = "https://pypi.python.org/packages/source/m/mpmath/mpmath-all-0.19.tar.gz"
 
     version('0.19', 'd1b7e19dd6830d0d7b5e1bc93d46c02c')
-
-    extends('python')
-
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix=%s' % prefix)

var/spack/repos/builtin/packages/py-mx/package.py

@@ -25,7 +25,7 @@
 from spack import *
 
 
-class PyMx(Package):
+class PyMx(PythonPackage):
     """The eGenix.com mx Base Distribution for Python is a collection of
        professional quality software tools which enhance Python's
        usability in many important areas such as fast text searching,
@@ -36,8 +36,3 @@ class PyMx(Package):
     url      = "https://downloads.egenix.com/python/egenix-mx-base-3.2.8.tar.gz"
 
     version('3.2.8', '9d9d3a25f9dc051a15e97f452413423b')
-
-    extends('python')
-
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix=%s' % prefix)

var/spack/repos/builtin/packages/py-mysqldb1/package.py

@@ -25,15 +25,14 @@
 from spack import *
 
 
-class PyMysqldb1(Package):
+class PyMysqldb1(PythonPackage):
     """Legacy mysql bindings for python"""
     homepage = "https://github.com/farcepest/MySQLdb1"
     url      = "https://github.com/farcepest/MySQLdb1/archive/MySQLdb-1.2.5.tar.gz"
 
-    version('1.2.5', '332c8f4955b6bc0c79ea15170bf7321b')
+    version('1.2.5', '332c8f4955b6bc0c79ea15170bf7321b',
+            url="https://github.com/farcepest/MySQLdb1/archive/MySQLdb-1.2.5.tar.gz")
 
-    extends('python')
-    depends_on('py-setuptools', type='build')
+    # FIXME: Missing dependency on mysql
 
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix=%s' % prefix)
+    depends_on('py-setuptools', type='build')

var/spack/repos/builtin/packages/py-nbconvert/package.py

@@ -25,30 +25,31 @@
 from spack import *
 
 
-class PyNbconvert(Package):
+class PyNbconvert(PythonPackage):
     """Jupyter Notebook Conversion"""
 
     homepage = "https://github.com/jupyter/nbconvert"
     url      = "https://github.com/jupyter/nbconvert/archive/4.2.0.tar.gz"
 
-    version('4.2.0' , '8bd88771cc00f575d5edcd0b5197f964')
-    version('4.1.0' , '06655576713ba1ff7cece2b92760c187')
-    version('4.0.0' , '9661620b1e10a7b46f314588d2d0932f')
-
-    extends('python')
+    version('4.2.0', '8bd88771cc00f575d5edcd0b5197f964')
+    version('4.1.0', '06655576713ba1ff7cece2b92760c187')
+    version('4.0.0', '9661620b1e10a7b46f314588d2d0932f')
 
     depends_on('py-setuptools', type='build')
     depends_on('py-pycurl', type='build')
     depends_on('python@2.7:2.7.999,3.3:')
-    depends_on('py-mistune')
-    depends_on('py-jinja2')
-    depends_on('py-pygments')
-    depends_on('py-traitlets')
-    depends_on('py-jupyter-core')
-    depends_on('py-nbformat')
-    depends_on('py-entrypoints')
-    depends_on('py-tornado')
-    depends_on('py-jupyter-client')
+    depends_on('py-mistune', type=('build', 'run'))
+    depends_on('py-jinja2', type=('build', 'run'))
+    depends_on('py-pygments', type=('build', 'run'))
+    depends_on('py-traitlets', type=('build', 'run'))
+    depends_on('py-jupyter-core', type=('build', 'run'))
+    depends_on('py-nbformat', type=('build', 'run'))
+    depends_on('py-entrypoints', type=('build', 'run'))
+    depends_on('py-tornado', type=('build', 'run'))
+    depends_on('py-jupyter-client', type=('build', 'run'))
 
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix={0}'.format(prefix))
+    # FIXME:
+    # Failed, try again after installing PycURL with `pip install pycurl` to avoid outdated SSL.  # noqa
+    # Failed to download css from https://cdn.jupyter.org/notebook/4.1.0/style/style.min.css: [Errno socket error] [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:661)  # noqa
+    # Downloading CSS: https://cdn.jupyter.org/notebook/4.1.0/style/style.min.css  # noqa
+    # error: Need Notebook CSS to proceed: nbconvert/resources/style.min.css

var/spack/repos/builtin/packages/py-nbformat/package.py

@@ -25,7 +25,7 @@
 from spack import *
 
 
-class PyNbformat(Package):
+class PyNbformat(PythonPackage):
     """The Jupyter Notebook format"""
 
     homepage = "https://github.com/jupyter/nbformat"
@@ -35,13 +35,8 @@ class PyNbformat(Package):
     version('4.0.1', 'ab7172e517c9d561c0c01eef5631b4c8')
     version('4.0.0', '7cf61359fa4e9cf3ef5e969e2fcb909e')
 
-    extends('python')
-
     depends_on('py-setuptools', type='build')
-    depends_on('py-ipython-genutils')
-    depends_on('py-traitlets')
-    depends_on('py-jsonschema')
-    depends_on('py-jupyter-core')
-
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix={0}'.format(prefix))
+    depends_on('py-ipython-genutils', type=('build', 'run'))
+    depends_on('py-traitlets', type=('build', 'run'))
+    depends_on('py-jsonschema', type=('build', 'run'))
+    depends_on('py-jupyter-core', type=('build', 'run'))

var/spack/repos/builtin/packages/py-nestle/package.py

@@ -25,7 +25,7 @@
 from spack import *
 
 
-class PyNestle(Package):
+class PyNestle(PythonPackage):
     """Nested sampling algorithms for evaluating Bayesian evidence."""
 
     homepage = "http://kbarbary.github.io/nestle/"
@@ -34,11 +34,7 @@ class PyNestle(Package):
     version('0.1.1', '4875c0f9a0a8e263c1d7f5fa6ce604c5')
 
     # Required dependencies
-    extends('python')
     depends_on('py-numpy', type=('build', 'run'))
 
     # Optional dependencies
     depends_on('py-scipy', type=('build', 'run'))
-
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix={0}'.format(prefix))

var/spack/repos/builtin/packages/py-netcdf/package.py

@@ -25,18 +25,14 @@
 from spack import *
 
 
-class PyNetcdf(Package):
+class PyNetcdf(PythonPackage):
     """Python interface to the netCDF Library."""
     homepage = "http://unidata.github.io/netcdf4-python"
     url      = "https://github.com/Unidata/netcdf4-python/tarball/v1.2.3.1rel"
 
     version('1.2.3.1', '4fc4320d4f2a77b894ebf8da1c9895af')
 
-    extends('python')
     depends_on('py-numpy', type=('build', 'run'))
     depends_on('py-cython', type=('build', 'run'))
     depends_on('py-setuptools', type=('build', 'run'))
     depends_on('netcdf')
-
-    def install(self, spec, prefix):
-        setup_py('install', '--prefix=%s' % prefix)