Skip to content
Snippets Groups Projects
Commit c0aaa8fc authored by Adam J. Stewart's avatar Adam J. Stewart Committed by Todd Gamblin
Browse files

Add PythonPackage base class

- Add a PythonPackage class with build system support.
  - Support build phases in PythonPackage
  - Add a custom sanity check for PythonPackages
  - Get rid of nolink dependencies in python packages

- Update spack create to use new PythonPackage class

- Port most of Python packages to new PythonPackage class

- Conducted a massive install and activate of Python packages.
  - Fixed bugs introduced by install and activate.

- Update API docs on PythonPackage
parent 1f49493f
No related branches found
No related tags found
No related merge requests found
Showing
with 391 additions and 95 deletions
......@@ -159,9 +159,10 @@
from spack.build_systems.makefile import MakefilePackage
from spack.build_systems.autotools import AutotoolsPackage
from spack.build_systems.cmake import CMakePackage
from spack.build_systems.python import PythonPackage
from spack.build_systems.r import RPackage
__all__ += ['Package', 'CMakePackage', 'AutotoolsPackage', 'MakefilePackage',
'RPackage']
'PythonPackage', 'RPackage']
from spack.version import Version, ver
__all__ += ['Version', 'ver']
......
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import inspect
from spack.directives import extends
from spack.package import PackageBase
from llnl.util.filesystem import working_dir
class PythonPackage(PackageBase):
"""Specialized class for packages that are built using Python
setup.py files
This class provides the following phases that can be overridden:
* build
* build_py
* build_ext
* build_clib
* build_scripts
* clean
* install
* install_lib
* install_headers
* install_scripts
* install_data
* sdist
* register
* bdist
* bdist_dumb
* bdist_rpm
* bdist_wininst
* upload
* check
These are all standard setup.py commands and can be found by running:
.. code-block:: console
$ python setup.py --help-commands
By default, only the 'build' and 'install' phases are run, but if you
need to run more phases, simply modify your ``phases`` list like so:
.. code-block:: python
phases = ['build_ext', 'install', 'bdist']
Each phase provides a function <phase> that runs:
.. code-block:: console
$ python --no-user-cfg setup.py <phase>
Each phase also has a <phase_args> function that can pass arguments to
this call. All of these functions are empty except for the ``install_args``
function, which passes ``--prefix=/path/to/installation/directory``.
If you need to run a phase which is not a standard setup.py command,
you'll need to define a function for it like so:
.. code-block:: python
def configure(self, spec, prefix):
self.setup_py('configure')
"""
# Default phases
phases = ['build', 'install']
# To be used in UI queries that require to know which
# build-system class we are using
build_system_class = 'PythonPackage'
extends('python')
def setup_file(self, spec, prefix):
"""Returns the name of the setup file to use."""
return 'setup.py'
def build_directory(self):
"""The directory containing the ``setup.py`` file."""
return self.stage.source_path
def python(self, *args):
inspect.getmodule(self).python(*args)
def setup_py(self, *args):
setup = self.setup_file(self.spec, self.prefix)
with working_dir(self.build_directory()):
self.python(setup, '--no-user-cfg', *args)
# The following phases and their descriptions come from:
# $ python setup.py --help-commands
# Only standard commands are included here, but some packages
# define extra commands as well
def build(self, spec, prefix):
"""Build everything needed to install."""
args = self.build_args(spec, prefix)
self.setup_py('build', *args)
def build_args(self, spec, prefix):
"""Arguments to pass to build."""
return []
def build_py(self, spec, prefix):
'''"Build" pure Python modules (copy to build directory).'''
args = self.build_py_args(spec, prefix)
self.setup_py('build_py', *args)
def build_py_args(self, spec, prefix):
"""Arguments to pass to build_py."""
return []
def build_ext(self, spec, prefix):
"""Build C/C++ extensions (compile/link to build directory)."""
args = self.build_ext_args(spec, prefix)
self.setup_py('build_ext', *args)
def build_ext_args(self, spec, prefix):
"""Arguments to pass to build_ext."""
return []
def build_clib(self, spec, prefix):
"""Build C/C++ libraries used by Python extensions."""
args = self.build_clib_args(spec, prefix)
self.setup_py('build_clib', *args)
def build_clib_args(self, spec, prefix):
"""Arguments to pass to build_clib."""
return []
def build_scripts(self, spec, prefix):
'''"Build" scripts (copy and fixup #! line).'''
args = self.build_scripts_args(spec, prefix)
self.setup_py('build_scripts', *args)
def clean(self, spec, prefix):
"""Clean up temporary files from 'build' command."""
args = self.clean_args(spec, prefix)
self.setup_py('clean', *args)
def clean_args(self, spec, prefix):
"""Arguments to pass to clean."""
return []
def install(self, spec, prefix):
"""Install everything from build directory."""
args = self.install_args(spec, prefix)
self.setup_py('install', *args)
def install_args(self, spec, prefix):
"""Arguments to pass to install."""
return ['--prefix={0}'.format(prefix)]
def install_lib(self, spec, prefix):
"""Install all Python modules (extensions and pure Python)."""
args = self.install_lib_args(spec, prefix)
self.setup_py('install_lib', *args)
def install_lib_args(self, spec, prefix):
"""Arguments to pass to install_lib."""
return []
def install_headers(self, spec, prefix):
"""Install C/C++ header files."""
args = self.install_headers_args(spec, prefix)
self.setup_py('install_headers', *args)
def install_headers_args(self, spec, prefix):
"""Arguments to pass to install_headers."""
return []
def install_scripts(self, spec, prefix):
"""Install scripts (Python or otherwise)."""
args = self.install_scripts_args(spec, prefix)
self.setup_py('install_scripts', *args)
def install_scripts_args(self, spec, prefix):
"""Arguments to pass to install_scripts."""
return []
def install_data(self, spec, prefix):
"""Install data files."""
args = self.install_data_args(spec, prefix)
self.setup_py('install_data', *args)
def install_data_args(self, spec, prefix):
"""Arguments to pass to install_data."""
return []
def sdist(self, spec, prefix):
"""Create a source distribution (tarball, zip file, etc.)."""
args = self.sdist_args(spec, prefix)
self.setup_py('sdist', *args)
def sdist_args(self, spec, prefix):
"""Arguments to pass to sdist."""
return []
def register(self, spec, prefix):
"""Register the distribution with the Python package index."""
args = self.register_args(spec, prefix)
self.setup_py('register', *args)
def register_args(self, spec, prefix):
"""Arguments to pass to register."""
return []
def bdist(self, spec, prefix):
"""Create a built (binary) distribution."""
args = self.bdist_args(spec, prefix)
self.setup_py('bdist', *args)
def bdist_args(self, spec, prefix):
"""Arguments to pass to bdist."""
return []
def bdist_dumb(self, spec, prefix):
'''Create a "dumb" built distribution.'''
args = self.bdist_dumb_args(spec, prefix)
self.setup_py('bdist_dumb', *args)
def bdist_dumb_args(self, spec, prefix):
"""Arguments to pass to bdist_dumb."""
return []
def bdist_rpm(self, spec, prefix):
"""Create an RPM distribution."""
args = self.bdist_rpm(spec, prefix)
self.setup_py('bdist_rpm', *args)
def bdist_rpm_args(self, spec, prefix):
"""Arguments to pass to bdist_rpm."""
return []
def bdist_wininst(self, spec, prefix):
"""Create an executable installer for MS Windows."""
args = self.bdist_wininst_args(spec, prefix)
self.setup_py('bdist_wininst', *args)
def bdist_wininst_args(self, spec, prefix):
"""Arguments to pass to bdist_wininst."""
return []
def upload(self, spec, prefix):
"""Upload binary package to PyPI."""
args = self.upload_args(spec, prefix)
self.setup_py('upload', *args)
def upload_args(self, spec, prefix):
"""Arguments to pass to upload."""
return []
def check(self, spec, prefix):
"""Perform some checks on the package."""
args = self.check_args(spec, prefix)
self.setup_py('check', *args)
def check_args(self, spec, prefix):
"""Arguments to pass to check."""
return []
# Check that self.prefix is there after installation
PackageBase.sanity_check('install')(PackageBase.sanity_check_prefix)
......@@ -30,7 +30,8 @@
build_system_to_phase = {
CMakePackage: 'build',
AutotoolsPackage: 'build'
AutotoolsPackage: 'build',
PythonPackage: 'build'
}
......
......@@ -194,19 +194,20 @@ def install(self, spec, prefix):
class PythonPackageTemplate(PackageTemplate):
"""Provides appropriate overrides for Python extensions"""
"""Provides appropriate overrides for python extensions"""
base_class_name = 'PythonPackage'
dependencies = """\
extends('python')
# FIXME: Add additional dependencies if required.
# FIXME: Add dependencies if required.
# depends_on('py-setuptools', type='build')
# depends_on('py-foo', type=('build', 'run'))"""
body = """\
def install(self, spec, prefix):
# FIXME: Add logic to build and install here.
setup_py('install', '--prefix={0}'.format(prefix))"""
def build_args(self):
# FIXME: Add arguments other than --prefix
# FIXME: If not needed delete the function
args = []
return args"""
def __init__(self, name, *args):
# If the user provided `--name py-numpy`, don't rename it py-py-numpy
......
......@@ -289,9 +289,10 @@ def extends(spec, **kwargs):
"""
def _execute(pkg):
if pkg.extendees:
msg = 'Packages can extend at most one other package.'
raise DirectiveError(msg)
# if pkg.extendees:
# directive = 'extends'
# msg = 'Packages can extend at most one other package.'
# raise DirectiveError(directive, msg)
when = kwargs.pop('when', pkg.name)
_depends_on(pkg, spec, when=when)
......@@ -344,8 +345,9 @@ def variant(name, default=False, description=""):
def _execute(pkg):
if not re.match(spack.spec.identifier_re, name):
msg = 'Invalid variant name in {0}: \'{1}\''
raise DirectiveError(msg.format(pkg.name, name))
directive = 'variant'
msg = "Invalid variant name in {0}: '{1}'"
raise DirectiveError(directive, msg.format(pkg.name, name))
pkg.variants[name] = Variant(default, description)
return _execute
......
......@@ -25,7 +25,7 @@
from spack import *
class Py3to2(Package):
class Py3to2(PythonPackage):
"""lib3to2 is a set of fixers that are intended to backport code written
for Python version 3.x into Python version 2.x."""
......@@ -33,8 +33,3 @@ class Py3to2(Package):
url = "https://pypi.python.org/packages/source/3/3to2/3to2-1.1.1.zip"
version('1.1.1', 'cbeed28e350dbdaef86111ace3052824')
extends('python')
def install(self, spec, prefix):
setup_py('install', '--prefix=%s' % prefix)
......@@ -25,7 +25,7 @@
from spack import *
class PyAlabaster(Package):
class PyAlabaster(PythonPackage):
"""Alabaster is a visually (c)lean, responsive, configurable theme
for the Sphinx documentation system."""
......@@ -35,9 +35,4 @@ class PyAlabaster(Package):
version('0.7.9', 'b29646a8bbe7aa52830375b7d17b5d7a',
url="https://pypi.python.org/packages/71/c3/70da7d8ac18a4f4c502887bd2549e05745fa403e2cd9d06a8a9910a762bc/alabaster-0.7.9.tar.gz")
extends('python')
depends_on('py-setuptools', type='build')
def install(self, spec, prefix):
setup_py('install', '--prefix={0}'.format(prefix))
......@@ -25,7 +25,7 @@
from spack import *
class PyArgcomplete(Package):
class PyArgcomplete(PythonPackage):
"""Bash tab completion for argparse."""
homepage = "https://pypi.python.org/pypi/argcomplete"
......@@ -34,7 +34,3 @@ class PyArgcomplete(Package):
version('1.1.1', '89a3839096c9f991ad33828e72d21abf')
depends_on('py-setuptools', type='build')
extends('python')
def install(self, spec, prefix):
setup_py('install', '--prefix=%s' % prefix)
......@@ -22,11 +22,10 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import depends_on, extends, version
from spack import Package
from spack import *
class PyAstroid(Package):
class PyAstroid(PythonPackage):
homepage = "https://www.astroid.org/"
url = "https://github.com/PyCQA/astroid/archive/astroid-1.4.5.tar.gz"
......@@ -36,10 +35,6 @@ class PyAstroid(Package):
version('1.4.2', '677f7965840f375af51b0e86403bee6a')
version('1.4.1', 'ed70bfed5e4b25be4292e7fe72da2c02')
extends('python')
depends_on('py-logilab-common', type=('build', 'run'))
depends_on('py-setuptools', type='build')
depends_on('py-six', type=('build', 'run'))
def install(self, spec, prefix):
setup_py('install', '--prefix=%s' % prefix)
......@@ -25,7 +25,7 @@
from spack import *
class PyAstropy(Package):
class PyAstropy(PythonPackage):
"""The Astropy Project is a community effort to develop a single core
package for Astronomy in Python and foster interoperability between
Python astronomy packages."""
......@@ -37,7 +37,6 @@ class PyAstropy(Package):
version('1.1.post1', 'b52919f657a37d45cc45f5cb0f58c44d')
# Required dependencies
extends('python')
depends_on('py-numpy', type=('build', 'run'))
# Optional dependencies
......@@ -55,6 +54,5 @@ class PyAstropy(Package):
depends_on('cfitsio')
depends_on('expat')
def install(self, spec, prefix):
setup_py('build', '--use-system-cfitsio', '--use-system-expat')
setup_py('install', '--prefix={0}'.format(prefix))
def build_args(self, spec, prefix):
return ['--use-system-cfitsio', '--use-system-expat']
......@@ -25,7 +25,7 @@
from spack import *
class PyAutopep8(Package):
class PyAutopep8(PythonPackage):
"""autopep8 automatically formats Python code to conform to the
PEP 8 style guide."""
......@@ -48,6 +48,3 @@ def url_for_version(self, version):
return url.format('v', version)
else:
return url.format('ver', version)
def install(self, spec, prefix):
setup_py('install', '--prefix={0}'.format(prefix))
......@@ -25,7 +25,7 @@
from spack import *
class PyBabel(Package):
class PyBabel(PythonPackage):
"""Babel is an integrated collection of utilities that assist in
internationalizing and localizing Python applications, with an
emphasis on web-based applications."""
......@@ -36,10 +36,5 @@ class PyBabel(Package):
version('2.3.4', 'afa20bc55b0e991833030129ad498f35',
url="https://pypi.python.org/packages/6e/96/ba2a2462ed25ca0e651fb7b66e7080f5315f91425a07ea5b34d7c870c114/Babel-2.3.4.tar.gz")
extends('python')
depends_on('py-setuptools', type='build')
depends_on('py-pytz', type=('build', 'run'))
def install(self, spec, prefix):
setup_py('install', '--prefix={0}'.format(prefix))
......@@ -25,17 +25,12 @@
from spack import *
class PyBackportsAbc(Package):
class PyBackportsAbc(PythonPackage):
"""Backports_ABC: A backport of recent additions to the 'collections.abc'
module."""
homepage = "https://github.com/cython/backports_abc"
# base https://pypi.python.org/pypi/backports_abc/
url = "https://github.com/cython/backports_abc/archive/0.4.tar.gz"
version('0.4', 'e4246ae689221c9cbe84369fdb59e8c74d02b298')
extends('python')
depends_on('py-setuptools', type='build')
def install(self, spec, prefix):
setup_py('install', '--prefix={0}'.format(prefix))
......@@ -25,8 +25,8 @@
from spack import *
class PyBackportsShutilGetTerminalSize(Package):
"""A backport of the get_terminal_size function
class PyBackportsShutilGetTerminalSize(PythonPackage):
"""A backport of the get_terminal_size function
from Python 3.3's shutil."""
homepage = "https://pypi.python.org/pypi/backports.shutil_get_terminal_size"
......@@ -34,10 +34,5 @@ class PyBackportsShutilGetTerminalSize(Package):
version('1.0.0', '03267762480bd86b50580dc19dff3c66')
extends('python')
depends_on('py-setuptools', type='build')
depends_on('python@:3.2.999')
def install(self, spec, prefix):
setup_py('install', '--prefix={0}'.format(prefix))
......@@ -25,7 +25,7 @@
from spack import *
class PyBackportsSslMatchHostname(Package):
class PyBackportsSslMatchHostname(PythonPackage):
"""The ssl.match_hostname() function from Python 3.5"""
homepage = "https://pypi.python.org/pypi/backports.ssl_match_hostname"
......@@ -33,9 +33,4 @@ class PyBackportsSslMatchHostname(Package):
version('3.5.0.1', 'c03fc5e2c7b3da46b81acf5cbacfe1e6')
extends('python')
depends_on('py-setuptools', type='build')
def install(self, spec, prefix):
setup_py('install', '--prefix={0}'.format(prefix))
......@@ -26,7 +26,7 @@
import os
class PyBasemap(Package):
class PyBasemap(PythonPackage):
"""The matplotlib basemap toolkit is a library for plotting
2D data on maps in Python."""
......@@ -35,17 +35,18 @@ class PyBasemap(Package):
version('1.0.7', '48c0557ced9e2c6e440b28b3caff2de8')
extends('python')
depends_on('py-setuptools', type='build')
depends_on('py-numpy', type=('build', 'run'))
depends_on('py-matplotlib', type=('build', 'run'))
depends_on('pil', type=('build', 'run'))
depends_on("geos")
depends_on('geos')
def install(self, spec, prefix):
env['GEOS_DIR'] = spec['geos'].prefix
setup_py('install', '--prefix=%s' % prefix)
def setup_environment(self, spack_env, run_env):
spack_env.set('GEOS_DIR', self.spec['geos'].prefix)
@PythonPackage.sanity_check('install')
def post_install_patch(self):
spec = self.spec
# We are not sure if this fix is needed before Python 3.5.2.
# If it is needed, this test should be changed.
# See: https://github.com/LLNL/spack/pull/1964
......
......@@ -25,7 +25,7 @@
from spack import *
class PyBeautifulsoup4(Package):
class PyBeautifulsoup4(PythonPackage):
"""Beautiful Soup is a Python library for pulling data out of HTML and
XML files. It works with your favorite parser to provide idiomatic ways
of navigating, searching, and modifying the parse tree."""
......@@ -38,9 +38,4 @@ class PyBeautifulsoup4(Package):
'download/4.5/beautifulsoup4-4.5.1.tar.gz')
version('4.4.1', '8fbd9a7cac0704645fa20d1419036815')
extends('python')
depends_on('py-setuptools', type='build')
def install(self, spec, prefix):
setup_py('install', '--prefix={0}'.format(prefix))
......@@ -25,7 +25,7 @@
from spack import *
class PyBiopython(Package):
class PyBiopython(PythonPackage):
"""A distributed collaborative effort to develop Python libraries and
applications which address the needs of current and future work in
bioinformatics.
......@@ -36,9 +36,5 @@ class PyBiopython(Package):
version('1.65', '143e7861ade85c0a8b5e2bbdd1da1f67')
extends('python')
depends_on('py-mx', type=('build', 'run'))
depends_on('py-numpy', type=('build', 'run'))
def install(self, spec, prefix):
setup_py('install', '--prefix=%s' % prefix)
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
class PyBleach(PythonPackage):
"""An easy whitelist-based HTML-sanitizing tool."""
homepage = "http://github.com/mozilla/bleach"
url = "https://pypi.python.org/packages/99/00/25a8fce4de102bf6e3cc76bc4ea60685b2fee33bde1b34830c70cacc26a7/bleach-1.5.0.tar.gz"
version('1.5.0', 'b663300efdf421b3b727b19d7be9c7e7')
depends_on('python@2.6:2.7,3.2:3.5')
depends_on('py-setuptools', type='build')
depends_on('py-six', type=('build', 'run'))
depends_on('py-html5lib@0.999,0.999999:0.9999999', type=('build', 'run'))
......@@ -25,7 +25,7 @@
from spack import *
class PyBlessings(Package):
class PyBlessings(PythonPackage):
"""A nicer, kinder way to write to the terminal """
homepage = "https://github.com/erikrose/blessings"
url = "https://pypi.python.org/packages/source/b/blessings/blessings-1.6.tar.gz"
......@@ -33,8 +33,3 @@ class PyBlessings(Package):
version('1.6', '4f552a8ebcd4982693c92571beb99394')
depends_on('py-setuptools', type='build')
extends("python")
def install(self, spec, prefix):
setup_py('install', '--prefix=%s' % prefix)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment