Merge pull request #977 from trws/explicit-docs

initial docs for find and flake8 cleanup

lib/spack/docs/basic_usage.rst

@@ -246,6 +246,12 @@ Packages are divided into groups according to their architecture and
 compiler.  Within each group, Spack tries to keep the view simple, and
 only shows the version of installed packages.
 
+``spack find`` can filter the package list based on the package name, spec, or
+a number of properties of their installation status.  For example, missing
+dependencies of a spec can be shown with ``-m``, packages which were
+explicitly installed with ``spack install <package>`` can be singled out with
+``-e`` and those which have been pulled in only as dependencies with ``-E``.
+
 In some cases, there may be different configurations of the *same*
 version of a package installed.  For example, there are two
 installations of of ``libdwarf@20130729`` above.  We can look at them

lib/spack/spack/cmd/find.py

@@ -37,71 +37,59 @@
 
 def setup_parser(subparser):
     format_group = subparser.add_mutually_exclusive_group()
-    format_group.add_argument('-s',
-                              '--short',
+    format_group.add_argument('-s', '--short',
                               action='store_const',
                               dest='mode',
                               const='short',
                               help='Show only specs (default)')
-    format_group.add_argument('-p',
-                              '--paths',
+    format_group.add_argument('-p', '--paths',
                               action='store_const',
                               dest='mode',
                               const='paths',
                               help='Show paths to package install directories')
     format_group.add_argument(
-        '-d',
-        '--deps',
+        '-d', '--deps',
         action='store_const',
         dest='mode',
         const='deps',
         help='Show full dependency DAG of installed packages')
 
-    subparser.add_argument('-l',
-                           '--long',
+    subparser.add_argument('-l', '--long',
                            action='store_true',
                            dest='long',
                            help='Show dependency hashes as well as versions.')
-    subparser.add_argument('-L',
-                           '--very-long',
+    subparser.add_argument('-L', '--very-long',
                            action='store_true',
                            dest='very_long',
                            help='Show dependency hashes as well as versions.')
-    subparser.add_argument('-f',
-                           '--show-flags',
+    subparser.add_argument('-f', '--show-flags',
                            action='store_true',
                            dest='show_flags',
                            help='Show spec compiler flags.')
 
     subparser.add_argument(
-        '-e',
-        '--explicit',
+        '-e', '--explicit',
         action='store_true',
         help='Show only specs that were installed explicitly')
     subparser.add_argument(
-        '-E',
-        '--implicit',
+        '-E', '--implicit',
         action='store_true',
         help='Show only specs that were installed as dependencies')
     subparser.add_argument(
-        '-u',
-        '--unknown',
+        '-u', '--unknown',
         action='store_true',
         dest='unknown',
         help='Show only specs Spack does not have a package for.')
     subparser.add_argument(
-        '-m',
-        '--missing',
+        '-m', '--missing',
         action='store_true',
         dest='missing',
         help='Show missing dependencies as well as installed specs.')
-    subparser.add_argument('-M',
-                           '--only-missing',
+    subparser.add_argument('-M', '--only-missing',
                            action='store_true',
                            dest='only_missing',
                            help='Show only missing dependencies.')
-    subparser.add_argument('-N',
-                           '--namespace',
+    subparser.add_argument('-N', '--namespace',
                            action='store_true',
                            help='Show fully qualified package names.')
 
@@ -187,7 +175,9 @@ def fmt(s):
                     print(hsh + spec.format(format_string, color=True) + '\n')
 
         else:
-            raise ValueError("Invalid mode for display_specs: %s. Must be one of (paths, deps, short)." % mode)  # NOQA: ignore=E501
+            raise ValueError(
+                "Invalid mode for display_specs: %s. Must be one of (paths,"
+                "deps, short)." % mode)  # NOQA: ignore=E501
 
 
 def find(parser, args):

lib/spack/spack/database.py

@@ -40,7 +40,6 @@
 
 """
 import os
-import time
 import socket
 
 import yaml
@@ -56,6 +55,7 @@
 from spack.error import SpackError
 from spack.repository import UnknownPackageError
 
+
 # DB goes in this directory underneath the root
 _db_dirname = '.spack-db'
 
@@ -69,10 +69,12 @@
 def _autospec(function):
     """Decorator that automatically converts the argument of a single-arg
        function to a Spec."""
+
     def converter(self, spec_like, *args, **kwargs):
         if not isinstance(spec_like, spack.spec.Spec):
             spec_like = spack.spec.Spec(spec_like)
         return function(self, spec_like, *args, **kwargs)
+
     return converter
 
 
@@ -92,6 +94,7 @@ class InstallRecord(object):
     dependents left.
 
     """
+
     def __init__(self, spec, path, installed, ref_count=0, explicit=False):
         self.spec = spec
         self.path = str(path)
@@ -100,16 +103,19 @@ def __init__(self, spec, path, installed, ref_count=0, explicit=False):
         self.explicit = explicit
 
     def to_dict(self):
-        return { 'spec'      : self.spec.to_node_dict(),
-                 'path'      : self.path,
-                 'installed' : self.installed,
-                 'ref_count' : self.ref_count,
-                 'explicit'  : self.explicit }
+        return {
+            'spec': self.spec.to_node_dict(),
+            'path': self.path,
+            'installed': self.installed,
+            'ref_count': self.ref_count,
+            'explicit': self.explicit
+        }
 
     @classmethod
     def from_dict(cls, spec, dictionary):
         d = dictionary
-        return InstallRecord(spec, d['path'], d['installed'], d['ref_count'], d.get('explicit', False))
+        return InstallRecord(spec, d['path'], d['installed'], d['ref_count'],
+                             d.get('explicit', False))
 
 
 class Database(object):
@@ -144,7 +150,7 @@ def __init__(self, root, db_dir=None):
 
         # Set up layout of database files within the db dir
         self._index_path = join_path(self._db_dir, 'index.yaml')
-        self._lock_path  = join_path(self._db_dir, 'lock')
+        self._lock_path = join_path(self._db_dir, 'lock')
 
         # Create needed directories and files
         if not os.path.exists(self._db_dir):
@@ -157,17 +163,14 @@ def __init__(self, root, db_dir=None):
         self.lock = Lock(self._lock_path)
         self._data = {}
 
-
     def write_transaction(self, timeout=_db_lock_timeout):
         """Get a write lock context manager for use in a `with` block."""
         return WriteTransaction(self, self._read, self._write, timeout)
 
-
     def read_transaction(self, timeout=_db_lock_timeout):
         """Get a read lock context manager for use in a `with` block."""
         return ReadTransaction(self, self._read, None, timeout)
 
-
     def _write_to_yaml(self, stream):
         """Write out the databsae to a YAML file.
 
@@ -183,9 +186,9 @@ def _write_to_yaml(self, stream):
         # different paths, it can't differentiate.
         # TODO: fix this before we support multiple install locations.
         database = {
-            'database' : {
-                'installs' : installs,
-                'version' : str(_db_version)
+            'database': {
+                'installs': installs,
+                'version': str(_db_version)
             }
         }
 
@@ -194,15 +197,11 @@ def _write_to_yaml(self, stream):
         except YAMLError as e:
             raise SpackYAMLError("error writing YAML database:", str(e))
 
-
     def _read_spec_from_yaml(self, hash_key, installs, parent_key=None):
         """Recursively construct a spec from a hash in a YAML database.
 
         Does not do any locking.
         """
-        if hash_key not in installs:
-            parent = read_spec(installs[parent_key]['path'])
-
         spec_dict = installs[hash_key]['spec']
 
         # Install records don't include hash with spec, so we add it in here
@@ -224,7 +223,6 @@ def _read_spec_from_yaml(self, hash_key, installs, parent_key=None):
         spec._mark_concrete()
         return spec
 
-
     def _read_from_yaml(self, stream):
         """
         Fill database from YAML, do not maintain old data
@@ -246,15 +244,15 @@ def _read_from_yaml(self, stream):
             return
 
         def check(cond, msg):
-            if not cond: raise CorruptDatabaseError(self._index_path, msg)
+            if not cond:
+                raise CorruptDatabaseError(self._index_path, msg)
 
         check('database' in yfile, "No 'database' attribute in YAML.")
 
         # High-level file checks
         db = yfile['database']
         check('installs' in db, "No 'installs' in YAML DB.")
-        check('version'  in db, "No 'version' in YAML DB.")
-
+        check('version' in db, "No 'version' in YAML DB.")
 
         installs = db['installs']
 
@@ -277,25 +275,25 @@ def check(cond, msg):
                 # hashes are the same.
                 spec_hash = spec.dag_hash()
                 if not spec_hash == hash_key:
-                    tty.warn("Hash mismatch in database: %s -> spec with hash %s"
-                             % (hash_key, spec_hash))
-                    continue    # TODO: is skipping the right thing to do?
+                    tty.warn(
+                        "Hash mismatch in database: %s -> spec with hash %s" %
+                        (hash_key, spec_hash))
+                    continue  # TODO: is skipping the right thing to do?
 
                 # Insert the brand new spec in the database.  Each
                 # spec has its own copies of its dependency specs.
-                # TODO: would a more immmutable spec implementation simplify this?
+                # TODO: would a more immmutable spec implementation simplify
+                #       this?
                 data[hash_key] = InstallRecord.from_dict(spec, rec)
 
             except Exception as e:
                 tty.warn("Invalid database reecord:",
                          "file:  %s" % self._index_path,
-                         "hash:  %s" % hash_key,
-                         "cause: %s" % str(e))
+                         "hash:  %s" % hash_key, "cause: %s" % str(e))
                 raise
 
         self._data = data
 
-
     def reindex(self, directory_layout):
         """Build database index from scratch based from a directory layout.
 
@@ -320,7 +318,6 @@ def reindex(self, directory_layout):
                 self._data = old_data
                 raise
 
-
     def _check_ref_counts(self):
         """Ensure consistency of reference counts in the DB.
 
@@ -342,9 +339,8 @@ def _check_ref_counts(self):
             found = rec.ref_count
             if not expected == found:
                 raise AssertionError(
-                    "Invalid ref_count: %s: %d (expected %d), in DB %s"
-                    % (key, found, expected, self._index_path))
-
+                    "Invalid ref_count: %s: %d (expected %d), in DB %s" %
+                    (key, found, expected, self._index_path))
 
     def _write(self):
         """Write the in-memory database index to its file path.
@@ -366,7 +362,6 @@ def _write(self):
                 os.remove(temp_file)
             raise
 
-
     def _read(self):
         """Re-read Database from the data in the set location.
 
@@ -381,7 +376,6 @@ def _read(self):
             # reindex() takes its own write lock, so no lock here.
             self.reindex(spack.install_layout)
 
-
     def _add(self, spec, path, directory_layout=None, explicit=False):
         """Add an install record for spec at path to the database.
 
@@ -404,11 +398,11 @@ def _add(self, spec, path, directory_layout=None, explicit=False):
             rec.path = path
 
         else:
-            self._data[key] = InstallRecord(spec, path, True, explicit=explicit)
+            self._data[key] = InstallRecord(spec, path, True,
+                                            explicit=explicit)
             for dep in spec.dependencies.values():
                 self._increment_ref_count(dep, directory_layout)
 
-
     def _increment_ref_count(self, spec, directory_layout=None):
         """Recursively examine dependencies and update their DB entries."""
         key = spec.dag_hash()
@@ -438,28 +432,25 @@ def add(self, spec, path, explicit=False):
         with self.write_transaction():
             self._add(spec, path, explicit=explicit)
 
-
     def _get_matching_spec_key(self, spec, **kwargs):
         """Get the exact spec OR get a single spec that matches."""
         key = spec.dag_hash()
-        if not key in self._data:
+        if key not in self._data:
             match = self.query_one(spec, **kwargs)
             if match:
                 return match.dag_hash()
             raise KeyError("No such spec in database! %s" % spec)
         return key
 
-
     @_autospec
     def get_record(self, spec, **kwargs):
         key = self._get_matching_spec_key(spec, **kwargs)
         return self._data[key]
 
-
     def _decrement_ref_count(self, spec):
         key = spec.dag_hash()
 
-        if not key in self._data:
+        if key not in self._data:
             # TODO: print something here?  DB is corrupt, but
             # not much we can do.
             return
@@ -472,7 +463,6 @@ def _decrement_ref_count(self, spec):
             for dep in spec.dependencies.values():
                 self._decrement_ref_count(dep)
 
-
     def _remove(self, spec):
         """Non-locking version of remove(); does real work.
         """
@@ -491,7 +481,6 @@ def _remove(self, spec):
         # query spec was passed in.
         return rec.spec
 
-
     @_autospec
     def remove(self, spec):
         """Removes a spec from the database.  To be called on uninstall.
@@ -508,7 +497,6 @@ def remove(self, spec):
         with self.write_transaction():
             return self._remove(spec)
 
-
     @_autospec
     def installed_extensions_for(self, extendee_spec):
         """
@@ -519,12 +507,11 @@ def installed_extensions_for(self, extendee_spec):
             try:
                 if s.package.extends(extendee_spec):
                     yield s.package
-            except UnknownPackageError as e:
+            except UnknownPackageError:
                 continue
             # skips unknown packages
             # TODO: conditional way to do this instead of catching exceptions
 
-
     def query(self, query_spec=any, known=any, installed=True, explicit=any):
         """Run a query on the database.
 
@@ -567,14 +554,14 @@ def query(self, query_spec=any, known=any, installed=True, explicit=any):
                     continue
                 if explicit is not any and rec.explicit != explicit:
                     continue
-                if known is not any and spack.repo.exists(rec.spec.name) != known:
+                if known is not any and spack.repo.exists(
+                        rec.spec.name) != known:
                     continue
                 if query_spec is any or rec.spec.satisfies(query_spec):
                     results.append(rec.spec)
 
             return sorted(results)
 
-
     def query_one(self, query_spec, known=any, installed=True):
         """Query for exactly one spec that matches the query spec.
 
@@ -586,10 +573,9 @@ def query_one(self, query_spec, known=any, installed=True):
         assert len(concrete_specs) <= 1
         return concrete_specs[0] if concrete_specs else None
 
-
     def missing(self, spec):
         with self.read_transaction():
-            key =  spec.dag_hash()
+            key = spec.dag_hash()
             return key in self._data and not self._data[key].installed
 
 
@@ -601,7 +587,10 @@ class _Transaction(object):
 
     Timeout for lock is customizable.
     """
-    def __init__(self, db, acquire_fn=None, release_fn=None,
+
+    def __init__(self, db,
+                 acquire_fn=None,
+                 release_fn=None,
                  timeout=_db_lock_timeout):
         self._db = db
         self._timeout = timeout
@@ -636,11 +625,11 @@ def _exit(self):
 class CorruptDatabaseError(SpackError):
     def __init__(self, path, msg=''):
         super(CorruptDatabaseError, self).__init__(
-            "Spack database is corrupt: %s.  %s" %(path, msg))
+            "Spack database is corrupt: %s.  %s" % (path, msg))
 
 
 class InvalidDatabaseVersionError(SpackError):
     def __init__(self, expected, found):
         super(InvalidDatabaseVersionError, self).__init__(
-            "Expected database version %s but found version %s"
-            % (expected, found))
+            "Expected database version %s but found version %s" %
+            (expected, found))

var/spack/repos/builtin/packages/py-autopep8/package.py

+from spack import *
+
+class PyAutopep8(Package):
+    """Automatic pep8 formatter"""
+    homepage = "https://github.com/hhatto/autopep8"
+    url      = "https://github.com/hhatto/autopep8/archive/ver1.2.2.tar.gz"
+
+    version('1.2.2', 'def3d023fc9dfd1b7113602e965ad8e1')
+
+    extends('python')
+    depends_on('py-setuptools')
+    depends_on('py-pep8')
+
+    def install(self, spec, prefix):
+        python('setup.py', 'install', '--prefix=%s' % prefix)
+

var/spack/repos/builtin/packages/py-pep8/package.py

+from spack import *
+
+class PyPep8(Package):
+    """python pep8 format checker"""
+    homepage = "https://github.com/PyCQA/pycodestyle"
+    url      = "https://github.com/PyCQA/pycodestyle/archive/1.7.0.tar.gz"
+
+    version('1.7.0', '31070a3a6391928893cbf5fa523eb8d9')
+
+    extends('python')
+    depends_on('py-setuptools')
+
+    def install(self, spec, prefix):
+        python('setup.py', 'install', '--prefix=%s' % prefix)
+