relicense: add `spack license` command

- `spack license list-files`: list all files that should have license headers
- `spack license list-lgpl`:  list files still under LGPL-2.1
- `spack license verify`:     check that license headers are correct

- Added `spack license verify` to style tests

lib/spack/spack/cmd/license.py

+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from __future__ import print_function
+
+import os
+import re
+
+import llnl.util.tty as tty
+
+import spack.paths
+from spack.util.executable import which
+
+description = 'list and check license headers on files in spack'
+section = "developer"
+level = "long"
+
+#: need the git command to check new files
+git = which('git')
+
+#: SPDX license id must appear in the first <license_lines> lines of a file
+license_lines = 6
+
+#: Spack's license identifier
+apache2_mit_spdx = "(Apache-2.0 OR MIT)"
+
+#: regular expressions for licensed files.
+licensed_files = [
+    # spack scripts
+    r'^bin/spack$',
+    r'^bin/spack-python$',
+    r'^bin/sbang$',
+
+    # all of spack core
+    r'^lib/spack/spack/.*\.py$',
+    r'^lib/spack/spack/.*\.sh$',
+    r'^lib/spack/llnl/.*\.py$',
+    r'^lib/spack/env/cc$',
+
+    # rst files in documentation
+    r'^lib/spack/docs/.*\.rst$',
+    r'^lib/spack/docs/.*\.py$',
+
+    # 2 files in external
+    r'^lib/spack/external/__init__.py$',
+    r'^lib/spack/external/ordereddict_backport.py$',
+
+    # shell scripts in share
+    r'^share/spack/.*\.sh$',
+    r'^share/spack/.*\.bash$',
+    r'^share/spack/.*\.csh$',
+    r'^share/spack/qa/run-[^/]*$',
+
+    # all packages
+    r'^var/spack/repos/.*/package.py$'
+]
+
+#: licensed files that can have LGPL language in them
+#: so far, just this command -- so it can find LGPL things elsewhere
+lgpl_exceptions = [
+    r'lib/spack/spack/cmd/license.py',
+    r'lib/spack/spack/test/cmd/license.py',
+]
+
+
+def _all_spack_files(root=spack.paths.prefix):
+    """Generates root-relative paths of all files in the spack repository."""
+    for cur_root, folders, files in os.walk(root):
+        for filename in files:
+            path = os.path.join(cur_root, filename)
+            yield os.path.relpath(path, root)
+
+
+def _licensed_files(root=spack.paths.prefix):
+    for relpath in _all_spack_files(root):
+        if any(regex.match(relpath) for regex in licensed_files):
+            yield relpath
+
+
+def list_files(args):
+    """list files in spack that should have license headers"""
+    for relpath in _licensed_files():
+        print(os.path.join(spack.paths.spack_root, relpath))
+
+
+def verify(args):
+    """verify that files in spack have the right license header"""
+    errors = 0
+    missing = 0
+    old_license = 0
+
+    for relpath in _licensed_files(args.root):
+        path = os.path.join(args.root, relpath)
+        with open(path) as f:
+            lines = [line for line in f]
+
+        if not any(re.match(regex, relpath) for regex in lgpl_exceptions):
+            if any(re.match(r'^# This program is free software', line)
+                   for line in lines):
+                print('%s: has old LGPL license header' % path)
+                old_license += 1
+                continue
+
+        # how we'll find licenses in files
+        spdx_expr = r'SPDX-License-Identifier: ([^\n]*)'
+
+        # check first <license_lines> lines for required header
+        first_n_lines = ''.join(lines[:license_lines])
+        match = re.search(spdx_expr, first_n_lines)
+
+        if not match:
+            print('%s: no license header' % path)
+            missing += 1
+            continue
+
+        correct = apache2_mit_spdx
+        actual = match.group(1)
+        if actual != correct:
+            print("%s: labeled as '%s', but should be '%s'"
+                  % (path, actual, correct))
+            errors += 1
+            continue
+
+    if any([errors, missing, old_license]):
+        tty.die(
+            '%d improperly licensed files' % (errors + missing + old_license),
+            'files with no SPDX-License-Identifier:      %d' % missing,
+            'files with wrong SPDX-License-Identifier:   %d' % errors,
+            'files with old license header:              %d' % old_license)
+    else:
+        tty.msg('No license issues found.')
+
+
+def setup_parser(subparser):
+    sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='license_command')
+    sp.add_parser('list-files', help=list_files.__doc__)
+
+    verify_parser = sp.add_parser('verify', help=verify.__doc__)
+    verify_parser.add_argument(
+        '--root', action='store', default=spack.paths.prefix,
+        help='scan a different prefix for license issues')
+
+
+def license(parser, args):
+    if not git:
+        tty.die('spack license requires git in your environment')
+
+    licensed_files[:] = [re.compile(regex) for regex in licensed_files]
+
+    commands = {
+        'list-files': list_files,
+        'verify': verify,
+    }
+    return commands[args.license_command](args)

lib/spack/spack/test/cmd/license.py

+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os.path
+import re
+
+from llnl.util.filesystem import touch, mkdirp
+
+import spack.paths
+from spack.main import SpackCommand
+
+license = SpackCommand('license')
+
+
+def test_list_files():
+    files = license('list-files').strip().split('\n')
+    assert all(f.startswith(spack.paths.prefix) for f in files)
+    assert os.path.join(spack.paths.bin_path, 'spack') in files
+    assert os.path.abspath(__file__) in files
+
+
+def test_verify(tmpdir):
+    source_dir = tmpdir.join('lib', 'spack', 'spack')
+    mkdirp(str(source_dir))
+
+    no_header = source_dir.join('no_header.py')
+    touch(str(no_header))
+
+    lgpl_header = source_dir.join('lgpl_header.py')
+    with lgpl_header.open('w') as f:
+        f.write("""\
+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: LGPL-2.1-only
+""")
+
+    old_lgpl_header = source_dir.join('old_lgpl_header.py')
+    with old_lgpl_header.open('w') as f:
+        f.write("""\
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+""")
+
+    correct_header = source_dir.join('correct_header.py')
+    with correct_header.open('w') as f:
+        f.write("""\
+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+""")
+
+    out = license('verify', '--root', str(tmpdir), fail_on_error=False)
+
+    assert str(no_header) in out
+    assert str(lgpl_header) in out
+    assert str(old_lgpl_header) in out
+    assert str(correct_header) not in out
+    assert '3 improperly licensed files' in out
+    assert re.search('files with no SPDX-License-Identifier:\s*1', out)
+    assert re.search('files with wrong SPDX-License-Identifier:\s*1', out)
+    assert re.search('files with old license header:\s*1', out)
+
+    assert license.returncode == 1

share/spack/qa/run-flake8-tests

@@ -17,4 +17,8 @@
 . "$(dirname $0)/setup.sh"
 check_dependencies flake8
 
+# verify that the code style is correct
 spack flake8
+
+# verify that the license headers are present
+spack license verify