From 24775697f5b8b38bea407fd5b2594e483f39d150 Mon Sep 17 00:00:00 2001
From: Peter Scheibel <scheibel1@llnl.gov>
Date: Wed, 3 Jun 2020 17:43:51 -0700
Subject: [PATCH] Mirrors: add option to exclude packages from "mirror create"
(#14154)
* add an --exclude-file option to 'spack mirror create' which allows a user to specify a file of specs to exclude when creating a mirror. this is anticipated to be useful especially when using the '--all' option
* allow specifying number of versions when mirroring all packages
* when mirroring all specs within an environment, include dependencies of root specs
* add '--exclude-specs' option to allow user to specify that specs should be excluded on the command line
* add test for excluding specs
---
lib/spack/spack/cmd/mirror.py | 46 ++++++++++++++++++++++-----
lib/spack/spack/environment.py | 25 ++++++---------
lib/spack/spack/test/cmd/mirror.py | 50 ++++++++++++++++++++++++++++++
share/spack/spack-completion.bash | 2 +-
4 files changed, 99 insertions(+), 24 deletions(-)
diff --git a/lib/spack/spack/cmd/mirror.py b/lib/spack/spack/cmd/mirror.py
index 1473550a56..2d338204d3 100644
--- a/lib/spack/spack/cmd/mirror.py
+++ b/lib/spack/spack/cmd/mirror.py
@@ -45,6 +45,15 @@ def setup_parser(subparser):
" (this requires significant time and space)")
create_parser.add_argument(
'-f', '--file', help="file with specs of packages to put in mirror")
+ create_parser.add_argument(
+ '--exclude-file',
+ help="specs which Spack should not try to add to a mirror"
+ " (listed in a file, one per line)")
+ create_parser.add_argument(
+ '--exclude-specs',
+ help="specs which Spack should not try to add to a mirror"
+ " (specified on command line)")
+
create_parser.add_argument(
'--skip-unstable-versions', action='store_true',
help="don't cache versions unless they identify a stable (unchanging)"
@@ -232,9 +241,7 @@ def _read_specs_from_file(filename):
return specs
-def mirror_create(args):
- """Create a directory to be used as a spack mirror, and fill it with
- package archives."""
+def _determine_specs_to_mirror(args):
if args.specs and args.all:
raise SpackError("Cannot specify specs on command line if you"
" chose to mirror all specs with '--all'")
@@ -264,6 +271,7 @@ def mirror_create(args):
tty.die("Cannot pass specs on the command line with --file.")
specs = _read_specs_from_file(args.file)
+ env_specs = None
if not specs:
# If nothing is passed, use environment or all if no active env
if not args.all:
@@ -273,12 +281,9 @@ def mirror_create(args):
env = ev.get_env(args, 'mirror')
if env:
- mirror_specs = env.specs_by_hash.values()
+ env_specs = env.all_specs()
else:
specs = [Spec(n) for n in spack.repo.all_package_names()]
- mirror_specs = spack.mirror.get_all_versions(specs)
- mirror_specs.sort(
- key=lambda s: (s.name, s.version))
else:
# If the user asked for dependencies, traverse spec DAG get them.
if args.dependencies:
@@ -297,11 +302,38 @@ def mirror_create(args):
msg = 'Skipping {0} as it is an external spec.'
tty.msg(msg.format(spec.cshort_spec))
+ if env_specs:
+ if args.versions_per_spec:
+ tty.warn("Ignoring '--versions-per-spec' for mirroring specs"
+ " in environment.")
+ mirror_specs = env_specs
+ else:
if num_versions == 'all':
mirror_specs = spack.mirror.get_all_versions(specs)
else:
mirror_specs = spack.mirror.get_matching_versions(
specs, num_versions=num_versions)
+ mirror_specs.sort(
+ key=lambda s: (s.name, s.version))
+
+ exclude_specs = []
+ if args.exclude_file:
+ exclude_specs.extend(_read_specs_from_file(args.exclude_file))
+ if args.exclude_specs:
+ exclude_specs.extend(
+ spack.cmd.parse_specs(str(args.exclude_specs).split()))
+ if exclude_specs:
+ mirror_specs = list(
+ x for x in mirror_specs
+ if not any(x.satisfies(y, strict=True) for y in exclude_specs))
+
+ return mirror_specs
+
+
+def mirror_create(args):
+ """Create a directory to be used as a spack mirror, and fill it with
+ package archives."""
+ mirror_specs = _determine_specs_to_mirror(args)
mirror = spack.mirror.Mirror(
args.directory or spack.config.get('config:source_cache'))
diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py
index 8d3b1438e1..00af2df2d9 100644
--- a/lib/spack/spack/environment.py
+++ b/lib/spack/spack/environment.py
@@ -1232,26 +1232,19 @@ def install_all(self, args=None):
self._install(spec, **kwargs)
- def all_specs_by_hash(self):
- """Map of hashes to spec for all specs in this environment."""
- # Note this uses dag-hashes calculated without build deps as keys,
- # whereas the environment tracks specs based on dag-hashes calculated
- # with all dependencies. This function should not be used by an
- # Environment object for management of its own data structures
- hashes = {}
- for h in self.concretized_order:
- specs = self.specs_by_hash[h].traverse(deptype=('link', 'run'))
- for spec in specs:
- hashes[spec.dag_hash()] = spec
- return hashes
-
def all_specs(self):
"""Return all specs, even those a user spec would shadow."""
- return sorted(self.all_specs_by_hash().values())
+ all_specs = set()
+ for h in self.concretized_order:
+ all_specs.update(self.specs_by_hash[h].traverse())
+
+ return sorted(all_specs)
def all_hashes(self):
- """Return all specs, even those a user spec would shadow."""
- return list(self.all_specs_by_hash().keys())
+ """Return hashes of all specs.
+
+ Note these hashes exclude build dependencies."""
+ return list(set(s.dag_hash() for s in self.all_specs()))
def roots(self):
"""Specs explicitly requested by the user *in this environment*.
diff --git a/lib/spack/spack/test/cmd/mirror.py b/lib/spack/spack/test/cmd/mirror.py
index 4bb4fad224..f6fe0b24dd 100644
--- a/lib/spack/spack/test/cmd/mirror.py
+++ b/lib/spack/spack/test/cmd/mirror.py
@@ -89,6 +89,56 @@ def test_mirror_skip_unstable(tmpdir_factory, mock_packages, config,
set(['trivial-pkg-with-valid-hash']))
+class MockMirrorArgs(object):
+ def __init__(self, specs=None, all=False, file=None,
+ versions_per_spec=None, dependencies=False,
+ exclude_file=None, exclude_specs=None):
+ self.specs = specs or []
+ self.all = all
+ self.file = file
+ self.versions_per_spec = versions_per_spec
+ self.dependencies = dependencies
+ self.exclude_file = exclude_file
+ self.exclude_specs = exclude_specs
+
+
+def test_exclude_specs(mock_packages):
+ args = MockMirrorArgs(
+ specs=['mpich'],
+ versions_per_spec='all',
+ exclude_specs="mpich@3.0.1:3.0.2 mpich@1.0")
+
+ mirror_specs = spack.cmd.mirror._determine_specs_to_mirror(args)
+ expected_include = set(spack.spec.Spec(x) for x in
+ ['mpich@3.0.3', 'mpich@3.0.4', 'mpich@3.0'])
+ expected_exclude = set(spack.spec.Spec(x) for x in
+ ['mpich@3.0.1', 'mpich@3.0.2', 'mpich@1.0'])
+ assert expected_include <= set(mirror_specs)
+ assert (not expected_exclude & set(mirror_specs))
+
+
+def test_exclude_file(mock_packages, tmpdir):
+ exclude_path = os.path.join(str(tmpdir), 'test-exclude.txt')
+ with open(exclude_path, 'w') as exclude_file:
+ exclude_file.write("""\
+mpich@3.0.1:3.0.2
+mpich@1.0
+""")
+
+ args = MockMirrorArgs(
+ specs=['mpich'],
+ versions_per_spec='all',
+ exclude_file=exclude_path)
+
+ mirror_specs = spack.cmd.mirror._determine_specs_to_mirror(args)
+ expected_include = set(spack.spec.Spec(x) for x in
+ ['mpich@3.0.3', 'mpich@3.0.4', 'mpich@3.0'])
+ expected_exclude = set(spack.spec.Spec(x) for x in
+ ['mpich@3.0.1', 'mpich@3.0.2', 'mpich@1.0'])
+ assert expected_include <= set(mirror_specs)
+ assert (not expected_exclude & set(mirror_specs))
+
+
def test_mirror_crud(tmp_scope, capsys):
with capsys.disabled():
mirror('add', '--scope', tmp_scope, 'mirror', 'http://spack.io')
diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash
index 6801862bee..fa1e79c9c7 100755
--- a/share/spack/spack-completion.bash
+++ b/share/spack/spack-completion.bash
@@ -1042,7 +1042,7 @@ _spack_mirror() {
_spack_mirror_create() {
if $list_options
then
- SPACK_COMPREPLY="-h --help -d --directory -a --all -f --file --skip-unstable-versions -D --dependencies -n --versions-per-spec"
+ SPACK_COMPREPLY="-h --help -d --directory -a --all -f --file --exclude-file --exclude-specs --skip-unstable-versions -D --dependencies -n --versions-per-spec"
else
_all_packages
fi
--
GitLab