From 23a7feb91741f8be42b17fe1d02fbbddaa751810 Mon Sep 17 00:00:00 2001
From: Patrick Gartung <gartung@fnal.gov>
Date: Thu, 30 Jan 2020 10:56:10 -0600
Subject: [PATCH] Limit the number of spec files downloaded to find matches for
buildcaches (#14659)
* Limit the number of spec flies downloaded to find matches
---
lib/spack/spack/binary_distribution.py | 34 ++++++++++++--------------
lib/spack/spack/cmd/buildcache.py | 14 ++++++++---
lib/spack/spack/package.py | 2 +-
lib/spack/spack/test/cmd/buildcache.py | 2 +-
share/spack/spack-completion.bash | 2 +-
5 files changed, 29 insertions(+), 25 deletions(-)
diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py
index fd89a80fa8..bb5f3d86d0 100644
--- a/lib/spack/spack/binary_distribution.py
+++ b/lib/spack/spack/binary_distribution.py
@@ -664,7 +664,7 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
_cached_specs = None
-def get_specs(force=False, use_arch=False):
+def get_specs(force=False, use_arch=False, names=[]):
"""
Get spec.yaml's for build caches available on mirror
"""
@@ -672,6 +672,15 @@ def get_specs(force=False, use_arch=False):
arch = architecture.Arch(architecture.platform(),
'default_os', 'default_target')
+ arch_pattern = ('([^-]*-[^-]*-[^-]*)')
+ if use_arch:
+ arch_pattern = '(%s-%s-[^-]*)' % (arch.platform, arch.os)
+
+ names_or_hashes = [name.replace('/', '') for name in names]
+ names_pattern = '|'.join(names_or_hashes)
+ regex_pattern = '%s(.*)(%s)(.*)(spec.yaml$)' % (arch_pattern,
+ names_pattern)
+ name_re = re.compile(regex_pattern)
if _cached_specs:
tty.debug("Using previously-retrieved specs")
@@ -692,30 +701,19 @@ def get_specs(force=False, use_arch=False):
if os.path.exists(mirror_dir):
files = os.listdir(mirror_dir)
for file in files:
- if re.search('spec.yaml', file):
+ m = name_re.search(file)
+ if m:
link = url_util.join(fetch_url_build_cache, file)
- if use_arch and re.search('%s-%s' %
- (arch.platform,
- arch.os),
- file):
- urls.add(link)
- else:
- urls.add(link)
+ urls.add(link)
else:
tty.msg("Finding buildcaches at %s" %
url_util.format(fetch_url_build_cache))
p, links = web_util.spider(
url_util.join(fetch_url_build_cache, 'index.html'))
for link in links:
- if re.search("spec.yaml", link):
- if use_arch and re.search('%s-%s' %
- (arch.platform,
- arch.os),
- link):
- urls.add(link)
- else:
- urls.add(link)
-
+ m = name_re.search(link)
+ if m:
+ urls.add(link)
_cached_specs = []
for link in urls:
with Stage(link, name="build_cache", keep=True) as stage:
diff --git a/lib/spack/spack/cmd/buildcache.py b/lib/spack/spack/cmd/buildcache.py
index cbcbc2c0cb..d3481bb6e6 100644
--- a/lib/spack/spack/cmd/buildcache.py
+++ b/lib/spack/spack/cmd/buildcache.py
@@ -87,6 +87,8 @@ def setup_parser(subparser):
help='show variants in output (can be long)')
listcache.add_argument('-f', '--force', action='store_true',
help="force new download of specs")
+ listcache.add_argument('-a', '--arch', action='store_true',
+ help="only list spec for the default architecture")
arguments.add_common_arguments(listcache, ['specs'])
listcache.set_defaults(func=listspecs)
@@ -263,10 +265,10 @@ def match_downloaded_specs(pkgs, allow_multiple_matches=False, force=False):
# List of specs that match expressions given via command line
specs_from_cli = []
has_errors = False
- specs = bindist.get_specs(force)
for pkg in pkgs:
matches = []
tty.msg("buildcache spec(s) matching %s \n" % pkg)
+ specs = bindist.get_specs(names=[pkg])
for spec in sorted(specs):
if pkg.startswith('/'):
pkghash = pkg.replace('/', '')
@@ -415,10 +417,14 @@ def install_tarball(spec, args):
def listspecs(args):
"""list binary packages available from mirrors"""
- specs = bindist.get_specs(args.force)
+ specs = list()
if args.specs:
- constraints = set(args.specs)
- specs = [s for s in specs if any(s.satisfies(c) for c in constraints)]
+ for s in bindist.get_specs(args.force, args.arch,
+ args.specs):
+ if s not in set(specs):
+ specs.append(s)
+ else:
+ specs = bindist.get_specs(force=args.force, use_arch=args.arch)
display_specs(specs, args, all_headers=True)
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 6856e3a397..6ad1ebe1f0 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -1510,7 +1510,7 @@ def _update_explicit_entry_in_db(self, rec, explicit):
def try_install_from_binary_cache(self, explicit):
tty.msg('Searching for binary cache of %s' % self.name)
- specs = binary_distribution.get_specs(use_arch=True)
+ specs = binary_distribution.get_specs(use_arch=True, names=[self.name])
binary_spec = spack.spec.Spec.from_dict(self.spec.to_dict())
binary_spec._mark_concrete()
if binary_spec not in specs:
diff --git a/lib/spack/spack/test/cmd/buildcache.py b/lib/spack/spack/test/cmd/buildcache.py
index f7b7611c33..fdd76bff20 100644
--- a/lib/spack/spack/test/cmd/buildcache.py
+++ b/lib/spack/spack/test/cmd/buildcache.py
@@ -17,7 +17,7 @@
def mock_get_specs(database, monkeypatch):
specs = database.query_local()
monkeypatch.setattr(
- spack.binary_distribution, 'get_specs', lambda x: specs
+ spack.binary_distribution, 'get_specs', lambda x, y, z: specs
)
diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash
index 20a5d936ea..b408d0b234 100755
--- a/share/spack/spack-completion.bash
+++ b/share/spack/spack-completion.bash
@@ -400,7 +400,7 @@ _spack_buildcache_install() {
_spack_buildcache_list() {
if $list_options
then
- SPACK_COMPREPLY="-h --help -l --long -L --very-long -v --variants -f --force"
+ SPACK_COMPREPLY="-h --help -l --long -L --very-long -v --variants -f --force -a --arch"
else
_all_packages
fi
--
GitLab