From 1c0a92662bee39bbd56ea5457aad3a31e45151b6 Mon Sep 17 00:00:00 2001
From: Tamara Dahlgren <35777542+tldahlgren@users.noreply.github.com>
Date: Wed, 19 Aug 2020 12:10:18 -0700
Subject: [PATCH] Restore curl progress output (#18127)
Restores the fetching progress bar sans failure outputs; restores non-debug reporting of using fetch cache for installed packages; and adds a unit test.
* Add status bar check to test and fetch output when already installed
---
lib/spack/spack/fetch_strategy.py | 20 +++++++++++++++-----
lib/spack/spack/test/url_fetch.py | 21 +++++++++++++++++++++
2 files changed, 36 insertions(+), 5 deletions(-)
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
index fb87373897..97d95c2fe9 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -295,6 +295,9 @@ def fetch(self):
url = None
errors = []
for url in self.candidate_urls:
+ if not self._existing_url(url):
+ continue
+
try:
partial_file, save_file = self._fetch_from_url(url)
if save_file:
@@ -309,13 +312,22 @@ def fetch(self):
if not self.archive_file:
raise FailedDownloadError(url)
+ def _existing_url(self, url):
+ tty.debug('Checking existence of {0}'.format(url))
+ curl = self.curl
+ # Telling curl to fetch the first byte (-r 0-0) is supposed to be
+ # portable.
+ curl_args = ['--stderr', '-', '-s', '-f', '-r', '0-0', url]
+ _ = curl(*curl_args, fail_on_error=False, output=os.devnull)
+ return curl.returncode == 0
+
def _fetch_from_url(self, url):
save_file = None
partial_file = None
if self.stage.save_filename:
save_file = self.stage.save_filename
partial_file = self.stage.save_filename + '.part'
- tty.debug('Fetching {0}'.format(url))
+ tty.msg('Fetching {0}'.format(url))
if partial_file:
save_args = ['-C',
'-', # continue partial downloads
@@ -330,8 +342,6 @@ def _fetch_from_url(self, url):
'-', # print out HTML headers
'-L', # resolve 3xx redirects
url,
- '--stderr', # redirect stderr output
- '-', # redirect to stdout
]
if not spack.config.get('config:verify_ssl'):
@@ -340,7 +350,7 @@ def _fetch_from_url(self, url):
if sys.stdout.isatty() and tty.msg_enabled():
curl_args.append('-#') # status bar when using a tty
else:
- curl_args.append('-sS') # just errors when not.
+ curl_args.append('-sS') # show errors if fail
connect_timeout = spack.config.get('config:connect_timeout', 10)
@@ -569,7 +579,7 @@ def fetch(self):
raise
# Notify the user how we fetched.
- tty.debug('Using cached archive: {0}'.format(path))
+ tty.msg('Using cached archive: {0}'.format(path))
class VCSFetchStrategy(FetchStrategy):
diff --git a/lib/spack/spack/test/url_fetch.py b/lib/spack/spack/test/url_fetch.py
index 20648b4766..8a9a175a37 100644
--- a/lib/spack/spack/test/url_fetch.py
+++ b/lib/spack/spack/test/url_fetch.py
@@ -6,8 +6,10 @@
import collections
import os
import pytest
+import sys
from llnl.util.filesystem import working_dir, is_exe
+import llnl.util.tty as tty
import spack.repo
import spack.config
@@ -173,6 +175,25 @@ def test_unknown_hash(checksum_type):
crypto.Checker('a')
+def test_url_with_status_bar(tmpdir, mock_archive, monkeypatch, capfd):
+ """Ensure fetch with status bar option succeeds."""
+ def is_true():
+ return True
+
+ testpath = str(tmpdir)
+
+ monkeypatch.setattr(sys.stdout, 'isatty', is_true)
+ monkeypatch.setattr(tty, 'msg_enabled', is_true)
+
+ fetcher = fs.URLFetchStrategy(mock_archive.url)
+ with Stage(fetcher, path=testpath) as stage:
+ assert fetcher.archive_file is None
+ stage.fetch()
+
+ status = capfd.readouterr()[1]
+ assert '##### 100.0%' in status
+
+
def test_url_extra_fetch(tmpdir, mock_archive):
"""Ensure a fetch after downloading is effectively a no-op."""
testpath = str(tmpdir)
--
GitLab