Restore curl progress output (#18127)

Restores the fetching progress bar sans failure outputs; restores non-debug reporting of using fetch cache for installed packages; and adds a unit test.

* Add status bar check to test and fetch output when already installed

lib/spack/spack/fetch_strategy.py

@@ -295,6 +295,9 @@ def fetch(self):
         url = None
         errors = []
         for url in self.candidate_urls:
+            if not self._existing_url(url):
+                continue
+
             try:
                 partial_file, save_file = self._fetch_from_url(url)
                 if save_file:
@@ -309,13 +312,22 @@ def fetch(self):
         if not self.archive_file:
             raise FailedDownloadError(url)
 
+    def _existing_url(self, url):
+        tty.debug('Checking existence of {0}'.format(url))
+        curl = self.curl
+        # Telling curl to fetch the first byte (-r 0-0) is supposed to be
+        # portable.
+        curl_args = ['--stderr', '-', '-s', '-f', '-r', '0-0', url]
+        _ = curl(*curl_args, fail_on_error=False, output=os.devnull)
+        return curl.returncode == 0
+
     def _fetch_from_url(self, url):
         save_file = None
         partial_file = None
         if self.stage.save_filename:
             save_file = self.stage.save_filename
             partial_file = self.stage.save_filename + '.part'
-        tty.debug('Fetching {0}'.format(url))
+        tty.msg('Fetching {0}'.format(url))
         if partial_file:
             save_args = ['-C',
                          '-',  # continue partial downloads
@@ -330,8 +342,6 @@ def _fetch_from_url(self, url):
             '-',  # print out HTML headers
             '-L',  # resolve 3xx redirects
             url,
-            '--stderr',  # redirect stderr output
-            '-',         # redirect to stdout
         ]
 
         if not spack.config.get('config:verify_ssl'):
@@ -340,7 +350,7 @@ def _fetch_from_url(self, url):
         if sys.stdout.isatty() and tty.msg_enabled():
             curl_args.append('-#')  # status bar when using a tty
         else:
-            curl_args.append('-sS')  # just errors when not.
+            curl_args.append('-sS')  # show errors if fail
 
         connect_timeout = spack.config.get('config:connect_timeout', 10)
 
@@ -569,7 +579,7 @@ def fetch(self):
                 raise
 
         # Notify the user how we fetched.
-        tty.debug('Using cached archive: {0}'.format(path))
+        tty.msg('Using cached archive: {0}'.format(path))
 
 
 class VCSFetchStrategy(FetchStrategy):

lib/spack/spack/test/url_fetch.py

@@ -6,8 +6,10 @@
 import collections
 import os
 import pytest
+import sys
 
 from llnl.util.filesystem import working_dir, is_exe
+import llnl.util.tty as tty
 
 import spack.repo
 import spack.config
@@ -173,6 +175,25 @@ def test_unknown_hash(checksum_type):
         crypto.Checker('a')
 
 
+def test_url_with_status_bar(tmpdir, mock_archive, monkeypatch, capfd):
+    """Ensure fetch with status bar option succeeds."""
+    def is_true():
+        return True
+
+    testpath = str(tmpdir)
+
+    monkeypatch.setattr(sys.stdout, 'isatty', is_true)
+    monkeypatch.setattr(tty, 'msg_enabled', is_true)
+
+    fetcher = fs.URLFetchStrategy(mock_archive.url)
+    with Stage(fetcher, path=testpath) as stage:
+        assert fetcher.archive_file is None
+        stage.fetch()
+
+    status = capfd.readouterr()[1]
+    assert '##### 100.0%' in status
+
+
 def test_url_extra_fetch(tmpdir, mock_archive):
     """Ensure a fetch after downloading is effectively a no-op."""
     testpath = str(tmpdir)