Merge pull request #850 from mathstuf/partial-downloads

fetch_strategy: download to temporary files

lib/spack/spack/fetch_strategy.py

@@ -157,12 +157,26 @@ def fetch(self):
             tty.msg("Already downloaded %s" % self.archive_file)
             return
 
+        possible_files = self.stage.expected_archive_files
+        save_file = None
+        partial_file = None
+        if possible_files:
+            save_file = self.stage.expected_archive_files[0]
+            partial_file = self.stage.expected_archive_files[0] + '.part'
+
         tty.msg("Trying to fetch from %s" % self.url)
 
-        curl_args = ['-O',  # save file to disk
+        if partial_file:
+            save_args = ['-C', '-',          # continue partial downloads
+                         '-o', partial_file] # use a .part file
+        else:
+            save_args = ['-O']
+
+        curl_args = save_args + [
                      '-f',  # fail on >400 errors
                      '-D', '-',  # print out HTML headers
-                     '-L', self.url, ]
+                     '-L',  # resolve 3xx redirects
+                     self.url, ]
 
         if sys.stdout.isatty():
             curl_args.append('-#')  # status bar when using a tty
@@ -178,6 +192,9 @@ def fetch(self):
             if self.archive_file:
                 os.remove(self.archive_file)
 
+            if partial_file and os.path.exists(partial_file):
+                os.remove(partial_file)
+
             if spack.curl.returncode == 22:
                 # This is a 404.  Curl will print the error.
                 raise FailedDownloadError(
@@ -209,6 +226,9 @@ def fetch(self):
                      "'spack clean <package>' to remove the bad archive, then fix",
                      "your internet gateway issue and install again.")
 
+        if save_file:
+            os.rename(partial_file, save_file)
+
         if not self.archive_file:
             raise FailedDownloadError(self.url)
 

lib/spack/spack/stage.py

@@ -210,6 +210,18 @@ def _need_to_create_path(self):
 
         return False
 
+    @property
+    def expected_archive_files(self):
+        """Possible archive file paths."""
+        paths = []
+        if isinstance(self.fetcher, fs.URLFetchStrategy):
+            paths.append(os.path.join(self.path, os.path.basename(self.fetcher.url)))
+
+        if self.mirror_path:
+            paths.append(os.path.join(self.path, os.path.basename(self.mirror_path)))
+
+        return paths
+
     @property
     def archive_file(self):
         """Path to the source archive within this stage directory."""