Make `insecure` option work with curl AND git. (#1786)

bin/spack

@@ -161,7 +161,7 @@ def main():
     # If the user asked for it, don't check ssl certs.
     if args.insecure:
         tty.warn("You asked for --insecure. Will NOT check SSL certificates.")
-        spack.curl.add_default_arg('-k')
+        spack.insecure = True
 
     # Try to load the particular command asked for and run it
     command = spack.cmd.get_command(args.command.replace('-', '_'))

lib/spack/spack/__init__.py

@@ -129,8 +129,9 @@
 # User's editor from the environment
 editor = Executable(os.environ.get("EDITOR", "vi"))
 
-# Curl tool for fetching files.
-curl = which("curl", required=True)
+# If this is enabled, tools that use SSL should not verify
+# certifiates. e.g., curl should use the -k option.
+insecure = False
 
 # Whether to build in tmp space or directly in the stage_path.
 # If this is true, then spack will make stage directories in

lib/spack/spack/fetch_strategy.py

@@ -158,12 +158,20 @@ def __init__(self, url=None, digest=None, **kwargs):
             self.digest = digest
 
         self.expand_archive = kwargs.get('expand', True)
+        self.extra_curl_options = kwargs.get('curl_options', [])
+        self._curl = None
 
         self.extension = kwargs.get('extension', None)
 
         if not self.url:
             raise ValueError("URLFetchStrategy requires a url for fetching.")
 
+    @property
+    def curl(self):
+        if not self._curl:
+            self._curl = which('curl', required=True)
+        return self._curl
+
     @_needs_stage
     def fetch(self):
         self.stage.chdir()
@@ -196,15 +204,21 @@ def fetch(self):
             self.url,
         ]
 
+        if spack.insecure:
+            curl_args.append('-k')
+
         if sys.stdout.isatty():
             curl_args.append('-#')  # status bar when using a tty
         else:
             curl_args.append('-sS')  # just errors when not.
 
+        curl_args += self.extra_curl_options
+
         # Run curl but grab the mime type from the http headers
-        headers = spack.curl(*curl_args, output=str, fail_on_error=False)
+        curl = self.curl
+        headers = curl(*curl_args, output=str, fail_on_error=False)
 
-        if spack.curl.returncode != 0:
+        if curl.returncode != 0:
             # clean up archive on failure.
             if self.archive_file:
                 os.remove(self.archive_file)
@@ -212,12 +226,12 @@ def fetch(self):
             if partial_file and os.path.exists(partial_file):
                 os.remove(partial_file)
 
-            if spack.curl.returncode == 22:
+            if curl.returncode == 22:
                 # This is a 404.  Curl will print the error.
                 raise FailedDownloadError(
                     self.url, "URL %s was not found!" % self.url)
 
-            elif spack.curl.returncode == 60:
+            elif curl.returncode == 60:
                 # This is a certificate error.  Suggest spack -k
                 raise FailedDownloadError(
                     self.url,
@@ -233,7 +247,7 @@ def fetch(self):
                 # error, but print a spack message too
                 raise FailedDownloadError(
                     self.url,
-                    "Curl failed with error %d" % spack.curl.returncode)
+                    "Curl failed with error %d" % curl.returncode)
 
         # Check if we somehow got an HTML file rather than the archive we
         # asked for.  We only look at the last content type, to handle
@@ -530,6 +544,12 @@ def git_version(self):
     def git(self):
         if not self._git:
             self._git = which('git', required=True)
+
+            # If the user asked for insecure fetching, make that work
+            # with git as well.
+            if spack.insecure:
+                self._git.add_default_env('GIT_SSL_NO_VERIFY', 'true')
+
         return self._git
 
     @_needs_stage

lib/spack/spack/util/executable.py

@@ -40,6 +40,7 @@ class Executable(object):
 
     def __init__(self, name):
         self.exe = name.split(' ')
+        self.default_env = {}
         self.returncode = None
 
         if not self.exe:
@@ -48,6 +49,9 @@ def __init__(self, name):
     def add_default_arg(self, arg):
         self.exe.append(arg)
 
+    def add_default_env(self, key, value):
+        self.default_env[key] = value
+
     @property
     def command(self):
         return ' '.join(self.exe)
@@ -103,7 +107,13 @@ def __call__(self, *args, **kwargs):
         fail_on_error = kwargs.pop("fail_on_error", True)
         ignore_errors = kwargs.pop("ignore_errors", ())
 
+        # environment
         env = kwargs.get('env', None)
+        if env is None:
+            env = os.environ.copy()
+            env.update(self.default_env)
+        else:
+            env = self.default_env.copy().update(env)
 
         # TODO: This is deprecated.  Remove in a future version.
         return_output = kwargs.pop("return_output", False)
@@ -149,6 +159,7 @@ def streamify(arg, mode):
 
         cmd_line = "'%s'" % "' '".join(
             map(lambda arg: arg.replace("'", "'\"'\"'"), cmd))
+
         tty.debug(cmd_line)
 
         try:

var/spack/repos/builtin/packages/jdk/package.py

@@ -26,10 +26,7 @@
 # Author: Justin Too <too1@llnl.gov>
 #
 import distutils.dir_util
-
-import spack
 from spack import *
-import llnl.util.tty as tty
 
 
 class Jdk(Package):
@@ -37,11 +34,6 @@ class Jdk(Package):
        in the form of a binary product aimed at Java developers."""
     homepage = "http://www.oracle.com/technetwork/java/javase/downloads/index.html"
 
-    version('8u66-linux-x64', '88f31f3d642c3287134297b8c10e61bf',
-            url="http://download.oracle.com/otn-pub/java/jdk/8u66-b17/jdk-8u66-linux-x64.tar.gz")
-    version('8u92-linux-x64', '65a1cc17ea362453a6e0eb4f13be76e4',
-            url="http://download.oracle.com/otn-pub/java/jdk/8u92-b14/jdk-8u92-linux-x64.tar.gz")
-
     # Oracle requires that you accept their License Agreement in order
     # to access the Java packages in download.oracle.com. In order to
     # automate this process, we need to utilize these additional curl
@@ -53,18 +45,12 @@ class Jdk(Package):
         '-H',  # specify required License Agreement cookie
         'Cookie: oraclelicense=accept-securebackup-cookie']
 
-    def do_fetch(self, mirror_only=False):
-        # Add our custom curl commandline options
-        tty.msg(
-            "[Jdk] Adding required commandline options to curl " +
-            "before performing fetch: %s" %
-            (self.curl_options))
-
-        for option in self.curl_options:
-            spack.curl.add_default_arg(option)
-
-        # Now perform the actual fetch
-        super(Jdk, self).do_fetch(mirror_only)
+    version('8u66-linux-x64', '88f31f3d642c3287134297b8c10e61bf',
+            url="http://download.oracle.com/otn-pub/java/jdk/8u66-b17/jdk-8u66-linux-x64.tar.gz",
+            curl_options=curl_options)
+    version('8u92-linux-x64', '65a1cc17ea362453a6e0eb4f13be76e4',
+            url="http://download.oracle.com/otn-pub/java/jdk/8u92-b14/jdk-8u92-linux-x64.tar.gz",
+            curl_options=curl_options)
 
     def install(self, spec, prefix):
         distutils.dir_util.copy_tree(".", prefix)