PythonPackage builds flat installs instead of egg directories.

- Spack doesn't need eggs -- it manages its own directories

- Simplify install layout and reduce sys.path searches by installing all
  packages flat (eggs are deprecated for wheels, and this is also what
  wheels do).

- We now supply the --single-version-externally-managed argument to
  `setup.py install` for setuptools packages and setuptools.

- modify packages to only use setuptools args if setuptools is an
  immediate dependency

- Remove setuptools from packages that do not need it.

  - Some packages use setuptools *only* when certain args (likeb
    'develop' or 'bdist') are supplied to setup.py, and they specifically
    do not use setuptools for installation.

  - Spack never calls setup.py this way, so just removing the setuptools
    dependency works for these packages.

lib/spack/spack/build_systems/python.py

@@ -222,7 +222,24 @@ def install(self, spec, prefix):
 
     def install_args(self, spec, prefix):
         """Arguments to pass to install."""
-        return ['--prefix={0}'.format(prefix)]
+        args = ['--prefix={0}'.format(prefix)]
+
+        # This option causes python packages (including setuptools) NOT
+        # to create eggs or easy-install.pth files.  Instead, they
+        # install naturally into $prefix/pythonX.Y/site-packages.
+        #
+        # Eggs add an extra level of indirection to sys.path, slowing
+        # down large HPC runs.  They are also deprecated in favor of
+        # wheels, which use a normal layout when installed.
+        #
+        # Spack manages the package directory on its own by symlinking
+        # extensions into the site-packages directory, so we don't really
+        # need the .pth files or egg directories, anyway.
+        if ('py-setuptools' == spec.name or          # this is setuptools, or
+            'py-setuptools' in spec._dependencies):  # it's an immediate dep
+            args += ['--single-version-externally-managed', '--root=/']
+
+        return args
 
     def install_lib(self, spec, prefix):
         """Install all Python modules (extensions and pure Python)."""

var/spack/repos/builtin/packages/npm/package.py

@@ -36,7 +36,7 @@ class Npm(AutotoolsPackage):
     version('3.10.9', 'ec1eb22b466ce87cdd0b90182acce07f')
     version('3.10.5', '46002413f4a71de9b0da5b506bf1d992')
 
-    depends_on('node-js')
+    depends_on('node-js', type=('build', 'run'))
 
     def setup_dependent_environment(self, spack_env, run_env, dependent_spec):
         npm_config_cache_dir = "%s/npm-cache" % dependent_spec.prefix

var/spack/repos/builtin/packages/py-backports-ssl-match-hostname/package.py

@@ -32,7 +32,3 @@ class PyBackportsSslMatchHostname(PythonPackage):
     url      = "https://pypi.io/packages/source/b/backports.ssl_match_hostname/backports.ssl_match_hostname-3.5.0.1.tar.gz"
 
     version('3.5.0.1', 'c03fc5e2c7b3da46b81acf5cbacfe1e6')
-
-    # newer setuptools version mess with "namespace" packages in an
-    # incompatible way cf. https://github.com/pypa/setuptools/issues/900
-    depends_on('py-setuptools@:30.999.999', type='build')

var/spack/repos/builtin/packages/py-basemap/package.py

@@ -35,7 +35,6 @@ class PyBasemap(PythonPackage):
 
     version('1.0.7', '48c0557ced9e2c6e440b28b3caff2de8')
 
-    depends_on('py-setuptools', type='build')
     depends_on('py-numpy', type=('build', 'run'))
     depends_on('py-matplotlib', type=('build', 'run'))
     depends_on('pil', type=('build', 'run'))

var/spack/repos/builtin/packages/py-ipykernel/package.py

@@ -43,7 +43,6 @@ class PyIpykernel(PythonPackage):
     version('4.1.0', '638a43e4f8a15872f749090c3f0827b6')
 
     depends_on('python@2.7:2.7.999,3.3:')
-    depends_on('py-setuptools', type='build')
     depends_on('py-traitlets@4.1.0:', type=('build', 'run'))
     depends_on('py-tornado@4.0:', type=('build', 'run'))
     depends_on('py-ipython@4.0:', type=('build', 'run'))

var/spack/repos/builtin/packages/py-ipython-genutils/package.py

@@ -33,5 +33,4 @@ class PyIpythonGenutils(PythonPackage):
 
     version('0.1.0', '9a8afbe0978adbcbfcb3b35b2d015a56')
 
-    depends_on('py-setuptools', type='build')
     depends_on('python@2.7:2.7.999,3.3:')

var/spack/repos/builtin/packages/py-ipython/package.py

@@ -38,7 +38,6 @@ class PyIpython(PythonPackage):
     version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf')
 
     depends_on('python@2.7:2.8,3.3:')
-    depends_on('py-setuptools@18.5:', type=('build', 'run'))
 
     # These dependencies breaks concretization
     # See https://github.com/LLNL/spack/issues/2793

var/spack/repos/builtin/packages/py-ipywidgets/package.py

@@ -33,7 +33,6 @@ class PyIpywidgets(PythonPackage):
 
     version('5.2.2', '112f3daa4aa0f42f8dda831cea3649c8')
 
-    depends_on('py-setuptools', type='build')
     depends_on('python@2.7:2.7.999,3.3:')
     depends_on('py-ipython@4.0.0:', type=('build', 'run'))
     depends_on('py-ipykernel@4.2.2:', type=('build', 'run'))

var/spack/repos/builtin/packages/py-jupyter-client/package.py

@@ -40,7 +40,6 @@ class PyJupyterClient(PythonPackage):
     version('4.1.0', 'cf42048b889c8434fbb5813a9eec1d34')
     version('4.0.0', '00fa63c67cb3adf359d09dc4d803aff5')
 
-    depends_on('py-setuptools', type='build')
     depends_on('python@2.7:2.7.999,3.3:')
     depends_on('py-traitlets', type=('build', 'run'))
     depends_on('py-jupyter-core', type=('build', 'run'))

var/spack/repos/builtin/packages/py-jupyter-console/package.py

@@ -37,7 +37,6 @@ class PyJupyterConsole(PythonPackage):
     version('4.0.3', '0e928ea261e7f8154698cf69ed4f2459')
     version('4.0.2', 'f2e174938c91136549b908bd39fa5d59')
 
-    depends_on('py-setuptools', type='build')
     depends_on('python@2.7:2.7.999,3.3:')
     depends_on('py-jupyter-client', type=('build', 'run'))
     depends_on('py-ipython', type=('build', 'run'))

var/spack/repos/builtin/packages/py-jupyter-core/package.py

@@ -42,6 +42,5 @@ class PyJupyterCore(PythonPackage):
     version('4.0.1', 'f849136b2badaaba2a8a3b397bf04639')
     version('4.0',   'b6b37cb4f40bd0fcd20433cb2cc7a4c1')
 
-    depends_on('py-setuptools', type='build')
     depends_on('python@2.7:2.7.999,3.3:')
     depends_on('py-traitlets', type=('build', 'run'))

var/spack/repos/builtin/packages/py-jupyter-notebook/package.py

@@ -44,9 +44,9 @@ class PyJupyterNotebook(PythonPackage):
 
     variant('terminal', default=False, description="Enable terminal functionality")
 
-    depends_on('py-setuptools', type='build')
     depends_on('python@2.7:2.7.999,3.3:')
     depends_on('npm', type='build')
+    depends_on('node-js', type=('build', 'run'))
     depends_on('py-jinja2', type=('build', 'run'))
     depends_on('py-tornado@4:', type=('build', 'run'))
     depends_on('py-ipython-genutils', type=('build', 'run'))

var/spack/repos/builtin/packages/py-nbconvert/package.py

@@ -22,6 +22,7 @@
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+import os
 from spack import *
 
 
@@ -35,7 +36,6 @@ class PyNbconvert(PythonPackage):
     version('4.1.0', '06655576713ba1ff7cece2b92760c187')
     version('4.0.0', '9661620b1e10a7b46f314588d2d0932f')
 
-    depends_on('py-setuptools', type='build')
     depends_on('py-pycurl', type='build')
     depends_on('python@2.7:2.7.999,3.3:')
     depends_on('py-mistune', type=('build', 'run'))
@@ -48,8 +48,8 @@ class PyNbconvert(PythonPackage):
     depends_on('py-tornado', type=('build', 'run'))
     depends_on('py-jupyter-client', type=('build', 'run'))
 
-    # FIXME:
-    # Failed, try again after installing PycURL with `pip install pycurl` to avoid outdated SSL.  # noqa
-    # Failed to download css from https://cdn.jupyter.org/notebook/4.1.0/style/style.min.css: [Errno socket error] [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:661)  # noqa
-    # Downloading CSS: https://cdn.jupyter.org/notebook/4.1.0/style/style.min.css  # noqa
-    # error: Need Notebook CSS to proceed: nbconvert/resources/style.min.css
+    def patch(self):
+        # We bundle this with the spack package so that the installer
+        # doesn't try to download it.
+        install(os.path.join(self.package_dir, 'style.min.css'),
+                os.path.join('nbconvert', 'resources', 'style.min.css'))

var/spack/repos/builtin/packages/py-nbformat/package.py

@@ -35,7 +35,6 @@ class PyNbformat(PythonPackage):
     version('4.0.1', 'ab7172e517c9d561c0c01eef5631b4c8')
     version('4.0.0', '7cf61359fa4e9cf3ef5e969e2fcb909e')
 
-    depends_on('py-setuptools', type='build')
     depends_on('py-ipython-genutils', type=('build', 'run'))
     depends_on('py-traitlets', type=('build', 'run'))
     depends_on('py-jsonschema', type=('build', 'run'))

var/spack/repos/builtin/packages/py-ptyprocess/package.py

@@ -32,5 +32,3 @@ class PyPtyprocess(PythonPackage):
     url      = "https://pypi.io/packages/source/p/ptyprocess/ptyprocess-0.5.1.tar.gz"
 
     version('0.5.1', '94e537122914cc9ec9c1eadcd36e73a1')
-
-    depends_on('py-setuptools', type='build')

var/spack/repos/builtin/packages/py-pycurl/package.py

@@ -35,5 +35,4 @@ class PyPycurl(PythonPackage):
     version('7.43.0', 'c94bdba01da6004fa38325e9bd6b9760')
 
     depends_on('python@2.6:')
-    depends_on('py-setuptools', type='build')
     depends_on('curl@7.19.0:')

var/spack/repos/builtin/packages/py-traitlets/package.py

@@ -40,7 +40,6 @@ class PyTraitlets(PythonPackage):
     version('4.0.0', 'b5b95ea5941fd9619b4704dfd8201568')
     version('4.0',   '14544e25ccf8e920ed1cbf833852481f')
 
-    depends_on('py-setuptools', type='build')
     depends_on('py-decorator', type=('build', 'run'))
     depends_on('py-ipython-genutils', type=('build', 'run'))
 

var/spack/repos/builtin/packages/py-zmq/package.py

@@ -33,7 +33,6 @@ class PyZmq(PythonPackage):
     version('16.0.2', '4cf14a2995742253b2b009541f4436f4')
     version('14.7.0', 'bf304fb73d72aee314ff82d3554328c179938ecf')
 
-    depends_on('py-setuptools', type='build')
     depends_on('py-cython@0.16:', type=('build', 'run'))
     depends_on('py-py', type=('build', 'run'))
     depends_on('py-cffi', type=('build', 'run'))