From 47b3dda1aa92ab98450950ba6383282cbf158857 Mon Sep 17 00:00:00 2001
From: Scott Wittenburg <scott.wittenburg@kitware.com>
Date: Tue, 12 May 2020 16:16:52 -0600
Subject: [PATCH] Support os-specific $padding in config:install_tree
Providing only $padding or ${padding} results in an attempt to
substitute a padding of maximum system path length, while leaving
room for the parts of the install path spack generates. Providing
$padding-<len> or ${padding-<len>} simply substitutes padding of
the specified length.
---
lib/spack/spack/test/config.py | 46 ++++++++++++++++-----
lib/spack/spack/util/path.py | 73 ++++++++++++++++++++++++++++++++++
2 files changed, 108 insertions(+), 11 deletions(-)
diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py
index b8598616d5..8212db6c21 100644
--- a/lib/spack/spack/test/config.py
+++ b/lib/spack/spack/test/config.py
@@ -23,7 +23,7 @@
import spack.schema.mirrors
import spack.schema.repos
import spack.util.spack_yaml as syaml
-from spack.util.path import canonicalize_path
+import spack.util.path as spack_path
# sample config data
@@ -272,31 +272,31 @@ def test_substitute_config_variables(mock_low_high_config):
assert os.path.join(
'/foo/bar/baz', prefix
- ) == canonicalize_path('/foo/bar/baz/$spack')
+ ) == spack_path.canonicalize_path('/foo/bar/baz/$spack')
assert os.path.join(
spack.paths.prefix, 'foo/bar/baz'
- ) == canonicalize_path('$spack/foo/bar/baz/')
+ ) == spack_path.canonicalize_path('$spack/foo/bar/baz/')
assert os.path.join(
'/foo/bar/baz', prefix, 'foo/bar/baz'
- ) == canonicalize_path('/foo/bar/baz/$spack/foo/bar/baz/')
+ ) == spack_path.canonicalize_path('/foo/bar/baz/$spack/foo/bar/baz/')
assert os.path.join(
'/foo/bar/baz', prefix
- ) == canonicalize_path('/foo/bar/baz/${spack}')
+ ) == spack_path.canonicalize_path('/foo/bar/baz/${spack}')
assert os.path.join(
spack.paths.prefix, 'foo/bar/baz'
- ) == canonicalize_path('${spack}/foo/bar/baz/')
+ ) == spack_path.canonicalize_path('${spack}/foo/bar/baz/')
assert os.path.join(
'/foo/bar/baz', prefix, 'foo/bar/baz'
- ) == canonicalize_path('/foo/bar/baz/${spack}/foo/bar/baz/')
+ ) == spack_path.canonicalize_path('/foo/bar/baz/${spack}/foo/bar/baz/')
assert os.path.join(
'/foo/bar/baz', prefix, 'foo/bar/baz'
- ) != canonicalize_path('/foo/bar/baz/${spack/foo/bar/baz/')
+ ) != spack_path.canonicalize_path('/foo/bar/baz/${spack/foo/bar/baz/')
packages_merge_low = {
@@ -345,19 +345,43 @@ def test_merge_with_defaults(mock_low_high_config, write_config_file):
def test_substitute_user(mock_low_high_config):
user = getpass.getuser()
- assert '/foo/bar/' + user + '/baz' == canonicalize_path(
+ assert '/foo/bar/' + user + '/baz' == spack_path.canonicalize_path(
'/foo/bar/$user/baz'
)
def test_substitute_tempdir(mock_low_high_config):
tempdir = tempfile.gettempdir()
- assert tempdir == canonicalize_path('$tempdir')
- assert tempdir + '/foo/bar/baz' == canonicalize_path(
+ assert tempdir == spack_path.canonicalize_path('$tempdir')
+ assert tempdir + '/foo/bar/baz' == spack_path.canonicalize_path(
'$tempdir/foo/bar/baz'
)
+def test_substitute_padding(mock_low_high_config):
+ max_system_path = spack_path.get_system_path_max()
+ expected_length = (max_system_path -
+ spack_path.SPACK_MAX_INSTALL_PATH_LENGTH)
+
+ install_path = spack_path.canonicalize_path('/foo/bar/${padding}/baz')
+
+ assert spack_path.SPACK_PATH_PADDING_CHARS in install_path
+ assert len(install_path) == expected_length
+
+ install_path = spack_path.canonicalize_path('/foo/bar/baz/gah/$padding')
+
+ assert spack_path.SPACK_PATH_PADDING_CHARS in install_path
+ assert len(install_path) == expected_length
+
+ i_path = spack_path.canonicalize_path('/foo/$padding:10')
+ i_expect = os.path.join('/foo', spack_path.SPACK_PATH_PADDING_CHARS[:10])
+ assert i_path == i_expect
+
+ i_path = spack_path.canonicalize_path('/foo/${padding:20}')
+ i_expect = os.path.join('/foo', spack_path.SPACK_PATH_PADDING_CHARS[:20])
+ assert i_path == i_expect
+
+
def test_read_config(mock_low_high_config, write_config_file):
write_config_file('config', config_low, 'low')
assert spack.config.get('config') == config_low['config']
diff --git a/lib/spack/spack/util/path.py b/lib/spack/spack/util/path.py
index 9d5413c609..8bcf598882 100644
--- a/lib/spack/spack/util/path.py
+++ b/lib/spack/spack/util/path.py
@@ -10,8 +10,12 @@
import os
import re
import getpass
+import subprocess
import tempfile
+import llnl.util.tty as tty
+from llnl.util.lang import memoized
+
import spack.paths
@@ -27,6 +31,38 @@
'tempdir': tempfile.gettempdir(),
}
+# This is intended to be longer than the part of the install path
+# spack generates from the root path we give it. Included in the
+# estimate:
+#
+# os-arch -> 30
+# compiler -> 30
+# package name -> 50 (longest is currently 47 characters)
+# version -> 20
+# hash -> 32
+# buffer -> 138
+# ---------------------
+# total -> 300
+SPACK_MAX_INSTALL_PATH_LENGTH = 300
+SPACK_PATH_PADDING_CHARS = 'spack_path_placeholder'
+
+
+@memoized
+def get_system_path_max():
+ # Choose a conservative default
+ sys_max_path_length = 256
+ try:
+ path_max_proc = subprocess.Popen(['getconf', 'PATH_MAX', '/'],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ proc_output = str(path_max_proc.communicate()[0].decode())
+ sys_max_path_length = int(proc_output)
+ except (ValueError, subprocess.CalledProcessError, OSError):
+ tty.msg('Unable to find system max path length, using: {0}'.format(
+ sys_max_path_length))
+
+ return sys_max_path_length
+
def substitute_config_variables(path):
"""Substitute placeholders into paths.
@@ -58,8 +94,45 @@ def substitute_path_variables(path):
return path
+def _get_padding_string(length):
+ spack_path_padding_size = len(SPACK_PATH_PADDING_CHARS)
+ num_reps = int(length / (spack_path_padding_size + 1))
+ extra_chars = length % (spack_path_padding_size + 1)
+ reps_list = [SPACK_PATH_PADDING_CHARS for i in range(num_reps)]
+ reps_list.append(SPACK_PATH_PADDING_CHARS[:extra_chars])
+ return os.path.sep.join(reps_list)
+
+
+def _add_computed_padding(path):
+ """Subtitute in padding of os-specific length. The intent is to leave
+ SPACK_MAX_INSTALL_PATH_LENGTH characters available for parts of the
+ path generated by spack. This is to allow for not-completely-known
+ lengths of things like os/arch, compiler, package name, hash length,
+ etc.
+ """
+ padding_regex = re.compile(r'(\$[\w\d\:]+\b|\$\{[\w\d\:]+\})')
+ m = padding_regex.search(path)
+ if m and m.group(0).strip('${}').startswith('padding'):
+ padding_part = m.group(0)
+ len_pad_part = len(m.group(0))
+ p_match = re.search(r'\:(\d+)', padding_part)
+ if p_match:
+ computed_padding = _get_padding_string(int(p_match.group(1)))
+ else:
+ # Take whatever has been computed/substituted so far and add some
+ # room
+ path_len = len(path) - len_pad_part + SPACK_MAX_INSTALL_PATH_LENGTH
+ system_max_path = get_system_path_max()
+ needed_pad_len = system_max_path - path_len
+ computed_padding = _get_padding_string(needed_pad_len)
+ return padding_regex.sub(computed_padding, path)
+ return path
+
+
def canonicalize_path(path):
"""Same as substitute_path_variables, but also take absolute path."""
path = substitute_path_variables(path)
path = os.path.abspath(path)
+ path = _add_computed_padding(path)
+
return path
--
GitLab