From 0cd6555388ad6fafc110bc2aa60f256acf920bcd Mon Sep 17 00:00:00 2001
From: Todd Gamblin <tgamblin@llnl.gov>
Date: Tue, 7 Mar 2017 15:18:48 -0800
Subject: [PATCH] Resolve Python2/Python3 unicode issues by using str()
- Remove ascii encoding assumption from spack_yaml
- proc.communicate() returns bytes; convert to str before adding.
- Fix various byte string/unicode issues for Python 2/3 support
- Need to decode subprocess output as utf-8 in from_sourcing_files.
- Fix comments in strify()
---
lib/spack/spack/environment.py | 2 +-
lib/spack/spack/stage.py | 2 +-
lib/spack/spack/test/architecture.py | 4 ++--
lib/spack/spack/util/crypto.py | 8 +++++++-
lib/spack/spack/util/executable.py | 4 ++--
lib/spack/spack/util/spack_json.py | 1 +
lib/spack/spack/util/spack_yaml.py | 7 +------
7 files changed, 15 insertions(+), 13 deletions(-)
diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py
index 76b8e132d4..eadfa45efb 100644
--- a/lib/spack/spack/environment.py
+++ b/lib/spack/spack/environment.py
@@ -310,7 +310,7 @@ def from_sourcing_files(*args, **kwargs):
proc.wait()
if proc.returncode != 0:
raise RuntimeError('sourcing files returned a non-zero exit code')
- output = ''.join([line for line in proc.stdout])
+ output = ''.join([line.decode('utf-8') for line in proc.stdout])
# Construct a dictionaries of the environment before and after
# sourcing the files, so that we can diff them.
diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py
index 03b7f5ef33..cf294be93b 100644
--- a/lib/spack/spack/stage.py
+++ b/lib/spack/spack/stage.py
@@ -226,7 +226,7 @@ def __init__(
self._lock = None
if lock:
if self.name not in Stage.stage_locks:
- sha1 = hashlib.sha1(self.name).digest()
+ sha1 = hashlib.sha1(self.name.encode('utf-8')).digest()
lock_id = prefix_bits(sha1, bit_length(sys.maxsize))
stage_lock_path = join_path(spack.stage_path, '.lock')
diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py
index fb4113361c..8f257cf0dc 100644
--- a/lib/spack/spack/test/architecture.py
+++ b/lib/spack/spack/test/architecture.py
@@ -138,8 +138,8 @@ def test_user_defaults(config):
def test_user_input_combination(config):
platform = spack.architecture.platform()
- os_list = platform.operating_sys.keys()
- target_list = platform.targets.keys()
+ os_list = list(platform.operating_sys.keys())
+ target_list = list(platform.targets.keys())
additional = ["fe", "be", "frontend", "backend"]
os_list.extend(additional)
diff --git a/lib/spack/spack/util/crypto.py b/lib/spack/spack/util/crypto.py
index d074716022..2965168056 100644
--- a/lib/spack/spack/util/crypto.py
+++ b/lib/spack/spack/util/crypto.py
@@ -22,6 +22,7 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
+import sys
import hashlib
"""Set of acceptable hashes that Spack will use."""
@@ -104,11 +105,16 @@ def check(self, filename):
def prefix_bits(byte_array, bits):
"""Return the first <bits> bits of a byte array as an integer."""
+ if sys.version_info < (3,):
+ b2i = ord # In Python 2, indexing byte_array gives str
+ else:
+ b2i = lambda b: b # In Python 3, indexing byte_array gives int
+
result = 0
n = 0
for i, b in enumerate(byte_array):
n += 8
- result = (result << 8) | ord(b)
+ result = (result << 8) | b2i(b)
if n >= bits:
break
diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py
index 372d0019e7..1d7f019fdf 100644
--- a/lib/spack/spack/util/executable.py
+++ b/lib/spack/spack/util/executable.py
@@ -178,9 +178,9 @@ def streamify(arg, mode):
if output is str or error is str:
result = ''
if output is str:
- result += out
+ result += out.decode('utf-8')
if error is str:
- result += err
+ result += err.decode('utf-8')
return result
except OSError as e:
diff --git a/lib/spack/spack/util/spack_json.py b/lib/spack/spack/util/spack_json.py
index 0090cf89ca..6b26ad5a98 100644
--- a/lib/spack/spack/util/spack_json.py
+++ b/lib/spack/spack/util/spack_json.py
@@ -68,6 +68,7 @@ def _byteify(data, ignore_dicts=False):
return dict((_byteify(key, ignore_dicts=True),
_byteify(value, ignore_dicts=True)) for key, value in
iteritems(data))
+
# if it's anything else, return it in its original form
return data
diff --git a/lib/spack/spack/util/spack_yaml.py b/lib/spack/spack/util/spack_yaml.py
index c49393af9c..a8b773ac0c 100644
--- a/lib/spack/spack/util/spack_yaml.py
+++ b/lib/spack/spack/util/spack_yaml.py
@@ -85,11 +85,6 @@ class OrderedLineLoader(Loader):
def construct_yaml_str(self, node):
value = self.construct_scalar(node)
- try:
- value = value.encode('ascii')
- except UnicodeEncodeError:
- pass
-
value = syaml_str(value)
mark(value, node)
@@ -181,7 +176,7 @@ def represent_mapping(self, tag, mapping, flow_style=None):
# if it's a syaml_dict, preserve OrderedDict order.
# Otherwise do the default thing.
sort = not isinstance(mapping, syaml_dict)
- mapping = mapping.items()
+ mapping = list(mapping.items())
if sort:
mapping.sort()
--
GitLab