From 6b90017efa1f3157fe4be7d0c7b199b6e51b9fa8 Mon Sep 17 00:00:00 2001
From: Todd Gamblin <tgamblin@llnl.gov>
Date: Wed, 28 Jan 2015 22:05:57 -0800
Subject: [PATCH] Fixed dumb link_tree bug, added test for link tree.
---
lib/spack/llnl/util/filesystem.py | 8 +-
lib/spack/llnl/util/link_tree.py | 197 +++++++++++++++++-------------
lib/spack/spack/test/__init__.py | 3 +-
lib/spack/spack/test/link_tree.py | 153 +++++++++++++++++++++++
4 files changed, 274 insertions(+), 87 deletions(-)
create mode 100644 lib/spack/spack/test/link_tree.py
diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py
index 0578415653..576aeb16bd 100644
--- a/lib/spack/llnl/util/filesystem.py
+++ b/lib/spack/llnl/util/filesystem.py
@@ -23,7 +23,7 @@
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
__all__ = ['set_install_permissions', 'install', 'expand_user', 'working_dir',
- 'touch', 'mkdirp', 'force_remove', 'join_path', 'ancestor',
+ 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor',
'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe']
import os
@@ -204,6 +204,12 @@ def touch(path):
os.utime(path, None)
+def touchp(path):
+ """Like touch, but creates any parent directories needed for the file."""
+ mkdirp(os.path.dirname(path))
+ touch(path)
+
+
def join_path(prefix, *args):
path = str(prefix)
for elt in args:
diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py
index 2d7126be2c..887f6f4d26 100644
--- a/lib/spack/llnl/util/link_tree.py
+++ b/lib/spack/llnl/util/link_tree.py
@@ -29,108 +29,116 @@
import shutil
from llnl.util.filesystem import mkdirp
+empty_file_name = '.spack-empty'
-class LinkTree(object):
- """Class to create trees of symbolic links from a source directory.
- LinkTree objects are constructed with a source root. Their
- methods allow you to create and delete trees of symbolic links
- back to the source tree in specific destination directories.
- Trees comprise symlinks only to files; directries are never
- symlinked to, to prevent the source directory from ever being
- modified.
+def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
+ """Traverse two filesystem trees simultaneously.
+
+ Walks the LinkTree directory in pre or post order. Yields each
+ file in the source directory with a matching path from the dest
+ directory, along with whether the file is a directory.
+ e.g., for this tree::
+
+ root/
+ a/
+ file1
+ file2
+ b/
+ file3
+
+ When called on dest, this yields::
+
+ ('root', 'dest')
+ ('root/a', 'dest/a')
+ ('root/a/file1', 'dest/a/file1')
+ ('root/a/file2', 'dest/a/file2')
+ ('root/b', 'dest/b')
+ ('root/b/file3', 'dest/b/file3')
+
+ Optional args:
+
+ order=[pre|post] -- Whether to do pre- or post-order traveral.
+
+ ignore=<predicate> -- Predicate indicating which files to ignore.
+
+ follow_nonexisting -- Whether to descend into directories in
+ src that do not exit in dest. Default True.
+
+ follow_links -- Whether to descend into symlinks in src.
"""
- def __init__(self, source_root):
- self._root = source_root
+ follow_nonexisting = kwargs.get('follow_nonexisting', True)
+ follow_links = kwargs.get('follow_link', False)
+ # Yield in pre or post order?
+ order = kwargs.get('order', 'pre')
+ if order not in ('pre', 'post'):
+ raise ValueError("Order must be 'pre' or 'post'.")
- def traverse(self, dest_root, **kwargs):
- """Traverse LinkTree root and dest simultaneously.
+ # List of relative paths to ignore under the src root.
+ ignore = kwargs.get('ignore', lambda filename: False)
- Walks the LinkTree directory in pre or post order. Yields
- each file in the source directory with a matching path from
- the dest directory. e.g., for this tree::
+ # Don't descend into ignored directories
+ if ignore(rel_path):
+ return
- root/
- a/
- file1
- file2
- b/
- file3
+ source_path = os.path.join(source_root, rel_path)
+ dest_path = os.path.join(dest_root, rel_path)
- When called on dest, this yields::
+ # preorder yields directories before children
+ if order == 'pre':
+ yield (source_path, dest_path)
- ('root', 'dest')
- ('root/a', 'dest/a')
- ('root/a/file1', 'dest/a/file1')
- ('root/a/file2', 'dest/a/file2')
- ('root/b', 'dest/b')
- ('root/b/file3', 'dest/b/file3')
+ for f in os.listdir(source_path):
+ source_child = os.path.join(source_path, f)
+ dest_child = os.path.join(dest_path, f)
- Optional args:
+ # Treat as a directory
+ if os.path.isdir(source_child) and (
+ follow_links or not os.path.islink(source_child)):
- order=[pre|post] -- Whether to do pre- or post-order traveral.
+ # When follow_nonexisting isn't set, don't descend into dirs
+ # in source that do not exist in dest
+ if follow_nonexisting or os.path.exists(dest_child):
+ tuples = traverse_tree(source_child, dest_child, rel_path, **kwargs)
+ for t in tuples: yield t
- ignore=<predicate> -- Predicate indicating which files to ignore.
+ # Treat as a file.
+ elif not ignore(os.path.join(rel_path, f)):
+ yield (source_child, dest_child)
+
+ if order == 'post':
+ yield (source_path, dest_path)
- follow_nonexisting -- Whether to descend into directories in
- src that do not exit in dest.
- """
- # Yield directories before or after their contents.
- order = kwargs.get('order', 'pre')
- if order not in ('pre', 'post'):
- raise ValueError("Order must be 'pre' or 'post'.")
-
- # List of relative paths to ignore under the src root.
- ignore = kwargs.get('ignore', lambda filename: False)
-
- # Whether to descend when dirs dont' exist in dest.
- follow_nonexisting = kwargs.get('follow_nonexisting', True)
-
- for dirpath, dirnames, filenames in os.walk(self._root):
- rel_path = dirpath[len(self._root):]
- rel_path = rel_path.lstrip(os.path.sep)
- dest_dirpath = os.path.join(dest_root, rel_path)
-
- # Don't descend into ignored directories
- if ignore(dest_dirpath):
- return
-
- # Don't descend into dirs in dest that do not exist in src.
- if not follow_nonexisting:
- dirnames[:] = [
- d for d in dirnames
- if os.path.exists(os.path.join(dest_dirpath, d))]
-
- # preorder yields directories before children
- if order == 'pre':
- yield (dirpath, dest_dirpath)
-
- for name in filenames:
- src_file = os.path.join(dirpath, name)
- dest_file = os.path.join(dest_dirpath, name)
-
- # Ignore particular paths inside the install root.
- src_relpath = src_file[len(self._root):]
- src_relpath = src_relpath.lstrip(os.path.sep)
- if ignore(src_relpath):
- continue
- yield (src_file, dest_file)
+class LinkTree(object):
+ """Class to create trees of symbolic links from a source directory.
- # postorder yields directories after children
- if order == 'post':
- yield (dirpath, dest_dirpath)
+ LinkTree objects are constructed with a source root. Their
+ methods allow you to create and delete trees of symbolic links
+ back to the source tree in specific destination directories.
+ Trees comprise symlinks only to files; directries are never
+ symlinked to, to prevent the source directory from ever being
+ modified.
+
+ """
+ def __init__(self, source_root):
+ if not os.path.exists(source_root):
+ raise IOError("No such file or directory: '%s'", source_root)
+ self._root = source_root
def find_conflict(self, dest_root, **kwargs):
- """Returns the first file in dest that also exists in src."""
+ """Returns the first file in dest that conflicts with src"""
kwargs['follow_nonexisting'] = False
- for src, dest in self.traverse(dest_root, **kwargs):
- if os.path.exists(dest) and not os.path.isdir(dest):
+ for src, dest in traverse_tree(self._root, dest_root, **kwargs):
+ if os.path.isdir(src):
+ if os.path.exists(dest) and not os.path.isdir(dest):
+ return dest
+ elif os.path.exists(dest):
return dest
return None
@@ -138,9 +146,20 @@ def find_conflict(self, dest_root, **kwargs):
def merge(self, dest_root, **kwargs):
"""Link all files in src into dest, creating directories if necessary."""
kwargs['order'] = 'pre'
- for src, dest in self.traverse(dest_root, **kwargs):
+ for src, dest in traverse_tree(self._root, dest_root, **kwargs):
if os.path.isdir(src):
- mkdirp(dest)
+ if not os.path.exists(dest):
+ mkdirp(dest)
+ continue
+
+ if not os.path.isdir(dest):
+ raise ValueError("File blocks directory: %s" % dest)
+
+ # mark empty directories so they aren't removed on unmerge.
+ if not os.listdir(dest):
+ marker = os.path.join(dest, empty_file_name)
+ touch(marker)
+
else:
assert(not os.path.exists(dest))
os.symlink(src, dest)
@@ -153,12 +172,20 @@ def unmerge(self, dest_root, **kwargs):
"""
kwargs['order'] = 'post'
- for src, dest in self.traverse(dest_root, **kwargs):
- if os.path.isdir(dest):
+ for src, dest in traverse_tree(self._root, dest_root, **kwargs):
+ if os.path.isdir(src):
+ if not os.path.isdir(dest):
+ raise ValueError("File blocks directory: %s" % dest)
+
+ # remove directory if it is empty.
if not os.listdir(dest):
- # TODO: what if empty directories were present pre-merge?
shutil.rmtree(dest, ignore_errors=True)
+ # remove empty dir marker if present.
+ marker = os.path.join(dest, empty_file_name)
+ if os.path.exists(marker):
+ os.remove(marker)
+
elif os.path.exists(dest):
if not os.path.islink(dest):
raise ValueError("%s is not a link tree!" % dest)
diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py
index 0eda667abc..c53e6774fc 100644
--- a/lib/spack/spack/test/__init__.py
+++ b/lib/spack/spack/test/__init__.py
@@ -51,7 +51,8 @@
'hg_fetch',
'mirror',
'url_extrapolate',
- 'cc']
+ 'cc',
+ 'link_tree']
def list_tests():
diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py
new file mode 100644
index 0000000000..bc7c2c6b5e
--- /dev/null
+++ b/lib/spack/spack/test/link_tree.py
@@ -0,0 +1,153 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import os
+import unittest
+import shutil
+import tempfile
+from contextlib import closing
+
+from llnl.util.filesystem import *
+from llnl.util.link_tree import LinkTree
+
+from spack.stage import Stage
+
+
+class LinkTreeTest(unittest.TestCase):
+ """Tests Spack's LinkTree class."""
+
+ def setUp(self):
+ self.stage = Stage('link-tree-test')
+
+ with working_dir(self.stage.path):
+ touchp('source/1')
+ touchp('source/a/b/2')
+ touchp('source/a/b/3')
+ touchp('source/c/4')
+ touchp('source/c/d/5')
+ touchp('source/c/d/6')
+ touchp('source/c/d/e/7')
+
+ source_path = os.path.join(self.stage.path, 'source')
+ self.link_tree = LinkTree(source_path)
+
+
+ def tearDown(self):
+ if self.stage:
+ self.stage.destroy()
+
+
+ def check_file_link(self, filename):
+ self.assertTrue(os.path.isfile(filename))
+ self.assertTrue(os.path.islink(filename))
+
+
+ def check_dir(self, filename):
+ self.assertTrue(os.path.isdir(filename))
+
+
+ def test_merge_to_new_directory(self):
+ with working_dir(self.stage.path):
+ self.link_tree.merge('dest')
+
+ self.check_file_link('dest/1')
+ self.check_file_link('dest/a/b/2')
+ self.check_file_link('dest/a/b/3')
+ self.check_file_link('dest/c/4')
+ self.check_file_link('dest/c/d/5')
+ self.check_file_link('dest/c/d/6')
+ self.check_file_link('dest/c/d/e/7')
+
+ self.link_tree.unmerge('dest')
+
+ self.assertFalse(os.path.exists('dest'))
+
+
+ def test_merge_to_existing_directory(self):
+ with working_dir(self.stage.path):
+
+ touchp('dest/x')
+ touchp('dest/a/b/y')
+
+ self.link_tree.merge('dest')
+
+ self.check_file_link('dest/1')
+ self.check_file_link('dest/a/b/2')
+ self.check_file_link('dest/a/b/3')
+ self.check_file_link('dest/c/4')
+ self.check_file_link('dest/c/d/5')
+ self.check_file_link('dest/c/d/6')
+ self.check_file_link('dest/c/d/e/7')
+
+ self.assertTrue(os.path.isfile('dest/x'))
+ self.assertTrue(os.path.isfile('dest/a/b/y'))
+
+ self.link_tree.unmerge('dest')
+
+ self.assertTrue(os.path.isfile('dest/x'))
+ self.assertTrue(os.path.isfile('dest/a/b/y'))
+
+ self.assertFalse(os.path.isfile('dest/1'))
+ self.assertFalse(os.path.isfile('dest/a/b/2'))
+ self.assertFalse(os.path.isfile('dest/a/b/3'))
+ self.assertFalse(os.path.isfile('dest/c/4'))
+ self.assertFalse(os.path.isfile('dest/c/d/5'))
+ self.assertFalse(os.path.isfile('dest/c/d/6'))
+ self.assertFalse(os.path.isfile('dest/c/d/e/7'))
+
+
+ def test_merge_with_empty_directories(self):
+ with working_dir(self.stage.path):
+ mkdirp('dest/f/g')
+ mkdirp('dest/a/b/h')
+
+ self.link_tree.merge('dest')
+ self.link_tree.unmerge('dest')
+
+ self.assertFalse(os.path.exists('dest/1'))
+ self.assertFalse(os.path.exists('dest/a/b/2'))
+ self.assertFalse(os.path.exists('dest/a/b/3'))
+ self.assertFalse(os.path.exists('dest/c/4'))
+ self.assertFalse(os.path.exists('dest/c/d/5'))
+ self.assertFalse(os.path.exists('dest/c/d/6'))
+ self.assertFalse(os.path.exists('dest/c/d/e/7'))
+
+ self.assertTrue(os.path.isdir('dest/a/b/h'))
+ self.assertTrue(os.path.isdir('dest/f/g'))
+
+
+ def test_ignore(self):
+ with working_dir(self.stage.path):
+ touchp('source/.spec')
+ touchp('dest/.spec')
+
+ self.link_tree.merge('dest', ignore=lambda x: x == '.spec')
+ self.link_tree.unmerge('dest', ignore=lambda x: x == '.spec')
+
+ self.assertFalse(os.path.exists('dest/1'))
+ self.assertFalse(os.path.exists('dest/a'))
+ self.assertFalse(os.path.exists('dest/c'))
+
+ self.assertTrue(os.path.isfile('source/.spec'))
+ self.assertTrue(os.path.isfile('dest/.spec'))
--
GitLab