diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py
index cc87b7eac2bfe85332671806de5b83ef0917da67..c84828951d45521a91c02385f454dc5df405e583 100644
--- a/lib/spack/llnl/util/lang.py
+++ b/lib/spack/llnl/util/lang.py
@@ -112,7 +112,7 @@ def partition_list(elements, predicate):
def caller_locals():
"""This will return the locals of the *parent* of the caller.
- This allows a fucntion to insert variables into its caller's
+ This allows a function to insert variables into its caller's
scope. Yes, this is some black magic, and yes it's useful
for implementing things like depends_on and provides.
"""
diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py
index 2a818e8d0c76838867127bbae5e5c61e70b4d632..aa9fbd8d33958682fa847e68562c768e1ce62989 100644
--- a/lib/spack/spack/directives.py
+++ b/lib/spack/spack/directives.py
@@ -42,15 +42,19 @@ class OpenMpi(Package):
* ``extends``
* ``patch``
* ``variant``
+ * ``resource``
"""
-__all__ = [ 'depends_on', 'extends', 'provides', 'patch', 'version',
- 'variant' ]
+__all__ = ['depends_on', 'extends', 'provides', 'patch', 'version',
+ 'variant', 'resource']
import re
import inspect
+import os.path
+import functools
from llnl.util.lang import *
+from llnl.util.filesystem import join_path
import spack
import spack.spec
@@ -60,7 +64,8 @@ class OpenMpi(Package):
from spack.patch import Patch
from spack.variant import Variant
from spack.spec import Spec, parse_anonymous_spec
-
+from spack.resource import Resource
+from spack.fetch_strategy import from_kwargs
#
# This is a list of all directives, built up as they are defined in
@@ -79,8 +84,8 @@ class directive(object):
"""Decorator for Spack directives.
Spack directives allow you to modify a package while it is being
- defined, e.g. to add version or depenency information. Directives
- are one of the key pieces of Spack's package "langauge", which is
+ defined, e.g. to add version or dependency information. Directives
+ are one of the key pieces of Spack's package "language", which is
embedded in python.
Here's an example directive:
@@ -141,6 +146,7 @@ def ensure_dicts(self, pkg):
def __call__(self, directive_function):
directives[directive_function.__name__] = self
+ @functools.wraps(directive_function)
def wrapped(*args, **kwargs):
pkg = DictWrapper(caller_locals())
self.ensure_dicts(pkg)
@@ -259,6 +265,43 @@ def variant(pkg, name, default=False, description=""):
pkg.variants[name] = Variant(default, description)
+@directive('resources')
+def resource(pkg, **kwargs):
+ """
+ Define an external resource to be fetched and staged when building the package. Based on the keywords present in the
+ dictionary the appropriate FetchStrategy will be used for the resource. Resources are fetched and staged in their
+ own folder inside spack stage area, and then linked into the stage area of the package that needs them.
+
+ List of recognized keywords:
+
+ * 'when' : (optional) represents the condition upon which the resource is needed
+ * 'destination' : (optional) path where to link the resource. This path must be relative to the main package stage
+ area.
+ * 'placement' : (optional) gives the possibility to fine tune how the resource is linked into the main package stage
+ area.
+ """
+ when = kwargs.get('when', pkg.name)
+ destination = kwargs.get('destination', "")
+ placement = kwargs.get('placement', None)
+ # Check if the path is relative
+ if os.path.isabs(destination):
+ message = "The destination keyword of a resource directive can't be an absolute path.\n"
+ message += "\tdestination : '{dest}\n'".format(dest=destination)
+ raise RuntimeError(message)
+ # Check if the path falls within the main package stage area
+ test_path = 'stage_folder_root/'
+ normalized_destination = os.path.normpath(join_path(test_path, destination)) # Normalized absolute path
+ if test_path not in normalized_destination:
+ message = "The destination folder of a resource must fall within the main package stage directory.\n"
+ message += "\tdestination : '{dest}'\n".format(dest=destination)
+ raise RuntimeError(message)
+ when_spec = parse_anonymous_spec(when, pkg.name)
+ resources = pkg.resources.setdefault(when_spec, [])
+ fetcher = from_kwargs(**kwargs)
+ name = kwargs.get('name')
+ resources.append(Resource(name, fetcher, destination, placement))
+
+
class DirectiveError(spack.error.SpackError):
"""This is raised when something is wrong with a package directive."""
def __init__(self, directive, message):
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
index 8c4bd4ed8a9895c5d9530da4c3fa9bea75f7cc97..a9374fb34b99ba460487fbf2259bc9fa7bd993b3 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -634,6 +634,22 @@ def from_url(url):
return URLFetchStrategy(url)
+def from_kwargs(**kwargs):
+ """
+ Construct the appropriate FetchStrategy from the given keyword arguments.
+
+ :param kwargs: dictionary of keyword arguments
+ :return: fetcher or raise a FetchError exception
+ """
+ for fetcher in all_strategies:
+ if fetcher.matches(kwargs):
+ return fetcher(**kwargs)
+ # Raise an error in case we can't instantiate any known strategy
+ message = "Cannot instantiate any FetchStrategy"
+ long_message = message + " from the given arguments : {arguments}".format(srguments=kwargs)
+ raise FetchError(message, long_message)
+
+
def args_are_for(args, fetcher):
fetcher.matches(args)
diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py
index 6fbf82de14328e9c03ccd3fa81cecdf9326df325..1d9b0e7ef25496cd4afeabff6cd57a38c535c239 100644
--- a/lib/spack/spack/mirror.py
+++ b/lib/spack/spack/mirror.py
@@ -26,7 +26,7 @@
This file contains code for creating spack mirror directories. A
mirror is an organized hierarchy containing specially named archive
files. This enabled spack to know where to find files in a mirror if
-the main server for a particualr package is down. Or, if the computer
+the main server for a particular package is down. Or, if the computer
where spack is run is not connected to the internet, it allows spack
to download packages directly from a mirror (e.g., on an intranet).
"""
@@ -42,7 +42,7 @@
from spack.spec import Spec
from spack.stage import Stage
from spack.version import *
-from spack.util.compression import extension
+from spack.util.compression import extension, allowed_archive
def mirror_archive_filename(spec):
@@ -87,11 +87,26 @@ def get_matching_versions(specs, **kwargs):
if v.satisfies(spec.versions):
s = Spec(pkg.name)
s.versions = VersionList([v])
+ s.variants = spec.variants.copy()
matching.append(s)
return matching
+def suggest_archive_basename(resource):
+ """
+ Return a tentative basename for an archive. Raise an exception if the name is among the allowed archive types.
+
+ :param fetcher:
+ :return:
+ """
+ basename = os.path.basename(resource.fetcher.url)
+ if not allowed_archive(basename):
+ raise RuntimeError("%s is not an allowed archive tye" % basename)
+ return basename
+
+
+
def create(path, specs, **kwargs):
"""Create a directory to be used as a spack mirror, and fill it with
package archives.
@@ -108,7 +123,7 @@ def create(path, specs, **kwargs):
Return Value:
Returns a tuple of lists: (present, mirrored, error)
- * present: Package specs that were already prsent.
+ * present: Package specs that were already present.
* mirrored: Package specs that were successfully mirrored.
* error: Package specs that failed to mirror due to some error.
@@ -140,6 +155,7 @@ def create(path, specs, **kwargs):
error = []
# Iterate through packages and download all the safe tarballs for each of them
+ everything_already_exists = True
for spec in version_specs:
pkg = spec.package
@@ -152,26 +168,47 @@ def create(path, specs, **kwargs):
if os.path.exists(archive_path):
tty.msg("Already added %s" % spec.format("$_$@"))
+ else:
+ everything_already_exists = False
+ # Set up a stage and a fetcher for the download
+ unique_fetch_name = spec.format("$_$@")
+ fetcher = fs.for_package_version(pkg, pkg.version)
+ stage = Stage(fetcher, name=unique_fetch_name)
+ fetcher.set_stage(stage)
+
+ # Do the fetch and checksum if necessary
+ fetcher.fetch()
+ if not kwargs.get('no_checksum', False):
+ fetcher.check()
+ tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
+
+ # Fetchers have to know how to archive their files. Use
+ # that to move/copy/create an archive in the mirror.
+ fetcher.archive(archive_path)
+ tty.msg("Added %s." % spec.format("$_$@"))
+
+ # Fetch resources if they are associated with the spec
+ resources = pkg._get_resources()
+ for resource in resources:
+ resource_archive_path = join_path(subdir, suggest_archive_basename(resource))
+ if os.path.exists(resource_archive_path):
+ tty.msg("Already added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
+ continue
+ everything_already_exists = False
+ resource_stage_folder = pkg._resource_stage(resource)
+ resource_stage = Stage(resource.fetcher, name=resource_stage_folder)
+ resource.fetcher.set_stage(resource_stage)
+ resource.fetcher.fetch()
+ if not kwargs.get('no_checksum', False):
+ resource.fetcher.check()
+ tty.msg("Checksum passed for the resource %s (%s@%s)" % (resource.name, pkg.name, pkg.version))
+ resource.fetcher.archive(resource_archive_path)
+ tty.msg("Added resource %s (%s@%s)." % (resource.name, pkg.name, pkg.version))
+
+ if everything_already_exists:
present.append(spec)
- continue
-
- # Set up a stage and a fetcher for the download
- unique_fetch_name = spec.format("$_$@")
- fetcher = fs.for_package_version(pkg, pkg.version)
- stage = Stage(fetcher, name=unique_fetch_name)
- fetcher.set_stage(stage)
-
- # Do the fetch and checksum if necessary
- fetcher.fetch()
- if not kwargs.get('no_checksum', False):
- fetcher.check()
- tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
-
- # Fetchers have to know how to archive their files. Use
- # that to move/copy/create an archive in the mirror.
- fetcher.archive(archive_path)
- tty.msg("Added %s." % spec.format("$_$@"))
- mirrored.append(spec)
+ else:
+ mirrored.append(spec)
except Exception, e:
if spack.debug:
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
index 4d75726e06ec28a2f717cb381c0fed3d062cd284..b95afb073dfe21b04d6f696fc99ed556ba6d593b 100644
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -655,26 +655,62 @@ def do_fetch(self):
"Will not fetch %s." % self.spec.format('$_$@'), checksum_msg)
self.stage.fetch()
+
+ ##########
+ # Fetch resources
+ resources = self._get_resources()
+ for resource in resources:
+ resource_stage_folder = self._resource_stage(resource)
+ # FIXME : works only for URLFetchStrategy
+ resource_mirror = join_path(self.name, os.path.basename(resource.fetcher.url))
+ resource_stage = Stage(resource.fetcher, name=resource_stage_folder, mirror_path=resource_mirror)
+ resource.fetcher.set_stage(resource_stage)
+ # Delegate to stage object to trigger mirror logic
+ resource_stage.fetch()
+ resource_stage.check()
+ ##########
+
self._fetch_time = time.time() - start_time
if spack.do_checksum and self.version in self.versions:
self.stage.check()
-
def do_stage(self):
"""Unpacks the fetched tarball, then changes into the expanded tarball
directory."""
if not self.spec.concrete:
raise ValueError("Can only stage concrete packages.")
- self.do_fetch()
+ def _expand_archive(stage, name=self.name):
+ archive_dir = stage.source_path
+ if not archive_dir:
+ stage.expand_archive()
+ tty.msg("Created stage in %s." % stage.path)
+ else:
+ tty.msg("Already staged %s in %s." % (name, stage.path))
- archive_dir = self.stage.source_path
- if not archive_dir:
- self.stage.expand_archive()
- tty.msg("Created stage in %s." % self.stage.path)
- else:
- tty.msg("Already staged %s in %s." % (self.name, self.stage.path))
+
+ self.do_fetch()
+ _expand_archive(self.stage)
+
+ ##########
+ # Stage resources in appropriate path
+ resources = self._get_resources()
+ for resource in resources:
+ stage = resource.fetcher.stage
+ _expand_archive(stage, resource.name)
+ # Turn placement into a dict with relative paths
+ placement = os.path.basename(stage.source_path) if resource.placement is None else resource.placement
+ if not isinstance(placement, dict):
+ placement = {'': placement}
+ # Make the paths in the dictionary absolute and link
+ for key, value in placement.iteritems():
+ link_path = join_path(self.stage.source_path, resource.destination, value)
+ source_path = join_path(stage.source_path, key)
+ if not os.path.exists(link_path):
+ # Create a symlink
+ os.symlink(source_path, link_path)
+ ##########
self.stage.chdir_to_source()
@@ -746,6 +782,19 @@ def do_fake_install(self):
mkdirp(self.prefix.man1)
+ def _get_resources(self):
+ resources = []
+ # Select the resources that are needed for this build
+ for when_spec, resource_list in self.resources.items():
+ if when_spec in self.spec:
+ resources.extend(resource_list)
+ return resources
+
+ def _resource_stage(self, resource):
+ pieces = ['resource', resource.name, self.spec.dag_hash()]
+ resource_stage_folder = '-'.join(pieces)
+ return resource_stage_folder
+
def _build_logger(self, log_path):
"""Create a context manager to log build output."""
diff --git a/lib/spack/spack/resource.py b/lib/spack/spack/resource.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d081b45c9625c7511fe9f446811bace5b328fc9
--- /dev/null
+++ b/lib/spack/spack/resource.py
@@ -0,0 +1,39 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""
+Describes an optional resource needed for a build. Typically a bunch of sources that can be built in-tree within another
+package to enable optional features.
+"""
+
+
+class Resource(object):
+ """
+ Represents an optional resource. Aggregates a name, a fetcher, a destination and a placement
+ """
+ def __init__(self, name, fetcher, destination, placement):
+ self.name = name
+ self.fetcher = fetcher
+ self.destination = destination
+ self.placement = placement
diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py
index 20a5ac2c940a101b4ae297715af33270ef64337a..e46e08d5f11b7609ee5835d37f105fd07aa5a6a8 100644
--- a/var/spack/packages/clang/package.py
+++ b/var/spack/packages/clang/package.py
@@ -22,8 +22,13 @@
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
+
+
from spack import *
+import os
+import os.path
+
class Clang(Package):
"""The goal of the Clang project is to create a new C, C++,
Objective C and Objective C++ front-end for the LLVM compiler.
@@ -39,13 +44,52 @@ class Clang(Package):
version('3.6.2', 'ff862793682f714bb7862325b9c06e20', url='http://llvm.org/releases/3.6.2/cfe-3.6.2.src.tar.xz')
version('3.5.1', '93f9532f8f7e6f1d8e5c1116907051cb', url='http://llvm.org/releases/3.5.1/cfe-3.5.1.src.tar.xz')
+ ##########
+ # @3.7.0
+ resource(name='clang-tools-extra',
+ url='http://llvm.org/releases/3.7.0/clang-tools-extra-3.7.0.src.tar.xz',
+ md5='d5a87dacb65d981a427a536f6964642e', destination='tools', when='@3.7.0')
+ ##########
+
def install(self, spec, prefix):
env['CXXFLAGS'] = self.compiler.cxx11_flag
with working_dir('spack-build', create=True):
+
+ options = []
+ if '@3.7.0:' in spec:
+ options.append('-DCLANG_DEFAULT_OPENMP_RUNTIME:STRING=libomp')
+ options.extend(std_cmake_args)
+
cmake('..',
- '-DCLANG_PATH_TO_LLVM_BUILD=%s' % spec['llvm'].prefix,
- '-DLLVM_MAIN_SRC_DIR=%s' % spec['llvm'].prefix,
- *std_cmake_args)
+ '-DCLANG_PATH_TO_LLVM_BUILD:PATH=%s' % spec['llvm'].prefix,
+ '-DLLVM_MAIN_SRC_DIR:PATH=%s' % spec['llvm'].prefix,
+ *options)
make()
make("install")
+ # CLang doesn't look in llvm folders for system headers...
+ self.link_llvm_directories(spec)
+
+ def link_llvm_directories(self, spec):
+
+ def clang_include_dir_at(root):
+ return join_path(root, 'include')
+
+ def clang_lib_dir_at(root):
+ return join_path(root, 'lib/clang/', str(self.version), 'include')
+
+ def do_link(source_dir, destination_dir):
+ if os.path.exists(source_dir):
+ for name in os.listdir(source_dir):
+ source = join_path(source_dir, name)
+ link = join_path(destination_dir, name)
+ os.symlink(source, link)
+
+ # Link folder and files in include
+ llvm_dir = clang_include_dir_at(spec['llvm'].prefix)
+ clang_dir = clang_include_dir_at(self.prefix)
+ do_link(llvm_dir, clang_dir)
+ # Link folder and files in lib
+ llvm_dir = clang_lib_dir_at(spec['llvm'].prefix)
+ clang_dir = clang_lib_dir_at(self.prefix)
+ do_link(llvm_dir, clang_dir)
\ No newline at end of file
diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py
index b3ca488809af329513d65adc9255b1ac14c34f09..a3307584e08b2b5bc57fe98af798ca375d963579 100644
--- a/var/spack/packages/llvm/package.py
+++ b/var/spack/packages/llvm/package.py
@@ -42,13 +42,31 @@ class Llvm(Package):
depends_on('python@2.7:')
+ variant('libcxx', default=False, description="Builds the LLVM Standard C++ library targeting C++11")
+
+ ##########
+ # @3.7.0
+ resource(name='compiler-rt',
+ url='http://llvm.org/releases/3.7.0/compiler-rt-3.7.0.src.tar.xz', md5='383c10affd513026f08936b5525523f5',
+ destination='projects', when='@3.7.0')
+ resource(name='openmp',
+ url='http://llvm.org/releases/3.7.0/openmp-3.7.0.src.tar.xz', md5='f482c86fdead50ba246a1a2b0bbf206f',
+ destination='projects', when='@3.7.0')
+ resource(name='libcxx',
+ url='http://llvm.org/releases/3.7.0/libcxx-3.7.0.src.tar.xz', md5='46aa5175cbe1ad42d6e9c995968e56dd',
+ destination='projects', placement='libcxx', when='+libcxx@3.7.0')
+ resource(name='libcxxabi',
+ url='http://llvm.org/releases/3.7.0/libcxxabi-3.7.0.src.tar.xz', md5='5aa769e2fca79fa5335cfae8f6258772',
+ destination='projects', placement='libcxxabi', when='+libcxx@3.7.0')
+ ##########
+
def install(self, spec, prefix):
env['CXXFLAGS'] = self.compiler.cxx11_flag
with working_dir('spack-build', create=True):
cmake('..',
- '-DLLVM_REQUIRES_RTTI=1',
- '-DPYTHON_EXECUTABLE=%s/bin/python' % spec['python'].prefix,
+ '-DLLVM_REQUIRES_RTTI:BOOL=ON',
+ '-DPYTHON_EXECUTABLE:PATH=%s/bin/python' % spec['python'].prefix,
*std_cmake_args)
make()
make("install")