cc: package search paths come before dependency paths (#4692)

Spack currently prepends include paths, library paths, and rpaths to the compile line.  This causes problems when a header or library in the package has the same name as one exported by one of its dependencies.  The *dependency's* header will be preferred over the package's, which is not what most builds expect.  This also breaks some of our production codes.

This restores the original cc behavior (from *very* early Spack) of parsing compiler arguments out by type (`-L`, `-I`, `-Wl,-rpath`) and reconstituting the full command at the end.

`<includes> <other_args> <library dirs> <rpaths>`

This differs from the original behavior in one significant way, though: it *appends* the library arguments so that dependency libraries do not shadow those in the build. 

This is safe because semantics aren't affected by *interleaving* `-I`, `-L`, and `-Wl,-rpath` arguments with others, only with each other (so the order fo two `-L` args affects the search path, but we search for all libraries on the command line using the same search path).

We preserve the following:
1. Any system directory in the paths will be listed last.
2. The root package's include/library/RPATH flags come before flags of the same type for any dependency.
3. Order will be preserved within flags passed by the build (except system paths, which are moved to be last)
4. Flags for dependencies will appear between the root flags and the system flags, and the flags for any dependency will come before those for *its* dependencies (this is for completeness -- we already guarantee this in `build_environment.py`)

lib/spack/env/cc

@@ -52,6 +52,7 @@ parameters=(
     SPACK_F77_RPATH_ARG
     SPACK_FC_RPATH_ARG
     SPACK_SHORT_SPEC
+    SPACK_SYSTEM_DIRS
 )
 
 # The compiler input variables are checked for sanity later:
@@ -228,7 +229,92 @@ fi
 
 # Save original command for debug logging
 input_command="$@"
-args=("$@")
+args=()
+
+#
+# Parse the command line args, trying hard to keep
+# non-rpath linker arguments in the proper order w.r.t. other command
+# line arguments.  This is important for things like groups.
+#
+# -l arguments are treated as 'other_args' to ensure that they stay in
+# any groups they are a part of. Dependency library -l statements are
+# categorized as 'libs'
+#
+# The various categories will be recombined with compiler flags into 
+# args variable later.
+#
+includes=()
+libdirs=()
+libs=()
+rpaths=()
+other_args=()
+
+while [ -n "$1" ]; do
+    case "$1" in
+        -I*)
+            arg="${1#-I}"
+            if [ -z "$arg" ]; then shift; arg="$1"; fi
+            includes+=("$arg")
+            ;;
+        -L*)
+            arg="${1#-L}"
+            if [ -z "$arg" ]; then shift; arg="$1"; fi
+            libdirs+=("$arg")
+            ;;
+        -l*)
+            arg="${1#-l}"
+            if [ -z "$arg" ]; then shift; arg="$1"; fi
+            other_args+=("-l$arg")
+            ;;
+        -Wl,*)
+            arg="${1#-Wl,}"
+            if [ -z "$arg" ]; then shift; arg="$1"; fi
+            if [[ "$arg" = -rpath=* ]]; then
+                rpaths+=("${arg#-rpath=}")
+	    elif [[ "$arg" = -rpath,* ]]; then
+		rpaths+=("${arg#-rpath,}")
+            elif [[ "$arg" = -rpath ]]; then
+                shift; arg="$1"
+                if [[ "$arg" != -Wl,* ]]; then
+                    die "-Wl,-rpath was not followed by -Wl,*"
+                fi
+                rpaths+=("${arg#-Wl,}")
+            else
+                other_args+=("-Wl,$arg")
+            fi
+            ;;
+        -Xlinker,*)
+            arg="${1#-Xlinker,}"
+            if [ -z "$arg" ]; then shift; arg="$1"; fi
+            if [[ "$arg" = -rpath=* ]]; then
+                rpaths+=("${arg#-rpath=}")
+            elif [[ "$arg" = -rpath ]]; then
+                shift; arg="$1"
+                if [[ "$arg" != -Xlinker,* ]]; then
+                    die "-Xlinker,-rpath was not followed by -Xlinker,*"
+                fi
+                rpaths+=("${arg#-Xlinker,}")
+            else
+                other_args+=("-Xlinker,$arg")
+            fi
+            ;;
+        -Xlinker)
+            if [[ "$2" == "-rpath" ]]; then
+		if [[ "$3" != "-Xlinker" ]]; then
+                    die "-Xlinker,-rpath was not followed by -Xlinker,*"
+		fi
+		shift 3;
+		rpaths+=("$1")
+            else
+                other_args+=("$1")
+            fi
+            ;;
+        *)
+            other_args+=("$1")
+            ;;
+    esac
+    shift
+done
 
 # Prepend cppflags, cflags, cxxflags, fcflags, fflags, and ldflags
 
@@ -266,92 +352,93 @@ case "$mode" in cc|ccld)
         ;;
 esac
 
+# Include all -L's and prefix/whatever dirs in rpath
+$add_rpaths && rpaths+=("$SPACK_PREFIX/lib")
+$add_rpaths && rpaths+=("$SPACK_PREFIX/lib64")
+
 # Read spack dependencies from the path environment variable
 IFS=':' read -ra deps <<< "$SPACK_DEPENDENCIES"
 for dep in "${deps[@]}"; do
-    # Prepend include directories
+    # Append include directories
     if [[ -d $dep/include ]]; then
         if [[ $mode == cpp || $mode == cc || $mode == as || $mode == ccld ]]; then
-            args=("-I$dep/include" "${args[@]}")
+            includes=("${includes[@]}" "$dep/include")
         fi
     fi
 
-    # Prepend lib and RPATH directories
+    # Append lib and RPATH directories
     if [[ -d $dep/lib ]]; then
-        if [[ $mode == ccld ]]; then
-            if [[ $SPACK_RPATH_DEPS == *$dep* ]]; then
-                $add_rpaths && args=("$rpath$dep/lib" "${args[@]}")
-            fi
-            if [[ $SPACK_LINK_DEPS == *$dep* ]]; then
-                args=("-L$dep/lib" "${args[@]}")
-            fi
-        elif [[ $mode == ld ]]; then
-            if [[ $SPACK_RPATH_DEPS == *$dep* ]]; then
-                $add_rpaths && args=("-rpath" "$dep/lib" "${args[@]}")
-            fi
-            if [[ $SPACK_LINK_DEPS == *$dep* ]]; then
-                args=("-L$dep/lib" "${args[@]}")
-            fi
+        if [[ $SPACK_RPATH_DEPS == *$dep* ]]; then
+            $add_rpaths && rpaths=("${rpaths[@]}" "$dep/lib")
+        fi
+        if [[ $SPACK_LINK_DEPS == *$dep* ]]; then
+            libdirs=("${libdirs[@]}" "$dep/lib")
         fi
     fi
 
-    # Prepend lib64 and RPATH directories
+    # Append lib64 and RPATH directories
     if [[ -d $dep/lib64 ]]; then
-        if [[ $mode == ccld ]]; then
-            if [[ $SPACK_RPATH_DEPS == *$dep* ]]; then
-                $add_rpaths && args=("$rpath$dep/lib64" "${args[@]}")
-            fi
-            if [[ $SPACK_LINK_DEPS == *$dep* ]]; then
-                args=("-L$dep/lib64" "${args[@]}")
-            fi
-        elif [[ $mode == ld ]]; then
-            if [[ $SPACK_RPATH_DEPS == *$dep* ]]; then
-                $add_rpaths && args=("-rpath" "$dep/lib64" "${args[@]}")
-            fi
-            if [[ $SPACK_LINK_DEPS == *$dep* ]]; then
-                args=("-L$dep/lib64" "${args[@]}")
-            fi
+        if [[ $SPACK_RPATH_DEPS == *$dep* ]]; then
+            $add_rpaths && rpaths+=("$dep/lib64")
+        fi
+        if [[ $SPACK_LINK_DEPS == *$dep* ]]; then
+            libdirs+=("$dep/lib64")
         fi
     fi
 done
 
-# Include all -L's and prefix/whatever dirs in rpath
-if [[ $mode == ccld ]]; then
-    $add_rpaths && args=("$rpath$SPACK_PREFIX/lib64" "${args[@]}")
-    $add_rpaths && args=("$rpath$SPACK_PREFIX/lib"   "${args[@]}")
-elif [[ $mode == ld ]]; then
-    $add_rpaths && args=("-rpath" "$SPACK_PREFIX/lib64" "${args[@]}")
-    $add_rpaths && args=("-rpath" "$SPACK_PREFIX/lib"   "${args[@]}")
-fi
-
 # Set extra RPATHs
 IFS=':' read -ra extra_rpaths <<< "$SPACK_COMPILER_EXTRA_RPATHS"
-for extra_rpath in "${extra_rpaths[@]}"; do
-    if [[ $mode == ccld ]]; then
-        $add_rpaths && args=("$rpath$extra_rpath" "${args[@]}")
-        args=("-L$extra_rpath" "${args[@]}")
-    elif [[ $mode == ld ]]; then
-        $add_rpaths && args=("-rpath" "$extra_rpath" "${args[@]}")
-        args=("-L$extra_rpath" "${args[@]}")
-    fi
+for extra_rpath in "${extra_rpaths[@]}"; do 
+    $add_rpaths && rpaths+=("$extra_rpath")
+    libdirs+=("$extra_rpath")
 done
 
 # Add SPACK_LDLIBS to args
 case "$mode" in
     ld|ccld)
-        args=("${args[@]}" ${SPACK_LDLIBS[@]}) ;;
+	for lib in ${SPACK_LDLIBS[@]}; do
+	    libs+=("${lib#-l}")
+	done
 esac
 
-#ccache only supports C languages, so filtering out Fortran
-if [[ ( ${lang_flags} = "C" || ${lang_flags} = "CXX" ) && ${SPACK_CCACHE_BINARY} ]]; then
-    full_command=("${SPACK_CCACHE_BINARY}" "$command" "${args[@]}")
-    # #3761#issuecomment-294352232
-    # workaround for stage being a temp folder
-    export CCACHE_NOHASHDIR=yes
-else
-   full_command=("$command" "${args[@]}")
+# Filter system locations to the end of each sublist of args
+# (includes, library dirs, rpaths)
+for sd in ${SPACK_SYSTEM_DIRS[@]}; do
+    stripped_includes=`echo $includes | sed "s#\b$sd/\? \b##g"`
+    stripped_libdirs=`echo $libdirs | sed "s#\b$sd/\? \b##g"`
+    stripped_rpaths=`echo $rpaths | sed "s#\b$sd/\? \b##g"`
+    if [[ "$includes" != "$stripped_includes" ]]; then
+	$includes="$stripped_includes $sd"
+    fi
+    if [[ "$libdirs" != "$stripped_libdirs" ]]; then
+	$libdirs="$stripped_libdirs $sd"
+    fi
+    if [[ "$rpaths" != "$stripped_rpaths" ]]; then
+	$rpaths="$stripped_rpaths $sd"
+    fi
+done
+
+# Put the arguments together into one list
+# Includes come first, then other args, library dirs, and rpaths
+# rpaths get appropriate flag for ld vs ccld mode
+for dir in "${includes[@]}";  do args+=("-I$dir"); done
+args+=("${other_args[@]}")
+for dir in "${libdirs[@]}"; do args+=("-L$dir"); done
+for lib in "${libs[@]}"; do args+=("-l$lib"); done
+if [ "$mode" = ccld ]; then
+    for dir in "${rpaths[@]}"; do
+        args+=("$rpath$dir")
+    done
+elif [ "$mode" = ld ]; then
+    for dir in "${rpaths[@]}"; do
+        args+=("-rpath" "$dir")
+    done
 fi
 
+full_command=("$command")
+full_command+=("${args[@]}")
+
 # In test command mode, write out full command for Spack tests.
 if [[ $SPACK_TEST_COMMAND == dump-args ]]; then
     echo "${full_command[@]}"
@@ -370,4 +457,4 @@ if [[ $SPACK_DEBUG == TRUE ]]; then
     echo "[$mode] ${full_command[@]}" >> "$output_log"
 fi
 
-exec "${full_command[@]}"
+exec "${full_command[@]}" 

lib/spack/spack/build_environment.py

@@ -73,6 +73,7 @@
 from spack.environment import EnvironmentModifications, validate
 from spack.environment import preserve_environment
 from spack.util.environment import env_flag, filter_system_paths, get_path
+from spack.util.environment import system_dirs
 from spack.util.executable import Executable
 from spack.util.module_cmd import load_module, get_path_from_module
 from spack.util.log_parse import parse_log_events, make_log_context
@@ -99,6 +100,7 @@
 SPACK_DEBUG_LOG_ID = 'SPACK_DEBUG_LOG_ID'
 SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR'
 SPACK_CCACHE_BINARY = 'SPACK_CCACHE_BINARY'
+SPACK_SYSTEM_DIRS = 'SPACK_SYSTEM_DIRS'
 
 
 # Platform-specific library suffix.
@@ -202,6 +204,8 @@ def set_compiler_environment_variables(pkg, env):
 
     env.set('SPACK_COMPILER_SPEC', str(pkg.spec.compiler))
 
+    env.set('SPACK_SYSTEM_DIRS', ' '.join(system_dirs))
+
     compiler.setup_custom_environment(pkg, env)
 
     return env

lib/spack/spack/test/build_environment.py

@@ -58,6 +58,8 @@ def build_environment():
     os.environ['SPACK_F77_RPATH_ARG'] = "-Wl,-rpath,"
     os.environ['SPACK_FC_RPATH_ARG']  = "-Wl,-rpath,"
 
+    os.environ['SPACK_SYSTEM_DIRS'] = '/usr/include /usr/lib'
+
     if 'SPACK_DEPENDENCIES' in os.environ:
         del os.environ['SPACK_DEPENDENCIES']
 
@@ -67,7 +69,8 @@ def build_environment():
                  'SPACK_ENV_PATH', 'SPACK_DEBUG_LOG_DIR',
                  'SPACK_COMPILER_SPEC', 'SPACK_SHORT_SPEC',
                  'SPACK_CC_RPATH_ARG', 'SPACK_CXX_RPATH_ARG',
-                 'SPACK_F77_RPATH_ARG', 'SPACK_FC_RPATH_ARG'):
+                 'SPACK_F77_RPATH_ARG', 'SPACK_FC_RPATH_ARG',
+                 'SPACK_SYSTEM_DIRS'):
         del os.environ[name]
 
 
@@ -96,8 +99,8 @@ def test_static_to_shared_library(build_environment):
                 shared_lib = '{0}.{1}'.format(
                     os.path.splitext(static_lib)[0], dso_suffix)
 
-            assert output == expected[arch].format(
-                static_lib, shared_lib, os.path.basename(shared_lib))
+            assert set(output.split()) == set(expected[arch].format(
+                static_lib, shared_lib, os.path.basename(shared_lib)).split())
 
 
 @pytest.mark.regression('8345')

lib/spack/spack/test/cc.py

@@ -166,50 +166,71 @@ def test_flags(self):
 
         # Test ldflags added properly in ld mode
         self.check_ld('dump-args', test_command,
-                      "ld " +
-                      '-rpath ' + self.prefix + '/lib ' +
-                      '-rpath ' + self.prefix + '/lib64 ' +
-                      '-L foo ' +
-                      ' '.join(test_command) + ' ' +
-                      '-lfoo')
+                      'ld -L foo ' +
+                      '-I/test/include -I/other/include arg1 ' +
+                      '-Wl,--start-group arg2 arg3 -llib1 -llib2 arg4 ' +
+                      '-Wl,--end-group ' +
+                      '-llib3 -llib4 arg5 arg6 ' +
+                      '-L/test/lib -L/other/lib ' +
+                      '-lfoo ' +
+                      '-rpath /first/rpath -rpath /second/rpath ' +
+                      '-rpath /third/rpath -rpath /fourth/rpath ' +
+                      '-rpath /spack-test-prefix/lib ' +
+                      '-rpath /spack-test-prefix/lib64')
 
         # Test cppflags added properly in cpp mode
         self.check_cpp('dump-args', test_command,
                        "cpp " +
                        '-g -O1 ' +
-                       ' '.join(test_command))
+                       '-I/test/include -I/other/include arg1 ' +
+                       '-Wl,--start-group arg2 arg3 -llib1 -llib2 arg4 ' +
+                       '-Wl,--end-group ' +
+                       '-llib3 -llib4 arg5 arg6 ' +
+                       '-L/test/lib -L/other/lib')
 
         # Test ldflags, cppflags, and language specific flags are added in
         # proper order
         self.check_cc('dump-args', test_command,
                       self.realcc + ' ' +
-                      '-Wl,-rpath,' + self.prefix + '/lib ' +
-                      '-Wl,-rpath,' + self.prefix + '/lib64 ' +
-                      '-g -O1 ' +
-                      '-Wall ' +
-                      '-L foo ' +
-                      ' '.join(test_command) + ' ' +
-                      '-lfoo')
+                      '-g -O1 -Wall -L foo ' +
+                      '-I/test/include -I/other/include arg1 ' +
+                      '-Wl,--start-group arg2 arg3 -llib1 -llib2 arg4 ' +
+                      '-Wl,--end-group ' +
+                      '-llib3 -llib4 arg5 arg6 ' +
+                      '-L/test/lib -L/other/lib ' +
+                      '-lfoo ' +
+                      '-Wl,-rpath,/first/rpath -Wl,-rpath,/second/rpath ' +
+                      '-Wl,-rpath,/third/rpath -Wl,-rpath,/fourth/rpath ' +
+                      '-Wl,-rpath,/spack-test-prefix/lib ' +
+                      '-Wl,-rpath,/spack-test-prefix/lib64')
 
         self.check_cxx('dump-args', test_command,
                        self.realcc + ' ' +
-                       '-Wl,-rpath,' + self.prefix + '/lib ' +
-                       '-Wl,-rpath,' + self.prefix + '/lib64 ' +
-                       '-g -O1 ' +
-                       '-Werror ' +
-                       '-L foo ' +
-                       ' '.join(test_command) + ' ' +
-                       '-lfoo')
+                       '-g -O1 -Werror -L foo ' +
+                       '-I/test/include -I/other/include arg1 ' +
+                       '-Wl,--start-group arg2 arg3 -llib1 -llib2 arg4 ' +
+                       '-Wl,--end-group ' +
+                       '-llib3 -llib4 arg5 arg6 ' +
+                       '-L/test/lib -L/other/lib ' +
+                       '-lfoo ' +
+                       '-Wl,-rpath,/first/rpath -Wl,-rpath,/second/rpath ' +
+                       '-Wl,-rpath,/third/rpath -Wl,-rpath,/fourth/rpath ' +
+                       '-Wl,-rpath,/spack-test-prefix/lib ' +
+                       '-Wl,-rpath,/spack-test-prefix/lib64')
 
         self.check_fc('dump-args', test_command,
                       self.realcc + ' ' +
-                      '-Wl,-rpath,' + self.prefix + '/lib ' +
-                      '-Wl,-rpath,' + self.prefix + '/lib64 ' +
-                      '-w ' +
-                      '-g -O1 ' +
-                      '-L foo ' +
-                      ' '.join(test_command) + ' ' +
-                      '-lfoo')
+                      '-w -g -O1 -L foo ' +
+                      '-I/test/include -I/other/include arg1 ' +
+                      '-Wl,--start-group arg2 arg3 -llib1 -llib2 arg4 ' +
+                      '-Wl,--end-group ' +
+                      '-llib3 -llib4 arg5 arg6 ' +
+                      '-L/test/lib -L/other/lib ' +
+                      '-lfoo ' +
+                      '-Wl,-rpath,/first/rpath -Wl,-rpath,/second/rpath ' +
+                      '-Wl,-rpath,/third/rpath -Wl,-rpath,/fourth/rpath ' +
+                      '-Wl,-rpath,/spack-test-prefix/lib ' +
+                      '-Wl,-rpath,/spack-test-prefix/lib64')
 
         del os.environ['SPACK_CFLAGS']
         del os.environ['SPACK_CXXFLAGS']
@@ -222,9 +243,15 @@ def test_dep_rpath(self):
         """Ensure RPATHs for root package are added."""
         self.check_cc('dump-args', test_command,
                       self.realcc + ' ' +
-                      '-Wl,-rpath,' + self.prefix + '/lib ' +
-                      '-Wl,-rpath,' + self.prefix + '/lib64 ' +
-                      ' '.join(test_command))
+                      '-I/test/include -I/other/include arg1 ' +
+                      '-Wl,--start-group arg2 arg3 -llib1 -llib2 arg4 ' +
+                      '-Wl,--end-group ' +
+                      '-llib3 -llib4 arg5 arg6 ' +
+                      '-L/test/lib -L/other/lib ' +
+                      '-Wl,-rpath,/first/rpath -Wl,-rpath,/second/rpath ' +
+                      '-Wl,-rpath,/third/rpath -Wl,-rpath,/fourth/rpath ' +
+                      '-Wl,-rpath,/spack-test-prefix/lib ' +
+                      '-Wl,-rpath,/spack-test-prefix/lib64')
 
     def test_dep_include(self):
         """Ensure a single dependency include directory is added."""
@@ -233,10 +260,16 @@ def test_dep_include(self):
         os.environ['SPACK_LINK_DEPS'] = os.environ['SPACK_DEPENDENCIES']
         self.check_cc('dump-args', test_command,
                       self.realcc + ' ' +
-                      '-Wl,-rpath,' + self.prefix + '/lib ' +
-                      '-Wl,-rpath,' + self.prefix + '/lib64 ' +
-                      '-I' + self.dep4 + '/include ' +
-                      ' '.join(test_command))
+                      '-I/test/include -I/other/include ' +
+                      '-I' + self.dep4 + '/include arg1 ' +
+                      '-Wl,--start-group arg2 arg3 -llib1 -llib2 arg4 ' +
+                      '-Wl,--end-group ' +
+                      '-llib3 -llib4 arg5 arg6 ' +
+                      '-L/test/lib -L/other/lib ' +
+                      '-Wl,-rpath,/first/rpath -Wl,-rpath,/second/rpath ' +
+                      '-Wl,-rpath,/third/rpath -Wl,-rpath,/fourth/rpath ' +
+                      '-Wl,-rpath,/spack-test-prefix/lib ' +
+                      '-Wl,-rpath,/spack-test-prefix/lib64')
 
     def test_dep_lib(self):
         """Ensure a single dependency RPATH is added."""
@@ -245,11 +278,17 @@ def test_dep_lib(self):
         os.environ['SPACK_LINK_DEPS'] = os.environ['SPACK_DEPENDENCIES']
         self.check_cc('dump-args', test_command,
                       self.realcc + ' ' +
-                      '-Wl,-rpath,' + self.prefix + '/lib ' +
-                      '-Wl,-rpath,' + self.prefix + '/lib64 ' +
+                      '-I/test/include -I/other/include arg1 ' +
+                      '-Wl,--start-group arg2 arg3 -llib1 -llib2 arg4 ' +
+                      '-Wl,--end-group ' +
+                      '-llib3 -llib4 arg5 arg6 ' +
+                      '-L/test/lib -L/other/lib ' +
                       '-L' + self.dep2 + '/lib64 ' +
-                      '-Wl,-rpath,' + self.dep2 + '/lib64 ' +
-                      ' '.join(test_command))
+                      '-Wl,-rpath,/first/rpath -Wl,-rpath,/second/rpath ' +
+                      '-Wl,-rpath,/third/rpath -Wl,-rpath,/fourth/rpath ' +
+                      '-Wl,-rpath,/spack-test-prefix/lib ' +
+                      '-Wl,-rpath,/spack-test-prefix/lib64 ' +
+                      '-Wl,-rpath,' + self.dep2 + '/lib64')
 
     def test_dep_lib_no_rpath(self):
         """Ensure a single dependency link flag is added with no dep RPATH."""
@@ -257,10 +296,16 @@ def test_dep_lib_no_rpath(self):
         os.environ['SPACK_LINK_DEPS'] = os.environ['SPACK_DEPENDENCIES']
         self.check_cc('dump-args', test_command,
                       self.realcc + ' ' +
-                      '-Wl,-rpath,' + self.prefix + '/lib ' +
-                      '-Wl,-rpath,' + self.prefix + '/lib64 ' +
+                      '-I/test/include -I/other/include arg1 ' +
+                      '-Wl,--start-group arg2 arg3 -llib1 -llib2 arg4 ' +
+                      '-Wl,--end-group ' +
+                      '-llib3 -llib4 arg5 arg6 ' +
+                      '-L/test/lib -L/other/lib ' +
                       '-L' + self.dep2 + '/lib64 ' +
-                      ' '.join(test_command))
+                      '-Wl,-rpath,/first/rpath -Wl,-rpath,/second/rpath ' +
+                      '-Wl,-rpath,/third/rpath -Wl,-rpath,/fourth/rpath ' +
+                      '-Wl,-rpath,/spack-test-prefix/lib ' +
+                      '-Wl,-rpath,/spack-test-prefix/lib64')
 
     def test_dep_lib_no_lib(self):
         """Ensure a single dependency RPATH is added with no -L."""
@@ -268,10 +313,16 @@ def test_dep_lib_no_lib(self):
         os.environ['SPACK_RPATH_DEPS'] = os.environ['SPACK_DEPENDENCIES']
         self.check_cc('dump-args', test_command,
                       self.realcc + ' ' +
-                      '-Wl,-rpath,' + self.prefix + '/lib ' +
-                      '-Wl,-rpath,' + self.prefix + '/lib64 ' +
-                      '-Wl,-rpath,' + self.dep2 + '/lib64 ' +
-                      ' '.join(test_command))
+                      '-I/test/include -I/other/include arg1 ' +
+                      '-Wl,--start-group arg2 arg3 -llib1 -llib2 arg4 ' +
+                      '-Wl,--end-group ' +
+                      '-llib3 -llib4 arg5 arg6 ' +
+                      '-L/test/lib -L/other/lib ' +
+                      '-Wl,-rpath,/first/rpath -Wl,-rpath,/second/rpath ' +
+                      '-Wl,-rpath,/third/rpath -Wl,-rpath,/fourth/rpath ' +
+                      '-Wl,-rpath,/spack-test-prefix/lib ' +
+                      '-Wl,-rpath,/spack-test-prefix/lib64 ' +
+                      '-Wl,-rpath,' + self.dep2 + '/lib64')
 
     def test_all_deps(self):
         """Ensure includes and RPATHs for all deps are added. """
@@ -285,23 +336,25 @@ def test_all_deps(self):
         # to.  We could loosen that if it becomes necessary
         self.check_cc('dump-args', test_command,
                       self.realcc + ' ' +
-                      '-Wl,-rpath,' + self.prefix + '/lib ' +
-                      '-Wl,-rpath,' + self.prefix + '/lib64 ' +
-
-                      '-I' + self.dep4 + '/include ' +
-
-                      '-L' + self.dep3 + '/lib64 ' +
-                      '-Wl,-rpath,' + self.dep3 + '/lib64 ' +
+                      '-I/test/include -I/other/include ' +
+                      '-I' + self.dep1 + '/include ' +
                       '-I' + self.dep3 + '/include ' +
-
-                      '-L' + self.dep2 + '/lib64 ' +
-                      '-Wl,-rpath,' + self.dep2 + '/lib64 ' +
-
+                      '-I' + self.dep4 + '/include ' +
+                      'arg1 ' +
+                      '-Wl,--start-group arg2 arg3 -llib1 -llib2 arg4 ' +
+                      '-Wl,--end-group ' +
+                      '-llib3 -llib4 arg5 arg6 ' +
+                      '-L/test/lib -L/other/lib ' +
                       '-L' + self.dep1 + '/lib ' +
+                      '-L' + self.dep2 + '/lib64 ' +
+                      '-L' + self.dep3 + '/lib64 ' +
+                      '-Wl,-rpath,/first/rpath -Wl,-rpath,/second/rpath ' +
+                      '-Wl,-rpath,/third/rpath -Wl,-rpath,/fourth/rpath ' +
+                      '-Wl,-rpath,/spack-test-prefix/lib ' +
+                      '-Wl,-rpath,/spack-test-prefix/lib64 ' +
                       '-Wl,-rpath,' + self.dep1 + '/lib ' +
-                      '-I' + self.dep1 + '/include ' +
-
-                      ' '.join(test_command))
+                      '-Wl,-rpath,' + self.dep2 + '/lib64 ' +
+                      '-Wl,-rpath,' + self.dep3 + '/lib64')
 
     def test_ld_deps(self):
         """Ensure no (extra) -I args or -Wl, are passed in ld mode."""
@@ -312,19 +365,22 @@ def test_ld_deps(self):
 
         self.check_ld('dump-args', test_command,
                       'ld ' +
-                      '-rpath ' + self.prefix + '/lib ' +
-                      '-rpath ' + self.prefix + '/lib64 ' +
-
-                      '-L' + self.dep3 + '/lib64 ' +
-                      '-rpath ' + self.dep3 + '/lib64 ' +
-
-                      '-L' + self.dep2 + '/lib64 ' +
-                      '-rpath ' + self.dep2 + '/lib64 ' +
-
+                      '-I/test/include -I/other/include ' +
+                      'arg1 ' +
+                      '-Wl,--start-group arg2 arg3 -llib1 -llib2 arg4 ' +
+                      '-Wl,--end-group ' +
+                      '-llib3 -llib4 arg5 arg6 ' +
+                      '-L/test/lib -L/other/lib ' +
                       '-L' + self.dep1 + '/lib ' +
+                      '-L' + self.dep2 + '/lib64 ' +
+                      '-L' + self.dep3 + '/lib64 ' +
+                      '-rpath /first/rpath -rpath /second/rpath ' +
+                      '-rpath /third/rpath -rpath /fourth/rpath ' +
+                      '-rpath /spack-test-prefix/lib ' +
+                      '-rpath /spack-test-prefix/lib64 ' +
                       '-rpath ' + self.dep1 + '/lib ' +
-
-                      ' '.join(test_command))
+                      '-rpath ' + self.dep2 + '/lib64 ' +
+                      '-rpath ' + self.dep3 + '/lib64')
 
     def test_ld_deps_no_rpath(self):
         """Ensure SPACK_RPATH_DEPS controls RPATHs for ld."""
@@ -334,14 +390,19 @@ def test_ld_deps_no_rpath(self):
 
         self.check_ld('dump-args', test_command,
                       'ld ' +
-                      '-rpath ' + self.prefix + '/lib ' +
-                      '-rpath ' + self.prefix + '/lib64 ' +
-
-                      '-L' + self.dep3 + '/lib64 ' +
-                      '-L' + self.dep2 + '/lib64 ' +
+                      '-I/test/include -I/other/include ' +
+                      'arg1 ' +
+                      '-Wl,--start-group arg2 arg3 -llib1 -llib2 arg4 ' +
+                      '-Wl,--end-group ' +
+                      '-llib3 -llib4 arg5 arg6 ' +
+                      '-L/test/lib -L/other/lib ' +
                       '-L' + self.dep1 + '/lib ' +
-
-                      ' '.join(test_command))
+                      '-L' + self.dep2 + '/lib64 ' +
+                      '-L' + self.dep3 + '/lib64 ' +
+                      '-rpath /first/rpath -rpath /second/rpath ' +
+                      '-rpath /third/rpath -rpath /fourth/rpath ' +
+                      '-rpath /spack-test-prefix/lib ' +
+                      '-rpath /spack-test-prefix/lib64')
 
     def test_ld_deps_no_link(self):
         """Ensure SPACK_LINK_DEPS controls -L for ld."""
@@ -351,14 +412,19 @@ def test_ld_deps_no_link(self):
 
         self.check_ld('dump-args', test_command,
                       'ld ' +
-                      '-rpath ' + self.prefix + '/lib ' +
-                      '-rpath ' + self.prefix + '/lib64 ' +
-
-                      '-rpath ' + self.dep3 + '/lib64 ' +
-                      '-rpath ' + self.dep2 + '/lib64 ' +
+                      '-I/test/include -I/other/include ' +
+                      'arg1 ' +
+                      '-Wl,--start-group arg2 arg3 -llib1 -llib2 arg4 ' +
+                      '-Wl,--end-group ' +
+                      '-llib3 -llib4 arg5 arg6 ' +
+                      '-L/test/lib -L/other/lib ' +
+                      '-rpath /first/rpath -rpath /second/rpath ' +
+                      '-rpath /third/rpath -rpath /fourth/rpath ' +
+                      '-rpath /spack-test-prefix/lib ' +
+                      '-rpath /spack-test-prefix/lib64 ' +
                       '-rpath ' + self.dep1 + '/lib ' +
-
-                      ' '.join(test_command))
+                      '-rpath ' + self.dep2 + '/lib64 ' +
+                      '-rpath ' + self.dep3 + '/lib64')
 
     def test_ld_deps_reentrant(self):
         """Make sure ld -r is handled correctly on OS's where it doesn't
@@ -371,18 +437,30 @@ def test_ld_deps_reentrant(self):
         reentrant_test_command = ['-r'] + test_command
         self.check_ld('dump-args', reentrant_test_command,
                       'ld ' +
-                      '-rpath ' + self.prefix + '/lib ' +
-                      '-rpath ' + self.prefix + '/lib64 ' +
-
+                      '-I/test/include -I/other/include ' +
+                      '-r arg1 ' +
+                      '-Wl,--start-group arg2 arg3 -llib1 -llib2 arg4 ' +
+                      '-Wl,--end-group ' +
+                      '-llib3 -llib4 arg5 arg6 ' +
+                      '-L/test/lib -L/other/lib ' +
                       '-L' + self.dep1 + '/lib ' +
-                      '-rpath ' + self.dep1 + '/lib ' +
-
-                      '-r ' +
-                      ' '.join(test_command))
-
+                      '-rpath /first/rpath -rpath /second/rpath ' +
+                      '-rpath /third/rpath -rpath /fourth/rpath ' +
+                      '-rpath /spack-test-prefix/lib ' +
+                      '-rpath /spack-test-prefix/lib64 ' +
+                      '-rpath ' + self.dep1 + '/lib')
+
+        # rpaths from the underlying command will still appear
+        # Spack will not add its own rpaths.
         os.environ['SPACK_SHORT_SPEC'] = "foo@1.2=darwin-x86_64"
         self.check_ld('dump-args', reentrant_test_command,
                       'ld ' +
+                      '-I/test/include -I/other/include ' +
+                      '-r arg1 ' +
+                      '-Wl,--start-group arg2 arg3 -llib1 -llib2 arg4 ' +
+                      '-Wl,--end-group ' +
+                      '-llib3 -llib4 arg5 arg6 ' +
+                      '-L/test/lib -L/other/lib ' +
                       '-L' + self.dep1 + '/lib ' +
-                      '-r ' +
-                      ' '.join(test_command))
+                      '-rpath /first/rpath -rpath /second/rpath ' +
+                      '-rpath /third/rpath -rpath /fourth/rpath')