From 47e9f7aac9c0418fd1e31e88bd49017c429565db Mon Sep 17 00:00:00 2001
From: Todd Gamblin <tgamblin@llnl.gov>
Date: Wed, 3 Jul 2019 01:56:13 -0700
Subject: [PATCH] tests: add tests for setup-env.sh
- tests use a shell-script harness and test all Spack commands that
require special shell support.
- tests work in bash, zsh, and dash
- run setup-env.sh tests on macos and linux builds.
- we run them on macos and linux
---
.codecov.yml | 2 +
.travis.yml | 33 ++--
share/spack/qa/run-unit-tests | 30 ++++
share/spack/qa/setup-env-test.sh | 289 +++++++++++++++++++++++++++++++
share/spack/qa/setup.sh | 18 +-
5 files changed, 343 insertions(+), 29 deletions(-)
create mode 100755 share/spack/qa/setup-env-test.sh
diff --git a/.codecov.yml b/.codecov.yml
index 042cc86e9c..c06cb57f88 100644
--- a/.codecov.yml
+++ b/.codecov.yml
@@ -11,5 +11,7 @@ ignore:
- lib/spack/spack/test/.*
- lib/spack/docs/.*
- lib/spack/external/.*
+ - share/spack/qa/.*
+ - share/spack/spack-completion.bash
comment: off
diff --git a/.travis.yml b/.travis.yml
index 91a95486e9..465232093a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -50,24 +50,6 @@ jobs:
os: linux
language: python
env: [ TEST_SUITE=unit, COVERAGE=true ]
- addons:
- apt:
- packages:
- - cmake
- - gfortran
- - graphviz
- - gnupg2
- - kcov
- - mercurial
- - ninja-build
- - perl
- - perl-base
- - realpath
- - patchelf
- - r-base
- - r-base-core
- - r-base-dev
-
- python: '3.7'
sudo: required
os: linux
@@ -159,6 +141,7 @@ addons:
- r-base
- r-base-core
- r-base-dev
+ - zsh
# for Mac builds, we use Homebrew
homebrew:
packages:
@@ -166,6 +149,8 @@ addons:
- gcc
- gnupg2
- ccache
+ - dash
+ - kcov
update: true
# ~/.ccache needs to be cached directly as Travis is not taking care of it
@@ -223,11 +208,13 @@ script:
after_success:
- ccache -s
- - if [[ "$TEST_SUITE" == "unit" || "$TEST_SUITE" == "build" ]]; then
- codecov --env PYTHON_VERSION
- --required
- --flags "${TEST_SUITE}${TRAVIS_OS_NAME}";
- fi
+ - case "$TEST_SUITE" in
+ unit)
+ codecov --env PYTHON_VERSION
+ --required
+ --flags "${TEST_SUITE}${TRAVIS_OS_NAME}";
+ ;;
+ esac
#=============================================================================
# Notifications
diff --git a/share/spack/qa/run-unit-tests b/share/spack/qa/run-unit-tests
index 49dbda0d32..4403d53a76 100755
--- a/share/spack/qa/run-unit-tests
+++ b/share/spack/qa/run-unit-tests
@@ -16,6 +16,12 @@
# Optionally add one or more unit tests
# to only run these tests.
#
+
+#-----------------------------------------------------------
+# Run a few initial commands and set up test environment
+#-----------------------------------------------------------
+ORIGINAL_PATH="$PATH"
+
. "$(dirname $0)/setup.sh"
check_dependencies ${coverage} git hg svn
@@ -33,9 +39,33 @@ bin/spack help -a
# Profile and print top 20 lines for a simple call to spack spec
bin/spack -p --lines 20 spec mpileaks%gcc ^elfutils@0.170
+#-----------------------------------------------------------
# Run unit tests with code coverage
+#-----------------------------------------------------------
extra_args=""
if [[ -n "$@" ]]; then
extra_args="-k $@"
fi
${coverage_run} bin/spack test --verbose "$extra_args"
+
+#-----------------------------------------------------------
+# Run tests for setup-env.sh
+#-----------------------------------------------------------
+# Clean the environment by removing Spack from the path and getting rid of
+# the spack shell function
+export PATH="$ORIGINAL_PATH"
+unset spack
+
+# start in the spack root directory
+cd $SPACK_ROOT
+
+# Run bash tests with coverage enabled, but pipe output to /dev/null
+# because it seems that kcov seems to undo the script's redirection
+if [ "$BASH_COVERAGE" = true ]; then
+ ${QA_DIR}/bashcov ${QA_DIR}/setup-env-test.sh &> /dev/null
+fi
+
+# run the test scripts for their output (these will print nicely)
+bash ${QA_DIR}/setup-env-test.sh
+zsh ${QA_DIR}/setup-env-test.sh
+dash ${QA_DIR}/setup-env-test.sh
diff --git a/share/spack/qa/setup-env-test.sh b/share/spack/qa/setup-env-test.sh
new file mode 100755
index 0000000000..b75d006e75
--- /dev/null
+++ b/share/spack/qa/setup-env-test.sh
@@ -0,0 +1,289 @@
+#!/bin/sh
+#
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+#
+# This script tests that Spack's setup-env.sh init script works.
+#
+# The tests are portable to bash, zsh, and bourne shell, and can be run
+# in any of these shells.
+#
+
+# ------------------------------------------------------------------------
+# Functions for color output.
+# ------------------------------------------------------------------------
+
+# Colors for output
+red='\033[1;31m'
+cyan='\033[1;36m'
+green='\033[1;32m'
+reset='\033[0m'
+
+echo_red() {
+ printf "${red}$*${reset}\n"
+}
+
+echo_green() {
+ printf "${green}$*${reset}\n"
+}
+
+echo_msg() {
+ printf "${cyan}$*${reset}\n"
+}
+
+# ------------------------------------------------------------------------
+# Generic functions for testing shell code.
+# ------------------------------------------------------------------------
+
+# counts of test successes and failures.
+success=0
+errors=0
+
+# Print out a header for a group of tests.
+title() {
+ echo
+ echo_msg "$@"
+ echo_msg "---------------------------------"
+}
+
+# echo FAIL in red text; increment failures
+fail() {
+ echo_red FAIL
+ errors=$((errors+1))
+}
+
+#
+# Echo SUCCESS in green; increment successes
+#
+pass() {
+ echo_green SUCCESS
+ success=$((success+1))
+}
+
+#
+# Run a command and suppress output unless it fails.
+# On failure, echo the exit code and output.
+#
+succeeds() {
+ printf "'%s' succeeds ... " "$*"
+ output=$("$@" 2>&1)
+ err="$?"
+
+ if [ "$err" != 0 ]; then
+ fail
+ echo_red "Command failed with error $err."
+ if [ -n "$output" ]; then
+ echo_msg "Output:"
+ echo "$output"
+ else
+ echo_msg "No output."
+ fi
+ else
+ pass
+ fi
+}
+
+#
+# Run a command and suppress output unless it succeeds.
+# If the command succeeds, echo the output.
+#
+fails() {
+ printf "'%s' fails ... " "$*"
+ output=$("$@" 2>&1)
+ err="$?"
+
+ if [ "$err" = 0 ]; then
+ fail
+ echo_red "Command failed with error $err."
+ if [ -n "$output" ]; then
+ echo_msg "Output:"
+ echo "$output"
+ else
+ echo_msg "No output."
+ fi
+ else
+ pass
+ fi
+}
+
+#
+# Ensure that a string is in the output of a command.
+# Suppresses output on success.
+# On failure, echo the exit code and output.
+#
+contains() {
+ string="$1"
+ shift
+
+ printf "'%s' output contains '$string' ... " "$*"
+ output=$("$@" 2>&1)
+ err="$?"
+
+ if [ "${output#*$string}" = "${output}" ]; then
+ fail
+ echo_red "Command exited with error $err."
+ echo_red "'$string' was not in output."
+ if [ -n "$output" ]; then
+ echo_msg "Output:"
+ echo "$output"
+ else
+ echo_msg "No output."
+ fi
+ else
+ pass
+ fi
+}
+
+# -----------------------------------------------------------------------
+# Instead of invoking the module/use/dotkit commands, we print the
+# arguments that Spack invokes the command with, so we can check that
+# Spack passes the expected arguments in the tests below.
+#
+# We make that happen by defining the sh functions below.
+# -----------------------------------------------------------------------
+module() {
+ echo module "$@"
+}
+
+use() {
+ echo use "$@"
+}
+
+unuse() {
+ echo unuse "$@"
+}
+
+# -----------------------------------------------------------------------
+# Setup test environment and do some preliminary checks
+# -----------------------------------------------------------------------
+
+# Make sure no environment is active
+unset SPACK_ENV
+
+# Source setup-env.sh before tests
+. share/spack/setup-env.sh
+
+title "Testing setup-env.sh with $_sp_shell"
+
+# spack command is now avaialble
+succeeds which spack
+
+# mock cd command (intentionally define only AFTER setup-env.sh)
+cd() {
+ echo cd "$@"
+}
+
+# create a fake mock package install and store its location for later
+title "Setup"
+echo "Creating a mock package installation"
+spack -m install --fake a
+a_install=$(spack location -i a)
+a_module=$(spack -m module tcl find a)
+a_dotkit=$(spack -m module dotkit find a)
+
+b_install=$(spack location -i b)
+b_module=$(spack -m module tcl find b)
+b_dotkit=$(spack -m module dotkit find b)
+
+# ensure that we uninstall b on exit
+cleanup() {
+ title "Cleanup"
+ echo "Removing test package before exiting test script."
+ spack -m uninstall -yf b
+ spack -m uninstall -yf a
+
+ echo
+ echo "$success tests succeeded."
+ echo "$errors tests failed."
+ if [ "$errors" = 0 ]; then
+ pass
+ exit 0
+ else
+ fail
+ exit 1
+ fi
+}
+trap cleanup EXIT
+
+# -----------------------------------------------------------------------
+# Test all spack commands with special env support
+# -----------------------------------------------------------------------
+title 'Testing `spack`'
+contains 'usage: spack ' spack
+contains "usage: spack " spack -h
+contains "usage: spack " spack help
+contains "usage: spack " spack -H
+contains "usage: spack " spack help --all
+
+title 'Testing `spack cd`'
+contains "usage: spack cd " spack cd -h
+contains "usage: spack cd " spack cd --help
+contains "cd $b_install" spack cd -i b
+
+title 'Testing `spack module`'
+contains "usage: spack module " spack -m module -h
+contains "usage: spack module " spack -m module --help
+contains "usage: spack module " spack -m module
+
+title 'Testing `spack load`'
+contains "module load $b_module" spack -m load b
+fails spack -m load -l
+contains "module load -l --arg $b_module" spack -m load -l --arg b
+contains "module load $b_module $a_module" spack -m load -r a
+contains "module load $b_module $a_module" spack -m load --dependencies a
+fails spack -m load d
+contains "usage: spack load " spack -m load -h
+contains "usage: spack load " spack -m load -h d
+contains "usage: spack load " spack -m load --help
+
+title 'Testing `spack unload`'
+contains "module unload $b_module" spack -m unload b
+fails spack -m unload -l
+contains "module unload -l --arg $b_module" spack -m unload -l --arg b
+fails spack -m unload d
+contains "usage: spack unload " spack -m unload -h
+contains "usage: spack unload " spack -m unload -h d
+contains "usage: spack unload " spack -m unload --help
+
+title 'Testing `spack use`'
+contains "use $b_dotkit" spack -m use b
+fails spack -m use -l
+contains "use -l --arg $b_dotkit" spack -m use -l --arg b
+contains "use $b_dotkit $a_dotkit" spack -m use -r a
+contains "use $b_dotkit $a_dotkit" spack -m use --dependencies a
+fails spack -m use d
+contains "usage: spack use " spack -m use -h
+contains "usage: spack use " spack -m use -h d
+contains "usage: spack use " spack -m use --help
+
+title 'Testing `spack unuse`'
+contains "unuse $b_dotkit" spack -m unuse b
+fails spack -m unuse -l
+contains "unuse -l --arg $b_dotkit" spack -m unuse -l --arg b
+fails spack -m unuse d
+contains "usage: spack unuse " spack -m unuse -h
+contains "usage: spack unuse " spack -m unuse -h d
+contains "usage: spack unuse " spack -m unuse --help
+
+title 'Testing `spack env`'
+contains "usage: spack env " spack env -h
+contains "usage: spack env " spack env --help
+
+title 'Testing `spack env activate`'
+contains "No such environment:" spack env activate no_such_environment
+contains "usage: spack env activate " spack env activate
+contains "usage: spack env activate " spack env activate -h
+contains "usage: spack env activate " spack env activate --help
+
+title 'Testing `spack env deactivate`'
+contains "Error: No environment is currently active" spack env deactivate
+contains "usage: spack env deactivate " spack env deactivate no_such_environment
+contains "usage: spack env deactivate " spack env deactivate -h
+contains "usage: spack env deactivate " spack env deactivate --help
+
+title 'Testing `spack env list`'
+contains " spack env list " spack env list -h
+contains " spack env list " spack env list --help
diff --git a/share/spack/qa/setup.sh b/share/spack/qa/setup.sh
index 6256269211..3ff226d057 100755
--- a/share/spack/qa/setup.sh
+++ b/share/spack/qa/setup.sh
@@ -20,18 +20,24 @@ export SPACK_ROOT=$(realpath "$QA_DIR/../../..")
coverage=""
coverage_run=""
+# bash coverage depends on some other factors -- there are issues with
+# kcov for Python 2.6, unit tests, and build tests.
+if [[ $TEST_SUITE == unit && # kcov segfaults for the MPICH build test
+ $TRAVIS_OS_NAME == linux &&
+ $TRAVIS_PYTHON_VERSION != 2.6 ]];
+then
+ BASH_COVERAGE="true"
+else
+ BASH_COVERAGE="false"
+fi
+
# Set up some variables for running coverage tests.
if [[ "$COVERAGE" == "true" ]]; then
# these set up coverage for Python
coverage=coverage
coverage_run="coverage run"
- # make a coverage directory for kcov, and patch cc to use our bashcov
- # script instead of plain bash
- if [[ $TEST_SUITE == unit && # kcov segfaults for the MPICH build test
- $TRAVIS_OS_NAME == linux &&
- $TRAVIS_PYTHON_VERSION != 2.6 ]];
- then
+ if [ "$BASH_COVERAGE" = true ]; then
mkdir -p coverage
cc_script="$SPACK_ROOT/lib/spack/env/cc"
bashcov=$(realpath ${QA_DIR}/bashcov)
--
GitLab