From bb5a433a46a8ffc69b01e8477eb6646f4c66f9d2 Mon Sep 17 00:00:00 2001
From: Todd Gamblin <tgamblin@llnl.gov>
Date: Thu, 27 Apr 2017 11:47:56 -0700
Subject: [PATCH] Separate integration tests; simplify test scripts (#4006)
* Separate build integration tests; simplify test scripts
- Move build tests out of the regular Travis unit tests, add more smoke
test packages to build.
- Run all test scripts with bash -e, which fails on error.
- Factor coverage out into a Travis environment variable, so it's more
obvious from .travis.yml which tests contribute to coverage and which
don't.
- Factor dependency checking and much of the front-matter in tests
scripts into a setup.sh script, which is sourced by all the test
scripts. Extra cruft in each tests script now reduced to 2 lines at
the beginning.
---
.travis.yml | 14 +++-
share/spack/qa/check_dependencies | 96 ------------------------
share/spack/qa/run-build-tests | 29 ++++++++
share/spack/qa/run-doc-tests | 29 +-------
share/spack/qa/run-flake8-tests | 23 +-----
share/spack/qa/run-unit-tests | 40 ++--------
share/spack/qa/setup.sh | 118 ++++++++++++++++++++++++++++++
7 files changed, 172 insertions(+), 177 deletions(-)
delete mode 100755 share/spack/qa/check_dependencies
create mode 100755 share/spack/qa/run-build-tests
create mode 100755 share/spack/qa/setup.sh
diff --git a/.travis.yml b/.travis.yml
index 60b3b1cc31..4b95903d16 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,7 +21,11 @@ matrix:
- python: '2.7'
os: linux
language: python
- env: TEST_SUITE=unit
+ env: [ TEST_SUITE=unit, COVERAGE=true ]
+ - python: '2.7'
+ os: linux
+ language: python
+ env: [ TEST_SUITE=build, COVERAGE=true, 'SPEC=hypre^mpich' ]
- python: '3.3'
os: linux
language: python
@@ -37,7 +41,11 @@ matrix:
- python: '3.6'
os: linux
language: python
- env: TEST_SUITE=unit
+ env: [ TEST_SUITE=unit, COVERAGE=true ]
+ - python: '3.6'
+ os: linux
+ language: python
+ env: [ TEST_SUITE=build, COVERAGE=true, 'SPEC=hypre^mpich' ]
- python: '2.7'
os: linux
language: python
@@ -48,7 +56,7 @@ matrix:
env: TEST_SUITE=doc
- os: osx
language: generic
- env: [ TEST_SUITE=unit, PYTHON_VERSION=2.7 ]
+ env: [ TEST_SUITE=unit, PYTHON_VERSION=2.7, COVERAGE=true ]
#=============================================================================
# Environment
diff --git a/share/spack/qa/check_dependencies b/share/spack/qa/check_dependencies
deleted file mode 100755
index e999463b03..0000000000
--- a/share/spack/qa/check_dependencies
+++ /dev/null
@@ -1,96 +0,0 @@
-#!/usr/bin/env bash
-#
-# Description:
-# Check to see if dependencies are installed.
-# If not, warn the user and tell them how to
-# install these dependencies.
-#
-# Usage:
-# check-deps <dep> ...
-#
-# Options:
-# One or more dependencies. Must use name of binary.
-
-for dep in "$@"; do
- if ! which $dep &> /dev/null; then
- # Map binary name to package name
- case $dep in
- sphinx-apidoc|sphinx-build)
- spack_package=py-sphinx
- pip_package=sphinx
- ;;
- coverage)
- spack_package=py-coverage
- pip_package=coverage
- ;;
- flake8)
- spack_package=py-flake8
- pip_package=flake8
- ;;
- dot)
- spack_package=graphviz
- ;;
- git)
- spack_package=git
- ;;
- hg)
- spack_package=mercurial
- pip_package=mercurial
- ;;
- svn)
- spack_package=subversion
- ;;
- *)
- spack_package=$dep
- pip_package=$dep
- ;;
- esac
-
- echo "ERROR: $dep is required to run this script."
- echo
-
- if [[ $spack_package ]]; then
- echo "To install with Spack, run:"
- echo " $ spack install $spack_package"
- fi
-
- if [[ $pip_package ]]; then
- echo "To install with pip, run:"
- echo " $ pip install $pip_package"
- fi
-
- if [[ $spack_package || $pip_package ]]; then
- echo "Then add the bin directory to your PATH."
- fi
-
- exit 1
- fi
-
- # Flake8 and Sphinx require setuptools in order to run.
- # Otherwise, they print out this error message:
- #
- # Traceback (most recent call last):
- # File: "/usr/bin/flake8", line 5, in <module>
- # from pkg_resources import load_entry_point
- # ImportError: No module named pkg_resources
- #
- # Print a more useful error message if setuptools not found.
- if [[ $dep == flake8 || $dep == sphinx* ]]; then
- # Find which Python is being run
- # Spack-installed packages have a hard-coded shebang
- python_cmd=$(head -n 1 $(which $dep) | cut -c 3-)
- # May not have a shebang
- if [[ $python_cmd != *python* ]]; then
- python_cmd=python
- fi
- # Check if setuptools is in the PYTHONPATH
- if ! $python_cmd -c "import setuptools" 2> /dev/null; then
- echo "ERROR: setuptools is required to run $dep."
- echo "Please add it to your PYTHONPATH."
-
- exit 1
- fi
- fi
-done
-
-echo "Dependencies found."
diff --git a/share/spack/qa/run-build-tests b/share/spack/qa/run-build-tests
new file mode 100755
index 0000000000..b5d5aed28f
--- /dev/null
+++ b/share/spack/qa/run-build-tests
@@ -0,0 +1,29 @@
+#!/bin/bash -e
+#
+# Description:
+# Runs Spack build smoke tests. This installs a few packages that
+# cover different parts of the build system. It is not an exhaustive
+# test of Spack's packages.
+#
+# Usage:
+# run-build-tests
+#
+. "$(dirname $0)/setup.sh"
+check_dependencies ${coverage} git hg svn
+
+# Move to root directory of Spack
+# Allows script to be run from anywhere
+cd "$SPACK_ROOT"
+
+# Make sure we have a spec to build.
+if [ -z "$SPEC" ]; then
+ echo "Error: run-build-tests requires the $SPEC to build to be set."
+ exit 1
+fi
+
+# Print compiler information
+spack config get compilers
+
+# Run some build smoke tests, potentially with code coverage
+${coverage_run} bin/spack install -v ${SPEC}
+${coverage_combine}
diff --git a/share/spack/qa/run-doc-tests b/share/spack/qa/run-doc-tests
index ca892d7eb4..b9a05aa3c8 100755
--- a/share/spack/qa/run-doc-tests
+++ b/share/spack/qa/run-doc-tests
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash -e
#
# Description:
# Builds Spack documentation and checks for
@@ -8,33 +8,12 @@
# Usage:
# run-doc-tests
#
-# Notes:
-# Requires sphinx, graphviz, git, mercurial, and subversion.
-#
-
-QA_DIR="$(dirname "$0")"
-SPACK_ROOT="$QA_DIR/../../.."
-DOC_DIR="$SPACK_ROOT/lib/spack/docs"
-
-# Array of dependencies
-deps=(
- sphinx-apidoc
- sphinx-build
- dot
- git
- hg
- svn
-)
-
-# Check for dependencies
-"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1
-
-# Add Spack to the PATH.
-export PATH="$SPACK_ROOT/bin:$PATH"
+. "$(dirname $0)/setup.sh"
+check_dependencies sphinx-apidoc sphinx-build dot git hg svn
# Move to documentation directory
# Allows script to be run from anywhere
-cd "$DOC_DIR"
+cd "$SPACK_ROOT/lib/spack/docs"
# Treat warnings as fatal errors
make clean --silent
diff --git a/share/spack/qa/run-flake8-tests b/share/spack/qa/run-flake8-tests
index 83469eeb9d..29fc15f9d7 100755
--- a/share/spack/qa/run-flake8-tests
+++ b/share/spack/qa/run-flake8-tests
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash -e
#
# Description:
# Runs source code style checks on Spack.
@@ -8,22 +8,7 @@
# Usage:
# run-flake8-tests
#
-# Notes:
-# Requires flake8.
-#
-
-QA_DIR="$(dirname "$0")"
-SPACK_ROOT="$QA_DIR/../../.."
-
-# Array of dependencies
-deps=(
- flake8
-)
-
-# Check for dependencies
-"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1
-
-# Add Spack to the PATH.
-export PATH="$SPACK_ROOT/bin:$PATH"
+. "$(dirname $0)/setup.sh"
+check_dependencies flake8
-exec spack flake8
+spack flake8
diff --git a/share/spack/qa/run-unit-tests b/share/spack/qa/run-unit-tests
index c266665ccb..7e300280ff 100755
--- a/share/spack/qa/run-unit-tests
+++ b/share/spack/qa/run-unit-tests
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash -e
#
# Description:
# Runs Spack unit tests.
@@ -10,44 +10,16 @@
# Optionally add one or more unit tests
# to only run these tests.
#
-# Notes:
-# Requires coverage, git, mercurial, and subversion.
-#
-
-QA_DIR="$(dirname "$0")"
-SPACK_ROOT="$QA_DIR/../../.."
-
-# Array of dependencies
-deps=(
- coverage
- git
- hg
- svn
-)
-
-# Check for dependencies
-"$QA_DIR/check_dependencies" "${deps[@]}" || exit 1
-
-# Add Spack to the PATH.
-export PATH="$SPACK_ROOT/bin:$PATH"
+. "$(dirname $0)/setup.sh"
+check_dependencies ${coverage} git hg svn
# Move to root directory of Spack
# Allows script to be run from anywhere
cd "$SPACK_ROOT"
-# Run integration tests
-# TODO: should these be separated into a different test suite?
-source "$SPACK_ROOT/share/spack/setup-env.sh"
-spack compilers
+# Print compiler information
spack config get compilers
# Run unit tests with code coverage
-py_ver=$(python -c 'import platform; print(platform.python_version())')
-if [[ "$py_ver" == 2.7* || "$py_ver" == 3.6* ]];
-then
- coverage run bin/spack install -v libdwarf
- coverage run bin/spack test "$@" && coverage combine
-else
- spack install -v libdwarf
- spack test "$@"
-fi
+${coverage_run} bin/spack test "$@"
+${coverage_combine}
diff --git a/share/spack/qa/setup.sh b/share/spack/qa/setup.sh
new file mode 100755
index 0000000000..98c79a0457
--- /dev/null
+++ b/share/spack/qa/setup.sh
@@ -0,0 +1,118 @@
+#!/bin/bash -e
+#
+# Description:
+# Common setup code to be sourced by Spack's test scripts.
+#
+
+QA_DIR="$(dirname ${BASH_SOURCE[0]})"
+SPACK_ROOT="$QA_DIR/../../.."
+
+# Source the setup script
+. "$SPACK_ROOT/share/spack/setup-env.sh"
+
+# Set up some variables for running coverage tests.
+if [[ "$COVERAGE" == true ]]; then
+ coverage=coverage
+ coverage_run="coverage run"
+ coverage_combine="coverage combine"
+else
+ coverage=""
+ coverage_run=""
+ coverage_combine=""
+fi
+
+#
+# Description:
+# Check to see if dependencies are installed.
+# If not, warn the user and tell them how to
+# install these dependencies.
+#
+# Usage:
+# check-deps <dep> ...
+#
+# Options:
+# One or more dependencies. Must use name of binary.
+check_dependencies() {
+ for dep in "$@"; do
+ if ! which $dep &> /dev/null; then
+ # Map binary name to package name
+ case $dep in
+ sphinx-apidoc|sphinx-build)
+ spack_package=py-sphinx
+ pip_package=sphinx
+ ;;
+ coverage)
+ spack_package=py-coverage
+ pip_package=coverage
+ ;;
+ flake8)
+ spack_package=py-flake8
+ pip_package=flake8
+ ;;
+ dot)
+ spack_package=graphviz
+ ;;
+ git)
+ spack_package=git
+ ;;
+ hg)
+ spack_package=mercurial
+ pip_package=mercurial
+ ;;
+ svn)
+ spack_package=subversion
+ ;;
+ *)
+ spack_package=$dep
+ pip_package=$dep
+ ;;
+ esac
+
+ echo "ERROR: $dep is required to run this script."
+ echo
+
+ if [[ $spack_package ]]; then
+ echo "To install with Spack, run:"
+ echo " $ spack install $spack_package"
+ fi
+
+ if [[ $pip_package ]]; then
+ echo "To install with pip, run:"
+ echo " $ pip install $pip_package"
+ fi
+
+ if [[ $spack_package || $pip_package ]]; then
+ echo "Then add the bin directory to your PATH."
+ fi
+
+ exit 1
+ fi
+
+ # Flake8 and Sphinx require setuptools in order to run.
+ # Otherwise, they print out this error message:
+ #
+ # Traceback (most recent call last):
+ # File: "/usr/bin/flake8", line 5, in <module>
+ # from pkg_resources import load_entry_point
+ # ImportError: No module named pkg_resources
+ #
+ # Print a more useful error message if setuptools not found.
+ if [[ $dep == flake8 || $dep == sphinx* ]]; then
+ # Find which Python is being run
+ # Spack-installed packages have a hard-coded shebang
+ python_cmd=$(head -n 1 $(which $dep) | cut -c 3-)
+ # May not have a shebang
+ if [[ $python_cmd != *python* ]]; then
+ python_cmd=python
+ fi
+ # Check if setuptools is in the PYTHONPATH
+ if ! $python_cmd -c "import setuptools" 2> /dev/null; then
+ echo "ERROR: setuptools is required to run $dep."
+ echo "Please add it to your PYTHONPATH."
+
+ exit 1
+ fi
+ fi
+ done
+ echo "Dependencies found."
+}
--
GitLab