Remove support for generating dotkit files (#11986)

Dotkit is being used only at a few sites and has been deprecated on new
machines. This commit removes all the code that provide support for the
generation of dotkit module files.

A new validator named "deprecatedProperties" has been added to the
jsonschema validators. It permits to prompt a warning message or exit
with an error if a property that has been marked as deprecated is
encountered.

* Removed references to dotkit in the docs
* Removed references to dotkit in setup-env-test.sh
* Added a unit test for the 'deprecatedProperties' schema validator

lib/spack/spack/test/cmd/module.py

@@ -21,7 +21,6 @@ def _module_files(module_type, *specs):
 
 @pytest.fixture(scope='module', autouse=True)
 def ensure_module_files_are_there(database):
-    module('dotkit', 'refresh', '-y')
     module('tcl', 'refresh', '-y')
 
 
@@ -39,7 +38,7 @@ def failure_args(request):
 
 
 @pytest.fixture(
-    params=['dotkit', 'tcl', 'lmod']
+    params=['tcl', 'lmod']
 )
 def module_type(request):
     return request.param

lib/spack/spack/test/data/config/modules.yaml

@@ -16,7 +16,6 @@
 modules:
   enable:
     - tcl
-    - dotkit
   prefix_inspections:
     bin:
       - PATH

lib/spack/spack/test/data/modules/dotkit/autoload_direct.yaml

-enable:
-  - dotkit
-dotkit:
-  all:
-    autoload: 'direct'

lib/spack/spack/test/data/modules/dotkit/override_template.yaml

-enable:
-  - dotkit
-dotkit:
-  all:
-    template: 'override_from_modules.txt'

lib/spack/spack/test/modules/dotkit.py

-# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-
-import pytest
-import spack.modules.dotkit
-
-#: Class of the writer tested in this module
-writer_cls = spack.modules.dotkit.DotkitModulefileWriter
-
-
-@pytest.mark.usefixtures('config', 'mock_packages')
-class TestDotkit(object):
-
-    def test_dotkit(self, modulefile_content, module_configuration):
-        """Tests the generation of a dotkit file that loads dependencies
-        automatically.
-        """
-
-        module_configuration('autoload_direct')
-        content = modulefile_content('mpileaks arch=x86-linux')
-
-        assert '#c spack' in content
-        assert '#d mpileaks @2.3' in content
-        assert len([x for x in content if 'dk_op' in x]) == 2
-
-    def test_override_template_in_package(
-            self, modulefile_content, module_configuration
-    ):
-        """Tests overriding a template from and attribute in the package."""
-
-        module_configuration('autoload_direct')
-        content = modulefile_content('override-module-templates')
-
-        assert 'Override successful!' in content
-
-    def test_override_template_in_modules_yaml(
-            self, modulefile_content, module_configuration
-    ):
-        """Tests overriding a template from `modules.yaml`"""
-
-        module_configuration('override_template')
-
-        # Check that this takes precedence over an attribute in the package
-        content = modulefile_content('override-module-templates')
-        assert 'Override even better!' in content
-
-        content = modulefile_content('mpileaks arch=x86-linux')
-        assert 'Override even better!' in content

lib/spack/spack/test/schema.py

@@ -114,3 +114,28 @@ def test_schema_validation(meta_schema, config_name):
 
     # If this validation throws the test won't pass
     jsonschema.validate(schema, meta_schema)
+
+
+def test_deprecated_properties(module_suffixes_schema):
+    # Test that an error is reported when 'error: True'
+    module_suffixes_schema['deprecatedProperties'] = {
+        'properties': ['tcl'],
+        'message': '{property} not allowed',
+        'error': True
+    }
+    v = spack.schema.Validator(module_suffixes_schema)
+    data = {'tcl': {'all': {'suffixes': {'^python': 'py'}}}}
+
+    with pytest.raises(jsonschema.ValidationError, match='tcl not allowed'):
+        v.validate(data)
+
+    # Test that just a warning is reported when 'error: False'
+    module_suffixes_schema['deprecatedProperties'] = {
+        'properties': ['tcl'],
+        'message': '{property} not allowed',
+        'error': False
+    }
+    v = spack.schema.Validator(module_suffixes_schema)
+    data = {'tcl': {'all': {'suffixes': {'^python': 'py'}}}}
+    # The next validation doesn't raise anymore
+    v.validate(data)

share/spack/csh/spack.csh

@@ -5,27 +5,27 @@
 
 ########################################################################
 # This is a wrapper around the spack command that forwards calls to
-# 'spack use' and 'spack unuse' to shell functions.  This in turn
-# allows them to be used to invoke dotkit functions.
+# 'spack load' and 'spack unload' to shell functions.  This in turn
+# allows them to be used to invoke environment-modules functions.
 #
-# 'spack use' is smarter than just 'use' because it converts its
-# arguments into a unique spack spec that is then passed to dotkit
+# 'spack load' is smarter than just 'load' because it converts its
+# arguments into a unique Spack spec that is then passed to environment-modules
 # commands.  This allows the user to use packages without knowing all
 # their installation details.
 #
-# e.g., rather than requring a full spec for libelf, the user can type:
+# e.g., rather than requiring a full spec for libelf, the user can type:
 #
-#     spack use libelf
+#     spack load libelf
 #
-# This will first find the available libelf dotkits and use a
+# This will first find the available libelf module file and use a
 # matching one.  If there are two versions of libelf, the user would
 # need to be more specific, e.g.:
 #
-#     spack use libelf@0.8.13
+#     spack load libelf@0.8.13
 #
 # This is very similar to how regular spack commands work and it
 # avoids the need to come up with a user-friendly naming scheme for
-# spack dotfiles.
+# spack module files.
 ########################################################################
 # accumulate initial flags for main spack command
 set _sp_flags = ""
@@ -104,8 +104,6 @@ case env:
                 breaksw
         endsw
     endif
-case use:
-case unuse:
 case load:
 case unload:
     set _sp_module_args=""""
@@ -115,22 +113,10 @@ case unload:
         set _sp_spec = ($_sp_spec)
     endif
 
-    # Here the user has run use or unuse with a spec.  Find a matching
+    # Here the user has run load or unload with a spec.  Find a matching
     # spec using 'spack module find', then use the appropriate module
     # tool's commands to add/remove the result from the environment.
     switch ($_sp_subcommand)
-        case "use":
-            set _sp_full_spec = ( "`\spack $_sp_flags module dotkit find $_sp_spec`" )
-            if ( $? == 0 ) then
-                use $_sp_module_args $_sp_full_spec
-            endif
-            breaksw
-        case "unuse":
-            set _sp_full_spec = ( "`\spack $_sp_flags module dotkit find $_sp_spec`" )
-            if ( $? == 0 ) then
-                unuse $_sp_module_args $_sp_full_spec
-            endif
-            breaksw
         case "load":
             # _sp_module_args may be "-r" for recursive spec retrieval
             set _sp_full_spec = ( "`\spack $_sp_flags module tcl find $_sp_module_args $_sp_spec`" )

share/spack/qa/setup-env-test.sh

@@ -167,7 +167,7 @@ is_not_set() {
 
 
 # -----------------------------------------------------------------------
-# Instead of invoking the module/use/dotkit commands, we print the
+# Instead of invoking the module commands, we print the
 # arguments that Spack invokes the command with, so we can check that
 # Spack passes the expected arguments in the tests below.
 #
@@ -177,14 +177,6 @@ module() {
     echo module "$@"
 }
 
-use() {
-    echo use "$@"
-}
-
-unuse() {
-    echo unuse "$@"
-}
-
 # -----------------------------------------------------------------------
 # Setup test environment and do some preliminary checks
 # -----------------------------------------------------------------------
@@ -219,11 +211,9 @@ echo "Creating a mock package installation"
 spack -m install --fake a
 a_install=$(spack location -i a)
 a_module=$(spack -m module tcl find a)
-a_dotkit=$(spack -m module dotkit find a)
 
 b_install=$(spack location -i b)
 b_module=$(spack -m module tcl find b)
-b_dotkit=$(spack -m module dotkit find b)
 
 # create a test environment for tesitng environment commands
 echo "Creating a mock environment"
@@ -304,26 +294,6 @@ contains "usage: spack unload " spack -m unload -h
 contains "usage: spack unload " spack -m unload -h d
 contains "usage: spack unload " spack -m unload --help
 
-title 'Testing `spack use`'
-contains "use $b_dotkit" spack -m use b
-fails spack -m use -l
-contains "use -l --arg $b_dotkit" spack -m use -l --arg b
-contains "use $b_dotkit $a_dotkit" spack -m use -r a
-contains "use $b_dotkit $a_dotkit" spack -m use --dependencies a
-fails spack -m use d
-contains "usage: spack use " spack -m use -h
-contains "usage: spack use " spack -m use -h d
-contains "usage: spack use " spack -m use --help
-
-title 'Testing `spack unuse`'
-contains "unuse $b_dotkit" spack -m unuse b
-fails spack -m unuse -l
-contains "unuse -l --arg $b_dotkit" spack -m unuse -l --arg b
-fails spack -m unuse d
-contains "usage: spack unuse "  spack -m unuse -h
-contains "usage: spack unuse "  spack -m unuse -h d
-contains "usage: spack unuse "  spack -m unuse --help
-
 title 'Testing `spack env`'
 contains "usage: spack env " spack env -h
 contains "usage: spack env " spack env --help

share/spack/setup-env.csh

@@ -6,7 +6,7 @@
 
 #
 # This file is part of Spack and sets up the spack environment for
-# csh and tcsh.  This includes dotkit support, module support, and
+# csh and tcsh.  This includes environment modules and lmod support, and
 # it also puts spack in your path.  Source it like this:
 #
 #    setenv SPACK_ROOT /path/to/spack
@@ -24,7 +24,7 @@ if ($?SPACK_ROOT) then
     _spack_pathadd PATH "$SPACK_ROOT/bin"
     eval `spack --print-shell-vars csh`
 
-    # Set up modules and dotkit search paths in the user environment
+    # Set up module search paths in the user environment
     set tcl_roots = `echo $_sp_tcl_roots:q | sed 's/:/ /g'`
     set compatible_sys_types = `echo $_sp_compatible_sys_types:q | sed 's/:/ /g'`
     foreach tcl_root ($tcl_roots:q)
@@ -33,10 +33,6 @@ if ($?SPACK_ROOT) then
         end
     end
 
-    set dotkit_roots = `echo $_sp_dotkit_roots:q | sed 's/:/ /g'`
-    foreach dotkit_root ($dotkit_roots)
-        _spack_pathadd DK_NODE "$dotkit_root/$_sp_sys_type"
-    end
 else
     echo "ERROR: Sourcing spack setup-env.csh requires setting SPACK_ROOT to "
     echo "       the root of your spack installation."

share/spack/setup-env.sh

@@ -7,8 +7,8 @@
 ########################################################################
 #
 # This file is part of Spack and sets up the spack environment for bash,
-# zsh, and dash (sh).  This includes dotkit support, module support, and
-# it also puts spack in your path.  The script also checks that at least
+# zsh, and dash (sh).  This includes environment modules and lmod support,
+# and it also puts spack in your path. The script also checks that at least
 # module support exists, and provides suggestions if it doesn't. Source
 # it like this:
 #
@@ -16,27 +16,27 @@
 #
 ########################################################################
 # This is a wrapper around the spack command that forwards calls to
-# 'spack use' and 'spack unuse' to shell functions.  This in turn
-# allows them to be used to invoke dotkit functions.
+# 'spack load' and 'spack unload' to shell functions.  This in turn
+# allows them to be used to invoke environment modules functions.
 #
-# 'spack use' is smarter than just 'use' because it converts its
-# arguments into a unique spack spec that is then passed to dotkit
+# 'spack load' is smarter than just 'load' because it converts its
+# arguments into a unique Spack spec that is then passed to module
 # commands.  This allows the user to use packages without knowing all
 # their installation details.
 #
 # e.g., rather than requiring a full spec for libelf, the user can type:
 #
-#     spack use libelf
+#     spack load libelf
 #
-# This will first find the available libelf dotkits and use a
+# This will first find the available libelf module file and use a
 # matching one.  If there are two versions of libelf, the user would
 # need to be more specific, e.g.:
 #
-#     spack use libelf@0.8.13
+#     spack load libelf@0.8.13
 #
 # This is very similar to how regular spack commands work and it
 # avoids the need to come up with a user-friendly naming scheme for
-# spack dotfiles.
+# spack module files.
 ########################################################################
 
 spack() {
@@ -140,7 +140,7 @@ spack() {
             fi
             return
             ;;
-        "use"|"unuse"|"load"|"unload")
+        "load"|"unload")
             # Shift any other args for use off before parsing spec.
             _sp_subcommand_args=""
             _sp_module_args=""
@@ -161,20 +161,6 @@ spack() {
             # tool's commands to add/remove the result from the environment.
             # If spack module command comes back with an error, do nothing.
             case $_sp_subcommand in
-                "use")
-                    if _sp_full_spec=$(command spack $_sp_flags module dotkit find $_sp_subcommand_args "$@"); then
-                        use $_sp_module_args $_sp_full_spec
-                    else
-                        $(exit 1)
-                    fi
-                    ;;
-                "unuse")
-                    if _sp_full_spec=$(command spack $_sp_flags module dotkit find $_sp_subcommand_args "$@"); then
-                        unuse $_sp_module_args $_sp_full_spec
-                    else
-                        $(exit 1)
-                    fi
-                    ;;
                 "load")
                     if _sp_full_spec=$(command spack $_sp_flags module tcl find $_sp_subcommand_args "$@"); then
                         module load $_sp_module_args $_sp_full_spec
@@ -356,7 +342,6 @@ _sp_multi_pathadd() {
     done
 }
 _sp_multi_pathadd MODULEPATH "$_sp_tcl_roots"
-_sp_multi_pathadd DK_NODE "$_sp_dotkit_roots"
 
 # Add programmable tab completion for Bash
 #

share/spack/spack-completion.bash

@@ -760,7 +760,7 @@ function _spack_module {
     then
         compgen -W "-h --help" -- "$cur"
     else
-        compgen -W "lmod tcl dotkit" -- "$cur"
+        compgen -W "lmod tcl" -- "$cur"
     fi
 }
 
@@ -812,53 +812,6 @@ function _spack_module_tcl_rm {
     fi
 }
 
-function _spack_module_dotkit {
-    if $list_options
-    then
-        compgen -W "-h --help" -- "$cur"
-    else
-        compgen -W "refresh find rm loads" -- "$cur"
-    fi
-}
-
-
-function _spack_module_dotkit_find {
-    if $list_options
-    then
-        compgen -W "-h --help --full-path -r --dependencies" -- "$cur"
-    else
-        compgen -W "$(_installed_packages)" -- "$cur"
-    fi
-}
-
-function _spack_module_dotkit_loads {
-    if $list_options
-    then
-        compgen -W "-h --help --input-only -p --prefix -x --exclude
-                    -r --dependencies" -- "$cur"
-    else
-        compgen -W "$(_installed_packages)" -- "$cur"
-    fi
-
-}
-
-function _spack_module_dotkit_refresh {
-    if $list_options
-    then
-        compgen -W "-h --help --delete-tree -y --yes-to-all" -- "$cur"
-    else
-        compgen -W "$(_installed_packages)" -- "$cur"
-    fi
-}
-
-function _spack_module_dotkit_rm {
-    if $list_options
-    then
-        compgen -W "-h --help -y --yes-to-all" -- "$cur"
-    else
-        compgen -W "$(_installed_packages)" -- "$cur"
-    fi
-}
 
 function _spack_module_lmod {
     if $list_options

share/spack/templates/modules/modulefile.dk

-{% block header %}
-{% if category %}
-#c {{ category }}
-{% endif %}
-{% if short_description %}
-#d {{ short_description }}
-{% endif %}
-{% if long_description %}
-{{ long_description| textwrap(72)| prepend_to_line('#h ')| join() }}
-{% endif %}
-{% endblock %}
-
-{% block autoloads %}
-{% for module in autoload %}
-dk_op {{ module }}
-{% endfor %}
-{% endblock %}
-
-{% block environment %}
-{% for command_name, cmd in environment_modifications %}
-{% if command_name == 'PrependPath' %}
-dk_alter {{ cmd.name }} {{ cmd.value }}
-{% endif %}
-{% if command_name == 'RemovePath' %}
-dk_unalter {{ cmd.name }} {{ cmd.value }}
-{% endif %}
-{% if command_name == 'SetEnv' %}
-dk_setenv {{ cmd.name }} {{ cmd.value }}
-{% endif %}
-{% endfor %}
-{% endblock %}

var/spack/repos/builtin.mock/packages/override-module-templates/package.py

@@ -12,7 +12,6 @@ class OverrideModuleTemplates(Package):
 
     version('1.0', 'foobarbaz')
 
-    dotkit_template = 'override.txt'
     tcl_template = 'override.txt'
     lmod_template = 'override.txt'
 

var/spack/repos/builtin/packages/go/package.py

@@ -134,6 +134,6 @@ def setup_dependent_environment(self, spack_env, run_env, dependent_spec):
         # This *MUST* be first, this is where new code is installed
         spack_env.set('GOPATH', ':'.join(path_components))
 
-        # Allow packages to find this when using module or dotkit
+        # Allow packages to find this when using module files
         run_env.prepend_path('GOPATH', ':'.join(
             [dependent_spec.prefix] + path_components))