Merge pull request #1 from boegel/clean_module_generator

sync with develop

Author: damianam

.travis.yml

@@ -7,8 +7,8 @@ env:
     - LMOD_VERSION=5.8 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
     - LMOD_VERSION=6.6.3
     - LMOD_VERSION=6.6.3 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
-    - LMOD_VERSION=7.1
-    - LMOD_VERSION=7.1 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
+    - LMOD_VERSION=7.4
+    - LMOD_VERSION=7.4 TEST_EASYBUILD_MODULE_SYNTAX=Tcl
     - ENV_MOD_VERSION=3.2.10 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesC TEST_EASYBUILD_MODULE_SYNTAX=Tcl
     - ENV_MOD_TCL_VERSION=1.147 TEST_EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl TEST_EASYBUILD_MODULE_SYNTAX=Tcl
 matrix:
@@ -43,7 +43,7 @@ before_install:
     # pydot (dep for python-graph-dot) 1.2.0 and more recent doesn't work with Python 2.6
     - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then pip install pydot==1.1.0; else pip install pydot; fi
     # optional Python packages for EasyBuild
-    - pip install autopep8 GC3Pie pep8 python-graph-dot python-hglib PyYAML
+    - pip install autopep8 GC3Pie pycodestyle python-graph-dot python-hglib PyYAML
     # git config is required to make actual git commits (cfr. tests for GitRepository)
     - git config --global user.name "Travis CI"
     - git config --global user.email "[email protected]"

RELEASE_NOTES

@@ -3,6 +3,48 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html.
 
+v3.1.2 (March 20th 2017)
+------------------------
+
+bugfix release
+- fix broken packaging support by fixing ``run_cmd`` bug with ``shell=False`` (#2153)
+- minor enhancements, including:
+  - implement change_dir function in filetools module (#2155)
+  - use checker_state in trailing whitespace style check + add dedicated test (#2160, #2169)
+  - consider both pycodestyle and pep8 Python modules in style checks (#2161)
+  - make bootstrap script aware various modules-related $EASYBUILD_* environment variables (#2170)
+- various bug fixes, including:
+  - interpret statements that extend $MODULEPATH in modpath_extensions_for method (#2104)
+    - also fixes inclusion of superfluous load statements in modules installed in user HMNS module tree (cfr. #2172)
+  - take into account that $PATH or $PYTHONPATH may be empty when running tests (#2149)
+  - handle duplicates in --include-* (#2151)
+  - exclude dependencies of dependencies that extend $MODULEPATH (#2152)
+  - add ld.bfd to RPATH wrappers (#2156)
+  - fix test_vsc_location, vsc.__file__ may not be available when vsc is installed as a namespace package (#2159)
+  - fix reported problems with scripts & docs tests (#2164)
+  - fix --try-software-version using non-greedy matching + lookahead assertion, add test for tweak_one (#2166)
+  - avoid creating empty modulefile directories when using modaltsoftname (#2168)
+  - fix fftw_libs for MKL without interface libraries (#2171)
+
+
+v3.1.1 (March 7th 2017)
+-----------------------
+
+bugfix release
+- minor enhancements, including:
+  - print more useful error message when no compiler-specific optarch flag is defined (#1950)
+  - add ec parameter to expand_toolchain_load() (#2103)
+  - clarify unstable/closed PR warning message (#2129)
+- various bug fixes, including:
+  - ensure that $EBEXTSLIST* is also included in generated module under --module-only (#2112)
+  - fix formatting issues in generated documentation for --list-software and --avail-easyconfig-licenses (#2121)
+  - fix problem with backticks in description breaking 'fpm' packaging command (#2124)
+  - replace --enable-new-dtags with --disable-new-dtags instead of removing it in RPATH wrapper script (#2131)
+  - only perform is_short_modname_for sanity check in det_short_module_name if modaltsoftname is available (#2138)
+  - fix logic in make_module_dep w.r.t. excluding loads for toolchain & toolchain components (#2140)
+  - skip test_check_style if pep8 is not available (#2142)
+
+
 v3.1.0 (February 3rd 2017)
 --------------------------
 

easybuild/framework/easyblock.py

@@ -65,7 +65,7 @@
 from easybuild.tools.config import install_path, log_path, package_path, source_paths
 from easybuild.tools.environment import restore_env, sanitize_env
 from easybuild.tools.filetools import DEFAULT_CHECKSUM
-from easybuild.tools.filetools import adjust_permissions, apply_patch, convert_name, derive_alt_pypi_url
+from easybuild.tools.filetools import adjust_permissions, apply_patch, change_dir, convert_name, derive_alt_pypi_url
 from easybuild.tools.filetools import compute_checksum, download_file, encode_class_name, extract_file
 from easybuild.tools.filetools import is_alt_pypi_url, mkdir, move_logs, read_file, remove_file, rmtree2, write_file
 from easybuild.tools.filetools import verify_checksum, weld_paths
@@ -912,21 +912,23 @@ def make_module_dep(self, unload_info=None):
 
         :param unload_info: dictionary with full module names as keys and module name to unload first as corr. value
         """
-        deps = []
         mns = ActiveMNS()
         unload_info = unload_info or {}
 
-        # include load statements for toolchain, either directly or for toolchain dependencies
+        # include toolchain as first dependency to load
+        tc_mod = None
         if self.toolchain.name != DUMMY_TOOLCHAIN_NAME:
-            if mns.expand_toolchain_load():
-                mod_names = self.toolchain.toolchain_dep_mods
-                deps.extend(mod_names)
-                self.log.debug("Adding toolchain components as module dependencies: %s" % mod_names)
-            else:
-                deps.append(self.toolchain.det_short_module_name())
-                self.log.debug("Adding toolchain %s as a module dependency" % deps[-1])
+            tc_mod = self.toolchain.det_short_module_name()
+            self.log.debug("Toolchain to load in generated module (before excluding any deps): %s", tc_mod)
+
+        # expand toolchain into toolchain components if desired
+        tc_dep_mods = None
+        if mns.expand_toolchain_load(ec=self.cfg):
+            tc_dep_mods = self.toolchain.toolchain_dep_mods
+            self.log.debug("Toolchain components to load in generated module (before excluding any): %s", tc_dep_mods)
 
         # include load/unload statements for dependencies
+        deps = []
         self.log.debug("List of deps considered to load in generated module: %s", self.toolchain.dependencies)
         for dep in self.toolchain.dependencies:
             if dep['build_only']:
@@ -935,35 +937,52 @@ def make_module_dep(self, unload_info=None):
                 modname = dep['short_mod_name']
                 self.log.debug("Adding %s as a module dependency" % modname)
                 deps.append(modname)
-
         self.log.debug("List of deps to load in generated module (before excluding any): %s", deps)
 
         # exclude dependencies that extend $MODULEPATH and form the path to the top of the module tree (if any)
         full_mod_subdir = os.path.join(self.installdir_mod, self.mod_subdir)
         init_modpaths = mns.det_init_modulepaths(self.cfg)
         top_paths = [self.installdir_mod] + [os.path.join(self.installdir_mod, p) for p in init_modpaths]
+
+        all_deps = [d for d in [tc_mod] + (tc_dep_mods or []) + deps if d is not None]
         excluded_deps = self.modules_tool.path_to_top_of_module_tree(top_paths, self.cfg.short_mod_name,
-                                                                     full_mod_subdir, deps)
+                                                                     full_mod_subdir, all_deps)
+
+        # if the toolchain is excluded, so should the toolchain components
+        if tc_mod in excluded_deps and tc_dep_mods:
+            excluded_deps.extend(tc_dep_mods)
+
         self.log.debug("List of excluded deps: %s", excluded_deps)
 
-        # load modules that open up the module tree before checking deps of deps (in reverse order)
-        self.modules_tool.load(excluded_deps[::-1])
+        # expand toolchain into toolchain components if desired
+        if tc_dep_mods is not None:
+            deps = tc_dep_mods + deps
+        elif tc_mod is not None:
+            deps = [tc_mod] + deps
 
+        # filter dependencies to avoid including loads for toolchain or toolchain components that extend $MODULEPATH
+        # with location to where this module is being installed (full_mod_subdir);
+        # if the modules that extend $MODULEPATH are not loaded this module is not available, so there is not
+        # point in loading them again (in fact, it may cause problems when reloading this module due to a load storm)
         deps = [d for d in deps if d not in excluded_deps]
-        for dep in excluded_deps:
-            excluded_dep_deps = dependencies_for(dep, self.modules_tool)
-            self.log.debug("List of dependencies for excluded dependency %s: %s" % (dep, excluded_dep_deps))
+
+        # load modules that open up the module tree before checking deps of deps (in reverse order)
+        self.modules_tool.load(excluded_deps[::-1])
+ 
+        for excluded_dep in excluded_deps:
+            excluded_dep_deps = dependencies_for(excluded_dep, self.modules_tool)
+            self.log.debug("List of dependencies for excluded dependency %s: %s" % (excluded_dep, excluded_dep_deps))
             deps = [d for d in deps if d not in excluded_dep_deps]
 
-        self.log.debug("List of retained deps to load in generated module: %s" % deps)
-        recursive_unload = self.cfg['recursive_module_unload']
+        self.log.debug("List of retained deps to load in generated module: %s", deps)
 
+        # include load statements for retained dependencies
         loads = []
         for dep in deps:
             unload_modules = []
             if dep in unload_info:
                 unload_modules.append(unload_info[dep])
-            loads.append(self.module_generator.load_module(dep, recursive_unload=recursive_unload,
+            loads.append(self.module_generator.load_module(dep, recursive_unload=self.cfg['recursive_module_unload'],
                                                            unload_modules=unload_modules))
 
         # Force unloading any other modules
@@ -1111,10 +1130,7 @@ def make_module_req(self):
 
         lines = []
         if os.path.isdir(self.installdir):
-            try:
-                os.chdir(self.installdir)
-            except OSError, err:
-                raise EasyBuildError("Failed to change to %s: %s", self.installdir, err)
+            change_dir(self.installdir)
 
             lines.append('\n')
 
@@ -1143,10 +1159,7 @@ def make_module_req(self):
                     lines.append(self.module_generator.prepend_paths(key, paths))
             if self.dry_run:
                 self.dry_run_msg('')
-            try:
-                os.chdir(self.orig_workdir)
-            except OSError, err:
-                raise EasyBuildError("Failed to change back to %s: %s", self.orig_workdir, err)
+            change_dir(self.orig_workdir)
 
         return ''.join(lines)
 
@@ -1332,11 +1345,8 @@ def guess_start_dir(self):
 
         self.log.info("Using %s as start dir", self.cfg['start_dir'])
 
-        try:
-            os.chdir(self.cfg['start_dir'])
-            self.log.debug("Changed to real build directory %s (start_dir)" % self.cfg['start_dir'])
-        except OSError, err:
-            raise EasyBuildError("Can't change to real build directory %s: %s", self.cfg['start_dir'], err)
+        change_dir(self.cfg['start_dir'])
+        self.log.debug("Changed to real build directory %s (start_dir)", self.cfg['start_dir'])
 
     def handle_iterate_opts(self):
         """Handle options relevant during iterated part of build/install procedure."""
@@ -1509,13 +1519,13 @@ def fetch_step(self, skip_checksums=False):
         # create parent dirs in install and modules path already
         # this is required when building in parallel
         mod_symlink_paths = ActiveMNS().det_module_symlink_paths(self.cfg)
-        parent_subdir = os.path.dirname(self.install_subdir)
+        mod_subdir = os.path.dirname(ActiveMNS().det_full_module_name(self.cfg))
         pardirs = [
             self.installdir,
-            os.path.join(self.installdir_mod, parent_subdir),
+            os.path.join(self.installdir_mod, mod_subdir),
         ]
         for mod_symlink_path in mod_symlink_paths:
-            pardirs.append(os.path.join(install_path('mod'), mod_symlink_path, parent_subdir))
+            pardirs.append(os.path.join(install_path('mod'), mod_symlink_path, mod_subdir))
 
         self.log.info("Checking dirs that need to be created: %s" % pardirs)
         for pardir in pardirs:
@@ -1707,7 +1717,7 @@ def extensions_step(self, fetch=False):
             print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent)
 
             # always go back to original work dir to avoid running stuff from a dir that no longer exists
-            os.chdir(self.orig_workdir)
+            change_dir(self.orig_workdir)
 
             cls, inst = None, None
             class_name = encode_class_name(ext['name'])
@@ -1717,7 +1727,8 @@ def extensions_step(self, fetch=False):
             try:
                 # no error when importing class fails, in case we run into an existing easyblock
                 # with a similar name (e.g., Perl Extension 'GO' vs 'Go' for which 'EB_Go' is available)
-                cls = get_easyblock_class(None, name=ext['name'], default_fallback=False, error_on_failed_import=False)
+                cls = get_easyblock_class(None, name=ext['name'], error_on_failed_import=False,
+                                          error_on_missing_easyblock=False)
                 self.log.debug("Obtained class %s for extension %s" % (cls, ext['name']))
                 if cls is not None:
                     inst = cls(self, ext)
@@ -2033,10 +2044,7 @@ def _sanity_check_step(self, custom_paths=None, custom_commands=None, extension=
 
         # chdir to installdir (better environment for running tests)
         if os.path.isdir(self.installdir):
-            try:
-                os.chdir(self.installdir)
-            except OSError, err:
-                raise EasyBuildError("Failed to move to installdir %s: %s", self.installdir, err)
+            change_dir(self.installdir)
 
         # run sanity check commands
         for command in commands:
@@ -2074,6 +2082,18 @@ def _sanity_check_step(self, custom_paths=None, custom_commands=None, extension=
         else:
             self.log.debug("Sanity check passed!")
 
+    def _set_module_as_default(self):
+        """
+        Defining default module Version
+
+        sets the default module version except if we are in dry run.
+        """
+        if self.dry_run:
+            dry_run_msg("Marked %s v%s as default version" % (self.name, self.version))
+        else:
+            mod_folderpath = os.path.dirname(self.module_generator.get_module_filepath())
+            self.module_generator.set_as_default(mod_folderpath, self.version)
+
     def cleanup_step(self):
         """
         Cleanup leftover mess: remove/clean build directory
@@ -2082,11 +2102,12 @@ def cleanup_step(self):
         cleanup_builddir is False, otherwise we remove the installation
         """
         if not self.build_in_installdir and build_option('cleanup_builddir'):
-            try:
-                os.chdir(self.orig_workdir)  # make sure we're out of the dir we're removing
 
-                self.log.info("Cleaning up builddir %s (in %s)" % (self.builddir, os.getcwd()))
+            # make sure we're out of the dir we're removing
+            change_dir(self.orig_workdir)
+            self.log.info("Cleaning up builddir %s (in %s)", self.builddir, os.getcwd())
 
+            try:
                 rmtree2(self.builddir)
                 base = os.path.dirname(self.builddir)
 
@@ -2137,7 +2158,6 @@ def make_module_step(self, fake=False):
                 self.dry_run_msg("Generating module file %s, with contents:\n", mod_filepath)
                 for line in txt.split('\n'):
                     self.dry_run_msg(' ' * 4 + line)
-
         else:
             write_file(mod_filepath, txt)
             self.log.info("Module file %s written: %s", mod_filepath, txt)
@@ -2162,6 +2182,9 @@ def make_module_step(self, fake=False):
             if not fake:
                 self.make_devel_module()
 
+        if build_option('set_default_module'):
+            self._set_module_as_default()
+
         return modpath
 
     def permissions_step(self):
@@ -2218,7 +2241,7 @@ def test_cases_step(self):
         Run provided test cases.
         """
         for test in self.cfg['tests']:
-            os.chdir(self.orig_workdir)
+            change_dir(self.orig_workdir)
             if os.path.isabs(test):
                 path = test
             else:
@@ -2514,7 +2537,7 @@ def build_and_install_one(ecdict, init_env):
     ended = 'ended'
 
     # make sure we're back in original directory before we finish up
-    os.chdir(cwd)
+    change_dir(cwd)
 
     application_log = None
 
@@ -2709,7 +2732,7 @@ def perform_step(step, obj, method, logfile):
             start_time = time.time()
 
             # start with a clean slate
-            os.chdir(base_dir)
+            change_dir(base_dir)
             restore_env(base_env)
             sanitize_env()