run pip check only once for PythonBundle

Author: Flamefire

easybuild/easyblocks/generic/pythonbundle.py

@@ -28,13 +28,11 @@
 @author: Kenneth Hoste (Ghent University)
 """
 import os
-import sys
 
 from easybuild.easyblocks.generic.bundle import Bundle
 from easybuild.easyblocks.generic.pythonpackage import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES
-from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, pick_python_cmd
+from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, find_python_cmd, run_pip_check
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.filetools import which
 from easybuild.tools.modules import get_software_root
 import easybuild.tools.environment as env
 
@@ -74,49 +72,33 @@ def __init__(self, *args, **kwargs):
 
             self.log.info("exts_default_options: %s", self.cfg['exts_default_options'])
 
+        self.python_cmd = None
         self.pylibdir = None
-        self.all_pylibdirs = []
+        self.all_pylibdirs = None
 
         # figure out whether this bundle of Python packages is being installed for multiple Python versions
         self.multi_python = 'Python' in self.cfg['multi_deps']
 
-    def prepare_step(self, *args, **kwargs):
-        """Prepare for installing bundle of Python packages."""
-        super(Bundle, self).prepare_step(*args, **kwargs)
+    def prepare_python(self):
+        """Python-specific preparations."""
 
-        python_root = get_software_root('Python')
-        if python_root is None:
+        if get_software_root('Python') is None:
             raise EasyBuildError("Python not included as dependency!")
+        self.python_cmd = find_python_cmd(self.log, self.cfg['req_py_majver'], self.cfg['req_py_minver'], required=True)
 
-        # when system Python is used, the first 'python' command in $PATH will not be $EBROOTPYTHON/bin/python,
-        # since $EBROOTPYTHON is set to just 'Python' in that case
-        # (see handling of allow_system_deps in EasyBlock.prepare_step)
-        if which('python') == os.path.join(python_root, 'bin', 'python'):
-            # if we're using a proper Python dependency, let det_pylibdir use 'python' like it does by default
-            python_cmd = None
-        else:
-            # since det_pylibdir will use 'python' by default as command to determine Python lib directory,
-            # we need to intervene when the system Python is used, by specifying version requirements
-            # to pick_python_cmd so the right 'python' command is used;
-            # if we're using the system Python and no Python version requirements are specified,
-            # use major/minor version of Python being used in this EasyBuild session (as we also do in PythonPackage)
-            req_py_majver = self.cfg['req_py_majver']
-            if req_py_majver is None:
-                req_py_majver = sys.version_info[0]
-            req_py_minver = self.cfg['req_py_minver']
-            if req_py_minver is None:
-                req_py_minver = sys.version_info[1]
-
-            python_cmd = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver)
-
-        self.all_pylibdirs = get_pylibdirs(python_cmd=python_cmd)
+        self.all_pylibdirs = get_pylibdirs(python_cmd=self.python_cmd)
         self.pylibdir = self.all_pylibdirs[0]
 
         # if 'python' is not used, we need to take that into account in the extensions filter
         # (which is also used during the sanity check)
-        if python_cmd:
+        if self.python_cmd != 'python':
             orig_exts_filter = EXTS_FILTER_PYTHON_PACKAGES
-            self.cfg['exts_filter'] = (orig_exts_filter[0].replace('python', python_cmd), orig_exts_filter[1])
+            self.cfg['exts_filter'] = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1])
+
+    def prepare_step(self, *args, **kwargs):
+        """Prepare for installing bundle of Python packages."""
+        super(Bundle, self).prepare_step(*args, **kwargs)
+        self.prepare_python()
 
     def extensions_step(self, *args, **kwargs):
         """Install extensions (usually PythonPackages)"""
@@ -158,6 +140,13 @@ def make_module_extra(self, *args, **kwargs):
     def sanity_check_step(self, *args, **kwargs):
         """Custom sanity check for bundle of Python package."""
 
+        if self.pylibdir is None:
+            # Python attributes not set up yet, happens e.g. with --sanity-check-only
+            # Load module first to get the right python command
+            self.fake_mod_data = self.sanity_check_load_module(extension=kwargs.get('extension', False),
+                                                               extra_modules=kwargs.get('extra_modules', None))
+            self.prepare_python()
+
         # inject directory path that uses %(pyshortver)s template into default value for sanity_check_paths
         # this is relevant for installations of Python bundles for multiple Python versions (via multi_deps)
         # (we can not pass this via custom_paths, since then the %(pyshortver)s template value will not be resolved)
@@ -168,3 +157,24 @@ def sanity_check_step(self, *args, **kwargs):
             }
 
         super(Bundle, self).sanity_check_step(*args, **kwargs)
+
+        # After the super-call self.ext_instances is initialized, so we can check the extensions
+        sanity_pip_check = self.cfg['sanity_pip_check']
+        unversioned_packages = set(self.cfg['unversioned_packages'])
+        has_sanity_pip_check_mismatch = False
+        all_unversioned_packages = unversioned_packages.copy()
+        for ext in self.ext_instances:
+            if isinstance(ext, PythonPackage):
+                if ext.cfg['sanity_pip_check'] != sanity_pip_check:
+                    has_sanity_pip_check_mismatch = True
+                all_unversioned_packages.update(ext.cfg['unversioned_packages'])
+
+        if has_sanity_pip_check_mismatch:
+            self.log.deprecated('For bundles of PythonPackage the sanity_pip_check option '
+                                'in the main EasyConfig must be used', '5.0')
+            sanity_pip_check = True  # Either the main set it or any extension enabled it
+        if all_unversioned_packages != unversioned_packages:
+            self.log.deprecated('For bundles of PythonPackage the unversioned_packages option '
+                                'in the main EasyConfig must be used', '5.0')
+        if sanity_pip_check:
+            run_pip_check(self.log, self.python_cmd, all_unversioned_packages)