Skip to content

Commit 01c4005

Browse files
boegelboegel
authored
Merge pull request #3343 from Micket/ebpythonprefixes
update easyblock to let EasyBuild framework take care of prepend to `$PYTHONPATH` or `$EBPYTHONPREFIXES` + make `PythonBundle` and `PythonPackage` aware of `--prefer-python-search-path` EasyBuild configuration option
2 parents 710a3f6 + cb728d4 commit 01c4005

File tree

14 files changed

+39
-96
lines changed

14 files changed

+39
-96
lines changed

‎easybuild/easyblocks/a/amber.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -381,7 +381,5 @@ def make_module_extra(self):
381381
txt = super(EB_Amber, self).make_module_extra()
382382

383383
txt += self.module_generator.set_environment('AMBERHOME', self.installdir)
384-
if self.pylibdir:
385-
txt += self.module_generator.prepend_paths('PYTHONPATH', self.pylibdir)
386384

387385
return txt

‎easybuild/easyblocks/c/cplex.py‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,6 @@
3838

3939
import easybuild.tools.environment as env
4040
from easybuild.easyblocks.generic.binary import Binary
41-
from easybuild.easyblocks.generic.pythonpackage import det_pylibdir
42-
from easybuild.easyblocks.python import EBPYTHONPREFIXES
4341
from easybuild.tools.build_log import EasyBuildError
4442
from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir
4543
from easybuild.tools.modules import get_software_root
@@ -150,12 +148,6 @@ def make_module_extra(self):
150148
txt += self.module_generator.set_environment('CPLEX_HOME', os.path.join(self.installdir, 'cplex'))
151149
txt += self.module_generator.set_environment('CPLEXDIR', os.path.join(self.installdir, 'cplex'))
152150

153-
if self.with_python:
154-
if self.multi_python:
155-
txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '')
156-
else:
157-
txt += self.module_generator.prepend_paths('PYTHONPATH', [det_pylibdir()])
158-
159151
self.log.debug("make_module_extra added %s" % txt)
160152
return txt
161153

‎easybuild/easyblocks/e/esmf.py‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -148,23 +148,14 @@ def install_step(self):
148148
run_shell_cmd(cmd)
149149

150150
def make_module_extra(self):
151-
"""Add install path to PYTHONPATH or EBPYTHONPREFIXES"""
151+
"""Set $ESMFMKFILE environment variable"""
152152
txt = super(EB_ESMF, self).make_module_extra()
153153

154154
# set environment variable ESMFMKFILE
155155
# see section 9.9 in https://earthsystemmodeling.org/docs/release/latest/ESMF_usrdoc/node10.html
156156
esmf_mkfile_path = os.path.join(self.installdir, "lib", "esmf.mk")
157157
txt += self.module_generator.set_environment('ESMFMKFILE', esmf_mkfile_path)
158158

159-
if self.cfg['multi_deps'] and 'Python' in self.cfg['multi_deps']:
160-
txt += self.module_generator.prepend_paths('EBPYTHONPREFIXES', '')
161-
else:
162-
python = get_software_version('Python')
163-
if python:
164-
pyshortver = '.'.join(get_software_version('Python').split('.')[:2])
165-
pythonpath = os.path.join('lib', 'python%s' % pyshortver, 'site-packages')
166-
txt += self.module_generator.prepend_paths('PYTHONPATH', [pythonpath])
167-
168159
return txt
169160

170161
def sanity_check_step(self):

‎easybuild/easyblocks/g/gurobi.py‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
"""
3232
import os
3333

34-
from easybuild.easyblocks.generic.pythonpackage import det_pylibdir
3534
from easybuild.easyblocks.generic.tarball import Tarball
3635
from easybuild.framework.easyconfig import CUSTOM
3736
from easybuild.tools import LooseVersion
@@ -101,10 +100,6 @@ def make_module_extra(self):
101100
txt = super(EB_Gurobi, self).make_module_extra()
102101
txt += self.module_generator.set_environment('GUROBI_HOME', self.installdir)
103102
txt += self.module_generator.set_environment('GRB_LICENSE_FILE', self.license_file)
104-
105-
if get_software_root('Python'):
106-
txt += self.module_generator.prepend_paths('PYTHONPATH', det_pylibdir())
107-
108103
txt += self.module_generator.prepend_paths('MATLABPATH', 'matlab')
109104

110105
return txt

‎easybuild/easyblocks/generic/pythonbundle.py‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,10 @@
3131
import sys
3232

3333
from easybuild.easyblocks.generic.bundle import Bundle
34-
from easybuild.easyblocks.generic.pythonpackage import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES
34+
from easybuild.easyblocks.generic.pythonpackage import EXTS_FILTER_PYTHON_PACKAGES
3535
from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, pick_python_cmd
3636
from easybuild.tools.build_log import EasyBuildError
37+
from easybuild.tools.config import build_option, PYTHONPATH, EBPYTHONPREFIXES
3738
from easybuild.tools.filetools import which
3839
from easybuild.tools.modules import get_software_root
3940
import easybuild.tools.environment as env
@@ -146,9 +147,20 @@ def make_module_extra(self, *args, **kwargs):
146147
txt = super(Bundle, self).make_module_extra(*args, **kwargs)
147148

148149
# update $EBPYTHONPREFIXES rather than $PYTHONPATH
149-
# if this Python package was installed for multiple Python versions
150-
if self.multi_python:
151-
txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '')
150+
# if this Python package was installed for multiple Python versions, or if we prefer it
151+
use_ebpythonprefixes = False
152+
runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
153+
154+
if 'Python' in runtime_deps:
155+
self.log.info("Found Python runtime dependency, so considering $EBPYTHONPREFIXES...")
156+
if build_option('prefer_python_search_path') == EBPYTHONPREFIXES:
157+
self.log.info("Preferred Python search path is $EBPYTHONPREFIXES, so using that")
158+
use_ebpythonprefixes = True
159+
160+
if self.multi_python or use_ebpythonprefixes:
161+
path = '' # EBPYTHONPREFIXES are relative to the install dir
162+
if path not in self.module_generator.added_paths_per_key[EBPYTHONPREFIXES]:
163+
txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, path)
152164
else:
153165

154166
# the temporary module file that is generated before installing extensions
@@ -163,7 +175,8 @@ def make_module_extra(self, *args, **kwargs):
163175
]
164176

165177
for pylibdir in new_pylibdirs:
166-
txt += self.module_generator.prepend_paths('PYTHONPATH', pylibdir)
178+
if pylibdir not in self.module_generator.added_paths_per_key[PYTHONPATH]:
179+
txt += self.module_generator.prepend_paths(PYTHONPATH, pylibdir)
167180

168181
return txt
169182

‎easybuild/easyblocks/generic/pythonpackage.py‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,13 @@
4242

4343
import easybuild.tools.environment as env
4444
from easybuild.base import fancylogger
45-
from easybuild.easyblocks.python import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES
45+
from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES
4646
from easybuild.framework.easyconfig import CUSTOM
4747
from easybuild.framework.easyconfig.default import DEFAULT_CONFIG
4848
from easybuild.framework.easyconfig.templates import PYPI_SOURCE
4949
from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
5050
from easybuild.tools.build_log import EasyBuildError, print_msg
51-
from easybuild.tools.config import build_option
51+
from easybuild.tools.config import build_option, PYTHONPATH, EBPYTHONPREFIXES
5252
from easybuild.tools.filetools import change_dir, mkdir, remove_dir, symlink, which
5353
from easybuild.tools.modules import get_software_root
5454
from easybuild.tools.run import run_shell_cmd, subprocess_popen_text
@@ -1162,15 +1162,27 @@ def make_module_extra(self, *args, **kwargs):
11621162
txt = ''
11631163

11641164
# update $EBPYTHONPREFIXES rather than $PYTHONPATH
1165-
# if this Python package was installed for multiple Python versions
1166-
if self.multi_python:
1167-
txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '')
1165+
# if this Python package was installed for multiple Python versions, or if we prefer it;
1166+
# note: although EasyBuild framework also has logic for this in EasyBlock.make_module_extra,
1167+
# we retain full control here, since the logic is slightly different
1168+
use_ebpythonprefixes = False
1169+
runtime_deps = [dep['name'] for dep in self.cfg.dependencies(runtime_only=True)]
1170+
1171+
if 'Python' in runtime_deps:
1172+
self.log.info("Found Python runtime dependency, so considering $EBPYTHONPREFIXES...")
1173+
if build_option('prefer_python_search_path') == EBPYTHONPREFIXES:
1174+
self.log.info("Preferred Python search path is $EBPYTHONPREFIXES, so using that")
1175+
use_ebpythonprefixes = True
1176+
1177+
if self.multi_python or use_ebpythonprefixes:
1178+
path = '' # EBPYTHONPREFIXES are relative to the install dir
1179+
txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, path)
11681180
elif self.require_python:
11691181
self.set_pylibdirs()
11701182
for path in self.all_pylibdirs:
11711183
fullpath = os.path.join(self.installdir, path)
11721184
# only extend $PYTHONPATH with existing, non-empty directories
11731185
if os.path.exists(fullpath) and os.listdir(fullpath):
1174-
txt += self.module_generator.prepend_paths('PYTHONPATH', path)
1186+
txt += self.module_generator.prepend_paths(PYTHONPATH, path)
11751187

11761188
return super(PythonPackage, self).make_module_extra(txt, *args, **kwargs)

‎easybuild/easyblocks/l/lammps.py‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -548,19 +548,6 @@ def sanity_check_step(self, *args, **kwargs):
548548

549549
return super(EB_LAMMPS, self).sanity_check_step(custom_commands=custom_commands, custom_paths=custom_paths)
550550

551-
def make_module_extra(self):
552-
"""Add install path to PYTHONPATH"""
553-
554-
txt = super(EB_LAMMPS, self).make_module_extra()
555-
556-
python = get_software_version('Python')
557-
if python:
558-
pyshortver = '.'.join(get_software_version('Python').split('.')[:2])
559-
pythonpath = os.path.join('lib', 'python%s' % pyshortver, 'site-packages')
560-
txt += self.module_generator.prepend_paths('PYTHONPATH', [pythonpath])
561-
562-
return txt
563-
564551

565552
def get_cuda_gpu_arch(cuda_cc):
566553
"""Return CUDA gpu ARCH in LAMMPS required format. Example: 'sm_32' """

‎easybuild/easyblocks/m/mxnet.py‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -252,12 +252,6 @@ def make_module_extra(self, *args, **kwargs):
252252
"""Custom variables for MXNet module."""
253253
txt = super(EB_MXNet, self).make_module_extra(*args, **kwargs)
254254

255-
for path in self.py_ext.all_pylibdirs:
256-
fullpath = os.path.join(self.installdir, path)
257-
# only extend $PYTHONPATH with existing, non-empty directories
258-
if os.path.exists(fullpath) and os.listdir(fullpath):
259-
txt += self.module_generator.prepend_paths('PYTHONPATH', path)
260-
261255
txt += self.module_generator.prepend_paths("R_LIBS", ['']) # prepend R_LIBS with install path
262256

263257
return txt

‎easybuild/easyblocks/n/neuron.py‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -283,11 +283,4 @@ def make_module_extra(self):
283283
else:
284284
self.log.debug("%s not set: %s" % (var, os.environ.get(var, None)))
285285

286-
if self.with_python:
287-
if self.cfg['multi_deps'] and 'Python' in self.cfg['multi_deps']:
288-
txt += self.module_generator.prepend_paths('EBPYTHONPREFIXES', '')
289-
else:
290-
txt += self.module_generator.prepend_paths('PYTHONPATH', [self.pylibdir])
291-
# also adds lib/python to PYTHONPATH
292-
txt += self.module_generator.prepend_paths('PYTHONPATH', ['lib/python'])
293286
return txt

‎easybuild/easyblocks/o/openbabel.py‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import glob
3232
import os
3333
from easybuild.easyblocks.generic.cmakemake import CMakeMake
34-
from easybuild.easyblocks.generic.pythonpackage import det_pylibdir
3534
from easybuild.framework.easyconfig import CUSTOM
3635
from easybuild.tools.build_log import EasyBuildError
3736
from easybuild.tools.modules import get_software_root, get_software_version
@@ -108,14 +107,6 @@ def sanity_check_step(self):
108107
def make_module_extra(self):
109108
"""Custom variables for OpenBabel module."""
110109
txt = super(EB_OpenBabel, self).make_module_extra()
111-
if self.with_python:
112-
if LooseVersion(self.version) >= LooseVersion('2.4'):
113-
# since OpenBabel 2.4.0 the Python bindings under
114-
# ${PREFIX}/lib/python2.7/site-packages rather than ${PREFIX}/lib
115-
ob_pythonpath = det_pylibdir()
116-
else:
117-
ob_pythonpath = 'lib'
118-
txt += self.module_generator.prepend_paths('PYTHONPATH', [ob_pythonpath])
119110
babel_libdir = os.path.join(self.installdir, 'lib', 'openbabel', self.version)
120111
txt += self.module_generator.set_environment('BABEL_LIBDIR', babel_libdir)
121112
babel_datadir = os.path.join(self.installdir, 'share', 'openbabel', self.version)

0 commit comments

Comments
 (0)