Merge branch 'it4i-karolina'

easyblocks/a/aocc.py

+##
+# Copyright 2020-2021 Forschungszentrum Juelich GmbH
+#
+# This file is triple-licensed under GPLv2 (see below), MIT, and
+# BSD three-clause licenses.
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+Support for installing AOCC, implemented as an easyblock.
+
+@author: Sebastian Achilles (Forschungszentrum Juelich GmbH)
+"""
+
+import os
+import stat
+
+from distutils.version import LooseVersion
+
+from easybuild.easyblocks.generic.packedbinary import PackedBinary
+from easybuild.framework.easyconfig import CUSTOM
+from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.filetools import adjust_permissions, move_file, write_file
+from easybuild.tools.systemtools import get_shared_lib_ext
+
+# Wrapper script definition
+WRAPPER_TEMPLATE = """#!/bin/sh
+
+%(compiler_name)s --gcc-toolchain=$EBROOTGCCCORE "$@"
+"""
+
+
+class EB_AOCC(PackedBinary):
+    """
+    Support for installing the AOCC compilers
+    """
+
+    @staticmethod
+    def extra_options():
+        extra_vars = {
+            'clangversion': [None, "Clang Version on which AOCC is based on (10.0.0 or 11.0.0 or ...)", CUSTOM],
+        }
+        return PackedBinary.extra_options(extra_vars)
+
+    def __init__(self, *args, **kwargs):
+        """Easyblock constructor, define custom class variables specific to AOCC."""
+        super(EB_AOCC, self).__init__(*args, **kwargs)
+
+        self.clangversion = self.cfg['clangversion']
+
+    def _aocc_guess_clang_version(self):
+        map_aocc_to_clang_ver = {
+            '2.3.0': '11.0.0',
+            '3.0.0': '12.0.0',
+            '3.1.0': '12.0.0',
+        }
+
+        if self.version in map_aocc_to_clang_ver:
+            return map_aocc_to_clang_ver[self.version]
+        else:
+            error_lines = [
+                "AOCC is based on Clang. Guessing Clang version in easyblock failed.",
+                "You should either:",
+                "- specify `clangversion` in the easyconfig;",
+                "- extend `map_aocc_to_clang_ver` in the easyblock;",
+            ]
+            raise EasyBuildError('\n'.join(error_lines))
+
+    def install_step(self):
+        # EULA for AOCC must be accepted via --accept-eula-for EasyBuild configuration option,
+        # or via 'accept_eula = True' in easyconfig file
+        self.check_accepted_eula(more_info='http://developer.amd.com/wordpress/media/files/AOCC_EULA.pdf')
+
+        # AOCC is based on Clang. Try to guess the clangversion from the AOCC version
+        # if clangversion is not specified in the easyconfig
+        if self.clangversion is None:
+            self.clangversion = self._aocc_guess_clang_version()
+
+        super(EB_AOCC, self).install_step()
+
+    def post_install_step(self):
+        """Create wrappers for the compilers to make sure compilers picks up GCCcore as GCC toolchain"""
+
+        orig_compiler_tmpl = '%s.orig'
+
+        def create_wrapper(wrapper_comp):
+            """Create for a particular compiler, with a particular name"""
+            wrapper_f = os.path.join(self.installdir, 'bin', wrapper_comp)
+            write_file(wrapper_f, WRAPPER_TEMPLATE % {'compiler_name': orig_compiler_tmpl % wrapper_comp})
+            perms = stat.S_IXUSR | stat.S_IRUSR | stat.S_IXGRP | stat.S_IRGRP | stat.S_IXOTH | stat.S_IROTH
+            adjust_permissions(wrapper_f, perms)
+
+        compilers_to_wrap = [
+            'clang',
+            'clang++',
+            'clang-%s' % LooseVersion(self.clangversion).version[0],
+            'clang-cpp',
+            'flang',
+        ]
+
+        # Rename original compilers and prepare wrappers to pick up GCCcore as GCC toolchain for the compilers
+        for comp in compilers_to_wrap:
+            actual_compiler = os.path.join(self.installdir, 'bin', comp)
+            if os.path.isfile(actual_compiler):
+                move_file(actual_compiler, orig_compiler_tmpl % actual_compiler)
+            else:
+                err_str = "Tried to move '%s' to '%s', but it does not exist!"
+                raise EasyBuildError(err_str, actual_compiler, '%s.orig' % actual_compiler)
+
+            if not os.path.exists(actual_compiler):
+                create_wrapper(comp)
+                self.log.info("Wrapper for %s successfully created", comp)
+            else:
+                err_str = "Creating wrapper for '%s' not possible, since original compiler was not renamed!"
+                raise EasyBuildError(err_str, actual_compiler)
+
+        super(EB_AOCC, self).post_install_step()
+
+    def sanity_check_step(self):
+        """Custom sanity check for AOCC, based on sanity check for Clang."""
+        shlib_ext = get_shared_lib_ext()
+        custom_paths = {
+            'files': [
+                'bin/clang', 'bin/clang++', 'bin/flang', 'bin/lld', 'bin/llvm-ar', 'bin/llvm-as', 'bin/llvm-config',
+                'bin/llvm-link', 'bin/llvm-nm', 'bin/llvm-symbolizer', 'bin/opt', 'bin/scan-build', 'bin/scan-view',
+                'include/clang-c/Index.h', 'include/llvm-c/Core.h', 'lib/clang/%s/include/omp.h' % self.clangversion,
+                'lib/clang/%s/include/stddef.h' % self.clangversion, 
+                'lib/libclang.%s' % shlib_ext, 'lib/libomp.%s' % shlib_ext,
+            ],
+            'dirs': ['include/llvm', 'lib/clang/%s/lib' % self.clangversion, 'lib32'],
+        }
+
+        custom_commands = [
+            "clang --help",
+            "clang++ --help",
+            "clang-%s --help" % LooseVersion(self.clangversion).version[0],
+            "clang-cpp --help",
+            "flang --help",
+            "llvm-config --cxxflags",
+        ]
+        super(EB_AOCC, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)
+
+    def make_module_extra(self):
+        """Custom variables for AOCC module."""
+        txt = super(EB_AOCC, self).make_module_extra()
+        # we set the symbolizer path so that asan/tsan give meanfull output by default
+        asan_symbolizer_path = os.path.join(self.installdir, 'bin', 'llvm-symbolizer')
+        txt += self.module_generator.set_environment('ASAN_SYMBOLIZER_PATH', asan_symbolizer_path)
+        # setting the AOCChome path
+        txt += self.module_generator.set_environment('AOCChome', self.installdir)
+        return txt
+
+    def make_module_req_guess(self):
+        """
+        A dictionary of possible directories to look for.
+        Include C_INCLUDE_PATH and CPLUS_INCLUDE_PATH as an addition to default ones
+        """
+        guesses = super(EB_AOCC, self).make_module_req_guess()
+        guesses['C_INCLUDE_PATH'] = ['include']
+        guesses['CPLUS_INCLUDE_PATH'] = ['include']
+        return guesses

easyblocks/i/intel_compilers.py

+# #
+# Copyright 2021-2021 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+# #
+"""
+EasyBuild support for installing Intel compilers, implemented as an easyblock
+
+@author: Kenneth Hoste (Ghent University)
+"""
+import os
+from distutils.version import LooseVersion
+
+from easybuild.easyblocks.generic.intelbase import IntelBase
+from easybuild.easyblocks.t.tbb import get_tbb_gccprefix
+from easybuild.tools.build_log import EasyBuildError, print_msg
+from easybuild.tools.run import run_cmd
+
+
+class EB_intel_minus_compilers(IntelBase):
+    """
+    Support for installing Intel compilers, starting with verion 2021.x (oneAPI)
+    """
+
+    def __init__(self, *args, **kwargs):
+        """
+        Easyblock constructor: check version
+        """
+        super(EB_intel_minus_compilers, self).__init__(*args, **kwargs)
+
+        # this easyblock is only valid for recent versions of the Intel compilers (2021.x, oneAPI)
+        if LooseVersion(self.version) < LooseVersion('2021'):
+            raise EasyBuildError("Invalid version %s, should be >= 2021.x" % self.version)
+
+        self.compilers_subdir = os.path.join('compiler', self.version, 'linux')
+        # note that tbb may have a lower version number than the compiler, so use 'latest' symlink
+        # for example compiler 2021.1.2 has tbb 2021.1.1.
+        self.tbb_subdir = os.path.join('tbb', 'latest')
+
+    def prepare_step(self, *args, **kwargs):
+        """
+        Prepare environment for installing.
+
+        Specify that oneAPI versions of Intel compilers don't require a runtime license.
+        """
+        # avoid that IntelBase trips over not having license info specified
+        kwargs['requires_runtime_license'] = False
+
+        super(EB_intel_minus_compilers, self).prepare_step(*args, **kwargs)
+
+    def configure_step(self):
+        """Configure installation."""
+
+        # redefine $HOME for install step, to avoid that anything is stored in $HOME/intel
+        # (like the 'installercache' database)
+        self.cfg['preinstallopts'] += " HOME=%s " % self.builddir
+
+    def install_step(self):
+        """
+        Install step: install each 'source file' one by one.
+        Installing the Intel compilers could be done via a single installation file (HPC Toolkit),
+        or with separate installation files (patch releases of the C++ and Fortran compilers).
+        """
+        srcs = self.src[:]
+        cnt = len(srcs)
+        for idx, src in enumerate(srcs):
+            print_msg("installing part %d/%s (%s)..." % (idx + 1, cnt, src['name']))
+            self.src = [src]
+            super(EB_intel_minus_compilers, self).install_step()
+
+    def sanity_check_step(self):
+        """
+        Custom sanity check for Intel compilers.
+        """
+
+        classic_compiler_cmds = ['icc', 'icpc', 'ifort']
+        oneapi_compiler_cmds = [
+            'dpcpp',  # Intel oneAPI Data Parallel C++ compiler
+            'icx',  # oneAPI Intel C compiler
+            'icpx',  # oneAPI Intel C++ compiler
+            'ifx',  # oneAPI Intel Fortran compiler
+        ]
+        bindir = os.path.join(self.compilers_subdir, 'bin')
+        classic_compiler_paths = [os.path.join(bindir, x) for x in oneapi_compiler_cmds]
+        oneapi_compiler_paths = [os.path.join(bindir, 'intel64', x) for x in classic_compiler_cmds]
+
+        custom_paths = {
+            'files': classic_compiler_paths + oneapi_compiler_paths,
+            'dirs': [self.compilers_subdir],
+        }
+
+        all_compiler_cmds = classic_compiler_cmds + oneapi_compiler_cmds
+        custom_commands = ["which %s" % c for c in all_compiler_cmds]
+
+        # only for 2021.x versions do all compiler commands have the expected version;
+        # for example: for 2022.0.1, icc has version 2021.5.0, icpx has 2022.0.0
+        if LooseVersion(self.version) >= LooseVersion('2022.0'):
+            custom_commands.extend("%s --version" % c for c in all_compiler_cmds)
+        else:
+            custom_commands.extend("%s --version | grep %s" % (c, self.version) for c in all_compiler_cmds)
+
+        super(EB_intel_minus_compilers, self).sanity_check_step(custom_paths=custom_paths,
+                                                                custom_commands=custom_commands)
+
+    def make_module_req_guess(self):
+        """
+        Paths to consider for prepend-paths statements in module file
+        """
+        libdirs = [
+            'lib',
+            os.path.join('lib', 'x64'),
+            os.path.join('compiler', 'lib', 'intel64_lin'),
+        ]
+        libdirs = [os.path.join(self.compilers_subdir, x) for x in libdirs]
+        # resolve 'latest' symlink for tbb (if module guess is run with install in place)
+        if os.path.islink(os.path.join(self.installdir, self.tbb_subdir)):
+            tbb_version = os.readlink(os.path.join(self.installdir, self.tbb_subdir))
+        else:
+            tbb_version = 'latest'
+        tbb_subdir = os.path.join('tbb', tbb_version)
+        tbb_libsubdir = os.path.join(tbb_subdir, 'lib', 'intel64')
+        libdirs.append(os.path.join(tbb_libsubdir,
+                                    get_tbb_gccprefix(os.path.join(self.installdir, tbb_libsubdir))))
+        guesses = {
+            'PATH': [
+                os.path.join(self.compilers_subdir, 'bin'),
+                os.path.join(self.compilers_subdir, 'bin', 'intel64'),
+            ],
+            'LD_LIBRARY_PATH': libdirs,
+            'LIBRARY_PATH': libdirs,
+            'OCL_ICD_FILENAMES': [
+                os.path.join(self.compilers_subdir, 'lib', 'x64', 'libintelocl.so'),
+            ],
+            'CPATH': [
+                os.path.join(tbb_subdir, 'include'),
+            ],
+            'TBBROOT': [tbb_subdir],
+        }
+        return guesses
+
+    def make_module_extra(self):
+        """Additional custom variables for intel-compiler"""
+        txt = super(EB_intel_minus_compilers, self).make_module_extra()
+
+        # On Debian/Ubuntu, /usr/include/x86_64-linux-gnu, or whatever dir gcc uses, needs to be included
+        # in $CPATH for Intel C compiler
+        multiarch_out, ec = run_cmd("gcc -print-multiarch", simple=False)
+        multiarch_out = multiarch_out.strip()
+        if ec == 0 and multiarch_out:
+            multiarch_inc_dir, ec = run_cmd("gcc -E -Wp,-v -xc /dev/null 2>&1 | grep %s$" % multiarch_out)
+            if ec == 0 and multiarch_inc_dir:
+                multiarch_inc_dir = multiarch_inc_dir.strip()
+                self.log.info("Adding multiarch include path %s to $CPATH in generated module file", multiarch_inc_dir)
+                # system location must be appended at the end, so use append_paths
+                txt += self.module_generator.append_paths('CPATH', [multiarch_inc_dir], allow_abs=True)
+
+        return txt