Skip to content
Snippets Groups Projects
Commit 4ff8cbe5 authored by Lukáš Krupčík's avatar Lukáš Krupčík
Browse files

Merge branch 'it4i-salomon'

parents 7b62bb79 6083c47c
Branches
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
easyblocks/a/amber.py deleted 100644 → 0
+ 0
− 237
View file @ 7b62bb79
##
# Copyright 2009-2018 Ghent University
# Copyright 2015-2018 Stanford University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
# the Hercules foundation (http://www.herculesstichting.be/in_English)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# https://github.com/easybuilders/easybuild
#
# EasyBuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation v2.
#
# EasyBuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
##
"""
EasyBuild support for Amber, implemented as an easyblock
Original author: Benjamin Roberts (The University of Auckland)
Modified by Stephane Thiell (Stanford University) for Amber14
Enhanced/cleaned up by Kenneth Hoste (HPC-UGent)
"""
import os
import easybuild.tools.environment as env
import easybuild.tools.toolchain as toolchain
from easybuild.easyblocks.generic.configuremake import ConfigureMake
from easybuild.easyblocks.generic.pythonpackage import det_pylibdir
from easybuild.framework.easyconfig import CUSTOM, MANDATORY, BUILD
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.modules import get_software_root, get_software_version
from easybuild.tools.run import run_cmd
class EB_Amber(ConfigureMake):
"""Easyblock for building and installing Amber"""
@staticmethod
def extra_options(extra_vars=None):
"""Extra easyconfig parameters specific to ConfigureMake."""
extra_vars = dict(ConfigureMake.extra_options(extra_vars))
extra_vars.update({
# 'Amber': [True, "Build Amber in addition to AmberTools", CUSTOM],
'patchlevels': ["latest", "(AmberTools, Amber) updates to be applied", CUSTOM],
# The following is necessary because some patches to the Amber update
# script update the update script itself, in which case it will quit
# and insist on being run again. We don't know how many times will
# be needed, but the number of times is patchlevel specific.
'patchruns': [1, "Number of times to run Amber's update script before building", CUSTOM],
# enable testing by default
'runtest': [True, "Run tests after each build", CUSTOM],
})
return ConfigureMake.extra_options(extra_vars)
def __init__(self, *args, **kwargs):
"""Easyblock constructor: initialise class variables."""
super(EB_Amber, self).__init__(*args, **kwargs)
self.build_in_installdir = True
self.pylibdir = None
self.with_cuda = False
self.with_mpi = False
env.setvar('AMBERHOME', self.installdir)
def extract_step(self):
"""Extract sources; strip off parent directory during unpack"""
self.cfg.update('unpack_options', "--strip-components=1")
super(EB_Amber, self).extract_step()
def patch_step(self, *args, **kwargs):
"""Patch Amber using 'update_amber' tool, prior to applying listed patch files (if any)."""
if self.cfg['patchlevels'] == "latest":
cmd = "./update_amber --update"
# Run as many times as specified. It is the responsibility
# of the easyconfig author to get this right, especially if
# he or she selects "latest". (Note: "latest" is not
# recommended for this reason and others.)
for _ in range(self.cfg['patchruns']):
run_cmd(cmd, log_all=True)
else:
for (tree, patch_level) in zip(['AmberTools', 'Amber'], self.cfg['patchlevels']):
if patch_level == 0:
continue
cmd = "./update_amber --update-to %s/%s" % (tree, patch_level)
# Run as many times as specified. It is the responsibility
# of the easyconfig author to get this right.
for _ in range(self.cfg['patchruns']):
run_cmd(cmd, log_all=True)
super(EB_Amber, self).patch_step(*args, **kwargs)
def configure_step(self):
"""Configuring Amber is done in install step."""
pass
def build_step(self):
"""Building Amber is done in install step."""
pass
def test_step(self):
"""Testing Amber build is done in install step."""
pass
def install_step(self):
"""Custom build, test & install procedure for Amber."""
# unset $LIBS since it breaks the build
env.unset_env_vars(['LIBS'])
# define environment variables for MPI, BLAS/LAPACK & dependencies
mklroot = get_software_root('imkl')
openblasroot = get_software_root('OpenBLAS')
if mklroot:
env.setvar('MKL_HOME', mklroot)
elif openblasroot:
lapack = os.getenv('LIBLAPACK')
if lapack is None:
raise EasyBuildError("LIBLAPACK (from OpenBLAS) not found in environement.")
else:
env.setvar('GOTO', lapack)
mpiroot = get_software_root(self.toolchain.MPI_MODULE_NAME[0])
if mpiroot and self.toolchain.options.get('usempi', None):
env.setvar('MPI_HOME', mpiroot)
self.with_mpi = True
if self.toolchain.mpi_family() == toolchain.INTELMPI:
self.mpi_option = '-intelmpi'
else:
self.mpi_option = '-mpi'
common_configopts = [self.cfg['configopts'], '--no-updates', '-noX11']
if self.name == 'Amber':
common_configopts.append('-static')
netcdfroot = get_software_root('netCDF')
if netcdfroot:
common_configopts.extend(["--with-netcdf", netcdfroot])
pythonroot = get_software_root('Python')
if pythonroot:
common_configopts.extend(["--with-python", os.path.join(pythonroot, 'bin', 'python')])
self.pylibdir = det_pylibdir()
pythonpath = os.environ.get('PYTHONPATH', '')
env.setvar('PYTHONPATH', os.pathsep.join([os.path.join(self.installdir, self.pylibdir), pythonpath]))
comp_fam = self.toolchain.comp_family()
if comp_fam == toolchain.INTELCOMP:
comp_str = 'intel'
elif comp_fam == toolchain.GCC:
comp_str = 'gnu'
else:
raise EasyBuildError("Don't know how to compile with compiler family '%s' -- check EasyBlock?", comp_fam)
# The NAB compiles need openmp flag
if self.toolchain.options.get('openmp', None):
env.setvar('CUSTOMBUILDFLAGS', self.toolchain.get_flag('openmp'))
# compose list of build targets
build_targets = [('', 'test')]
if self.with_mpi:
build_targets.append((self.mpi_option, 'test.parallel'))
# hardcode to 4 MPI processes, minimal required to run all tests
env.setvar('DO_PARALLEL', 'mpirun -np 4')
cudaroot = get_software_root('CUDA')
if cudaroot:
env.setvar('CUDA_HOME', cudaroot)
self.with_cuda = True
build_targets.append(('-cuda', 'test.cuda'))
if self.with_mpi:
build_targets.append(("-cuda %s" % self.mpi_option, 'test.cuda_parallel'))
ld_lib_path = os.environ.get('LD_LIBRARY_PATH', '')
env.setvar('LD_LIBRARY_PATH', os.pathsep.join([os.path.join(self.installdir, 'lib'), ld_lib_path]))
for flag, testrule in build_targets:
# configure
cmd = "%s ./configure %s" % (self.cfg['preconfigopts'], ' '.join(common_configopts + [flag, comp_str]))
(out, _) = run_cmd(cmd, log_all=True, simple=False)
# build in situ using 'make install'
# note: not 'build'
super(EB_Amber, self).install_step()
# test
if self.cfg['runtest']:
run_cmd("make %s" % testrule, log_all=True, simple=False)
# clean, overruling the normal 'build'
run_cmd("make clean")
def sanity_check_step(self):
"""Custom sanity check for Amber."""
binaries = ['sander', 'tleap']
if self.name == 'Amber':
binaries.append('pmemd')
if self.with_cuda:
binaries.append('pmemd.cuda')
if self.with_mpi:
binaries.append('pmemd.cuda.MPI')
if self.with_mpi:
binaries.extend(['sander.MPI'])
if self.name == 'Amber':
binaries.append('pmemd.MPI')
custom_paths = {
'files': [os.path.join(self.installdir, 'bin', binary) for binary in binaries],
'dirs': [],
}
super(EB_Amber, self).sanity_check_step(custom_paths=custom_paths)
def make_module_extra(self):
"""Add module entries specific to Amber/AmberTools"""
txt = super(EB_Amber, self).make_module_extra()
txt += self.module_generator.set_environment('AMBERHOME', self.installdir)
if self.pylibdir:
txt += self.module_generator.prepend_paths('PYTHONPATH', self.pylibdir)
return txt
easyblocks/l/lammps.py deleted 100644 → 0
+ 0
− 47
View file @ 7b62bb79
#!/usr/bin/env python
#
# Copyright (C) 2015 IT4Innovations
# Lumir Jasiok
# lumir.jasiok@vsb.cz
# http://www.it4i.cz
#
#
#
"""
EasyBuild support for building and installing LAMPPS,
implemented as an easyblock
@author: Lumir Jasiok (IT4 Innovations)
"""
import os
import shutil
from easybuild.framework.easyblock import EasyBlock
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.run import run_cmd
class EB_LAMMPS(EasyBlock):
"""Support for building and installing LAMMPS."""
def build_step(self):
"""Run simply make mkl"""
cmd = "%s make mkl" % self.cfg['prebuildopts']
run_cmd(cmd, log_all=True)
srcdir = os.path.join(self.cfg['start_dir'], 'src')
targetdir = os.path.join(self.installdir, 'bin')
self.log.info("LAMMPS install directory is %s" % self.installdir)
try:
if os.path.isdir(self.installdir):
self.log.info("Directory %s exists, deleting"
% self.installdir)
shutil.rmtree(self.installdir)
os.mkdir(self.installdir)
os.mkdir(targetdir)
shutil.copy2(os.path.join(srcdir, 'lmp_mkl'),
os.path.join(targetdir, 'lmp_mkl'))
except Exception, err:
raise EasyBuildError("Failed to install LAMMPS binary, %s" % err)
easyblocks/m/matlab.py deleted 100644 → 0
+ 0
− 161
View file @ 7b62bb79
##
# Copyright 2009-2018 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# https://github.com/easybuilders/easybuild
#
# EasyBuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation v2.
#
# EasyBuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
##
"""
EasyBuild support for installing MATLAB, implemented as an easyblock
@author: Stijn De Weirdt (Ghent University)
@author: Dries Verdegem (Ghent University)
@author: Kenneth Hoste (Ghent University)
@author: Pieter De Baets (Ghent University)
@author: Jens Timmerman (Ghent University)
@author: Fotis Georgatos (Uni.Lu, NTUA)
"""
import re
import os
import shutil
import stat
from distutils.version import LooseVersion
from easybuild.easyblocks.generic.packedbinary import PackedBinary
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import adjust_permissions, change_dir, read_file, write_file
from easybuild.tools.run import run_cmd
from easybuild.tools.systemtools import get_shared_lib_ext
class EB_MATLAB(PackedBinary):
"""Support for installing MATLAB."""
def __init__(self, *args, **kwargs):
"""Add extra config options specific to MATLAB."""
super(EB_MATLAB, self).__init__(*args, **kwargs)
self.comp_fam = None
self.configfile = os.path.join(self.builddir, 'my_installer_input.txt')
@staticmethod
def extra_options():
extra_vars = {
'java_options': ['-Xmx256m', "$_JAVA_OPTIONS value set for install and in module file.", CUSTOM],
}
return PackedBinary.extra_options(extra_vars)
def configure_step(self):
"""Configure MATLAB installation: create license file."""
licserv = self.cfg['license_server']
if licserv is None:
licserv = os.getenv('EB_MATLAB_LICENSE_SERVER', 'license.example.com')
licport = self.cfg['license_server_port']
if licport is None:
licport = os.getenv('EB_MATLAB_LICENSE_SERVER_PORT', '00000')
key = self.cfg['key']
if key is None:
key = os.getenv('EB_MATLAB_KEY', '00000-00000-00000-00000-00000-00000-00000-00000-00000-00000')
# create license file
lictxt = '\n'.join([
"SERVER %s 000000000000 %s" % (licserv, licport),
"USE_SERVER",
])
licfile = os.path.join(self.builddir, 'matlab.lic')
write_file(licfile, lictxt)
try:
shutil.copyfile(os.path.join(self.cfg['start_dir'], 'installer_input.txt'), self.configfile)
config = read_file(self.configfile)
regdest = re.compile(r"^# destinationFolder=.*", re.M)
regkey = re.compile(r"^# fileInstallationKey=.*", re.M)
regagree = re.compile(r"^# agreeToLicense=.*", re.M)
regmode = re.compile(r"^# mode=.*", re.M)
reglicpath = re.compile(r"^# licensePath=.*", re.M)
config = regdest.sub("destinationFolder=%s" % self.installdir, config)
config = regkey.sub("fileInstallationKey=%s" % key, config)
config = regagree.sub("agreeToLicense=Yes", config)
config = regmode.sub("mode=silent", config)
config = reglicpath.sub("licensePath=%s" % licfile, config)
write_file(self.configfile, config)
except IOError, err:
raise EasyBuildError("Failed to create installation config file %s: %s", self.configfile, err)
self.log.debug('configuration file written to %s:\n %s', self.configfile, config)
def install_step(self):
"""MATLAB install procedure using 'install' command."""
src = os.path.join(self.cfg['start_dir'], 'install')
# make sure install script is executable
adjust_permissions(src, stat.S_IXUSR)
if LooseVersion(self.version) >= LooseVersion('2016b'):
jdir = os.path.join(self.cfg['start_dir'], 'sys', 'java', 'jre', 'glnxa64', 'jre', 'bin')
for perm_dir in [os.path.join(self.cfg['start_dir'], 'bin', 'glnxa64'), jdir]:
adjust_permissions(perm_dir, stat.S_IXUSR)
# make sure $DISPLAY is not defined, which may lead to (hard to trace) problems
# this is a workaround for not being able to specify --nodisplay to the install scripts
if 'DISPLAY' in os.environ:
os.environ.pop('DISPLAY')
if '_JAVA_OPTIONS' not in self.cfg['preinstallopts']:
java_opts = 'export _JAVA_OPTIONS="%s" && ' % self.cfg['java_options']
self.cfg['preinstallopts'] = java_opts + self.cfg['preinstallopts']
if LooseVersion(self.version) >= LooseVersion('2016b'):
change_dir(self.builddir)
cmd = "%s %s -v -inputFile %s %s" % (self.cfg['preinstallopts'], src, self.configfile, self.cfg['installopts'])
(out, _) = run_cmd(cmd, log_all=True, simple=False)
# check installer output for known signs of trouble
patterns = [
"Error: You have entered an invalid File Installation Key",
]
for pattern in patterns:
regex = re.compile(pattern, re.I)
if regex.search(out):
raise EasyBuildError("Found error pattern '%s' in output of installation command '%s': %s",
regex.pattern, cmd, out)
def sanity_check_step(self):
"""Custom sanity check for MATLAB."""
custom_paths = {
'files': ["bin/matlab"],
'dirs': ["java/jar"],
}
super(EB_MATLAB, self).sanity_check_step(custom_paths=custom_paths)
def make_module_extra(self):
"""Extend PATH and set proper _JAVA_OPTIONS (e.g., -Xmx)."""
txt = super(EB_MATLAB, self).make_module_extra()
txt += self.module_generator.set_environment('_JAVA_OPTIONS', self.cfg['java_options'])
return txt
easyblocks/m/mkl.py deleted 100644 → 0
+ 0
− 425
View file @ 7b62bb79
# #
# Copyright 2009-2017 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# https://github.com/easybuilders/easybuild
#
# EasyBuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation v2.
#
# EasyBuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
# #
"""
EasyBuild support for installing the Intel Math Kernel Library (MKL), implemented as an easyblock
@author: Stijn De Weirdt (Ghent University)
@author: Dries Verdegem (Ghent University)
@author: Kenneth Hoste (Ghent University)
@author: Pieter De Baets (Ghent University)
@author: Jens Timmerman (Ghent University)
@author: Ward Poelmans (Ghent University)
@author: Lumir Jasiok (IT4Innovations)
"""
import itertools
import os
import shutil
import tempfile
from distutils.version import LooseVersion
import easybuild.tools.environment as env
import easybuild.tools.toolchain as toolchain
from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, LICENSE_FILE_NAME_2012
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import apply_regex_substitutions, rmtree2
from easybuild.tools.modules import get_software_root
from easybuild.tools.run import run_cmd
from easybuild.tools.systemtools import get_shared_lib_ext
class EB_mkl(IntelBase):
"""
Class that can be used to install mkl
- tested with 10.2.1.017
-- will fail for all older versions (due to newer silent installer)
"""
@staticmethod
def extra_options():
"""Add easyconfig parameters custom to mkl (e.g. interfaces)."""
extra_vars = {
'interfaces': [True, "Indicates whether interfaces should be built", CUSTOM],
}
return IntelBase.extra_options(extra_vars)
def __init__(self, *args, **kwargs):
super(EB_mkl, self).__init__(*args, **kwargs)
# make sure $MKLROOT isn't set, it's known to cause problems with the installation
self.cfg.update('unwanted_env_vars', ['MKLROOT'])
def prepare_step(self, *args, **kwargs):
if LooseVersion(self.version) >= LooseVersion('2017.2.174'):
kwargs['requires_runtime_license'] = False
super(EB_mkl, self).prepare_step(*args, **kwargs)
else:
super(EB_mkl, self).prepare_step(*args, **kwargs)
def install_step(self):
"""
Actual installation
- create silent cfg file
- execute command
"""
silent_cfg_names_map = None
silent_cfg_extras = None
if LooseVersion(self.version) < LooseVersion('11.1'):
# since mkl v11.1, silent.cfg has been slightly changed to be 'more standard'
silent_cfg_names_map = {
'activation_name': ACTIVATION_NAME_2012,
'license_file_name': LICENSE_FILE_NAME_2012,
}
if LooseVersion(self.version) >= LooseVersion('11.1') and self.install_components is None:
silent_cfg_extras = {
'COMPONENTS': 'ALL',
}
super(EB_mkl, self).install_step(silent_cfg_names_map=silent_cfg_names_map, silent_cfg_extras=silent_cfg_extras)
def make_module_req_guess(self):
"""
A dictionary of possible directories to look for
"""
if LooseVersion(self.version) >= LooseVersion('10.3'):
if self.cfg['m32']:
raise EasyBuildError("32-bit not supported yet for IMKL v%s (>= 10.3)", self.version)
else:
retdict = {
'PATH': ['bin', 'mkl/bin', 'mkl/bin/intel64', 'composerxe-2011/bin'],
'LD_LIBRARY_PATH': ['lib/intel64', 'mkl/lib/intel64'],
'LIBRARY_PATH': ['lib/intel64', 'mkl/lib/intel64'],
'MANPATH': ['man', 'man/en_US'],
'CPATH': ['mkl/include', 'mkl/include/fftw'],
}
if LooseVersion(self.version) >= LooseVersion('11.0'):
if LooseVersion(self.version) >= LooseVersion('11.3'):
retdict['MIC_LD_LIBRARY_PATH'] = ['lib/intel64_lin_mic', 'mkl/lib/mic'];
elif LooseVersion(self.version) >= LooseVersion('11.1'):
retdict['MIC_LD_LIBRARY_PATH'] = ['lib/mic', 'mkl/lib/mic'];
else:
retdict['MIC_LD_LIBRARY_PATH'] = ['compiler/lib/mic', 'mkl/lib/mic'];
return retdict;
else:
if self.cfg['m32']:
return {
'PATH': ['bin', 'bin/ia32', 'tbb/bin/ia32'],
'LD_LIBRARY_PATH': ['lib', 'lib/32'],
'LIBRARY_PATH': ['lib', 'lib/32'],
'MANPATH': ['man', 'share/man', 'man/en_US'],
'CPATH': ['include'],
}
else:
return {
'PATH': ['bin', 'bin/intel64', 'tbb/bin/em64t'],
'LD_LIBRARY_PATH': ['lib', 'lib/em64t'],
'LIBRARY_PATH': ['lib', 'lib/em64t'],
'MANPATH': ['man', 'share/man', 'man/en_US'],
'CPATH': ['include'],
}
def make_module_extra(self):
"""Overwritten from Application to add extra txt"""
txt = super(EB_mkl, self).make_module_extra()
txt += self.module_generator.set_environment('MKLROOT', os.path.join(self.installdir, 'mkl'))
return txt
def post_install_step(self):
"""
Install group libraries and interfaces (if desired).
"""
super(EB_mkl, self).post_install_step()
shlib_ext = get_shared_lib_ext()
# reload the dependencies
self.load_dependency_modules()
if self.cfg['m32']:
extra = {
'libmkl.%s' % shlib_ext : 'GROUP (-lmkl_intel -lmkl_intel_thread -lmkl_core)',
'libmkl_em64t.a': 'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)',
'libmkl_solver.a': 'GROUP (libmkl_solver.a)',
'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_core.a)',
'libmkl_lapack.a': 'GROUP (libmkl_intel.a libmkl_intel_thread.a libmkl_core.a)',
'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)'
}
else:
extra = {
'libmkl.%s' % shlib_ext: 'GROUP (-lmkl_intel_lp64 -lmkl_intel_thread -lmkl_core)',
'libmkl_em64t.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)',
'libmkl_solver.a': 'GROUP (libmkl_solver_lp64.a)',
'libmkl_scalapack.a': 'GROUP (libmkl_scalapack_lp64.a)',
'libmkl_lapack.a': 'GROUP (libmkl_intel_lp64.a libmkl_intel_thread.a libmkl_core.a)',
'libmkl_cdft.a': 'GROUP (libmkl_cdft_core.a)'
}
if LooseVersion(self.version) >= LooseVersion('10.3'):
libsubdir = os.path.join('mkl', 'lib', 'intel64')
else:
if self.cfg['m32']:
libsubdir = os.path.join('lib', '32')
else:
libsubdir = os.path.join('lib', 'em64t')
for fil, txt in extra.items():
dest = os.path.join(self.installdir, libsubdir, fil)
if not os.path.exists(dest):
try:
f = open(dest, 'w')
f.write(txt)
f.close()
self.log.info("File %s written" % dest)
except IOError, err:
raise EasyBuildError("Can't write file %s: %s", dest, err)
# build the mkl interfaces, if desired
if self.cfg['interfaces']:
if LooseVersion(self.version) >= LooseVersion('10.3'):
intsubdir = os.path.join('mkl', 'interfaces')
inttarget = 'libintel64'
else:
intsubdir = 'interfaces'
if self.cfg['m32']:
inttarget = 'lib32'
else:
inttarget = 'libem64t'
cmd = "make -f makefile %s" % inttarget
# blas95 and lapack95 need more work, ignore for now
# blas95 and lapack also need include/.mod to be processed
fftw2libs = ['fftw2xc', 'fftw2xf']
fftw3libs = ['fftw3xc', 'fftw3xf']
cdftlibs = ['fftw2x_cdft']
if LooseVersion(self.version) >= LooseVersion('10.3'):
cdftlibs.append('fftw3x_cdft')
interfacedir = os.path.join(self.installdir, intsubdir)
try:
os.chdir(interfacedir)
self.log.info("Changed to interfaces directory %s" % interfacedir)
except OSError, err:
raise EasyBuildError("Can't change to interfaces directory %s", interfacedir)
compopt = None
# determine whether we're using a non-Intel GCC-based or PGI-based toolchain
# can't use toolchain.comp_family, because of dummy toolchain used when installing mkl
if get_software_root('icc') is None:
# check for PGI first, since there's a GCC underneath PGI too...
if get_software_root('PGI'):
compopt = 'compiler=pgi'
elif get_software_root('GCC'):
compopt = 'compiler=gnu'
else:
raise EasyBuildError("Not using Intel/GCC/PGI compilers, don't know how to build wrapper libs")
else:
compopt = 'compiler=intel'
# patch makefiles for cdft wrappers when PGI is used as compiler
if get_software_root('PGI'):
regex_subs = [
# pgi should be considered as a valid compiler
("intel gnu", "intel gnu pgi"),
# transform 'gnu' case to 'pgi' case
(r"ifeq \(\$\(compiler\),gnu\)", "ifeq ($(compiler),pgi)"),
('=gcc', '=pgcc'),
# correct flag to use C99 standard
('-std=c99', '-c99'),
# -Wall and -Werror are not valid options for pgcc, no close equivalent
('-Wall', ''),
('-Werror', ''),
]
for lib in cdftlibs:
apply_regex_substitutions(os.path.join(interfacedir, lib, 'makefile'), regex_subs)
for lib in fftw2libs + fftw3libs + cdftlibs:
buildopts = [compopt]
if lib in fftw3libs:
buildopts.append('install_to=$INSTALL_DIR')
elif lib in cdftlibs:
mpi_spec = None
# check whether MPI_FAMILY constant is defined, so mpi_family() can be used
if hasattr(self.toolchain, 'MPI_FAMILY') and self.toolchain.MPI_FAMILY is not None:
mpi_spec_by_fam = {
toolchain.MPICH: 'mpich2', # MPICH is MPICH v3.x, which is MPICH2 compatible
toolchain.MPICH2: 'mpich2',
toolchain.MVAPICH2: 'mpich2',
toolchain.OPENMPI: 'openmpi',
}
mpi_fam = self.toolchain.mpi_family()
mpi_spec = mpi_spec_by_fam.get(mpi_fam)
self.log.debug("Determined MPI specification based on MPI toolchain component: %s" % mpi_spec)
else:
# can't use toolchain.mpi_family, because of dummy toolchain
if get_software_root('MPICH2') or get_software_root('MVAPICH2'):
mpi_spec = 'mpich2'
elif get_software_root('OpenMPI'):
mpi_spec = 'openmpi'
self.log.debug("Determined MPI specification based on loaded MPI module: %s" % mpi_spec)
if mpi_spec is not None:
buildopts.append('mpi=%s' % mpi_spec)
precflags = ['']
if lib.startswith('fftw2x') and not self.cfg['m32']:
# build both single and double precision variants
precflags = ['PRECISION=MKL_DOUBLE', 'PRECISION=MKL_SINGLE']
intflags = ['']
if lib in cdftlibs and not self.cfg['m32']:
# build both 32-bit and 64-bit interfaces
intflags = ['interface=lp64', 'interface=ilp64']
allopts = [list(opts) for opts in itertools.product(intflags, precflags)]
for flags, extraopts in itertools.product(['', '-fPIC'], allopts):
tup = (lib, flags, buildopts, extraopts)
self.log.debug("Building lib %s with: flags %s, buildopts %s, extraopts %s" % tup)
tmpbuild = tempfile.mkdtemp(dir=self.builddir)
self.log.debug("Created temporary directory %s" % tmpbuild)
# always set INSTALL_DIR, SPEC_OPT, COPTS and CFLAGS
# fftw2x(c|f): use $INSTALL_DIR, $CFLAGS and $COPTS
# fftw3x(c|f): use $CFLAGS
# fftw*cdft: use $INSTALL_DIR and $SPEC_OPT
env.setvar('INSTALL_DIR', tmpbuild)
env.setvar('SPEC_OPT', flags)
env.setvar('COPTS', flags)
env.setvar('CFLAGS', flags)
try:
intdir = os.path.join(interfacedir, lib)
os.chdir(intdir)
self.log.info("Changed to interface %s directory %s" % (lib, intdir))
except OSError, err:
raise EasyBuildError("Can't change to interface %s directory %s: %s", lib, intdir, err)
fullcmd = "%s %s" % (cmd, ' '.join(buildopts + extraopts))
res = run_cmd(fullcmd, log_all=True, simple=True)
if not res:
raise EasyBuildError("Building %s (flags: %s, fullcmd: %s) failed", lib, flags, fullcmd)
for fn in os.listdir(tmpbuild):
src = os.path.join(tmpbuild, fn)
if flags == '-fPIC':
# add _pic to filename
ff = fn.split('.')
fn = '.'.join(ff[:-1]) + '_pic.' + ff[-1]
dest = os.path.join(self.installdir, libsubdir, fn)
try:
if os.path.isfile(src):
shutil.move(src, dest)
self.log.info("Moved %s to %s" % (src, dest))
except OSError, err:
raise EasyBuildError("Failed to move %s to %s: %s", src, dest, err)
rmtree2(tmpbuild)
def sanity_check_step(self):
"""Custom sanity check paths for Intel MKL."""
shlib_ext = get_shared_lib_ext()
mklfiles = None
mkldirs = None
ver = LooseVersion(self.version)
libs = ['libmkl_core.%s' % shlib_ext, 'libmkl_gnu_thread.%s' % shlib_ext,
'libmkl_intel_thread.%s' % shlib_ext, 'libmkl_sequential.%s' % shlib_ext]
extralibs = ['libmkl_blacs_intelmpi_%(suff)s.' + shlib_ext, 'libmkl_scalapack_%(suff)s.' + shlib_ext]
if self.cfg['interfaces']:
compsuff = '_intel'
if get_software_root('icc') is None:
# check for PGI first, since there's a GCC underneath PGI too...
if get_software_root('PGI'):
compsuff = '_pgi'
elif get_software_root('GCC'):
compsuff = '_gnu'
else:
raise EasyBuildError("Not using Intel/GCC/PGI, don't know compiler suffix for FFTW libraries.")
precs = ['_double', '_single']
if ver < LooseVersion('11'):
# no precision suffix in libfftw2 libs before mkl v11
precs = ['']
fftw_vers = ['2x%s%s' % (x, prec) for x in ['c', 'f'] for prec in precs] + ['3xc', '3xf']
pics = ['', '_pic']
libs = ['libfftw%s%s%s.a' % (fftwver, compsuff, pic) for fftwver in fftw_vers for pic in pics]
fftw_cdft_vers = ['2x_cdft_DOUBLE']
if not self.cfg['m32']:
fftw_cdft_vers.append('2x_cdft_SINGLE')
if ver >= LooseVersion('10.3'):
fftw_cdft_vers.append('3x_cdft')
if ver >= LooseVersion('11.0.2'):
bits = ['_lp64']
if not self.cfg['m32']:
bits.append('_ilp64')
else:
# no bits suffix in cdft libs before mkl v11.0.2
bits = ['']
libs += ['libfftw%s%s%s.a' % x for x in itertools.product(fftw_cdft_vers, bits, pics)]
if ver >= LooseVersion('10.3'):
if self.cfg['m32']:
raise EasyBuildError("Sanity check for 32-bit not implemented yet for IMKL v%s (>= 10.3)", self.version)
else:
mkldirs = ['bin', 'mkl/bin', 'mkl/lib/intel64', 'mkl/include']
if ver < LooseVersion('11.3'):
mkldirs.append('mkl/bin/intel64')
libs += [lib % {'suff': suff} for lib in extralibs for suff in ['lp64', 'ilp64']]
mklfiles = ['mkl/lib/intel64/libmkl.%s' % shlib_ext, 'mkl/include/mkl.h'] + \
['mkl/lib/intel64/%s' % lib for lib in libs]
if ver >= LooseVersion('10.3.4') and ver < LooseVersion('11.1'):
mkldirs += ['compiler/lib/intel64']
else:
if ver >= LooseVersion('2017.0.0'):
mkldirs += ['lib/intel64_lin']
else:
mkldirs += ['lib/intel64']
else:
if self.cfg['m32']:
mklfiles = ['lib/32/libmkl.%s' % shlib_ext, 'include/mkl.h'] + \
['lib/32/%s' % lib for lib in libs]
mkldirs = ['lib/32', 'include/32', 'interfaces']
else:
libs += [lib % {'suff': suff} for lib in extralibs for suff in ['lp64', 'ilp64']]
mklfiles = ['lib/em64t/libmkl.%s' % shlib_ext, 'include/mkl.h'] + \
['lib/em64t/%s' % lib for lib in libs]
mkldirs = ['lib/em64t', 'include/em64t', 'interfaces']
custom_paths = {
'files': mklfiles,
'dirs': mkldirs,
}
super(EB_mkl, self).sanity_check_step(custom_paths=custom_paths)
easyblocks/s/siesta.py deleted 100644 → 0
+ 0
− 330
View file @ 7b62bb79
##
# Copyright 2009-2017 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# http://github.com/hpcugent/easybuild
#
# EasyBuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation v2.
#
# EasyBuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
##
"""
EasyBuild support for building and installing Siesta, implemented as an easyblock
@author: Miguel Dias Costa (National University of Singapore)
@author: Ake Sandgren (Umea University)
"""
import os
import stat
import easybuild.tools.toolchain as toolchain
from distutils.version import LooseVersion
from easybuild.easyblocks.generic.configuremake import ConfigureMake
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools.build_log import EasyBuildError, print_msg
from easybuild.tools.filetools import adjust_permissions, apply_regex_substitutions, change_dir, copy_dir, copy_file, mkdir
from easybuild.tools.modules import get_software_root
from easybuild.tools.run import run_cmd
class siesta(ConfigureMake):
"""
Support for building/installing Siesta.
- avoid parallel build for older versions
"""
@staticmethod
def extra_options(extra_vars=None):
"""Define extra options for Siesta"""
extra = {
'with_transiesta': [True, "Build transiesta", CUSTOM],
'with_utils': [True, "Build all utils", CUSTOM],
}
return ConfigureMake.extra_options(extra_vars=extra)
def configure_step(self):
"""
Custom configure and build procedure for Siesta.
- There are two main builds to do, siesta and transiesta
- In addition there are multiple support tools to build
"""
start_dir = self.cfg['start_dir']
obj_dir = os.path.join(start_dir, 'Obj')
arch_make = os.path.join(obj_dir, 'arch.make')
bindir = os.path.join(start_dir, 'bin')
par = ''
if LooseVersion(self.version) >= LooseVersion('4.1'):
par = '-j %s' % self.cfg['parallel']
# enable OpenMP support if desired
env_var_suff = ''
if self.toolchain.options.get('openmp', None):
env_var_suff = '_MT'
scalapack = os.environ['LIBSCALAPACK' + env_var_suff]
blacs = os.environ['LIBSCALAPACK' + env_var_suff]
lapack = os.environ['LIBLAPACK' + env_var_suff]
blas = os.environ['LIBBLAS' + env_var_suff]
if get_software_root('imkl') or get_software_root('FFTW'):
fftw = os.environ['LIBFFT' + env_var_suff]
else:
fftw = None
regex_newlines = []
regex_subs = [
('dc_lapack.a', ''),
(r'^NETCDF_INTERFACE\s*=.*$', ''),
('libsiestaBLAS.a', ''),
('libsiestaLAPACK.a', ''),
# Needed here to allow 4.1-b1 to be built with openmp
(r"^(LDFLAGS\s*=).*$", r"\1 %s %s" % (os.environ['FCFLAGS'], os.environ['LDFLAGS'])),
]
netcdff_loc = get_software_root('netCDF-Fortran')
if netcdff_loc:
# Needed for gfortran at least
regex_newlines.append((r"^(ARFLAGS_EXTRA\s*=.*)$", r"\1\nNETCDF_INCFLAGS = -I%s/include" % netcdff_loc))
if fftw:
fft_inc, fft_lib = os.environ['FFT_INC_DIR'], os.environ['FFT_LIB_DIR']
fppflags = r"\1\nFFTW_INCFLAGS = -I%s\nFFTW_LIBS = -L%s %s" % (fft_inc, fft_lib, fftw)
regex_newlines.append((r'(FPPFLAGS\s*=.*)$', fppflags))
# Make a temp installdir during the build of the various parts
mkdir(bindir)
# change to actual build dir
change_dir(obj_dir)
# Populate start_dir with makefiles
run_cmd(os.path.join(start_dir, 'Src', 'obj_setup.sh'), log_all=True, simple=True, log_output=True)
if LooseVersion(self.version) < LooseVersion('4.1-b2'):
# MPI?
if self.toolchain.options.get('usempi', None):
self.cfg.update('configopts', '--enable-mpi')
# BLAS and LAPACK
self.cfg.update('configopts', '--with-blas="%s"' % blas)
self.cfg.update('configopts', '--with-lapack="%s"' % lapack)
# ScaLAPACK (and BLACS)
self.cfg.update('configopts', '--with-scalapack="%s"' % scalapack)
self.cfg.update('configopts', '--with-blacs="%s"' % blacs)
# NetCDF-Fortran
if netcdff_loc:
self.cfg.update('configopts', '--with-netcdf=-lnetcdff')
# Configure is run in obj_dir, configure script is in ../Src
super(EB_Siesta, self).configure_step(cmd_prefix='../Src/')
if LooseVersion(self.version) > LooseVersion('4.0'):
regex_subs_Makefile = [
(r'CFLAGS\)-c', r'CFLAGS) -c'),
]
apply_regex_substitutions('Makefile', regex_subs_Makefile)
else: # there's no configure on newer versions
if self.toolchain.comp_family() in [toolchain.INTELCOMP]:
copy_file(os.path.join(obj_dir, 'intel.make'), arch_make)
elif self.toolchain.comp_family() in [toolchain.GCC]:
copy_file(os.path.join(obj_dir, 'gfortran.make'), arch_make)
else:
raise EasyBuildError("There is currently no support for compiler: %s", self.toolchain.comp_family())
if self.toolchain.options.get('usempi', None):
regex_subs.extend([
(r"^(CC\s*=\s*).*$", r"\1%s" % os.environ['MPICC']),
(r"^(FC\s*=\s*).*$", r"\1%s" % os.environ['MPIF90']),
(r"^(FPPFLAGS\s*=.*)$", r"\1 -DMPI"),
])
regex_newlines.append((r"^(FPPFLAGS\s*=.*)$", r"\1\nMPI_INTERFACE = libmpi_f90.a\nMPI_INCLUDE = ."))
complibs = scalapack
else:
complibs = lapack
regex_subs.extend([
(r"^(LIBS\s*=\s).*$", r"\1 %s" % complibs),
# Needed for a couple of the utils
(r"^(FFLAGS\s*=\s*).*$", r"\1 -fPIC %s" % os.environ['FCFLAGS']),
])
regex_newlines.append((r"^(COMP_LIBS\s*=.*)$", r"\1\nWXML = libwxml.a"))
if netcdff_loc:
regex_subs.extend([
(r"^(COMP_LIBS\s*=.*)$", r"\1 libncdf.a libfdict.a"),
(r"^(LIBS\s*=.*)$", r"\1 $(NETCDF_LIBS)"),
(r"^(FPPFLAGS\s*=.*)$", r"\1 -DCDF -DNCDF -DNCDF_4 -DNCDF_PARALLEL"),
])
#we are using netCDF with HDF5
regex_newlines.append((r"^(COMP_LIBS\s*=.*)$", r"\1\nNETCDF_LIBS = -lnetcdff -lnetcdf -lhdf5_fortran -lhdf5 -lz"))
apply_regex_substitutions(arch_make, regex_subs)
# individually apply substitutions that add lines
for regex_nl in regex_newlines:
apply_regex_substitutions(arch_make, [regex_nl])
print_msg("building siesta...")
run_cmd('make %s' % par, log_all=True, simple=True, log_output=True)
# Put binary in temporary install dir
copy_file(os.path.join(obj_dir, 'siesta'), bindir)
if self.cfg['with_utils']:
# Make the utils
change_dir(os.path.join(start_dir, 'Util'))
# clean_all.sh might be missing executable bit...
adjust_permissions('./clean_all.sh', stat.S_IXUSR, recursive=False, relative=True)
run_cmd('./clean_all.sh', log_all=True, simple=True, log_output=True)
if LooseVersion(self.version) >= LooseVersion('4.1'):
regex_subs_TS = [
(r"^default:.*$", r""),
(r"^EXE\s*=.*$", r""),
(r"^(include\s*..ARCH_MAKE.*)$", r"EXE=tshs2tshs\ndefault: $(EXE)\n\1"),
(r"^(INCFLAGS.*)$", r"\1 -I%s" % obj_dir),
]
makefile = os.path.join(start_dir, 'Util', 'TS', 'tshs2tshs', 'Makefile')
apply_regex_substitutions(makefile, regex_subs_TS)
# SUFFIX rules in wrong place
regex_subs_suffix = [
(r'^(\.SUFFIXES:.*)$', r''),
(r'^(include\s*\$\(ARCH_MAKE\).*)$', r'\1\n.SUFFIXES:\n.SUFFIXES: .c .f .F .o .a .f90 .F90'),
]
makefile = os.path.join(start_dir, 'Util', 'Sockets', 'Makefile')
apply_regex_substitutions(makefile, regex_subs_suffix)
makefile = os.path.join(start_dir, 'Util', 'SiestaSubroutine', 'SimpleTest', 'Src', 'Makefile')
apply_regex_substitutions(makefile, regex_subs_suffix)
regex_subs_UtilLDFLAGS = [
(r'(\$\(FC\)\s*-o\s)', r'$(FC) %s %s -o ' % (os.environ['FCFLAGS'], os.environ['LDFLAGS'])),
]
makefile = os.path.join(start_dir, 'Util', 'Optimizer', 'Makefile')
apply_regex_substitutions(makefile, regex_subs_UtilLDFLAGS)
makefile = os.path.join(start_dir, 'Util', 'JobList', 'Src', 'Makefile')
apply_regex_substitutions(makefile, regex_subs_UtilLDFLAGS)
print_msg("building utils...")
run_cmd('./build_all.sh', log_all=True, simple=True, log_output=True)
# Now move all the built utils to the temp installdir
expected_utils = [
'Bands/eigfat2plot',
'CMLComp/ccViz',
'Contrib/APostnikov/eig2bxsf', 'Contrib/APostnikov/rho2xsf',
'Contrib/APostnikov/vib2xsf', 'Contrib/APostnikov/fmpdos',
'Contrib/APostnikov/xv2xsf', 'Contrib/APostnikov/md2axsf',
'COOP/mprop', 'COOP/fat',
'Denchar/Src/denchar',
'DensityMatrix/dm2cdf', 'DensityMatrix/cdf2dm',
'Eig2DOS/Eig2DOS',
'Gen-basis/ioncat', 'Gen-basis/gen-basis',
'Grid/cdf2grid', 'Grid/cdf_laplacian', 'Grid/cdf2xsf',
'Grid/grid2cube',
'Grid/grid_rotate', 'Grid/g2c_ng', 'Grid/grid2cdf', 'Grid/grid2val',
'Helpers/get_chem_labels',
'HSX/hs2hsx', 'HSX/hsx2hs',
'JobList/Src/getResults', 'JobList/Src/countJobs',
'JobList/Src/runJobs', 'JobList/Src/horizontal',
'Macroave/Src/macroave',
'ON/lwf2cdf',
'Optimizer/simplex', 'Optimizer/swarm',
'pdosxml/pdosxml',
'Projections/orbmol_proj',
'SiestaSubroutine/FmixMD/Src/driver',
'SiestaSubroutine/FmixMD/Src/para',
'SiestaSubroutine/FmixMD/Src/simple',
'STM/simple-stm/plstm', 'STM/ol-stm/Src/stm',
'VCA/mixps', 'VCA/fractional',
'Vibra/Src/vibra', 'Vibra/Src/fcbuild',
'WFS/info_wfsx', 'WFS/wfsx2wfs',
'WFS/readwfx', 'WFS/wfsnc2wfsx', 'WFS/readwf', 'WFS/wfs2wfsx',
]
if LooseVersion(self.version) <= LooseVersion('4.0'):
expected_utils.extend([
'Bands/new.gnubands',
'TBTrans/tbtrans',
])
if LooseVersion(self.version) >= LooseVersion('4.0'):
expected_utils.extend([
'SiestaSubroutine/ProtoNEB/Src/protoNEB',
'SiestaSubroutine/SimpleTest/Src/simple_pipes_parallel',
'SiestaSubroutine/SimpleTest/Src/simple_pipes_serial',
'Sockets/f2fmaster', 'Sockets/f2fslave',
])
if LooseVersion(self.version) >= LooseVersion('4.1'):
expected_utils.extend([
'Bands/gnubands',
'Grimme/fdf2grimme',
'SpPivot/pvtsp',
'TS/ts2ts/ts2ts', 'TS/tshs2tshs/tshs2tshs', 'TS/TBtrans/tbtrans',
])
for util in expected_utils:
copy_file(os.path.join(start_dir, 'Util', util), bindir)
if self.cfg['with_transiesta']:
# Build transiesta
change_dir(obj_dir)
run_cmd('make clean', log_all=True, simple=True, log_output=True)
print_msg("building transiesta...")
run_cmd('make %s transiesta' % par, log_all=True, simple=True, log_output=True)
copy_file(os.path.join(obj_dir, 'transiesta'), bindir)
def build_step(self):
"""No build step for Siesta."""
pass
def install_step(self):
"""Custom install procedure for Siesta: copy binaries."""
bindir = os.path.join(self.installdir, 'bin')
copy_dir(os.path.join(self.cfg['start_dir'], 'bin'), bindir)
def sanity_check_step(self):
"""Custom sanity check for Siesta."""
bins = ['bin/siesta']
if self.cfg['with_transiesta']:
bins.append('bin/transiesta')
if self.cfg['with_utils']:
bins.append('bin/denchar')
custom_paths = {
'files': bins,
'dirs': [],
}
custom_commands = []
if self.toolchain.options.get('usempi', None):
# make sure Siesta was indeed built with support for running in parallel
custom_commands.append("echo 'SystemName test' | mpirun -np 2 siesta 2>/dev/null | grep PARALLEL")
super(siesta, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment