Skip to content
Snippets Groups Projects
Commit 60b830cb authored by Lukáš Krupčík's avatar Lukáš Krupčík
Browse files

Merge branch 'it4i-anselm'

parents 73e2ad91 a8f6c681
Branches
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 527 additions and 0 deletions
easyblocks/n/nwchem.py 0 → 100644
+ 527
− 0
View file @ 60b830cb
##
# Copyright 2009-2019 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# https://github.com/easybuilders/easybuild
#
# EasyBuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation v2.
#
# EasyBuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
##
"""
EasyBuild support for building and installing NWChem, implemented as an easyblock
@author: Kenneth Hoste (Ghent University)
"""
import os
import re
import shutil
import stat
import tempfile
import easybuild.tools.config as config
import easybuild.tools.environment as env
import easybuild.tools.toolchain as toolchain
from distutils.version import LooseVersion
from easybuild.easyblocks.generic.configuremake import ConfigureMake
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, remove_file, symlink, write_file
from easybuild.tools.modules import get_software_libdir, get_software_root, get_software_version
from easybuild.tools.run import run_cmd
class EB_NWChem(ConfigureMake):
"""Support for building/installing NWChem."""
def __init__(self, *args, **kwargs):
"""Initialisation of custom class variables for NWChem."""
super(EB_NWChem, self).__init__(*args, **kwargs)
self.test_cases_dir = None
# path for symlink to local copy of default .nwchemrc, required by NWChem at runtime
# this path is hardcoded by NWChem, and there's no way to make it use a config file at another path...
self.home_nwchemrc = os.path.join(os.getenv('HOME'), '.nwchemrc')
# temporary directory that is common across multiple nodes in a cluster;
# we can't rely on tempfile.gettempdir() since that follows $TMPDIR,
# which is typically set to a unique directory in jobs;
# use /tmp as default, allow customisation via $EB_NWCHEM_TMPDIR environment variable
common_tmp_dir = os.getenv('EB_NWCHEM_TMPDIR', '/tmp')
# local NWChem .nwchemrc config file, to which symlink will point
# using this approach, multiple parallel builds (on different nodes) can use the same symlink
self.local_nwchemrc = os.path.join(common_tmp_dir, os.getenv('USER'), 'easybuild_nwchem', '.nwchemrc')
@staticmethod
def extra_options():
"""Custom easyconfig parameters for NWChem."""
extra_vars = {
'target': ["LINUX64", "Target platform", CUSTOM],
# possible options for ARMCI_NETWORK on LINUX64 with Infiniband:
# OPENIB, MPI-MT, MPI-SPAWN, MELLANOX
'armci_network': ["OPENIB", "Network protocol to use", CUSTOM],
'msg_comms': ["MPI", "Type of message communication", CUSTOM],
'modules': ["all", "NWChem modules to build", CUSTOM],
'lib_defines': ["", "Additional defines for C preprocessor", CUSTOM],
'tests': [True, "Run example test cases", CUSTOM],
# lots of tests fail, so allow a certain fail ratio
'max_fail_ratio': [0.5, "Maximum test case fail ratio", CUSTOM],
}
return ConfigureMake.extra_options(extra_vars)
def setvar_env_makeopt(self, name, value):
"""Set a variable both in the environment and a an option to make."""
env.setvar(name, value)
self.cfg.update('buildopts', "%s='%s'" % (name, value))
def configure_step(self):
"""Custom configuration procedure for NWChem."""
# check whether a (valid) symlink to a .nwchemrc config file exists (via a dummy file if necessary)
# fail early if the link is not what's we expect, since running the test cases will likely fail in this case
try:
if os.path.exists(self.home_nwchemrc) or os.path.islink(self.home_nwchemrc):
# create a dummy file to check symlink
if not os.path.exists(self.local_nwchemrc):
write_file(self.local_nwchemrc, 'dummy')
self.log.debug("Contents of %s: %s", os.path.dirname(self.local_nwchemrc),
os.listdir(os.path.dirname(self.local_nwchemrc)))
if os.path.islink(self.home_nwchemrc):
home_nwchemrc_target = os.readlink(self.home_nwchemrc)
if home_nwchemrc_target != self.local_nwchemrc:
raise EasyBuildError("Found %s, but it's not a symlink to %s. "
"Please (re)move %s while installing NWChem; it can be restored later",
self.home_nwchemrc, self.local_nwchemrc, self.home_nwchemrc)
# ok to remove, we'll recreate it anyway
remove_file(self.local_nwchemrc)
except (IOError, OSError) as err:
raise EasyBuildError("Failed to validate %s symlink: %s", self.home_nwchemrc, err)
# building NWChem in a long path name is an issue, so let's try to make sure we have a short one
try:
# NWChem insists that version is in name of build dir
tmpdir = tempfile.mkdtemp(suffix='-%s-%s' % (self.name, self.version))
# remove created directory, since we're not going to use it as is
os.rmdir(tmpdir)
# avoid having '['/']' characters in build dir name, NWChem doesn't like that
start_dir = tmpdir.replace('[', '_').replace(']', '_')
mkdir(os.path.dirname(start_dir), parents=True)
symlink(self.cfg['start_dir'], start_dir)
change_dir(start_dir)
self.cfg['start_dir'] = start_dir
except OSError as err:
raise EasyBuildError("Failed to symlink build dir to a shorter path name: %s", err)
# change to actual build dir
change_dir('src')
nwchem_modules = self.cfg['modules']
# set required NWChem environment variables
env.setvar('NWCHEM_TOP', self.cfg['start_dir'])
if len(self.cfg['start_dir']) > 64:
# workaround for:
# "The directory name chosen for NWCHEM_TOP is longer than the maximum allowed value of 64 characters"
# see also https://svn.pnl.gov/svn/nwchem/trunk/src/util/util_nwchem_srcdir.F
self.setvar_env_makeopt('NWCHEM_LONG_PATHS', 'Y')
env.setvar('NWCHEM_TARGET', self.cfg['target'])
env.setvar('MSG_COMMS', self.cfg['msg_comms'])
env.setvar('ARMCI_NETWORK', self.cfg['armci_network'])
if self.cfg['armci_network'] in ["OPENIB"]:
env.setvar('IB_INCLUDE', "/usr/include")
env.setvar('IB_LIB', "/usr/lib64")
env.setvar('IB_LIB_NAME', "-libumad -libverbs -lpthread")
if 'python' in self.cfg['modules']:
python_root = get_software_root('Python')
if not python_root:
raise EasyBuildError("Python module not loaded, you should add Python as a dependency.")
env.setvar('PYTHONHOME', python_root)
pyver = '.'.join(get_software_version('Python').split('.')[0:2])
env.setvar('PYTHONVERSION', pyver)
# if libreadline is loaded, assume it was a dependency for Python
# pass -lreadline to avoid linking issues (libpython2.7.a doesn't include readline symbols)
libreadline = get_software_root('libreadline')
if libreadline:
libreadline_libdir = os.path.join(libreadline, get_software_libdir('libreadline'))
ncurses = get_software_root('ncurses')
if not ncurses:
raise EasyBuildError("ncurses is not loaded, but required to link with libreadline")
ncurses_libdir = os.path.join(ncurses, get_software_libdir('ncurses'))
readline_libs = ' '.join([
os.path.join(libreadline_libdir, 'libreadline.a'),
os.path.join(ncurses_libdir, 'libcurses.a'),
])
extra_libs = os.environ.get('EXTRA_LIBS', '')
env.setvar('EXTRA_LIBS', ' '.join([extra_libs, readline_libs]))
env.setvar('LARGE_FILES', 'TRUE')
env.setvar('USE_NOFSCHECK', 'TRUE')
env.setvar('CCSDTLR', 'y') # enable CCSDTLR
env.setvar('CCSDTQ', 'y') # enable CCSDTQ (compilation is long, executable is big)
if LooseVersion(self.version) >= LooseVersion("6.2"):
env.setvar('MRCC_METHODS','y') # enable multireference coupled cluster capability
if LooseVersion(self.version) >= LooseVersion("6.5"):
env.setvar('EACCSD','y') # enable EOM electron-attachemnt coupled cluster capability
env.setvar('IPCCSD','y') # enable EOM ionization-potential coupled cluster capability
env.setvar('USE_NOIO', 'TRUE') # avoid doing I/O for the ddscf, mp2 and ccsd modules
for var in ['USE_MPI', 'USE_MPIF', 'USE_MPIF4']:
env.setvar(var, 'y')
for var in ['CC', 'CXX', 'F90']:
env.setvar('MPI_%s' % var, os.getenv('MPI%s' % var))
libmpi = ""
# for NWChem 6.6 and newer, $LIBMPI & co should no longer be
# set, the correct values are determined by the NWChem build
# procedure automatically, see
# http://www.nwchem-sw.org/index.php/Compiling_NWChem#MPI_variables
if LooseVersion(self.version) < LooseVersion("6.6"):
env.setvar('MPI_LOC', os.path.dirname(os.getenv('MPI_INC_DIR')))
env.setvar('MPI_LIB', os.getenv('MPI_LIB_DIR'))
env.setvar('MPI_INCLUDE', os.getenv('MPI_INC_DIR'))
mpi_family = self.toolchain.mpi_family()
if mpi_family in toolchain.OPENMPI:
ompi_ver = get_software_version('OpenMPI')
if LooseVersion(ompi_ver) < LooseVersion("1.10"):
if LooseVersion(ompi_ver) < LooseVersion("1.8"):
libmpi = "-lmpi_f90 -lmpi_f77 -lmpi -ldl -Wl,--export-dynamic -lnsl -lutil"
else:
libmpi = "-lmpi_usempi -lmpi_mpifh -lmpi"
else:
libmpi = "-lmpi_usempif08 -lmpi_usempi_ignore_tkr -lmpi_mpifh -lmpi"
elif mpi_family in [toolchain.INTELMPI]:
if self.cfg['armci_network'] in ["MPI-MT"]:
libmpi = "-lmpigf -lmpigi -lmpi_ilp64 -lmpi_mt"
else:
libmpi = "-lmpigf -lmpigi -lmpi_ilp64 -lmpi"
elif mpi_family in [toolchain.MPICH, toolchain.MPICH2]:
libmpi = "-lmpichf90 -lmpich -lopa -lmpl -lrt -lpthread"
else:
raise EasyBuildError("Don't know how to set LIBMPI for %s", mpi_family)
env.setvar('LIBMPI', libmpi)
if self.cfg['armci_network'] in ["OPENIB"]:
libmpi += " -libumad -libverbs -lpthread"
# compiler optimization flags: set environment variables _and_ add them to list of make options
self.setvar_env_makeopt('COPTIMIZE', os.getenv('CFLAGS'))
self.setvar_env_makeopt('FOPTIMIZE', os.getenv('FFLAGS'))
# BLAS and ScaLAPACK
self.setvar_env_makeopt('BLASOPT', '%s -L%s %s %s' % (os.getenv('LDFLAGS'), os.getenv('MPI_LIB_DIR'),
os.getenv('LIBSCALAPACK_MT'), libmpi))
self.setvar_env_makeopt('SCALAPACK', '%s %s' % (os.getenv('LDFLAGS'), os.getenv('LIBSCALAPACK_MT')))
if self.toolchain.options['i8']:
size = 8
self.setvar_env_makeopt('USE_SCALAPACK_I8', 'y')
self.cfg.update('lib_defines', '-DSCALAPACK_I8')
else:
self.setvar_env_makeopt('HAS_BLAS', 'yes')
self.setvar_env_makeopt('USE_SCALAPACK', 'y')
size = 4
# set sizes
for lib in ['BLAS', 'LAPACK', 'SCALAPACK']:
self.setvar_env_makeopt('%s_SIZE' % lib, str(size))
env.setvar('NWCHEM_MODULES', nwchem_modules)
env.setvar('LIB_DEFINES', self.cfg['lib_defines'])
# clean first (why not)
# run_cmd("make clean", simple=True, log_all=True, log_ok=True)
# configure build
cmd = "%s " % self.cfg['preconfigopts']
cmd += "make %s nwchem_config" % self.cfg['buildopts']
run_cmd(cmd, simple=True, log_all=True, log_ok=True, log_output=True)
def build_step(self):
"""Custom build procedure for NWChem."""
# set FC
self.setvar_env_makeopt('FC', os.getenv('F77'))
# check whether 64-bit integers should be used, and act on it
if not self.toolchain.options['i8']:
if self.cfg['parallel']:
self.cfg.update('buildopts', '-j %s' % self.cfg['parallel'])
run_cmd("make %s 64_to_32" % self.cfg['buildopts'], simple=True, log_all=True, log_ok=True, log_output=True)
self.setvar_env_makeopt('USE_64TO32', "y")
# unset env vars that cause trouble during NWChem build or cause build to generate incorrect stuff
for var in ['CFLAGS', 'FFLAGS', 'LIBS']:
val = os.getenv(var)
if val:
self.log.info("%s was defined as '%s', need to unset it to avoid problems..." % (var, val))
os.unsetenv(var)
os.environ.pop(var)
super(EB_NWChem, self).build_step(verbose=True)
# build version info
try:
self.log.info("Building version info...")
cwd = os.getcwd()
change_dir(os.path.join(self.cfg['start_dir'], 'src', 'util'))
run_cmd("make version", simple=True, log_all=True, log_ok=True, log_output=True)
run_cmd("make", simple=True, log_all=True, log_ok=True, log_output=True)
change_dir(os.path.join(self.cfg['start_dir'], 'src'))
run_cmd("make link", simple=True, log_all=True, log_ok=True, log_output=True)
change_dir(cwd)
except OSError as err:
raise EasyBuildError("Failed to build version info: %s", err)
# run getmem.nwchem script to assess memory availability and make an educated guess
# this is an alternative to specifying -DDFLT_TOT_MEM via LIB_DEFINES
# this recompiles the appropriate files and relinks
if not 'DDFLT_TOT_MEM' in self.cfg['lib_defines']:
change_dir(os.path.join(self.cfg['start_dir'], 'contrib'))
run_cmd("./getmem.nwchem", simple=True, log_all=True, log_ok=True, log_output=True)
change_dir(self.cfg['start_dir'])
def install_step(self):
"""Custom install procedure for NWChem."""
try:
# binary
bindir = os.path.join(self.installdir, 'bin')
mkdir(bindir)
shutil.copy(os.path.join(self.cfg['start_dir'], 'bin', self.cfg['target'], 'nwchem'),
bindir)
# data
shutil.copytree(os.path.join(self.cfg['start_dir'], 'src', 'data'),
os.path.join(self.installdir, 'data'))
shutil.copytree(os.path.join(self.cfg['start_dir'], 'src', 'basis', 'libraries'),
os.path.join(self.installdir, 'data', 'libraries'))
shutil.copytree(os.path.join(self.cfg['start_dir'], 'src', 'nwpw', 'libraryps'),
os.path.join(self.installdir, 'data', 'libraryps'))
except OSError as err:
raise EasyBuildError("Failed to install NWChem: %s", err)
# create NWChem settings file
default_nwchemrc = os.path.join(self.installdir, 'data', 'default.nwchemrc')
txt = '\n'.join([
"nwchem_basis_library %(path)s/data/libraries/",
"nwchem_nwpw_library %(path)s/data/libraryps/",
"ffield amber",
"amber_1 %(path)s/data/amber_s/",
"amber_2 %(path)s/data/amber_q/",
"amber_3 %(path)s/data/amber_x/",
"amber_4 %(path)s/data/amber_u/",
"spce %(path)s/data/solvents/spce.rst",
"charmm_s %(path)s/data/charmm_s/",
"charmm_x %(path)s/data/charmm_x/",
]) % {'path': self.installdir}
write_file(default_nwchemrc, txt)
# fix permissions in data directory
datadir = os.path.join(self.installdir, 'data')
adjust_permissions(datadir, stat.S_IROTH, add=True, recursive=True)
adjust_permissions(datadir, stat.S_IXOTH, add=True, recursive=True, onlydirs=True)
def sanity_check_step(self):
"""Custom sanity check for NWChem."""
custom_paths = {
'files': ['bin/nwchem'],
'dirs': [os.path.join('data', x) for x in ['amber_q', 'amber_s', 'amber_t', 'amber_u', 'amber_x',
'charmm_s', 'charmm_x', 'solvents', 'libraries', 'libraryps']],
}
super(EB_NWChem, self).sanity_check_step(custom_paths=custom_paths)
def make_module_extra(self):
"""Custom extra module file entries for NWChem."""
txt = super(EB_NWChem, self).make_module_extra()
# check whether Python module is loaded for compatibility with --module-only
python = get_software_root('Python')
if python:
txt += self.module_generator.set_environment('PYTHONHOME', python)
# '/' at the end is critical for NWCHEM_BASIS_LIBRARY!
datadir = os.path.join(self.installdir, 'data')
txt += self.module_generator.set_environment('NWCHEM_BASIS_LIBRARY', os.path.join(datadir, 'libraries/'))
if LooseVersion(self.version) >= LooseVersion("6.3"):
txt += self.module_generator.set_environment('NWCHEM_NWPW_LIBRARY', os.path.join(datadir, 'libraryps/'))
return txt
def cleanup_step(self):
"""Copy stuff from build directory we still need, if any."""
try:
exs_dir = os.path.join(self.cfg['start_dir'], 'examples')
self.examples_dir = os.path.join(tempfile.mkdtemp(), 'examples')
shutil.copytree(exs_dir, self.examples_dir)
self.log.info("Copied %s to %s." % (exs_dir, self.examples_dir))
except OSError as err:
raise EasyBuildError("Failed to copy examples: %s", err)
super(EB_NWChem, self).cleanup_step()
def test_cases_step(self):
"""Run provided list of test cases, or provided examples is no test cases were specified."""
# run all examples if no test cases were specified
# order and grouping is important for some of these tests (e.g., [o]h3tr*
# Some of the examples are deleted
# missing md parameter files: dna.nw, mache.nw, 18c6NaK.nw, membrane.nw, sdm.nw
# method not implemented (unknown thory) or keyword not found: triplet.nw, C2H6.nw, pspw_MgO.nw, ccsdt_polar_small.nw, CG.nw
# no convergence: diamond.nw
# Too much memory required: ccsd_polar_big.nw
if type(self.cfg['tests']) is bool:
examples = [('qmd', ['3carbo_dft.nw', '3carbo.nw', 'h2o_scf.nw']),
('pspw', ['C2.nw', 'C6.nw', 'Carbene.nw', 'Na16.nw', 'NaCl.nw']),
('tcepolar', ['ccsd_polar_small.nw']),
('dirdyvtst/h3', ['h3tr1.nw', 'h3tr2.nw']),
('dirdyvtst/h3', ['h3tr3.nw']), ('dirdyvtst/h3', ['h3tr4.nw']), ('dirdyvtst/h3', ['h3tr5.nw']),
('dirdyvtst/oh3', ['oh3tr1.nw', 'oh3tr2.nw']),
('dirdyvtst/oh3', ['oh3tr3.nw']), ('dirdyvtst/oh3', ['oh3tr4.nw']), ('dirdyvtst/oh3', ['oh3tr5.nw']),
('pspw/session1', ['band.nw', 'si4.linear.nw', 'si4.rhombus.nw', 'S2-drift.nw',
'silicon.nw', 'S2.nw', 'si4.rectangle.nw']),
('md/myo', ['myo.nw']), ('md/nak', ['NaK.nw']), ('md/crown', ['crown.nw']), ('md/hrc', ['hrc.nw']),
('md/benzene', ['benzene.nw'])]
self.cfg['tests'] = [(os.path.join(self.examples_dir, d), l) for (d, l) in examples]
self.log.info("List of examples to be run as test cases: %s" % self.cfg['tests'])
try:
# symlink $HOME/.nwchemrc to local copy of default nwchemrc
default_nwchemrc = os.path.join(self.installdir, 'data', 'default.nwchemrc')
# make a local copy of the default .nwchemrc file at a fixed path, so we can symlink to it
# this makes sure that multiple parallel builds can reuse the same symlink, even for different builds
# there is apparently no way to point NWChem to a particular config file other that $HOME/.nwchemrc
try:
local_nwchemrc_dir = os.path.dirname(self.local_nwchemrc)
if not os.path.exists(local_nwchemrc_dir):
os.makedirs(local_nwchemrc_dir)
shutil.copy2(default_nwchemrc, self.local_nwchemrc)
# only try to create symlink if it's not there yet
# we've verified earlier that the symlink is what we expect it to be if it's there
if not os.path.islink(self.home_nwchemrc):
symlink(self.local_nwchemrc, self.home_nwchemrc)
except OSError as err:
raise EasyBuildError("Failed to symlink %s to %s: %s", self.home_nwchemrc, self.local_nwchemrc, err)
# run tests, keep track of fail ratio
cwd = os.getcwd()
fail = 0.0
tot = 0.0
success_regexp = re.compile("Total times\s*cpu:.*wall:.*")
test_cases_logfn = os.path.join(self.installdir, config.log_path(), 'test_cases.log')
test_cases_log = open(test_cases_logfn, "w")
for (testdir, tests) in self.cfg['tests']:
# run test in a temporary dir
tmpdir = tempfile.mkdtemp(prefix='nwchem_test_')
change_dir(tmpdir)
# copy all files in test case dir
for item in os.listdir(testdir):
test_file = os.path.join(testdir, item)
if os.path.isfile(test_file):
self.log.debug("Copying %s to %s" % (test_file, tmpdir))
shutil.copy2(test_file, tmpdir)
# run tests
for testx in tests:
cmd = "nwchem %s" % testx
msg = "Running test '%s' (from %s) in %s..." % (cmd, testdir, tmpdir)
self.log.info(msg)
test_cases_log.write("\n%s\n" % msg)
(out, ec) = run_cmd(cmd, simple=False, log_all=False, log_ok=False, log_output=True)
# check exit code and output
if ec:
msg = "Test %s failed (exit code: %s)!" % (testx, ec)
self.log.warning(msg)
test_cases_log.write('FAIL: %s' % msg)
fail += 1
else:
if success_regexp.search(out):
msg = "Test %s successful!" % testx
self.log.info(msg)
test_cases_log.write('SUCCESS: %s' % msg)
else:
msg = "No 'Total times' found for test %s (but exit code is %s)!" % (testx, ec)
self.log.warning(msg)
test_cases_log.write('FAIL: %s' % msg)
fail += 1
test_cases_log.write("\nOUTPUT:\n\n%s\n\n" % out)
tot += 1
# go back
change_dir(cwd)
shutil.rmtree(tmpdir)
fail_ratio = fail / tot
fail_pcnt = fail_ratio * 100
msg = "%d of %d tests failed (%s%%)!" % (fail, tot, fail_pcnt)
self.log.info(msg)
test_cases_log.write('\n\nSUMMARY: %s' % msg)
test_cases_log.close()
self.log.info("Log for test cases saved at %s" % test_cases_logfn)
if fail_ratio > self.cfg['max_fail_ratio']:
max_fail_pcnt = self.cfg['max_fail_ratio'] * 100
raise EasyBuildError("Over %s%% of test cases failed, assuming broken build.", max_fail_pcnt)
# cleanup
try:
shutil.rmtree(self.examples_dir)
shutil.rmtree(local_nwchemrc_dir)
except OSError as err:
raise EasyBuildError("Cleanup failed: %s", err)
# set post msg w.r.t. cleaning up $HOME/.nwchemrc symlink
self.postmsg += "\nRemember to clean up %s after all NWChem builds are finished." % self.home_nwchemrc
except OSError as err:
raise EasyBuildError("Failed to run test cases: %s", err)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment