Skip to content
Snippets Groups Projects
Commit 83654a04 authored by Lukáš Krupčík's avatar Lukáš Krupčík
Browse files

modified: install/CENTOS7LATEST 

	modified:   install/CONFIG
	modified:   install/UBUNTULATEST
	deleted:    CentOS/7.5-ESPRESO/CentOS-7.5-Bootstrap
	deleted:    CentOS/7.5-ESPRESO/app/log_crawler.py
	deleted:    CentOS/7.5-ESPRESO/app/notifications_monitor.py
	deleted:    CentOS/7.5-ESPRESO/app/startup.py
	deleted:    CentOS/7.5-ESPRESO/app/wait_a_while.py
	deleted:    CentOS/7.5-ESPRESO/heat.cpp
	deleted:    Debian/8.0/Debian-8.0-Bootstrap
	deleted:    Ubuntu/16.04-OpenFOAM/README
	deleted:    Ubuntu/16.04-OpenFOAM/Ubuntu-16.04-Bootstrap
	deleted:    Ubuntu/16.04-OpenFOAM/app/check
	deleted:    Ubuntu/16.04-OpenFOAM/app/monitor
	deleted:    Ubuntu/16.04-OpenFOAM/app/startup
	deleted:    Ubuntu/16.04-OpenFOAM/hpc_image
	deleted:    Ubuntu/16.04-OpenFOAM/openfoam6-linux
	deleted:    Ubuntu/16.04-OpenFOAM/openfoam_hacks/aliases
	deleted:    Ubuntu/16.04-OpenFOAM/openfoam_hacks/bashrc
	deleted:    Ubuntu/16.04-OpenFOAM/openfoam_hacks/exec
	deleted:    Ubuntu/16.04-OpenFOAM/recipe-openfoam.txt
	deleted:    Ubuntu/16.04-OpenFOAM/singularity_image.tar
	deleted:    Ubuntu/16.04/Ubuntu-16.04-Bootstrap
parent dfcb11ef
Branches
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 0 additions and 1828 deletions
CentOS/7.5-ESPRESO/CentOS-7.5-Bootstrap deleted 100644 → 0
+ 0
− 58
View file @ dfcb11ef
Bootstrap: docker
From: centos:latest
%post
yum -y update && yum -y upgrade
# CentOS repo
yum -y install python gcc make wget vim mc procps tcl gcc-c++ gcc-gfortran pkg-config acl acpid alsa-lib alsa-lib-devel amtu apr aspell at atk atk-devel attr audit audit-libs audit-libs-python authconfig autoconf autofs automake avahi-glib avahi-libs basesystem bash bc bind-libs bind-utils binutils binutils-devel bison bison-devel bluez-libs bridge-utils busybox bzip2 bzip2-devel bzip2-libs ca-certificates cairo cairo-devel c-ares ccid cdparanoia-libs centos-indexhtml centos-release checkpolicy chkconfig cloog-ppl compat-libcap1 compat-libstdc++-33 compat-readline5 ConsoleKit ConsoleKit-libs coolkey coreutils coreutils-libs cpio cpp cppunit cpupowerutils cpuspeed cracklib cracklib-dicts crash crda cronie cronie-anacron crontabs cryptsetup-luks cryptsetup-luks-libs cups cups-libs curl cvs cyrus-sasl cyrus-sasl-devel cyrus-sasl-gssapi cyrus-sasl-lib cyrus-sasl-plain dash db4 db4-cxx db4-devel db4-utils dbus dbus-devel dbus-glib dbus-glib-devel dbus-libs dbus-python dejavu-fonts-common dejavu-lgc-sans-mono-fonts dejavu-sans-mono-fonts desktop-file-utils device-mapper device-mapper-event device-mapper-event-libs device-mapper-libs device-mapper-multipath device-mapper-multipath-libs device-mapper-persistent-data diffutils dmidecode dmraid dmraid-events dmz-cursor-themes docbook-dtds docbook-style-dsssl docbook-style-xsl docbook-utils dos2unix dosfstools dracut dracut-kernel dracut-network dump e2fsprogs e2fsprogs-devel e2fsprogs-libs ed eggdbus eject elfutils-libelf elfutils-libelf-devel elfutils-libs emacs emacs-common environment-modules ethtool expat expat-devel expect file file-devel file-libs filesystem findutils finger fipscheck fipscheck-lib flac fontconfig fontconfig-devel fontpackages-filesystem foomatic foomatic-db foomatic-db-filesystem foomatic-db-ppds freeglut freeglut-devel freetype freetype-devel ftp fuse fuse-libs gamin gamin-python gawk gcc gcc-c++ gcc-gfortran GConf2 GConf2-devel gd gdb gdbm gdbm-devel gdk-pixbuf2 gdk-pixbuf2-devel genisoimage gettext gettext-devel gettext-libs ghostscript ghostscript-fonts giflib git glib2 glib2-devel glibc glibc-common glibc-debuginfo glibc-debuginfo-common glibc-devel glibc-headers glx-utils gmp gmp-devel gnome-icon-theme gnome-keyring gnome-keyring-devel gnome-themes gnome-vfs2 gnome-vfs2-devel gnupg2 gnuplot gnuplot-common gnutls gnutls-devel gperf gpgme gpm gpm-libs grep groff grub grubby gstreamer gstreamer-plugins-base gstreamer-tools gtk2 gtk2-devel gtk2-engines gtk-doc gzip hal hal-devel hal-info hal-libs hdparm hesiod hicolor-icon-theme hunspell hwdata hwloc hwloc-devel ilmbase ImageMagick indent info initscripts iproute iptables iptables-ipv6 iptstate iputils irqbalance iscsi-initiator-utils iso-codes iw jasper-libs jwhois kbd kbd-misc kernel kernel kernel-devel kernel-firmware kernel-headers keyutils keyutils-libs keyutils-libs-devel kpartx krb5-devel krb5-libs krb5-workstation ksh lcms-libs less lftp libacl libacl-devel libaio libart_lgpl libart_lgpl-devel libasyncns libattr libattr-devel libbasicobjects libblkid libbonobo libbonobo-devel libbonoboui libbonoboui-devel libcanberra libcanberra-gtk2 libcap libcap-devel libcap-ng libcgroup libcollection libcom_err libcom_err-devel libcroco libcroco-devel libcxgb3 libcxgb4 libdaemon libdhash libdrm libdrm-devel libedit libedit-devel libevent libexif libffi libfontenc libgcc libgcj libgcrypt libgcrypt-devel libgfortran libglade2 libglade2-devel libgnome libgnomecanvas libgnomecanvas-devel libgnome-devel libgomp libgpg-error libgpg-error-devel libgsf libgsf-devel libgssglue libgudev1 libhugetlbfs libibcm libibcm-devel libibverbs libibverbs-devel libibverbs-devel-static libibverbs-utils libICE libICE-devel libicu libIDL libIDL-devel libidn libidn-devel libini_config libipa_hbac libjpeg-turbo libjpeg-turbo-devel libldb libmcpp libmlx4 libmlx5 libmng libmng-devel libmthca libmthca-static libnetfilter_conntrack libnfnetlink libnih libnl libnl-devel libnotify libogg liboil libotf libpath_utils libpcap libpciaccess libpciaccess-devel libpng libpng-devel libproxy libproxy-bin libproxy-python libref_array librsvg2 librsvg2-devel libselinux libselinux-devel libselinux-python libselinux-utils libsemanage libsepol libsepol-devel libSM libSM-devel libsndfile libss libssh2 libsss_idmap libstdc++ libstdc++-devel libsysfs libsysfs-devel libtalloc libtasn1 libtasn1-devel libtdb libtevent libthai libtheora libtiff libtiff-devel libtirpc libtool-ltdl libudev libudev-devel libunistring libusb libusb1 libusb-devel libuser libutempter libuuid libvisual libvorbis libwmf libwmf-lite libwnck libX11 libX11-common libX11-devel libXau libXau-devel libXaw libXaw-devel libxcb libxcb-devel libXcomposite libXcomposite-devel libXcursor libXcursor-devel libXdamage libXdamage-devel libXdmcp libXdmcp-devel libXext libXext-devel libXfixes libXfixes-devel libXfont libXft libXft-devel libXi libXi-devel libXinerama libXinerama-devel libxkbfile libxml2 libxml2-devel libxml2-python libXmu libXmu-devel libXp libXpm libXpm-devel libXrandr libXrandr-devel libXrender libXrender-devel libXres libXScrnSaver libXScrnSaver-devel libxslt libxslt-devel libXt libXt-devel libXtst libXtst-devel libXv libXv-devel libXxf86misc libXxf86misc-devel libXxf86vm libXxf86vm-devel lm_sensors-libs lockdev logrotate lsof lvm2 lvm2-libs lynx lzo m17n-db m17n-db-datafiles m17n-lib m2crypto m4 mailcap mailx make MAKEDEV man man-pages mc mcpp mcstrans mdadm mesa-dri1-drivers mesa-dri-drivers mesa-dri-filesystem mesa-libGL mesa-libGL-devel mesa-libGLU mesa-libGLU-devel mesa-private-llvm mgetty microcode_ctl mingetty mkbootdisk mlocate module-init-tools mpfr mtools mtr mysql-libs nano nc ncurses ncurses-base ncurses-devel ncurses-libs net-snmp net-snmp-libs net-tools newt newt-python nfs-utils nfs-utils-lib nmap notification-daemon nscd nspr nspr-devel nss nss_db nss-devel nss-softokn nss-softokn-debuginfo nss-softokn-devel nss-softokn-freebl nss-softokn-freebl-devel nss-sysinit nss-tools nss-util nss-util-devel ntp ntpdate ntsysv OpenEXR-libs OpenIPMI OpenIPMI-libs openjade openjpeg-libs openldap openldap-devel openmotif openmotif22 opensp openssh openssh-clients openssh-server ORBit2 ORBit2-devel p11-kit p11-kit-trust pam pam-devel pam_krb5 pam_passwdqc pam_pkcs11 pango pango-devel paps paps-libs parted passwd patch pax pciutils pciutils-devel pciutils-libs pcmciautils pcp pcp-conf pcp-libs pcre pcsc-lite pcsc-lite-libs perf perl perl-AppConfig perl-Authen-SASL perl-Bit-Vector perl-Carp-Clan perl-CGI perl-Class-Singleton perl-Compress-Raw-Zlib perl-Compress-Zlib perl-Convert-ASN1 perl-Crypt-PasswdMD5 perl-Date-Calc perl-Date-Manip perl-DateTime perl-DBD-Pg perl-DBI perl-devel perl-Digest-HMAC perl-Digest-SHA1 perl-Error perl-ExtUtils-MakeMaker perl-ExtUtils-ParseXS perl-File-Which perl-Git perl-GSSAPI perl-HTML-Parser perl-HTML-Tagset perl-IO-Compress-Base perl-IO-Compress-Zlib perl-IO-Socket-SSL perl-JSON perl-LDAP perl-libs perl-libwww-perl perl-List-MoreUtils perl-Module-Pluggable perl-Net-LibIDN perl-Net-SSLeay perl-Net-Telnet perl-Params-Validate perl-PCP-PMDA perl-Pod-Escapes perl-Pod-Simple perl-SGMLSpm perl-Socket6 perl-String-CRC32 perl-TermReadKey perl-Test-Harness perl-Test-Simple perl-Text-Iconv perl-Time-HiRes perl-URI perl-version perl-WWW-Curl perl-XML-Filter-BufferText perl-XML-LibXML perl-XML-NamespaceSupport perl-XML-Parser perl-XML-SAX perl-XML-SAX-Writer perl-XML-Simple perl-YAML-Syck pinentry pinfo pixman pixman-devel pkgconfig plymouth plymouth-core-libs plymouth-scripts pm-utils policycoreutils polkit poppler poppler-data poppler-utils popt popt-devel portreserve postfix postgresql postgresql-devel postgresql-libs postgresql-odbc ppl ppp prelink procmail procps psacct psmisc pth pulseaudio-libs pycairo pycairo-devel pygobject2 pygobject2-codegen pygobject2-devel pygobject2-doc pygpgme pygtk2 pygtk2-codegen pygtk2-devel pygtk2-doc pyOpenSSL pytalloc python python-argparse python-devel python-iniparse python-libs python-pcp python-pycurl python-setuptools python-sssdconfig python-urlgrabber qt qt3 qt-sqlite qt-x11 quota rarian rarian-compat rdate rdma readahead readline readline-devel redhat-logos redhat-lsb redhat-lsb-compat redhat-lsb-core redhat-lsb-graphics redhat-lsb-printing redhat-menus redhat-rpm-config rmt rootfiles rpcbind rpm rpm-devel rpm-libs rpm-python rp-pppoe rrdtool rsh rsync ruby ruby-libs samba samba4-libs samba-common samba-winbind samba-winbind-clients screen SDL SDL-devel sed selinux-policy setserial setup setuptool sgml-common sgpio shadow-utils shared-mime-info slang smartmontools snappy sos sound-theme-freedesktop sqlite sqlite-devel sssd sssd-ad sssd-client sssd-common sssd-common-pac sssd-ipa sssd-krb5 sssd-krb5-common sssd-ldap sssd-proxy startup-notification startup-notification-devel strace stunnel sudo swig symlinks sysfsutils syslinux syslinux-nonlinux sysstat system-gnome-theme system-icon-theme systemtap-devel sysvinit-tools t1lib talk tar tcl tcl-devel tcpdump tcp_wrappers tcp_wrappers-libs tcsh telnet time tk tmpwatch traceroute tree tunctl tzdata udev udftools unix2dos unixODBC unixODBC-devel unzip upstart urw-fonts usbutils usermode ustr util-linux-ng vconfig vim-common vim-enhanced vim-filesystem vim-minimal virt-what wget which wireless-tools words wpa_supplicant xcb-util xdg-utils xerces-c xerces-c-devel xinetd xkeyboard-config xml-common xmlrpc-c xmlrpc-c-client xorg-x11-fonts-100dpi xorg-x11-fonts-75dpi xorg-x11-font-utils xorg-x11-proto-devel xorg-x11-server-common xorg-x11-server-utils xorg-x11-xauth xorg-x11-xkb-utils xterm xz xz-libs xz-lzma-compat ypbind yp-tools yum yum-metadata-parser yum-plugin-auto-update-debug-info yum-plugin-fastestmirror yum-plugin-priorities yum-plugin-security yum-plugin-versionlock yum-utils zip zlib zlib-devel
# Fedora repo
yum -y install bash-completion pigz tmux
yum -y groupinstall "Development Tools"
cd /tmp
# OpenMPI
wget https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.1.tar.gz
tar xvf openmpi-2.1.1.tar.gz && cd openmpi-2.1.1
./configure --prefix=/usr/local
make -j 16
make install
# pip
curl -s -O https://bootstrap.pypa.io/get-pip.py
python get-pip.py
pip install setuptools numpy scipy pandas wheel --upgrade
# lua
wget http://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/l/lua-posix-32-2.el7.x86_64.rpm
wget http://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/l/lua-filesystem-1.6.2-2.el7.x86_64.rpm
wget http://mirror.centos.org/centos/7/os/x86_64/Packages/lua-devel-5.1.4-15.el7.x86_64.rpm
wget http://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/l/lua-bitop-1.0.2-3.el7.x86_64.rpm
rpm -ihv lua-devel-5.1.4-15.el7.x86_64.rpm
rpm -ihv lua-filesystem-1.6.2-2.el7.x86_64.rpm
rpm -ihv lua-bitop-1.0.2-3.el7.x86_64.rpm
rpm -ihv lua-posix-32-2.el7.x86_64.rpm
# Lmod
wget https://github.com/TACC/Lmod/archive/7.7.7.tar.gz
tar xvf 7.7.7.tar.gz
cd Lmod-7.7.7
./configure --prefix=/opt/apps
make install
# APPS and SCRATCH
mkdir /apps
mkdir /scratch
%environment
source /opt/apps/lmod/7.7.7/init/profile
clearMT
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
export -f ml
export -f module
export -f clearMT
CentOS/7.5-ESPRESO/app/log_crawler.py deleted 100644 → 0
+ 0
− 151
View file @ dfcb11ef
import sys
import time
import os
import json
import logging
import shutil
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("Log crawler")
PROGRESS_HEAD = '''<html>
<head>
<title>HPC job progress</title>
<script type="text/javascript">
function abort() {
notify_running_job("ABORT");
}
</script>
</head>'''
PROGRESS_BODY='''<body style="margin: 20px; padding: 20px;">
<h1>HPC job progress</h1>
<div>
<h3>This is a waiter. It's waiting on the HPC cluster. It has waited {}/{} seconds.</h3>
<p>(Status from {}.)</p>
<p>Click the Abort button to stop the waiter early. Please click only once,
it will take a while until the request is processed. (The status might even
update once more before the job is aborted.)</p>
<input type="button" value="Abort" onclick="abort()">
</div>
</body>
</html>
'''
PROGRESS_BODY_UNKNOWN='''<body style="margin: 20px; padding: 20px;">
<h1>HPC job progress</h1>
<div>
<h3>This is a waiter. It's waiting on the HPC cluster. Its status is unknown.</h3>
<input type="button" value="Abort job" onclick="abort()">
</div>
</body>
</html>
'''
def main():
fn_log = sys.argv[1]
fn_status = sys.argv[2]
crawl_log(fn_log, fn_status)
def crawl_log(fn_log, fn_status):
"""Monitors a log file and creates html status pages from it
The log file is expected to grow only by one line at a time, with new lines
added to the end of the file. Each line is expected to be either a json
string containing the fields 'timestamp', 'elapsed_time', and 'total_time',
or the string 'FINISHED', which triggers the crawler to stop.
Note that this crawler is "memory-less", meaning that it cares only about
the last, newest line added to the log file while all prior lines are
disregarded.
Args:
fn_log (str): Filename of the log file to monitor
fn_status (str): Filename to write the status output to
"""
logger.info("Log crawler starting up")
cached_stamp = 0
should_stop = False
while not should_stop:
if not os.path.exists(fn_log):
logger.info("Log file doesn't exist")
else:
# Check if "last modified" time of the log file has changed
stamp = os.stat(fn_log).st_mtime
if not stamp != cached_stamp:
logger.info("Log file hasn't changed since last check")
else:
logger.info("Log file has changed")
cached_stamp = stamp
status = get_last_line(fn_log)
# We can stop crawling the logs if the main app is done
if status.strip() == "FINISHED":
should_stop = True
logger.info("Found FINISHED log, will stop now!")
# Otherwise, parse json and write status from it
else:
try:
log = json.loads(status)
write_status(log, fn_status)
except json.JSONDecodeError:
write_status_unknown(fn_status)
time.sleep(1)
logger.info("Log crawler terminating")
def get_last_line(fn):
"""Returns the last line of a file
Args:
fn (str): File name of the file to read from
"""
with open(fn, 'r') as fin:
for line in fin:
pass
return line
def write_status(log, fn_status):
"""Writes a status html page using the information in 'log'
Args:
log (dict): Log containing the field 'timestamp', 'elapsed_time', and
'total_time'
fn_status (str): Filename to write the status html to
"""
status = PROGRESS_HEAD + PROGRESS_BODY.format(
log["elapsed_time"],
log["total_time"],
log["timestamp"],
)
# First write to a temporary file, then copy to actual status file. That
# way, the status file won't be corrupt if the writing process takes a
# while.
fn_status_new = fn_status + '.new'
with open(fn_status_new, 'w') as fout:
fout.write(status)
shutil.copyfile(fn_status_new, fn_status)
logger.info("Wrote status file")
def write_status_unknown(fn_status):
"""Writes a status html page when no actual status is known
Args:
fn_status (str): Filename to write the status html to
"""
with open(fn_status, 'w') as fout:
fout.write(PROGRESS_HEAD + PROGRESS_BODY_UNKNOWN)
logger.info("Wrote unknown status")
if __name__ == "__main__":
main()
\ No newline at end of file
CentOS/7.5-ESPRESO/app/notifications_monitor.py deleted 100644 → 0
+ 0
− 65
View file @ dfcb11ef
import os
import time
import sys
import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("Notifications monitor")
def main():
fn_notifications = sys.argv[1]
fn_abort = sys.argv[2]
monitor(fn_notifications, fn_abort)
def monitor(fn_notifications, fn_abort):
"""
"""
logger.info("Notifications monitor starting up")
cached_stamp = 0
while True:
if not os.path.exists(fn_notifications):
logger.info("Notifications file doesn't exist")
else:
# Check if "last modified" time of the file has changed
stamp = os.stat(fn_notifications).st_mtime
if not stamp != cached_stamp:
logger.info("Notifications file hasn't changed since last check")
else:
logger.info("Notifications file has changed")
cached_stamp = stamp
command = get_last_line(fn_notifications).strip()
if command == "ABORT":
logger.info("Received abort command")
# Abort the main app by creating a specific file
open(fn_abort, 'a').close()
else:
logger.info("Ignoring unknown command {}".format(command))
time.sleep(1)
def get_last_line(fn):
"""Returns the last line of a file
Returns an empty string if the file is empty.
Args:
fn (str): File name of the file to read from
"""
with open(fn, 'r') as fin:
i = -1
for i, line in enumerate(fin):
pass
if i >= 0:
return line
else:
return ''
if __name__ == "__main__":
main()
\ No newline at end of file
CentOS/7.5-ESPRESO/app/startup.py deleted 100644 → 0
+ 0
− 94
View file @ dfcb11ef
import subprocess
import sys
import os
import string
import random
import shutil
import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("Startup")
def main():
# Input arguments
seconds_to_wait = int(sys.argv[1])
app_folder = sys.argv[2]
# Pre-defined files for communication with the "outside world"
fn_status = '/service/status.html'
fn_result = '/service/result.txt'
fn_notifications = '/service/notifications.txt'
# Write dummy value into result file
logger.info("Writing dummy result")
with open(fn_result, 'w') as fout:
fout.write("UNSET")
# Define files for the main app in a unique folder
ID = generate_id('/home')
unique_folder = os.path.join('/home', ID)
os.mkdir(unique_folder)
fn_log = os.path.join(unique_folder, 'main_log.txt')
fn_abort = os.path.join(unique_folder, 'main_abort.txt')
logger.info("Created unique folder {}".format(unique_folder))
# 1: Start log crawler as a background process
# The log crawler monitors a single log file and creates status html pages
# from it
# The log crawler will automatically terminate when it reads the finishing
# log from the main application
logger.info("Starting log crawler")
log_crawler_script = os.path.join(app_folder, 'log_crawler.py')
command = ['python', log_crawler_script, fn_log, fn_status]
proc_log = subprocess.Popen(command)
# 2: Start the notifications monitor as a background process
# It monitors the notifications file (which is written to when messages are
# sent to the HPC service running this Singularity image) and translates
# messages into comands to the main application. Here, this is done simply
# by creating a specific file the main app looks for. Only a single command
# (abort) is supported.
# The notifications monitor does _not_ terminate by itself.
logger.info("Starting notifications monitor")
not_monitor_script = os.path.join(app_folder, 'notifications_monitor.py')
command = ['python', not_monitor_script, fn_notifications, fn_abort]
proc_not = subprocess.Popen(command)
# 3: Start the main application as a foreground process
# The main application will write to the log file monitored by the crawler,
# and it will write its final result into the result file.
# It will also check periodically for the existence of 'fn_abort' and quit
# if this file exists. (This is a cheap way of sending a command to the
# main app.)
logger.info("Starting waiter")
waiter_script = os.path.join(app_folder, 'wait_a_while.py')
command = ['python', waiter_script, str(seconds_to_wait), fn_log,
fn_result, fn_abort]
subprocess.run(command)
logger.info("Waiter finished, terminating processes")
# 4: Terminate notifications monitor and remove log folders
# As the main app is finished at this point, we can safely terminate the
# notifications monitor
proc_log.terminate()
proc_not.terminate()
shutil.rmtree(unique_folder)
# (Optional) 5: Process end result
# In case the final result of the main application (see step 3) is _not_
# just a simple file, we should call another script here which processes
# this end result and writes into the /service/result.txt file.
def generate_id(folder, size=16, chars=string.ascii_uppercase + string.digits):
"""Generates a random ID string which guaranteed not to exist in a folder"""
while True:
ID = ''.join(random.choice(chars) for _ in range(size))
if not os.path.exists(os.path.join(folder, ID)):
break
return ID
if __name__ == "__main__":
main()
\ No newline at end of file
CentOS/7.5-ESPRESO/app/wait_a_while.py deleted 100644 → 0
+ 0
− 64
View file @ dfcb11ef
import sys
import time
import datetime
import json
import os
import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("Waiter")
def main():
seconds_to_wait = int(sys.argv[1])
fn_log = sys.argv[2]
fn_result = sys.argv[3]
fn_abort = sys.argv[4]
logger.info("Waiter starting up, will wait {} seconds".format(seconds_to_wait))
for current_time in range(seconds_to_wait):
# Perform part of "complicated computation"
time.sleep(1)
# Write log
write_log(fn_log, current_time, seconds_to_wait)
# Check if we should abort
if os.path.exists(fn_abort):
write_final_log(fn_log)
write_result(fn_result, current_time, aborted=True)
return 0
write_final_log(fn_log)
write_result(fn_result, seconds_to_wait)
def write_log(fn_log, elapsed_time, total_time):
log = {
'timestamp': str(datetime.datetime.now()),
'elapsed_time': str(elapsed_time),
'total_time': str(total_time),
}
with open(fn_log, 'a') as fout:
json.dump(log, fout)
fout.write("\n")
logger.info("Log written: {}/{} seconds waited".format(elapsed_time, total_time))
def write_final_log(fn_log):
with open(fn_log, 'a') as fout:
fout.write("FINISHED\n")
logger.info("Final log written")
def write_result(fn, seconds_waited, aborted=False):
with open(fn, 'w') as f:
if aborted:
f.write('Done. My waiting was aborted after {} seconds.'.format(seconds_waited))
else:
f.write('Done. I have waited {} seconds.'.format(seconds_waited))
logger.info("Result written")
if __name__ == "__main__":
main()
CentOS/7.5-ESPRESO/heat.cpp deleted 100644 → 0
+ 0
− 145
View file @ dfcb11ef
# ESPRESO Configuration File
#BENCHMARK ARG0 [ TETRA4, TETRA10, PYRAMID5, PYRAMID13, PRISMA6, PRISMA15, HEXA8, HEXA20 ]
#BENCHMARK ARG10 [ TOTAL_FETI, HYBRID_FETI ]
DEFAULT_ARGS {
0 HEXA8;
1 1;
2 1;
3 1;
4 2;
5 2;
6 2;
7 5;
8 5;
9 5;
10 TOTAL_FETI;
}
INPUT GENERATOR;
PHYSICS HEAT_TRANSFER_3D;
DECOMPOSITION {
BALANCE_CLUSTERS TRUE;
}
GENERATOR {
SHAPE GRID;
GRID {
LENGTH_X 1;
LENGTH_Y 1;
LENGTH_Z 1;
NODES {
BOTTOM <1 , 1> <0 , 1> <0 , 1>;
TOP <0 , 0> <0 , 1> <0 , 1>;
}
ELEMENT_TYPE [ARG0];
BLOCKS_X 1;
BLOCKS_Y 1;
BLOCKS_Z 1;
CLUSTERS_X [ARG1];
CLUSTERS_Y [ARG2];
CLUSTERS_Z [ARG3];
DOMAINS_X [ARG4];
DOMAINS_Y [ARG5];
DOMAINS_Z [ARG6];
ELEMENTS_X [ARG7];
ELEMENTS_Y [ARG8];
ELEMENTS_Z [ARG9];
}
}
HEAT_TRANSFER_3D {
LOAD_STEPS 1;
MATERIALS {
1 {
COORDINATE_SYSTEM {
TYPE CYLINDRICAL;
CENTER { X .5; Y .5; Z .5; }
ROTATION { X 90; }
}
DENS 1;
CP 1;
THERMAL_CONDUCTIVITY {
MODEL DIAGONAL;
KXX 1;
KYY 10;
KZZ 10;
}
}
}
MATERIAL_SET {
ALL_ELEMENTS 1;
}
INITIAL_TEMPERATURE {
ALL_ELEMENTS 200;
}
STABILIZATION CAU;
SIGMA 0;
LOAD_STEPS_SETTINGS {
1 {
DURATION_TIME 1;
TYPE STEADY_STATE;
MODE LINEAR;
SOLVER FETI;
FETI {
METHOD [ARG10];
PRECONDITIONER DIRICHLET;
PRECISION 1E-08;
ITERATIVE_SOLVER PCG;
REGULARIZATION ANALYTIC;
}
TEMPERATURE {
TOP 100;
BOTTOM 300;
}
}
}
}
OUTPUT {
RESULTS_STORE_FREQUENCY EVERY_TIMESTEP;
MONITORS_STORE_FREQUENCY EVERY_TIMESTEP;
MONITORING {
1 {
REGION TOP;
STATISTICS MAX;
PROPERTY TEMPERATURE;
}
2 {
REGION BOTTOM;
STATISTICS MIN;
PROPERTY TEMPERATURE;
}
5 {
REGION ALL_ELEMENTS;
STATISTICS AVG;
PROPERTY TEMPERATURE;
}
}
}
Debian/8.0/Debian-8.0-Bootstrap deleted 100644 → 0
+ 0
− 57
View file @ dfcb11ef
Bootstrap: docker
From: debian:jessie
%post
apt -y update && apt -y upgrade
apt -y install python curl wget gcc g++ make vim mc libncurses-dev rsync tclsh lua5.2 liblua5.2 gfortran ssh
curl -s -O https://bootstrap.pypa.io/get-pip.py
python get-pip.py
pip install setuptools python-graph-core python-graph-dot tabulate numpy scipy pandas wheel --upgrade
wget https://github.com/luarocks/luarocks/archive/v2.4.2.tar.gz
tar xvf v2.4.2.tar.gz
cd luarocks-2.4.2
./configure
make build
make install
mkdir ~/.luarocks
echo "fs_use_modules = false" > ~/.luarocks/config.lua
luarocks install luaposix
luarocks install luafilesystem
LUAROCKS_PREFIX=/usr/local
export LUA_PATH="$LUAROCKS_PREFIX/share/lua/5.1/?.lua;$LUAROCKS_PREFIX/share/lua/5.1/?/init.lua;;"
export LUA_CPATH="$LUAROCKS_PREFIX/lib/lua/5.1/?.so;;"
echo "# !/bin/bash" > /bin/logger
echo "exit 0" >> /bin/logger
chmod +x /bin/logger
# Lmod
wget https://github.com/TACC/Lmod/archive/7.7.7.tar.gz
tar xvf 7.7.7.tar.gz
cd Lmod-7.7.7
./configure --prefix=/opt/apps
make install
# OpenMPI
wget https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.1.tar.gz
tar xvf openmpi-2.1.1.tar.gz && cd openmpi-2.1.1
./configure --prefix=/usr/local
make -j 16
make install
cd /bin
rm sh && ln -s /bin/bash sh
# APPS and SCRATCH
mkdir /apps
mkdir /scratch
%environment
source /opt/apps/lmod/7.7.7/init/profile
clearMT
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
export -f ml
export -f module
export -f clearMT
Ubuntu/16.04-OpenFOAM/README deleted 100644 → 0
+ 0
− 74
View file @ dfcb11ef
A few instruction for building an openfoam image to be used on the CFG platform.
Openfoam in this image can be used directly, by calling the solver, or through a python procedure useful to monitor the calculation on the CFG platform.
Usually I build a sandbox writable image. When it's ok, I build a read-only image to be uploaded on CFG platform.
openfoamdev/ is the sandbox image, to be used for development and debug
openfoam.simg is a non-writable image deployable on CFG platform
BUILDING OF THE IMAGE
To build the writable image, use the recipe-openfoam.txt recipe as follows
sudo singularity build -s openfoamdev recipe-openfoam.txt
Files in folders app and openfoam_hacks are needed with this recipe.
In the app folders there are files needed for checking and monitoring the calculation.
They can be modified or not added.
Files in openfoam_hacks are needed to allows openfoam to run in the container.
TESTING OF THE IMAGE
To test the container, the two mountpoints requested by the CGF platform are to be creted.
Of course they can reside in a folder different than ~.
mkdir ~/scratch
mkdir ~/service
- Serial test (direct call of the solver). Specify the solver and the case folder (-case option)
singularity -B ~/service:service ~/scratch:/scratch exec --cleanenv openfoamdev ###SOLVER###
- Parallel test (direct call of the solver). Specify the solver, the case folder (-case option) and the number of processors.
Of course, modify the system/decomposeParDict file accordingly.
singularity -B ~/service:service ~/scratch:/scratch exec --cleanenv openfoamdev decomposePar
mpirun -np ###PROCS#### singularity -B ~/service:service ~/scratch:/scratch exec --cleanenv openfoamdev ###SOLVER### -parallel
NOTE When executing directly a solver in parallel mode, this can be a truly MPI calculation.
Instead, when testing the python procedure, only processors on one single node can be used.
- Python procedure test.
When using the python procedure, the solver, the case folder and the number of processors are specified on command line.
The decomposePar utility is executed, then the solver is started through an mpirun command and the the check utility is started.
As stated before, in this way only processors on a single node can be used. This will hopefully change in future.
A webpage is displayed in "~/service/status.html". The abort button can't work locally, but only on the CFG platform.
Anyway, the interrupt procedure is implemented. See /app/notification file.
singularity -B ~/service:service ~/scratch:/scratch --cleanenv exec openfoamdev /app/startup -s SOLVER -f FOLDER -p NUMBER_OF_PROCESSORS
The /app/check file is to be modified in order to display the desided webpage.
The parsing procedure is taylored for interFoam solver and can fail when parsing output of different solvers, so please modify it and test it locally.
DEPLOYMENT ON CFG PLATFORM
To build to a non writable image:
sudo singularity build openfoam.simg recipe-openfoam.txt
The hpc_image script can be used to upload and register the new image on the CFG platform,
or to unregister and delete. Specify username, password and project in the script before using it.
To upload and register
./hpc_image upload openfoam.simg
To unregister and delete
./hpc_image delete openfoam.simg
Ubuntu/16.04-OpenFOAM/Ubuntu-16.04-Bootstrap deleted 100644 → 0
+ 0
− 105
View file @ dfcb11ef
Bootstrap: docker
From: ubuntu:16.04
%post
apt -y update && apt -y upgrade
apt -y install python curl wget gcc g++ make vim mc libncurses-dev rsync tclsh lua5.2 liblua5.2 gfortran ssh apt-transport-https
apt -y install dkms infiniband-diags libibverbs* ibacm librdmacm* libmlx4* libmlx5* mstflint libibcm.* libibmad.* libibumad* opensm srptools libmlx4-dev librdmacm-dev rdmacm-utils ibverbs-utils perftest vlan ibutils
apt -y install libtool autoconf automake build-essential ibutils ibverbs-utils rdmacm-utils infiniband-diags perftest librdmacm-dev libibverbs-dev libmlx4-1 numactl libnuma-dev autoconf automake gcc g++ git libtool pkg-config
apt -y install libnl-3-200 libnl-route-3-200 libnl-route-3-dev libnl-utils
curl -s -O https://bootstrap.pypa.io/get-pip.py
python get-pip.py
pip install setuptools python-graph-core python-graph-dot tabulate numpy scipy pandas wheel --upgrade
cd /tmp
wget https://github.com/luarocks/luarocks/archive/v2.4.2.tar.gz
tar xvf v2.4.2.tar.gz
cd luarocks-2.4.2
./configure
make build
make install
mkdir ~/.luarocks
echo "fs_use_modules = false" > ~/.luarocks/config.lua
luarocks install luaposix
luarocks install luafilesystem
LUAROCKS_PREFIX=/usr/local
export LUA_PATH="$LUAROCKS_PREFIX/share/lua/5.1/?.lua;$LUAROCKS_PREFIX/share/lua/5.1/?/init.lua;;"
export LUA_CPATH="$LUAROCKS_PREFIX/lib/lua/5.1/?.so;;"
echo "# !/bin/bash" > /bin/logger
echo "exit 0" >> /bin/logger
chmod +x /bin/logger
# Lmod
cd /tmp
wget https://github.com/TACC/Lmod/archive/7.7.7.tar.gz
tar xvf 7.7.7.tar.gz
cd Lmod-7.7.7
./configure --prefix=/opt/apps
make install
# OpenMPI
cd /tmp
wget https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.1.tar.gz
tar xvf openmpi-2.1.1.tar.gz && cd openmpi-2.1.1
./configure --prefix=/usr/local \
--enable-orterun-prefix-by-default \
--enable-mpirun-prefix-by-default \
--enable-static \
--enable-shared \
--with-verbs
make -j 16
make install
# APPS and SCRATCH
mkdir /apps
mkdir /scratch
cd /bin
rm sh && ln -s /bin/bash sh
### folders and files ###
mkdir /service
mv /opt/startup /app
mv /opt/check /app
mv /opt/monitor /app
### set-up python3 ###
#apt install -y python3-pip python3-mpi4py
#pip3 install numpy matplotlib
### install openfoam ###
apt install -y software-properties-common wget
add-apt-repository "http://dl.openfoam.org/ubuntu"
sh -c "wget -O - http://dl.openfoam.org/gpg.key | apt-key add -"
apt update
apt install -y openfoam6
echo '. /opt/openfoam6/etc/bashrc' >>$SINGULARITY_ENVIRONMENT
### openfoam hacks ###
#mv -f /opt/bashrc /opt/openfoam6/etc
#mv -f /opt/aliases /opt/openfoam6/etc/config.sh
#mv -f /opt/exec /.singularity.d/actions
%environment
source /opt/apps/lmod/7.7.7/init/profile
clearMT
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
export -f ml
export -f module
export -f clearMT
%files
app/startup opt/
app/check opt/
app/monitor opt/
openfoam_hacks/bashrc opt/
openfoam_hacks/aliases opt/
openfoam_hacks/exec opt/
Ubuntu/16.04-OpenFOAM/app/check deleted 100755 → 0
+ 0
− 183
View file @ dfcb11ef
#!/usr/bin/python3
'''
This file parses the solver log file and produces the webpage displayed
during the simulation. Also a webpage result is written after the end of the
simulation.
The webpage displays just the residuals and the abort button. Of course it
can be customized.
'''
import time
import sys
from subprocess import check_output
from os.path import isfile, basename
from os import getcwd
from shutil import copy
import logging
import base64
import re
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
PROGRESS_HEAD = '''<html>
<head>
<title>OPENFOAM job progress</title>
<script type="text/javascript">
function abort() {
notify_running_job("ABORT");
}
</script>
</head>'''
PROGRESS_BODY='''<body style="margin: 20px; padding: 20px;">
<h3 align=center>Simulation {}</h3>
<p style="text-align:center;">
<!-- Images to be embedded encoded in base64-->
<img src = "data:image/png;base64,{}" alt ="missing image">
</p>
<p style="text-align:center;">
<input type="button" value="Abort simulation" onclick="abort()">
</p>
</body>
</html>
'''
'''
Regular expression needed to parse the log file
'''
_rx_dict = {
'time': re.compile(r'^Time = (?P<time>(\d+.\d+e[+-]\d+)|(\d+[.\d+]*))'),
'timestep': re.compile(r'^deltaT = (?P<timestep>(\d+.\d+e[+-]\d+)|(\d+[.\d+]*))'),
'residual': re.compile(r'''
^\w+: \s+ # solver name
Solving \s for \s (?P<quantity>\w+(.\w+)*), \s # field
Initial \s residual \s = \s
(?P<residual>(-?\d+.\d+e[+-]\d+)|(-?\d+[.\d+]*)) # residual value
''',
re.VERBOSE),
'clock': re.compile(r'ExecutionTime = (?P<clock>\d+.\d+) s')
}
def _parse_line(line):
'''
Check if the line matches one of the regex in _rx_dict
'''
for key, rx in _rx_dict.items():
match = rx.search(line)
if match:
return key, match
return None, None
def give_nth_word(line,n):
try:
out = line.split()[n-1].strip(",")
except IndexError:
return ""
return out
def parse_logfile(logfilename,data,offset=0):
'''
Parses the file LOGFILENAME starting from OFFSET line, putting requested
data in list DATA and updating the OFFSET variable.
It is executed every 3 seconds (see main function).
Residuals, timestep, simulation time and clock time are parsed and
appended to data, which is a list of list.
'''
time_data = {}
content = check_output(["cat",logfilename])
lines = content.decode("utf-8").splitlines(True)
for index,line in enumerate(lines):
if index > offset:
key, match = _parse_line(line)
if key == 'time':
time_data['time'] = float(match.group('time'))
if key == 'residual':
time_data[match.group('quantity')] = float(match.group('residual'))
if key == 'timestep':
time_data['timestep'] = float(match.group('timestep'))
if key == 'clock':
time_data['clock'] = float(match.group('clock'))
offset = index
data[0].append(time_data['time'])
data[1].append(time_data['timestep'])
data[2].append(time_data['Ux'])
data[3].append(time_data['Uy'])
data[4].append(time_data['Uz'])
data[5].append(time_data['p_rgh'])
data[6].append(time_data['clock'])
return offset
def check_if_finished(logfilename):
with open(logfilename,'r') as logfile:
for line in logfile:
pass
if give_nth_word(line,1) == "Finalising":
return True
else:
return False
def plot_images(data):
'''
Plot residual data and save it in a png file. This file will be embedded
in the webpage.
'''
try:
fig = plt.figure()
ax1 = fig.add_subplot(1,1,1)
ax1.set_xlabel('Time [s]')
ax1.set_ylabel('Residuals')
ax1.semilogy(data[0],data[2], label='ux')
ax1.semilogy(data[0],data[3], label='uy')
ax1.semilogy(data[0],data[4], label='uz')
ax1.semilogy(data[0],data[5], label='p')
leg = ax1.legend()
plt.savefig('/service/output.png')
plt.close()
except UserWarning:
return
return
def main():
'''
Main function.
Every 3 seconds, che logfile is parsed, it is checked if the simulation is
finished, then the status webpage is updated.
In the end, the same webpage is written in the result file.
'''
folder = sys.argv[1]
casename = basename((getcwd()))
logfilename = folder + '/' + casename + '.log'
statusfilename = '/service/status.html'
resultfilename = '/service/result.txt'
time.sleep(5)
finished = False
log_offset = 0
output_data = [[],[],[],[],[],[],[]]
### time, timestep, ux, uy, uz, pres, clock
while not finished:
time.sleep(3)
log_offset = parse_logfile(logfilename,output_data,log_offset)
finished = check_if_finished(logfilename)
plot_images(output_data)
with open('/service/output.png','rb') as imgfile:
b64image = base64.b64encode(imgfile.read()).decode()
with open(statusfilename,'w') as statfile:
statfile.write(str(PROGRESS_HEAD + PROGRESS_BODY.format(casename,b64image)))
with open(resultfilename,'w') as resfile:
resfile.write(str(PROGRESS_HEAD + PROGRESS_BODY.format(casename,b64image)))
if __name__ == "__main__":
main()
Ubuntu/16.04-OpenFOAM/app/monitor deleted 100755 → 0
+ 0
− 62
View file @ dfcb11ef
#!/usr/bin/env python3
'''
This file checks if something is written in the notification file
If the message ABORT is written, the controlDict file is modified in order to
stop the simulation.
Other answers to other command could be implemented, if needed.
'''
import os
import sys
import logging
import time
from subprocess import check_output
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger("Notifications monitor")
def modify_dict(dictionary,keyword,value):
spaces = " " * (16-len(keyword))
try:
with open(dictionary,"r") as old_filedict:
with open(dictionary+"new","w") as new_filedict:
for line in old_filedict:
keys = line.split()
if (len(keys) > 1) and not "//" in keys[0]:
if keyword == keys[0]:
line = keyword + spaces + str(value) + ";\n"
print(line,end='',file=new_filedict)
os.remove(dictionary)
os.rename(dictionary+"new",dictionary)
except FileNotFoundError:
print("File " + dictionary + " not found.\n")
exit()
return
def main():
logger.info("Notification monitor started")
folder = sys.argv[1]
dictFile = folder + '/system/controlDict'
cached_stamp = 0
while True:
if os.path.exists('/service/notifications.txt'):
stamp = os.stat('/service/notifications.txt').st_mtime
if stamp != cached_stamp:
logger.info("New notifications received")
cached_stamp = stamp
command = check_output(["tail","-1","/service/notifications.txt"]).strip().decode()
if command == "ABORT":
logger.info("Received abort command")
modify_dict(dictFile,"stopAt","writeNow")
else:
logger.info("Ignoring unknown command {}".format(command))
time.sleep(1)
if __name__ == "__main__":
main()
Ubuntu/16.04-OpenFOAM/app/startup deleted 100755 → 0
+ 0
− 106
View file @ dfcb11ef
#!/usr/bin/env python3
'''
This is the principal file.
It parses the command line, in which requested openFoam solver, case folder
and number of processors are specified.
It decomposes the case, calls the check and monitor scripts, then calls
the solver with the correct parameters and, finally, reconstructs the case.
cmdline syntax:
startup -s SOLVER [-f FOLDER] -p PROCESSORS
The execution is parallel, but since mpi is called inside the container,
processes are spawn only on one node, so there can be at maximum 16 processes
on Anselm ann 24 on Salomon.
A truly parallel execution would probably require an mpi4py procedure.
(Leonardo) It seems that subprocess commands don't work in conjunction with
an mpi4py script. I have no idea on how to launch an MPI
openfoam solver from inside a MPI python script.
'''
from subprocess import Popen, run, DEVNULL
from os import getcwd, chdir, rename, remove
from os.path import basename, abspath
import logging
import argparse
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger("Openfoam image logger")
def command_line():
'''
Define and parses the command line arguments.
'''
parser = argparse.ArgumentParser()
parser.add_argument('-solver',action='store',dest='solver')
parser.add_argument('-folder', action='store', dest='folder')
parser.add_argument('-processor',action='store', type=int, dest='np')
args = vars(parser.parse_args())
solver = args['solver']
folder = args['folder']
np = args['np']
return solver,folder,np
def modify_dict(dictionary,keyword,value):
'''
Modify the VALUE of the desired KEYWORD in the desider DICTIONARY
'''
spaces = " " * (16-len(keyword))
try:
with open(dictionary,"r") as old_filedict:
with open(dictionary+"new","w") as new_filedict:
for line in old_filedict:
keys = line.split()
if (len(keys) > 1) and not "//" in keys[0]:
if keyword == keys[0]:
line = keyword + spaces + str(value) + ";\n"
print(line,end='',file=new_filedict)
remove(dictionary)
rename(dictionary+"new",dictionary)
except FileNotFoundError:
print("File " + dictionary + " not found.\n")
exit()
return
def main():
solver,folder,np = command_line()
# folder check
if folder is not None:
try:
chdir(folder)
except FileNotFoundError:
logger.error("Folder {} not found".format(folder))
exit()
else:
folder = getcwd()
folder = abspath(getcwd())
casename = basename(getcwd())
logfilename = folder + '/' + casename + '.log'
statusfilename = '/service/status.html'
with open(logfilename,'w') as logfile:
logger.info("Running decomposePar utility")
run(["decomposePar","-force"],stdout=logfile)
check_proc = Popen(["/app/check",folder])
monitor_proc = Popen(["/app/monitor",folder])
logger.info("Running {} on {} processors".format(solver,str(np)))
run(["mpirun","-np",str(np),solver,"-parallel","-case",folder],stdout=logfile)
logger.info("Running reconstructPar")
run(["reconstructPar"],stdout=DEVNULL)
monitor_proc.terminate()
### in case of abort command, set the controlDict file in the original state
ctrlDict = folder + '/system/controlDict'
modify_dict(ctrlDict,"stopAt","endTime")
logger.info("Calculation ended")
if __name__ == "__main__":
main()
Ubuntu/16.04-OpenFOAM/hpc_image deleted 100644 → 0
+ 0
− 71
View file @ dfcb11ef
#!/usr/bin/python3
"""Ugly but working hard-coded test script for the HPC client"""
"""
For upload and register, hpc_image upload image_name
For unregister and delete, hpc_image delete image_name
N.B. Images are uploaded in /home/images folder in anselm cluster
N.B. Folder /home/images has to exist on the cluster
N.B. Username, password and project should be filled with needed values
"""
import os
import filecmp
import json
import clfpy as cf
import sys
auth_url = "https://api.hetcomp.org/authManager/AuthManager?wsdl"
hpc_url = "https://api.hetcomp.org/hpc_anselm/Images?wsdl"
gss_url = "https://api.hetcomp.org/gss-0.1/FileUtilities?wsdl"
username = ""
password = ""
project = ''
print("Obtaining session token ...")
auth = cf.AuthClient(auth_url)
session_token = auth.get_session_token(username, project, password)
gss = cf.GssClient(gss_url)
hpc = cf.HpcImagesClient(hpc_url)
def upload():
print("Uploading and registering a new image")
image_filepath = sys.argv[2]
gss_ID = "it4i_anselm://home/images/" + sys.argv[2]
print(gss.upload(gss_ID, session_token, image_filepath))
print(hpc.register_image(session_token, sys.argv[2], gss_ID))
def delete():
print("Removing and deleting image")
print(hpc.delete_image(session_token, sys.argv[2]))
gss_ID = "it4i_anselm://home/images/" + sys.argv[2]
print(gss.delete(gss_ID, session_token))
def main():
if sys.argv[1] == "upload":
upload()
elif sys.argv[1] == "delete":
delete()
if __name__ == "__main__":
main()
Ubuntu/16.04-OpenFOAM/openfoam6-linux deleted 100644 → 0
+ 0
− 144
View file @ dfcb11ef
#!/bin/sh
#------------------------------------------------------------------------------
# ========= |
# \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
# \\ / O peration |
# \\ / A nd | Copyright (C) 2017 OpenFOAM Foundation
# \\/ M anipulation |
#-------------------------------------------------------------------------------
# License
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License
# along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
#
# Script
# openfoam6-linux
#
# Description
# Run script for an OpenFOAM 6 Docker image at:
# https://hub.docker.com/r/openfoam
#
#------------------------------------------------------------------------------
Script=${0##*/}
VER=6
usage () {
exec 1>&2
while [ "$#" -ge 1 ]; do echo "$1"; shift; done
cat <<USAGE
Usage: ${0##*/} [OPTIONS]
options:
-d | -dir host directory mounted (defaults to current directory)
-x | -xhost use custom X authority and give container host network
-h | -help help
Launches the OpenFOAM ${VER} Docker image.
- Requires installation of docker-engine.
- Runs a "containerized" bash shell environment where the user can run OpenFOAM
and, optionally, ParaView (see below).
- The container mounts the user's file system so that case files are stored
permanently. The container mounts the current directory by default, but the
user can also specify a particular directory using the "-d" option.
- Mounting the user's HOME directory is disallowed.
- The '-xhost' option is useful when accessing the host via 'ssh -X'.
This option should only be used when strictly necessary, as it relies on the
option '--net=host' when launching the container in Docker, which will
give to the container full access to the Docker host network stack and
potentially the host's system services that rely on network communication,
making it potentially insecure.
Example:
To store data in ${HOME}/OpenFOAM/${USER}-${VER}, the user can launch
${Script} either by:
cd ${HOME}/OpenFOAM/${USER}-${VER} && ${Script}
or
${Script} -d ${HOME}/OpenFOAM/${USER}-${VER}
Further Information:
http://openfoam.org/download/6-linux
Note:
The container user name appears as "openfoam" but it is just an alias.
USAGE
exit 1
}
DOCKER_IMAGE='openfoam/openfoam6-paraview54'
MOUNT_DIR=$(pwd)
CUSTOM_XAUTH=""
DOCKER_OPTIONS=""
while [ "$#" -gt 0 ]
do
case "$1" in
-d | -dir)
[ "$#" -ge 2 ] || usage "'$1' option requires an argument"
MOUNT_DIR=$2
shift 2
;;
-x | -xhost)
CUSTOM_XAUTH=yes
shift
;;
-h | -help)
usage
;;
*)
usage "Invalid option '$1'"
;;
esac
done
[ -d "$MOUNT_DIR" ] || usage "No directory exists: $MOUNT_DIR"
MOUNT_DIR=$(cd "$MOUNT_DIR" && pwd -P)
[ "$MOUNT_DIR" = "$(cd "$HOME" && pwd -P)" ] && \
usage "Mount directory cannot be the user's home directory" \
"Make a subdirectory and run from there, e.g." \
" mkdir -p ${HOME}/OpenFOAM/$(whoami)-${VER}" \
" ${Script} -d ${HOME}/OpenFOAM/$(whoami)-${VER}"
if [ -n "$CUSTOM_XAUTH" ]
then
XAUTH_PATH="${MOUNT_DIR}/.docker.xauth.$$"
touch "${XAUTH_PATH}"
# Generate a custom X-authority file that allows any hostname
xauth nlist "$DISPLAY" | sed -e 's/^..../ffff/' | \
xauth -f "$XAUTH_PATH" nmerge -
DOCKER_OPTIONS="-e XAUTHORITY=$XAUTH_PATH
-v $XAUTH_PATH:$XAUTH_PATH
--net=host"
fi
USER_ID=$(id -u 2> /dev/null)
[ -n "$USER_ID" ] || usage "Cannot determine current user ID"
GROUP_ID=$(id -g)
HOME_DIR='/home/openfoam'
echo "Launching $0"
echo "User: \"$(id -un)\" (ID $USER_ID, group ID $GROUP_ID)"
docker run -it \
--rm \
-e DISPLAY=$DISPLAY \
-u $USER_ID:$GROUP_ID \
-v /tmp/.X11-unix:/tmp/.X11-unix \
-v $MOUNT_DIR:$HOME_DIR \
$DOCKER_OPTIONS \
$DOCKER_IMAGE
[ -n "$CUSTOM_XAUTH" -a -e "${XAUTH_PATH}" ] && rm "${XAUTH_PATH}"
Ubuntu/16.04-OpenFOAM/openfoam_hacks/aliases deleted 100755 → 0
+ 0
− 120
View file @ dfcb11ef
#----------------------------------*-sh-*--------------------------------------
# ========= |
# \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
# \\ / O peration |
# \\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation
# \\/ M anipulation |
#------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM.
#
# OpenFOAM is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License
# along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
#
# File
# etc/config.sh/aliases
#
# Description
# Aliases for working with OpenFOAM
# Sourced from OpenFOAM-<VERSION>/etc/bashrc and/or ~/.bashrc
#
#------------------------------------------------------------------------------
# Change compiled version aliases
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
alias wmSet='. $WM_PROJECT_DIR/etc/bashrc'
alias wm64='wmSet WM_ARCH_OPTION=64'
alias wm32='wmSet WM_ARCH_OPTION=32'
alias wmSP='wmSet WM_PRECISION_OPTION=SP'
alias wmDP='wmSet WM_PRECISION_OPTION=DP'
# Clear env
alias wmUnset='. $WM_PROJECT_DIR/etc/config.sh/unset'
# Toggle wmakeScheduler on/off
# - also need to set WM_HOSTS
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
alias wmSchedOn='export WM_SCHEDULER=$WM_PROJECT_DIR/wmake/wmakeScheduler'
alias wmSchedOff='unset WM_SCHEDULER'
# Change directory aliases
# ~~~~~~~~~~~~~~~~~~~~~~~~
alias foam='cd $WM_PROJECT_DIR'
if [ -n "$WM_PROJECT_SITE" ]
then
alias foamSite='cd $WM_PROJECT_SITE'
else
alias foamSite='cd $WM_PROJECT_INST_DIR/site'
fi
alias src='cd $FOAM_SRC'
alias lib='cd $FOAM_LIBBIN'
alias app='cd $FOAM_APP'
alias sol='cd $FOAM_SOLVERS'
alias util='cd $FOAM_UTILITIES'
alias tut='cd $FOAM_TUTORIALS'
alias run='cd $FOAM_RUN'
# Refresh the environment
# ~~~~~~~~~~~~~~~~~~~~~~~
# For backward-compatibility unalias wmRefresh if it is defined as an alias
[ "$(type -t wmRefresh)" = "alias" ] && unalias wmRefresh || unset wmRefresh
#if declare -f wmRefresh > /dev/null
#then
# unset wmRefresh
#else
# unalias wmRefresh 2> /dev/null
#fi
wmRefresh()
{
wmProjectDir=$WM_PROJECT_DIR
foamSettings=$FOAM_SETTINGS
wmUnset
. $wmProjectDir/etc/bashrc $foamSettings
}
# Change OpenFOAM version
# ~~~~~~~~~~~~~~~~~~~~~~~
unset foamVersion
foamVersion()
{
if [ "$1" ]; then
foamInstDir=$FOAM_INST_DIR
wmUnset
. $foamInstDir/OpenFOAM-$1/etc/bashrc
foam
echo "Changed to OpenFOAM-$1" 1>&2
else
echo "OpenFOAM-$WM_PROJECT_VERSION" 1>&2
fi
}
# Change ParaView version
# ~~~~~~~~~~~~~~~~~~~~~~~
unset foamPV
foamPV()
{
. $WM_PROJECT_DIR/etc/config.sh/paraview ParaView_VERSION=$1
echo "paraview-$ParaView_VERSION (major: $ParaView_MAJOR)" 1>&2
}
#------------------------------------------------------------------------------
Ubuntu/16.04-OpenFOAM/openfoam_hacks/bashrc deleted 100755 → 0
+ 0
− 215
View file @ dfcb11ef
#----------------------------------*-sh-*--------------------------------------
# ========= |
# \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
# \\ / O peration |
# \\ / A nd | Copyright (C) 2011-2017 OpenFOAM Foundation
# \\/ M anipulation |
#------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM.
#
# OpenFOAM is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License
# along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
#
# File
# etc/bashrc
#
# Description
# Startup file for OpenFOAM
# Sourced from ~/.profile or ~/.bashrc
# Should be usable by any POSIX-compliant shell (eg, ksh)
#
#------------------------------------------------------------------------------
export WM_PROJECT=OpenFOAM
export WM_PROJECT_VERSION=5.0
################################################################################
# USER EDITABLE PART: Changes made here may be lost with the next upgrade
#
# FOAM_INST_DIR is the location of the OpenFOAM installation which defaults to
# the directory containing this file if sourced by a bash shell.
#
# Please set to the appropriate path if the default is not correct.
#
[ "$BASH" -o "$ZSH_NAME" ]
#export FOAM_INST_DIR=$(cd $(dirname ${BASH_SOURCE:-$0})/../.. && pwd -P) || \
export FOAM_INST_DIR=/opt
# export FOAM_INST_DIR=~$WM_PROJECT
# export FOAM_INST_DIR=/opt/$WM_PROJECT
# export FOAM_INST_DIR=/usr/local/$WM_PROJECT
#
# END OF (NORMAL) USER EDITABLE PART
################################################################################
# The default environment variables below can be overridden in a prefs.sh file
# located in ~/.OpenFOAM/$WM_PROJECT_VERSION, ~/.OpenFOAM,
# $FOAM_INST_DIR/site/$WM_PROJECT_VERSION or $FOAM_INST_DIR/site
#- Compiler location:
# WM_COMPILER_TYPE= system | ThirdParty (OpenFOAM)
export WM_COMPILER_TYPE=system
#- Compiler:
# WM_COMPILER = Gcc | Gcc48 ... Gcc62 | Clang | Icc
export WM_COMPILER=Gcc
unset WM_COMPILER_ARCH WM_COMPILER_LIB_ARCH
#- Memory addressing:
# On a 64bit OS this can be 32bit or 64bit
# On a 32bit OS addressing is 32bit and this option is not used
# WM_ARCH_OPTION = 32 | 64
export WM_ARCH_OPTION=64
#- Precision:
# WM_PRECISION_OPTION = DP | SP
export WM_PRECISION_OPTION=DP
#- Label size:
# WM_LABEL_SIZE = 32 | 64
export WM_LABEL_SIZE=32
#- Optimised, debug, profiling:
# WM_COMPILE_OPTION = Opt | Debug | Prof
export WM_COMPILE_OPTION=Opt
#- MPI implementation:
# WM_MPLIB = SYSTEMOPENMPI | OPENMPI | SYSTEMMPI | MPICH | MPICH-GM | HPMPI
# | MPI | FJMPI | QSMPI | SGIMPI | INTELMPI
export WM_MPLIB=SYSTEMOPENMPI
#- Operating System:
# WM_OSTYPE = POSIX | ???
export WM_OSTYPE=POSIX
#- Floating-point signal handling:
# set or unset
export FOAM_SIGFPE=
#- memory initialisation:
# set or unset
#export FOAM_SETNAN=
################################################################################
# The old dirs to be cleaned from the environment variables
foamOldDirs="$WM_PROJECT_DIR $WM_THIRD_PARTY_DIR \
$HOME/$WM_PROJECT/$USER $FOAM_USER_APPBIN $FOAM_USER_LIBBIN \
$WM_PROJECT_SITE $FOAM_SITE_APPBIN $FOAM_SITE_LIBBIN"
# Location of installation
# ~~~~~~~~~~~~~~~~~~~~~~~~
export WM_PROJECT_INST_DIR=$FOAM_INST_DIR
export WM_PROJECT_DIR=$WM_PROJECT_INST_DIR/openfoam5
# Location of third-party software
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
export WM_THIRD_PARTY_DIR=$WM_PROJECT_INST_DIR/ThirdParty-$WM_PROJECT_VERSION
# Location of site-specific templates etc
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# unset is equivalent to $WM_PROJECT_INST_DIR/site
if [ -d "$WM_PROJECT_SITE" ]
then
export WM_PROJECT_SITE
else
unset WM_PROJECT_SITE
fi
# Location of user files
# ~~~~~~~~~~~~~~~~~~~~~~
export WM_PROJECT_USER_DIR=$HOME/$WM_PROJECT/$USER-$WM_PROJECT_VERSION
# Source initialization functions
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
. $WM_PROJECT_DIR/etc/config.sh/functions
# Add in preset user or site preferences:
_foamSource `$WM_PROJECT_DIR/bin/foamEtcFile prefs.sh`
# Evaluate command-line parameters and record settings for later
# these can be used to set/unset values, or specify alternative pref files
export FOAM_SETTINGS="$@"
# _foamEval $@
# Clean standard environment variables (PATH, LD_LIBRARY_PATH, MANPATH)
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
foamClean=$WM_PROJECT_DIR/bin/foamCleanPath
#- Clean PATH
cleaned=`$foamClean "$PATH" "$foamOldDirs"` && PATH="$cleaned"
#- Clean LD_LIBRARY_PATH
cleaned=`$foamClean "$LD_LIBRARY_PATH" "$foamOldDirs"` \
&& LD_LIBRARY_PATH="$cleaned"
#- Clean MANPATH
cleaned=`$foamClean "$MANPATH" "$foamOldDirs"` && MANPATH="$cleaned"
export PATH LD_LIBRARY_PATH MANPATH
# Source project setup files
# ~~~~~~~~~~~~~~~~~~~~~~~~~~
_foamSource $WM_PROJECT_DIR/etc/config.sh/settings
_foamSource $WM_PROJECT_DIR/etc/config.sh/aliases
# Source user setup files for optional packages
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
_foamSource `$WM_PROJECT_DIR/bin/foamEtcFile config.sh/mpi`
_foamSource `$WM_PROJECT_DIR/bin/foamEtcFile config.sh/paraview`
_foamSource `$WM_PROJECT_DIR/bin/foamEtcFile config.sh/ensight`
_foamSource `$WM_PROJECT_DIR/bin/foamEtcFile config.sh/gperftools`
if [ ! -z "$FOAMY_HEX_MESH" ]
then
_foamSource `$WM_PROJECT_DIR/bin/foamEtcFile config.sh/CGAL`
fi
# Clean environment paths again. Only remove duplicates
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#- Clean PATH
cleaned=`$foamClean "$PATH"` && PATH="$cleaned"
#- Clean LD_LIBRARY_PATH
cleaned=`$foamClean "$LD_LIBRARY_PATH"` && LD_LIBRARY_PATH="$cleaned"
#- Clean MANPATH (trailing ':' to find system pages)
cleaned=`$foamClean "$MANPATH"`: && MANPATH="$cleaned"
export PATH LD_LIBRARY_PATH MANPATH
#- Clean LD_PRELOAD
if [ -n "$LD_PRELOAD" ]
then
cleaned=`$foamClean "$LD_PRELOAD"` && LD_PRELOAD="$cleaned"
export LD_PRELOAD
fi
# Cleanup environment:
# ~~~~~~~~~~~~~~~~~~~~
unset cleaned foamClean foamOldDirs
# Unload initialization functions:
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
. $WM_PROJECT_DIR/etc/config.sh/functions
# Source the bash completion file:
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[ "$BASH" ] && . $WM_PROJECT_DIR/etc/config.sh/bash_completion
#------------------------------------------------------------------------------
Ubuntu/16.04-OpenFOAM/openfoam_hacks/exec deleted 100755 → 0
+ 0
− 9
View file @ dfcb11ef
#!/bin/bash
for script in /.singularity.d/env/*.sh; do
if [ -f "$script" ]; then
. "$script"
fi
done
exec "$@"
Ubuntu/16.04-OpenFOAM/recipe-openfoam.txt deleted 100644 → 0
+ 0
− 48
View file @ dfcb11ef
Bootstrap: docker
From: ubuntu:17.10
%post
### folders and files ###
mkdir /scratch
mkdir /service
mkdir /app
mv /opt/startup /app
mv /opt/check /app
mv /opt/monitor /app
### update & upgrade ###
apt update
apt upgrade -y
apt install -y nano ssh
### set-up python3 ###
apt install -y python3-pip python3-mpi4py
pip3 install numpy matplotlib
### install openfoam ###
apt install -y software-properties-common wget
add-apt-repository "http://dl.openfoam.org/ubuntu"
sh -c "wget -O - http://dl.openfoam.org/gpg.key | apt-key add -"
apt update
apt install -y openfoam5
echo '. /opt/openfoam5/etc/bashrc' >>$SINGULARITY_ENVIRONMENT
### openfoam hacks ###
mv -f /opt/bashrc /opt/openfoam5/etc
mv -f /opt/aliases /opt/openfoam5/etc/config.sh
mv -f /opt/exec /.singularity.d/actions
%files
app/startup opt/
app/check opt/
app/monitor opt/
openfoam_hacks/bashrc opt/
openfoam_hacks/aliases opt/
openfoam_hacks/exec opt/
Ubuntu/16.04-OpenFOAM/singularity_image.tar deleted 100644 → 0
+ 0
− 0
View file @ dfcb11ef
File deleted
Ubuntu/16.04/Ubuntu-16.04-Bootstrap deleted 100644 → 0
+ 0
− 57
View file @ dfcb11ef
Bootstrap: docker
From: ubuntu:latest
%post
apt -y update && apt -y upgrade
apt -y install python curl wget gcc g++ make vim mc libncurses-dev rsync tclsh lua5.2 liblua5.2 gfortran ssh
curl -s -O https://bootstrap.pypa.io/get-pip.py
python get-pip.py
pip install setuptools python-graph-core python-graph-dot tabulate numpy scipy pandas wheel --upgrade
wget https://github.com/luarocks/luarocks/archive/v2.4.2.tar.gz
tar xvf v2.4.2.tar.gz
cd luarocks-2.4.2
./configure
make build
make install
mkdir ~/.luarocks
echo "fs_use_modules = false" > ~/.luarocks/config.lua
luarocks install luaposix
luarocks install luafilesystem
LUAROCKS_PREFIX=/usr/local
export LUA_PATH="$LUAROCKS_PREFIX/share/lua/5.1/?.lua;$LUAROCKS_PREFIX/share/lua/5.1/?/init.lua;;"
export LUA_CPATH="$LUAROCKS_PREFIX/lib/lua/5.1/?.so;;"
echo "# !/bin/bash" > /bin/logger
echo "exit 0" >> /bin/logger
chmod +x /bin/logger
# Lmod
wget https://github.com/TACC/Lmod/archive/7.7.7.tar.gz
tar xvf 7.7.7.tar.gz
cd Lmod-7.7.7
./configure --prefix=/opt/apps
make install
# OpenMPI
wget https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.1.tar.gz
tar xvf openmpi-2.1.1.tar.gz && cd openmpi-2.1.1
./configure --prefix=/usr/local
make -j 16
make install
# APPS and SCRATCH
mkdir /apps
mkdir /scratch
cd /bin
rm sh && ln -s /bin/bash sh
%environment
source /opt/apps/lmod/7.7.7/init/profile
clearMT
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
export -f ml
export -f module
export -f clearMT
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment