Skip to content
Snippets Groups Projects
Commit 2766d2da authored by Lukáš Krupčík's avatar Lukáš Krupčík
Browse files

fix

parent 96b84681
Branches
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
.gitignore
+ 5
− 3
View file @ 2766d2da
*.img *.img
*.swp *.swp
*.simg *.simg
CentOS-7.5/ CentOS-7.5
CentOS/7.5-intel/ CentOS/7.5-intel
CentOS/7.5-VARROCentOS/7.5-VARROC/ CentOS/7.5-VARROC
CentOS/7.5-DHI
CentOS/7.5-ESPRESO/CentOS-7.5-ESPRESO.simg.backup
OpenFOAM/ OpenFOAM/
Ubuntu/16.04-OpenFOAM/ Ubuntu/16.04-OpenFOAM/
CentOS/7.5-ESPRESO/app/log_crawler.py 0 → 100644
+ 151
− 0
View file @ 2766d2da
import sys
import time
import os
import json
import logging
import shutil
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("Log crawler")
PROGRESS_HEAD = '''<html>
<head>
<title>HPC job progress</title>
<script type="text/javascript">
function abort() {
notify_running_job("ABORT");
}
</script>
</head>'''
PROGRESS_BODY='''<body style="margin: 20px; padding: 20px;">
<h1>HPC job progress</h1>
<div>
<h3>This is a waiter. It's waiting on the HPC cluster. It has waited {}/{} seconds.</h3>
<p>(Status from {}.)</p>
<p>Click the Abort button to stop the waiter early. Please click only once,
it will take a while until the request is processed. (The status might even
update once more before the job is aborted.)</p>
<input type="button" value="Abort" onclick="abort()">
</div>
</body>
</html>
'''
PROGRESS_BODY_UNKNOWN='''<body style="margin: 20px; padding: 20px;">
<h1>HPC job progress</h1>
<div>
<h3>This is a waiter. It's waiting on the HPC cluster. Its status is unknown.</h3>
<input type="button" value="Abort job" onclick="abort()">
</div>
</body>
</html>
'''
def main():
fn_log = sys.argv[1]
fn_status = sys.argv[2]
crawl_log(fn_log, fn_status)
def crawl_log(fn_log, fn_status):
"""Monitors a log file and creates html status pages from it
The log file is expected to grow only by one line at a time, with new lines
added to the end of the file. Each line is expected to be either a json
string containing the fields 'timestamp', 'elapsed_time', and 'total_time',
or the string 'FINISHED', which triggers the crawler to stop.
Note that this crawler is "memory-less", meaning that it cares only about
the last, newest line added to the log file while all prior lines are
disregarded.
Args:
fn_log (str): Filename of the log file to monitor
fn_status (str): Filename to write the status output to
"""
logger.info("Log crawler starting up")
cached_stamp = 0
should_stop = False
while not should_stop:
if not os.path.exists(fn_log):
logger.info("Log file doesn't exist")
else:
# Check if "last modified" time of the log file has changed
stamp = os.stat(fn_log).st_mtime
if not stamp != cached_stamp:
logger.info("Log file hasn't changed since last check")
else:
logger.info("Log file has changed")
cached_stamp = stamp
status = get_last_line(fn_log)
# We can stop crawling the logs if the main app is done
if status.strip() == "FINISHED":
should_stop = True
logger.info("Found FINISHED log, will stop now!")
# Otherwise, parse json and write status from it
else:
try:
log = json.loads(status)
write_status(log, fn_status)
except json.JSONDecodeError:
write_status_unknown(fn_status)
time.sleep(1)
logger.info("Log crawler terminating")
def get_last_line(fn):
"""Returns the last line of a file
Args:
fn (str): File name of the file to read from
"""
with open(fn, 'r') as fin:
for line in fin:
pass
return line
def write_status(log, fn_status):
"""Writes a status html page using the information in 'log'
Args:
log (dict): Log containing the field 'timestamp', 'elapsed_time', and
'total_time'
fn_status (str): Filename to write the status html to
"""
status = PROGRESS_HEAD + PROGRESS_BODY.format(
log["elapsed_time"],
log["total_time"],
log["timestamp"],
)
# First write to a temporary file, then copy to actual status file. That
# way, the status file won't be corrupt if the writing process takes a
# while.
fn_status_new = fn_status + '.new'
with open(fn_status_new, 'w') as fout:
fout.write(status)
shutil.copyfile(fn_status_new, fn_status)
logger.info("Wrote status file")
def write_status_unknown(fn_status):
"""Writes a status html page when no actual status is known
Args:
fn_status (str): Filename to write the status html to
"""
with open(fn_status, 'w') as fout:
fout.write(PROGRESS_HEAD + PROGRESS_BODY_UNKNOWN)
logger.info("Wrote unknown status")
if __name__ == "__main__":
main()
\ No newline at end of file
CentOS/7.5-ESPRESO/app/notifications_monitor.py 0 → 100644
+ 65
− 0
View file @ 2766d2da
import os
import time
import sys
import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("Notifications monitor")
def main():
fn_notifications = sys.argv[1]
fn_abort = sys.argv[2]
monitor(fn_notifications, fn_abort)
def monitor(fn_notifications, fn_abort):
"""
"""
logger.info("Notifications monitor starting up")
cached_stamp = 0
while True:
if not os.path.exists(fn_notifications):
logger.info("Notifications file doesn't exist")
else:
# Check if "last modified" time of the file has changed
stamp = os.stat(fn_notifications).st_mtime
if not stamp != cached_stamp:
logger.info("Notifications file hasn't changed since last check")
else:
logger.info("Notifications file has changed")
cached_stamp = stamp
command = get_last_line(fn_notifications).strip()
if command == "ABORT":
logger.info("Received abort command")
# Abort the main app by creating a specific file
open(fn_abort, 'a').close()
else:
logger.info("Ignoring unknown command {}".format(command))
time.sleep(1)
def get_last_line(fn):
"""Returns the last line of a file
Returns an empty string if the file is empty.
Args:
fn (str): File name of the file to read from
"""
with open(fn, 'r') as fin:
i = -1
for i, line in enumerate(fin):
pass
if i >= 0:
return line
else:
return ''
if __name__ == "__main__":
main()
\ No newline at end of file
CentOS/7.5-ESPRESO/app/startup.py 0 → 100644
+ 94
− 0
View file @ 2766d2da
import subprocess
import sys
import os
import string
import random
import shutil
import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("Startup")
def main():
# Input arguments
seconds_to_wait = int(sys.argv[1])
app_folder = sys.argv[2]
# Pre-defined files for communication with the "outside world"
fn_status = '/service/status.html'
fn_result = '/service/result.txt'
fn_notifications = '/service/notifications.txt'
# Write dummy value into result file
logger.info("Writing dummy result")
with open(fn_result, 'w') as fout:
fout.write("UNSET")
# Define files for the main app in a unique folder
ID = generate_id('/home')
unique_folder = os.path.join('/home', ID)
os.mkdir(unique_folder)
fn_log = os.path.join(unique_folder, 'main_log.txt')
fn_abort = os.path.join(unique_folder, 'main_abort.txt')
logger.info("Created unique folder {}".format(unique_folder))
# 1: Start log crawler as a background process
# The log crawler monitors a single log file and creates status html pages
# from it
# The log crawler will automatically terminate when it reads the finishing
# log from the main application
logger.info("Starting log crawler")
log_crawler_script = os.path.join(app_folder, 'log_crawler.py')
command = ['python', log_crawler_script, fn_log, fn_status]
proc_log = subprocess.Popen(command)
# 2: Start the notifications monitor as a background process
# It monitors the notifications file (which is written to when messages are
# sent to the HPC service running this Singularity image) and translates
# messages into comands to the main application. Here, this is done simply
# by creating a specific file the main app looks for. Only a single command
# (abort) is supported.
# The notifications monitor does _not_ terminate by itself.
logger.info("Starting notifications monitor")
not_monitor_script = os.path.join(app_folder, 'notifications_monitor.py')
command = ['python', not_monitor_script, fn_notifications, fn_abort]
proc_not = subprocess.Popen(command)
# 3: Start the main application as a foreground process
# The main application will write to the log file monitored by the crawler,
# and it will write its final result into the result file.
# It will also check periodically for the existence of 'fn_abort' and quit
# if this file exists. (This is a cheap way of sending a command to the
# main app.)
logger.info("Starting waiter")
waiter_script = os.path.join(app_folder, 'wait_a_while.py')
command = ['python', waiter_script, str(seconds_to_wait), fn_log,
fn_result, fn_abort]
subprocess.run(command)
logger.info("Waiter finished, terminating processes")
# 4: Terminate notifications monitor and remove log folders
# As the main app is finished at this point, we can safely terminate the
# notifications monitor
proc_log.terminate()
proc_not.terminate()
shutil.rmtree(unique_folder)
# (Optional) 5: Process end result
# In case the final result of the main application (see step 3) is _not_
# just a simple file, we should call another script here which processes
# this end result and writes into the /service/result.txt file.
def generate_id(folder, size=16, chars=string.ascii_uppercase + string.digits):
"""Generates a random ID string which guaranteed not to exist in a folder"""
while True:
ID = ''.join(random.choice(chars) for _ in range(size))
if not os.path.exists(os.path.join(folder, ID)):
break
return ID
if __name__ == "__main__":
main()
\ No newline at end of file
CentOS/7.5-ESPRESO/app/wait_a_while.py 0 → 100644
+ 64
− 0
View file @ 2766d2da
import sys
import time
import datetime
import json
import os
import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("Waiter")
def main():
seconds_to_wait = int(sys.argv[1])
fn_log = sys.argv[2]
fn_result = sys.argv[3]
fn_abort = sys.argv[4]
logger.info("Waiter starting up, will wait {} seconds".format(seconds_to_wait))
for current_time in range(seconds_to_wait):
# Perform part of "complicated computation"
time.sleep(1)
# Write log
write_log(fn_log, current_time, seconds_to_wait)
# Check if we should abort
if os.path.exists(fn_abort):
write_final_log(fn_log)
write_result(fn_result, current_time, aborted=True)
return 0
write_final_log(fn_log)
write_result(fn_result, seconds_to_wait)
def write_log(fn_log, elapsed_time, total_time):
log = {
'timestamp': str(datetime.datetime.now()),
'elapsed_time': str(elapsed_time),
'total_time': str(total_time),
}
with open(fn_log, 'a') as fout:
json.dump(log, fout)
fout.write("\n")
logger.info("Log written: {}/{} seconds waited".format(elapsed_time, total_time))
def write_final_log(fn_log):
with open(fn_log, 'a') as fout:
fout.write("FINISHED\n")
logger.info("Final log written")
def write_result(fn, seconds_waited, aborted=False):
with open(fn, 'w') as f:
if aborted:
f.write('Done. My waiting was aborted after {} seconds.'.format(seconds_waited))
else:
f.write('Done. I have waited {} seconds.'.format(seconds_waited))
logger.info("Result written")
if __name__ == "__main__":
main()
CentOS/7.5-ESPRESO/heat.cpp 0 → 100644
+ 145
− 0
View file @ 2766d2da
# ESPRESO Configuration File
#BENCHMARK ARG0 [ TETRA4, TETRA10, PYRAMID5, PYRAMID13, PRISMA6, PRISMA15, HEXA8, HEXA20 ]
#BENCHMARK ARG10 [ TOTAL_FETI, HYBRID_FETI ]
DEFAULT_ARGS {
0 HEXA8;
1 1;
2 1;
3 1;
4 2;
5 2;
6 2;
7 5;
8 5;
9 5;
10 TOTAL_FETI;
}
INPUT GENERATOR;
PHYSICS HEAT_TRANSFER_3D;
DECOMPOSITION {
BALANCE_CLUSTERS TRUE;
}
GENERATOR {
SHAPE GRID;
GRID {
LENGTH_X 1;
LENGTH_Y 1;
LENGTH_Z 1;
NODES {
BOTTOM <1 , 1> <0 , 1> <0 , 1>;
TOP <0 , 0> <0 , 1> <0 , 1>;
}
ELEMENT_TYPE [ARG0];
BLOCKS_X 1;
BLOCKS_Y 1;
BLOCKS_Z 1;
CLUSTERS_X [ARG1];
CLUSTERS_Y [ARG2];
CLUSTERS_Z [ARG3];
DOMAINS_X [ARG4];
DOMAINS_Y [ARG5];
DOMAINS_Z [ARG6];
ELEMENTS_X [ARG7];
ELEMENTS_Y [ARG8];
ELEMENTS_Z [ARG9];
}
}
HEAT_TRANSFER_3D {
LOAD_STEPS 1;
MATERIALS {
1 {
COORDINATE_SYSTEM {
TYPE CYLINDRICAL;
CENTER { X .5; Y .5; Z .5; }
ROTATION { X 90; }
}
DENS 1;
CP 1;
THERMAL_CONDUCTIVITY {
MODEL DIAGONAL;
KXX 1;
KYY 10;
KZZ 10;
}
}
}
MATERIAL_SET {
ALL_ELEMENTS 1;
}
INITIAL_TEMPERATURE {
ALL_ELEMENTS 200;
}
STABILIZATION CAU;
SIGMA 0;
LOAD_STEPS_SETTINGS {
1 {
DURATION_TIME 1;
TYPE STEADY_STATE;
MODE LINEAR;
SOLVER FETI;
FETI {
METHOD [ARG10];
PRECONDITIONER DIRICHLET;
PRECISION 1E-08;
ITERATIVE_SOLVER PCG;
REGULARIZATION ANALYTIC;
}
TEMPERATURE {
TOP 100;
BOTTOM 300;
}
}
}
}
OUTPUT {
RESULTS_STORE_FREQUENCY EVERY_TIMESTEP;
MONITORS_STORE_FREQUENCY EVERY_TIMESTEP;
MONITORING {
1 {
REGION TOP;
STATISTICS MAX;
PROPERTY TEMPERATURE;
}
2 {
REGION BOTTOM;
STATISTICS MIN;
PROPERTY TEMPERATURE;
}
5 {
REGION ALL_ELEMENTS;
STATISTICS AVG;
PROPERTY TEMPERATURE;
}
}
}
bin/1.0.lua 0 → 100644
+ 21
− 0
View file @ 2766d2da
help([[
Description
===========
Wrapers for Singularity containers.
More information
================
- Homepage: https://docs.it4i.cz
]])
whatis([[Description: Wrapers for Singularity containers (mailto:easybuild@vsb.cz).]])
whatis([[Homepage: https://docs.it4i.cz]])
local root = "/apps/all/OS/"
conflict("SingularityWrapers")
add_property("state", "singularity")
prepend_path("PATH", pathJoin(root, "bin"))
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment