Skip to content
Snippets Groups Projects
Commit 20f7ca03 authored by David Hrbáč's avatar David Hrbáč
Browse files

Merge branch 'py' into 'master' 

Update python requirements

See merge request !276
parents 40a61078 0596399a
No related branches found
No related tags found
5 merge requests!368Update prace.md to document the change from qprace to qprod as the default...,!367Update prace.md to document the change from qprace to qprod as the default...,!366Update prace.md to document the change from qprace to qprod as the default...,!323extended-acls-storage-section,!276Update python requirements
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 22
− 6
View file @ 20f7ca03
......@@ -14,11 +14,23 @@ docs:
script:
- mdl -r ~MD013,~MD010,~MD014,~MD024,~MD026,~MD029,~MD033,~MD036,~MD037,~MD046 *.md docs.it4i # BUGS
pylint:
stage: test
image: it4innovations/docker-pycheck:latest
script:
- pylint $(find . -name "*.py" -not -name "feslicescript.py")
pysafety:
stage: test
image: it4innovations/docker-pycheck:latest
script:
- cat requirements.txt | safety check --stdin --full-report
capitalize:
stage: test
image: it4innovations/docker-mkdocscheck:latest
script:
- find mkdocs.yml docs.it4i/ \( -name '*.md' -o -name '*.yml' \) -print0 | xargs -0 -n1 scripts/titlemd_test.py
- find mkdocs.yml docs.it4i/ \( -name '*.md' -o -name '*.yml' \) -print0 | xargs -0 -n1 scripts/titlemd.py --test
ext_links:
stage: after_test
......@@ -45,10 +57,10 @@ ext_links:
mkdocs:
stage: build
image: it4innovations/docker-mkdocscheck:latest
before_script:
- python -V # Print out python version for debugging
script:
- mkdocs -V
- python -V # debug
- pip freeze # debug
- mkdocs -V # debug
# add version to footer
- bash scripts/add_version.sh
# get modules list from clusters
......@@ -58,8 +70,12 @@ mkdocs:
# generate ULT for code link
- sed -i "s/master/$CI_BUILD_REF_NAME/g" material/partials/toc.html
# regenerate modules matrix
- python scripts/modules-matrix.py > docs.it4i/modules-matrix.md
- python scripts/modules-json.py > docs.it4i/modules-matrix.json
- source /opt/.venv3/bin/activate
- python -V # debug
- pip freeze # debug
- python scripts/modules_matrix.py > docs.it4i/modules-matrix.md
- python scripts/modules_matrix.py --json > docs.it4i/modules-matrix.json
- declare -Ff deactivate && deactivate
- curl -f0 https://code.it4i.cz/sccs/scs-api-public/raw/master/scs_api.server_public.md -o docs.it4i/apiv1.md
# build pages
- mkdocs build
......
requirements.txt
+ 12
− 11
View file @ 20f7ca03
backports-abc==0.5
Click==7.0
Click==7.1.1
EditorConfig==0.12.2
futures==3.2.0
Jinja2==2.10
jsbeautifier==1.9.0
livereload==2.6.0
Markdown==3.0.1
futures==3.3.0
Jinja2==2.11.1
jsbeautifier==1.10.3
livereload==2.6.1
Markdown==3.1.1
MarkupSafe==1.1.1
mkdocs==1.0.4
mkdocs-material==4.0.1
Pygments==2.3.1
pymdown-extensions==6.0
PyYAML==3.13
mkdocs-material==4.6.0
packaging==20.3
Pygments==2.5.2
pymdown-extensions==6.2.1
PyYAML==5.3.1
singledispatch==3.4.0.3
six==1.12.0
six==1.14.0
tornado==5.1.1
scripts/combinations.py
+ 10
− 5
View file @ 20f7ca03
#!/usr/bin/python
# -*- coding: utf-8 -*-
""" combinations """
import itertools
import re
l = ['A', 'S', 'U', 'T', 'D']
mask = ''.join(reversed(l))
for i in range(1,len(l)+1):
for comb in itertools.combinations(l, i):
CHARS = ['A', 'S', 'U', 'T', 'D']
MASK = ''.join(reversed(CHARS))
for i in range(1, len(CHARS)+1):
for comb in itertools.combinations(CHARS, i):
reg = "[^%s]" % ''.join(comb)
print re.sub(reg,"-", mask)
print re.sub(reg, "-", MASK)
scripts/matrix.py
+ 7
− 10
View file @ 20f7ca03
#!/usr/bin/python
# -*- coding: utf-8 -*-
import csv
import collections
""" matrix """
import itertools
import re
from distutils.version import LooseVersion
l = ['A', 'S', 'U', 'P', 'D', 'B']
mask = ''.join(reversed(l))
from itertools import product
for bits in product([0, 1], repeat=len(l)):
CHARS = ['A', 'S', 'U', 'P', 'D', 'B']
MASK = ''.join(reversed(CHARS))
for bits in itertools.product([0, 1], repeat=len(CHARS)):
s = "".join(str(bit) for bit in bits)
ns = ""
for i in range(len(s)):
for i, _ in enumerate(s):
if s[i] == "1":
ns += mask[i]
ns += MASK[i]
else:
ns += "-"
print ns
scripts/modules-json.py deleted 100755 → 0
+ 0
− 73
View file @ 40a61078
#!/usr/bin/python
# -*- coding: utf-8 -*-
import csv
import collections
import json
import itertools
import re
from distutils.version import LooseVersion
def get_data(filename):
'''function to read the data form the input csv file to use in the analysis'''
reader = [] # Just in case the file open fails
with open(filename, 'rb') as f:
reader = csv.reader(f, delimiter=',')
# returns all the data from the csv file in list form
# f.close() # May need to close the file when done
return list(reader) # only return the reader when you have finished.
your_list = []
your_list += get_data('./scripts/anselm.csv')
your_list += get_data('./scripts/salomon.csv')
your_list += get_data('./scripts/uv2000.csv')
your_list += get_data('./scripts/phi.csv')
your_list += get_data('./scripts/dgx.csv')
your_list += get_data('./scripts/barbora.csv')
counts = dict()
for i in your_list:
counts[i[0]] = counts.get(i[0], 0) + int(i[1])
# 1 2 4 8 16 32
l = ['A', 'S', 'U', 'P', 'D', 'B']
c = []
mask = ''.join(reversed(l))
from itertools import product
for bits in product([0, 1], repeat=len(l)):
s = "".join(str(bit) for bit in bits)
ns = ""
for i in range(len(s)):
if s[i] == "1":
ns += mask[i]
else:
ns += "-"
c.append(ns)
software = dict()
versions = ''
clusters = ''
prev = ''
for m, i in sorted(counts.items()):
# print m
split = m.split('/')
# print split
if len(split) > 1:
a = split[0]
b = split[1]
if split[0] <> prev:
software[a] = {}
software[a][b] = '`' + c[i] + '`'
prev = a
packages = {}
for m in sorted(software.items(), key=lambda i: i[0].lower()):
packages[m[0]] = sorted(m[1], key=LooseVersion)[len(m[1]) - 1]
data = {'total': len(packages), 'projects': packages}
print json.dumps(data)
scripts/modules-matrix.py deleted 100755 → 0
+ 0
− 94
View file @ 40a61078
#!/usr/bin/python
# -*- coding: utf-8 -*-
import csv
import collections
import itertools
import re
from distutils.version import LooseVersion
def get_data(filename):
'''function to read the data form the input csv file to use in the analysis'''
reader = [] # Just in case the file open fails
with open(filename, 'rb') as f:
reader = csv.reader(f, delimiter=',')
# returns all the data from the csv file in list form
# f.close() # May need to close the file when done
return list(reader) # only return the reader when you have finished.
your_list = []
your_list += get_data('./scripts/anselm.csv')
your_list += get_data('./scripts/salomon.csv')
your_list += get_data('./scripts/uv2000.csv')
your_list += get_data('./scripts/phi.csv')
your_list += get_data('./scripts/dgx.csv')
your_list += get_data('./scripts/barbora.csv')
print your_list
counts = dict()
for i in your_list:
# print i[0]
# print int(i[1])
counts[i[0]] = counts.get(i[0], 0) + int(i[1])
# 1 2 4 8 16 32
l = ['A', 'S', 'U', 'P', 'D', 'B']
c = []
mask = ''.join(reversed(l))
from itertools import product
for bits in product([0, 1], repeat=len(l)):
s = "".join(str(bit) for bit in bits)
ns = ""
for i in range(len(s)):
if s[i] == "1":
ns += mask[i]
else:
ns += "-"
c.append(ns)
print '!!! Hint "Cluster Acronyms"'
print ' ```'
print ' B D P U S A'
print ' | | | | | |'
print ' | | | | | +----> Anselm'
print ' | | | | +------> Salomon'
print ' | | | +--------> UV2000'
print ' | | +----------> Phi'
print ' | +------------> DGX-2'
print ' +--------------> Barbora'
print ' ```'
print
print '| Module </br><form><input id="searchInput" placeholder="🔍 Filter" style="width: 8rem; border-radius: 0.2rem; color: black; padding-left: .2rem;"></form> | Versions | Clusters |'
print "| ------ | -------- | -------- |"
software = dict()
versions = ''
clusters = ''
prev = ''
for m, i in sorted(counts.items()):
# print m
split = m.split('/')
# print split
if len(split) > 1:
a = split[0]
b = split[1]
if split[0] <> prev:
software[a] = {}
software[a][b] = '`' + c[i] + '`'
prev = a
for m in sorted(software.items(), key=lambda i: i[0].lower()):
software = m[0]
versions = []
clusters = []
for key in sorted(m[1], key=LooseVersion):
versions.append(key)
clusters.append(m[1][key])
print "| %s | %s | %s |" % (software, '</br>'.join(versions), '</br>'.join(clusters))
print
print '---8<--- "modules_matrix_search.md"'
scripts/modules_matrix.py 0 → 100755
+ 141
− 0
View file @ 20f7ca03
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
""" modules_matrix """
from __future__ import print_function
import argparse
import csv
import itertools
import json
import os.path
import packaging.specifiers
def arg_parse():
"""
argument parser
"""
parser = argparse.ArgumentParser(
description="Module_matrix"
)
parser.add_argument('--json',
action='store_true',
help="get json")
return parser.parse_args()
def get_data(filename):
'''function to read the data form the input csv file to use in the analysis'''
reader = [] # Just in case the file open fails
with open(filename, 'r') as fdata:
reader = csv.reader(fdata, delimiter=',')
# returns all the data from the csv file in list form
# print(list(reader))
return list(reader) # only return the reader when you have finished.
def get_datalist():
""" get_datalist """
datalist = []
for name in ['anselm', 'salomon', 'uv2000', 'phi', 'dgx', 'barbora']:
path = os.path.join('scripts', "{}.csv".format(name))
datalist += get_data(path)
return datalist
def get_counts(datalist):
""" get_counts """
counts = dict()
for i in datalist:
counts[i[0]] = counts.get(i[0], 0) + int(i[1])
return counts
def get_matrix():
""" get_matrix """
# 1 2 4 8 16 32
chars = ['A', 'S', 'U', 'P', 'D', 'B']
arr = []
mask = ''.join(reversed(chars))
for bits in itertools.product([0, 1], repeat=len(chars)):
sbit = "".join(str(bit) for bit in bits)
nst = ""
for i, _ in enumerate(sbit):
if sbit[i] == "1":
nst += mask[i]
else:
nst += "-"
arr.append(nst)
return arr
def get_software(datalist):
""" get_software """
matrix = get_matrix()
counts = get_counts(datalist)
software = dict()
prev = ''
for mat, i in sorted(counts.items()):
split = mat.split('/')
if len(split) > 1:
if split[0] != prev:
software[split[0]] = {}
software[split[0]][split[1]] = '`' + matrix[i] + '`'
prev = split[0]
return software
def packages_json(software):
""" packages_json """
packages = {}
for module in sorted(software.items(), key=lambda i: i[0].lower()):
packages[module[0]] = sorted(module[1],
key=packaging.specifiers.LegacyVersion)[len(module[1]) - 1]
data = {'total': len(packages), 'projects': packages}
return data
def print_software(software):
""" print_software """
versions = ''
clusters = ''
for module in sorted(software.items(), key=lambda i: i[0].lower()):
software = module[0]
versions = []
clusters = []
for key in sorted(module[1].keys(), key=packaging.specifiers.LegacyVersion):
versions.append(key)
clusters.append(module[1][key])
print("| {} | {} | {} |".format(software, '</br>'.join(versions), '</br>'.join(clusters)))
print()
print('---8<--- "modules_matrix_search.md"')
def print_hint():
""" print_hint """
print('!!! Hint "Cluster Acronyms"')
print(' ```')
print(' B D P U S A')
print(' | | | | | |')
print(' | | | | | +----> Anselm')
print(' | | | | +------> Salomon')
print(' | | | +--------> UV2000')
print(' | | +----------> Phi')
print(' | +------------> DGX-2')
print(' +--------------> Barbora')
print(' ```')
print()
print("{} {} {}".format('| Module </br><form><input id="searchInput" placeholder="🔍 Filter"',
'style="width: 8rem; border-radius: 0.2rem; color: black;',
'padding-left: .2rem;"></form> | Versions | Clusters |'))
print("| ------ | -------- | -------- |")
def main():
"""
main function
"""
arg = arg_parse()
datalist = get_datalist()
software = get_software(datalist)
if arg.json:
print(json.dumps(packages_json(software)))
else:
print(datalist)
print_hint()
print_software(software)
if __name__ == "__main__":
main()
scripts/titlemd.py
+ 153
− 41
View file @ 20f7ca03
#!/usr/bin/python
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
""" titlemd """
import fnmatch
import os
from __future__ import print_function
import argparse
import sys
try:
from titlecase import titlecase
from titlecase import titlecase
except ImportError:
print("Please install titlecase")
def main(location):
# Spelling exceptions
with open('.spelling') as f:
spelling = f.readlines()
def abbreviations(word, **kwargs):
if word+"\n" in spelling:
return word
# Open the file and read the lines as a list
with open(location) as f:
lines = f.readlines()
with open(location, 'w') as f:
# Loop through the list of lines and titlecase
# any line beginning with '#'.
prev_line = lines.pop(0)
disabled = 0
for line in lines:
if line.startswith("``") and disabled == 0:
disabled = 1
else:
if line.startswith("``") and disabled == 1:
disabled = 0
if line.startswith('#') and disabled == 0:
line = titlecase(line[:(line.find("]"))], callback=abbreviations)+line[(line.find("]")):]
if (line.startswith('---') or line.startswith('===')) and disabled == 0:
prev_line = titlecase(prev_line[:(prev_line.find("]"))], callback=abbreviations)+prev_line[(prev_line.find("]")):]
f.write(prev_line)
prev_line = line
f.write(prev_line)
print("Please install titlecase")
def arg_parse():
"""
argument parser
"""
parser = argparse.ArgumentParser(
description="Titlemd"
)
parser.add_argument('-t', '--test',
action='store_true',
help="test")
parser.add_argument('location',
nargs='?',
default='.',
help="location, default current directory")
return parser.parse_args()
def mkdocs_available(location):
""" Is mkdocs.yml available? """
if location.find("mkdocs.yml") != -1:
return True
return False
def linestart(line, disabled, test, prev_line=None):
""" linestart """
if test:
if (line.startswith("``") or line.startswith("extra:")) and not disabled:
return True
else:
if (line.startswith("``") or prev_line.startswith("pages:")) and disabled:
return False
else:
if line.startswith("``") and not disabled:
return True
else:
if line.startswith("``") and disabled:
return False
return disabled
def testdata(arg):
""" test """
# Spelling exceptions
with open('.spelling') as fdata:
spelling = fdata.readlines()
# pylint: disable=unused-argument,inconsistent-return-statements
def abbreviations(word, **kwargs):
""" abbreviations """
if word+"\n" in spelling:
return word
# Open the file and read the lines as a list
with open(arg.location) as fdata:
lines = fdata.readlines()
# Loop through the list of lines and titlecase
# any line beginning with '#'.
return_value = 0
prev_line = lines[0]
echo_filename = False
disabled = mkdocs_available(arg.location)
for line in lines:
disabled = linestart(line, disabled, arg.test, prev_line)
if line.startswith('#') and not disabled and not mkdocs_available(arg.location):
if line != titlecase(line[:(line.find("]"))],
callback=abbreviations)+line[(line.find("]")):]:
if return_value == 0 and not echo_filename:
print("%s" % arg.location)
echo_filename = True
print("-"+line, end="")
print("+"+titlecase(line[:(line.find("]"))],
callback=abbreviations)+line[(line.find("]")):], end="")
print()
return_value = 1
if (line.startswith('---') or line.startswith('===')) and not disabled:
if prev_line != titlecase(prev_line[:(prev_line.find("]"))],
callback=abbreviations)+prev_line[(prev_line.find("]")):]:
if return_value == 0 and not echo_filename:
print("%s" % arg.location)
echo_filename = True
print("-"+prev_line, end="")
print("+"+titlecase(prev_line[:(prev_line.find("]"))],
callback=abbreviations)+prev_line[(prev_line.find("]")):],
end="")
print()
return_value = 1
if (mkdocs_available(arg.location) and not line.startswith('#') and not disabled):
if line != titlecase(line[:(line.find(":"))],
callback=abbreviations)+line[(line.find(":")):]:
if return_value == 0 and not echo_filename:
print("%s" % arg.location)
echo_filename = True
print("-"+line, end="")
print("+"+titlecase(line[:(line.find(":"))],
callback=abbreviations)+line[(line.find(":")):], end="")
print()
return_value = 1
prev_line = line
return return_value
def writedata(arg):
""" writedata """
# Spelling exceptions
with open('.spelling') as fdata:
spelling = fdata.readlines()
# pylint: disable=unused-argument,inconsistent-return-statements
def abbreviations(word, **kwargs):
""" abbreviations """
if word+"\n" in spelling:
return word
# Open the file and read the lines as a list
with open(arg.location) as fdata:
lines = fdata.readlines()
with open(arg.location, 'w') as fdata:
# Loop through the list of lines and titlecase
# any line beginning with '#'.
prev_line = lines[0]
disabled = False
for line in lines:
disabled = linestart(line, disabled, arg.test)
if line.startswith('#') and not disabled:
line = titlecase(line[:(line.find("]"))],
callback=abbreviations)+line[(line.find("]")):]
if (line.startswith('---') or line.startswith('===')) and not disabled:
prev_line = titlecase(prev_line[:(prev_line.find("]"))],
callback=abbreviations)+prev_line[(prev_line.find("]")):]
fdata.write(prev_line)
prev_line = line
fdata.write(prev_line)
def main():
"""
main function
"""
arg = arg_parse()
if arg.test:
sys.exit(testdata(arg))
else:
writedata(arg)
if __name__ == "__main__":
try:
main(sys.argv[1])
except IndexError:
main('.')
main()
scripts/titlemd_test.py deleted 100755 → 0
+ 0
− 73
View file @ 40a61078
#!/usr/bin/env python
from __future__ import print_function
import fnmatch
import os
import sys
try:
from titlecase import titlecase
except ImportError:
print("Please install titlecase")
def main(location):
# Spelling exceptions
with open('.spelling') as f:
spelling = f.readlines()
def abbreviations(word, **kwargs):
if word+"\n" in spelling:
return word
# Open the file and read the lines as a list
with open(location) as f:
lines = f.readlines()
# Loop through the list of lines and titlecase
# any line beginning with '#'.
return_value = 0
prev_line = lines[0]
disabled = 0
echo_filename = False
if location.find("mkdocs.yml") != -1:
disabled = 1
for line in lines:
if (line.startswith("``") or line.startswith("extra:")) and disabled == 0:
disabled = 1
else:
if (line.startswith("``") or prev_line.startswith("pages:")) and disabled == 1:
disabled = 0
if line.startswith('#') and (disabled == 0) and (location.find("mkdocs.yml") == -1):
if line != titlecase(line[:(line.find("]"))], callback=abbreviations)+line[(line.find("]")):]:
if return_value == 0 and echo_filename == False:
print("%s" % location)
echo_filename = True
print("-"+line,end="")
print("+"+titlecase(line[:(line.find("]"))], callback=abbreviations)+line[(line.find("]")):],end="")
print()
return_value = 1
if (line.startswith('---') or line.startswith('===')) and disabled == 0:
if prev_line != titlecase(prev_line[:(prev_line.find("]"))], callback=abbreviations)+prev_line[(prev_line.find("]")):]:
if return_value == 0 and echo_filename == False:
print("%s" % location)
echo_filename = True
print("-"+prev_line,end="")
print("+"+titlecase(prev_line[:(prev_line.find("]"))], callback=abbreviations)+prev_line[(prev_line.find("]")):],end="")
print()
return_value = 1
if ((location.find("mkdocs.yml") != -1) and not line.startswith('#') and disabled == 0):
if line != titlecase(line[:(line.find(":"))], callback=abbreviations)+line[(line.find(":")):]:
if return_value == 0 and echo_filename == False:
print("%s" % location)
echo_filename = True
print("-"+line,end="")
print("+"+titlecase(line[:(line.find(":"))], callback=abbreviations)+line[(line.find(":")):],end="")
print()
return_value = 1
prev_line = line
exit(return_value)
if __name__ == "__main__":
try:
main(sys.argv[1])
except IndexError:
main('.')
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment