diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 51d8fa2e89e6d53325b4ec99d7d2b78849a51628..d72ee28f9054dc4e29a0e072af036d9c21bfb20c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,11 +14,23 @@ docs: script: - mdl -r ~MD013,~MD010,~MD014,~MD024,~MD026,~MD029,~MD033,~MD036,~MD037,~MD046 *.md docs.it4i # BUGS +pylint: + stage: test + image: it4innovations/docker-pycheck:latest + script: + - pylint $(find . -name "*.py" -not -name "feslicescript.py") + +pysafety: + stage: test + image: it4innovations/docker-pycheck:latest + script: + - cat requirements.txt | safety check --stdin --full-report + capitalize: stage: test image: it4innovations/docker-mkdocscheck:latest script: - - find mkdocs.yml docs.it4i/ \( -name '*.md' -o -name '*.yml' \) -print0 | xargs -0 -n1 scripts/titlemd_test.py + - find mkdocs.yml docs.it4i/ \( -name '*.md' -o -name '*.yml' \) -print0 | xargs -0 -n1 scripts/titlemd.py --test ext_links: stage: after_test @@ -45,10 +57,10 @@ ext_links: mkdocs: stage: build image: it4innovations/docker-mkdocscheck:latest - before_script: - - python -V # Print out python version for debugging script: - - mkdocs -V + - python -V # debug + - pip freeze # debug + - mkdocs -V # debug # add version to footer - bash scripts/add_version.sh # get modules list from clusters @@ -58,8 +70,12 @@ mkdocs: # generate ULT for code link - sed -i "s/master/$CI_BUILD_REF_NAME/g" material/partials/toc.html # regenerate modules matrix - - python scripts/modules-matrix.py > docs.it4i/modules-matrix.md - - python scripts/modules-json.py > docs.it4i/modules-matrix.json + - source /opt/.venv3/bin/activate + - python -V # debug + - pip freeze # debug + - python scripts/modules_matrix.py > docs.it4i/modules-matrix.md + - python scripts/modules_matrix.py --json > docs.it4i/modules-matrix.json + - declare -Ff deactivate && deactivate - curl -f0 https://code.it4i.cz/sccs/scs-api-public/raw/master/scs_api.server_public.md -o docs.it4i/apiv1.md # build pages - mkdocs build diff --git a/requirements.txt b/requirements.txt index e341139126ee854088483283131e0c37b2cc25ce..9e09158e9087bf53b9c0d3f154daae1b973e8643 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,18 @@ backports-abc==0.5 -Click==7.0 +Click==7.1.1 EditorConfig==0.12.2 -futures==3.2.0 -Jinja2==2.10 -jsbeautifier==1.9.0 -livereload==2.6.0 -Markdown==3.0.1 +futures==3.3.0 +Jinja2==2.11.1 +jsbeautifier==1.10.3 +livereload==2.6.1 +Markdown==3.1.1 MarkupSafe==1.1.1 mkdocs==1.0.4 -mkdocs-material==4.0.1 -Pygments==2.3.1 -pymdown-extensions==6.0 -PyYAML==3.13 +mkdocs-material==4.6.0 +packaging==20.3 +Pygments==2.5.2 +pymdown-extensions==6.2.1 +PyYAML==5.3.1 singledispatch==3.4.0.3 -six==1.12.0 +six==1.14.0 tornado==5.1.1 diff --git a/scripts/combinations.py b/scripts/combinations.py index 067f39f0cb9b1ed19c8f5ad831fc175e3bb44fa8..67cf39b0df3854f20807cd44493049a35666ca21 100644 --- a/scripts/combinations.py +++ b/scripts/combinations.py @@ -1,8 +1,13 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" combinations """ + import itertools import re -l = ['A', 'S', 'U', 'T', 'D'] -mask = ''.join(reversed(l)) -for i in range(1,len(l)+1): - for comb in itertools.combinations(l, i): + +CHARS = ['A', 'S', 'U', 'T', 'D'] +MASK = ''.join(reversed(CHARS)) +for i in range(1, len(CHARS)+1): + for comb in itertools.combinations(CHARS, i): reg = "[^%s]" % ''.join(comb) - print re.sub(reg,"-", mask) + print re.sub(reg, "-", MASK) diff --git a/scripts/matrix.py b/scripts/matrix.py index 33a18175af409b8d4be31742f1beedd62188c641..a285cabbd5a0507bc5bafbfc3084eb6e5ca757a7 100644 --- a/scripts/matrix.py +++ b/scripts/matrix.py @@ -1,20 +1,17 @@ #!/usr/bin/python # -*- coding: utf-8 -*- -import csv -import collections +""" matrix """ + import itertools -import re -from distutils.version import LooseVersion -l = ['A', 'S', 'U', 'P', 'D', 'B'] -mask = ''.join(reversed(l)) -from itertools import product -for bits in product([0, 1], repeat=len(l)): +CHARS = ['A', 'S', 'U', 'P', 'D', 'B'] +MASK = ''.join(reversed(CHARS)) +for bits in itertools.product([0, 1], repeat=len(CHARS)): s = "".join(str(bit) for bit in bits) ns = "" - for i in range(len(s)): + for i, _ in enumerate(s): if s[i] == "1": - ns += mask[i] + ns += MASK[i] else: ns += "-" print ns diff --git a/scripts/modules-json.py b/scripts/modules-json.py deleted file mode 100755 index e394df2385f313f4f088274bda1859082170cb73..0000000000000000000000000000000000000000 --- a/scripts/modules-json.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -import csv -import collections -import json -import itertools -import re - -from distutils.version import LooseVersion - - -def get_data(filename): - '''function to read the data form the input csv file to use in the analysis''' - reader = [] # Just in case the file open fails - with open(filename, 'rb') as f: - reader = csv.reader(f, delimiter=',') - # returns all the data from the csv file in list form - # f.close() # May need to close the file when done - return list(reader) # only return the reader when you have finished. - -your_list = [] -your_list += get_data('./scripts/anselm.csv') -your_list += get_data('./scripts/salomon.csv') -your_list += get_data('./scripts/uv2000.csv') -your_list += get_data('./scripts/phi.csv') -your_list += get_data('./scripts/dgx.csv') -your_list += get_data('./scripts/barbora.csv') - -counts = dict() -for i in your_list: - counts[i[0]] = counts.get(i[0], 0) + int(i[1]) - -# 1 2 4 8 16 32 -l = ['A', 'S', 'U', 'P', 'D', 'B'] -c = [] -mask = ''.join(reversed(l)) -from itertools import product -for bits in product([0, 1], repeat=len(l)): - s = "".join(str(bit) for bit in bits) - ns = "" - for i in range(len(s)): - if s[i] == "1": - ns += mask[i] - else: - ns += "-" - c.append(ns) - -software = dict() -versions = '' -clusters = '' -prev = '' - -for m, i in sorted(counts.items()): - # print m - split = m.split('/') - # print split - if len(split) > 1: - a = split[0] - b = split[1] - if split[0] <> prev: - software[a] = {} - software[a][b] = '`' + c[i] + '`' - prev = a - - -packages = {} - -for m in sorted(software.items(), key=lambda i: i[0].lower()): - packages[m[0]] = sorted(m[1], key=LooseVersion)[len(m[1]) - 1] - -data = {'total': len(packages), 'projects': packages} -print json.dumps(data) diff --git a/scripts/modules-matrix.py b/scripts/modules-matrix.py deleted file mode 100755 index c66f8723a85573d405c1ed51c9ace3058efdcfd6..0000000000000000000000000000000000000000 --- a/scripts/modules-matrix.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- - -import csv -import collections -import itertools -import re - -from distutils.version import LooseVersion - - -def get_data(filename): - '''function to read the data form the input csv file to use in the analysis''' - reader = [] # Just in case the file open fails - with open(filename, 'rb') as f: - reader = csv.reader(f, delimiter=',') - # returns all the data from the csv file in list form - # f.close() # May need to close the file when done - return list(reader) # only return the reader when you have finished. - -your_list = [] -your_list += get_data('./scripts/anselm.csv') -your_list += get_data('./scripts/salomon.csv') -your_list += get_data('./scripts/uv2000.csv') -your_list += get_data('./scripts/phi.csv') -your_list += get_data('./scripts/dgx.csv') -your_list += get_data('./scripts/barbora.csv') - -print your_list - -counts = dict() -for i in your_list: - # print i[0] - # print int(i[1]) - counts[i[0]] = counts.get(i[0], 0) + int(i[1]) - -# 1 2 4 8 16 32 -l = ['A', 'S', 'U', 'P', 'D', 'B'] -c = [] -mask = ''.join(reversed(l)) -from itertools import product -for bits in product([0, 1], repeat=len(l)): - s = "".join(str(bit) for bit in bits) - ns = "" - for i in range(len(s)): - if s[i] == "1": - ns += mask[i] - else: - ns += "-" - c.append(ns) - -print '!!! Hint "Cluster Acronyms"' -print ' ```' -print ' B D P U S A' -print ' | | | | | |' -print ' | | | | | +----> Anselm' -print ' | | | | +------> Salomon' -print ' | | | +--------> UV2000' -print ' | | +----------> Phi' -print ' | +------------> DGX-2' -print ' +--------------> Barbora' -print ' ```' -print -print '| Module </br><form><input id="searchInput" placeholder="🔍 Filter" style="width: 8rem; border-radius: 0.2rem; color: black; padding-left: .2rem;"></form> | Versions | Clusters |' -print "| ------ | -------- | -------- |" - -software = dict() -versions = '' -clusters = '' -prev = '' - -for m, i in sorted(counts.items()): - # print m - split = m.split('/') -# print split - if len(split) > 1: - a = split[0] - b = split[1] - if split[0] <> prev: - software[a] = {} - software[a][b] = '`' + c[i] + '`' - prev = a - -for m in sorted(software.items(), key=lambda i: i[0].lower()): - software = m[0] - versions = [] - clusters = [] - for key in sorted(m[1], key=LooseVersion): - versions.append(key) - clusters.append(m[1][key]) - print "| %s | %s | %s |" % (software, '</br>'.join(versions), '</br>'.join(clusters)) - -print -print '---8<--- "modules_matrix_search.md"' diff --git a/scripts/modules_matrix.py b/scripts/modules_matrix.py new file mode 100755 index 0000000000000000000000000000000000000000..a2e3acb5e0d4ca10b08a23fe08ad7b451df785eb --- /dev/null +++ b/scripts/modules_matrix.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- +""" modules_matrix """ + +from __future__ import print_function + +import argparse +import csv +import itertools +import json +import os.path +import packaging.specifiers + +def arg_parse(): + """ + argument parser + """ + parser = argparse.ArgumentParser( + description="Module_matrix" + ) + parser.add_argument('--json', + action='store_true', + help="get json") + return parser.parse_args() + +def get_data(filename): + '''function to read the data form the input csv file to use in the analysis''' + reader = [] # Just in case the file open fails + with open(filename, 'r') as fdata: + reader = csv.reader(fdata, delimiter=',') + # returns all the data from the csv file in list form +# print(list(reader)) + return list(reader) # only return the reader when you have finished. + +def get_datalist(): + """ get_datalist """ + datalist = [] + for name in ['anselm', 'salomon', 'uv2000', 'phi', 'dgx', 'barbora']: + path = os.path.join('scripts', "{}.csv".format(name)) + datalist += get_data(path) + return datalist + +def get_counts(datalist): + """ get_counts """ + counts = dict() + for i in datalist: + counts[i[0]] = counts.get(i[0], 0) + int(i[1]) + return counts + +def get_matrix(): + """ get_matrix """ + # 1 2 4 8 16 32 + chars = ['A', 'S', 'U', 'P', 'D', 'B'] + arr = [] + mask = ''.join(reversed(chars)) + for bits in itertools.product([0, 1], repeat=len(chars)): + sbit = "".join(str(bit) for bit in bits) + nst = "" + for i, _ in enumerate(sbit): + if sbit[i] == "1": + nst += mask[i] + else: + nst += "-" + arr.append(nst) + return arr + +def get_software(datalist): + """ get_software """ + matrix = get_matrix() + counts = get_counts(datalist) + software = dict() + prev = '' + for mat, i in sorted(counts.items()): + split = mat.split('/') + if len(split) > 1: + if split[0] != prev: + software[split[0]] = {} + software[split[0]][split[1]] = '`' + matrix[i] + '`' + prev = split[0] + return software + +def packages_json(software): + """ packages_json """ + packages = {} + for module in sorted(software.items(), key=lambda i: i[0].lower()): + packages[module[0]] = sorted(module[1], + key=packaging.specifiers.LegacyVersion)[len(module[1]) - 1] + data = {'total': len(packages), 'projects': packages} + return data + +def print_software(software): + """ print_software """ + versions = '' + clusters = '' + for module in sorted(software.items(), key=lambda i: i[0].lower()): + software = module[0] + versions = [] + clusters = [] + for key in sorted(module[1].keys(), key=packaging.specifiers.LegacyVersion): + versions.append(key) + clusters.append(module[1][key]) + print("| {} | {} | {} |".format(software, '</br>'.join(versions), '</br>'.join(clusters))) + print() + print('---8<--- "modules_matrix_search.md"') + +def print_hint(): + """ print_hint """ + print('!!! Hint "Cluster Acronyms"') + print(' ```') + print(' B D P U S A') + print(' | | | | | |') + print(' | | | | | +----> Anselm') + print(' | | | | +------> Salomon') + print(' | | | +--------> UV2000') + print(' | | +----------> Phi') + print(' | +------------> DGX-2') + print(' +--------------> Barbora') + print(' ```') + print() + print("{} {} {}".format('| Module </br><form><input id="searchInput" placeholder="🔍 Filter"', + 'style="width: 8rem; border-radius: 0.2rem; color: black;', + 'padding-left: .2rem;"></form> | Versions | Clusters |')) + print("| ------ | -------- | -------- |") + +def main(): + """ + main function + """ + arg = arg_parse() + datalist = get_datalist() + software = get_software(datalist) + + if arg.json: + print(json.dumps(packages_json(software))) + else: + print(datalist) + print_hint() + print_software(software) + +if __name__ == "__main__": + main() diff --git a/scripts/titlemd.py b/scripts/titlemd.py index 2fc5cb89c9cd25b4076cd7800b91fbf8b8e899c7..39fb6b0a03e0cbbe61bde62b55f9dbf30909d0ad 100755 --- a/scripts/titlemd.py +++ b/scripts/titlemd.py @@ -1,48 +1,160 @@ -#!/usr/bin/python +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- +""" titlemd """ -import fnmatch -import os +from __future__ import print_function + +import argparse import sys try: - from titlecase import titlecase + from titlecase import titlecase except ImportError: - print("Please install titlecase") - -def main(location): - # Spelling exceptions - with open('.spelling') as f: - spelling = f.readlines() - - def abbreviations(word, **kwargs): - if word+"\n" in spelling: - return word - - # Open the file and read the lines as a list - with open(location) as f: - lines = f.readlines() - - with open(location, 'w') as f: - # Loop through the list of lines and titlecase - # any line beginning with '#'. - prev_line = lines.pop(0) - disabled = 0 - for line in lines: - if line.startswith("``") and disabled == 0: - disabled = 1 - else: - if line.startswith("``") and disabled == 1: - disabled = 0 - if line.startswith('#') and disabled == 0: - line = titlecase(line[:(line.find("]"))], callback=abbreviations)+line[(line.find("]")):] - if (line.startswith('---') or line.startswith('===')) and disabled == 0: - prev_line = titlecase(prev_line[:(prev_line.find("]"))], callback=abbreviations)+prev_line[(prev_line.find("]")):] - f.write(prev_line) - prev_line = line - f.write(prev_line) + print("Please install titlecase") + +def arg_parse(): + """ + argument parser + """ + parser = argparse.ArgumentParser( + description="Titlemd" + ) + parser.add_argument('-t', '--test', + action='store_true', + help="test") + parser.add_argument('location', + nargs='?', + default='.', + help="location, default current directory") + return parser.parse_args() + +def mkdocs_available(location): + """ Is mkdocs.yml available? """ + if location.find("mkdocs.yml") != -1: + return True + return False + +def linestart(line, disabled, test, prev_line=None): + """ linestart """ + if test: + if (line.startswith("``") or line.startswith("extra:")) and not disabled: + return True + else: + if (line.startswith("``") or prev_line.startswith("pages:")) and disabled: + return False + else: + if line.startswith("``") and not disabled: + return True + else: + if line.startswith("``") and disabled: + return False + return disabled + +def testdata(arg): + """ test """ + # Spelling exceptions + with open('.spelling') as fdata: + spelling = fdata.readlines() + + # pylint: disable=unused-argument,inconsistent-return-statements + def abbreviations(word, **kwargs): + """ abbreviations """ + if word+"\n" in spelling: + return word + + # Open the file and read the lines as a list + with open(arg.location) as fdata: + lines = fdata.readlines() + + # Loop through the list of lines and titlecase + # any line beginning with '#'. + return_value = 0 + prev_line = lines[0] + echo_filename = False + disabled = mkdocs_available(arg.location) + for line in lines: + disabled = linestart(line, disabled, arg.test, prev_line) + if line.startswith('#') and not disabled and not mkdocs_available(arg.location): + if line != titlecase(line[:(line.find("]"))], + callback=abbreviations)+line[(line.find("]")):]: + if return_value == 0 and not echo_filename: + print("%s" % arg.location) + echo_filename = True + print("-"+line, end="") + print("+"+titlecase(line[:(line.find("]"))], + callback=abbreviations)+line[(line.find("]")):], end="") + print() + return_value = 1 + if (line.startswith('---') or line.startswith('===')) and not disabled: + if prev_line != titlecase(prev_line[:(prev_line.find("]"))], + callback=abbreviations)+prev_line[(prev_line.find("]")):]: + if return_value == 0 and not echo_filename: + print("%s" % arg.location) + echo_filename = True + print("-"+prev_line, end="") + print("+"+titlecase(prev_line[:(prev_line.find("]"))], + callback=abbreviations)+prev_line[(prev_line.find("]")):], + end="") + print() + return_value = 1 + if (mkdocs_available(arg.location) and not line.startswith('#') and not disabled): + if line != titlecase(line[:(line.find(":"))], + callback=abbreviations)+line[(line.find(":")):]: + if return_value == 0 and not echo_filename: + print("%s" % arg.location) + echo_filename = True + print("-"+line, end="") + print("+"+titlecase(line[:(line.find(":"))], + callback=abbreviations)+line[(line.find(":")):], end="") + print() + return_value = 1 + prev_line = line + return return_value + +def writedata(arg): + """ writedata """ + # Spelling exceptions + with open('.spelling') as fdata: + spelling = fdata.readlines() + + # pylint: disable=unused-argument,inconsistent-return-statements + def abbreviations(word, **kwargs): + """ abbreviations """ + if word+"\n" in spelling: + return word + + # Open the file and read the lines as a list + with open(arg.location) as fdata: + lines = fdata.readlines() + + with open(arg.location, 'w') as fdata: + # Loop through the list of lines and titlecase + # any line beginning with '#'. + prev_line = lines[0] + disabled = False + for line in lines: + disabled = linestart(line, disabled, arg.test) + if line.startswith('#') and not disabled: + line = titlecase(line[:(line.find("]"))], + callback=abbreviations)+line[(line.find("]")):] + if (line.startswith('---') or line.startswith('===')) and not disabled: + prev_line = titlecase(prev_line[:(prev_line.find("]"))], + callback=abbreviations)+prev_line[(prev_line.find("]")):] + fdata.write(prev_line) + prev_line = line + fdata.write(prev_line) + +def main(): + """ + main function + """ + + arg = arg_parse() + + if arg.test: + sys.exit(testdata(arg)) + else: + writedata(arg) if __name__ == "__main__": - try: - main(sys.argv[1]) - except IndexError: - main('.') + main() diff --git a/scripts/titlemd_test.py b/scripts/titlemd_test.py deleted file mode 100755 index 49287c0ec98e78539157fdb4a8b27b1b550616a9..0000000000000000000000000000000000000000 --- a/scripts/titlemd_test.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python -from __future__ import print_function - -import fnmatch -import os -import sys - -try: - from titlecase import titlecase -except ImportError: - print("Please install titlecase") - -def main(location): - # Spelling exceptions - with open('.spelling') as f: - spelling = f.readlines() - - def abbreviations(word, **kwargs): - if word+"\n" in spelling: - return word - - # Open the file and read the lines as a list - with open(location) as f: - lines = f.readlines() - - # Loop through the list of lines and titlecase - # any line beginning with '#'. - return_value = 0 - prev_line = lines[0] - disabled = 0 - echo_filename = False - if location.find("mkdocs.yml") != -1: - disabled = 1 - for line in lines: - if (line.startswith("``") or line.startswith("extra:")) and disabled == 0: - disabled = 1 - else: - if (line.startswith("``") or prev_line.startswith("pages:")) and disabled == 1: - disabled = 0 - if line.startswith('#') and (disabled == 0) and (location.find("mkdocs.yml") == -1): - if line != titlecase(line[:(line.find("]"))], callback=abbreviations)+line[(line.find("]")):]: - if return_value == 0 and echo_filename == False: - print("%s" % location) - echo_filename = True - print("-"+line,end="") - print("+"+titlecase(line[:(line.find("]"))], callback=abbreviations)+line[(line.find("]")):],end="") - print() - return_value = 1 - if (line.startswith('---') or line.startswith('===')) and disabled == 0: - if prev_line != titlecase(prev_line[:(prev_line.find("]"))], callback=abbreviations)+prev_line[(prev_line.find("]")):]: - if return_value == 0 and echo_filename == False: - print("%s" % location) - echo_filename = True - print("-"+prev_line,end="") - print("+"+titlecase(prev_line[:(prev_line.find("]"))], callback=abbreviations)+prev_line[(prev_line.find("]")):],end="") - print() - return_value = 1 - if ((location.find("mkdocs.yml") != -1) and not line.startswith('#') and disabled == 0): - if line != titlecase(line[:(line.find(":"))], callback=abbreviations)+line[(line.find(":")):]: - if return_value == 0 and echo_filename == False: - print("%s" % location) - echo_filename = True - print("-"+line,end="") - print("+"+titlecase(line[:(line.find(":"))], callback=abbreviations)+line[(line.find(":")):],end="") - print() - return_value = 1 - prev_line = line - exit(return_value) -if __name__ == "__main__": - try: - main(sys.argv[1]) - except IndexError: - main('.')