Python refactoring

.gitlab-ci.yml

@@ -14,11 +14,23 @@ docs:
   script:
   - mdl -r ~MD013,~MD010,~MD014,~MD024,~MD026,~MD029,~MD033,~MD036,~MD037,~MD046 *.md docs.it4i # BUGS
 
+pylint:
+  stage: test
+  image: it4innovations/docker-pycheck:latest
+  script:
+  - pylint $(find . -name "*.py" -not -name "feslicescript.py")
+
+pysafety:
+  stage: test
+  image: it4innovations/docker-pycheck:latest
+  script:
+  - cat requirements.txt | safety check --stdin --full-report
+
 capitalize:
   stage: test
   image: it4innovations/docker-mkdocscheck:latest
   script:
-  - find mkdocs.yml docs.it4i/ \( -name '*.md' -o -name '*.yml' \) -print0 | xargs -0 -n1 scripts/titlemd_test.py
+  - find mkdocs.yml docs.it4i/ \( -name '*.md' -o -name '*.yml' \) -print0 | xargs -0 -n1 scripts/titlemd.py --test
 
 ext_links:
   stage: after_test
@@ -45,10 +57,10 @@ ext_links:
 mkdocs:
   stage: build
   image: it4innovations/docker-mkdocscheck:latest
-  before_script:
-  - python -V               # Print out python version for debugging
   script:
-  - mkdocs -V
+  - python -V  # debug
+  - pip freeze # debug
+  - mkdocs -V # debug
     # add version to footer
   - bash scripts/add_version.sh
     # get modules list from clusters
@@ -58,8 +70,12 @@ mkdocs:
     # generate ULT for code link
   - sed -i "s/master/$CI_BUILD_REF_NAME/g" material/partials/toc.html
     # regenerate modules matrix
-  - python scripts/modules-matrix.py > docs.it4i/modules-matrix.md
-  - python scripts/modules-json.py > docs.it4i/modules-matrix.json
+  - source /opt/.venv3/bin/activate
+  - python -V # debug
+  - pip freeze # debug
+  - python scripts/modules_matrix.py > docs.it4i/modules-matrix.md
+  - python scripts/modules_matrix.py --json > docs.it4i/modules-matrix.json
+  - declare -Ff deactivate  && deactivate
   - curl -f0 https://code.it4i.cz/sccs/scs-api-public/raw/master/scs_api.server_public.md -o docs.it4i/apiv1.md
     # build pages
   - mkdocs build

requirements.txt

@@ -9,6 +9,7 @@ Markdown==3.1.1
 MarkupSafe==1.1.1
 mkdocs==1.0.4
 mkdocs-material==4.6.0
+packaging==20.3
 Pygments==2.5.2
 pymdown-extensions==6.2.1
 PyYAML==5.3.1

scripts/combinations.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+""" combinations """
+
 import itertools
 import re
-l = ['A', 'S', 'U', 'T', 'D']
-mask = ''.join(reversed(l))
-for i in range(1,len(l)+1):
-    for comb in itertools.combinations(l, i):
+
+CHARS = ['A', 'S', 'U', 'T', 'D']
+MASK = ''.join(reversed(CHARS))
+for i in range(1, len(CHARS)+1):
+    for comb in itertools.combinations(CHARS, i):
         reg = "[^%s]" % ''.join(comb)
-        print re.sub(reg,"-", mask)
+        print re.sub(reg, "-", MASK)

scripts/matrix.py

 #!/usr/bin/python
 # -*- coding: utf-8 -*-
-import csv
-import collections
+""" matrix """
+
 import itertools
-import re
 
-from distutils.version import LooseVersion
-l = ['A', 'S', 'U', 'P', 'D', 'B']
-mask = ''.join(reversed(l))
-from itertools import product
-for bits in product([0, 1], repeat=len(l)):
+CHARS = ['A', 'S', 'U', 'P', 'D', 'B']
+MASK = ''.join(reversed(CHARS))
+for bits in itertools.product([0, 1], repeat=len(CHARS)):
     s = "".join(str(bit) for bit in bits)
     ns = ""
-    for i in range(len(s)):
+    for i, _ in enumerate(s):
         if s[i] == "1":
-            ns += mask[i]
+            ns += MASK[i]
         else:
             ns += "-"
     print ns

scripts/modules-json.py

-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
-import csv
-import collections
-import json
-import itertools
-import re
-
-from distutils.version import LooseVersion
-
-
-def get_data(filename):
-    '''function to read the data form the input csv file to use in the analysis'''
-    reader = []  # Just in case the file open fails
-    with open(filename, 'rb') as f:
-        reader = csv.reader(f, delimiter=',')
-        # returns all the data from the csv file in list form
-        # f.close() # May need to close the file when done
-        return list(reader)  # only return the reader when you have finished.
-
-your_list = []
-your_list += get_data('./scripts/anselm.csv')
-your_list += get_data('./scripts/salomon.csv')
-your_list += get_data('./scripts/uv2000.csv')
-your_list += get_data('./scripts/phi.csv')
-your_list += get_data('./scripts/dgx.csv')
-your_list += get_data('./scripts/barbora.csv')
-
-counts = dict()
-for i in your_list:
-    counts[i[0]] = counts.get(i[0], 0) + int(i[1])
-
-#     1    2    4    8    16   32
-l = ['A', 'S', 'U', 'P', 'D', 'B']
-c = []
-mask = ''.join(reversed(l))
-from itertools import product
-for bits in product([0, 1], repeat=len(l)):
-    s = "".join(str(bit) for bit in bits)
-    ns = ""
-    for i in range(len(s)):
-        if s[i] == "1":
-            ns += mask[i]
-        else:
-            ns += "-"
-    c.append(ns)
-
-software = dict()
-versions = ''
-clusters = ''
-prev = ''
-
-for m, i in sorted(counts.items()):
-    # print m
-    split = m.split('/')
-    # print split
-    if len(split) > 1:
-        a = split[0]
-        b = split[1]
-        if split[0] <> prev:
-            software[a] = {}
-        software[a][b] = '`' + c[i] + '`'
-        prev = a
-
-
-packages = {}
-
-for m in sorted(software.items(), key=lambda i: i[0].lower()):
-    packages[m[0]] = sorted(m[1], key=LooseVersion)[len(m[1]) - 1]
-
-data = {'total': len(packages), 'projects': packages}
-print json.dumps(data)

scripts/modules-matrix.py

-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
-import csv
-import collections
-import itertools
-import re
-
-from distutils.version import LooseVersion
-
-
-def get_data(filename):
-    '''function to read the data form the input csv file to use in the analysis'''
-    reader = []  # Just in case the file open fails
-    with open(filename, 'rb') as f:
-        reader = csv.reader(f, delimiter=',')
-        # returns all the data from the csv file in list form
-        # f.close() # May need to close the file when done
-        return list(reader)  # only return the reader when you have finished.
-
-your_list = []
-your_list += get_data('./scripts/anselm.csv')
-your_list += get_data('./scripts/salomon.csv')
-your_list += get_data('./scripts/uv2000.csv')
-your_list += get_data('./scripts/phi.csv')
-your_list += get_data('./scripts/dgx.csv')
-your_list += get_data('./scripts/barbora.csv')
-
-print your_list
-
-counts = dict()
-for i in your_list:
-    # print i[0]
-    # print int(i[1])
-    counts[i[0]] = counts.get(i[0], 0) + int(i[1])
-
-#     1    2    4    8    16   32
-l = ['A', 'S', 'U', 'P', 'D', 'B']
-c = []
-mask = ''.join(reversed(l))
-from itertools import product
-for bits in product([0, 1], repeat=len(l)):
-    s = "".join(str(bit) for bit in bits)
-    ns = ""
-    for i in range(len(s)):
-        if s[i] == "1":
-            ns += mask[i]
-        else:
-            ns += "-"
-    c.append(ns)
-
-print '!!! Hint "Cluster Acronyms"'
-print '    ```'
-print '    B D P U S A'
-print '    | | | | | |'
-print '    | | | | | +----> Anselm'
-print '    | | | | +------> Salomon'
-print '    | | | +--------> UV2000'
-print '    | | +----------> Phi'
-print '    | +------------> DGX-2'
-print '    +--------------> Barbora'
-print '    ```'
-print
-print '| Module </br><form><input id="searchInput" placeholder="🔍 Filter" style="width: 8rem; border-radius: 0.2rem; color: black; padding-left: .2rem;"></form> | Versions | Clusters |'
-print "| ------ | -------- | -------- |"
-
-software = dict()
-versions = ''
-clusters = ''
-prev = ''
-
-for m, i in sorted(counts.items()):
-    #  print m
-    split = m.split('/')
-#  print split
-    if len(split) > 1:
-        a = split[0]
-        b = split[1]
-        if split[0] <> prev:
-            software[a] = {}
-        software[a][b] = '`' + c[i] + '`'
-        prev = a
-
-for m in sorted(software.items(), key=lambda i: i[0].lower()):
-    software = m[0]
-    versions = []
-    clusters = []
-    for key in sorted(m[1], key=LooseVersion):
-        versions.append(key)
-        clusters.append(m[1][key])
-    print "| %s | %s | %s |" % (software, '</br>'.join(versions), '</br>'.join(clusters))
-
-print
-print '---8<--- "modules_matrix_search.md"'

scripts/modules_matrix.py

+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+""" modules_matrix """
+
+from __future__ import print_function
+
+import argparse
+import csv
+import itertools
+import json
+import os.path
+import packaging.specifiers
+
+def arg_parse():
+    """
+    argument parser
+    """
+    parser = argparse.ArgumentParser(
+        description="Module_matrix"
+    )
+    parser.add_argument('--json',
+                        action='store_true',
+                        help="get json")
+    return parser.parse_args()
+
+def get_data(filename):
+    '''function to read the data form the input csv file to use in the analysis'''
+    reader = []  # Just in case the file open fails
+    with open(filename, 'r') as fdata:
+        reader = csv.reader(fdata, delimiter=',')
+        # returns all the data from the csv file in list form
+#        print(list(reader))
+        return list(reader)  # only return the reader when you have finished.
+
+def get_datalist():
+    """ get_datalist """
+    datalist = []
+    for name in ['anselm', 'salomon', 'uv2000', 'phi', 'dgx', 'barbora']:
+        path = os.path.join('scripts', "{}.csv".format(name))
+        datalist += get_data(path)
+    return datalist
+
+def get_counts(datalist):
+    """ get_counts """
+    counts = dict()
+    for i in datalist:
+        counts[i[0]] = counts.get(i[0], 0) + int(i[1])
+    return counts
+
+def get_matrix():
+    """ get_matrix """
+    #     1    2    4    8    16   32
+    chars = ['A', 'S', 'U', 'P', 'D', 'B']
+    arr = []
+    mask = ''.join(reversed(chars))
+    for bits in itertools.product([0, 1], repeat=len(chars)):
+        sbit = "".join(str(bit) for bit in bits)
+        nst = ""
+        for i, _ in enumerate(sbit):
+            if sbit[i] == "1":
+                nst += mask[i]
+            else:
+                nst += "-"
+        arr.append(nst)
+    return arr
+
+def get_software(datalist):
+    """ get_software """
+    matrix = get_matrix()
+    counts = get_counts(datalist)
+    software = dict()
+    prev = ''
+    for mat, i in sorted(counts.items()):
+        split = mat.split('/')
+        if len(split) > 1:
+            if split[0] != prev:
+                software[split[0]] = {}
+            software[split[0]][split[1]] = '`' + matrix[i] + '`'
+            prev = split[0]
+    return software
+
+def packages_json(software):
+    """ packages_json """
+    packages = {}
+    for module in sorted(software.items(), key=lambda i: i[0].lower()):
+        packages[module[0]] = sorted(module[1],
+                                     key=packaging.specifiers.LegacyVersion)[len(module[1]) - 1]
+    data = {'total': len(packages), 'projects': packages}
+    return data
+
+def print_software(software):
+    """ print_software """
+    versions = ''
+    clusters = ''
+    for module in sorted(software.items(), key=lambda i: i[0].lower()):
+        software = module[0]
+        versions = []
+        clusters = []
+        for key in sorted(module[1].keys(), key=packaging.specifiers.LegacyVersion):
+            versions.append(key)
+            clusters.append(module[1][key])
+        print("| {} | {} | {} |".format(software, '</br>'.join(versions), '</br>'.join(clusters)))
+    print()
+    print('---8<--- "modules_matrix_search.md"')
+
+def print_hint():
+    """ print_hint """
+    print('!!! Hint "Cluster Acronyms"')
+    print('    ```')
+    print('    B D P U S A')
+    print('    | | | | | |')
+    print('    | | | | | +----> Anselm')
+    print('    | | | | +------> Salomon')
+    print('    | | | +--------> UV2000')
+    print('    | | +----------> Phi')
+    print('    | +------------> DGX-2')
+    print('    +--------------> Barbora')
+    print('    ```')
+    print()
+    print("{} {} {}".format('| Module </br><form><input id="searchInput" placeholder="🔍 Filter"',
+                            'style="width: 8rem; border-radius: 0.2rem; color: black;',
+                            'padding-left: .2rem;"></form> | Versions | Clusters |'))
+    print("| ------ | -------- | -------- |")
+
+def main():
+    """
+    main function
+    """
+    arg = arg_parse()
+    datalist = get_datalist()
+    software = get_software(datalist)
+
+    if arg.json:
+        print(json.dumps(packages_json(software)))
+    else:
+        print(datalist)
+        print_hint()
+        print_software(software)
+
+if __name__ == "__main__":
+    main()

scripts/titlemd.py

-#!/usr/bin/python
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+""" titlemd """
 
-import fnmatch
-import os
+from __future__ import print_function
+
+import argparse
 import sys
 
 try:
-  from titlecase import titlecase
+    from titlecase import titlecase
 except ImportError:
-  print("Please install titlecase")
-
-def main(location):
-      # Spelling exceptions
-      with open('.spelling') as f:
-          spelling = f.readlines()
-
-      def abbreviations(word, **kwargs):
-          if word+"\n" in spelling:
-              return word
-
-      # Open the file and read the lines as a list
-      with open(location) as f:
-          lines = f.readlines()
-
-      with open(location, 'w') as f:
-          # Loop through the list of lines and titlecase
-          # any line beginning with '#'.
-          prev_line = lines.pop(0)
-          disabled = 0
-          for line in lines:
-              if line.startswith("``") and disabled == 0:
-                  disabled = 1
-              else:
-                  if line.startswith("``") and disabled == 1:
-                      disabled = 0
-              if line.startswith('#') and disabled == 0:
-                  line = titlecase(line[:(line.find("]"))], callback=abbreviations)+line[(line.find("]")):] 
-              if (line.startswith('---') or line.startswith('===')) and disabled == 0:
-                  prev_line = titlecase(prev_line[:(prev_line.find("]"))], callback=abbreviations)+prev_line[(prev_line.find("]")):]
-              f.write(prev_line)
-              prev_line = line
-          f.write(prev_line)
+    print("Please install titlecase")
+
+def arg_parse():
+    """
+    argument parser
+    """
+    parser = argparse.ArgumentParser(
+        description="Titlemd"
+    )
+    parser.add_argument('-t', '--test',
+                        action='store_true',
+                        help="test")
+    parser.add_argument('location',
+                        nargs='?',
+                        default='.',
+                        help="location, default current directory")
+    return parser.parse_args()
+
+def mkdocs_available(location):
+    """ Is mkdocs.yml available? """
+    if location.find("mkdocs.yml") != -1:
+        return True
+    return False
+
+def linestart(line, disabled, test, prev_line=None):
+    """ linestart """
+    if test:
+        if (line.startswith("``") or line.startswith("extra:")) and not disabled:
+            return True
+        else:
+            if (line.startswith("``") or prev_line.startswith("pages:")) and disabled:
+                return False
+    else:
+        if line.startswith("``") and not disabled:
+            return True
+        else:
+            if line.startswith("``") and disabled:
+                return False
+    return disabled
+
+def testdata(arg):
+    """ test """
+    # Spelling exceptions
+    with open('.spelling') as fdata:
+        spelling = fdata.readlines()
+
+    # pylint: disable=unused-argument,inconsistent-return-statements
+    def abbreviations(word, **kwargs):
+        """ abbreviations """
+        if word+"\n" in spelling:
+            return word
+
+    # Open the file and read the lines as a list
+    with open(arg.location) as fdata:
+        lines = fdata.readlines()
+
+    # Loop through the list of lines and titlecase
+    # any line beginning with '#'.
+    return_value = 0
+    prev_line = lines[0]
+    echo_filename = False
+    disabled = mkdocs_available(arg.location)
+    for line in lines:
+        disabled = linestart(line, disabled, arg.test, prev_line)
+        if line.startswith('#') and not disabled and not mkdocs_available(arg.location):
+            if line != titlecase(line[:(line.find("]"))],
+                                 callback=abbreviations)+line[(line.find("]")):]:
+                if return_value == 0 and not echo_filename:
+                    print("%s" % arg.location)
+                    echo_filename = True
+                print("-"+line, end="")
+                print("+"+titlecase(line[:(line.find("]"))],
+                                    callback=abbreviations)+line[(line.find("]")):], end="")
+                print()
+                return_value = 1
+        if (line.startswith('---') or line.startswith('===')) and not disabled:
+            if prev_line != titlecase(prev_line[:(prev_line.find("]"))],
+                                      callback=abbreviations)+prev_line[(prev_line.find("]")):]:
+                if return_value == 0 and not echo_filename:
+                    print("%s" % arg.location)
+                    echo_filename = True
+                print("-"+prev_line, end="")
+                print("+"+titlecase(prev_line[:(prev_line.find("]"))],
+                                    callback=abbreviations)+prev_line[(prev_line.find("]")):],
+                      end="")
+                print()
+                return_value = 1
+        if (mkdocs_available(arg.location) and not line.startswith('#') and not disabled):
+            if line != titlecase(line[:(line.find(":"))],
+                                 callback=abbreviations)+line[(line.find(":")):]:
+                if return_value == 0 and not echo_filename:
+                    print("%s" % arg.location)
+                    echo_filename = True
+                print("-"+line, end="")
+                print("+"+titlecase(line[:(line.find(":"))],
+                                    callback=abbreviations)+line[(line.find(":")):], end="")
+                print()
+                return_value = 1
+            prev_line = line
+    return return_value
+
+def writedata(arg):
+    """ writedata """
+    # Spelling exceptions
+    with open('.spelling') as fdata:
+        spelling = fdata.readlines()
+
+    # pylint: disable=unused-argument,inconsistent-return-statements
+    def abbreviations(word, **kwargs):
+        """ abbreviations """
+        if word+"\n" in spelling:
+            return word
+
+    # Open the file and read the lines as a list
+    with open(arg.location) as fdata:
+        lines = fdata.readlines()
+
+    with open(arg.location, 'w') as fdata:
+        # Loop through the list of lines and titlecase
+        # any line beginning with '#'.
+        prev_line = lines[0]
+        disabled = False
+        for line in lines:
+            disabled = linestart(line, disabled, arg.test)
+            if line.startswith('#') and not disabled:
+                line = titlecase(line[:(line.find("]"))],
+                                 callback=abbreviations)+line[(line.find("]")):]
+            if (line.startswith('---') or line.startswith('===')) and not disabled:
+                prev_line = titlecase(prev_line[:(prev_line.find("]"))],
+                                      callback=abbreviations)+prev_line[(prev_line.find("]")):]
+            fdata.write(prev_line)
+            prev_line = line
+        fdata.write(prev_line)
+
+def main():
+    """
+    main function
+    """
+
+    arg = arg_parse()
+
+    if arg.test:
+        sys.exit(testdata(arg))
+    else:
+        writedata(arg)
 
 if __name__ == "__main__":
-  try:
-      main(sys.argv[1])
-  except IndexError:
-      main('.')
+    main()

scripts/titlemd_test.py

-#!/usr/bin/env python
-from __future__ import print_function
-
-import fnmatch
-import os
-import sys
-
-try:
-  from titlecase import titlecase
-except ImportError:
-  print("Please install titlecase")
-
-def main(location):
-      # Spelling exceptions
-      with open('.spelling') as f:
-          spelling = f.readlines()
-
-      def abbreviations(word, **kwargs):
-          if word+"\n" in spelling:
-              return word
-
-      # Open the file and read the lines as a list
-      with open(location) as f:
-          lines = f.readlines()
-
-      # Loop through the list of lines and titlecase
-      # any line beginning with '#'.
-      return_value = 0
-      prev_line = lines[0]
-      disabled = 0
-      echo_filename = False
-      if location.find("mkdocs.yml") != -1:
-          disabled = 1
-      for line in lines:
-          if (line.startswith("``") or line.startswith("extra:")) and disabled == 0:
-              disabled = 1
-          else:
-              if (line.startswith("``") or prev_line.startswith("pages:")) and disabled == 1:
-                  disabled = 0
-          if line.startswith('#') and (disabled == 0) and (location.find("mkdocs.yml") == -1):
-            if line != titlecase(line[:(line.find("]"))], callback=abbreviations)+line[(line.find("]")):]:
-              if return_value == 0 and echo_filename == False:
-                print("%s" % location)
-                echo_filename = True
-              print("-"+line,end="") 
-              print("+"+titlecase(line[:(line.find("]"))], callback=abbreviations)+line[(line.find("]")):],end="")
-              print()
-              return_value = 1
-          if (line.startswith('---') or line.startswith('===')) and disabled == 0:
-            if prev_line != titlecase(prev_line[:(prev_line.find("]"))], callback=abbreviations)+prev_line[(prev_line.find("]")):]:
-              if return_value == 0 and echo_filename == False:
-                print("%s" % location)
-                echo_filename = True
-              print("-"+prev_line,end="")
-              print("+"+titlecase(prev_line[:(prev_line.find("]"))], callback=abbreviations)+prev_line[(prev_line.find("]")):],end="")
-              print()
-              return_value = 1
-          if ((location.find("mkdocs.yml") != -1) and not line.startswith('#') and disabled == 0):
-            if line != titlecase(line[:(line.find(":"))], callback=abbreviations)+line[(line.find(":")):]:
-              if return_value == 0 and echo_filename == False:
-                print("%s" % location)
-                echo_filename = True
-              print("-"+line,end="")
-              print("+"+titlecase(line[:(line.find(":"))], callback=abbreviations)+line[(line.find(":")):],end="")
-              print()
-              return_value = 1
-          prev_line = line
-      exit(return_value)
-if __name__ == "__main__":
-  try:
-      main(sys.argv[1])
-  except IndexError:
-      main('.')