modules.py

#!/usr/bin/env python3
"""Script to generate module lists"""

import os
import re

def get_software_list(path):
    """List software from given module class"""
    os.chdir(path)
    software_list = next(os.walk('.'))[1]
    # In case that there is 'all' module
    if 'all' in software_list:
        software_list.remove('all')
    return software_list

def get_software_versions(path):
    """List available versions of software"""
    os.chdir(path)
    return next(os.walk('.'))[2]

def get_module_description(module_path):
    """Return software homepage URL and description"""
    url = ""
    description = "Description not available."
    with open(module_path) as _file:
        content = _file.read()
    if module_path.endswith('lua'):
        data = re.search(r"whatis\((\[\[|\[==\[)Description: ?([\s\S]+)(?:\]\]|\]==\])\)\s+whatis\((?:\[\[|\[==\[)Homepage:\s(\S+)(?:\]\]|\]==\])\)",
                         content, re.DOTALL)
    else:
        data = re.search(r"module-whatis\s{(Description: )?(.+)\s-\sHomepage:\s(\S+)}",
                         content, re.DOTALL)

    try:
        description = data.group(2)
        url = data.group(3)
    except AttributeError:
        pass
    return url, description

if __name__ == "__main__":
    HTML_SOURCE_DATA = {}
    SUBDIRS = os.environ['MODULEPATH'].split(':')

    for subdir in sorted(SUBDIRS, key=lambda s: s.lower()):
        # Ignore the specific path
        if subdir == "/apps/all/Lmod/8.7.37/modulefiles/Core":
            continue
        if subdir == "/apps/all/Lmod/8.7.37/lmod/lmod/modulefiles/Core":
            continue
        if subdir == "/apps/all/Lmod/8.7.37/modulefiles/Linux":
            continue
        if subdir == "/apps/all/intel-compilers/2023.2.1/modulefiles":
            continue
        if subdir == "/opt/cray/pe/modulefiles":
            continue
        if subdir == "/opt/cray/pe/craype-targets/1.4.0/modulefiles":
            continue
        
        available_software = get_software_list(subdir)
        print("\n## {}\n".format(os.path.basename(subdir).title()))
        print("| Module | Description |")
        print("| ------ | ----------- |")
        HTML_SOURCE_DATA[subdir] = {}
        for soft in sorted(available_software, key=str.lower):
            software_path = os.path.join(subdir, soft)
            software_versions = get_software_versions(software_path)
            software_versions = [ver for ver in software_versions if not ver[0] == '.']

            if not software_versions:
                continue
            first_module_path = os.path.join(software_path, software_versions[0])
            software_url, software_description = get_module_description(first_module_path)
            HTML_SOURCE_DATA[subdir][soft] = {'versions': [],
                                              'url': software_url,
                                              'description': software_description}
            for version in software_versions:
                HTML_SOURCE_DATA[subdir][soft]['versions'].append(version)
            if software_url in ["", "(none)", "N/A"]:
                print("| {} | {} |".format(soft, " ".join(software_description.split())))
            else:
                print("| [{}]({}) | {} |".format(soft, software_url, " ".join(software_description.split())))