Merge branch '52-module-matrix-json-export' into 'master'

Resolve "Module matrix - json export"

Closes #52

See merge request !119

.gitlab-ci.yml

@@ -67,6 +67,7 @@ mkdocs:
   - bash scripts/get_modules.sh
     # regenerate modules matrix
   - python scripts/modules-matrix.py > docs.it4i/modules-matrix.md
+  - python scripts/modules-json.py > docs.it4i/modules-matrix.json
     # build pages
   - mkdocs build
     # compress search_index.json

docs.it4i/modules-matrix.json

+{"total": 415, "projects": {"MIKE": "default", "SIP": "4.17-Python-2.7.9", "HDF5": "1.10.0-patch1-intel-2016.01-mic", "p4vasp": "0.3.29-GNU-4.9.3-2.25", "Automake": "1.15-GNU-5.1.0-2.25", "netcdf": "4.3.0", "bullxde": "2.0", "APR-util": "1.5.4-foss-2015g", "ScaLAPACK": "2.0.2-OpenBLAS-0.2.14-LAPACK-3.5.0", "BerkeleyUPC": "2.16.2-gompi-2015b", "BWA": "0.7.5a-foss-2015g", "openmpi": "1.8.1-icc", "matlab": "R2014a-EDU", "sympy": "0.7.6-intel-2016.01-Python-2.7.9", "kbproto": "1.0.7-intel-2016a", "lsprepost": "4.2", "prace": "20160107-intel-2016.01", "mpt": "2.12", "Bison": "3.0.4-GCC-4.9.3", "totalview": "8.13", "Wine": "1.7.29-GNU-5.1.0-2.25", "opari2": "1.1.2-icc", "MAP": "5.0.1", "libyaml": "0.1.6-intel-2015b", "mercurial": "2.9.1", "beopest": "13.3", "perfsuite": "1a5.3", "PSBLAS-ext": "1.0-4-GCC-4.9.3-2.25", "OSPRay": "0.9.1", "S4MPLE": "1.0.0", "libxslt": "1.1.28-intel-2015b", "hwloc": "1.11.5-GCC-6.3.0-2.27", "libunistring": "0.9.3-intel-2015b", "QGIS": "2.12.3-foss-2015g", "ngsPipeline": "1.0.0", "boost": "1.56-icc-impi", "matplotlib": "1.4.3-intel-2015b-Python-2.7.9", "openfoam": "2.2.2-icc-openmpi1.8.1-DP", "Szip": "2.1-intel-2017a", "PROJ_4": "4.9.2-foss-2015g", "phono3py": "1.11.7.8-intel-2015b-Python-2.7.11", "CMake": "3.7.2-intel-2017a", "COMSOL": "51-EDU", "hdf5": "1.8.13", "gimkl": "2.11.5", "xineramaproto": "1.2.1-intel-2015b", "xextproto": "7.3.0-intel-2016a", "GLM": "0.9.7.2-intel-2017a", "SWIG": "3.0.7-Python-2.7.9", "tmux": "2.3", "ipm": "0.983-icc-impi", "SUMO": "0.27.1-foss-2015g", "ipp": "15.3.187", "hdf5-parallel": "1.8.13-gcc49", "PrgEnv-intel": "15.0.3", "libxcb": "1.11-Python-2.7.9", "MPI_NET": "1.2.0-intel-2016.01", "QEMU": "2.1.2-GCC-4.4.7-system-VDE2", "cp2k-mpi": "2.5.1-gcc", "OTF2": "2.0-intel-2015b-mic", "VirtualGL": "2.4.1", "Armadillo": "7.500.0-foss-2016a-Python-3.5.2", "netcdf-fortran": "4.2", "perfcatcher": "1.0", "tk": "8.5.15", "itac": "9.1.2.024", "LAPACKE": "3.5.0-LAPACK-3.5.0", "PrgEnv-gnu": "4.8.1", "libICE": "1.0.9-intel-2015b", "Rstudio": "0.97", "VisIt": "2.10.0", "virtualgl": "2.4", "Scipion": "1.0.1-Java-1.8.0_112-OpenMPI-1.10.2-GCC-5.3.0-2.26", "grace": "5.1.25-intel-2015b", "ANSYS": "18.0", "ATLAS": "3.10.1-GCC-4.9.3-2.25-LAPACK-3.4.2", "Scalasca": "2.3.1-intel-2015b", "BCFtools": "1.3-foss-2015g", "gcc": "5.4.0", "lxml": "3.4.4-intel-2015b-Python-2.7.9", "lsdyna": "7.x.x", "PGI": "16.10-GNU-4.9.3-2.25", "advisor_xe": "2015.1.10.380555", "CUDA": "8.0.44-intel-2017.00", "gatk": "2.6-4", "Spark": "1.5.2", "ifort": "2017.1.132-GCC-6.3.0-2.27", "lam": "7.1.4-icc", "PyYAML": "3.11-intel-2015b-Python-2.7.9", "tcsh": "6.19.00", "gperf": "3.0.4-intel-2016a", "METIS": "5.1.0-intel-2017.00", "Digimat": "5.0.1-EDU", "pigz": "2.3.3-GCC-6.2.0-2.27", "Autotools": "20150215-GNU-5.1.0-2.25", "parallel": "20150322-GNU-5.1.0-2.25", "bowtie2": "2.2.3", "QuantumESPRESSO": "5.4.0-intel-2017.00", "CP2K": "2.6.0-intel-2015b", "MATIO": "1.5.2-intel-2017a", "wine": "1.7.29", "libX11": "1.6.3-intel-2016a", "HyperWorks": "13.0", "hpg-aligner": "1.0.0", "PCRE": "8.39-intel-2017.00", "modflow-2005": "1.11.00", "EasyBuild": "3.1.0", "adios": "1.8.0", "GLOBUS": "globus", "picard": "2.1.0", "turbovnc": "1.2.3", "settarg": "7.2.2", "JOE": "4.2", "libSM": "1.2.2-intel-2015b", "pixman": "0.32.6-intel-2015b", "flex": "2.6.3-GCCcore-6.3.0", "libgdiplus": "3.12-GNU-5.1.0-2.25", "python": "3.4.2", "namd": "2.8", "APR": "1.5.2-foss-2015g", "aislinn": "20160105-Python-2.7.9-gompi-2015e", "inspector_xe": "2015.1.2.379161", "h5py": "2.4.0-ictce-7.3.5-Python-2.7.9-serial", "cURL": "7.51.0-intel-2017.00", "SIONlib": "1.6.1-tools", "bupc": "2.16.2", "PAPI": "5.4.3-pic", "PerfReports": "5.0.1", "cairo": "1.12.18-foss-2015b", "Harminv": "1.4-intel-2015b", "Perl": "5.24.0-GCC-4.9.3-2.25-bare", "Lua": "5.1.4-8", "fftw2-mpi": "2.1.5-icc", "mxml": "2.9", "Maven": "3.3.9", "GATK": "3.5-Java-1.7.0_79", "Trimmomatic": "0.35-Java-1.7.0_79", "GCCcore": "6.3.0", "GCC": "6.3.0-2.27", "xcb-proto": "1.11-Python-2.7.9", "hypermesh": "12.0.110", "imkl": "2017.1.132-iimpi-2017a", "Meep": "1.3-intel-2015b", "eudev": "3.1.5-intel-2016a", "Vampir": "9.0.0", "FastQC": "0.11.3", "PROJ": "4.9.2-intel-2017.00", "NASM": "2.11.08-intel-2017.00", "mvapich2": "1.9-icc", "iompi": "2017.01", "OpenCV": "3.0.0-intel-2015b", "ParaView": "5.0.0-binary", "ISL": "0.15-GNU-4.9.3-2.25", "intelpcm": "2.6", "Libint": "1.1.4-intel-2015b", "libreadline": "6.3-intel-2017a", "SpatiaLite": "4.3.0a-foss-2015g", "Clang": "3.7.0-GNU-5.1.0-2.25", "ParMETIS": "4.0.3-intel-2017a", "Mesa": "11.2.1-foss-2016a", "fftw3": "3.3.3-icc", "slepc": "3.7.2-icc16-impi5-mkl-opt", "MPFR": "3.1.5-intel-2017.00", "OpenCL-builder": "2015", "OpenCL-runtime": "15.1", "relion": "1.3", "XZ": "5.2.2-intel-2017.00", "libunwind": "1.1-GCC-5.4.0-2.26", "libevent": "2.1.8", "fftw2": "2.1.5-icc", "dytran": "2013.0.1", "ffmpeg": "2.4-intel-2015b", "M4": "1.4.18-GCCcore-6.3.0", "FFTW": "3.3.6-gompi-2017a", "PyQt": "4.11.4-foss-2015g-Python-2.7.9", "NWChem": "6.5.revision26243-intel-2015b-2014-09-10-Python-2.7.8", "hyperworks": "13.0", "ant": "1.9.3-Java-1.7.0_79", "Forge": "7.0", "arpack-ng": "3.4.0-intel-2017.00", "comsol": "50-EDU", "GEOS": "3.5.0-foss-2015g", "Singularity": "2.2-GCC-6.3.0-2.27", "VTune": "2016_update1", "digimat": "5.0.1", "LLVM": "3.9.0-intel-2017.00", "Qt": "4.8.6-foss-2015g", "fixesproto": "5.0-intel-2016a", "Molpro": "2010.1-patch-57-intel2015b", "libXdamage": "1.1.4-intel-2016a", "bullxmpi": "bullxmpi_1.2.4.1", "scalasca2": "2.0-icc-impi", "openssh-x509": "6.2p2", "mpi.net": "1.0.0-mono-3.12.1", "gimpi": "2.11.5", "R": "3.2.3-intel-2016.01", "Racket": "6.1.1-GNU-5.1.0-2.25", "SCOTCH": "6.0.4-intel-2017a", "fastqc": "0.11.2", "trilinos": "11.2.3-icc", "netcdf-parallel": "4.3.0", "chicken": "4.8.0.6", "OpenBLAS": "0.2.19-GCC-6.3.0-2.27-LAPACK-3.7.0", "blender": "2.71", "PCRE2": "10.22-intel-2017.00", "fontconfig": "2.11.94-intel-2017.00", "Octave": "4.0.1-gimkl-2.11.5", "DCW": "1.1.2", "Qwt": "6.1.2-foss-2015g", "Bash": "4.3", "freetype": "2.6.3-intel-2016a", "cube": "4.2.3-icc", "Valgrind": "3.11.0-intel-2015b", "iimpi": "2017a", "dhi-mike": "default", "tbb": "15.3.187", "guile": "1.8.8-intel-2015b", "PSBLAS": "3.3.4-3-GCC-4.9.3-2.25", "libXfont": "1.5.1-Python-2.7.9", "szip": "2.1", "memoryscape": "3.4", "vampir": "8.2", "libpciaccess": "0.13.4-intel-2016a", "JasPer": "1.900.1-intel-2015b", "racket": "6.0.1", "foss": "2017a", "Boost": "1.61.0-foss-2016a-serial", "FIAT": "1.6.0-intel-2016.01-Python-2.7.9", "PRACE": "prace", "gompi": "2017a", "lux": "1.3.1", "LibTIFF": "4.0.3-intel-2015b", "netCDF-Fortran": "4.4.0-intel-2016.01", "libpng": "1.6.16-intel-2015b", "SuiteSparse": "4.5.3-intel-2017a-ParMETIS-4.0.3", "FOX": "1.6.51-foss-2015g", "DDT": "5.0.1", "libctl": "3.2.2-intel-2015b", "mono": "3.12.1", "valgrind": "3.9.0-impi", "PLUMED": "2.3b-foss-2016a", "SnuCL": "1.3.3-gompi-2015e", "Tcl": "8.6.5-intel-2017a", "libXft": "2.3.2-intel-2015b", "binutils": "2.27-GCCcore-6.3.0", "GPI-2": "1.1.1-gompi-2015e-MPI", "xdrfile": "1.1.4-intel-2015b", "libGLU": "9.0.0-foss-2015g", "otf2": "1.4-icc", "util-linux": "2.28-intel-2016a", "lmod": "7.2.2", "MVAPICH2": "2.1-GNU-5.1.0-2.25", "byacc": "20150711-intel-2015b", "java": "1.7", "marc": "2013.1", "elmer": "7.0-r6695-opt", "HTSlib": "1.3-foss-2015g", "MATLAB": "2015b-EDU", "gupc": "4.8.0.3", "abinit": "7.10.1-icc-impi", "numpy": "1.9.1-intel-2015b-Python-2.7.9", "modflow-nwt": "1.0.9-aquaveo", "Adams": "2013.2", "ncurses": "6.0-intel-2017a", "MUMPS": "5.0.2-intel-2017a-parmetis", "Score-P": "3.0-intel-2015b", "ruby": "2.0.0-p247", "Subversion": "1.8.16-foss-2015g", "NAMD": "2.9-mpi", "zlib": "1.2.11-GCCcore-6.3.0", "xtrans": "1.3.5-intel-2016a", "snpEff": "3.6", "ABINIT": "7.10.1-intel-2015b", "libMesh": "0.9.5-intel-2016.01", "motif": "2.3.4-intel-2015b-libX11-1.6.2", "GNU": "5.1.0-2.25-intel-2015b", "almost": "2.1.0-intel-2015b", "libxml2": "2.9.3-intel-2017.00", "expat": "2.1.0-intel-2017.00", "Code_Saturne": "3.0.5", "opencl-rt": "4.5.0.8", "samtools": "0.1.19", "ictce": "8.3.5", "Python": "3.5.2-intel-2017.00", "make": "3.82-intel-2015b", "Mono": "4.2.2.10-intel-2016.01", "FreeFem++": "3.45-intel-2015b", "SAMtools": "1.3-foss-2015g", "SQLite": "3.13.0-intel-2017a", "HPL": "2.1-intel-2015b", "OpenDX": "4.4.4-foss-2015g", "Autoconf": "2.69-GNU-5.1.0-2.25", "RStudio": "0.98.1103", "globus": "globus", "fontsproto": "2.1.3-intel-2016a", "SDE": "7.41.0", "gzip": "1.6-intel-2015b", "gsl": "1.16-icc", "tcl": "8.5.15", "TotalView": "8.15.4-6-linux-x86-64", "MPICH": "3.2-GCC-5.3.1-snapshot-20160419-2.25", "GSL": "2.1-intel-2015b", "libXfixes": "5.0.1-intel-2016a", "OpenFOAM": "3.0.0-intel-2016.01", "SCons": "2.3.6-Python-2.7.9", "iccifort": "2017.1.132-GCC-6.3.0-2.27", "plasma": "2.6.0", "phonopy": "1.11.6.7-intel-2015b-Python-2.7.11", "libXdmcp": "1.1.2-intel-2016a", "Mercurial": "3.7.3-foss-2015g-Python-2.7.9", "xproto": "7.0.28-intel-2016a", "FLTK": "1.3.2-intel-2015b", "hpg-variant": "1.0.0", "libdrm": "2.4.68-intel-2016a", "intel": "2017a", "nwchem": "6.3-rev2-patch1-venus", "adams": "2013.2", "makedepend": "1.0.5-intel-2016a", "numactl": "2.0.11-GCC-6.3.0-2.27", "vtune_xe": "2015.3.0.403110", "Discovery_Studio": "4.0", "wien2k": "14.2", "help2man": "1.47.4-GCCcore-6.3.0", "xbitmaps": "1.1.1-intel-2015b", "Inspector": "2016_update1", "spGPU": "master-GCC-4.9.3-2.25", "JUnit": "4.11-Java-1.7.0_79", "nastran": "2013.1.1", "pkg-config": "0.29-intel-2016a", "Java": "1.8.0_112", "Marc": "2013.1.0", "magma": "1.3.0-mic", "libXrender": "0.9.8-intel-2015b", "inputproto": "2.3-intel-2015b", "libfontenc": "1.1.3-intel-2016a", "GDAL": "2.1.0-GNU-5.1.0-2.25-intel-2015b", "Cube": "4.3.4-intel-2015b", "icc": "2017.1.132-GCC-6.3.0-2.27", "qemu": "2.1.2-vde2", "fftw3-mpi": "3.3.3-icc", "Ruby": "2.3.1", "libXau": "1.0.8-intel-2016a", "Doxygen": "1.8.11-intel-2017a", "gpi2": "1.1.1", "dataspaces": "1.4.0", "RELION": "1.3-intel-2015b", "libXinerama": "1.1.3-intel-2015b", "Amber": "14", "MPC": "1.0.2-intel-2017.00", "LAMMPS": "28Jun14-intel-2015b", "libjpeg-turbo": "1.4.2-intel-2017.00", "perfboost": "1.0", "python-meep": "1.4.2-intel-2015b-Python-2.7.9-Meep-1.3", "petsc": "3.7.3-icc16-impi5-mkl-opt", "ScientificPython": "2.9.4-intel-2016.01-Python-2.7.9", "cuda": "7.5", "scorep": "1.2.3-icc-impi", "OPARI2": "2.0", "MLD2P4": "2.0-rc4-GCC-4.9.3-2.25", "maxwell": "3.0", "VampirServer": "9.0.0-intel-2015b", "spatialindex": "1.8.5-foss-2015g", "vde2": "2.3.2", "paraview": "4.0.1-gcc481-bullxmpi1.2.4.1-osmesa10.0", "Siesta": "4.1-b2-intel-2017.00", "VASP": "5.4.1-intel-2017.00-24Jun15", "git": "2.11.0-GNU-4.9.3-2.25", "lammps": "28Jun14", "mkl": "15.3.187", "xorg-macros": "1.19.0-intel-2016a", "likwid": "4.1.2-intel", "OpenCoarrays": "1.4.0-GCC-5.3.1-snapshot-20160419-2.25", "GROMACS": "5.1.2-intel-2016a-hybrid", "libXt": "1.1.5-foss-2015g", "tensorflow": "0.12.0", "libXext": "1.3.3-intel-2016a", "GMT": "5.2.1-foss-2015g", "molpro": "2010.1-p45-intel", "fds": "6.svn", "pest": "13.0", "PerformanceReports": "6.0.6", "netcdf-cxx": "4.2", "impi": "2017-BETA.ENG", "Lmod": "7.2.2", "libmatheval": "1.1.11-intel-2015b", "GLib": "2.40.0-GCC-4.4.7-system", "QCA": "2.1.0-foss-2015g", "scite": "3.4.3", "Tk": "8.6.5-intel-2017a", "hpg-fastq": "1.0.0", "SnpEff": "4.1_G", "libpthread-stubs": "0.3-intel-2016a", "bzip2": "1.0.6-intel-2017a", "cmake": "2.8.11-mic", "gnuplot": "4.6.5", "gettext": "0.19.6-intel-2017.00", "VDE2": "2.3.2-GCC-4.4.7-system", "Advisor": "2017", "glproto": "1.4.17-intel-2016a", "ORCA": "3_0_3-linux_x86-64", "llvm": "3.6.0", "papi": "5.4.0-mic", "Serf": "1.3.8-foss-2015g", "libxc": "2.2.1-intel-2015b", "libtool": "2.4.6-GCC-6.3.0-2.27", "libffi": "3.2.1-GCC-4.4.7-system", "libmesh": "0.9.3-petsc-3.4.4-icc-impi-mkl-opt", "opencl-sdk": "4.6.0.92", "GMP": "6.1.1-intel-2017.00", "PETSc": "3.6.3-intel-2015b-Python-2.7.11", "OpenMPI": "2.0.2-GCC-6.3.0-2.27", "netCDF": "4.4.1-intel-2017a", "Hypre": "2.10.1-intel-2015b", "renderproto": "0.11-intel-2015b", "oscar-modules": "1.0.3"}}

scripts/check_version.rb

+require 'faraday'
+require 'json'
+
+#print ARGV[0].downcase
+puts "Checking the ver for #{ARGV[0]}"
+
+response = Faraday.get "https://release-monitoring.org/api/projects/?pattern=#{ARGV[0].downcase}"
+
+fedora_j = JSON.parse( response.body )['projects']
+fedora = fedora_j.select { |h| h['name'] == ARGV[0].downcase }.first['version']
+
+response = Faraday.get "https://docs.it4i.cz/devel/52-module-matrix-json-export/modules-matrix.json"
+
+it4i = JSON.parse( response.body )['projects'][ARGV[0]]
+it4i_short = it4i.split('-')[0]
+puts "IT4Innovations: #{it4i_short} (#{it4i})"
+puts "Upstream      : #{fedora}"

scripts/modules-json.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+import csv
+import collections
+import json
+from distutils.version import LooseVersion
+
+def get_data(filename):
+    '''function to read the data form the input csv file to use in the analysis'''
+    reader = [] # Just in case the file open fails
+    with open(filename, 'rb') as f:
+        reader = csv.reader(f,delimiter=',')                      
+        #returns all the data from the csv file in list form
+        #f.close() # May need to close the file when done
+        return list(reader)  # only return the reader when you have finished.
+
+your_list = []
+your_list += get_data('./scripts/modules-anselm.csv')
+your_list += get_data('./scripts/modules-salomon.csv')
+your_list += get_data('./scripts/modules-salomon-uv.csv')
+#print your_list
+
+#a=[["python/2.8.1",1],["python/2.9.1",2],["python/2.8.1",4],["python/3.0.1",4]]
+counts = dict()
+for i in your_list:
+  #print i[0]
+  #print int(i[1])
+  counts[i[0]]=counts.get(i[0], 0) + int(i[1])
+
+#print sorted(counts.items())
+
+c=[
+"---",
+"--A",
+"-S-",
+"-SA",
+"U--",
+"U-A",
+"US-",
+"USA",
+]
+
+software = dict()
+versions = ''
+clusters = ''
+prev = ''
+
+for m,i in sorted(counts.items()):
+  #print m
+  split =  m.split('/')
+  #print split
+  if len(split) > 1:
+    a = split[0]
+    b = split[1]
+    if split[0] <> prev:
+      software[a] = {}
+    software[a][b] = '`' + c[i] + '`'
+    prev = a
+
+
+packages = {}
+
+for m in sorted(software.items(), key=lambda i: i[0].lower()):
+  packages[m[0]]=sorted(m[1], key=LooseVersion)[len(m[1])-1]
+
+data = {'total': len(packages), 'projects': packages } 
+print json.dumps(data)