CMake: move checker from Blender repo

Move this file from build_files/cmake/cmake_consistency_check.py

check_source/check_cmake_consistency.py

+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+# <pep8 compliant>
+
+# Note: this code should be cleaned up / refactored.
+
+import sys
+if sys.version_info.major < 3:
+    print("\nPython3.x needed, found %s.\nAborting!\n" %
+          sys.version.partition(" ")[0])
+    sys.exit(1)
+
+import os
+from os.path import (
+    dirname,
+    join,
+    normpath,
+    splitext,
+)
+
+from check_cmake_consistency_config import (
+    IGNORE_SOURCE,
+    IGNORE_SOURCE_MISSING,
+    IGNORE_CMAKE,
+    UTF8_CHECK,
+    SOURCE_DIR,
+    BUILD_DIR,
+)
+
+from typing import (
+    Callable,
+    Dict,
+    Generator,
+    Iterator,
+    List,
+    Optional,
+    Tuple,
+)
+
+
+global_h = set()
+global_c = set()
+global_refs: Dict[str, List[Tuple[str, int]]] = {}
+
+# Flatten `IGNORE_SOURCE_MISSING` to avoid nested looping.
+IGNORE_SOURCE_MISSING_FLAT = [
+    (k, ignore_path) for k, ig_list in IGNORE_SOURCE_MISSING
+    for ignore_path in ig_list
+]
+
+# Ignore cmake file, path pairs.
+global_ignore_source_missing: Dict[str, List[str]] = {}
+for k, v in IGNORE_SOURCE_MISSING_FLAT:
+    global_ignore_source_missing.setdefault(k, []).append(v)
+del IGNORE_SOURCE_MISSING_FLAT
+
+
+def replace_line(f: str, i: int, text: str, keep_indent: bool = True) -> None:
+    file_handle = open(f, 'r')
+    data = file_handle.readlines()
+    file_handle.close()
+
+    l = data[i]
+    ws = l[:len(l) - len(l.lstrip())]
+
+    data[i] = "%s%s\n" % (ws, text)
+
+    file_handle = open(f, 'w')
+    file_handle.writelines(data)
+    file_handle.close()
+
+
+def source_list(
+        path: str,
+        filename_check: Optional[Callable[[str], bool]] = None,
+) -> Generator[str, None, None]:
+    for dirpath, dirnames, filenames in os.walk(path):
+        # skip '.git'
+        dirnames[:] = [d for d in dirnames if not d.startswith(".")]
+
+        for filename in filenames:
+            if filename_check is None or filename_check(filename):
+                yield os.path.join(dirpath, filename)
+
+
+# extension checking
+def is_cmake(filename: str) -> bool:
+    ext = splitext(filename)[1]
+    return (ext == ".cmake") or (filename == "CMakeLists.txt")
+
+
+def is_c_header(filename: str) -> bool:
+    ext = splitext(filename)[1]
+    return (ext in {".h", ".hpp", ".hxx", ".hh"})
+
+
+def is_c(filename: str) -> bool:
+    ext = splitext(filename)[1]
+    return (ext in {".c", ".cpp", ".cxx", ".m", ".mm", ".rc", ".cc", ".inl", ".metal"})
+
+
+def is_c_any(filename: str) -> bool:
+    return is_c(filename) or is_c_header(filename)
+
+
+def cmake_get_src(f: str) -> None:
+
+    sources_h = []
+    sources_c = []
+
+    filen = open(f, "r", encoding="utf8")
+    it: Optional[Iterator[str]] = iter(filen)
+    found = False
+    i = 0
+    # print(f)
+
+    def is_definition(l: str, f: str, i: int, name: str) -> Tuple[bool, int]:
+        """
+        Return (is_definition, single_line_offset).
+        """
+        if l.startswith("unset("):
+            return False, -1
+
+        single_line_offset = -1
+        name_test = 'set(%s' % name
+        single_line_offset = l.find(name_test)
+        if (single_line_offset != -1) or ('set(' in l and l.endswith(name)):
+            if single_line_offset != -1:
+                single_line_offset += len(name_test)
+            # if len(l.split()) > 1:
+                # raise Exception("strict formatting not kept 'set(%s*' %s:%d" % (name, f, i))
+            if l.endswith(")"):
+                pass
+                while single_line_offset < len(l) and l[single_line_offset] != " ":
+                    single_line_offset += 1
+            else:
+                single_line_offset = -1
+            return True, single_line_offset
+
+        name_test = "list(APPEND %s" % name
+        single_line_offset = l.find(name_test)
+        if (single_line_offset != -1) or ('list(APPEND ' in l and l.endswith(name)):
+            if single_line_offset != -1:
+                single_line_offset += len(name_test)
+            if l.endswith(")"):
+                # raise Exception("strict formatting not kept 'list(APPEND %s...)' on 1 line %s:%d" % (name, f, i))
+                pass
+                while single_line_offset < len(l) and l[single_line_offset] != " ":
+                    single_line_offset += 1
+            else:
+                single_line_offset = -1
+            return True, single_line_offset
+        return False, -1
+
+    while it is not None:
+        context_name = ""
+        while it is not None:
+            i += 1
+            try:
+                l = next(it)
+            except StopIteration:
+                it = None
+                break
+            l = l.strip()
+            if not l.startswith("#"):
+                for var in ("SRC", "INC"):
+                    found, single_line_offset = is_definition(l, f, i, var)
+                    if found:
+                        context_name = var
+                        break
+                if found:
+                    break
+
+        if found:
+            tokens = []
+            if single_line_offset != -1:
+                end = False
+                for w in l[single_line_offset:].split():
+                    if w.startswith("#"):
+                        break
+                    if w.endswith(")"):
+                        w = w[:-1].rstrip()
+                        end = True
+                    tokens.append((w, i))
+                    if end:
+                        break
+                del end
+                if len(tokens) > 1:
+                    print("Expect multi-variable to be split across multiple lines! '%s' %s:%d" % (l, f, i))
+            else:
+                while it is not None:
+                    i += 1
+                    try:
+                        l = next(it)
+                    except StopIteration:
+                        it = None
+                        break
+                    l = l.strip()
+                    if not l.startswith("#"):
+                        # Remove in-line comments.
+                        l = l.split(" # ")[0].rstrip()
+                        if ")" in l:
+                            if l.strip() != ")":
+                                raise Exception("strict formatting not kept '*)' %s:%d" % (f, i))
+                            break
+                        tokens.append((l, i))
+
+            cmake_base = dirname(f)
+            cmake_base_bin = os.path.join(BUILD_DIR, os.path.relpath(cmake_base, SOURCE_DIR))
+
+            # Find known missing sources list (if we have one).
+            f_rel = os.path.relpath(f, SOURCE_DIR)
+            f_rel_key = f_rel
+            if os.sep != "/":
+                f_rel_key = f_rel_key.replace(os.sep, "/")
+            local_ignore_source_missing = global_ignore_source_missing.get(f_rel_key, [])
+
+
+            for l, line_number in tokens:
+                # replace dirs
+                l = l.replace("${CMAKE_SOURCE_DIR}", SOURCE_DIR)
+                l = l.replace("${CMAKE_CURRENT_SOURCE_DIR}", cmake_base)
+                l = l.replace("${CMAKE_CURRENT_BINARY_DIR}", cmake_base_bin)
+                l = l.strip('"')
+
+                if not l:
+                    pass
+                elif l in local_ignore_source_missing:
+                    local_ignore_source_missing.remove(l)
+                elif l.startswith("$"):
+                    if context_name == "SRC":
+                        # assume if it ends with context_name we know about it
+                        if not l.split("}")[0].endswith(context_name):
+                            print("Can't use var '%s' %s:%d" % (l, f, line_number))
+                elif len(l.split()) > 1:
+                    raise Exception("Multi-line define '%s' %s:%d" % (l, f, line_number))
+                else:
+                    new_file = normpath(join(cmake_base, l))
+
+                    if context_name == "SRC":
+                        if is_c_header(new_file):
+                            sources_h.append(new_file)
+                            global_refs.setdefault(new_file, []).append((f, line_number))
+                        elif is_c(new_file):
+                            sources_c.append(new_file)
+                            global_refs.setdefault(new_file, []).append((f, line_number))
+                        elif l in {"PARENT_SCOPE", }:
+                            # cmake var, ignore
+                            pass
+                        elif new_file.endswith(".list"):
+                            pass
+                        elif new_file.endswith(".def"):
+                            pass
+                        elif new_file.endswith(".cl"):  # opencl
+                            pass
+                        elif new_file.endswith(".cu"):  # cuda
+                            pass
+                        elif new_file.endswith(".osl"):  # open shading language
+                            pass
+                        elif new_file.endswith(".glsl"):
+                            pass
+                        else:
+                            raise Exception("unknown file type - not c or h %s -> %s" % (f, new_file))
+
+                    elif context_name == "INC":
+                        if new_file.startswith(BUILD_DIR):
+                            # assume generated path
+                            pass
+                        elif os.path.isdir(new_file):
+                            new_path_rel = os.path.relpath(new_file, cmake_base)
+
+                            if new_path_rel != l:
+                                print("overly relative path:\n  %s:%d\n  %s\n  %s" % (f, line_number, l, new_path_rel))
+
+                                # # Save time. just replace the line
+                                # replace_line(f, line_number - 1, new_path_rel)
+
+                        else:
+                            raise Exception("non existent include %s:%d -> %s" % (f, line_number, new_file))
+
+                    # print(new_file)
+
+            global_h.update(set(sources_h))
+            global_c.update(set(sources_c))
+            '''
+            if not sources_h and not sources_c:
+                raise Exception("No sources %s" % f)
+
+            sources_h_fs = list(source_list(cmake_base, is_c_header))
+            sources_c_fs = list(source_list(cmake_base, is_c))
+            '''
+            # find missing C files:
+            '''
+            for ff in sources_c_fs:
+                if ff not in sources_c:
+                    print("  missing: " + ff)
+            '''
+
+            # reset
+            del sources_h[:]
+            del sources_c[:]
+
+    filen.close()
+
+
+def is_ignore_source(f: str, ignore_used: List[bool]) -> bool:
+    for index, ignore_path in enumerate(IGNORE_SOURCE):
+        if ignore_path in f:
+            ignore_used[index] = True
+            return True
+    return False
+
+
+def is_ignore_cmake(f: str, ignore_used: List[bool]) -> bool:
+    for index, ignore_path in enumerate(IGNORE_CMAKE):
+        if ignore_path in f:
+            ignore_used[index] = True
+            return True
+    return False
+
+
+def main() -> None:
+
+    print("Scanning:", SOURCE_DIR)
+
+    ignore_used_source = [False] * len(IGNORE_SOURCE)
+    ignore_used_cmake = [False] * len(IGNORE_CMAKE)
+
+    for cmake in source_list(SOURCE_DIR, is_cmake):
+        if not is_ignore_cmake(cmake, ignore_used_cmake):
+            cmake_get_src(cmake)
+
+    # First do stupid check, do these files exist?
+    print("\nChecking for missing references:")
+    is_err = False
+    errs = []
+    for f in (global_h | global_c):
+        if f.startswith(BUILD_DIR):
+            continue
+
+        if not os.path.exists(f):
+            refs = global_refs[f]
+            if refs:
+                for cf, i in refs:
+                    errs.append((cf, i))
+            else:
+                raise Exception("CMake references missing, internal error, aborting!")
+            is_err = True
+
+    errs.sort()
+    errs.reverse()
+    for cf, i in errs:
+        print("%s:%d" % (cf, i))
+        # Write a 'sed' script, useful if we get a lot of these
+        # print("sed '%dd' '%s' > '%s.tmp' ; mv '%s.tmp' '%s'" % (i, cf, cf, cf, cf))
+
+    if is_err:
+        raise Exception("CMake references missing files, aborting!")
+    del is_err
+    del errs
+
+    # now check on files not accounted for.
+    print("\nC/C++ Files CMake does not know about...")
+    for cf in sorted(source_list(SOURCE_DIR, is_c)):
+        if not is_ignore_source(cf, ignore_used_source):
+            if cf not in global_c:
+                print("missing_c: ", cf)
+
+            # Check if automake builds a corresponding .o file.
+            '''
+            if cf in global_c:
+                out1 = os.path.splitext(cf)[0] + ".o"
+                out2 = os.path.splitext(cf)[0] + ".Po"
+                out2_dir, out2_file = out2 = os.path.split(out2)
+                out2 = os.path.join(out2_dir, ".deps", out2_file)
+                if not os.path.exists(out1) and not os.path.exists(out2):
+                    print("bad_c: ", cf)
+            '''
+
+    print("\nC/C++ Headers CMake does not know about...")
+    for hf in sorted(source_list(SOURCE_DIR, is_c_header)):
+        if not is_ignore_source(hf, ignore_used_source):
+            if hf not in global_h:
+                print("missing_h: ", hf)
+
+    if UTF8_CHECK:
+        # test encoding
+        import traceback
+        for files in (global_c, global_h):
+            for f in sorted(files):
+                if os.path.exists(f):
+                    # ignore outside of our source tree
+                    if "extern" not in f:
+                        i = 1
+                        try:
+                            for _ in open(f, "r", encoding="utf8"):
+                                i += 1
+                        except UnicodeDecodeError:
+                            print("Non utf8: %s:%d" % (f, i))
+                            if i > 1:
+                                traceback.print_exc()
+
+    # Check ignores aren't stale
+    print("\nCheck for unused 'IGNORE_SOURCE' paths...")
+    for index, ignore_path in enumerate(IGNORE_SOURCE):
+        if not ignore_used_source[index]:
+            print("unused ignore: %r" % ignore_path)
+
+    # Check ignores aren't stale
+    print("\nCheck for unused 'IGNORE_SOURCE_MISSING' paths...")
+    for k, v in sorted(global_ignore_source_missing.items()):
+        for ignore_path in v:
+            print("unused ignore: %r -> %r" % (ignore_path, k))
+
+    # Check ignores aren't stale
+    print("\nCheck for unused 'IGNORE_CMAKE' paths...")
+    for index, ignore_path in enumerate(IGNORE_CMAKE):
+        if not ignore_used_cmake[index]:
+            print("unused ignore: %r" % ignore_path)
+
+
+if __name__ == "__main__":
+    main()

check_source/check_cmake_consistency_config.py

+# SPDX-License-Identifier: GPL-2.0-or-later
+import os
+
+IGNORE_SOURCE = (
+    "/test/",
+    "/tests/gtests/",
+    "/release/",
+
+    # specific source files
+    "extern/audaspace/",
+
+    # Use for `WIN32` only.
+    "source/creator/blender_launcher_win32.c",
+
+    # specific source files
+    "extern/bullet2/src/BulletCollision/CollisionDispatch/btBox2dBox2dCollisionAlgorithm.cpp",
+    "extern/bullet2/src/BulletCollision/CollisionDispatch/btConvex2dConvex2dAlgorithm.cpp",
+    "extern/bullet2/src/BulletCollision/CollisionDispatch/btInternalEdgeUtility.cpp",
+    "extern/bullet2/src/BulletCollision/CollisionShapes/btBox2dShape.cpp",
+    "extern/bullet2/src/BulletCollision/CollisionShapes/btConvex2dShape.cpp",
+    "extern/bullet2/src/BulletDynamics/Character/btKinematicCharacterController.cpp",
+    "extern/bullet2/src/BulletDynamics/ConstraintSolver/btHinge2Constraint.cpp",
+    "extern/bullet2/src/BulletDynamics/ConstraintSolver/btUniversalConstraint.cpp",
+
+    "doc/doxygen/doxygen.extern.h",
+    "doc/doxygen/doxygen.intern.h",
+    "doc/doxygen/doxygen.main.h",
+    "doc/doxygen/doxygen.source.h",
+    "extern/bullet2/src/BulletCollision/CollisionDispatch/btBox2dBox2dCollisionAlgorithm.h",
+    "extern/bullet2/src/BulletCollision/CollisionDispatch/btConvex2dConvex2dAlgorithm.h",
+    "extern/bullet2/src/BulletCollision/CollisionDispatch/btInternalEdgeUtility.h",
+    "extern/bullet2/src/BulletCollision/CollisionShapes/btBox2dShape.h",
+    "extern/bullet2/src/BulletCollision/CollisionShapes/btConvex2dShape.h",
+    "extern/bullet2/src/BulletDynamics/Character/btKinematicCharacterController.h",
+    "extern/bullet2/src/BulletDynamics/ConstraintSolver/btHinge2Constraint.h",
+    "extern/bullet2/src/BulletDynamics/ConstraintSolver/btUniversalConstraint.h",
+
+    "build_files/build_environment/patches/config_gmpxx.h",
+)
+
+# Ignore cmake file, path pairs.
+IGNORE_SOURCE_MISSING = (
+    (   # Use for `WITH_NANOVDB`.
+        "intern/cycles/kernel/CMakeLists.txt", (
+            "nanovdb/util/CSampleFromVoxels.h",
+            "nanovdb/util/SampleFromVoxels.h",
+            "nanovdb/NanoVDB.h",
+            "nanovdb/CNanoVDB.h",
+        ),
+    ),
+)
+
+IGNORE_CMAKE = (
+    "extern/audaspace/CMakeLists.txt",
+)
+
+UTF8_CHECK = True
+
+SOURCE_DIR = os.path.normpath(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")))
+
+# doesn't have to exist, just use as reference
+BUILD_DIR = os.path.normpath(os.path.abspath(os.path.normpath(os.path.join(SOURCE_DIR, "..", "build"))))