Commit 0bf62364 authored by Campbell Barton's avatar Campbell Barton
Browse files

code_clean: rewrite header_clean.py, make it part of code_clean.py

parent 5e2423d8
......@@ -69,6 +69,15 @@ def line_from_span(text, start, end):
return text[start:end]
def files_recursive_with_ext(path, ext):
for dirpath, dirnames, filenames in os.walk(path):
# skip '.git' and other dot-files.
dirnames[:] = [d for d in dirnames if not d.startswith(".")]
for filename in filenames:
if filename.endswith(ext):
yield os.path.join(dirpath, filename)
# -----------------------------------------------------------------------------
# Execution Wrappers
......@@ -597,6 +606,81 @@ class edit_generators:
return edits
class header_clean(EditGenerator):
"""
Clean headers, ensuring that the headers removed are not used directly or indirectly.
Note that the `CFLAGS` should be set so missing prototypes error instead of warn:
With GCC: `-Werror=missing-prototypes`
"""
@staticmethod
def _header_guard_from_filename(f):
return '__%s__' % os.path.basename(f).replace('.', '_').upper()
@classmethod
def setup(cls):
# For each file replace pragma once with old-style header guard.
# This is needed so we can remove the header with the knowledge the source file didn't use it indirectly.
files = []
shared_edit_data = {
'files': files,
}
for f in files_recursive_with_ext(
os.path.join(SOURCE_DIR, 'source'),
('.h', '.hh', '.inl', '.hpp', '.hxx'),
):
with open(f, 'r', encoding='utf-8') as fh:
data = fh.read()
for match in re.finditer(r'^[ \t]*#\s*(pragma\s+once)\b', data, flags=re.MULTILINE):
header_guard = cls._header_guard_from_filename(f)
start, end = match.span()
src = data[start:end]
dst = (
'#ifndef %s\n#define %s' % (header_guard, header_guard)
)
dst_footer = '\n#endif /* %s */\n' % header_guard
files.append((f, src, dst, dst_footer))
data = data[:start] + dst + data[end:] + dst_footer
with open(f, 'w', encoding='utf-8') as fh:
fh.write(data)
break
return shared_edit_data
@staticmethod
def teardown(shared_edit_data):
files = shared_edit_data['files']
for f, src, dst, dst_footer in files:
with open(f, 'r', encoding='utf-8') as fh:
data = fh.read()
data = data.replace(
dst, src,
).replace(
dst_footer, '',
)
with open(f, 'w', encoding='utf-8') as fh:
fh.write(data)
@classmethod
def edit_list_from_file(cls, source, data, _shared_edit_data):
edits = []
# Remove include.
for match in re.finditer(r"^(([ \t]*#\s*include\s+\")([^\"]+)(\"[^\n]*\n))", data, flags=re.MULTILINE):
header_name = match.group(3)
header_guard = cls._header_guard_from_filename(header_name)
edits.append(Edit(
span=match.span(),
content='', # Remove the header.
content_fail='%s__ALWAYS_FAIL__%s' % (match.group(2), match.group(4)),
extra_build_args=('-D' + header_guard),
))
return edits
def test_edit(source, output, output_bytes, build_args, data, data_test, keep_edits=True, expect_failure=False):
"""
......
#!/usr/bin/env python3
# ##### BEGIN GPL LICENSE BLOCK #####
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####
# <pep8-80 compliant>
"""
Example:
./source/tools/utils/header_clean.py /src/cmake_debug --match ".*/editmesh_.*"
Note: currently this is limited to paths in "source/blender",
we could change this if it's needed.
Ensures headers are NOT removed:
- They aren't used in the current build configuration.
- They are needed but happen to be indirectly included by another header.
- They use '#include <...>', instead of quotes (keep system headers).
"""
import os
import sys
import subprocess
import re
# Copied form elsewhere...
def cmake_cache_var(cmake_dir, var):
cache_file = open(os.path.join(cmake_dir, "CMakeCache.txt"), encoding='utf-8')
lines = [l_strip for l in cache_file for l_strip in (l.strip(),)
if l_strip if not l_strip.startswith("//") if not l_strip.startswith("#")]
cache_file.close()
for l in lines:
if l.split(":")[0] == var:
return l.split("=", 1)[-1]
return None
# RE_CFILE_SEARCH = re.search('<title>(.*)</title>', html, re.IGNORECASE)
RE_CFILE_SEARCH = re.compile(r"\s\-c\s([\S]+)")
def process_commands(cmake_dir, data):
compiler = cmake_cache_var(cmake_dir, "CMAKE_C_COMPILER") # could do CXX too
file_args = []
for l in data:
if compiler in l:
# extract -c FILE
# c_file = l.split(" -c ", 1)[1].split()[0]
c_file_search = re.search(RE_CFILE_SEARCH, l)
if c_file_search is not None:
c_file = c_file_search.group(1)
file_args.append((c_file, l))
else:
# could print, NO C FILE FOUND?
pass
print(c_file)
file_args.sort()
return file_args
def find_build_args_ninja(build_dir):
cmake_dir = build_dir
make_exe = "ninja"
process = subprocess.Popen(
[make_exe, "-t", "commands"],
stdout=subprocess.PIPE,
cwd=build_dir,
)
while process.poll():
time.sleep(1)
out = process.stdout.read()
process.stdout.close()
# print("done!", len(out), "bytes")
data = out.decode("utf-8", errors="ignore").split("\n")
return process_commands(cmake_dir, data)
def find_build_args_make(build_dir):
make_exe = "make"
process = subprocess.Popen(
[make_exe, "--always-make", "--dry-run", "--keep-going", "VERBOSE=1"],
stdout=subprocess.PIPE,
cwd=build_dir,
)
while process.poll():
time.sleep(1)
out = process.stdout.read()
process.stdout.close()
# print("done!", len(out), "bytes")
data = out.decode("utf-8", errors="ignore").split("\n")
return process_commands(cmake_dir, data)
def wash_source_const(arg_group):
(source, build_args) = arg_group
# Here is where the fun happens, try make changes and see what happens
# 'char *' -> 'const char *'
lines = open(source, 'r', encoding='utf-8').read().split("\n")
def write_lines(lines):
with open(source, 'w', encoding='utf-8') as f:
f.write("\n".join(lines))
re_c = re.compile(r"([\s|, |\(])(\b[A-Za-z0-9_]+)(\s+[a-zA-Z0-9_]+\[\d+\])")
for i, l in enumerate(lines):
l_strip = l.strip()
if len(l_strip) == len(l):
continue
# -------- HACKY but works OK
if ";" in l_strip:
continue
'''
a = l_strip.find("=")
if a == -1:
a = 10000
b = l_strip.find(";")
if b == -1:
b == 10000
if " *" not in l_strip[:min(a, b)]:
continue
'''
# for t in ("bool", "char", "short", "int", "long", "float", "double"):
# if t in l_strip:
changed = True
while changed:
changed = False
l_prev = l
t = r"[A-Za-z0-9_]+"
l_new = re.sub(re_c, r"\1const \2\3", l, 1)
if l_new == l_prev:
break
if "const const" in l_new:
break
l_test = re.sub(re_c, r"\1TESTME \2\3", l, 1)
# l = l.replace(" %s *" % t, " const %s *" % t, 1)
# l_test = l.replace(" %s " % t, "TESTME", 1)
print(source, i + 1, l)
# first check this is even getting compiled
lines[i] = l_test
write_lines(lines)
# ensure this fails!, else we may be in an `#if 0` block
ret = os.system(build_args)
if ret != 0:
lines[i] = l_new
write_lines(lines)
ret = os.system(build_args)
if ret != 0:
lines[i] = l_prev
write_lines(lines)
else:
print("success!")
l = l_new
changed = True
else:
lines[i] = l_prev
write_lines(lines)
# print("building:", c)
def wash_source_replace(arg_group):
(source, build_args) = arg_group
# Here is where the fun happens, try make changes and see what happens
# 'char *' -> 'const char *'
lines = open(source, 'r', encoding='utf-8').read().split("\n")
def write_lines(lines):
with open(source, 'w', encoding='utf-8') as f:
f.write("\n".join(lines))
str_src = "CTX_wm_screen(C)"
str_dst = "sc"
for i, l in enumerate(lines):
if str_src not in l:
continue
l_prev = l
l_new = l_prev.replace(str_src, str_dst)
if l_new == l_prev:
continue
# avoid 'scene = scene'
if (str_dst + "=" + str_dst) in l_new.replace(" ", ""):
continue
l_test = l_prev.replace(str_src, "TESTME")
print(source, i + 1, l)
# first check this is even getting compiled
lines[i] = l_test
write_lines(lines)
# ensure this fails!, else we may be in an `#if 0` block
ret = os.system(build_args)
if ret != 0:
lines[i] = l_new
write_lines(lines)
ret = os.system(build_args)
if ret != 0:
lines[i] = l_prev
write_lines(lines)
else:
print("success!")
l = l_new
changed = True
else:
lines[i] = l_prev
write_lines(lines)
# Never remove these headers
HEADER_BLACKLIST = {
"BLI_strict_flags.h",
"BLI_utildefines.h",
}
def wash_source_include(arg_group):
(source, build_args, regex_list_header) = arg_group
# Here is where the fun happens, try make changes and see what happens
# 'char *' -> 'const char *'
lines = open(source, 'r', encoding='utf-8').read().split("\n")
def write_lines(lines):
with open(source, 'w', encoding='utf-8') as f:
f.write("\n".join(lines))
re_c = re.compile(r"\s*#\s*include\s+\"([a-zA-Z0-9_\-\.]+)\"")
# Make multiple configurations (debug/non-debug).
build_args_all = [build_args]
build_args_other = re.sub("($|\\s)(-DNDEBUG)\\b", "\\1-DDEBUG", build_args)
if build_args_other != build_args:
build_args_all.append(build_args_other)
build_args_other = re.sub("($|\\s)(-DDEBUG)\\b", "\\1-DNDEBUG", build_args)
if build_args_other != build_args:
build_args_all.append(build_args_other)
del build_args, build_args_other
i = 0
while i < len(lines):
l = lines[i]
# if "\"BKE_" not in l:
# i += 1
# continue
m = re.match(re_c, l)
if m is None:
i += 1
continue
l_header = m.group(1)
del m
if l_header in HEADER_BLACKLIST:
i += 1
continue
if regex_list_header:
if not any(regex.match(l_header) for regex in regex_list_header):
i += 1
continue
l_prev = l
l_new = ""
# this must fail!, else if '#if 0' or commented
l_test = "#include <THISISMISSING>"
print(source, i + 1, l)
# first check this is even getting compiled
lines[i] = l_test
write_lines(lines)
# ensure this fails!, else we may be in an `#if 0` block
for build_args in build_args_all:
ret = os.system(build_args)
if ret != 0:
break
if ret != 0:
lines[i] = l_new
# redefine to cause error
# if we're already including indirectly
l_guard = l_header.upper().replace(".", "_").replace("-", "_")
l_bad_guard = "#define __%s__" % l_guard
del l_guard
# add, remove bad definition of include guard
# we the include is indirect, this will fail.
lines.insert(0, l_bad_guard) # add guard
write_lines(lines)
lines.pop(0) # remove guard
del l_bad_guard
for build_args in build_args_all:
ret = os.system(build_args + " -Wno-unused-macros -Werror=missing-prototypes")
if ret != 0:
break
if ret != 0:
lines[i] = l_prev
write_lines(lines)
else:
print("success!")
l = l_new
changed = True
del lines[i]
write_lines(lines)
i -= 1
else:
lines[i] = l_prev
write_lines(lines)
i += 1
def header_clean_all(build_dir, regex_list, regex_list_header):
# currently only supports ninja or makefiles
build_file_ninja = os.path.join(build_dir, "build.ninja")
build_file_make = os.path.join(build_dir, "Makefile")
if os.path.exists(build_file_ninja):
print("Using Ninja")
args = find_build_args_ninja(build_dir)
elif os.path.exists(build_file_make):
print("Using Make")
args = find_build_args_make(build_dir)
else:
sys.stderr.write(
"Can't find Ninja or Makefile (%r or %r), aborting" %
(build_file_ninja, build_file_make)
)
return
# needed for when arguments are referenced relatively
os.chdir(build_dir)
# Weak, but we probably don't want to handle extern.
# this limit could be removed.
source_path = os.path.join("blender", "source")
def test_path(c):
index = c.rfind(source_path)
if index == -1:
return False
# Remove first part of the path, we don't want to match
# against paths in Blender's repo.
c_strip = c[index:]
for regex in regex_list:
if regex.match(c_strip) is not None:
return True
return False
if 1:
args = [(c, build_args, regex_list_header) for (c, build_args) in args if test_path(c)]
import multiprocessing
job_total = multiprocessing.cpu_count()
pool = multiprocessing.Pool(processes=job_total * 2)
pool.map(wash_source_include, args)
else:
# now we have commands
for i, (c, build_args) in enumerate(args):
if (source_path in c) and ("rna_" not in c):
wash_source_include((c, build_args, regex_list_header))
def create_parser():
import argparse
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawTextHelpFormatter,
)
parser.add_argument(
"build_dir",
help="list of files or directories to check",
)
parser.add_argument(
"--match",
nargs='+',
required=True,
metavar="REGEX",
help="Match file paths against this expression",
)
parser.add_argument(
"--match-header",
nargs='+',
required=False,
metavar="REGEX",
default=(),
help="Match file paths against this expression",
)
return parser
def main():
parser = create_parser()
args = parser.parse_args()
build_dir = args.build_dir
regex_list = []
regex_list_header = []
for i, expr in enumerate(args.match):
try:
regex_list.append(re.compile(expr))
except Exception as ex:
print(f"Error in expression: {expr}\n {ex}")
return 1
for i, expr in enumerate(args.match_header):
try:
regex_list_header.append(re.compile(expr))
except Exception as ex:
print(f"Error in expression: {expr}\n {ex}")
return 1
return header_clean_all(build_dir, regex_list, regex_list_header)
if __name__ == "__main__":
sys.exit(main())
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment