August 2019 - tor-commits - lists.torproject.org

[tor/master] checkIncludes: Separate file-handling from rule-handling
by dgoulet＠torproject.org 21 Aug '19

21 Aug '19

commit 47d9bcfef8cff23225850545400f60a93fe18f49 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Aug 5 11:58:12 2019 -0400 checkIncludes: Separate file-handling from rule-handling This is our shift from directory-at-a-time processing to file-at-a-time processing. --- scripts/maint/checkIncludes.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py index c4e77c71e..9daaf1363 100755 --- a/scripts/maint/checkIncludes.py +++ b/scripts/maint/checkIncludes.py @@ -126,9 +126,14 @@ class Rules(object): include_rules_cache = {} def load_include_rules(fname): - """ Read a rules file from 'fname', and return it as a Rules object. """ + """ Read a rules file from 'fname', and return it as a Rules object. + Return 'None' if fname does not exist. + """ if fname in include_rules_cache: return include_rules_cache[fname] + if not os.path.exists(fname): + include_rules_cache[fname] = None + return None result = Rules(os.path.split(fname)[0]) with open_file(fname) as f: for line in f: @@ -139,6 +144,11 @@ def load_include_rules(fname): include_rules_cache[fname] = result return result +def get_all_include_rules(): + return [ rules for (fname,rules) in + sorted(include_rules_cache.items()) + if rules is not None ] + def remove_self_edges(graph): """Takes a directed graph in as an adjacency mapping (a mapping from node to a list of the nodes to which it connects). @@ -187,18 +197,12 @@ if __name__ == '__main__': list_unused = False log_sorted_levels = False - uses_dirs = { } - for dirpath, dirnames, fnames in os.walk("src"): - if ".may_include" in fnames: - rules = load_include_rules(os.path.join(dirpath, RULES_FNAME)) - for fname in fnames: - if fname_is_c(fname): + for fname in fnames: + if fname_is_c(fname): + rules = load_include_rules(os.path.join(dirpath, RULES_FNAME)) + if rules is not None: rules.applyToFile(os.path.join(dirpath,fname)) - if list_unused: - rules.noteUnusedRules() - - uses_dirs[rules.incpath] = rules.getAllowedDirectories() if trouble: err( @@ -206,6 +210,14 @@ if __name__ == '__main__': files in its enclosing directory.""".format(RULES_FNAME)) sys.exit(1) + if list_unused: + for rules in get_all_include_rules(): + rules.noteUnusedRules() + + uses_dirs = { } + for rules in get_all_include_rules(): + uses_dirs[rules.incpath] = rules.getAllowedDirectories() + remove_self_edges(uses_dirs) all_levels = toposort(uses_dirs)

1 0

[tor/master] Move the executable part of checkIncludes.py inside an if block.
by dgoulet＠torproject.org 21 Aug '19

21 Aug '19

commit 475749351dcf7a89ad921f120a6daac80310edde Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Aug 5 11:30:22 2019 -0400 Move the executable part of checkIncludes.py inside an if block. I'll want to make this block into a series of functions in a subsequent commit, but I'm doing this separately to get the indentation change out of the way. This branch will end up with making checkIncludes.py an integrated part of practracker, for ticket 31176. --- scripts/maint/checkIncludes.py | 91 +++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 45 deletions(-) diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py index ec9350b9b..a6672b097 100755 --- a/scripts/maint/checkIncludes.py +++ b/scripts/maint/checkIncludes.py @@ -134,48 +134,49 @@ def load_include_rules(fname): result.addPattern(line) return result -list_unused = False -log_sorted_levels = False - -uses_dirs = { } - -for dirpath, dirnames, fnames in os.walk("src"): - if ".may_include" in fnames: - rules = load_include_rules(os.path.join(dirpath, RULES_FNAME)) - for fname in fnames: - if fname_is_c(fname): - rules.applyToFile(os.path.join(dirpath,fname)) - if list_unused: - rules.noteUnusedRules() - - uses_dirs[rules.incpath] = rules.getAllowedDirectories() - -if trouble: - err( -"""To change which includes are allowed in a C file, edit the {} -files in its enclosing directory.""".format(RULES_FNAME)) - sys.exit(1) - -all_levels = [] - -n = 0 -while uses_dirs: - n += 0 - cur_level = [] - for k in list(uses_dirs): - uses_dirs[k] = [ d for d in uses_dirs[k] - if (d in uses_dirs and d != k)] - if uses_dirs[k] == []: - cur_level.append(k) - for k in cur_level: - del uses_dirs[k] - n += 1 - if cur_level and log_sorted_levels: - print(n, cur_level) - if n > 100: - break - -if uses_dirs: - print("There are circular .may_include dependencies in here somewhere:", - uses_dirs) - sys.exit(1) +if __name__ == '__main__': + list_unused = False + log_sorted_levels = False + + uses_dirs = { } + + for dirpath, dirnames, fnames in os.walk("src"): + if ".may_include" in fnames: + rules = load_include_rules(os.path.join(dirpath, RULES_FNAME)) + for fname in fnames: + if fname_is_c(fname): + rules.applyToFile(os.path.join(dirpath,fname)) + if list_unused: + rules.noteUnusedRules() + + uses_dirs[rules.incpath] = rules.getAllowedDirectories() + + if trouble: + err( + """To change which includes are allowed in a C file, edit the {} + files in its enclosing directory.""".format(RULES_FNAME)) + sys.exit(1) + + all_levels = [] + + n = 0 + while uses_dirs: + n += 0 + cur_level = [] + for k in list(uses_dirs): + uses_dirs[k] = [ d for d in uses_dirs[k] + if (d in uses_dirs and d != k)] + if uses_dirs[k] == []: + cur_level.append(k) + for k in cur_level: + del uses_dirs[k] + n += 1 + if cur_level and log_sorted_levels: + print(n, cur_level) + if n > 100: + break + + if uses_dirs: + print("There are circular .may_include dependencies in here somewhere:", + uses_dirs) + sys.exit(1)

1 0

[tor/master] checkIncludes: introduce rules-file caching.
by dgoulet＠torproject.org 21 Aug '19

21 Aug '19

commit 3f35ac772b0428e77c93968fdfc6f4add1c9f15d Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Aug 5 11:35:13 2019 -0400 checkIncludes: introduce rules-file caching. We'll want this so that we can have each file evaluated independently, rather than a directory at a time. --- scripts/maint/checkIncludes.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py index a6672b097..a67397bfa 100755 --- a/scripts/maint/checkIncludes.py +++ b/scripts/maint/checkIncludes.py @@ -123,8 +123,12 @@ class Rules(object): return allowed +include_rules_cache = {} + def load_include_rules(fname): """ Read a rules file from 'fname', and return it as a Rules object. """ + if fname in include_rules_cache: + return include_rules_cache[fname] result = Rules(os.path.split(fname)[0]) with open_file(fname) as f: for line in f: @@ -132,6 +136,7 @@ def load_include_rules(fname): if line.startswith("#") or not line: continue result.addPattern(line) + include_rules_cache[fname] = result return result if __name__ == '__main__':

1 0

[tor/master] checkIncludes.py: extract topological sort code
by dgoulet＠torproject.org 21 Aug '19

21 Aug '19

commit 65a69f861e192ae77c7edafe46bc2674bc39b9ea Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Aug 5 11:49:52 2019 -0400 checkIncludes.py: extract topological sort code Our topological sort code really deserves a function of its own. Additionally, don't print from inside the topological sort code: instead, return a result, and let the caller print it. --- scripts/maint/checkIncludes.py | 67 +++++++++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 17 deletions(-) diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py index a67397bfa..c4e77c71e 100755 --- a/scripts/maint/checkIncludes.py +++ b/scripts/maint/checkIncludes.py @@ -139,6 +139,50 @@ def load_include_rules(fname): include_rules_cache[fname] = result return result +def remove_self_edges(graph): + """Takes a directed graph in as an adjacency mapping (a mapping from + node to a list of the nodes to which it connects). + + Remove all edges from a node to itself.""" + + for k in list(graph): + graph[k] = [ d for d in graph[k] if d != k ] + +def toposort(graph, limit=100): + """Takes a directed graph in as an adjacency mapping (a mapping from + node to a list of the nodes to which it connects). Tries to + perform a topological sort on the graph, arranging the nodes into + "levels", such that every member of each level is only reachable + by members of later levels. + + Returns a list of the members of each level. + + Modifies the input graph, removing every member that could be + sorted. If the graph does not become empty, then it contains a + cycle. + + "limit" is the max depth of the graph after which we give up trying + to sort it and conclude we have a cycle. + """ + all_levels = [] + + n = 0 + while graph: + n += 0 + cur_level = [] + all_levels.append(cur_level) + for k in list(graph): + graph[k] = [ d for d in graph[k] if d in graph ] + if graph[k] == []: + cur_level.append(k) + for k in cur_level: + del graph[k] + n += 1 + if n > limit: + break + + return all_levels + if __name__ == '__main__': list_unused = False log_sorted_levels = False @@ -162,24 +206,13 @@ if __name__ == '__main__': files in its enclosing directory.""".format(RULES_FNAME)) sys.exit(1) - all_levels = [] + remove_self_edges(uses_dirs) + all_levels = toposort(uses_dirs) - n = 0 - while uses_dirs: - n += 0 - cur_level = [] - for k in list(uses_dirs): - uses_dirs[k] = [ d for d in uses_dirs[k] - if (d in uses_dirs and d != k)] - if uses_dirs[k] == []: - cur_level.append(k) - for k in cur_level: - del uses_dirs[k] - n += 1 - if cur_level and log_sorted_levels: - print(n, cur_level) - if n > 100: - break + if log_sorted_levels: + for (n, cur_level) in enumerate(all_levels): + if cur_level: + print(n, cur_level) if uses_dirs: print("There are circular .may_include dependencies in here somewhere:",

1 0

[tor/master] checkIncludes: refactor to use error-iteration style
by dgoulet＠torproject.org 21 Aug '19

21 Aug '19

commit 3f4e89a7abb2a9027d83da48c84227a190f8b31d Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Aug 5 12:25:41 2019 -0400 checkIncludes: refactor to use error-iteration style This makes checkIncludes match practracker more closely, and lets us eliminate a global. --- scripts/maint/checkIncludes.py | 50 +++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py index 9daaf1363..c398dc7a5 100755 --- a/scripts/maint/checkIncludes.py +++ b/scripts/maint/checkIncludes.py @@ -23,9 +23,6 @@ import os import re import sys -# Global: Have there been any errors? -trouble = False - if sys.version_info[0] <= 2: def open_file(fname): return open(fname, 'r') @@ -36,13 +33,6 @@ else: def warn(msg): print(msg, file=sys.stderr) -def err(msg): - """ Declare that an error has happened, and remember that there has - been an error. """ - global trouble - trouble = True - print(msg, file=sys.stderr) - def fname_is_c(fname): """ Return true iff 'fname' is the name of a file that we should search for possibly disallowed #include directives. """ @@ -65,6 +55,14 @@ def pattern_is_normal(s): return True return False +class Error(object): + def __init__(self, location, msg): + self.location = location + self.msg = msg + + def __str__(self): + return "{} at {}".format(self.msg, self.location) + class Rules(object): """ A 'Rules' object is the parsed version of a .may_include file. """ def __init__(self, dirpath): @@ -88,7 +86,7 @@ class Rules(object): return True return False - def applyToLines(self, lines, context=""): + def applyToLines(self, lines, loc_prefix=""): lineno = 0 for line in lines: lineno += 1 @@ -96,18 +94,19 @@ class Rules(object): if m: include = m.group(1) if not self.includeOk(include): - err("Forbidden include of {} on line {}{}".format( - include, lineno, context)) + yield Error("{}{}".format(loc_prefix,str(lineno)), + "Forbidden include of {}".format(include)) def applyToFile(self, fname): with open_file(fname) as f: #print(fname) - self.applyToLines(iter(f), " of {}".format(fname)) + for error in self.applyToLines(iter(f), "{}:".format(fname)): + yield error def noteUnusedRules(self): for p in self.patterns: if p not in self.usedPatterns: - print("Pattern {} in {} was never used.".format(p, self.dirpath)) + warn("Pattern {} in {} was never used.".format(p, self.dirpath)) def getAllowedDirectories(self): allowed = [] @@ -145,6 +144,8 @@ def load_include_rules(fname): return result def get_all_include_rules(): + """Return a list of all the Rules objects we have loaded so far, + sorted by their directory names.""" return [ rules for (fname,rules) in sorted(include_rules_cache.items()) if rules is not None ] @@ -193,16 +194,29 @@ def toposort(graph, limit=100): return all_levels +def consider_include_rules(fname): + dirpath = os.path.split(fname)[0] + rules_fname = os.path.join(dirpath, RULES_FNAME) + rules = load_include_rules(os.path.join(dirpath, RULES_FNAME)) + if rules is None: + return + + for err in rules.applyToFile(fname): + yield err + if __name__ == '__main__': list_unused = False log_sorted_levels = False + trouble = False + for dirpath, dirnames, fnames in os.walk("src"): for fname in fnames: if fname_is_c(fname): - rules = load_include_rules(os.path.join(dirpath, RULES_FNAME)) - if rules is not None: - rules.applyToFile(os.path.join(dirpath,fname)) + fullpath = os.path.join(dirpath,fname) + for err in consider_include_rules(fullpath): + print(err, file=sys.stderr) + trouble = True if trouble: err(

1 0

[tor/master] Update pre-commit hook to find checkIncludes in its new location
by dgoulet＠torproject.org 21 Aug '19

21 Aug '19

commit 9abbde2c244f40fcf06c34cd72c360a9770fcd7c Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Aug 5 14:11:51 2019 -0400 Update pre-commit hook to find checkIncludes in its new location Also add a temporary script to redirect the hook, if people don't upgrade for a bit. --- scripts/git/pre-commit.git-hook | 4 ++-- scripts/maint/checkIncludes.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/scripts/git/pre-commit.git-hook b/scripts/git/pre-commit.git-hook index 2a2983719..7c7cf8857 100755 --- a/scripts/git/pre-commit.git-hook +++ b/scripts/git/pre-commit.git-hook @@ -36,8 +36,8 @@ elif [ -d src/common ]; then src/tools/*.[ch] fi -if test -e scripts/maint/checkIncludes.py; then - python scripts/maint/checkIncludes.py +if test -e scripts/maint/practracker/includes.py; then + python scripts/maint/practracker/includes.py fi if [ -e scripts/maint/practracker/practracker.py ]; then diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py new file mode 100755 index 000000000..926b201b3 --- /dev/null +++ b/scripts/maint/checkIncludes.py @@ -0,0 +1,14 @@ +#!/usr/bin/python +# Copyright 2018 The Tor Project, Inc. See LICENSE file for licensing info. + +# This file is no longer here; see practracker/includes.py for this +# functionality. This is a stub file that exists so that older git +# hooks will know where to look. + +import sys, os + +dirname = os.path.split(sys.argv[0])[0] +new_location = os.path.join(dirname, "practracker", "includes.py") +python = sys.executable + +os.execl(python, python, new_location, *sys.argv[1:])

1 0

[tor/master] checkIncludes: add a real main function and CLI
by dgoulet＠torproject.org 21 Aug '19

21 Aug '19

commit 9eb12dde181df868932e866c26e39b633aba72c1 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Aug 5 12:36:13 2019 -0400 checkIncludes: add a real main function and CLI --- scripts/maint/checkIncludes.py | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py index c398dc7a5..c35fcfd85 100755 --- a/scripts/maint/checkIncludes.py +++ b/scripts/maint/checkIncludes.py @@ -204,19 +204,27 @@ def consider_include_rules(fname): for err in rules.applyToFile(fname): yield err -if __name__ == '__main__': + list_unused = False log_sorted_levels = False - trouble = False +def walk_c_files(topdir="src"): + """Run through all c and h files under topdir, looking for + include-rule violations. Yield those violations.""" - for dirpath, dirnames, fnames in os.walk("src"): + for dirpath, dirnames, fnames in os.walk(topdir): for fname in fnames: if fname_is_c(fname): fullpath = os.path.join(dirpath,fname) for err in consider_include_rules(fullpath): - print(err, file=sys.stderr) - trouble = True + yield err + +def run_check_includes(topdir, list_unused=False, log_sorted_levels=False): + trouble = False + + for err in walk_c_files(topdir): + print(err, file=sys.stderr) + trouble = True if trouble: err( @@ -244,3 +252,23 @@ if __name__ == '__main__': print("There are circular .may_include dependencies in here somewhere:", uses_dirs) sys.exit(1) + +def main(argv): + import argparse + + progname = argv[0] + parser = argparse.ArgumentParser(prog=progname) + parser.add_argument("--toposort", action="store_true", + help="Print a topologically sorted list of modules") + parser.add_argument("--list-unused", action="store_true", + help="List unused lines in .may_include files.") + parser.add_argument("topdir", default="src", nargs="?", + help="Top-level directory for the tor source") + args = parser.parse_args(argv[1:]) + + run_check_includes(topdir=args.topdir, + log_sorted_levels=args.toposort, + list_unused=args.list_unused) + +if __name__ == '__main__': + main(sys.argv)

1 0

[tor/master] Move checkIncludes inside practracker
by dgoulet＠torproject.org 21 Aug '19

21 Aug '19

commit 6fb74753c2d6400ea6d7e118ce2428d0290728a8 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Aug 5 14:10:40 2019 -0400 Move checkIncludes inside practracker Update the makefile accordingly. --- Makefile.am | 4 ++-- scripts/maint/{checkIncludes.py => practracker/includes.py} | 0 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index 8eeba5edb..e973593bd 100644 --- a/Makefile.am +++ b/Makefile.am @@ -166,7 +166,7 @@ EXTRA_DIST+= \ Makefile.nmake \ README \ ReleaseNotes \ - scripts/maint/checkIncludes.py \ + scripts/maint/practracker/includes.py \ scripts/maint/checkSpace.pl \ scripts/maint/practracker @@ -366,7 +366,7 @@ endif check-includes: if USEPYTHON - $(PYTHON) $(top_srcdir)/scripts/maint/checkIncludes.py + $(PYTHON) $(top_srcdir)/scripts/maint/practracker/includes.py endif check-best-practices: diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/practracker/includes.py similarity index 100% rename from scripts/maint/checkIncludes.py rename to scripts/maint/practracker/includes.py

1 0

[tor/master] Teach include-checker about advisory rules
by dgoulet＠torproject.org 21 Aug '19

21 Aug '19

commit 720951f05664c38155a4515aa76b9c8d3a99269d Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Aug 5 17:04:00 2019 -0400 Teach include-checker about advisory rules A .may_includes file can be "advisory", which means that some violations of the rules are expected. We will track these violations with practracker, not as automatic errors. --- scripts/maint/practracker/includes.py | 24 ++++++++++++++++------ src/core/crypto/.may_include | 10 +++++++++ src/core/mainloop/.may_include | 20 ++++++++++++++++++ src/core/or/.may_include | 38 +++++++++++++++++++++++++++++++++++ src/core/proto/.may_include | 10 +++++++++ 5 files changed, 96 insertions(+), 6 deletions(-) diff --git a/scripts/maint/practracker/includes.py b/scripts/maint/practracker/includes.py index c35fcfd85..fbd68f4f5 100755 --- a/scripts/maint/practracker/includes.py +++ b/scripts/maint/practracker/includes.py @@ -56,9 +56,10 @@ def pattern_is_normal(s): return False class Error(object): - def __init__(self, location, msg): + def __init__(self, location, msg, is_advisory=False): self.location = location self.msg = msg + self.is_advisory = is_advisory def __str__(self): return "{} at {}".format(self.msg, self.location) @@ -73,8 +74,12 @@ class Rules(object): self.incpath = dirpath self.patterns = [] self.usedPatterns = set() + self.is_advisory = False def addPattern(self, pattern): + if pattern == "!advisory": + self.is_advisory = True + return if not pattern_is_normal(pattern): warn("Unusual pattern {} in {}".format(pattern, self.dirpath)) self.patterns.append(pattern) @@ -95,7 +100,8 @@ class Rules(object): include = m.group(1) if not self.includeOk(include): yield Error("{}{}".format(loc_prefix,str(lineno)), - "Forbidden include of {}".format(include)) + "Forbidden include of {}".format(include), + is_advisory=self.is_advisory) def applyToFile(self, fname): with open_file(fname) as f: @@ -204,7 +210,6 @@ def consider_include_rules(fname): for err in rules.applyToFile(fname): yield err - list_unused = False log_sorted_levels = False @@ -219,12 +224,16 @@ def walk_c_files(topdir="src"): for err in consider_include_rules(fullpath): yield err -def run_check_includes(topdir, list_unused=False, log_sorted_levels=False): +def run_check_includes(topdir, list_unused=False, log_sorted_levels=False, + list_advisories=False): trouble = False for err in walk_c_files(topdir): + if err.is_advisory and not list_advisories: + continue print(err, file=sys.stderr) - trouble = True + if not err.is_advisory: + trouble = True if trouble: err( @@ -262,13 +271,16 @@ def main(argv): help="Print a topologically sorted list of modules") parser.add_argument("--list-unused", action="store_true", help="List unused lines in .may_include files.") + parser.add_argument("--list-advisories", action="store_true", + help="List advisories as well as forbidden includes") parser.add_argument("topdir", default="src", nargs="?", help="Top-level directory for the tor source") args = parser.parse_args(argv[1:]) run_check_includes(topdir=args.topdir, log_sorted_levels=args.toposort, - list_unused=args.list_unused) + list_unused=args.list_unused, + list_advisories=args.list_advisories) if __name__ == '__main__': main(sys.argv) diff --git a/src/core/crypto/.may_include b/src/core/crypto/.may_include new file mode 100644 index 000000000..5782a3679 --- /dev/null +++ b/src/core/crypto/.may_include @@ -0,0 +1,10 @@ +!advisory + +orconfig.h + +lib/crypt_ops/*.h +lib/ctime/*.h +lib/cc/*.h +lib/log/*.h + +core/crypto/*.h diff --git a/src/core/mainloop/.may_include b/src/core/mainloop/.may_include new file mode 100644 index 000000000..79d6a130a --- /dev/null +++ b/src/core/mainloop/.may_include @@ -0,0 +1,20 @@ +!advisory + +orconfig.h + +lib/container/*.h +lib/dispatch/*.h +lib/evloop/*.h +lib/pubsub/*.h +lib/subsys/*.h +lib/buf/*.h +lib/crypt_ops/*.h +lib/err/*.h +lib/tls/*.h +lib/net/*.h +lib/evloop/*.h +lib/geoip/*.h +lib/sandbox/*.h +lib/compress/*.h + +core/mainloop/*.h \ No newline at end of file diff --git a/src/core/or/.may_include b/src/core/or/.may_include new file mode 100644 index 000000000..5173e8a2b --- /dev/null +++ b/src/core/or/.may_include @@ -0,0 +1,38 @@ +!advisory + +orconfig.h + +lib/arch/*.h +lib/buf/*.h +lib/cc/*.h +lib/compress/*.h +lib/container/*.h +lib/crypt_ops/*.h +lib/ctime/*.h +lib/defs/*.h +lib/encoding/*.h +lib/err/*.h +lib/evloop/*.h +lib/fs/*.h +lib/geoip/*.h +lib/intmath/*.h +lib/log/*.h +lib/malloc/*.h +lib/math/*.h +lib/net/*.h +lib/pubsub/*.h +lib/string/*.h +lib/subsys/*.h +lib/test/*.h +lib/testsupport/*.h +lib/thread/*.h +lib/time/*.h +lib/tls/*.h +lib/wallclock/*.h + +trunnel/*.h + +core/mainloop/*.h +core/proto/*.h +core/crypto/*.h +core/or/*.h \ No newline at end of file diff --git a/src/core/proto/.may_include b/src/core/proto/.may_include new file mode 100644 index 000000000..c1647a5cf --- /dev/null +++ b/src/core/proto/.may_include @@ -0,0 +1,10 @@ +!advisory + +orconfig.h + +lib/crypt_ops/*.h +lib/buf/*.h + +trunnel/*.h + +core/proto/*.h \ No newline at end of file

1 0

[tor/master] practracker: a violation of a .may_include rule is now a problem.
by dgoulet＠torproject.org 21 Aug '19

21 Aug '19

commit 6b26281b50f25f0175f3ba801951324a30ba7a32 Author: Nick Mathewson <nickm(a)torproject.org> Date: Mon Aug 5 17:17:50 2019 -0400 practracker: a violation of a .may_include rule is now a problem. We treat "0" as the expected number, and warn about everything else. The problem type is "dependency-violation". --- scripts/maint/practracker/practracker.py | 17 ++++++++++++++++- scripts/maint/practracker/problem.py | 17 +++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index 7e51edb48..66ebb1277 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -25,6 +25,7 @@ import os, sys import metrics import util import problem +import includes # The filename of the exceptions file (it should be placed in the practracker directory) EXCEPTIONS_FNAME = "./exceptions.txt" @@ -35,12 +36,15 @@ MAX_FILE_SIZE = 3000 # lines MAX_FUNCTION_SIZE = 100 # lines # Recommended number of #includes MAX_INCLUDE_COUNT = 50 +# Recommended number of dependency violations +MAX_DEP_VIOLATIONS = 0 # Map from problem type to functions that adjust for tolerance TOLERANCE_FNS = { 'include-count': lambda n: int(n*1.1), 'function-size': lambda n: int(n*1.1), - 'file-size': lambda n: int(n*1.02) + 'file-size': lambda n: int(n*1.02), + 'dependency-violation': lambda n: (n+2) } ####################################################### @@ -94,6 +98,7 @@ def consider_metrics_for_file(fname, f): Yield a sequence of problem.Item objects for all of the metrics in 'f'. """ + real_fname = fname # Strip the useless part of the path if fname.startswith(TOR_TOPDIR): fname = fname[len(TOR_TOPDIR):] @@ -112,6 +117,13 @@ def consider_metrics_for_file(fname, f): for item in consider_function_size(fname, f): yield item + n = 0 + for item in includes.consider_include_rules(real_fname): + n += 1 + if n: + yield problem.DependencyViolationItem(fname, n) + + HEADER="""\ # Welcome to the exceptions file for Tor's best-practices tracker! # @@ -167,6 +179,8 @@ def main(argv): help="Maximum includes per C file") parser.add_argument("--max-function-size", default=MAX_FUNCTION_SIZE, help="Maximum lines per function") + parser.add_argument("--max-dependency-violations", default=MAX_DEP_VIOLATIONS, + help="Maximum number of dependency violations to allow") parser.add_argument("topdir", default=".", nargs="?", help="Top-level directory for the tor source") args = parser.parse_args(argv[1:]) @@ -183,6 +197,7 @@ def main(argv): filt.addThreshold(problem.FileSizeItem("*", int(args.max_file_size))) filt.addThreshold(problem.IncludeCountItem("*", int(args.max_include_count))) filt.addThreshold(problem.FunctionSizeItem("*", int(args.max_function_size))) + filt.addThreshold(problem.DependencyViolationItem("*", int(args.max_dependency_violations))) # 1) Get all the .c files we care about files_list = util.get_tor_c_files(TOR_TOPDIR) diff --git a/scripts/maint/practracker/problem.py b/scripts/maint/practracker/problem.py index 73519d446..dafcda86f 100644 --- a/scripts/maint/practracker/problem.py +++ b/scripts/maint/practracker/problem.py @@ -191,6 +191,21 @@ class FunctionSizeItem(Item): def __init__(self, problem_location, metric_value): super(FunctionSizeItem, self).__init__("function-size", problem_location, metric_value) +class DependencyViolationItem(Item): + """ + Denotes a dependency violation in a .c or .h file. A dependency violation + occurs when a file includes a file from some module that is not listed + in its .may_include file. + + The 'problem_location' is the file that contains the problem. + + The 'metric_value' is the number of forbidden includes. + """ + def __init__(self, problem_location, metric_value): + super(DependencyViolationItem, self).__init__("dependency-violation", + problem_location, + metric_value) + comment_re = re.compile(r'#.*$') def get_old_problem_from_exception_str(exception_str): @@ -212,5 +227,7 @@ def get_old_problem_from_exception_str(exception_str): return IncludeCountItem(problem_location, metric_value) elif problem_type == "function-size": return FunctionSizeItem(problem_location, metric_value) + elif problem_type == "dependency-violation": + return DependencyViolationItem(problem_location, metric_value) else: raise ValueError("Unknown exception type {!r}".format(orig_str))

1 0