20036
|
1 import ast
|
|
2 import os
|
|
3 import sys
|
|
4
|
|
5 def dotted_name_of_path(path):
|
|
6 """Given a relative path to a source file, return its dotted module name.
|
|
7
|
|
8
|
|
9 >>> dotted_name_of_path('mercurial/error.py')
|
|
10 'mercurial.error'
|
|
11 """
|
|
12 parts = path.split('/')
|
|
13 parts[-1] = parts[-1][:-3] # remove .py
|
|
14 return '.'.join(parts)
|
|
15
|
|
16
|
|
17 def list_stdlib_modules():
|
|
18 """List the modules present in the stdlib.
|
|
19
|
|
20 >>> mods = set(list_stdlib_modules())
|
|
21 >>> 'BaseHTTPServer' in mods
|
|
22 True
|
|
23
|
|
24 os.path isn't really a module, so it's missing:
|
|
25
|
|
26 >>> 'os.path' in mods
|
|
27 False
|
|
28
|
|
29 sys requires special treatment, because it's baked into the
|
|
30 interpreter, but it should still appear:
|
|
31
|
|
32 >>> 'sys' in mods
|
|
33 True
|
|
34
|
|
35 >>> 'collections' in mods
|
|
36 True
|
|
37
|
|
38 >>> 'cStringIO' in mods
|
|
39 True
|
|
40 """
|
|
41 for m in sys.builtin_module_names:
|
|
42 yield m
|
|
43 # These modules only exist on windows, but we should always
|
|
44 # consider them stdlib.
|
|
45 for m in ['msvcrt', '_winreg']:
|
|
46 yield m
|
|
47 # These get missed too
|
|
48 for m in 'ctypes', 'email':
|
|
49 yield m
|
|
50 yield 'builtins' # python3 only
|
|
51 for libpath in sys.path:
|
|
52 # We want to walk everything in sys.path that starts with
|
|
53 # either sys.prefix or sys.exec_prefix.
|
|
54 if not (libpath.startswith(sys.prefix)
|
|
55 or libpath.startswith(sys.exec_prefix)):
|
|
56 continue
|
|
57 if 'site-packages' in libpath:
|
|
58 continue
|
|
59 for top, dirs, files in os.walk(libpath):
|
|
60 for name in files:
|
|
61 if name == '__init__.py':
|
|
62 continue
|
|
63 if not (name.endswith('.py') or name.endswith('.so')):
|
|
64 continue
|
|
65 full_path = os.path.join(top, name)
|
|
66 if 'site-packages' in full_path:
|
|
67 continue
|
|
68 rel_path = full_path[len(libpath) + 1:]
|
|
69 mod = dotted_name_of_path(rel_path)
|
|
70 yield mod
|
|
71
|
|
72 stdlib_modules = set(list_stdlib_modules())
|
|
73
|
|
74 def imported_modules(source):
|
|
75 """Given the source of a file as a string, yield the names
|
|
76 imported by that file.
|
|
77
|
|
78 >>> list(imported_modules(
|
|
79 ... 'import foo ; from baz import bar; import foo.qux'))
|
|
80 ['foo', 'baz.bar', 'foo.qux']
|
|
81 """
|
|
82 for node in ast.walk(ast.parse(source)):
|
|
83 if isinstance(node, ast.Import):
|
|
84 for n in node.names:
|
|
85 yield n.name
|
|
86 elif isinstance(node, ast.ImportFrom):
|
|
87 prefix = node.module + '.'
|
|
88 for n in node.names:
|
|
89 yield prefix + n.name
|
|
90
|
|
91 def verify_stdlib_on_own_line(source):
|
|
92 """Given some python source, verify that stdlib imports are done
|
|
93 in separate statements from relative local module imports.
|
|
94
|
|
95 Observing this limitation is important as it works around an
|
|
96 annoying lib2to3 bug in relative import rewrites:
|
|
97 http://bugs.python.org/issue19510.
|
|
98
|
|
99 >>> list(verify_stdlib_on_own_line('import sys, foo'))
|
|
100 ['mixed stdlib and relative imports:\\n foo, sys']
|
|
101 >>> list(verify_stdlib_on_own_line('import sys, os'))
|
|
102 []
|
|
103 >>> list(verify_stdlib_on_own_line('import foo, bar'))
|
|
104 []
|
|
105 """
|
|
106 for node in ast.walk(ast.parse(source)):
|
|
107 if isinstance(node, ast.Import):
|
|
108 from_stdlib = {}
|
|
109 for n in node.names:
|
|
110 from_stdlib[n.name] = n.name in stdlib_modules
|
|
111 num_std = len([x for x in from_stdlib.values() if x])
|
|
112 if num_std not in (len(from_stdlib.values()), 0):
|
|
113 yield ('mixed stdlib and relative imports:\n %s' %
|
|
114 ', '.join(sorted(from_stdlib.iterkeys())))
|
|
115
|
|
116 class CircularImport(Exception):
|
|
117 pass
|
|
118
|
|
119
|
|
120 def cyclekey(names):
|
|
121 return tuple(sorted(set(names)))
|
|
122
|
|
123 def check_one_mod(mod, imports, path=None, ignore=None):
|
|
124 if path is None:
|
|
125 path = []
|
|
126 if ignore is None:
|
|
127 ignore = []
|
|
128 path = path + [mod]
|
|
129 for i in sorted(imports.get(mod, [])):
|
|
130 if i not in stdlib_modules:
|
|
131 i = mod.rsplit('.', 1)[0] + '.' + i
|
|
132 if i in path:
|
|
133 firstspot = path.index(i)
|
|
134 cycle = path[firstspot:] + [i]
|
|
135 if cyclekey(cycle) not in ignore:
|
|
136 raise CircularImport(cycle)
|
|
137 continue
|
|
138 check_one_mod(i, imports, path=path, ignore=ignore)
|
|
139
|
|
140
|
|
141 def find_cycles(imports):
|
|
142 """Find cycles in an already-loaded import graph.
|
|
143
|
|
144 >>> imports = {'top.foo': ['bar', 'os.path', 'qux'],
|
|
145 ... 'top.bar': ['baz', 'sys'],
|
|
146 ... 'top.baz': ['foo'],
|
|
147 ... 'top.qux': ['foo']}
|
|
148 >>> print '\\n'.join(sorted(find_cycles(imports)))
|
|
149 top.bar -> top.baz -> top.foo -> top.bar
|
|
150 top.foo -> top.qux -> top.foo
|
|
151 """
|
|
152 cycles = {}
|
|
153 for mod in sorted(imports.iterkeys()):
|
|
154 try:
|
|
155 check_one_mod(mod, imports, ignore=cycles)
|
|
156 except CircularImport, e:
|
|
157 cycle = e.args[0]
|
|
158 cycles[cyclekey(cycle)] = ' -> '.join(rotatecycle(cycle))
|
|
159 return cycles.values()
|
|
160
|
|
161 def _cycle_sortkey(c):
|
|
162 return len(c), c
|
|
163
|
|
164 def main(argv):
|
|
165 if len(argv) < 2:
|
|
166 print 'Usage: %s file [file] [file] ...'
|
|
167 return 1
|
|
168 used_imports = {}
|
|
169 any_errors = False
|
|
170 for source_path in argv[1:]:
|
|
171 f = open(source_path)
|
|
172 modname = dotted_name_of_path(source_path)
|
|
173 src = f.read()
|
|
174 used_imports[modname] = sorted(imported_modules(src))
|
|
175 for error in verify_stdlib_on_own_line(src):
|
|
176 any_errors = True
|
|
177 print source_path, error
|
|
178 f.close()
|
|
179 cycles = find_cycles(used_imports)
|
|
180 if cycles:
|
|
181 firstmods = set()
|
|
182 for c in sorted(cycles, key=_cycle_sortkey):
|
|
183 first = c.split()[0]
|
|
184 # As a rough cut, ignore any cycle that starts with the
|
|
185 # same module as some other cycle. Otherwise we see lots
|
|
186 # of cycles that are effectively duplicates.
|
|
187 if first in firstmods:
|
|
188 continue
|
|
189 print 'Import cycle:', c
|
|
190 firstmods.add(first)
|
|
191 any_errors = True
|
|
192 return not any_errors
|
|
193
|
|
194 if __name__ == '__main__':
|
|
195 sys.exit(int(main(sys.argv)))
|