1 | # NO SHEBANG because we call it directly.
|
2 | """
|
3 | app_deps.py
|
4 |
|
5 | Dynamically discover Python and C modules. We import the main module and
|
6 | inspect sys.modules before and after. That is, we use the exact logic that the
|
7 | Python interpreter does.
|
8 |
|
9 | Usage:
|
10 | PYTHONPATH=... py_deps.py <main module>
|
11 |
|
12 | IMPORTANT: Run this script with -S so that system libraries aren't found.
|
13 | """
|
14 | from __future__ import print_function
|
15 |
|
16 | import sys
|
17 | OLD_MODULES = dict(sys.modules) # Make a copy
|
18 |
|
19 | import posix # Do it afterward so we don't mess up analysis.
|
20 |
|
21 | VERBOSE = False
|
22 |
|
23 | def log(msg, *args):
|
24 | if not VERBOSE:
|
25 | return
|
26 | if args:
|
27 | msg = msg % args
|
28 | print('\t', msg, file=sys.stderr)
|
29 |
|
30 |
|
31 | def ImportMain(main_module, old_modules):
|
32 | """Yields (module name, absolute path) pairs."""
|
33 |
|
34 | log('Importing %r', main_module)
|
35 | try:
|
36 | __import__(main_module)
|
37 | except ImportError as e:
|
38 | log('Error importing %r with sys.path %r', main_module, sys.path)
|
39 | # TODO: print better error.
|
40 | raise
|
41 |
|
42 | new_modules = sys.modules
|
43 | log('After importing: %d modules', len(new_modules))
|
44 |
|
45 | for name in sorted(new_modules):
|
46 | if name in old_modules:
|
47 | continue # exclude old modules
|
48 |
|
49 | module = new_modules[name]
|
50 |
|
51 | full_path = getattr(module, '__file__', None)
|
52 |
|
53 | # For some reason, there are entries like:
|
54 | # 'pan.core.os': None in sys.modules. Here's a hack to get rid of them.
|
55 | if module is None:
|
56 | continue
|
57 | # Not sure why, but some stdlib modules don't have a __file__ attribute,
|
58 | # e.g. "gc", "marshal", "thread". Doesn't matter for our purposes.
|
59 | if full_path is None:
|
60 | continue
|
61 | yield name, full_path
|
62 |
|
63 | # Special case for __future__. It's necessary, but doesn't get counted
|
64 | # because we import it first!
|
65 | module = sys.modules['__future__']
|
66 | full_path = getattr(module, '__file__', None)
|
67 | yield '__future__', full_path
|
68 |
|
69 |
|
70 | PY_MODULE = 0
|
71 | C_MODULE = 1
|
72 |
|
73 |
|
74 | def FilterModules(modules):
|
75 | """Look at __file__ of each module, and classify them as Python or C."""
|
76 |
|
77 | for module, full_path in modules:
|
78 | #print 'OLD', module, full_path
|
79 | num_parts = module.count('.') + 1
|
80 | i = len(full_path)
|
81 | # Do it once more in this case
|
82 | if full_path.endswith('/__init__.pyc') or \
|
83 | full_path.endswith('__init__.py'):
|
84 | i = full_path.rfind('/', 0, i)
|
85 | for _ in range(num_parts): # range for Python 3
|
86 | i = full_path.rfind('/', 0, i)
|
87 | #print i, full_path[i+1:]
|
88 | rel_path = full_path[i + 1:]
|
89 |
|
90 | # Depending on whether it's cached, the __file__ attribute on the module
|
91 | # ends with '.py' or '.pyc'.
|
92 | if full_path.endswith('.py'):
|
93 | yield PY_MODULE, full_path, rel_path
|
94 | elif full_path.endswith('.pyc'):
|
95 | yield PY_MODULE, full_path[:-1], rel_path[:-1]
|
96 | else:
|
97 | # .so file
|
98 | yield C_MODULE, module, full_path
|
99 |
|
100 |
|
101 | def main(argv):
|
102 | """Returns an exit code."""
|
103 |
|
104 | # Set an environment variable so dependencies in debug mode can be excluded.
|
105 | posix.environ['_OVM_DEPS'] = '1'
|
106 |
|
107 | action = argv[1]
|
108 | main_module = argv[2]
|
109 | log('Before importing: %d modules', len(OLD_MODULES))
|
110 |
|
111 | if action == 'both': # Write files for both .py and .so dependencies
|
112 | prefix = argv[3]
|
113 | py_out_path = prefix + '-cpython.txt'
|
114 | c_out_path = prefix + '-c.txt'
|
115 |
|
116 | modules = ImportMain(main_module, OLD_MODULES)
|
117 |
|
118 | with open(py_out_path, 'w') as py_out, open(c_out_path, 'w') as c_out:
|
119 | for mod_type, x, y in FilterModules(modules):
|
120 | if mod_type == PY_MODULE:
|
121 | print(x, y, file=py_out)
|
122 | print(x + 'c', y + 'c', file=py_out) # .pyc goes in bytecode.zip too
|
123 |
|
124 | elif mod_type == C_MODULE:
|
125 | print(x, y, file=c_out) # mod_name, full_path
|
126 |
|
127 | else:
|
128 | raise AssertionError(mod_type)
|
129 |
|
130 | elif action == 'py': # .py path -> .pyc relative path
|
131 | modules = ImportMain(main_module, OLD_MODULES)
|
132 | for mod_type, full_path, rel_path in FilterModules(modules):
|
133 | if mod_type == PY_MODULE:
|
134 | opy_input = full_path
|
135 | opy_output = rel_path + 'c' # output is .pyc
|
136 | print(opy_input, opy_output)
|
137 |
|
138 | elif action == 'py-manifest': # .py path -> .py relative path
|
139 | modules = ImportMain(main_module, OLD_MODULES)
|
140 | for mod_type, full_path, rel_path in FilterModules(modules):
|
141 | if mod_type == PY_MODULE:
|
142 | opy_input = full_path
|
143 | assert rel_path.endswith('.py')
|
144 | #mod_name = rel_path[:-3].replace('/', '.')
|
145 | print(opy_input, rel_path)
|
146 | else:
|
147 | raise RuntimeError('Invalid action %r' % action)
|
148 |
|
149 |
|
150 | if __name__ == '__main__':
|
151 | try:
|
152 | sys.exit(main(sys.argv))
|
153 | except RuntimeError as e:
|
154 | print('%s: %s' % (sys.argv[0], e.args[0]), file=sys.stderr)
|
155 | sys.exit(1)
|