| 1 | # NO SHEBANG because we call it directly.
|
| 2 | """
|
| 3 | app_deps.py
|
| 4 |
|
| 5 | Dynamically discover Python and C modules. We import the main module and
|
| 6 | inspect sys.modules before and after. That is, we use the exact logic that the
|
| 7 | Python interpreter does.
|
| 8 |
|
| 9 | Usage:
|
| 10 | PYTHONPATH=... py_deps.py <main module>
|
| 11 |
|
| 12 | IMPORTANT: Run this script with -S so that system libraries aren't found.
|
| 13 | """
|
| 14 | from __future__ import print_function
|
| 15 |
|
| 16 | import sys
|
| 17 | OLD_MODULES = dict(sys.modules) # Make a copy
|
| 18 |
|
| 19 | import posix # Do it afterward so we don't mess up analysis.
|
| 20 |
|
| 21 | VERBOSE = False
|
| 22 |
|
| 23 | def log(msg, *args):
|
| 24 | if not VERBOSE:
|
| 25 | return
|
| 26 | if args:
|
| 27 | msg = msg % args
|
| 28 | print('\t', msg, file=sys.stderr)
|
| 29 |
|
| 30 |
|
| 31 | def ImportMain(main_module, old_modules):
|
| 32 | """Yields (module name, absolute path) pairs."""
|
| 33 |
|
| 34 | log('Importing %r', main_module)
|
| 35 | try:
|
| 36 | __import__(main_module)
|
| 37 | except ImportError as e:
|
| 38 | log('Error importing %r with sys.path %r', main_module, sys.path)
|
| 39 | # TODO: print better error.
|
| 40 | raise
|
| 41 |
|
| 42 | new_modules = sys.modules
|
| 43 | log('After importing: %d modules', len(new_modules))
|
| 44 |
|
| 45 | for name in sorted(new_modules):
|
| 46 | if name in old_modules:
|
| 47 | continue # exclude old modules
|
| 48 |
|
| 49 | module = new_modules[name]
|
| 50 |
|
| 51 | full_path = getattr(module, '__file__', None)
|
| 52 |
|
| 53 | # For some reason, there are entries like:
|
| 54 | # 'pan.core.os': None in sys.modules. Here's a hack to get rid of them.
|
| 55 | if module is None:
|
| 56 | continue
|
| 57 | # Not sure why, but some stdlib modules don't have a __file__ attribute,
|
| 58 | # e.g. "gc", "marshal", "thread". Doesn't matter for our purposes.
|
| 59 | if full_path is None:
|
| 60 | continue
|
| 61 | yield name, full_path
|
| 62 |
|
| 63 | # Special case for __future__. It's necessary, but doesn't get counted
|
| 64 | # because we import it first!
|
| 65 | module = sys.modules['__future__']
|
| 66 | full_path = getattr(module, '__file__', None)
|
| 67 | yield '__future__', full_path
|
| 68 |
|
| 69 |
|
| 70 | PY_MODULE = 0
|
| 71 | C_MODULE = 1
|
| 72 |
|
| 73 |
|
| 74 | def FilterModules(modules):
|
| 75 | """Look at __file__ of each module, and classify them as Python or C."""
|
| 76 |
|
| 77 | for module, full_path in modules:
|
| 78 | #print 'OLD', module, full_path
|
| 79 | num_parts = module.count('.') + 1
|
| 80 | i = len(full_path)
|
| 81 | # Do it once more in this case
|
| 82 | if full_path.endswith('/__init__.pyc') or \
|
| 83 | full_path.endswith('__init__.py'):
|
| 84 | i = full_path.rfind('/', 0, i)
|
| 85 | for _ in range(num_parts): # range for Python 3
|
| 86 | i = full_path.rfind('/', 0, i)
|
| 87 | #print i, full_path[i+1:]
|
| 88 | rel_path = full_path[i + 1:]
|
| 89 |
|
| 90 | # Depending on whether it's cached, the __file__ attribute on the module
|
| 91 | # ends with '.py' or '.pyc'.
|
| 92 | if full_path.endswith('.py'):
|
| 93 | yield PY_MODULE, full_path, rel_path
|
| 94 | elif full_path.endswith('.pyc'):
|
| 95 | yield PY_MODULE, full_path[:-1], rel_path[:-1]
|
| 96 | else:
|
| 97 | # .so file
|
| 98 | yield C_MODULE, module, full_path
|
| 99 |
|
| 100 |
|
| 101 | def main(argv):
|
| 102 | """Returns an exit code."""
|
| 103 |
|
| 104 | # Set an environment variable so dependencies in debug mode can be excluded.
|
| 105 | posix.environ['_OVM_DEPS'] = '1'
|
| 106 |
|
| 107 | action = argv[1]
|
| 108 | main_module = argv[2]
|
| 109 | log('Before importing: %d modules', len(OLD_MODULES))
|
| 110 |
|
| 111 | if action == 'both': # Write files for both .py and .so dependencies
|
| 112 | prefix = argv[3]
|
| 113 | py_out_path = prefix + '-cpython.txt'
|
| 114 | c_out_path = prefix + '-c.txt'
|
| 115 |
|
| 116 | modules = ImportMain(main_module, OLD_MODULES)
|
| 117 |
|
| 118 | with open(py_out_path, 'w') as py_out, open(c_out_path, 'w') as c_out:
|
| 119 | for mod_type, x, y in FilterModules(modules):
|
| 120 | if mod_type == PY_MODULE:
|
| 121 | print(x, y, file=py_out)
|
| 122 | print(x + 'c', y + 'c', file=py_out) # .pyc goes in bytecode.zip too
|
| 123 |
|
| 124 | elif mod_type == C_MODULE:
|
| 125 | print(x, y, file=c_out) # mod_name, full_path
|
| 126 |
|
| 127 | else:
|
| 128 | raise AssertionError(mod_type)
|
| 129 |
|
| 130 | elif action == 'py': # .py path -> .pyc relative path
|
| 131 | modules = ImportMain(main_module, OLD_MODULES)
|
| 132 | for mod_type, full_path, rel_path in FilterModules(modules):
|
| 133 | if mod_type == PY_MODULE:
|
| 134 | opy_input = full_path
|
| 135 | opy_output = rel_path + 'c' # output is .pyc
|
| 136 | print(opy_input, opy_output)
|
| 137 |
|
| 138 | elif action == 'py-manifest': # .py path -> .py relative path
|
| 139 | modules = ImportMain(main_module, OLD_MODULES)
|
| 140 | for mod_type, full_path, rel_path in FilterModules(modules):
|
| 141 | if mod_type == PY_MODULE:
|
| 142 | opy_input = full_path
|
| 143 | assert rel_path.endswith('.py')
|
| 144 | #mod_name = rel_path[:-3].replace('/', '.')
|
| 145 | print(opy_input, rel_path)
|
| 146 | else:
|
| 147 | raise RuntimeError('Invalid action %r' % action)
|
| 148 |
|
| 149 |
|
| 150 | if __name__ == '__main__':
|
| 151 | try:
|
| 152 | sys.exit(main(sys.argv))
|
| 153 | except RuntimeError as e:
|
| 154 | print('%s: %s' % (sys.argv[0], e.args[0]), file=sys.stderr)
|
| 155 | sys.exit(1)
|