| 1 | #!/usr/bin/python -S
|
| 2 | from __future__ import print_function
|
| 3 | """
|
| 4 | py_deps.py
|
| 5 |
|
| 6 | Dynamically discover Python and C modules. We import the main module and
|
| 7 | inspect sys.modules before and after. That is, we use the exact logic that the
|
| 8 | Python interpreter does.
|
| 9 |
|
| 10 | Usage:
|
| 11 | PYTHONPATH=... py_deps.py <main module>
|
| 12 |
|
| 13 | IMPORTANT: Run this script with -S so that system libraries aren't found.
|
| 14 | """
|
| 15 |
|
| 16 | import sys
|
| 17 | OLD_MODULES = dict(sys.modules) # Make a copy
|
| 18 |
|
| 19 | import os # Do it here so we don't mess up analysis
|
| 20 |
|
| 21 |
|
| 22 | def log(msg, *args):
|
| 23 | if args:
|
| 24 | msg = msg % args
|
| 25 | print('\t', msg, file=sys.stderr)
|
| 26 |
|
| 27 |
|
| 28 | def ImportMain(main_module, old_modules):
|
| 29 | """Yields (module name, absolute path) pairs."""
|
| 30 |
|
| 31 | log('Importing %r', main_module)
|
| 32 | try:
|
| 33 | __import__(main_module)
|
| 34 | except ImportError, e:
|
| 35 | log('Error importing %r with sys.path %r', main_module, sys.path)
|
| 36 | # TODO: print better error.
|
| 37 | raise
|
| 38 |
|
| 39 | new_modules = sys.modules
|
| 40 | log('After importing: %d modules', len(new_modules))
|
| 41 |
|
| 42 | for name in sorted(new_modules):
|
| 43 | if name in old_modules:
|
| 44 | continue # exclude old modules
|
| 45 |
|
| 46 | module = new_modules[name]
|
| 47 |
|
| 48 | full_path = getattr(module, '__file__', None)
|
| 49 |
|
| 50 | # For some reason, there are entries like:
|
| 51 | # 'pan.core.os': None in sys.modules. Here's a hack to get rid of them.
|
| 52 | if module is None:
|
| 53 | continue
|
| 54 | # Not sure why, but some stdlib modules don't have a __file__ attribute,
|
| 55 | # e.g. "gc", "marshal", "thread". Doesn't matter for our purposes.
|
| 56 | if full_path is None:
|
| 57 | continue
|
| 58 | yield name, full_path
|
| 59 |
|
| 60 |
|
| 61 | PY_MODULE = 0
|
| 62 | C_MODULE = 1
|
| 63 |
|
| 64 |
|
| 65 | def FilterModules(modules):
|
| 66 | """Look at __file__ of each module, and classify them as Python or C."""
|
| 67 |
|
| 68 | for module, full_path in modules:
|
| 69 | #print 'OLD', module, full_path
|
| 70 | num_parts = module.count('.') + 1
|
| 71 | i = len(full_path)
|
| 72 | # Do it once more in this case
|
| 73 | if full_path.endswith('/__init__.pyc') or \
|
| 74 | full_path.endswith('__init__.py'):
|
| 75 | i = full_path.rfind('/', 0, i)
|
| 76 | for _ in xrange(num_parts):
|
| 77 | i = full_path.rfind('/', 0, i)
|
| 78 | #print i, full_path[i+1:]
|
| 79 | rel_path = full_path[i + 1:]
|
| 80 |
|
| 81 | # Depending on whether it's cached, the __file__ attribute on the module
|
| 82 | # ends with '.py' or '.pyc'.
|
| 83 | if full_path.endswith('.py'):
|
| 84 | yield PY_MODULE, full_path, rel_path
|
| 85 | elif full_path.endswith('.pyc'):
|
| 86 | yield PY_MODULE, full_path[:-1], rel_path[:-1]
|
| 87 | else:
|
| 88 | # .so file
|
| 89 | yield C_MODULE, module, full_path
|
| 90 |
|
| 91 |
|
| 92 | # TODO: Get rid of this?
|
| 93 | def CreateOptionsParser():
|
| 94 | parser = optparse.OptionParser()
|
| 95 | return parser
|
| 96 |
|
| 97 |
|
| 98 | def main(argv):
|
| 99 | """Returns an exit code."""
|
| 100 |
|
| 101 | #(opts, argv) = CreateOptionsParser().parse_args(argv)
|
| 102 | #if not argv:
|
| 103 | # raise Error('No modules specified.')
|
| 104 |
|
| 105 | # Set an environment variable so dependencies in debug mode can be excluded.
|
| 106 | os.environ['_OVM_DEPS'] = '1'
|
| 107 |
|
| 108 | action = argv[1]
|
| 109 | main_module = argv[2]
|
| 110 | log('Before importing: %d modules', len(OLD_MODULES))
|
| 111 |
|
| 112 | if action == 'both': # Write files for both .py and .so dependencies
|
| 113 | prefix = argv[3]
|
| 114 | py_out_path = prefix + '-cpython.txt'
|
| 115 | c_out_path = prefix + '-c.txt'
|
| 116 |
|
| 117 | modules = ImportMain(main_module, OLD_MODULES)
|
| 118 |
|
| 119 | with open(py_out_path, 'w') as py_out, open(c_out_path, 'w') as c_out:
|
| 120 | for mod_type, x, y in FilterModules(modules):
|
| 121 | if mod_type == PY_MODULE:
|
| 122 | print(x, y, file=py_out)
|
| 123 | print(x + 'c', y + 'c', file=py_out) # .pyc goes in bytecode.zip too
|
| 124 |
|
| 125 | elif mod_type == C_MODULE:
|
| 126 | print(x, y, file=c_out) # mod_name, full_path
|
| 127 |
|
| 128 | else:
|
| 129 | raise AssertionError(mod_type)
|
| 130 |
|
| 131 | elif action == 'py': # Just .py files
|
| 132 | modules = ImportMain(main_module, OLD_MODULES)
|
| 133 | for mod_type, full_path, rel_path in FilterModules(modules):
|
| 134 | if mod_type == PY_MODULE:
|
| 135 | opy_input = full_path
|
| 136 | opy_output = rel_path + 'c' # output is .pyc
|
| 137 | print(opy_input, opy_output)
|
| 138 |
|
| 139 | else:
|
| 140 | raise RuntimeError('Invalid action %r' % action)
|
| 141 |
|
| 142 |
|
| 143 | if __name__ == '__main__':
|
| 144 | try:
|
| 145 | sys.exit(main(sys.argv))
|
| 146 | except RuntimeError as e:
|
| 147 | print('%s: %s' % (sys.argv[0], e.args[0]), file=sys.stderr)
|
| 148 | sys.exit(1)
|