1 | #!/usr/bin/env python3
|
2 | """
|
3 | mycpp_main.py - Translate a subset of Python to C++, using MyPy's typed AST.
|
4 | """
|
5 | from __future__ import print_function
|
6 |
|
7 | import optparse
|
8 | import os
|
9 | import sys
|
10 |
|
11 | from typing import List, Optional, Tuple
|
12 |
|
13 | from mypy.build import build as mypy_build
|
14 | from mypy.build import BuildSource
|
15 | from mypy.main import process_options
|
16 |
|
17 | from mycpp import ir_pass
|
18 | from mycpp import const_pass
|
19 | from mycpp import cppgen_pass
|
20 | from mycpp import debug_pass
|
21 | from mycpp import control_flow_pass
|
22 | from mycpp import pass_state
|
23 | from mycpp.util import log
|
24 |
|
25 |
|
26 | def Options():
|
27 | """Returns an option parser instance."""
|
28 |
|
29 | p = optparse.OptionParser()
|
30 | p.add_option('-v',
|
31 | '--verbose',
|
32 | dest='verbose',
|
33 | action='store_true',
|
34 | default=False,
|
35 | help='Show details about translation')
|
36 |
|
37 | p.add_option('--cc-out',
|
38 | dest='cc_out',
|
39 | default=None,
|
40 | help='.cc file to write to')
|
41 |
|
42 | p.add_option('--to-header',
|
43 | dest='to_header',
|
44 | action='append',
|
45 | default=[],
|
46 | help='Export this module to a header, e.g. frontend.args')
|
47 |
|
48 | p.add_option('--header-out',
|
49 | dest='header_out',
|
50 | default=None,
|
51 | help='Write this header')
|
52 |
|
53 | p.add_option(
|
54 | '--stack-roots-warn',
|
55 | dest='stack_roots_warn',
|
56 | default=None,
|
57 | type='int',
|
58 | help='Emit warnings about functions with too many stack roots')
|
59 |
|
60 | return p
|
61 |
|
62 |
|
63 | # Copied from mypyc/build.py
|
64 | def get_mypy_config(
|
65 | paths: List[str], mypy_options: Optional[List[str]]
|
66 | ) -> Tuple[List[BuildSource], Options]:
|
67 | """Construct mypy BuildSources and Options from file and options lists"""
|
68 | # It is kind of silly to do this but oh well
|
69 | mypy_options = mypy_options or []
|
70 | mypy_options.append('--')
|
71 | mypy_options.extend(paths)
|
72 |
|
73 | sources, options = process_options(mypy_options)
|
74 |
|
75 | options.show_traceback = True
|
76 | # Needed to get types for all AST nodes
|
77 | options.export_types = True
|
78 | # TODO: Support incremental checking
|
79 | options.incremental = False
|
80 | # 10/2019: FIX for MyPy 0.730. Not sure why I need this but I do.
|
81 | options.preserve_asts = True
|
82 |
|
83 | # 1/2023: Workaround for conditional import in osh/builtin_comp.py
|
84 | # Same as devtools/types.sh
|
85 | options.warn_unused_ignores = False
|
86 |
|
87 | for source in sources:
|
88 | options.per_module_options.setdefault(source.module,
|
89 | {})['mypyc'] = True
|
90 |
|
91 | return sources, options
|
92 |
|
93 |
|
94 | _FIRST = ('asdl.runtime', 'core.vm')
|
95 |
|
96 | # should be LAST because they use base classes
|
97 | _LAST = ('builtin.bracket_osh', 'builtin.completion_osh', 'core.shell')
|
98 |
|
99 |
|
100 | def ModulesToCompile(result, mod_names):
|
101 | # HACK TO PUT asdl/runtime FIRST.
|
102 | #
|
103 | # Another fix is to hoist those to the declaration phase? Not sure if that
|
104 | # makes sense.
|
105 |
|
106 | # FIRST files. Somehow the MyPy builder reorders the modules.
|
107 | for name, module in result.files.items():
|
108 | if name in _FIRST:
|
109 | yield name, module
|
110 |
|
111 | for name, module in result.files.items():
|
112 | # Only translate files that were mentioned on the command line
|
113 | suffix = name.split('.')[-1]
|
114 | if suffix not in mod_names:
|
115 | continue
|
116 |
|
117 | if name in _FIRST: # We already did these
|
118 | continue
|
119 |
|
120 | if name in _LAST: # We'll do these later
|
121 | continue
|
122 |
|
123 | yield name, module
|
124 |
|
125 | # LAST files
|
126 | for name, module in result.files.items():
|
127 | if name in _LAST:
|
128 | yield name, module
|
129 |
|
130 |
|
131 | def main(argv):
|
132 | # TODO: Put these in the shell script
|
133 | mypy_options = [
|
134 | '--py2',
|
135 | '--strict',
|
136 | '--no-implicit-optional',
|
137 | '--no-strict-optional',
|
138 | # for consistency?
|
139 | '--follow-imports=silent',
|
140 | #'--verbose',
|
141 | ]
|
142 |
|
143 | o = Options()
|
144 | opts, argv = o.parse_args(argv)
|
145 |
|
146 | paths = argv[1:] # e.g. asdl/typed_arith_parse.py
|
147 |
|
148 | log('\tmycpp: LOADING %s', ' '.join(paths))
|
149 | #log('\tmycpp: MYPYPATH = %r', os.getenv('MYPYPATH'))
|
150 |
|
151 | if 0:
|
152 | print(opts)
|
153 | print(paths)
|
154 | return
|
155 |
|
156 | # e.g. asdl/typed_arith_parse.py -> 'typed_arith_parse'
|
157 | mod_names = [os.path.basename(p) for p in paths]
|
158 | mod_names = [os.path.splitext(name)[0] for name in mod_names]
|
159 |
|
160 | # Ditto
|
161 | to_header = opts.to_header
|
162 | #if to_header:
|
163 | if 0:
|
164 | to_header = [os.path.basename(p) for p in to_header]
|
165 | to_header = [os.path.splitext(name)[0] for name in to_header]
|
166 |
|
167 | #log('to_header %s', to_header)
|
168 |
|
169 | sources, options = get_mypy_config(paths, mypy_options)
|
170 | if 0:
|
171 | for source in sources:
|
172 | log('source %s', source)
|
173 | log('')
|
174 | #log('options %s', options)
|
175 |
|
176 | #result = emitmodule.parse_and_typecheck(sources, options)
|
177 | import time
|
178 | start_time = time.time()
|
179 | result = mypy_build(sources=sources, options=options)
|
180 | #log('elapsed 1: %f', time.time() - start_time)
|
181 |
|
182 | if result.errors:
|
183 | log('')
|
184 | log('-' * 80)
|
185 | for e in result.errors:
|
186 | log(e)
|
187 | log('-' * 80)
|
188 | log('')
|
189 | return 1
|
190 |
|
191 | # Important functions in mypyc/build.py:
|
192 | #
|
193 | # generate_c (251 lines)
|
194 | # parse_and_typecheck
|
195 | # compile_modules_to_c
|
196 |
|
197 | # mypyc/emitmodule.py (487 lines)
|
198 | # def compile_modules_to_c(result: BuildResult, module_names: List[str],
|
199 | # class ModuleGenerator:
|
200 | # # This generates a whole bunch of textual code!
|
201 |
|
202 | # literals, modules, errors = genops.build_ir(file_nodes, result.graph,
|
203 | # result.types)
|
204 |
|
205 | # TODO: Debug what comes out of here.
|
206 | #build.dump_graph(result.graph)
|
207 | #return
|
208 |
|
209 | # no-op
|
210 | if 0:
|
211 | for name in result.graph:
|
212 | log('result %s %s', name, result.graph[name])
|
213 | log('')
|
214 |
|
215 | # GLOBAL Constant pass over all modules. We want to collect duplicate
|
216 | # strings together. And have globally unique IDs str0, str1, ... strN.
|
217 | const_lookup = {} # Dict {StrExpr node => string name}
|
218 | const_code = []
|
219 | pass1 = const_pass.Collect(result.types, const_lookup, const_code)
|
220 |
|
221 | to_compile = list(ModulesToCompile(result, mod_names))
|
222 |
|
223 | # HACK: Why do I get oil.asdl.tdop in addition to asdl.tdop?
|
224 | #names = set(name for name, _ in to_compile)
|
225 |
|
226 | filtered = []
|
227 | seen = set()
|
228 | for name, module in to_compile:
|
229 | if name.startswith('oil.'):
|
230 | name = name[4:]
|
231 |
|
232 | # ditto with testpkg.module1
|
233 | if name.startswith('mycpp.'):
|
234 | name = name[6:]
|
235 |
|
236 | if name not in seen: # remove dupe
|
237 | filtered.append((name, module))
|
238 | seen.add(name)
|
239 |
|
240 | to_compile = filtered
|
241 |
|
242 | #import pickle
|
243 | if 0:
|
244 | for name, module in to_compile:
|
245 | log('to_compile %s', name)
|
246 | log('')
|
247 |
|
248 | # can't pickle but now I see deserialize() nodes and stuff
|
249 | #s = pickle.dumps(module)
|
250 | #log('%d pickle', len(s))
|
251 |
|
252 | # Print the tree for debugging
|
253 | if 0:
|
254 | for name, module in to_compile:
|
255 | builder = debug_pass.Print(result.types)
|
256 | builder.visit_mypy_file(module)
|
257 | return
|
258 |
|
259 | if opts.cc_out:
|
260 | f = open(opts.cc_out, 'w')
|
261 | else:
|
262 | f = sys.stdout
|
263 |
|
264 | f.write("""\
|
265 | // BEGIN mycpp output
|
266 |
|
267 | #include "mycpp/runtime.h"
|
268 |
|
269 | """)
|
270 |
|
271 | # Convert the mypy AST into our own IR.
|
272 | dot_exprs = {} # module name -> {expr node -> access type}
|
273 | log('\tmycpp pass: IR')
|
274 | for _, module in to_compile:
|
275 | p = ir_pass.Build(result.types)
|
276 | p.visit_mypy_file(module)
|
277 | dot_exprs[module.path] = p.dot_exprs
|
278 |
|
279 | # Collect constants and then emit code.
|
280 | log('\tmycpp pass: CONST')
|
281 | for name, module in to_compile:
|
282 | pass1.visit_mypy_file(module)
|
283 |
|
284 | # Instead of top-level code, should we generate a function and call it from
|
285 | # main?
|
286 | for line in const_code:
|
287 | f.write('%s\n' % line)
|
288 | f.write('\n')
|
289 |
|
290 | # Note: doesn't take into account module names!
|
291 | virtual = pass_state.Virtual()
|
292 |
|
293 | if opts.header_out:
|
294 | header_f = open(opts.header_out, 'w') # Not closed
|
295 |
|
296 | log('\tmycpp pass: FORWARD DECL')
|
297 |
|
298 | # Forward declarations first.
|
299 | # class Foo; class Bar;
|
300 | for name, module in to_compile:
|
301 | #log('forward decl name %s', name)
|
302 | if name in to_header:
|
303 | out_f = header_f
|
304 | else:
|
305 | out_f = f
|
306 | p2 = cppgen_pass.Generate(result.types,
|
307 | const_lookup,
|
308 | out_f,
|
309 | virtual=virtual,
|
310 | forward_decl=True,
|
311 | dot_exprs=dot_exprs[module.path])
|
312 |
|
313 | p2.visit_mypy_file(module)
|
314 | MaybeExitWithErrors(p2)
|
315 |
|
316 | # After seeing class and method names in the first pass, figure out which
|
317 | # ones are virtual. We use this info in the second pass.
|
318 | virtual.Calculate()
|
319 | if 0:
|
320 | log('virtuals %s', virtual.virtuals)
|
321 | log('has_vtable %s', virtual.has_vtable)
|
322 |
|
323 | local_vars = {} # FuncDef node -> (name, c_type) list
|
324 | ctx_member_vars = {
|
325 | } # Dict[ClassDef node for ctx_Foo, Dict[member_name: str, Type]]
|
326 |
|
327 | log('\tmycpp pass: PROTOTYPES')
|
328 |
|
329 | # First generate ALL C++ declarations / "headers".
|
330 | # class Foo { void method(); }; class Bar { void method(); };
|
331 | for name, module in to_compile:
|
332 | #log('decl name %s', name)
|
333 | if name in to_header:
|
334 | out_f = header_f
|
335 | else:
|
336 | out_f = f
|
337 | p3 = cppgen_pass.Generate(result.types,
|
338 | const_lookup,
|
339 | out_f,
|
340 | local_vars=local_vars,
|
341 | ctx_member_vars=ctx_member_vars,
|
342 | virtual=virtual,
|
343 | decl=True,
|
344 | dot_exprs=dot_exprs[module.path])
|
345 |
|
346 | p3.visit_mypy_file(module)
|
347 | MaybeExitWithErrors(p3)
|
348 |
|
349 | if 0:
|
350 | log('\tctx_member_vars')
|
351 | from pprint import pformat
|
352 | print(pformat(ctx_member_vars), file=sys.stderr)
|
353 |
|
354 | log('\tmycpp pass: CONTROL FLOW')
|
355 |
|
356 | cfgs = {} # fully qualified function name -> control flow graph
|
357 | for name, module in to_compile:
|
358 | cfg_pass = control_flow_pass.Build(result.types, virtual, local_vars,
|
359 | dot_exprs[module.path])
|
360 | cfg_pass.visit_mypy_file(module)
|
361 | cfgs.update(cfg_pass.cfgs)
|
362 |
|
363 | pass_state.DumpControlFlowGraphs(cfgs)
|
364 |
|
365 | log('\tmycpp pass: IMPL')
|
366 |
|
367 | # Now the definitions / implementations.
|
368 | # void Foo:method() { ... }
|
369 | # void Bar:method() { ... }
|
370 | for name, module in to_compile:
|
371 | p4 = cppgen_pass.Generate(result.types,
|
372 | const_lookup,
|
373 | f,
|
374 | local_vars=local_vars,
|
375 | ctx_member_vars=ctx_member_vars,
|
376 | stack_roots_warn=opts.stack_roots_warn,
|
377 | dot_exprs=dot_exprs[module.path])
|
378 | p4.visit_mypy_file(module)
|
379 | MaybeExitWithErrors(p4)
|
380 |
|
381 | return 0 # success
|
382 |
|
383 |
|
384 | def MaybeExitWithErrors(p):
|
385 | # Check for errors we collected
|
386 | num_errors = len(p.errors_keep_going)
|
387 | if num_errors != 0:
|
388 | log('')
|
389 | log('%s: %d translation errors (after type checking)', sys.argv[0],
|
390 | num_errors)
|
391 |
|
392 | # A little hack to tell the test-invalid-examples harness how many errors we had
|
393 | sys.exit(min(num_errors, 255))
|
394 |
|
395 |
|
396 | if __name__ == '__main__':
|
397 | try:
|
398 | sys.exit(main(sys.argv))
|
399 | except RuntimeError as e:
|
400 | print('FATAL: %s' % e, file=sys.stderr)
|
401 | sys.exit(1)
|