OILS / opy / _regtest / src / bin / oil.py View on Github | oilshell.org

623 lines, 407 significant
1#!/usr/bin/env python
2# Copyright 2016 Andy Chu. All rights reserved.
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8from __future__ import print_function
9"""
10oil.py - A busybox-like binary for oil.
11
12Based on argv[0], it acts like a few different programs.
13
14Builtins that can be exposed:
15
16- test / [ -- call BoolParser at runtime
17- 'time' -- because it has format strings, etc.
18- find/xargs equivalents (even if they are not compatible)
19 - list/each/every
20
21- echo: most likely don't care about this
22"""
23
24import os
25import sys
26import time # for perf measurement
27
28# TODO: Set PYTHONPATH from outside?
29this_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
30sys.path.append(os.path.join(this_dir, '..'))
31
32_trace_path = os.environ.get('_PY_TRACE')
33if _trace_path:
34 from benchmarks import pytrace
35 _tracer = pytrace.Tracer()
36 _tracer.Start()
37else:
38 _tracer = None
39
40# Uncomment this to see startup time problems.
41if os.environ.get('OIL_TIMING'):
42 start_time = time.time()
43 def _tlog(msg):
44 pid = os.getpid() # TODO: Maybe remove PID later.
45 print('[%d] %.3f %s' % (pid, (time.time() - start_time) * 1000, msg))
46else:
47 def _tlog(msg):
48 pass
49
50_tlog('before imports')
51
52import errno
53#import traceback # for debugging
54
55# Set in Modules/main.c.
56HAVE_READLINE = os.getenv('_HAVE_READLINE') != ''
57
58from asdl import format as fmt
59from asdl import encode
60
61from osh import word_parse # for tracing
62from osh import cmd_parse # for tracing
63
64from osh import ast_lib
65from osh import parse_lib
66
67from core import alloc
68from core import args
69from core import builtin
70from core import cmd_exec
71from osh.meta import Id
72from core import legacy
73from core import lexer # for tracing
74from core import process
75from core import reader
76from core import state
77from core import word
78from core import word_eval
79from core import ui
80from core import util
81
82if HAVE_READLINE:
83 from core import completion
84else:
85 completion = None
86
87from tools import deps
88from tools import osh2oil
89
90log = util.log
91
92_tlog('after imports')
93
94
95def InteractiveLoop(opts, ex, c_parser, w_parser, line_reader):
96 if opts.show_ast:
97 ast_f = fmt.DetectConsoleOutput(sys.stdout)
98 else:
99 ast_f = None
100
101 status = 0
102 while True:
103 try:
104 w = c_parser.Peek()
105 except KeyboardInterrupt:
106 print('Ctrl-C')
107 break
108
109 if w is None:
110 raise RuntimeError('Failed parse: %s' % c_parser.Error())
111 c_id = word.CommandId(w)
112 if c_id == Id.Op_Newline:
113 print('nothing to execute')
114 elif c_id == Id.Eof_Real:
115 print('EOF')
116 break
117 else:
118 node = c_parser.ParseCommandLine()
119
120 # TODO: Need an error for an empty command, which we ignore? GetLine
121 # could do that in the first position?
122 # ParseSimpleCommand fails with '\n' token?
123 if not node:
124 # TODO: PrintError here
125 raise RuntimeError('failed parse: %s' % c_parser.Error())
126
127 if ast_f:
128 ast_lib.PrettyPrint(node)
129
130 status, is_control_flow = ex.ExecuteAndCatch(node)
131 if is_control_flow: # exit or return
132 break
133
134 if opts.print_status:
135 print('STATUS', repr(status))
136
137 # Reset prompt to PS1.
138 line_reader.Reset()
139
140 # Reset internal newline state.
141 # NOTE: It would actually be correct to reinitialize all objects (except
142 # Env) on every iteration. But we know that the w_parser is the only thing
143 # that needs to be reset, for now.
144 w_parser.Reset()
145 c_parser.Reset()
146
147 return status
148
149
150# bash --noprofile --norc uses 'bash-4.3$ '
151OSH_PS1 = 'osh$ '
152
153
154def _ShowVersion():
155 util.ShowAppVersion('Oil')
156
157
158def OshMain(argv0, argv, login_shell):
159 spec = args.FlagsAndOptions()
160 spec.ShortFlag('-c', args.Str, quit_parsing_flags=True) # command string
161 spec.ShortFlag('-i') # interactive
162
163 # TODO: -h too
164 spec.LongFlag('--help')
165 spec.LongFlag('--version')
166 spec.LongFlag('--ast-format',
167 ['text', 'abbrev-text', 'html', 'abbrev-html', 'oheap', 'none'],
168 default='abbrev-text')
169 spec.LongFlag('--show-ast') # execute and show
170 spec.LongFlag('--fix')
171 spec.LongFlag('--debug-spans') # For oshc translate
172 spec.LongFlag('--print-status')
173 spec.LongFlag('--trace', ['cmd-parse', 'word-parse', 'lexer']) # NOTE: can only trace one now
174 spec.LongFlag('--hijack-shebang')
175
176 # For benchmarks/*.sh
177 spec.LongFlag('--parser-mem-dump', args.Str)
178 spec.LongFlag('--runtime-mem-dump', args.Str)
179
180 builtin.AddOptionsToArgSpec(spec)
181
182 try:
183 opts, opt_index = spec.Parse(argv)
184 except args.UsageError as e:
185 util.usage(str(e))
186 return 2
187
188 if opts.help:
189 loader = util.GetResourceLoader()
190 builtin.Help(['osh-usage'], loader)
191 return 0
192 if opts.version:
193 # OSH version is the only binary in Oil right now, so it's all one version.
194 _ShowVersion()
195 return 0
196
197 trace_state = util.TraceState()
198 if 'cmd-parse' == opts.trace:
199 util.WrapMethods(cmd_parse.CommandParser, trace_state)
200 if 'word-parse' == opts.trace:
201 util.WrapMethods(word_parse.WordParser, trace_state)
202 if 'lexer' == opts.trace:
203 util.WrapMethods(lexer.Lexer, trace_state)
204
205 if opt_index == len(argv):
206 dollar0 = argv0
207 else:
208 dollar0 = argv[opt_index] # the script name, or the arg after -c
209
210 # TODO: Create a --parse action or 'osh parse' or 'oil osh-parse'
211 # osh-fix
212 # It uses a different memory-management model. It's a batch program and not
213 # an interactive program.
214
215 pool = alloc.Pool()
216 arena = pool.NewArena()
217
218 # TODO: Maybe wrap this initialization sequence up in an oil_State, like
219 # lua_State.
220 status_lines = ui.MakeStatusLines()
221 mem = state.Mem(dollar0, argv[opt_index + 1:], os.environ, arena)
222 funcs = {}
223
224 # Passed to Executor for 'complete', and passed to completion.Init
225 if completion:
226 comp_lookup = completion.CompletionLookup()
227 else:
228 # TODO: NullLookup?
229 comp_lookup = None
230
231 exec_opts = state.ExecOpts(mem)
232 builtin.SetExecOpts(exec_opts, opts.opt_changes)
233
234 fd_state = process.FdState()
235 ex = cmd_exec.Executor(mem, fd_state, status_lines, funcs, completion,
236 comp_lookup, exec_opts, arena)
237
238 # NOTE: The rc file can contain both commands and functions... ideally we
239 # would only want to save nodes/lines for the functions.
240 try:
241 rc_path = 'oilrc'
242 arena.PushSource(rc_path)
243 with open(rc_path) as f:
244 rc_line_reader = reader.FileLineReader(f, arena)
245 _, rc_c_parser = parse_lib.MakeParser(rc_line_reader, arena)
246 try:
247 rc_node = rc_c_parser.ParseWholeFile()
248 if not rc_node:
249 err = rc_c_parser.Error()
250 ui.PrintErrorStack(err, arena, sys.stderr)
251 return 2 # parse error is code 2
252 finally:
253 arena.PopSource()
254
255 status = ex.Execute(rc_node)
256 #print('oilrc:', status, cflow, file=sys.stderr)
257 # Ignore bad status?
258 except IOError as e:
259 if e.errno != errno.ENOENT:
260 raise
261
262 if opts.c is not None:
263 arena.PushSource('<command string>')
264 line_reader = reader.StringLineReader(opts.c, arena)
265 interactive = False
266 elif opts.i: # force interactive
267 arena.PushSource('<stdin -i>')
268 line_reader = reader.InteractiveLineReader(OSH_PS1, arena)
269 interactive = True
270 else:
271 try:
272 script_name = argv[opt_index]
273 except IndexError:
274 if sys.stdin.isatty():
275 arena.PushSource('<interactive>')
276 line_reader = reader.InteractiveLineReader(OSH_PS1, arena)
277 interactive = True
278 else:
279 arena.PushSource('<stdin>')
280 line_reader = reader.FileLineReader(sys.stdin, arena)
281 interactive = False
282 else:
283 arena.PushSource(script_name)
284 try:
285 f = fd_state.Open(script_name)
286 except OSError as e:
287 util.error("Couldn't open %r: %s", script_name, os.strerror(e.errno))
288 return 1
289 line_reader = reader.FileLineReader(f, arena)
290 interactive = False
291
292 # TODO: assert arena.NumSourcePaths() == 1
293 # TODO: .rc file needs its own arena.
294 w_parser, c_parser = parse_lib.MakeParser(line_reader, arena)
295
296 if interactive:
297 # NOTE: We're using a different evaluator here. The completion system can
298 # also run functions... it gets the Executor through Executor._Complete.
299 if HAVE_READLINE:
300 splitter = legacy.SplitContext(mem)
301 ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter)
302 status_out = completion.StatusOutput(status_lines, exec_opts)
303 completion.Init(pool, builtin.BUILTIN_DEF, mem, funcs, comp_lookup,
304 status_out, ev)
305
306 return InteractiveLoop(opts, ex, c_parser, w_parser, line_reader)
307 else:
308 # Parse the whole thing up front
309 #print('Parsing file')
310
311 _tlog('ParseWholeFile')
312 # TODO: Do I need ParseAndEvalLoop? How is it different than
313 # InteractiveLoop?
314 try:
315 node = c_parser.ParseWholeFile()
316 except util.ParseError as e:
317 ui.PrettyPrintError(e, arena, sys.stderr)
318 print('parse error: %s' % e.UserErrorString(), file=sys.stderr)
319 return 2
320 else:
321 # TODO: Remove this older form of error handling.
322 if not node:
323 err = c_parser.Error()
324 assert err, err # can't be empty
325 ui.PrintErrorStack(err, arena, sys.stderr)
326 return 2 # parse error is code 2
327
328 do_exec = True
329 if opts.fix:
330 #log('SPANS: %s', arena.spans)
331 osh2oil.PrintAsOil(arena, node, opts.debug_spans)
332 do_exec = False
333 if exec_opts.noexec:
334 do_exec = False
335
336 # Do this after parsing the entire file. There could be another option to
337 # do it before exiting runtime?
338 if opts.parser_mem_dump:
339 # This might be superstition, but we want to let the value stabilize
340 # after parsing. bash -c 'cat /proc/$$/status' gives different results
341 # with a sleep.
342 time.sleep(0.001)
343 input_path = '/proc/%d/status' % os.getpid()
344 with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
345 contents = f.read()
346 f2.write(contents)
347 log('Wrote %s to %s (--parser-mem-dump)', input_path,
348 opts.parser_mem_dump)
349
350 # -n prints AST, --show-ast prints and executes
351 if exec_opts.noexec or opts.show_ast:
352 if opts.ast_format == 'none':
353 print('AST not printed.', file=sys.stderr)
354 elif opts.ast_format == 'oheap':
355 # TODO: Make this a separate flag?
356 if sys.stdout.isatty():
357 raise RuntimeError('ERROR: Not dumping binary data to a TTY.')
358 f = sys.stdout
359
360 enc = encode.Params()
361 out = encode.BinOutput(f)
362 encode.EncodeRoot(node, enc, out)
363
364 else: # text output
365 f = sys.stdout
366
367 if opts.ast_format in ('text', 'abbrev-text'):
368 ast_f = fmt.DetectConsoleOutput(f)
369 elif opts.ast_format in ('html', 'abbrev-html'):
370 ast_f = fmt.HtmlOutput(f)
371 else:
372 raise AssertionError
373 abbrev_hook = (
374 ast_lib.AbbreviateNodes if 'abbrev-' in opts.ast_format else None)
375 tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook)
376 ast_f.FileHeader()
377 fmt.PrintTree(tree, ast_f)
378 ast_f.FileFooter()
379 ast_f.write('\n')
380
381 #util.log("Execution skipped because 'noexec' is on ")
382 status = 0
383
384 if do_exec:
385 _tlog('Execute(node)')
386 status = ex.ExecuteAndRunExitTrap(node)
387 # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
388 # don't because they call sys.exit().
389 if opts.runtime_mem_dump:
390 # This might be superstition, but we want to let the value stabilize
391 # after parsing. bash -c 'cat /proc/$$/status' gives different results
392 # with a sleep.
393 time.sleep(0.001)
394 input_path = '/proc/%d/status' % os.getpid()
395 with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
396 contents = f.read()
397 f2.write(contents)
398 log('Wrote %s to %s (--runtime-mem-dump)', input_path,
399 opts.runtime_mem_dump)
400
401 else:
402 status = 0
403
404 return status
405
406
407def OilMain(argv):
408 spec = args.FlagsAndOptions()
409 # TODO: -h too
410 spec.LongFlag('--help')
411 spec.LongFlag('--version')
412 #builtin.AddOptionsToArgSpec(spec)
413
414 try:
415 opts, opt_index = spec.Parse(argv)
416 except args.UsageError as e:
417 util.usage(str(e))
418 return 2
419
420 if opts.help:
421 loader = util.GetResourceLoader()
422 builtin.Help(['oil-usage'], loader)
423 return 0
424 if opts.version:
425 # OSH version is the only binary in Oil right now, so it's all one version.
426 _ShowVersion()
427 return 0
428
429 raise NotImplementedError('oil')
430 return 0
431
432
433def WokMain(main_argv):
434 raise NotImplementedError('wok')
435
436
437def BoilMain(main_argv):
438 raise NotImplementedError('boil')
439
440
441# TODO: Hook up to completion.
442SUBCOMMANDS = ['translate', 'format', 'deps', 'undefined-vars']
443
444def OshCommandMain(argv):
445 """Run an 'oshc' tool.
446
447 'osh' is short for "osh compiler" or "osh command".
448
449 TODO:
450 - oshc --help
451
452 oshc deps
453 --path: the $PATH to use to find executables. What about libraries?
454
455 NOTE: we're leaving out su -c, find, xargs, etc.? Those should generally
456 run functions using the $0 pattern.
457 --chained-command sudo
458 """
459 try:
460 action = argv[0]
461 except IndexError:
462 raise args.UsageError('oshc: Missing required subcommand.')
463
464 if action not in SUBCOMMANDS:
465 raise args.UsageError('oshc: Invalid subcommand %r.' % action)
466
467 try:
468 script_name = argv[1]
469 except IndexError:
470 script_name = '<stdin>'
471 f = sys.stdin
472 else:
473 try:
474 f = open(script_name)
475 except IOError as e:
476 util.error("Couldn't open %r: %s", script_name, os.strerror(e.errno))
477 return 2
478
479 pool = alloc.Pool()
480 arena = pool.NewArena()
481 arena.PushSource(script_name)
482
483 line_reader = reader.FileLineReader(f, arena)
484 _, c_parser = parse_lib.MakeParser(line_reader, arena)
485
486 try:
487 node = c_parser.ParseWholeFile()
488 except util.ParseError as e:
489 ui.PrettyPrintError(e, arena, sys.stderr)
490 print('parse error: %s' % e.UserErrorString(), file=sys.stderr)
491 return 2
492 else:
493 # TODO: Remove this older form of error handling.
494 if not node:
495 err = c_parser.Error()
496 assert err, err # can't be empty
497 ui.PrintErrorStack(err, arena, sys.stderr)
498 return 2 # parse error is code 2
499
500 f.close()
501
502 # Columns for list-*
503 # path line name
504 # where name is the binary path, variable name, or library path.
505
506 # bin-deps and lib-deps can be used to make an app bundle.
507 # Maybe I should list them together? 'deps' can show 4 columns?
508 #
509 # path, line, type, name
510 #
511 # --pretty can show the LST location.
512
513 # stderr: show how we're following imports?
514
515 if action == 'translate':
516 # TODO: FIx this invocation up.
517 #debug_spans = opt.debug_spans
518 debug_spans = False
519 osh2oil.PrintAsOil(arena, node, debug_spans)
520
521 elif action == 'format':
522 # TODO: autoformat code
523 raise NotImplementedError(action)
524
525 elif action == 'deps':
526 deps.Deps(node)
527
528 elif action == 'undefined-vars': # could be environment variables
529 pass
530
531 else:
532 raise AssertionError # Checked above
533
534 return 0
535
536
537# The valid applets right now.
538# TODO: Hook up to completion.
539APPLETS = ['osh', 'oshc']
540
541
542def AppBundleMain(argv):
543 login_shell = False
544
545 b = os.path.basename(argv[0])
546 main_name, ext = os.path.splitext(b)
547 if main_name.startswith('-'):
548 login_shell = True
549 main_name = main_name[1:]
550
551 if main_name == 'oil' and ext: # oil.py or oil.ovm
552 try:
553 first_arg = argv[1]
554 except IndexError:
555 raise args.UsageError('Missing required applet name.')
556
557 if first_arg in ('-h', '--help'):
558 builtin.Help(['bundle-usage'], util.GetResourceLoader())
559 sys.exit(0)
560
561 if first_arg in ('-V', '--version'):
562 _ShowVersion()
563 sys.exit(0)
564
565 main_name = first_arg
566 if main_name.startswith('-'): # TODO: Remove duplication above
567 login_shell = True
568 main_name = main_name[1:]
569 argv0 = argv[1]
570 main_argv = argv[2:]
571 else:
572 argv0 = argv[0]
573 main_argv = argv[1:]
574
575 if main_name in ('osh', 'sh'):
576 status = OshMain(argv0, main_argv, login_shell)
577 _tlog('done osh main')
578 return status
579 elif main_name == 'oshc':
580 return OshCommandMain(main_argv)
581
582 elif main_name == 'oil':
583 return OilMain(main_argv)
584 elif main_name == 'wok':
585 return WokMain(main_argv)
586 elif main_name == 'boil':
587 return BoilMain(main_argv)
588
589 # For testing latency
590 elif main_name == 'true':
591 return 0
592 elif main_name == 'false':
593 return 1
594 else:
595 raise args.UsageError('Invalid applet name %r.' % main_name)
596
597
598def main(argv):
599 try:
600 sys.exit(AppBundleMain(argv))
601 except NotImplementedError as e:
602 raise
603 except args.UsageError as e:
604 #builtin.Help(['oil-usage'], util.GetResourceLoader())
605 log('oil: %s', e)
606 sys.exit(2)
607 except RuntimeError as e:
608 log('FATAL: %s', e)
609 sys.exit(1)
610 finally:
611 _tlog('Exiting main()')
612 if _trace_path:
613 _tracer.Stop(_trace_path)
614
615
616if __name__ == '__main__':
617 # NOTE: This could end up as opy.InferTypes(), opy.GenerateCode(), etc.
618 if os.getenv('CALLGRAPH') == '1':
619 from opy import callgraph
620 callgraph.Walk(main, sys.modules)
621 else:
622 main(sys.argv)
623