OILS / core / shell.py View on Github | oilshell.org

1139 lines, 733 significant
1"""
2core/shell.py -- Entry point for the shell interpreter.
3"""
4from __future__ import print_function
5
6from errno import ENOENT
7import time as time_
8
9from _devbuild.gen import arg_types
10from _devbuild.gen.option_asdl import option_i, builtin_i
11from _devbuild.gen.runtime_asdl import scope_e
12from _devbuild.gen.syntax_asdl import (loc, source, source_t, IntParamBox,
13 debug_frame, debug_frame_t)
14from _devbuild.gen.value_asdl import (value, value_e)
15from core import alloc
16from core import comp_ui
17from core import dev
18from core import error
19from core import executor
20from core import completion
21from core import main_loop
22from core import optview
23from core import pyos
24from core import process
25from core import pyutil
26from core import state
27from core import ui
28from core import util
29from core import vm
30
31from frontend import args
32from frontend import flag_def # side effect: flags are defined!
33
34unused1 = flag_def
35from frontend import flag_util
36from frontend import location
37from frontend import reader
38from frontend import parse_lib
39
40from builtin import assign_osh
41from builtin import bracket_osh
42from builtin import completion_osh
43from builtin import completion_ysh
44from builtin import dirs_osh
45from builtin import error_ysh
46from builtin import hay_ysh
47from builtin import io_osh
48from builtin import io_ysh
49from builtin import json_ysh
50from builtin import meta_osh
51from builtin import misc_osh
52from builtin import module_ysh
53from builtin import printf_osh
54from builtin import process_osh
55from builtin import pure_osh
56from builtin import pure_ysh
57from builtin import readline_osh
58from builtin import read_osh
59from builtin import trap_osh
60
61from builtin import func_eggex
62from builtin import func_hay
63from builtin import func_misc
64
65from builtin import method_dict
66from builtin import method_io
67from builtin import method_list
68from builtin import method_other
69from builtin import method_str
70
71from osh import cmd_eval
72from osh import glob_
73from osh import history
74from osh import prompt
75from osh import sh_expr_eval
76from osh import split
77from osh import word_eval
78
79from mycpp import mops
80from mycpp import mylib
81from mycpp.mylib import print_stderr, log
82from pylib import os_path
83from tools import deps
84from tools import fmt
85from tools import ysh_ify
86from ysh import expr_eval
87
88unused2 = log
89
90import libc
91import posix_ as posix
92
93from typing import List, Dict, Optional, TYPE_CHECKING, cast
94if TYPE_CHECKING:
95 from frontend.py_readline import Readline
96
97if mylib.PYTHON:
98 try:
99 from _devbuild.gen import help_meta # type: ignore
100 except ImportError:
101 help_meta = None
102
103
104def _InitDefaultCompletions(cmd_ev, complete_builtin, comp_lookup):
105 # type: (cmd_eval.CommandEvaluator, completion_osh.Complete, completion.Lookup) -> None
106
107 # register builtins and words
108 complete_builtin.Run(cmd_eval.MakeBuiltinArgv(['-E', '-A', 'command']))
109 # register path completion
110 # Add -o filenames? Or should that be automatic?
111 complete_builtin.Run(cmd_eval.MakeBuiltinArgv(['-D', '-A', 'file']))
112
113
114def _CompletionDemo(comp_lookup):
115 # type: (completion.Lookup) -> None
116
117 # Something for fun, to show off. Also: test that you don't repeatedly hit
118 # the file system / network / coprocess.
119 A1 = completion.TestAction(['foo.py', 'foo', 'bar.py'], 0.0)
120 l = [] # type: List[str]
121 for i in xrange(0, 5):
122 l.append('m%d' % i)
123
124 A2 = completion.TestAction(l, 0.1)
125 C1 = completion.UserSpec([A1, A2], [], [], completion.DefaultPredicate(),
126 '', '')
127 comp_lookup.RegisterName('slowc', {}, C1)
128
129
130def SourceStartupFile(
131 fd_state, # type: process.FdState
132 rc_path, # type: str
133 lang, # type: str
134 parse_ctx, # type: parse_lib.ParseContext
135 cmd_ev, # type: cmd_eval.CommandEvaluator
136 errfmt, # type: ui.ErrorFormatter
137):
138 # type: (...) -> None
139
140 # Right now this is called when the shell is interactive. (Maybe it should
141 # be called on login_shel too.)
142 #
143 # Terms:
144 # - interactive shell: Roughly speaking, no args or -c, and isatty() is true
145 # for stdin and stdout.
146 # - login shell: Started from the top level, e.g. from init or ssh.
147 #
148 # We're not going to copy everything bash does because it's too complex, but
149 # for reference:
150 # https://www.gnu.org/software/bash/manual/bash.html#Bash-Startup-Files
151 # Bash also has --login.
152
153 try:
154 f = fd_state.Open(rc_path)
155 except (IOError, OSError) as e:
156 # TODO: Could warn about nonexistent explicit --rcfile?
157 if e.errno != ENOENT:
158 raise # Goes to top level. Handle this better?
159 return
160
161 arena = parse_ctx.arena
162 rc_line_reader = reader.FileLineReader(f, arena)
163 rc_c_parser = parse_ctx.MakeOshParser(rc_line_reader)
164
165 with alloc.ctx_SourceCode(arena, source.SourcedFile(rc_path, loc.Missing)):
166 # TODO: handle status, e.g. 2 for ParseError
167 unused = main_loop.Batch(cmd_ev, rc_c_parser, errfmt)
168
169 f.close()
170
171
172class ShellOptHook(state.OptHook):
173
174 def __init__(self, readline):
175 # type: (Optional[Readline]) -> None
176 self.readline = readline
177
178 def OnChange(self, opt0_array, opt_name, b):
179 # type: (List[bool], str, bool) -> bool
180 """This method is called whenever an option is changed.
181
182 Returns success or failure.
183 """
184 if opt_name == 'vi' or opt_name == 'emacs':
185 # TODO: Replace with a hook? Just like setting LANG= can have a hook.
186 if self.readline:
187 self.readline.parse_and_bind("set editing-mode " + opt_name)
188 else:
189 print_stderr(
190 "Warning: Can't set option %r because shell wasn't compiled with GNU readline"
191 % opt_name)
192 return False
193
194 # Invert: they are mutually exclusive!
195 if opt_name == 'vi':
196 opt0_array[option_i.emacs] = not b
197 elif opt_name == 'emacs':
198 opt0_array[option_i.vi] = not b
199
200 return True
201
202
203def _SetGlobalFunc(mem, name, func):
204 # type: (state.Mem, str, vm._Callable) -> None
205 assert isinstance(func, vm._Callable), func
206
207 # Note: no location info for builtin functions?
208 mem.SetNamed(location.LName(name), value.BuiltinFunc(func),
209 scope_e.GlobalOnly)
210
211
212def InitAssignmentBuiltins(
213 mem, # type: state.Mem
214 procs, # type: state.Procs
215 exec_opts, # type: optview.Exec
216 errfmt, # type: ui.ErrorFormatter
217):
218 # type: (...) -> Dict[int, vm._AssignBuiltin]
219
220 assign_b = {} # type: Dict[int, vm._AssignBuiltin]
221
222 new_var = assign_osh.NewVar(mem, procs, exec_opts, errfmt)
223 assign_b[builtin_i.declare] = new_var
224 assign_b[builtin_i.typeset] = new_var
225 assign_b[builtin_i.local] = new_var
226
227 assign_b[builtin_i.export_] = assign_osh.Export(mem, errfmt)
228 assign_b[builtin_i.readonly] = assign_osh.Readonly(mem, errfmt)
229
230 return assign_b
231
232
233class ShellFiles(object):
234
235 def __init__(self, lang, home_dir, mem, flag):
236 # type: (str, str, state.Mem, arg_types.main) -> None
237 assert lang in ('osh', 'ysh'), lang
238 self.lang = lang
239 self.home_dir = home_dir
240 self.mem = mem
241 self.flag = flag
242
243 def _HistVar(self):
244 # type: () -> str
245 return 'HISTFILE' if self.lang == 'osh' else 'YSH_HISTFILE'
246
247 def _DefaultHistoryFile(self):
248 # type: () -> str
249 return os_path.join(self.home_dir,
250 '.local/share/oils/%s_history' % self.lang)
251
252 def InitAfterLoadingEnv(self):
253 # type: () -> None
254
255 hist_var = self._HistVar()
256 if self.mem.GetValue(hist_var).tag() == value_e.Undef:
257 # Note: if the directory doesn't exist, GNU readline ignores
258 state.SetGlobalString(self.mem, hist_var,
259 self._DefaultHistoryFile())
260
261 def HistoryFile(self):
262 # type: () -> Optional[str]
263 # TODO: In non-strict mode we should try to cast the HISTFILE value to a
264 # string following bash's rules
265
266 UP_val = self.mem.GetValue(self._HistVar())
267 if UP_val.tag() == value_e.Str:
268 val = cast(value.Str, UP_val)
269 return val.s
270 else:
271 # Note: if HISTFILE is an array, bash will return ${HISTFILE[0]}
272 return None
273 #return self._DefaultHistoryFile()
274
275 # TODO: can we recover line information here?
276 # might be useful to show where HISTFILE was set
277 #raise error.Strict("$HISTFILE should only ever be a string", loc.Missing)
278
279
280def Main(
281 lang, # type: str
282 arg_r, # type: args.Reader
283 environ, # type: Dict[str, str]
284 login_shell, # type: bool
285 loader, # type: pyutil._ResourceLoader
286 readline, # type: Optional[Readline]
287):
288 # type: (...) -> int
289 """The full shell lifecycle. Used by bin/osh and bin/ysh.
290
291 Args:
292 lang: 'osh' or 'ysh'
293 login_shell: Was - on argv[0]?
294 loader: to get help, version, grammar, etc.
295 readline: optional GNU readline
296 """
297 # Differences between osh and ysh:
298 # - oshrc vs yshrc
299 # - shopt -s ysh:all
300 # - Prompt
301 # - --help
302
303 argv0 = arg_r.Peek()
304 assert argv0 is not None
305 arg_r.Next()
306
307 assert lang in ('osh', 'ysh'), lang
308
309 try:
310 attrs = flag_util.ParseMore('main', arg_r)
311 except error.Usage as e:
312 print_stderr('%s usage error: %s' % (lang, e.msg))
313 return 2
314 flag = arg_types.main(attrs.attrs)
315
316 arena = alloc.Arena()
317 errfmt = ui.ErrorFormatter()
318
319 if flag.help:
320 util.HelpFlag(loader, '%s-usage' % lang, mylib.Stdout())
321 return 0
322 if flag.version:
323 util.VersionFlag(loader, mylib.Stdout())
324 return 0
325
326 if flag.tool == 'cat-em':
327 paths = arg_r.Rest()
328
329 status = 0
330 for p in paths:
331 try:
332 contents = loader.Get(p)
333 print(contents)
334 except (OSError, IOError):
335 print_stderr("cat-em: %r not found" % p)
336 status = 1
337 return status
338
339 debug_stack = [] # type: List[debug_frame_t]
340 if arg_r.AtEnd():
341 dollar0 = argv0
342 else:
343 dollar0 = arg_r.Peek() # the script name, or the arg after -c
344
345 frame0 = debug_frame.Main(dollar0)
346 debug_stack.append(frame0)
347
348 script_name = arg_r.Peek() # type: Optional[str]
349 arg_r.Next()
350 mem = state.Mem(dollar0, arg_r.Rest(), arena, debug_stack)
351
352 opt_hook = ShellOptHook(readline)
353 # Note: only MutableOpts needs mem, so it's not a true circular dep.
354 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, opt_hook)
355 mem.exec_opts = exec_opts # circular dep
356 mutable_opts.Init()
357
358 version_str = pyutil.GetVersion(loader)
359 state.InitMem(mem, environ, version_str)
360
361 if attrs.show_options: # special case: sh -o
362 mutable_opts.ShowOptions([])
363 return 0
364
365 # Set these BEFORE processing flags, so they can be overridden.
366 if lang == 'ysh':
367 mutable_opts.SetAnyOption('ysh:all', True)
368
369 pure_osh.SetOptionsFromFlags(mutable_opts, attrs.opt_changes,
370 attrs.shopt_changes)
371
372 # feedback between runtime and parser
373 aliases = {} # type: Dict[str, str]
374
375 ysh_grammar = pyutil.LoadYshGrammar(loader)
376
377 if flag.do_lossless and not exec_opts.noexec():
378 raise error.Usage('--one-pass-parse requires noexec (-n)', loc.Missing)
379
380 # Tools always use one pass parse
381 # Note: osh --tool syntax-tree is like osh -n --one-pass-parse
382 do_lossless = True if len(flag.tool) else flag.do_lossless
383
384 parse_ctx = parse_lib.ParseContext(arena,
385 parse_opts,
386 aliases,
387 ysh_grammar,
388 do_lossless=do_lossless)
389
390 # Three ParseContext instances SHARE aliases.
391 comp_arena = alloc.Arena()
392 comp_arena.PushSource(source.Unused('completion'))
393 trail1 = parse_lib.Trail()
394 # do_lossless needs to be turned on to complete inside backticks. TODO:
395 # fix the issue where ` gets erased because it's not part of
396 # set_completer_delims().
397 comp_ctx = parse_lib.ParseContext(comp_arena,
398 parse_opts,
399 aliases,
400 ysh_grammar,
401 do_lossless=True)
402 comp_ctx.Init_Trail(trail1)
403
404 hist_arena = alloc.Arena()
405 hist_arena.PushSource(source.Unused('history'))
406 trail2 = parse_lib.Trail()
407 hist_ctx = parse_lib.ParseContext(hist_arena, parse_opts, aliases,
408 ysh_grammar)
409 hist_ctx.Init_Trail(trail2)
410
411 # Deps helps manages dependencies. These dependencies are circular:
412 # - cmd_ev and word_ev, arith_ev -- for command sub, arith sub
413 # - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
414 # - cmd_ev and builtins (which execute code, like eval)
415 # - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
416 cmd_deps = cmd_eval.Deps()
417 cmd_deps.mutable_opts = mutable_opts
418
419 job_control = process.JobControl()
420 job_list = process.JobList()
421 fd_state = process.FdState(errfmt, job_control, job_list, mem, None, None)
422
423 my_pid = posix.getpid()
424
425 debug_path = ''
426 debug_dir = environ.get('OILS_DEBUG_DIR')
427 if flag.debug_file is not None:
428 # --debug-file takes precedence over OSH_DEBUG_DIR
429 debug_path = flag.debug_file
430 elif debug_dir is not None:
431 debug_path = os_path.join(debug_dir, '%d-osh.log' % my_pid)
432
433 if len(debug_path):
434 # This will be created as an empty file if it doesn't exist, or it could be
435 # a pipe.
436 try:
437 debug_f = util.DebugFile(
438 fd_state.OpenForWrite(debug_path)) # type: util._DebugFile
439 except (IOError, OSError) as e:
440 print_stderr("%s: Couldn't open %r: %s" %
441 (lang, debug_path, posix.strerror(e.errno)))
442 return 2
443 else:
444 debug_f = util.NullDebugFile()
445
446 if flag.xtrace_to_debug_file:
447 trace_f = debug_f
448 else:
449 trace_f = util.DebugFile(mylib.Stderr())
450
451 trace_dir = environ.get('OILS_TRACE_DIR', '')
452 dumps = environ.get('OILS_TRACE_DUMPS', '')
453 streams = environ.get('OILS_TRACE_STREAMS', '')
454 multi_trace = dev.MultiTracer(my_pid, trace_dir, dumps, streams, fd_state)
455
456 tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, trace_f,
457 multi_trace)
458 fd_state.tracer = tracer # circular dep
459
460 signal_safe = pyos.InitSignalSafe()
461 trap_state = trap_osh.TrapState(signal_safe)
462
463 waiter = process.Waiter(job_list, exec_opts, signal_safe, tracer)
464 fd_state.waiter = waiter
465
466 cmd_deps.debug_f = debug_f
467
468 now = time_.time()
469 iso_stamp = time_.strftime("%Y-%m-%d %H:%M:%S", time_.localtime(now))
470
471 argv_buf = mylib.BufWriter()
472 dev.PrintShellArgv(arg_r.argv, argv_buf)
473
474 debug_f.writeln('%s [%d] Oils started with argv %s' %
475 (iso_stamp, my_pid, argv_buf.getvalue()))
476 if len(debug_path):
477 debug_f.writeln('Writing logs to %r' % debug_path)
478
479 interp = environ.get('OILS_HIJACK_SHEBANG', '')
480 search_path = state.SearchPath(mem)
481 ext_prog = process.ExternalProgram(interp, fd_state, errfmt, debug_f)
482
483 splitter = split.SplitContext(mem)
484 # TODO: This is instantiation is duplicated in osh/word_eval.py
485 globber = glob_.Globber(exec_opts)
486
487 # This could just be OILS_TRACE_DUMPS='crash:argv0'
488 crash_dump_dir = environ.get('OILS_CRASH_DUMP_DIR', '')
489 cmd_deps.dumper = dev.CrashDumper(crash_dump_dir, fd_state)
490
491 comp_lookup = completion.Lookup()
492
493 # Various Global State objects to work around readline interfaces
494 compopt_state = completion.OptionState()
495
496 comp_ui_state = comp_ui.State()
497 prompt_state = comp_ui.PromptState()
498
499 # The login program is supposed to set $HOME
500 # https://superuser.com/questions/271925/where-is-the-home-environment-variable-set
501 # state.InitMem(mem) must happen first
502 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
503 home_dir = tilde_ev.GetMyHomeDir()
504 if home_dir is None:
505 # TODO: print errno from getpwuid()
506 print_stderr("%s: Failed to get home dir from $HOME or getpwuid()" %
507 lang)
508 return 1
509
510 sh_files = ShellFiles(lang, home_dir, mem, flag)
511 sh_files.InitAfterLoadingEnv()
512
513 #
514 # Executor and Evaluators (are circularly dependent)
515 #
516
517 # Global proc namespace. Funcs are defined in the common variable
518 # namespace.
519 procs = state.Procs(mem) # type: state.Procs
520
521 builtins = {} # type: Dict[int, vm._Builtin]
522
523 # e.g. s->startswith()
524 methods = {} # type: Dict[int, Dict[str, vm._Callable]]
525
526 hay_state = hay_ysh.HayState()
527
528 shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
529 hay_state, builtins, search_path,
530 ext_prog, waiter, tracer, job_control,
531 job_list, fd_state, trap_state, errfmt)
532
533 arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, mutable_opts,
534 parse_ctx, errfmt)
535 bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, mutable_opts,
536 parse_ctx, errfmt)
537 expr_ev = expr_eval.ExprEvaluator(mem, mutable_opts, methods, splitter,
538 errfmt)
539 word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, mutable_opts,
540 tilde_ev, splitter, errfmt)
541
542 assign_b = InitAssignmentBuiltins(mem, procs, exec_opts, errfmt)
543 cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs, assign_b,
544 arena, cmd_deps, trap_state,
545 signal_safe)
546
547 # PromptEvaluator rendering is needed in non-interactive shells for @P.
548 prompt_ev = prompt.Evaluator(lang, version_str, parse_ctx, mem)
549 global_io = value.IO(cmd_ev, prompt_ev)
550 global_guts = value.Guts(None)
551
552 # Wire up circular dependencies.
553 vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
554 prompt_ev, global_io, tracer)
555
556 unsafe_arith = sh_expr_eval.UnsafeArith(mem, exec_opts, mutable_opts,
557 parse_ctx, arith_ev, errfmt)
558 vm.InitUnsafeArith(mem, word_ev, unsafe_arith)
559
560 #
561 # Initialize Built-in Procs
562 #
563
564 b = builtins # short alias for initialization
565
566 if mylib.PYTHON:
567 if help_meta:
568 help_data = help_meta.TopicMetadata()
569 else:
570 help_data = {} # minimal build
571 else:
572 help_data = help_meta.TopicMetadata()
573 b[builtin_i.help] = misc_osh.Help(lang, loader, help_data, errfmt)
574
575 # Interpreter state
576 b[builtin_i.set] = pure_osh.Set(mutable_opts, mem)
577 b[builtin_i.shopt] = pure_osh.Shopt(mutable_opts, cmd_ev)
578
579 b[builtin_i.hash] = pure_osh.Hash(search_path) # not really pure
580 b[builtin_i.trap] = trap_osh.Trap(trap_state, parse_ctx, tracer, errfmt)
581
582 b[builtin_i.shvar] = pure_ysh.Shvar(mem, search_path, cmd_ev)
583 b[builtin_i.ctx] = pure_ysh.Ctx(mem, cmd_ev)
584 b[builtin_i.push_registers] = pure_ysh.PushRegisters(mem, cmd_ev)
585
586 # Hay
587 b[builtin_i.hay] = hay_ysh.Hay(hay_state, mutable_opts, mem, cmd_ev)
588 b[builtin_i.haynode] = hay_ysh.HayNode_(hay_state, mem, cmd_ev)
589
590 # Interpreter introspection
591 b[builtin_i.type] = meta_osh.Type(procs, aliases, search_path, errfmt)
592 b[builtin_i.builtin] = meta_osh.Builtin(shell_ex, errfmt)
593 b[builtin_i.command] = meta_osh.Command(shell_ex, procs, aliases,
594 search_path)
595 # Part of YSH, but similar to builtin/command
596 b[builtin_i.runproc] = meta_osh.RunProc(shell_ex, procs, errfmt)
597
598 # Meta builtins
599 source_builtin = meta_osh.Source(parse_ctx, search_path, cmd_ev, fd_state,
600 tracer, errfmt, loader)
601 b[builtin_i.source] = source_builtin
602 b[builtin_i.dot] = source_builtin
603 b[builtin_i.eval] = meta_osh.Eval(parse_ctx, exec_opts, cmd_ev, tracer,
604 errfmt)
605
606 # Module builtins
607 guards = {} # type: Dict[str, bool]
608 b[builtin_i.source_guard] = module_ysh.SourceGuard(guards, exec_opts,
609 errfmt)
610 b[builtin_i.is_main] = module_ysh.IsMain(mem)
611 b[builtin_i.use] = module_ysh.Use(mem, errfmt)
612
613 # Errors
614 b[builtin_i.error] = error_ysh.Error()
615 b[builtin_i.failed] = error_ysh.Failed(mem)
616 b[builtin_i.boolstatus] = error_ysh.BoolStatus(shell_ex, errfmt)
617 b[builtin_i.try_] = error_ysh.Try(mutable_opts, mem, cmd_ev, shell_ex,
618 errfmt)
619
620 # Pure builtins
621 true_ = pure_osh.Boolean(0)
622 b[builtin_i.colon] = true_ # a "special" builtin
623 b[builtin_i.true_] = true_
624 b[builtin_i.false_] = pure_osh.Boolean(1)
625
626 b[builtin_i.alias] = pure_osh.Alias(aliases, errfmt)
627 b[builtin_i.unalias] = pure_osh.UnAlias(aliases, errfmt)
628
629 b[builtin_i.getopts] = pure_osh.GetOpts(mem, errfmt)
630
631 b[builtin_i.shift] = assign_osh.Shift(mem)
632 b[builtin_i.unset] = assign_osh.Unset(mem, procs, unsafe_arith, errfmt)
633
634 b[builtin_i.append] = pure_ysh.Append(mem, errfmt)
635
636 # test / [ differ by need_right_bracket
637 b[builtin_i.test] = bracket_osh.Test(False, exec_opts, mem, errfmt)
638 b[builtin_i.bracket] = bracket_osh.Test(True, exec_opts, mem, errfmt)
639
640 # Output
641 b[builtin_i.echo] = io_osh.Echo(exec_opts)
642 b[builtin_i.printf] = printf_osh.Printf(mem, parse_ctx, unsafe_arith,
643 errfmt)
644 b[builtin_i.write] = io_ysh.Write(mem, errfmt)
645 b[builtin_i.fopen] = io_ysh.Fopen(mem, cmd_ev)
646
647 # (pp output format isn't stable)
648 b[builtin_i.pp] = io_ysh.Pp(mem, errfmt, procs, arena)
649
650 # Input
651 b[builtin_i.cat] = io_osh.Cat() # for $(<file)
652 b[builtin_i.read] = read_osh.Read(splitter, mem, parse_ctx, cmd_ev, errfmt)
653
654 mapfile = io_osh.MapFile(mem, errfmt, cmd_ev)
655 b[builtin_i.mapfile] = mapfile
656 b[builtin_i.readarray] = mapfile
657
658 # Dirs
659 dir_stack = dirs_osh.DirStack()
660 b[builtin_i.cd] = dirs_osh.Cd(mem, dir_stack, cmd_ev, errfmt)
661 b[builtin_i.pushd] = dirs_osh.Pushd(mem, dir_stack, errfmt)
662 b[builtin_i.popd] = dirs_osh.Popd(mem, dir_stack, errfmt)
663 b[builtin_i.dirs] = dirs_osh.Dirs(mem, dir_stack, errfmt)
664 b[builtin_i.pwd] = dirs_osh.Pwd(mem, errfmt)
665
666 b[builtin_i.times] = misc_osh.Times()
667
668 b[builtin_i.json] = json_ysh.Json(mem, errfmt, False)
669 b[builtin_i.json8] = json_ysh.Json(mem, errfmt, True)
670
671 ### Process builtins
672 b[builtin_i.exec_] = process_osh.Exec(mem, ext_prog, fd_state, search_path,
673 errfmt)
674 b[builtin_i.umask] = process_osh.Umask()
675 b[builtin_i.ulimit] = process_osh.Ulimit()
676 b[builtin_i.wait] = process_osh.Wait(waiter, job_list, mem, tracer, errfmt)
677
678 b[builtin_i.jobs] = process_osh.Jobs(job_list)
679 b[builtin_i.fg] = process_osh.Fg(job_control, job_list, waiter)
680 b[builtin_i.bg] = process_osh.Bg(job_list)
681
682 # Could be in process_ysh
683 b[builtin_i.fork] = process_osh.Fork(shell_ex)
684 b[builtin_i.forkwait] = process_osh.ForkWait(shell_ex)
685
686 # Interactive builtins depend on readline
687 b[builtin_i.bind] = readline_osh.Bind(readline, errfmt)
688 b[builtin_i.history] = readline_osh.History(readline, sh_files, errfmt,
689 mylib.Stdout())
690
691 # Completion
692 spec_builder = completion_osh.SpecBuilder(cmd_ev, parse_ctx, word_ev,
693 splitter, comp_lookup, help_data,
694 errfmt)
695 complete_builtin = completion_osh.Complete(spec_builder, comp_lookup)
696 b[builtin_i.complete] = complete_builtin
697 b[builtin_i.compgen] = completion_osh.CompGen(spec_builder)
698 b[builtin_i.compopt] = completion_osh.CompOpt(compopt_state, errfmt)
699 b[builtin_i.compadjust] = completion_osh.CompAdjust(mem)
700
701 comp_ev = word_eval.CompletionWordEvaluator(mem, exec_opts, mutable_opts,
702 tilde_ev, splitter, errfmt)
703
704 comp_ev.arith_ev = arith_ev
705 comp_ev.expr_ev = expr_ev
706 comp_ev.prompt_ev = prompt_ev
707 comp_ev.CheckCircularDeps()
708
709 root_comp = completion.RootCompleter(comp_ev, mem, comp_lookup,
710 compopt_state, comp_ui_state,
711 comp_ctx, debug_f)
712 b[builtin_i.compexport] = completion_ysh.CompExport(root_comp)
713
714 #
715 # Initialize Builtin-in Methods
716 #
717
718 methods[value_e.Str] = {
719 'startsWith': method_str.HasAffix(method_str.START),
720 'endsWith': method_str.HasAffix(method_str.END),
721 'trim': method_str.Trim(method_str.START | method_str.END),
722 'trimStart': method_str.Trim(method_str.START),
723 'trimEnd': method_str.Trim(method_str.END),
724 'upper': method_str.Upper(),
725 'lower': method_str.Lower(),
726
727 # finds a substring, optional position to start at
728 'find': None,
729
730 # replace substring, OR an eggex
731 # takes count=3, the max number of replacements to do.
732 'replace': method_str.Replace(mem, expr_ev),
733
734 # Like Python's re.search, except we put it on the string object
735 # It's more consistent with Str->find(substring, pos=0)
736 # It returns value.Match() rather than an integer
737 'search': method_str.SearchMatch(method_str.SEARCH),
738
739 # like Python's re.match()
740 'leftMatch': method_str.SearchMatch(method_str.LEFT_MATCH),
741
742 # like Python's re.fullmatch(), not sure if we really need it
743 'fullMatch': None,
744 }
745 methods[value_e.Dict] = {
746 'get': None, # doesn't raise an error
747 'erase': None, # ensures it doesn't exist
748 'keys': method_dict.Keys(),
749 'values': method_dict.Values(),
750
751 # I think items() isn't as necessary because dicts are ordered?
752 # YSH code shouldn't use the List of Lists representation.
753
754 # could be d->tally() or d->increment(), but inc() is short
755 #
756 # call d->inc('mycounter')
757 # call d->inc('mycounter', 3)
758 'inc': None,
759
760 # call d->accum('mygroup', 'value')
761 'accum': None,
762 }
763 methods[value_e.List] = {
764 'reverse': method_list.Reverse(),
765 'append': method_list.Append(),
766 'extend': method_list.Extend(),
767 'pop': method_list.Pop(),
768 'insert': None, # insert object before index
769 'remove': None, # insert object before index
770 'indexOf': method_list.IndexOf(), # return first index of value, or -1
771 # Python list() has index(), which raises ValueError
772 # But this is consistent with Str->find(), and doesn't
773 # use exceptions
774 'join': func_misc.Join(), # both a method and a func
775 }
776
777 methods[value_e.Match] = {
778 'group': func_eggex.MatchMethod(func_eggex.G, expr_ev),
779 'start': func_eggex.MatchMethod(func_eggex.S, None),
780 'end': func_eggex.MatchMethod(func_eggex.E, None),
781 }
782
783 methods[value_e.IO] = {
784 # io->eval(myblock) is the functional version of eval (myblock)
785 # Should we also have expr->eval() instead of evalExpr?
786 'eval': method_io.Eval(),
787
788 # identical to command sub
789 'captureStdout': method_io.CaptureStdout(),
790 'promptVal': method_io.PromptVal(),
791 'time': method_io.Time(),
792 'strftime': method_io.Strftime(),
793 }
794
795 methods[value_e.Place] = {
796 # instead of setplace keyword
797 'setValue': method_other.SetValue(mem),
798 }
799
800 methods[value_e.Command] = {
801 # var x = ^(echo hi)
802 # Export source code and line number
803 # Useful for test frameworks and so forth
804 'export': None,
805 }
806
807 #
808 # Initialize Built-in Funcs
809 #
810
811 parse_hay = func_hay.ParseHay(fd_state, parse_ctx, errfmt)
812 eval_hay = func_hay.EvalHay(hay_state, mutable_opts, mem, cmd_ev)
813 hay_func = func_hay.HayFunc(hay_state)
814
815 _SetGlobalFunc(mem, 'parseHay', parse_hay)
816 _SetGlobalFunc(mem, 'evalHay', eval_hay)
817 _SetGlobalFunc(mem, '_hay', hay_func)
818
819 _SetGlobalFunc(mem, 'len', func_misc.Len())
820 _SetGlobalFunc(mem, 'type', func_misc.Type())
821 _SetGlobalFunc(mem, 'repeat', func_misc.Repeat())
822
823 g = func_eggex.MatchFunc(func_eggex.G, expr_ev, mem)
824 _SetGlobalFunc(mem, '_group', g)
825 _SetGlobalFunc(mem, '_match', g) # TODO: remove this backward compat alias
826 _SetGlobalFunc(mem, '_start', func_eggex.MatchFunc(func_eggex.S, None,
827 mem))
828 _SetGlobalFunc(mem, '_end', func_eggex.MatchFunc(func_eggex.E, None, mem))
829
830 _SetGlobalFunc(mem, 'join', func_misc.Join())
831 _SetGlobalFunc(mem, 'maybe', func_misc.Maybe())
832 _SetGlobalFunc(mem, 'evalExpr', func_misc.EvalExpr(expr_ev))
833
834 # type conversions
835 _SetGlobalFunc(mem, 'bool', func_misc.Bool())
836 _SetGlobalFunc(mem, 'int', func_misc.Int())
837 _SetGlobalFunc(mem, 'float', func_misc.Float())
838 _SetGlobalFunc(mem, 'str', func_misc.Str_())
839 _SetGlobalFunc(mem, 'list', func_misc.List_())
840 _SetGlobalFunc(mem, 'dict', func_misc.Dict_())
841
842 _SetGlobalFunc(mem, 'runes', func_misc.Runes())
843 _SetGlobalFunc(mem, 'encodeRunes', func_misc.EncodeRunes())
844 _SetGlobalFunc(mem, 'bytes', func_misc.Bytes())
845 _SetGlobalFunc(mem, 'encodeBytes', func_misc.EncodeBytes())
846
847 # TODO: This should be Python style splitting
848 _SetGlobalFunc(mem, 'split', func_misc.Split(splitter))
849 _SetGlobalFunc(mem, 'shSplit', func_misc.Split(splitter))
850
851 _SetGlobalFunc(mem, 'glob', func_misc.Glob(globber))
852 _SetGlobalFunc(mem, 'shvarGet', func_misc.Shvar_get(mem))
853 _SetGlobalFunc(mem, 'getVar', func_misc.GetVar(mem))
854 _SetGlobalFunc(mem, 'assert_', func_misc.Assert())
855
856 _SetGlobalFunc(mem, 'toJson8', func_misc.ToJson8(True))
857 _SetGlobalFunc(mem, 'toJson', func_misc.ToJson8(False))
858
859 _SetGlobalFunc(mem, 'fromJson8', func_misc.FromJson8(True))
860 _SetGlobalFunc(mem, 'fromJson', func_misc.FromJson8(False))
861
862 _SetGlobalFunc(mem, '_a2sp', func_misc.BashArrayToSparse())
863 _SetGlobalFunc(mem, '_d2sp', func_misc.DictToSparse())
864 _SetGlobalFunc(mem, '_opsp', func_misc.SparseOp())
865
866 mem.SetNamed(location.LName('_io'), global_io, scope_e.GlobalOnly)
867 mem.SetNamed(location.LName('_guts'), global_guts, scope_e.GlobalOnly)
868
869 mem.SetNamed(location.LName('stdin'), value.Stdin, scope_e.GlobalOnly)
870
871 #
872 # Is the shell interactive?
873 #
874
875 # History evaluation is a no-op if readline is None.
876 hist_ev = history.Evaluator(readline, hist_ctx, debug_f)
877
878 if flag.c is not None:
879 src = source.CFlag # type: source_t
880 line_reader = reader.StringLineReader(flag.c,
881 arena) # type: reader._Reader
882 if flag.i: # -c and -i can be combined
883 mutable_opts.set_interactive()
884
885 elif flag.i: # force interactive
886 src = source.Stdin(' -i')
887 line_reader = reader.InteractiveLineReader(arena, prompt_ev, hist_ev,
888 readline, prompt_state)
889 mutable_opts.set_interactive()
890
891 else:
892 if script_name is None:
893 if flag.headless:
894 src = source.Headless
895 line_reader = None # unused!
896 # Not setting '-i' flag for now. Some people's bashrc may want it?
897 else:
898 stdin_ = mylib.Stdin()
899 # --tool never starts a prompt
900 if len(flag.tool) == 0 and stdin_.isatty():
901 src = source.Interactive
902 line_reader = reader.InteractiveLineReader(
903 arena, prompt_ev, hist_ev, readline, prompt_state)
904 mutable_opts.set_interactive()
905 else:
906 src = source.Stdin('')
907 line_reader = reader.FileLineReader(stdin_, arena)
908 else:
909 src = source.MainFile(script_name)
910 try:
911 f = fd_state.Open(script_name)
912 except (IOError, OSError) as e:
913 print_stderr("%s: Couldn't open %r: %s" %
914 (lang, script_name, posix.strerror(e.errno)))
915 return 1
916 line_reader = reader.FileLineReader(f, arena)
917
918 # Pretend it came from somewhere else
919 if flag.location_str is not None:
920 src = source.Synthetic(flag.location_str)
921 assert line_reader is not None
922 location_start_line = mops.BigTruncate(flag.location_start_line)
923 if location_start_line != -1:
924 line_reader.SetLineOffset(location_start_line)
925
926 arena.PushSource(src)
927
928 # Calculate ~/.config/oils/oshrc or yshrc. Used for both -i and --headless
929 # We avoid cluttering the user's home directory. Some users may want to ln
930 # -s ~/.config/oils/oshrc ~/oshrc or ~/.oshrc.
931
932 # https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other
933
934 config_dir = '.config/oils'
935 rc_paths = [] # type: List[str]
936 if not flag.norc and (flag.headless or exec_opts.interactive()):
937 # User's rcfile comes FIRST. Later we can add an 'after-rcdir' hook
938 rc_path = flag.rcfile
939 if rc_path is None:
940 rc_paths.append(
941 os_path.join(home_dir, '%s/%src' % (config_dir, lang)))
942 else:
943 rc_paths.append(rc_path)
944
945 # Load all files in ~/.config/oil/oshrc.d or oilrc.d
946 # This way "installers" can avoid mutating oshrc directly
947
948 rc_dir = flag.rcdir
949 if rc_dir is None:
950 rc_dir = os_path.join(home_dir, '%s/%src.d' % (config_dir, lang))
951
952 rc_paths.extend(libc.glob(os_path.join(rc_dir, '*')))
953 else:
954 if flag.rcfile is not None: # bash doesn't have this warning, but it's useful
955 print_stderr('%s warning: --rcfile ignored with --norc' % lang)
956 if flag.rcdir is not None:
957 print_stderr('%s warning: --rcdir ignored with --norc' % lang)
958
959 # Initialize even in non-interactive shell, for 'compexport'
960 _InitDefaultCompletions(cmd_ev, complete_builtin, comp_lookup)
961
962 if flag.headless:
963 state.InitInteractive(mem)
964 mutable_opts.set_redefine_proc_func()
965 mutable_opts.set_redefine_module()
966
967 # NOTE: rc files loaded AFTER _InitDefaultCompletions.
968 for rc_path in rc_paths:
969 with state.ctx_ThisDir(mem, rc_path):
970 try:
971 SourceStartupFile(fd_state, rc_path, lang, parse_ctx,
972 cmd_ev, errfmt)
973 except util.UserExit as e:
974 return e.status
975
976 loop = main_loop.Headless(cmd_ev, parse_ctx, errfmt)
977 try:
978 # TODO: What other exceptions happen here?
979 status = loop.Loop()
980 except util.UserExit as e:
981 status = e.status
982
983 # Same logic as interactive shell
984 mut_status = IntParamBox(status)
985 cmd_ev.MaybeRunExitTrap(mut_status)
986 status = mut_status.i
987
988 return status
989
990 # Note: headless mode above doesn't use c_parser
991 assert line_reader is not None
992 c_parser = parse_ctx.MakeOshParser(line_reader)
993
994 if exec_opts.interactive():
995 state.InitInteractive(mem)
996 # bash: 'set -o emacs' is the default only in the interactive shell
997 mutable_opts.set_emacs()
998 mutable_opts.set_redefine_proc_func()
999 mutable_opts.set_redefine_module()
1000
1001 if readline:
1002 term_width = 0
1003 if flag.completion_display == 'nice':
1004 try:
1005 term_width = libc.get_terminal_width()
1006 except (IOError, OSError): # stdin not a terminal
1007 pass
1008
1009 if term_width != 0:
1010 display = comp_ui.NiceDisplay(
1011 term_width, comp_ui_state, prompt_state, debug_f, readline,
1012 signal_safe) # type: comp_ui._IDisplay
1013 else:
1014 display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
1015 debug_f)
1016
1017 comp_ui.InitReadline(readline, sh_files.HistoryFile(), root_comp,
1018 display, debug_f)
1019
1020 _InitDefaultCompletions(cmd_ev, complete_builtin, comp_lookup)
1021 if flag.completion_demo:
1022 _CompletionDemo(comp_lookup)
1023
1024 else: # Without readline module
1025 display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
1026 debug_f)
1027
1028 process.InitInteractiveShell() # Set signal handlers
1029
1030 # The interactive shell leads a process group which controls the terminal.
1031 # It MUST give up the terminal afterward, otherwise we get SIGTTIN /
1032 # SIGTTOU bugs.
1033 with process.ctx_TerminalControl(job_control, errfmt):
1034
1035 # NOTE: rc files loaded AFTER _InitDefaultCompletions.
1036 for rc_path in rc_paths:
1037 with state.ctx_ThisDir(mem, rc_path):
1038 try:
1039 SourceStartupFile(fd_state, rc_path, lang, parse_ctx,
1040 cmd_ev, errfmt)
1041 except util.UserExit as e:
1042 return e.status
1043
1044 assert line_reader is not None
1045 line_reader.Reset() # After sourcing startup file, render $PS1
1046
1047 prompt_plugin = prompt.UserPlugin(mem, parse_ctx, cmd_ev, errfmt)
1048 try:
1049 status = main_loop.Interactive(flag, cmd_ev, c_parser, display,
1050 prompt_plugin, waiter, errfmt)
1051 except util.UserExit as e:
1052 status = e.status
1053
1054 mut_status = IntParamBox(status)
1055 cmd_ev.MaybeRunExitTrap(mut_status)
1056 status = mut_status.i
1057
1058 if readline:
1059 hist_file = sh_files.HistoryFile()
1060 if hist_file is not None:
1061 try:
1062 readline.write_history_file(hist_file)
1063 except (IOError, OSError):
1064 pass
1065
1066 return status
1067
1068 if flag.rcfile is not None: # bash doesn't have this warning, but it's useful
1069 print_stderr('%s warning: --rcfile ignored in non-interactive shell' %
1070 lang)
1071 if flag.rcdir is not None:
1072 print_stderr('%s warning: --rcdir ignored in non-interactive shell' %
1073 lang)
1074
1075 #
1076 # Tools that use the OSH/YSH parsing mode, etc.
1077 #
1078
1079 # flag.tool is '' if nothing is passed
1080 # osh --tool syntax-tree is equivalent to osh -n --one-pass-parse
1081 tool_name = 'syntax-tree' if exec_opts.noexec() else flag.tool
1082
1083 if len(tool_name):
1084 # Don't save tokens becaues it's slow
1085 if tool_name != 'syntax-tree':
1086 arena.SaveTokens()
1087
1088 try:
1089 node = main_loop.ParseWholeFile(c_parser)
1090 except error.Parse as e:
1091 errfmt.PrettyPrintError(e)
1092 return 2
1093
1094 if tool_name == 'syntax-tree':
1095 ui.PrintAst(node, flag)
1096
1097 elif tool_name == 'tokens':
1098 ysh_ify.PrintTokens(arena)
1099
1100 elif tool_name == 'lossless-cat': # for test/lossless.sh
1101 ysh_ify.LosslessCat(arena)
1102
1103 elif tool_name == 'fmt':
1104 fmt.Format(arena, node)
1105
1106 elif tool_name == 'test':
1107 raise AssertionError('TODO')
1108
1109 elif tool_name == 'ysh-ify':
1110 ysh_ify.Ysh_ify(arena, node)
1111
1112 elif tool_name == 'deps':
1113 if mylib.PYTHON:
1114 deps.Deps(node)
1115
1116 else:
1117 raise AssertionError(tool_name) # flag parser validated it
1118
1119 return 0
1120
1121 #
1122 # Run a shell script
1123 #
1124
1125 with state.ctx_ThisDir(mem, script_name):
1126 try:
1127 status = main_loop.Batch(cmd_ev,
1128 c_parser,
1129 errfmt,
1130 cmd_flags=cmd_eval.IsMainProgram)
1131 except util.UserExit as e:
1132 status = e.status
1133 mut_status = IntParamBox(status)
1134 cmd_ev.MaybeRunExitTrap(mut_status)
1135
1136 multi_trace.WriteDumps()
1137
1138 # NOTE: We haven't closed the file opened with fd_state.Open
1139 return mut_status.i