| 1 | """
 | 
| 2 | dev.py - Devtools / introspection.
 | 
| 3 | """
 | 
| 4 | from __future__ import print_function
 | 
| 5 | 
 | 
| 6 | from _devbuild.gen.option_asdl import option_i, builtin_i, builtin_t
 | 
| 7 | from _devbuild.gen.runtime_asdl import (cmd_value, scope_e, trace, trace_e,
 | 
| 8 |                                         trace_t)
 | 
| 9 | from _devbuild.gen.syntax_asdl import assign_op_e, Token
 | 
| 10 | from _devbuild.gen.value_asdl import (value, value_e, value_t, sh_lvalue,
 | 
| 11 |                                       sh_lvalue_e, LeftName)
 | 
| 12 | 
 | 
| 13 | from core import error
 | 
| 14 | from core import optview
 | 
| 15 | from core import num
 | 
| 16 | from core import state
 | 
| 17 | from core import ui
 | 
| 18 | from data_lang import j8
 | 
| 19 | from frontend import location
 | 
| 20 | from osh import word_
 | 
| 21 | from data_lang import j8_lite
 | 
| 22 | from pylib import os_path
 | 
| 23 | from mycpp import mops
 | 
| 24 | from mycpp import mylib
 | 
| 25 | from mycpp.mylib import tagswitch, iteritems, print_stderr, log
 | 
| 26 | 
 | 
| 27 | import posix_ as posix
 | 
| 28 | 
 | 
| 29 | from typing import List, Dict, Optional, Any, cast, TYPE_CHECKING
 | 
| 30 | if TYPE_CHECKING:
 | 
| 31 |     from _devbuild.gen.syntax_asdl import assign_op_t, CompoundWord
 | 
| 32 |     from _devbuild.gen.runtime_asdl import scope_t
 | 
| 33 |     from _devbuild.gen.value_asdl import sh_lvalue_t
 | 
| 34 |     from core import alloc
 | 
| 35 |     from core.error import _ErrorWithLocation
 | 
| 36 |     from core import process
 | 
| 37 |     from core import util
 | 
| 38 |     from frontend.parse_lib import ParseContext
 | 
| 39 |     from osh.word_eval import NormalWordEvaluator
 | 
| 40 |     from osh.cmd_eval import CommandEvaluator
 | 
| 41 | 
 | 
| 42 | _ = log
 | 
| 43 | 
 | 
| 44 | 
 | 
| 45 | class CrashDumper(object):
 | 
| 46 |     """Controls if we collect a crash dump, and where we write it to.
 | 
| 47 | 
 | 
| 48 |     An object that can be serialized to JSON.
 | 
| 49 | 
 | 
| 50 |     trap CRASHDUMP upload-to-server
 | 
| 51 | 
 | 
| 52 |     # it gets written to a file first
 | 
| 53 |     upload-to-server() {
 | 
| 54 |       local path=$1
 | 
| 55 |       curl -X POST https://osh-trace.oilshell.org  < $path
 | 
| 56 |     }
 | 
| 57 | 
 | 
| 58 |     Things to dump:
 | 
| 59 |     CommandEvaluator
 | 
| 60 |       functions, aliases, traps, completion hooks, fd_state, dir_stack
 | 
| 61 | 
 | 
| 62 |     debug info for the source?  Or does that come elsewhere?
 | 
| 63 | 
 | 
| 64 |     Yeah I think you should have two separate files.
 | 
| 65 |     - debug info for a given piece of code (needs hash)
 | 
| 66 |       - this could just be the raw source files?  Does it need anything else?
 | 
| 67 |       - I think it needs a hash so the VM dump can refer to it.
 | 
| 68 |     - vm dump.
 | 
| 69 |     - Combine those and you get a UI.
 | 
| 70 | 
 | 
| 71 |     One is constant at build time; the other is constant at runtime.
 | 
| 72 |     """
 | 
| 73 | 
 | 
| 74 |     def __init__(self, crash_dump_dir, fd_state):
 | 
| 75 |         # type: (str, process.FdState) -> None
 | 
| 76 |         self.crash_dump_dir = crash_dump_dir
 | 
| 77 |         self.fd_state = fd_state
 | 
| 78 | 
 | 
| 79 |         # whether we should collect a dump, at the highest level of the stack
 | 
| 80 |         self.do_collect = bool(crash_dump_dir)
 | 
| 81 |         self.collected = False  # whether we have anything to dump
 | 
| 82 | 
 | 
| 83 |         self.var_stack = None  # type: List[value_t]
 | 
| 84 |         self.argv_stack = None  # type: List[value_t]
 | 
| 85 |         self.debug_stack = None  # type: List[value_t]
 | 
| 86 |         self.error = None  # type: Dict[str, value_t]
 | 
| 87 | 
 | 
| 88 |     def MaybeRecord(self, cmd_ev, err):
 | 
| 89 |         # type: (CommandEvaluator, _ErrorWithLocation) -> None
 | 
| 90 |         """Collect data for a crash dump.
 | 
| 91 | 
 | 
| 92 |         Args:
 | 
| 93 |           cmd_ev: CommandEvaluator instance
 | 
| 94 |           error: _ErrorWithLocation (ParseError or error.FatalRuntime)
 | 
| 95 |         """
 | 
| 96 |         if not self.do_collect:  # Either we already did it, or there is no file
 | 
| 97 |             return
 | 
| 98 | 
 | 
| 99 |         self.var_stack, self.argv_stack, self.debug_stack = cmd_ev.mem.Dump()
 | 
| 100 |         blame_tok = location.TokenFor(err.location)
 | 
| 101 | 
 | 
| 102 |         self.error = {
 | 
| 103 |             'msg': value.Str(err.UserErrorString()),
 | 
| 104 |         }
 | 
| 105 | 
 | 
| 106 |         if blame_tok:
 | 
| 107 |             # Could also do msg % args separately, but JavaScript won't be able to
 | 
| 108 |             # render that.
 | 
| 109 |             self.error['source'] = value.Str(
 | 
| 110 |                 ui.GetLineSourceString(blame_tok.line))
 | 
| 111 |             self.error['line_num'] = num.ToBig(blame_tok.line.line_num)
 | 
| 112 |             self.error['line'] = value.Str(blame_tok.line.content)
 | 
| 113 | 
 | 
| 114 |         # TODO: Collect functions, aliases, etc.
 | 
| 115 |         self.do_collect = False
 | 
| 116 |         self.collected = True
 | 
| 117 | 
 | 
| 118 |     def MaybeDump(self, status):
 | 
| 119 |         # type: (int) -> None
 | 
| 120 |         """Write the dump as JSON.
 | 
| 121 | 
 | 
| 122 |         User can configure it two ways:
 | 
| 123 |         - dump unconditionally -- a daily cron job.  This would be fine.
 | 
| 124 |         - dump on non-zero exit code
 | 
| 125 | 
 | 
| 126 |         OILS_FAIL
 | 
| 127 |         Maybe counters are different than failure
 | 
| 128 | 
 | 
| 129 |         OILS_CRASH_DUMP='function alias trap completion stack' ?
 | 
| 130 |         OILS_COUNTER_DUMP='function alias trap completion'
 | 
| 131 |         and then
 | 
| 132 |         I think both of these should dump the (path, mtime, checksum) of the source
 | 
| 133 |         they ran?  And then you can match those up with source control or whatever?
 | 
| 134 |         """
 | 
| 135 |         if not self.collected:
 | 
| 136 |             return
 | 
| 137 | 
 | 
| 138 |         my_pid = posix.getpid()  # Get fresh PID here
 | 
| 139 | 
 | 
| 140 |         # Other things we need: the reason for the crash!  _ErrorWithLocation is
 | 
| 141 |         # required I think.
 | 
| 142 |         d = {
 | 
| 143 |             'var_stack': value.List(self.var_stack),
 | 
| 144 |             'argv_stack': value.List(self.argv_stack),
 | 
| 145 |             'debug_stack': value.List(self.debug_stack),
 | 
| 146 |             'error': value.Dict(self.error),
 | 
| 147 |             'status': num.ToBig(status),
 | 
| 148 |             'pid': num.ToBig(my_pid),
 | 
| 149 |         }  # type: Dict[str, value_t]
 | 
| 150 | 
 | 
| 151 |         path = os_path.join(self.crash_dump_dir,
 | 
| 152 |                             '%d-osh-crash-dump.json' % my_pid)
 | 
| 153 | 
 | 
| 154 |         # TODO: This should be JSON with unicode replacement char?
 | 
| 155 |         buf = mylib.BufWriter()
 | 
| 156 |         j8.PrintMessage(value.Dict(d), buf, 2)
 | 
| 157 |         json_str = buf.getvalue()
 | 
| 158 | 
 | 
| 159 |         try:
 | 
| 160 |             f = self.fd_state.OpenForWrite(path)
 | 
| 161 |         except (IOError, OSError) as e:
 | 
| 162 |             # Ignore error
 | 
| 163 |             return
 | 
| 164 | 
 | 
| 165 |         f.write(json_str)
 | 
| 166 | 
 | 
| 167 |         # TODO: mylib.Writer() needs close()?  Also for DebugFile()
 | 
| 168 |         #f.close()
 | 
| 169 | 
 | 
| 170 |         print_stderr('[%d] Wrote crash dump to %s' % (my_pid, path))
 | 
| 171 | 
 | 
| 172 | 
 | 
| 173 | class ctx_Tracer(object):
 | 
| 174 |     """A stack for tracing synchronous constructs."""
 | 
| 175 | 
 | 
| 176 |     def __init__(self, tracer, label, argv):
 | 
| 177 |         # type: (Tracer, str, Optional[List[str]]) -> None
 | 
| 178 |         self.arg = None  # type: Optional[str]
 | 
| 179 |         if label == 'proc':
 | 
| 180 |             self.arg = argv[0]
 | 
| 181 |         elif label == 'source':
 | 
| 182 |             self.arg = argv[1]
 | 
| 183 | 
 | 
| 184 |         tracer.PushMessage(label, argv)
 | 
| 185 |         self.label = label
 | 
| 186 |         self.tracer = tracer
 | 
| 187 | 
 | 
| 188 |     def __enter__(self):
 | 
| 189 |         # type: () -> None
 | 
| 190 |         pass
 | 
| 191 | 
 | 
| 192 |     def __exit__(self, type, value, traceback):
 | 
| 193 |         # type: (Any, Any, Any) -> None
 | 
| 194 |         self.tracer.PopMessage(self.label, self.arg)
 | 
| 195 | 
 | 
| 196 | 
 | 
| 197 | def _PrintShValue(val, buf):
 | 
| 198 |     # type: (value_t, mylib.BufWriter) -> None
 | 
| 199 |     """Print ShAssignment values.
 | 
| 200 | 
 | 
| 201 |     NOTE: This is a bit like _PrintVariables for declare -p
 | 
| 202 |     """
 | 
| 203 |     # I think this should never happen because it's for ShAssignment
 | 
| 204 |     result = '?'
 | 
| 205 | 
 | 
| 206 |     # Using maybe_shell_encode() because it's shell
 | 
| 207 |     UP_val = val
 | 
| 208 |     with tagswitch(val) as case:
 | 
| 209 |         if case(value_e.Str):
 | 
| 210 |             val = cast(value.Str, UP_val)
 | 
| 211 |             result = j8_lite.MaybeShellEncode(val.s)
 | 
| 212 | 
 | 
| 213 |         elif case(value_e.BashArray):
 | 
| 214 |             val = cast(value.BashArray, UP_val)
 | 
| 215 |             parts = ['(']
 | 
| 216 |             for s in val.strs:
 | 
| 217 |                 parts.append(j8_lite.MaybeShellEncode(s))
 | 
| 218 |             parts.append(')')
 | 
| 219 |             result = ' '.join(parts)
 | 
| 220 | 
 | 
| 221 |         elif case(value_e.BashAssoc):
 | 
| 222 |             val = cast(value.BashAssoc, UP_val)
 | 
| 223 |             parts = ['(']
 | 
| 224 |             for k, v in iteritems(val.d):
 | 
| 225 |                 # key must be quoted
 | 
| 226 |                 parts.append(
 | 
| 227 |                     '[%s]=%s' %
 | 
| 228 |                     (j8_lite.ShellEncode(k), j8_lite.MaybeShellEncode(v)))
 | 
| 229 |             parts.append(')')
 | 
| 230 |             result = ' '.join(parts)
 | 
| 231 | 
 | 
| 232 |     buf.write(result)
 | 
| 233 | 
 | 
| 234 | 
 | 
| 235 | def PrintShellArgv(argv, buf):
 | 
| 236 |     # type: (List[str], mylib.BufWriter) -> None
 | 
| 237 |     for i, arg in enumerate(argv):
 | 
| 238 |         if i != 0:
 | 
| 239 |             buf.write(' ')
 | 
| 240 |         buf.write(j8_lite.MaybeShellEncode(arg))
 | 
| 241 | 
 | 
| 242 | 
 | 
| 243 | def _PrintYshArgv(argv, buf):
 | 
| 244 |     # type: (List[str], mylib.BufWriter) -> None
 | 
| 245 | 
 | 
| 246 |     # We're printing $'hi\n' for OSH, but we might want to print u'hi\n' or
 | 
| 247 |     # b'\n' for YSH.  We could have a shopt --set xtrace_j8 or something.
 | 
| 248 |     #
 | 
| 249 |     # This used to be xtrace_rich, but I think that was too subtle.
 | 
| 250 | 
 | 
| 251 |     for arg in argv:
 | 
| 252 |         buf.write(' ')
 | 
| 253 |         # TODO: use unquoted -> POSIX '' -> b''
 | 
| 254 |         # This would use JSON "", which CONFLICTS with shell.  So we need
 | 
| 255 |         # another function.
 | 
| 256 |         #j8.EncodeString(arg, buf, unquoted_ok=True)
 | 
| 257 | 
 | 
| 258 |         buf.write(j8_lite.MaybeShellEncode(arg))
 | 
| 259 |     buf.write('\n')
 | 
| 260 | 
 | 
| 261 | 
 | 
| 262 | class MultiTracer(object):
 | 
| 263 |     """ Manages multi-process tracing and dumping.
 | 
| 264 | 
 | 
| 265 |     Use case:
 | 
| 266 | 
 | 
| 267 |     TODO: write a shim for everything that autoconf starts out with
 | 
| 268 | 
 | 
| 269 |     (1) How do you discover what is shelled out to? 
 | 
| 270 |         - you need a MULTIPROCESS tracing and MULTIPROCESS errors
 | 
| 271 | 
 | 
| 272 |     OILS_TRACE_DIR=_tmp/foo OILS_TRACE_STREAMS=xtrace:completion:gc \
 | 
| 273 |     OILS_TRACE_DUMPS=crash:argv0 \
 | 
| 274 |       osh ./configure
 | 
| 275 | 
 | 
| 276 |     - Streams are written continuously, they are O(n)
 | 
| 277 |     - Dumps are written once per shell process, they are O(1). This includes metrics.
 | 
| 278 | 
 | 
| 279 |     (2) Use that dump to generate stubs in _tmp/stubs
 | 
| 280 |         They will invoke benchmarks/time-helper, so we get timing and memory use
 | 
| 281 |         for each program. 
 | 
| 282 | 
 | 
| 283 |     (3) ORIG_PATH=$PATH PATH=_tmp/stubs:$PATH osh ./configure
 | 
| 284 | 
 | 
| 285 |     THen the stub looks like this?
 | 
| 286 | 
 | 
| 287 |     #!/bin/sh 
 | 
| 288 |     # _tmp/stubs/cc1
 | 
| 289 | 
 | 
| 290 |     PATH=$ORIG_PATH time-helper -x -e -- cc1 "$@"
 | 
| 291 |     """
 | 
| 292 | 
 | 
| 293 |     def __init__(self, shell_pid, out_dir, dumps, streams, fd_state):
 | 
| 294 |         # type: (int, str, str, str, process.FdState) -> None
 | 
| 295 |         """
 | 
| 296 |         out_dir could be auto-generated from root PID?
 | 
| 297 |         """
 | 
| 298 |         # All of these may be empty string
 | 
| 299 |         self.out_dir = out_dir
 | 
| 300 |         self.dumps = dumps
 | 
| 301 |         self.streams = streams
 | 
| 302 |         self.fd_state = fd_state
 | 
| 303 | 
 | 
| 304 |         self.this_pid = shell_pid
 | 
| 305 | 
 | 
| 306 |         # This is what we consider an O(1) metric.  Technically a shell program
 | 
| 307 |         # could run forever and keep invoking different binaries, but that is
 | 
| 308 |         # unlikely.  I guess we could limit it to 1,000 or 10,000 artifically
 | 
| 309 |         # or something.
 | 
| 310 |         self.hist_argv0 = {}  # type: Dict[str, int]
 | 
| 311 | 
 | 
| 312 |     def OnNewProcess(self, child_pid):
 | 
| 313 |         # type: (int) -> None
 | 
| 314 |         """
 | 
| 315 |         Right now we call this from
 | 
| 316 |            Process::StartProcess -> tracer.SetChildPid()
 | 
| 317 |         It would be more accurate to call it from SubProgramThunk.
 | 
| 318 | 
 | 
| 319 |         TODO: do we need a compound PID?
 | 
| 320 |         """
 | 
| 321 |         self.this_pid = child_pid
 | 
| 322 |         # each process keep track of direct children
 | 
| 323 |         self.hist_argv0.clear()
 | 
| 324 | 
 | 
| 325 |     def EmitArgv0(self, argv0):
 | 
| 326 |         # type: (str) -> None
 | 
| 327 | 
 | 
| 328 |         # TODO: Should we have word 0 in the source, and the FILE the $PATH
 | 
| 329 |         # lookup resolved to?
 | 
| 330 | 
 | 
| 331 |         if argv0 not in self.hist_argv0:
 | 
| 332 |             self.hist_argv0[argv0] = 1
 | 
| 333 |         else:
 | 
| 334 |             # TODO: mycpp doesn't allow +=
 | 
| 335 |             self.hist_argv0[argv0] = self.hist_argv0[argv0] + 1
 | 
| 336 | 
 | 
| 337 |     def WriteDumps(self):
 | 
| 338 |         # type: () -> None
 | 
| 339 |         if len(self.out_dir) == 0:
 | 
| 340 |             return
 | 
| 341 | 
 | 
| 342 |         # TSV8 table might be nicer for this
 | 
| 343 | 
 | 
| 344 |         metric_argv0 = []  # type: List[value_t]
 | 
| 345 |         for argv0, count in iteritems(self.hist_argv0):
 | 
| 346 |             a = value.Str(argv0)
 | 
| 347 |             c = value.Int(mops.IntWiden(count))
 | 
| 348 |             d = {'argv0': a, 'count': c}
 | 
| 349 |             metric_argv0.append(value.Dict(d))
 | 
| 350 | 
 | 
| 351 |         # Other things we need: the reason for the crash!  _ErrorWithLocation is
 | 
| 352 |         # required I think.
 | 
| 353 |         j = {
 | 
| 354 |             'pid': value.Int(mops.IntWiden(self.this_pid)),
 | 
| 355 |             'metric_argv0': value.List(metric_argv0),
 | 
| 356 |         }  # type: Dict[str, value_t]
 | 
| 357 | 
 | 
| 358 |         # dumps are named $PID.$channel.json
 | 
| 359 |         path = os_path.join(self.out_dir, '%d.argv0.json' % self.this_pid)
 | 
| 360 | 
 | 
| 361 |         buf = mylib.BufWriter()
 | 
| 362 |         j8.PrintMessage(value.Dict(j), buf, 2)
 | 
| 363 |         json8_str = buf.getvalue()
 | 
| 364 | 
 | 
| 365 |         try:
 | 
| 366 |             f = self.fd_state.OpenForWrite(path)
 | 
| 367 |         except (IOError, OSError) as e:
 | 
| 368 |             # Ignore error
 | 
| 369 |             return
 | 
| 370 | 
 | 
| 371 |         f.write(json8_str)
 | 
| 372 |         f.close()
 | 
| 373 | 
 | 
| 374 |         print_stderr('[%d] Wrote metrics dump to %s' % (self.this_pid, path))
 | 
| 375 | 
 | 
| 376 | 
 | 
| 377 | class Tracer(object):
 | 
| 378 |     """For OSH set -x, and YSH hierarchical, parsable tracing.
 | 
| 379 | 
 | 
| 380 |     See doc/xtrace.md for details.
 | 
| 381 | 
 | 
| 382 |     - TODO: Connect it somehow to tracers for other processes.  So you can make
 | 
| 383 |       an HTML report offline.
 | 
| 384 |       - Could inherit SHX_*
 | 
| 385 | 
 | 
| 386 |     https://www.gnu.org/software/bash/manual/html_node/Bash-Variables.html#Bash-Variables
 | 
| 387 | 
 | 
| 388 |     Other hooks:
 | 
| 389 | 
 | 
| 390 |     - Command completion starts other processes
 | 
| 391 |     - YSH command constructs: BareDecl, VarDecl, Mutation, Expr
 | 
| 392 |     """
 | 
| 393 | 
 | 
| 394 |     def __init__(
 | 
| 395 |             self,
 | 
| 396 |             parse_ctx,  # type: ParseContext
 | 
| 397 |             exec_opts,  # type: optview.Exec
 | 
| 398 |             mutable_opts,  # type: state.MutableOpts
 | 
| 399 |             mem,  # type: state.Mem
 | 
| 400 |             f,  # type: util._DebugFile
 | 
| 401 |             multi_trace,  # type: MultiTracer
 | 
| 402 |     ):
 | 
| 403 |         # type: (...) -> None
 | 
| 404 |         """
 | 
| 405 |         trace_dir comes from OILS_TRACE_DIR
 | 
| 406 |         """
 | 
| 407 |         self.parse_ctx = parse_ctx
 | 
| 408 |         self.exec_opts = exec_opts
 | 
| 409 |         self.mutable_opts = mutable_opts
 | 
| 410 |         self.mem = mem
 | 
| 411 |         self.f = f  # can be stderr, the --debug-file, etc.
 | 
| 412 |         self.multi_trace = multi_trace
 | 
| 413 | 
 | 
| 414 |         self.word_ev = None  # type: NormalWordEvaluator
 | 
| 415 | 
 | 
| 416 |         self.ind = 0  # changed by process, proc, source, eval
 | 
| 417 |         self.indents = ['']  # "pooled" to avoid allocations
 | 
| 418 | 
 | 
| 419 |         # PS4 value -> CompoundWord.  PS4 is scoped.
 | 
| 420 |         self.parse_cache = {}  # type: Dict[str, CompoundWord]
 | 
| 421 | 
 | 
| 422 |         # Mutate objects to save allocations
 | 
| 423 |         self.val_indent = value.Str('')
 | 
| 424 |         self.val_punct = value.Str('')
 | 
| 425 |         # TODO: show something for root process by default?  INTERLEAVED output
 | 
| 426 |         # can be confusing, e.g. debugging traps in forkred subinterpreter
 | 
| 427 |         # created by a pipeline.
 | 
| 428 |         self.val_pid_str = value.Str('')  # mutated by SetProcess
 | 
| 429 | 
 | 
| 430 |         # Can these be global constants?  I don't think we have that in ASDL yet.
 | 
| 431 |         self.lval_indent = location.LName('SHX_indent')
 | 
| 432 |         self.lval_punct = location.LName('SHX_punct')
 | 
| 433 |         self.lval_pid_str = location.LName('SHX_pid_str')
 | 
| 434 | 
 | 
| 435 |     def CheckCircularDeps(self):
 | 
| 436 |         # type: () -> None
 | 
| 437 |         assert self.word_ev is not None
 | 
| 438 | 
 | 
| 439 |     def _EvalPS4(self, punct):
 | 
| 440 |         # type: (str) -> str
 | 
| 441 |         """The prefix of each line."""
 | 
| 442 |         val = self.mem.GetValue('PS4')
 | 
| 443 |         if val.tag() == value_e.Str:
 | 
| 444 |             ps4 = cast(value.Str, val).s
 | 
| 445 |         else:
 | 
| 446 |             ps4 = ''
 | 
| 447 | 
 | 
| 448 |         # NOTE: This cache is slightly broken because aliases are mutable!  I think
 | 
| 449 |         # that is more or less harmless though.
 | 
| 450 |         ps4_word = self.parse_cache.get(ps4)
 | 
| 451 |         if ps4_word is None:
 | 
| 452 |             # We have to parse this at runtime.  PS4 should usually remain constant.
 | 
| 453 |             w_parser = self.parse_ctx.MakeWordParserForPlugin(ps4)
 | 
| 454 | 
 | 
| 455 |             # NOTE: could use source.Variable, like $PS1 prompt does
 | 
| 456 |             try:
 | 
| 457 |                 ps4_word = w_parser.ReadForPlugin()
 | 
| 458 |             except error.Parse as e:
 | 
| 459 |                 ps4_word = word_.ErrorWord("<ERROR: Can't parse PS4: %s>" %
 | 
| 460 |                                            e.UserErrorString())
 | 
| 461 |             self.parse_cache[ps4] = ps4_word
 | 
| 462 | 
 | 
| 463 |         # Mutate objects to save allocations
 | 
| 464 |         if self.exec_opts.xtrace_rich():
 | 
| 465 |             self.val_indent.s = self.indents[self.ind]
 | 
| 466 |         else:
 | 
| 467 |             self.val_indent.s = ''
 | 
| 468 |         self.val_punct.s = punct
 | 
| 469 | 
 | 
| 470 |         # Prevent infinite loop when PS4 has command sub!
 | 
| 471 |         assert self.exec_opts.xtrace()  # We shouldn't call this unless it's on
 | 
| 472 | 
 | 
| 473 |         # TODO: Remove allocation for [] ?
 | 
| 474 |         with state.ctx_Option(self.mutable_opts, [option_i.xtrace], False):
 | 
| 475 |             with state.ctx_Temp(self.mem):
 | 
| 476 |                 self.mem.SetNamed(self.lval_indent, self.val_indent,
 | 
| 477 |                                   scope_e.LocalOnly)
 | 
| 478 |                 self.mem.SetNamed(self.lval_punct, self.val_punct,
 | 
| 479 |                                   scope_e.LocalOnly)
 | 
| 480 |                 self.mem.SetNamed(self.lval_pid_str, self.val_pid_str,
 | 
| 481 |                                   scope_e.LocalOnly)
 | 
| 482 |                 prefix = self.word_ev.EvalForPlugin(ps4_word)
 | 
| 483 |         return prefix.s
 | 
| 484 | 
 | 
| 485 |     def _Inc(self):
 | 
| 486 |         # type: () -> None
 | 
| 487 |         self.ind += 1
 | 
| 488 |         if self.ind >= len(self.indents):  # make sure there are enough
 | 
| 489 |             self.indents.append('  ' * self.ind)
 | 
| 490 | 
 | 
| 491 |     def _Dec(self):
 | 
| 492 |         # type: () -> None
 | 
| 493 |         self.ind -= 1
 | 
| 494 | 
 | 
| 495 |     def _ShTraceBegin(self):
 | 
| 496 |         # type: () -> Optional[mylib.BufWriter]
 | 
| 497 |         if not self.exec_opts.xtrace() or not self.exec_opts.xtrace_details():
 | 
| 498 |             return None
 | 
| 499 | 
 | 
| 500 |         # Note: bash repeats the + for command sub, eval, source.  Other shells
 | 
| 501 |         # don't do it.  Leave this out for now.
 | 
| 502 |         prefix = self._EvalPS4('+')
 | 
| 503 |         buf = mylib.BufWriter()
 | 
| 504 |         buf.write(prefix)
 | 
| 505 |         return buf
 | 
| 506 | 
 | 
| 507 |     def _RichTraceBegin(self, punct):
 | 
| 508 |         # type: (str) -> Optional[mylib.BufWriter]
 | 
| 509 |         """For the stack printed by xtrace_rich."""
 | 
| 510 |         if not self.exec_opts.xtrace() or not self.exec_opts.xtrace_rich():
 | 
| 511 |             return None
 | 
| 512 | 
 | 
| 513 |         prefix = self._EvalPS4(punct)
 | 
| 514 |         buf = mylib.BufWriter()
 | 
| 515 |         buf.write(prefix)
 | 
| 516 |         return buf
 | 
| 517 | 
 | 
| 518 |     def OnProcessStart(self, pid, why):
 | 
| 519 |         # type: (int, trace_t) -> None
 | 
| 520 |         """
 | 
| 521 |         In parent, Process::StartProcess calls us with child PID
 | 
| 522 |         """
 | 
| 523 |         UP_why = why
 | 
| 524 |         with tagswitch(why) as case:
 | 
| 525 |             if case(trace_e.External):
 | 
| 526 |                 why = cast(trace.External, UP_why)
 | 
| 527 | 
 | 
| 528 |                 # There is the empty argv case of $(true), but it's never external
 | 
| 529 |                 assert len(why.argv) > 0
 | 
| 530 |                 self.multi_trace.EmitArgv0(why.argv[0])
 | 
| 531 | 
 | 
| 532 |         buf = self._RichTraceBegin('|')
 | 
| 533 |         if not buf:
 | 
| 534 |             return
 | 
| 535 | 
 | 
| 536 |         # TODO: ProcessSub and PipelinePart are commonly command.Simple, and also
 | 
| 537 |         # Fork/ForkWait through the BraceGroup.  We could print those argv arrays.
 | 
| 538 | 
 | 
| 539 |         with tagswitch(why) as case:
 | 
| 540 |             # Synchronous cases
 | 
| 541 |             if case(trace_e.External):
 | 
| 542 |                 why = cast(trace.External, UP_why)
 | 
| 543 |                 buf.write('command %d:' % pid)
 | 
| 544 |                 _PrintYshArgv(why.argv, buf)
 | 
| 545 | 
 | 
| 546 |             # Everything below is the same.  Could use string literals?
 | 
| 547 |             elif case(trace_e.ForkWait):
 | 
| 548 |                 buf.write('forkwait %d\n' % pid)
 | 
| 549 |             elif case(trace_e.CommandSub):
 | 
| 550 |                 buf.write('command sub %d\n' % pid)
 | 
| 551 | 
 | 
| 552 |             # Async cases
 | 
| 553 |             elif case(trace_e.ProcessSub):
 | 
| 554 |                 buf.write('proc sub %d\n' % pid)
 | 
| 555 |             elif case(trace_e.HereDoc):
 | 
| 556 |                 buf.write('here doc %d\n' % pid)
 | 
| 557 |             elif case(trace_e.Fork):
 | 
| 558 |                 buf.write('fork %d\n' % pid)
 | 
| 559 |             elif case(trace_e.PipelinePart):
 | 
| 560 |                 buf.write('part %d\n' % pid)
 | 
| 561 | 
 | 
| 562 |             else:
 | 
| 563 |                 raise AssertionError()
 | 
| 564 | 
 | 
| 565 |         self.f.write(buf.getvalue())
 | 
| 566 | 
 | 
| 567 |     def OnProcessEnd(self, pid, status):
 | 
| 568 |         # type: (int, int) -> None
 | 
| 569 |         buf = self._RichTraceBegin(';')
 | 
| 570 |         if not buf:
 | 
| 571 |             return
 | 
| 572 | 
 | 
| 573 |         buf.write('process %d: status %d\n' % (pid, status))
 | 
| 574 |         self.f.write(buf.getvalue())
 | 
| 575 | 
 | 
| 576 |     def OnNewProcess(self, child_pid):
 | 
| 577 |         # type: (int) -> None
 | 
| 578 |         """All trace lines have a PID prefix, except those from the root
 | 
| 579 |         process."""
 | 
| 580 |         self.val_pid_str.s = ' %d' % child_pid
 | 
| 581 |         self._Inc()
 | 
| 582 |         self.multi_trace.OnNewProcess(child_pid)
 | 
| 583 | 
 | 
| 584 |     def PushMessage(self, label, argv):
 | 
| 585 |         # type: (str, Optional[List[str]]) -> None
 | 
| 586 |         """For synchronous constructs that aren't processes."""
 | 
| 587 |         buf = self._RichTraceBegin('>')
 | 
| 588 |         if buf:
 | 
| 589 |             buf.write(label)
 | 
| 590 |             if label == 'proc':
 | 
| 591 |                 _PrintYshArgv(argv, buf)
 | 
| 592 |             elif label == 'source':
 | 
| 593 |                 _PrintYshArgv(argv[1:], buf)
 | 
| 594 |             elif label == 'wait':
 | 
| 595 |                 _PrintYshArgv(argv[1:], buf)
 | 
| 596 |             else:
 | 
| 597 |                 buf.write('\n')
 | 
| 598 |             self.f.write(buf.getvalue())
 | 
| 599 | 
 | 
| 600 |         self._Inc()
 | 
| 601 | 
 | 
| 602 |     def PopMessage(self, label, arg):
 | 
| 603 |         # type: (str, Optional[str]) -> None
 | 
| 604 |         """For synchronous constructs that aren't processes.
 | 
| 605 | 
 | 
| 606 |         e.g. source or proc
 | 
| 607 |         """
 | 
| 608 |         self._Dec()
 | 
| 609 | 
 | 
| 610 |         buf = self._RichTraceBegin('<')
 | 
| 611 |         if buf:
 | 
| 612 |             buf.write(label)
 | 
| 613 |             if arg is not None:
 | 
| 614 |                 buf.write(' ')
 | 
| 615 |                 # TODO: use unquoted -> POSIX '' -> b''
 | 
| 616 |                 buf.write(j8_lite.MaybeShellEncode(arg))
 | 
| 617 |             buf.write('\n')
 | 
| 618 |             self.f.write(buf.getvalue())
 | 
| 619 | 
 | 
| 620 |     def OtherMessage(self, message):
 | 
| 621 |         # type: (str) -> None
 | 
| 622 |         """Can be used when receiving signals."""
 | 
| 623 |         buf = self._RichTraceBegin('!')
 | 
| 624 |         if not buf:
 | 
| 625 |             return
 | 
| 626 | 
 | 
| 627 |         buf.write(message)
 | 
| 628 |         buf.write('\n')
 | 
| 629 |         self.f.write(buf.getvalue())
 | 
| 630 | 
 | 
| 631 |     def OnExec(self, argv):
 | 
| 632 |         # type: (List[str]) -> None
 | 
| 633 |         buf = self._RichTraceBegin('.')
 | 
| 634 |         if not buf:
 | 
| 635 |             return
 | 
| 636 |         buf.write('exec')
 | 
| 637 |         _PrintYshArgv(argv, buf)
 | 
| 638 |         self.f.write(buf.getvalue())
 | 
| 639 | 
 | 
| 640 |     def OnBuiltin(self, builtin_id, argv):
 | 
| 641 |         # type: (builtin_t, List[str]) -> None
 | 
| 642 |         if builtin_id in (builtin_i.eval, builtin_i.source, builtin_i.wait):
 | 
| 643 |             return  # These 3 builtins handled separately
 | 
| 644 | 
 | 
| 645 |         buf = self._RichTraceBegin('.')
 | 
| 646 |         if not buf:
 | 
| 647 |             return
 | 
| 648 |         buf.write('builtin')
 | 
| 649 |         _PrintYshArgv(argv, buf)
 | 
| 650 |         self.f.write(buf.getvalue())
 | 
| 651 | 
 | 
| 652 |     #
 | 
| 653 |     # Shell Tracing That Begins with _ShTraceBegin
 | 
| 654 |     #
 | 
| 655 | 
 | 
| 656 |     def OnSimpleCommand(self, argv):
 | 
| 657 |         # type: (List[str]) -> None
 | 
| 658 |         """For legacy set -x.
 | 
| 659 | 
 | 
| 660 |         Called before we know if it's a builtin, external, or proc.
 | 
| 661 |         """
 | 
| 662 |         buf = self._ShTraceBegin()
 | 
| 663 |         if not buf:
 | 
| 664 |             return
 | 
| 665 | 
 | 
| 666 |         # Redundant with OnProcessStart (external), PushMessage (proc), and OnBuiltin
 | 
| 667 |         if self.exec_opts.xtrace_rich():
 | 
| 668 |             return
 | 
| 669 | 
 | 
| 670 |         # Legacy: Use SHELL encoding, NOT _PrintYshArgv()
 | 
| 671 |         PrintShellArgv(argv, buf)
 | 
| 672 |         buf.write('\n')
 | 
| 673 |         self.f.write(buf.getvalue())
 | 
| 674 | 
 | 
| 675 |     def OnAssignBuiltin(self, cmd_val):
 | 
| 676 |         # type: (cmd_value.Assign) -> None
 | 
| 677 |         buf = self._ShTraceBegin()
 | 
| 678 |         if not buf:
 | 
| 679 |             return
 | 
| 680 | 
 | 
| 681 |         for i, arg in enumerate(cmd_val.argv):
 | 
| 682 |             if i != 0:
 | 
| 683 |                 buf.write(' ')
 | 
| 684 |             buf.write(arg)
 | 
| 685 | 
 | 
| 686 |         for pair in cmd_val.pairs:
 | 
| 687 |             buf.write(' ')
 | 
| 688 |             buf.write(pair.var_name)
 | 
| 689 |             buf.write('=')
 | 
| 690 |             if pair.rval:
 | 
| 691 |                 _PrintShValue(pair.rval, buf)
 | 
| 692 | 
 | 
| 693 |         buf.write('\n')
 | 
| 694 |         self.f.write(buf.getvalue())
 | 
| 695 | 
 | 
| 696 |     def OnShAssignment(self, lval, op, val, flags, which_scopes):
 | 
| 697 |         # type: (sh_lvalue_t, assign_op_t, value_t, int, scope_t) -> None
 | 
| 698 |         buf = self._ShTraceBegin()
 | 
| 699 |         if not buf:
 | 
| 700 |             return
 | 
| 701 | 
 | 
| 702 |         left = '?'
 | 
| 703 |         UP_lval = lval
 | 
| 704 |         with tagswitch(lval) as case:
 | 
| 705 |             if case(sh_lvalue_e.Var):
 | 
| 706 |                 lval = cast(LeftName, UP_lval)
 | 
| 707 |                 left = lval.name
 | 
| 708 |             elif case(sh_lvalue_e.Indexed):
 | 
| 709 |                 lval = cast(sh_lvalue.Indexed, UP_lval)
 | 
| 710 |                 left = '%s[%d]' % (lval.name, lval.index)
 | 
| 711 |             elif case(sh_lvalue_e.Keyed):
 | 
| 712 |                 lval = cast(sh_lvalue.Keyed, UP_lval)
 | 
| 713 |                 left = '%s[%s]' % (lval.name, j8_lite.MaybeShellEncode(
 | 
| 714 |                     lval.key))
 | 
| 715 |         buf.write(left)
 | 
| 716 | 
 | 
| 717 |         # Only two possibilities here
 | 
| 718 |         buf.write('+=' if op == assign_op_e.PlusEqual else '=')
 | 
| 719 | 
 | 
| 720 |         _PrintShValue(val, buf)
 | 
| 721 | 
 | 
| 722 |         buf.write('\n')
 | 
| 723 |         self.f.write(buf.getvalue())
 | 
| 724 | 
 | 
| 725 |     def OnControlFlow(self, keyword, arg):
 | 
| 726 |         # type: (str, int) -> None
 | 
| 727 | 
 | 
| 728 |         # This is NOT affected by xtrace_rich or xtrace_details.  Works in both.
 | 
| 729 |         if not self.exec_opts.xtrace():
 | 
| 730 |             return
 | 
| 731 | 
 | 
| 732 |         prefix = self._EvalPS4('+')
 | 
| 733 |         buf = mylib.BufWriter()
 | 
| 734 |         buf.write(prefix)
 | 
| 735 | 
 | 
| 736 |         buf.write(keyword)
 | 
| 737 |         buf.write(' ')
 | 
| 738 |         buf.write(str(arg))  # Note: 'return' is equivalent to 'return 0'
 | 
| 739 |         buf.write('\n')
 | 
| 740 | 
 | 
| 741 |         self.f.write(buf.getvalue())
 | 
| 742 | 
 | 
| 743 |     def PrintSourceCode(self, left_tok, right_tok, arena):
 | 
| 744 |         # type: (Token, Token, alloc.Arena) -> None
 | 
| 745 |         """For (( )) and [[ ]].
 | 
| 746 | 
 | 
| 747 |         Bash traces these.
 | 
| 748 |         """
 | 
| 749 |         buf = self._ShTraceBegin()
 | 
| 750 |         if not buf:
 | 
| 751 |             return
 | 
| 752 | 
 | 
| 753 |         line = left_tok.line.content
 | 
| 754 |         start = left_tok.col
 | 
| 755 | 
 | 
| 756 |         if left_tok.line == right_tok.line:
 | 
| 757 |             end = right_tok.col + right_tok.length
 | 
| 758 |             buf.write(line[start:end])
 | 
| 759 |         else:
 | 
| 760 |             # Print first line only
 | 
| 761 |             end = -1 if line.endswith('\n') else len(line)
 | 
| 762 |             buf.write(line[start:end])
 | 
| 763 |             buf.write(' ...')
 | 
| 764 | 
 | 
| 765 |         buf.write('\n')
 | 
| 766 |         self.f.write(buf.getvalue())
 |