| 1 | #!/usr/bin/env python2
|
| 2 | # Copyright 2016 Andy Chu. All rights reserved.
|
| 3 | # Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 | # you may not use this file except in compliance with the License.
|
| 5 | # You may obtain a copy of the License at
|
| 6 | #
|
| 7 | # http://www.apache.org/licenses/LICENSE-2.0
|
| 8 | """
|
| 9 | test_lib.py - Functions for testing.
|
| 10 | """
|
| 11 |
|
| 12 | import string
|
| 13 | import sys
|
| 14 |
|
| 15 | from _devbuild.gen.option_asdl import builtin_i, option_i
|
| 16 | from _devbuild.gen.runtime_asdl import cmd_value, scope_e
|
| 17 | from _devbuild.gen.syntax_asdl import loc, source, SourceLine, Token
|
| 18 | from _devbuild.gen.value_asdl import value
|
| 19 | from asdl import pybase
|
| 20 | from builtin import assign_osh
|
| 21 | from builtin import completion_osh
|
| 22 | from builtin import hay_ysh
|
| 23 | from builtin import io_osh
|
| 24 | from builtin import pure_osh
|
| 25 | from builtin import readline_osh
|
| 26 | from builtin import trap_osh
|
| 27 | from core import alloc
|
| 28 | from core import completion
|
| 29 | from core import dev
|
| 30 | from core import executor
|
| 31 | from core import main_loop
|
| 32 | from core import optview
|
| 33 | from core import process
|
| 34 | from core import pyos
|
| 35 | from core import pyutil
|
| 36 | from core import state
|
| 37 | from core import ui
|
| 38 | from core import util
|
| 39 | from core import vm
|
| 40 | from frontend import lexer
|
| 41 | from frontend import location
|
| 42 | from frontend import parse_lib
|
| 43 | from frontend import reader
|
| 44 | from osh import cmd_eval
|
| 45 | from osh import prompt
|
| 46 | from osh import sh_expr_eval
|
| 47 | from osh import split
|
| 48 | from osh import word_eval
|
| 49 | from ysh import expr_eval
|
| 50 | from mycpp import mylib
|
| 51 |
|
| 52 | import posix_ as posix
|
| 53 |
|
| 54 |
|
| 55 | def MakeBuiltinArgv(argv):
|
| 56 | return cmd_value.Argv(argv, [loc.Missing] * len(argv), None, None, None,
|
| 57 | None)
|
| 58 |
|
| 59 |
|
| 60 | def FakeTok(id_, val):
|
| 61 | # type: (int, str) -> Token
|
| 62 | src = source.Interactive
|
| 63 | line = SourceLine(1, val, src)
|
| 64 | return Token(id_, len(val), 0, line, None)
|
| 65 |
|
| 66 |
|
| 67 | def PrintableString(s):
|
| 68 | """For pretty-printing in tests."""
|
| 69 | if all(c in string.printable for c in s):
|
| 70 | return s
|
| 71 | return repr(s)
|
| 72 |
|
| 73 |
|
| 74 | def TokensEqual(left, right):
|
| 75 | # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
|
| 76 |
|
| 77 | if left.id != right.id:
|
| 78 | return False
|
| 79 |
|
| 80 | if left.line is not None:
|
| 81 | left_str = lexer.TokenVal(left)
|
| 82 | else:
|
| 83 | left_str = None
|
| 84 |
|
| 85 | if right.line is not None:
|
| 86 | right_str = lexer.TokenVal(right)
|
| 87 | else:
|
| 88 | right_str = None
|
| 89 |
|
| 90 | # Better error message sometimes:
|
| 91 | #assert left_str == right_str, '%r != %r' % (left_str, right_str)
|
| 92 | return left_str == right_str
|
| 93 |
|
| 94 |
|
| 95 | def TokenWordsEqual(left, right):
|
| 96 | # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
|
| 97 | return TokensEqual(left.token, right.token)
|
| 98 | #return left == right
|
| 99 |
|
| 100 |
|
| 101 | def AsdlEqual(left, right):
|
| 102 | """Check if generated ASDL instances are equal.
|
| 103 |
|
| 104 | We don't use equality in the actual code, so this is relegated to
|
| 105 | test_lib.
|
| 106 | """
|
| 107 | if left is None and right is None:
|
| 108 | return True
|
| 109 |
|
| 110 | if isinstance(left, (int, str, bool, pybase.SimpleObj)):
|
| 111 | return left == right
|
| 112 |
|
| 113 | if isinstance(left, list):
|
| 114 | if len(left) != len(right):
|
| 115 | return False
|
| 116 | for a, b in zip(left, right):
|
| 117 | if not AsdlEqual(a, b):
|
| 118 | return False
|
| 119 | return True
|
| 120 |
|
| 121 | if isinstance(left, pybase.CompoundObj):
|
| 122 | if left.tag() != right.tag():
|
| 123 | return False
|
| 124 |
|
| 125 | field_names = left.__slots__ # hack for now
|
| 126 | for name in field_names:
|
| 127 | # Special case: we are not testing locations right now.
|
| 128 | if name == 'span_id':
|
| 129 | continue
|
| 130 | a = getattr(left, name)
|
| 131 | b = getattr(right, name)
|
| 132 | if not AsdlEqual(a, b):
|
| 133 | return False
|
| 134 |
|
| 135 | return True
|
| 136 |
|
| 137 | raise AssertionError(left)
|
| 138 |
|
| 139 |
|
| 140 | def AssertAsdlEqual(test, left, right):
|
| 141 | test.assertTrue(AsdlEqual(left, right),
|
| 142 | 'Expected %s, got %s' % (left, right))
|
| 143 |
|
| 144 |
|
| 145 | def MakeArena(source_name):
|
| 146 | arena = alloc.Arena(save_tokens=True)
|
| 147 | arena.PushSource(source.MainFile(source_name))
|
| 148 | return arena
|
| 149 |
|
| 150 |
|
| 151 | def InitLineLexer(s, arena):
|
| 152 | line_lexer = lexer.LineLexer(arena)
|
| 153 | src = source.Interactive
|
| 154 | line_lexer.Reset(SourceLine(1, s, src), 0)
|
| 155 | return line_lexer
|
| 156 |
|
| 157 |
|
| 158 | def InitLexer(s, arena):
|
| 159 | """For tests only."""
|
| 160 | line_lexer = lexer.LineLexer(arena)
|
| 161 | line_reader = reader.StringLineReader(s, arena)
|
| 162 | lx = lexer.Lexer(line_lexer, line_reader)
|
| 163 | return line_reader, lx
|
| 164 |
|
| 165 |
|
| 166 | def InitWordEvaluator(exec_opts=None):
|
| 167 | arena = MakeArena('<InitWordEvaluator>')
|
| 168 | mem = state.Mem('', [], arena, [])
|
| 169 |
|
| 170 | if exec_opts is None:
|
| 171 | parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
|
| 172 | mem.exec_opts = exec_opts # circular dep
|
| 173 | state.InitMem(mem, {}, '0.1')
|
| 174 | mutable_opts.Init()
|
| 175 | else:
|
| 176 | mutable_opts = None
|
| 177 |
|
| 178 | cmd_deps = cmd_eval.Deps()
|
| 179 | cmd_deps.trap_nodes = []
|
| 180 |
|
| 181 | splitter = split.SplitContext(mem)
|
| 182 | errfmt = ui.ErrorFormatter()
|
| 183 |
|
| 184 | tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
|
| 185 | ev = word_eval.CompletionWordEvaluator(mem, exec_opts, mutable_opts,
|
| 186 | tilde_ev, splitter, errfmt)
|
| 187 | return ev
|
| 188 |
|
| 189 |
|
| 190 | def InitCommandEvaluator(parse_ctx=None,
|
| 191 | comp_lookup=None,
|
| 192 | arena=None,
|
| 193 | mem=None,
|
| 194 | aliases=None,
|
| 195 | ext_prog=None):
|
| 196 |
|
| 197 | opt0_array = state.InitOpts()
|
| 198 | opt_stacks = [None] * option_i.ARRAY_SIZE
|
| 199 | if parse_ctx:
|
| 200 | arena = parse_ctx.arena
|
| 201 | else:
|
| 202 | parse_ctx = InitParseContext()
|
| 203 |
|
| 204 | mem = mem or state.Mem('', [], arena, [])
|
| 205 | exec_opts = optview.Exec(opt0_array, opt_stacks)
|
| 206 | mutable_opts = state.MutableOpts(mem, opt0_array, opt_stacks, None)
|
| 207 | mem.exec_opts = exec_opts
|
| 208 | state.InitMem(mem, {}, '0.1')
|
| 209 | mutable_opts.Init()
|
| 210 |
|
| 211 | # No 'readline' in the tests.
|
| 212 |
|
| 213 | errfmt = ui.ErrorFormatter()
|
| 214 | job_control = process.JobControl()
|
| 215 | job_list = process.JobList()
|
| 216 | fd_state = process.FdState(errfmt, job_control, job_list, None, None, None)
|
| 217 | aliases = {} if aliases is None else aliases
|
| 218 | procs = state.Procs(mem)
|
| 219 | methods = {}
|
| 220 |
|
| 221 | compopt_state = completion.OptionState()
|
| 222 | comp_lookup = comp_lookup or completion.Lookup()
|
| 223 |
|
| 224 | readline = None # simulate not having it
|
| 225 |
|
| 226 | new_var = assign_osh.NewVar(mem, procs, exec_opts, errfmt)
|
| 227 | assign_builtins = {
|
| 228 | builtin_i.declare: new_var,
|
| 229 | builtin_i.typeset: new_var,
|
| 230 | builtin_i.local: new_var,
|
| 231 | builtin_i.export_: assign_osh.Export(mem, errfmt),
|
| 232 | builtin_i.readonly: assign_osh.Readonly(mem, errfmt),
|
| 233 | }
|
| 234 | builtins = { # Lookup
|
| 235 | builtin_i.echo: io_osh.Echo(exec_opts),
|
| 236 | builtin_i.shift: assign_osh.Shift(mem),
|
| 237 |
|
| 238 | builtin_i.history: readline_osh.History(
|
| 239 | readline,
|
| 240 | mem,
|
| 241 | errfmt,
|
| 242 | mylib.Stdout(),
|
| 243 | ),
|
| 244 |
|
| 245 | builtin_i.compopt: completion_osh.CompOpt(compopt_state, errfmt),
|
| 246 | builtin_i.compadjust: completion_osh.CompAdjust(mem),
|
| 247 |
|
| 248 | builtin_i.alias: pure_osh.Alias(aliases, errfmt),
|
| 249 | builtin_i.unalias: pure_osh.UnAlias(aliases, errfmt),
|
| 250 | }
|
| 251 |
|
| 252 | debug_f = util.DebugFile(sys.stderr)
|
| 253 | cmd_deps = cmd_eval.Deps()
|
| 254 | cmd_deps.mutable_opts = mutable_opts
|
| 255 |
|
| 256 | search_path = state.SearchPath(mem)
|
| 257 |
|
| 258 | ext_prog = \
|
| 259 | ext_prog or process.ExternalProgram('', fd_state, errfmt, debug_f)
|
| 260 |
|
| 261 | cmd_deps.dumper = dev.CrashDumper('', fd_state)
|
| 262 | cmd_deps.debug_f = debug_f
|
| 263 |
|
| 264 | splitter = split.SplitContext(mem)
|
| 265 |
|
| 266 | arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, mutable_opts,
|
| 267 | parse_ctx, errfmt)
|
| 268 | bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, mutable_opts,
|
| 269 | parse_ctx, errfmt)
|
| 270 | expr_ev = expr_eval.ExprEvaluator(mem, mutable_opts, methods, splitter,
|
| 271 | errfmt)
|
| 272 | tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
|
| 273 | word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, mutable_opts,
|
| 274 | tilde_ev, splitter, errfmt)
|
| 275 | signal_safe = pyos.InitSignalSafe()
|
| 276 | trap_state = trap_osh.TrapState(signal_safe)
|
| 277 | cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs,
|
| 278 | assign_builtins, arena, cmd_deps,
|
| 279 | trap_state, signal_safe)
|
| 280 |
|
| 281 | multi_trace = dev.MultiTracer(posix.getpid(), '', '', '', fd_state)
|
| 282 | tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, debug_f,
|
| 283 | multi_trace)
|
| 284 | waiter = process.Waiter(job_list, exec_opts, trap_state, tracer)
|
| 285 |
|
| 286 | hay_state = hay_ysh.HayState()
|
| 287 | shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
|
| 288 | hay_state, builtins, search_path,
|
| 289 | ext_prog, waiter, tracer, job_control,
|
| 290 | job_list, fd_state, trap_state, errfmt)
|
| 291 |
|
| 292 | assert cmd_ev.mutable_opts is not None, cmd_ev
|
| 293 | prompt_ev = prompt.Evaluator('osh', '0.0.0', parse_ctx, mem)
|
| 294 |
|
| 295 | global_io = value.IO(cmd_ev, prompt_ev)
|
| 296 | vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
|
| 297 | prompt_ev, global_io, tracer)
|
| 298 |
|
| 299 | try:
|
| 300 | from _devbuild.gen.help_meta import TOPICS
|
| 301 | except ImportError:
|
| 302 | TOPICS = None # minimal dev build
|
| 303 | spec_builder = completion_osh.SpecBuilder(cmd_ev, parse_ctx, word_ev,
|
| 304 | splitter, comp_lookup, TOPICS,
|
| 305 | errfmt)
|
| 306 |
|
| 307 | # Add some builtins that depend on the executor!
|
| 308 | complete_builtin = completion_osh.Complete(spec_builder, comp_lookup)
|
| 309 | builtins[builtin_i.complete] = complete_builtin
|
| 310 | builtins[builtin_i.compgen] = completion_osh.CompGen(spec_builder)
|
| 311 |
|
| 312 | return cmd_ev
|
| 313 |
|
| 314 |
|
| 315 | def EvalCode(code_str, parse_ctx, comp_lookup=None, mem=None, aliases=None):
|
| 316 | """Unit tests can evaluate code strings and then use the resulting
|
| 317 | CommandEvaluator."""
|
| 318 | arena = parse_ctx.arena
|
| 319 | errfmt = ui.ErrorFormatter()
|
| 320 |
|
| 321 | comp_lookup = comp_lookup or completion.Lookup()
|
| 322 | mem = mem or state.Mem('', [], arena, [])
|
| 323 | parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
|
| 324 | mem.exec_opts = exec_opts
|
| 325 |
|
| 326 | state.InitMem(mem, {}, '0.1')
|
| 327 | mutable_opts.Init()
|
| 328 |
|
| 329 | line_reader, _ = InitLexer(code_str, arena)
|
| 330 | c_parser = parse_ctx.MakeOshParser(line_reader)
|
| 331 |
|
| 332 | cmd_ev = InitCommandEvaluator(parse_ctx=parse_ctx,
|
| 333 | comp_lookup=comp_lookup,
|
| 334 | arena=arena,
|
| 335 | mem=mem,
|
| 336 | aliases=aliases)
|
| 337 |
|
| 338 | main_loop.Batch(cmd_ev, c_parser, errfmt) # Parse and execute!
|
| 339 | return cmd_ev
|
| 340 |
|
| 341 |
|
| 342 | def InitParseContext(arena=None,
|
| 343 | ysh_grammar=None,
|
| 344 | aliases=None,
|
| 345 | parse_opts=None,
|
| 346 | do_lossless=False):
|
| 347 | arena = arena or MakeArena('<test_lib>')
|
| 348 |
|
| 349 | if aliases is None:
|
| 350 | aliases = {}
|
| 351 |
|
| 352 | mem = state.Mem('', [], arena, [])
|
| 353 | if parse_opts is None:
|
| 354 | parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
|
| 355 |
|
| 356 | parse_ctx = parse_lib.ParseContext(arena,
|
| 357 | parse_opts,
|
| 358 | aliases,
|
| 359 | ysh_grammar,
|
| 360 | do_lossless=do_lossless)
|
| 361 |
|
| 362 | return parse_ctx
|
| 363 |
|
| 364 |
|
| 365 | def InitWordParser(word_str, oil_at=False, arena=None):
|
| 366 | arena = arena or MakeArena('<test_lib>')
|
| 367 |
|
| 368 | mem = state.Mem('', [], arena, [])
|
| 369 | parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
|
| 370 |
|
| 371 | # CUSTOM SETTING
|
| 372 | mutable_opts.opt0_array[option_i.parse_at] = oil_at
|
| 373 |
|
| 374 | loader = pyutil.GetResourceLoader()
|
| 375 | ysh_grammar = pyutil.LoadYshGrammar(loader)
|
| 376 | parse_ctx = parse_lib.ParseContext(arena, parse_opts, {}, ysh_grammar)
|
| 377 | line_reader, _ = InitLexer(word_str, arena)
|
| 378 | c_parser = parse_ctx.MakeOshParser(line_reader)
|
| 379 | # Hack
|
| 380 | return c_parser.w_parser
|
| 381 |
|
| 382 |
|
| 383 | def InitCommandParser(code_str, arena=None):
|
| 384 | arena = arena or MakeArena('<test_lib>')
|
| 385 |
|
| 386 | loader = pyutil.GetResourceLoader()
|
| 387 | ysh_grammar = pyutil.LoadYshGrammar(loader)
|
| 388 |
|
| 389 | parse_ctx = InitParseContext(arena=arena, ysh_grammar=ysh_grammar)
|
| 390 | line_reader, _ = InitLexer(code_str, arena)
|
| 391 | c_parser = parse_ctx.MakeOshParser(line_reader)
|
| 392 | return c_parser
|
| 393 |
|
| 394 |
|
| 395 | def SetLocalString(mem, name, s):
|
| 396 | # type: (state.Mem, str, str) -> None
|
| 397 | """Bind a local string."""
|
| 398 | assert isinstance(s, str)
|
| 399 | mem.SetNamed(location.LName(name), value.Str(s), scope_e.LocalOnly)
|