1 | #!/usr/bin/env python2
|
2 | # Copyright 2016 Andy Chu. All rights reserved.
|
3 | # Licensed under the Apache License, Version 2.0 (the "License");
|
4 | # you may not use this file except in compliance with the License.
|
5 | # You may obtain a copy of the License at
|
6 | #
|
7 | # http://www.apache.org/licenses/LICENSE-2.0
|
8 | """
|
9 | test_lib.py - Functions for testing.
|
10 | """
|
11 |
|
12 | import string
|
13 | import sys
|
14 |
|
15 | from _devbuild.gen.option_asdl import builtin_i, option_i
|
16 | from _devbuild.gen.runtime_asdl import cmd_value, scope_e
|
17 | from _devbuild.gen.syntax_asdl import loc, source, SourceLine, Token
|
18 | from _devbuild.gen.value_asdl import value
|
19 | from asdl import pybase
|
20 | from builtin import assign_osh
|
21 | from builtin import completion_osh
|
22 | from builtin import hay_ysh
|
23 | from builtin import io_osh
|
24 | from builtin import pure_osh
|
25 | from builtin import readline_osh
|
26 | from builtin import trap_osh
|
27 | from core import alloc
|
28 | from core import completion
|
29 | from core import dev
|
30 | from core import executor
|
31 | from core import main_loop
|
32 | from core import optview
|
33 | from core import process
|
34 | from core import pyos
|
35 | from core import pyutil
|
36 | from core import state
|
37 | from core import ui
|
38 | from core import util
|
39 | from core import vm
|
40 | from frontend import lexer
|
41 | from frontend import location
|
42 | from frontend import parse_lib
|
43 | from frontend import reader
|
44 | from osh import cmd_eval
|
45 | from osh import prompt
|
46 | from osh import sh_expr_eval
|
47 | from osh import split
|
48 | from osh import word_eval
|
49 | from ysh import expr_eval
|
50 | from mycpp import mylib
|
51 |
|
52 | import posix_ as posix
|
53 |
|
54 |
|
55 | def MakeBuiltinArgv(argv):
|
56 | return cmd_value.Argv(argv, [loc.Missing] * len(argv), None, None, None,
|
57 | None)
|
58 |
|
59 |
|
60 | def FakeTok(id_, val):
|
61 | # type: (int, str) -> Token
|
62 | src = source.Interactive
|
63 | line = SourceLine(1, val, src)
|
64 | return Token(id_, len(val), 0, line, None)
|
65 |
|
66 |
|
67 | def PrintableString(s):
|
68 | """For pretty-printing in tests."""
|
69 | if all(c in string.printable for c in s):
|
70 | return s
|
71 | return repr(s)
|
72 |
|
73 |
|
74 | def TokensEqual(left, right):
|
75 | # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
|
76 |
|
77 | if left.id != right.id:
|
78 | return False
|
79 |
|
80 | if left.line is not None:
|
81 | left_str = lexer.TokenVal(left)
|
82 | else:
|
83 | left_str = None
|
84 |
|
85 | if right.line is not None:
|
86 | right_str = lexer.TokenVal(right)
|
87 | else:
|
88 | right_str = None
|
89 |
|
90 | # Better error message sometimes:
|
91 | #assert left_str == right_str, '%r != %r' % (left_str, right_str)
|
92 | return left_str == right_str
|
93 |
|
94 |
|
95 | def TokenWordsEqual(left, right):
|
96 | # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
|
97 | return TokensEqual(left.token, right.token)
|
98 | #return left == right
|
99 |
|
100 |
|
101 | def AsdlEqual(left, right):
|
102 | """Check if generated ASDL instances are equal.
|
103 |
|
104 | We don't use equality in the actual code, so this is relegated to
|
105 | test_lib.
|
106 | """
|
107 | if left is None and right is None:
|
108 | return True
|
109 |
|
110 | if isinstance(left, (int, str, bool, pybase.SimpleObj)):
|
111 | return left == right
|
112 |
|
113 | if isinstance(left, list):
|
114 | if len(left) != len(right):
|
115 | return False
|
116 | for a, b in zip(left, right):
|
117 | if not AsdlEqual(a, b):
|
118 | return False
|
119 | return True
|
120 |
|
121 | if isinstance(left, pybase.CompoundObj):
|
122 | if left.tag() != right.tag():
|
123 | return False
|
124 |
|
125 | field_names = left.__slots__ # hack for now
|
126 | for name in field_names:
|
127 | # Special case: we are not testing locations right now.
|
128 | if name == 'span_id':
|
129 | continue
|
130 | a = getattr(left, name)
|
131 | b = getattr(right, name)
|
132 | if not AsdlEqual(a, b):
|
133 | return False
|
134 |
|
135 | return True
|
136 |
|
137 | raise AssertionError(left)
|
138 |
|
139 |
|
140 | def AssertAsdlEqual(test, left, right):
|
141 | test.assertTrue(AsdlEqual(left, right),
|
142 | 'Expected %s, got %s' % (left, right))
|
143 |
|
144 |
|
145 | def MakeArena(source_name):
|
146 | arena = alloc.Arena(save_tokens=True)
|
147 | arena.PushSource(source.MainFile(source_name))
|
148 | return arena
|
149 |
|
150 |
|
151 | def InitLineLexer(s, arena):
|
152 | line_lexer = lexer.LineLexer(arena)
|
153 | src = source.Interactive
|
154 | line_lexer.Reset(SourceLine(1, s, src), 0)
|
155 | return line_lexer
|
156 |
|
157 |
|
158 | def InitLexer(s, arena):
|
159 | """For tests only."""
|
160 | line_lexer = lexer.LineLexer(arena)
|
161 | line_reader = reader.StringLineReader(s, arena)
|
162 | lx = lexer.Lexer(line_lexer, line_reader)
|
163 | return line_reader, lx
|
164 |
|
165 |
|
166 | def InitWordEvaluator(exec_opts=None):
|
167 | arena = MakeArena('<InitWordEvaluator>')
|
168 | mem = state.Mem('', [], arena, [])
|
169 |
|
170 | if exec_opts is None:
|
171 | parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
|
172 | mem.exec_opts = exec_opts # circular dep
|
173 | state.InitMem(mem, {}, '0.1')
|
174 | mutable_opts.Init()
|
175 | else:
|
176 | mutable_opts = None
|
177 |
|
178 | cmd_deps = cmd_eval.Deps()
|
179 | cmd_deps.trap_nodes = []
|
180 |
|
181 | splitter = split.SplitContext(mem)
|
182 | errfmt = ui.ErrorFormatter()
|
183 |
|
184 | tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
|
185 | ev = word_eval.CompletionWordEvaluator(mem, exec_opts, mutable_opts,
|
186 | tilde_ev, splitter, errfmt)
|
187 | return ev
|
188 |
|
189 |
|
190 | def InitCommandEvaluator(parse_ctx=None,
|
191 | comp_lookup=None,
|
192 | arena=None,
|
193 | mem=None,
|
194 | aliases=None,
|
195 | ext_prog=None):
|
196 |
|
197 | opt0_array = state.InitOpts()
|
198 | opt_stacks = [None] * option_i.ARRAY_SIZE
|
199 | if parse_ctx:
|
200 | arena = parse_ctx.arena
|
201 | else:
|
202 | parse_ctx = InitParseContext()
|
203 |
|
204 | mem = mem or state.Mem('', [], arena, [])
|
205 | exec_opts = optview.Exec(opt0_array, opt_stacks)
|
206 | mutable_opts = state.MutableOpts(mem, opt0_array, opt_stacks, None)
|
207 | mem.exec_opts = exec_opts
|
208 | state.InitMem(mem, {}, '0.1')
|
209 | mutable_opts.Init()
|
210 |
|
211 | # No 'readline' in the tests.
|
212 |
|
213 | errfmt = ui.ErrorFormatter()
|
214 | job_control = process.JobControl()
|
215 | job_list = process.JobList()
|
216 | fd_state = process.FdState(errfmt, job_control, job_list, None, None, None)
|
217 | aliases = {} if aliases is None else aliases
|
218 | procs = {}
|
219 | methods = {}
|
220 |
|
221 | compopt_state = completion.OptionState()
|
222 | comp_lookup = comp_lookup or completion.Lookup()
|
223 |
|
224 | readline = None # simulate not having it
|
225 |
|
226 | new_var = assign_osh.NewVar(mem, procs, exec_opts, errfmt)
|
227 | assign_builtins = {
|
228 | builtin_i.declare: new_var,
|
229 | builtin_i.typeset: new_var,
|
230 | builtin_i.local: new_var,
|
231 | builtin_i.export_: assign_osh.Export(mem, errfmt),
|
232 | builtin_i.readonly: assign_osh.Readonly(mem, errfmt),
|
233 | }
|
234 | builtins = { # Lookup
|
235 | builtin_i.echo: io_osh.Echo(exec_opts),
|
236 | builtin_i.shift: assign_osh.Shift(mem),
|
237 |
|
238 | builtin_i.history: readline_osh.History(
|
239 | readline,
|
240 | mem,
|
241 | errfmt,
|
242 | mylib.Stdout(),
|
243 | ),
|
244 |
|
245 | builtin_i.compopt: completion_osh.CompOpt(compopt_state, errfmt),
|
246 | builtin_i.compadjust: completion_osh.CompAdjust(mem),
|
247 |
|
248 | builtin_i.alias: pure_osh.Alias(aliases, errfmt),
|
249 | builtin_i.unalias: pure_osh.UnAlias(aliases, errfmt),
|
250 | }
|
251 |
|
252 | debug_f = util.DebugFile(sys.stderr)
|
253 | cmd_deps = cmd_eval.Deps()
|
254 | cmd_deps.mutable_opts = mutable_opts
|
255 |
|
256 | search_path = state.SearchPath(mem)
|
257 |
|
258 | ext_prog = \
|
259 | ext_prog or process.ExternalProgram('', fd_state, errfmt, debug_f)
|
260 |
|
261 | cmd_deps.dumper = dev.CrashDumper('', fd_state)
|
262 | cmd_deps.debug_f = debug_f
|
263 |
|
264 | splitter = split.SplitContext(mem)
|
265 |
|
266 | arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, mutable_opts,
|
267 | parse_ctx, errfmt)
|
268 | bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, mutable_opts,
|
269 | parse_ctx, errfmt)
|
270 | expr_ev = expr_eval.ExprEvaluator(mem, mutable_opts, methods, splitter,
|
271 | errfmt)
|
272 | tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
|
273 | word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, mutable_opts,
|
274 | tilde_ev, splitter, errfmt)
|
275 | signal_safe = pyos.InitSignalSafe()
|
276 | trap_state = trap_osh.TrapState(signal_safe)
|
277 | cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs,
|
278 | assign_builtins, arena, cmd_deps,
|
279 | trap_state, signal_safe)
|
280 |
|
281 | multi_trace = dev.MultiTracer(posix.getpid(), '', '', '', fd_state)
|
282 | tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, debug_f,
|
283 | multi_trace)
|
284 | waiter = process.Waiter(job_list, exec_opts, trap_state, tracer)
|
285 |
|
286 | hay_state = hay_ysh.HayState()
|
287 | shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
|
288 | hay_state, builtins, search_path,
|
289 | ext_prog, waiter, tracer, job_control,
|
290 | job_list, fd_state, trap_state, errfmt)
|
291 |
|
292 | assert cmd_ev.mutable_opts is not None, cmd_ev
|
293 | prompt_ev = prompt.Evaluator('osh', '0.0.0', parse_ctx, mem)
|
294 |
|
295 | global_io = value.IO(cmd_ev, prompt_ev)
|
296 | vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
|
297 | prompt_ev, global_io, tracer)
|
298 |
|
299 | try:
|
300 | from _devbuild.gen.help_meta import TOPICS
|
301 | except ImportError:
|
302 | TOPICS = None # minimal dev build
|
303 | spec_builder = completion_osh.SpecBuilder(cmd_ev, parse_ctx, word_ev,
|
304 | splitter, comp_lookup, TOPICS,
|
305 | errfmt)
|
306 |
|
307 | # Add some builtins that depend on the executor!
|
308 | complete_builtin = completion_osh.Complete(spec_builder, comp_lookup)
|
309 | builtins[builtin_i.complete] = complete_builtin
|
310 | builtins[builtin_i.compgen] = completion_osh.CompGen(spec_builder)
|
311 |
|
312 | return cmd_ev
|
313 |
|
314 |
|
315 | def EvalCode(code_str, parse_ctx, comp_lookup=None, mem=None, aliases=None):
|
316 | """Unit tests can evaluate code strings and then use the resulting
|
317 | CommandEvaluator."""
|
318 | arena = parse_ctx.arena
|
319 | errfmt = ui.ErrorFormatter()
|
320 |
|
321 | comp_lookup = comp_lookup or completion.Lookup()
|
322 | mem = mem or state.Mem('', [], arena, [])
|
323 | parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
|
324 | mem.exec_opts = exec_opts
|
325 |
|
326 | state.InitMem(mem, {}, '0.1')
|
327 | mutable_opts.Init()
|
328 |
|
329 | line_reader, _ = InitLexer(code_str, arena)
|
330 | c_parser = parse_ctx.MakeOshParser(line_reader)
|
331 |
|
332 | cmd_ev = InitCommandEvaluator(parse_ctx=parse_ctx,
|
333 | comp_lookup=comp_lookup,
|
334 | arena=arena,
|
335 | mem=mem,
|
336 | aliases=aliases)
|
337 |
|
338 | main_loop.Batch(cmd_ev, c_parser, errfmt) # Parse and execute!
|
339 | return cmd_ev
|
340 |
|
341 |
|
342 | def InitParseContext(arena=None,
|
343 | ysh_grammar=None,
|
344 | aliases=None,
|
345 | parse_opts=None,
|
346 | do_lossless=False):
|
347 | arena = arena or MakeArena('<test_lib>')
|
348 |
|
349 | if aliases is None:
|
350 | aliases = {}
|
351 |
|
352 | mem = state.Mem('', [], arena, [])
|
353 | if parse_opts is None:
|
354 | parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
|
355 |
|
356 | parse_ctx = parse_lib.ParseContext(arena,
|
357 | parse_opts,
|
358 | aliases,
|
359 | ysh_grammar,
|
360 | do_lossless=do_lossless)
|
361 |
|
362 | return parse_ctx
|
363 |
|
364 |
|
365 | def InitWordParser(word_str, oil_at=False, arena=None):
|
366 | arena = arena or MakeArena('<test_lib>')
|
367 |
|
368 | mem = state.Mem('', [], arena, [])
|
369 | parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
|
370 |
|
371 | # CUSTOM SETTING
|
372 | mutable_opts.opt0_array[option_i.parse_at] = oil_at
|
373 |
|
374 | loader = pyutil.GetResourceLoader()
|
375 | ysh_grammar = pyutil.LoadYshGrammar(loader)
|
376 | parse_ctx = parse_lib.ParseContext(arena, parse_opts, {}, ysh_grammar)
|
377 | line_reader, _ = InitLexer(word_str, arena)
|
378 | c_parser = parse_ctx.MakeOshParser(line_reader)
|
379 | # Hack
|
380 | return c_parser.w_parser
|
381 |
|
382 |
|
383 | def InitCommandParser(code_str, arena=None):
|
384 | arena = arena or MakeArena('<test_lib>')
|
385 |
|
386 | loader = pyutil.GetResourceLoader()
|
387 | ysh_grammar = pyutil.LoadYshGrammar(loader)
|
388 |
|
389 | parse_ctx = InitParseContext(arena=arena, ysh_grammar=ysh_grammar)
|
390 | line_reader, _ = InitLexer(code_str, arena)
|
391 | c_parser = parse_ctx.MakeOshParser(line_reader)
|
392 | return c_parser
|
393 |
|
394 |
|
395 | def SetLocalString(mem, name, s):
|
396 | # type: (state.Mem, str, str) -> None
|
397 | """Bind a local string."""
|
398 | assert isinstance(s, str)
|
399 | mem.SetNamed(location.LName(name), value.Str(s), scope_e.LocalOnly)
|