OILS / core / test_lib.py View on Github | oilshell.org

400 lines, 280 significant
1#!/usr/bin/env python2
2# Copyright 2016 Andy Chu. All rights reserved.
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8"""
9test_lib.py - Functions for testing.
10"""
11
12import string
13import sys
14
15from _devbuild.gen.option_asdl import builtin_i, option_i
16from _devbuild.gen.runtime_asdl import cmd_value, scope_e
17from _devbuild.gen.syntax_asdl import loc, source, SourceLine, Token
18from _devbuild.gen.value_asdl import value
19from asdl import pybase
20from builtin import assign_osh
21from builtin import completion_osh
22from builtin import hay_ysh
23from builtin import io_osh
24from builtin import pure_osh
25from builtin import readline_osh
26from builtin import trap_osh
27from core import alloc
28from core import completion
29from core import dev
30from core import executor
31from core import main_loop
32from core import optview
33from core import process
34from core import pyos
35from core import pyutil
36from core import state
37from core import ui
38from core import util
39from core import vm
40from frontend import lexer
41from frontend import location
42from frontend import parse_lib
43from frontend import reader
44from osh import cmd_eval
45from osh import prompt
46from osh import sh_expr_eval
47from osh import split
48from osh import word_eval
49from ysh import expr_eval
50from mycpp import mylib
51
52import posix_ as posix
53
54
55def MakeBuiltinArgv(argv):
56 return cmd_value.Argv(argv, [loc.Missing] * len(argv), None, None, None,
57 None)
58
59
60def FakeTok(id_, val):
61 # type: (int, str) -> Token
62 src = source.Interactive
63 line = SourceLine(1, val, src)
64 return Token(id_, len(val), 0, line, None)
65
66
67def PrintableString(s):
68 """For pretty-printing in tests."""
69 if all(c in string.printable for c in s):
70 return s
71 return repr(s)
72
73
74def TokensEqual(left, right):
75 # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
76
77 if left.id != right.id:
78 return False
79
80 if left.line is not None:
81 left_str = lexer.TokenVal(left)
82 else:
83 left_str = None
84
85 if right.line is not None:
86 right_str = lexer.TokenVal(right)
87 else:
88 right_str = None
89
90 # Better error message sometimes:
91 #assert left_str == right_str, '%r != %r' % (left_str, right_str)
92 return left_str == right_str
93
94
95def TokenWordsEqual(left, right):
96 # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
97 return TokensEqual(left.token, right.token)
98 #return left == right
99
100
101def AsdlEqual(left, right):
102 """Check if generated ASDL instances are equal.
103
104 We don't use equality in the actual code, so this is relegated to
105 test_lib.
106 """
107 if left is None and right is None:
108 return True
109
110 if isinstance(left, (int, str, bool, pybase.SimpleObj)):
111 return left == right
112
113 if isinstance(left, list):
114 if len(left) != len(right):
115 return False
116 for a, b in zip(left, right):
117 if not AsdlEqual(a, b):
118 return False
119 return True
120
121 if isinstance(left, pybase.CompoundObj):
122 if left.tag() != right.tag():
123 return False
124
125 field_names = left.__slots__ # hack for now
126 for name in field_names:
127 # Special case: we are not testing locations right now.
128 if name == 'span_id':
129 continue
130 a = getattr(left, name)
131 b = getattr(right, name)
132 if not AsdlEqual(a, b):
133 return False
134
135 return True
136
137 raise AssertionError(left)
138
139
140def AssertAsdlEqual(test, left, right):
141 test.assertTrue(AsdlEqual(left, right),
142 'Expected %s, got %s' % (left, right))
143
144
145def MakeArena(source_name):
146 arena = alloc.Arena(save_tokens=True)
147 arena.PushSource(source.MainFile(source_name))
148 return arena
149
150
151def InitLineLexer(s, arena):
152 line_lexer = lexer.LineLexer(arena)
153 src = source.Interactive
154 line_lexer.Reset(SourceLine(1, s, src), 0)
155 return line_lexer
156
157
158def InitLexer(s, arena):
159 """For tests only."""
160 line_lexer = lexer.LineLexer(arena)
161 line_reader = reader.StringLineReader(s, arena)
162 lx = lexer.Lexer(line_lexer, line_reader)
163 return line_reader, lx
164
165
166def InitWordEvaluator(exec_opts=None):
167 arena = MakeArena('<InitWordEvaluator>')
168 mem = state.Mem('', [], arena, [])
169
170 if exec_opts is None:
171 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
172 mem.exec_opts = exec_opts # circular dep
173 state.InitMem(mem, {}, '0.1')
174 mutable_opts.Init()
175 else:
176 mutable_opts = None
177
178 cmd_deps = cmd_eval.Deps()
179 cmd_deps.trap_nodes = []
180
181 splitter = split.SplitContext(mem)
182 errfmt = ui.ErrorFormatter()
183
184 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
185 ev = word_eval.CompletionWordEvaluator(mem, exec_opts, mutable_opts,
186 tilde_ev, splitter, errfmt)
187 return ev
188
189
190def InitCommandEvaluator(parse_ctx=None,
191 comp_lookup=None,
192 arena=None,
193 mem=None,
194 aliases=None,
195 ext_prog=None):
196
197 opt0_array = state.InitOpts()
198 opt_stacks = [None] * option_i.ARRAY_SIZE
199 if parse_ctx:
200 arena = parse_ctx.arena
201 else:
202 parse_ctx = InitParseContext()
203
204 mem = mem or state.Mem('', [], arena, [])
205 exec_opts = optview.Exec(opt0_array, opt_stacks)
206 mutable_opts = state.MutableOpts(mem, opt0_array, opt_stacks, None)
207 mem.exec_opts = exec_opts
208 state.InitMem(mem, {}, '0.1')
209 mutable_opts.Init()
210
211 # No 'readline' in the tests.
212
213 errfmt = ui.ErrorFormatter()
214 job_control = process.JobControl()
215 job_list = process.JobList()
216 fd_state = process.FdState(errfmt, job_control, job_list, None, None, None,
217 exec_opts)
218 aliases = {} if aliases is None else aliases
219 procs = {}
220 methods = {}
221
222 compopt_state = completion.OptionState()
223 comp_lookup = comp_lookup or completion.Lookup()
224
225 readline = None # simulate not having it
226
227 new_var = assign_osh.NewVar(mem, procs, errfmt)
228 assign_builtins = {
229 builtin_i.declare: new_var,
230 builtin_i.typeset: new_var,
231 builtin_i.local: new_var,
232 builtin_i.export_: assign_osh.Export(mem, errfmt),
233 builtin_i.readonly: assign_osh.Readonly(mem, errfmt),
234 }
235 builtins = { # Lookup
236 builtin_i.echo: io_osh.Echo(exec_opts),
237 builtin_i.shift: assign_osh.Shift(mem),
238
239 builtin_i.history: readline_osh.History(
240 readline,
241 mem,
242 errfmt,
243 mylib.Stdout(),
244 ),
245
246 builtin_i.compopt: completion_osh.CompOpt(compopt_state, errfmt),
247 builtin_i.compadjust: completion_osh.CompAdjust(mem),
248
249 builtin_i.alias: pure_osh.Alias(aliases, errfmt),
250 builtin_i.unalias: pure_osh.UnAlias(aliases, errfmt),
251 }
252
253 debug_f = util.DebugFile(sys.stderr)
254 cmd_deps = cmd_eval.Deps()
255 cmd_deps.mutable_opts = mutable_opts
256
257 search_path = state.SearchPath(mem)
258
259 ext_prog = \
260 ext_prog or process.ExternalProgram('', fd_state, errfmt, debug_f)
261
262 cmd_deps.dumper = dev.CrashDumper('', fd_state)
263 cmd_deps.debug_f = debug_f
264
265 splitter = split.SplitContext(mem)
266
267 arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, mutable_opts,
268 parse_ctx, errfmt)
269 bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, mutable_opts,
270 parse_ctx, errfmt)
271 expr_ev = expr_eval.ExprEvaluator(mem, mutable_opts, methods, splitter,
272 errfmt)
273 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
274 word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, mutable_opts,
275 tilde_ev, splitter, errfmt)
276 signal_safe = pyos.InitSignalSafe()
277 trap_state = trap_osh.TrapState(signal_safe)
278 cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs,
279 assign_builtins, arena, cmd_deps,
280 trap_state, signal_safe)
281
282 multi_trace = dev.MultiTracer(posix.getpid(), '', '', '', fd_state)
283 tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, debug_f,
284 multi_trace)
285 waiter = process.Waiter(job_list, exec_opts, trap_state, tracer)
286
287 hay_state = hay_ysh.HayState()
288 shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
289 hay_state, builtins, search_path,
290 ext_prog, waiter, tracer, job_control,
291 job_list, fd_state, trap_state, errfmt)
292
293 assert cmd_ev.mutable_opts is not None, cmd_ev
294 prompt_ev = prompt.Evaluator('osh', '0.0.0', parse_ctx, mem)
295
296 global_io = value.IO(cmd_ev, prompt_ev)
297 vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
298 prompt_ev, global_io, tracer)
299
300 try:
301 from _devbuild.gen.help_meta import TOPICS
302 except ImportError:
303 TOPICS = None # minimal dev build
304 spec_builder = completion_osh.SpecBuilder(cmd_ev, parse_ctx, word_ev,
305 splitter, comp_lookup, TOPICS,
306 errfmt)
307
308 # Add some builtins that depend on the executor!
309 complete_builtin = completion_osh.Complete(spec_builder, comp_lookup)
310 builtins[builtin_i.complete] = complete_builtin
311 builtins[builtin_i.compgen] = completion_osh.CompGen(spec_builder)
312
313 return cmd_ev
314
315
316def EvalCode(code_str, parse_ctx, comp_lookup=None, mem=None, aliases=None):
317 """Unit tests can evaluate code strings and then use the resulting
318 CommandEvaluator."""
319 arena = parse_ctx.arena
320 errfmt = ui.ErrorFormatter()
321
322 comp_lookup = comp_lookup or completion.Lookup()
323 mem = mem or state.Mem('', [], arena, [])
324 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
325 mem.exec_opts = exec_opts
326
327 state.InitMem(mem, {}, '0.1')
328 mutable_opts.Init()
329
330 line_reader, _ = InitLexer(code_str, arena)
331 c_parser = parse_ctx.MakeOshParser(line_reader)
332
333 cmd_ev = InitCommandEvaluator(parse_ctx=parse_ctx,
334 comp_lookup=comp_lookup,
335 arena=arena,
336 mem=mem,
337 aliases=aliases)
338
339 main_loop.Batch(cmd_ev, c_parser, errfmt) # Parse and execute!
340 return cmd_ev
341
342
343def InitParseContext(arena=None,
344 ysh_grammar=None,
345 aliases=None,
346 parse_opts=None,
347 do_lossless=False):
348 arena = arena or MakeArena('<test_lib>')
349
350 if aliases is None:
351 aliases = {}
352
353 mem = state.Mem('', [], arena, [])
354 if parse_opts is None:
355 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
356
357 parse_ctx = parse_lib.ParseContext(arena,
358 parse_opts,
359 aliases,
360 ysh_grammar,
361 do_lossless=do_lossless)
362
363 return parse_ctx
364
365
366def InitWordParser(word_str, oil_at=False, arena=None):
367 arena = arena or MakeArena('<test_lib>')
368
369 mem = state.Mem('', [], arena, [])
370 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
371
372 # CUSTOM SETTING
373 mutable_opts.opt0_array[option_i.parse_at] = oil_at
374
375 loader = pyutil.GetResourceLoader()
376 ysh_grammar = pyutil.LoadYshGrammar(loader)
377 parse_ctx = parse_lib.ParseContext(arena, parse_opts, {}, ysh_grammar)
378 line_reader, _ = InitLexer(word_str, arena)
379 c_parser = parse_ctx.MakeOshParser(line_reader)
380 # Hack
381 return c_parser.w_parser
382
383
384def InitCommandParser(code_str, arena=None):
385 arena = arena or MakeArena('<test_lib>')
386
387 loader = pyutil.GetResourceLoader()
388 ysh_grammar = pyutil.LoadYshGrammar(loader)
389
390 parse_ctx = InitParseContext(arena=arena, ysh_grammar=ysh_grammar)
391 line_reader, _ = InitLexer(code_str, arena)
392 c_parser = parse_ctx.MakeOshParser(line_reader)
393 return c_parser
394
395
396def SetLocalString(mem, name, s):
397 # type: (state.Mem, str, str) -> None
398 """Bind a local string."""
399 assert isinstance(s, str)
400 mem.SetNamed(location.LName(name), value.Str(s), scope_e.LocalOnly)