OILS / core / test_lib.py View on Github | oilshell.org

399 lines, 279 significant
1#!/usr/bin/env python2
2# Copyright 2016 Andy Chu. All rights reserved.
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8"""
9test_lib.py - Functions for testing.
10"""
11
12import string
13import sys
14
15from _devbuild.gen.option_asdl import builtin_i, option_i
16from _devbuild.gen.runtime_asdl import cmd_value, scope_e
17from _devbuild.gen.syntax_asdl import loc, source, SourceLine, Token
18from _devbuild.gen.value_asdl import value
19from asdl import pybase
20from builtin import assign_osh
21from builtin import completion_osh
22from builtin import hay_ysh
23from builtin import io_osh
24from builtin import pure_osh
25from builtin import readline_osh
26from builtin import trap_osh
27from core import alloc
28from core import completion
29from core import dev
30from core import executor
31from core import main_loop
32from core import optview
33from core import process
34from core import pyos
35from core import pyutil
36from core import state
37from core import ui
38from core import util
39from core import vm
40from frontend import lexer
41from frontend import location
42from frontend import parse_lib
43from frontend import reader
44from osh import cmd_eval
45from osh import prompt
46from osh import sh_expr_eval
47from osh import split
48from osh import word_eval
49from ysh import expr_eval
50from mycpp import mylib
51
52import posix_ as posix
53
54
55def MakeBuiltinArgv(argv):
56 return cmd_value.Argv(argv, [loc.Missing] * len(argv), None, None, None,
57 None)
58
59
60def FakeTok(id_, val):
61 # type: (int, str) -> Token
62 src = source.Interactive
63 line = SourceLine(1, val, src)
64 return Token(id_, len(val), 0, line, None)
65
66
67def PrintableString(s):
68 """For pretty-printing in tests."""
69 if all(c in string.printable for c in s):
70 return s
71 return repr(s)
72
73
74def TokensEqual(left, right):
75 # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
76
77 if left.id != right.id:
78 return False
79
80 if left.line is not None:
81 left_str = lexer.TokenVal(left)
82 else:
83 left_str = None
84
85 if right.line is not None:
86 right_str = lexer.TokenVal(right)
87 else:
88 right_str = None
89
90 # Better error message sometimes:
91 #assert left_str == right_str, '%r != %r' % (left_str, right_str)
92 return left_str == right_str
93
94
95def TokenWordsEqual(left, right):
96 # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
97 return TokensEqual(left.token, right.token)
98 #return left == right
99
100
101def AsdlEqual(left, right):
102 """Check if generated ASDL instances are equal.
103
104 We don't use equality in the actual code, so this is relegated to
105 test_lib.
106 """
107 if left is None and right is None:
108 return True
109
110 if isinstance(left, (int, str, bool, pybase.SimpleObj)):
111 return left == right
112
113 if isinstance(left, list):
114 if len(left) != len(right):
115 return False
116 for a, b in zip(left, right):
117 if not AsdlEqual(a, b):
118 return False
119 return True
120
121 if isinstance(left, pybase.CompoundObj):
122 if left.tag() != right.tag():
123 return False
124
125 field_names = left.__slots__ # hack for now
126 for name in field_names:
127 # Special case: we are not testing locations right now.
128 if name == 'span_id':
129 continue
130 a = getattr(left, name)
131 b = getattr(right, name)
132 if not AsdlEqual(a, b):
133 return False
134
135 return True
136
137 raise AssertionError(left)
138
139
140def AssertAsdlEqual(test, left, right):
141 test.assertTrue(AsdlEqual(left, right),
142 'Expected %s, got %s' % (left, right))
143
144
145def MakeArena(source_name):
146 arena = alloc.Arena(save_tokens=True)
147 arena.PushSource(source.MainFile(source_name))
148 return arena
149
150
151def InitLineLexer(s, arena):
152 line_lexer = lexer.LineLexer(arena)
153 src = source.Interactive
154 line_lexer.Reset(SourceLine(1, s, src), 0)
155 return line_lexer
156
157
158def InitLexer(s, arena):
159 """For tests only."""
160 line_lexer = lexer.LineLexer(arena)
161 line_reader = reader.StringLineReader(s, arena)
162 lx = lexer.Lexer(line_lexer, line_reader)
163 return line_reader, lx
164
165
166def InitWordEvaluator(exec_opts=None):
167 arena = MakeArena('<InitWordEvaluator>')
168 mem = state.Mem('', [], arena, [])
169
170 if exec_opts is None:
171 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
172 mem.exec_opts = exec_opts # circular dep
173 state.InitMem(mem, {}, '0.1')
174 mutable_opts.Init()
175 else:
176 mutable_opts = None
177
178 cmd_deps = cmd_eval.Deps()
179 cmd_deps.trap_nodes = []
180
181 splitter = split.SplitContext(mem)
182 errfmt = ui.ErrorFormatter()
183
184 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
185 ev = word_eval.CompletionWordEvaluator(mem, exec_opts, mutable_opts,
186 tilde_ev, splitter, errfmt)
187 return ev
188
189
190def InitCommandEvaluator(parse_ctx=None,
191 comp_lookup=None,
192 arena=None,
193 mem=None,
194 aliases=None,
195 ext_prog=None):
196
197 opt0_array = state.InitOpts()
198 opt_stacks = [None] * option_i.ARRAY_SIZE
199 if parse_ctx:
200 arena = parse_ctx.arena
201 else:
202 parse_ctx = InitParseContext()
203
204 mem = mem or state.Mem('', [], arena, [])
205 exec_opts = optview.Exec(opt0_array, opt_stacks)
206 mutable_opts = state.MutableOpts(mem, opt0_array, opt_stacks, None)
207 mem.exec_opts = exec_opts
208 state.InitMem(mem, {}, '0.1')
209 mutable_opts.Init()
210
211 # No 'readline' in the tests.
212
213 errfmt = ui.ErrorFormatter()
214 job_control = process.JobControl()
215 job_list = process.JobList()
216 fd_state = process.FdState(errfmt, job_control, job_list, None, None, None)
217 aliases = {} if aliases is None else aliases
218 procs = {}
219 methods = {}
220
221 compopt_state = completion.OptionState()
222 comp_lookup = comp_lookup or completion.Lookup()
223
224 readline = None # simulate not having it
225
226 new_var = assign_osh.NewVar(mem, procs, errfmt)
227 assign_builtins = {
228 builtin_i.declare: new_var,
229 builtin_i.typeset: new_var,
230 builtin_i.local: new_var,
231 builtin_i.export_: assign_osh.Export(mem, errfmt),
232 builtin_i.readonly: assign_osh.Readonly(mem, errfmt),
233 }
234 builtins = { # Lookup
235 builtin_i.echo: io_osh.Echo(exec_opts),
236 builtin_i.shift: assign_osh.Shift(mem),
237
238 builtin_i.history: readline_osh.History(
239 readline,
240 mem,
241 errfmt,
242 mylib.Stdout(),
243 ),
244
245 builtin_i.compopt: completion_osh.CompOpt(compopt_state, errfmt),
246 builtin_i.compadjust: completion_osh.CompAdjust(mem),
247
248 builtin_i.alias: pure_osh.Alias(aliases, errfmt),
249 builtin_i.unalias: pure_osh.UnAlias(aliases, errfmt),
250 }
251
252 debug_f = util.DebugFile(sys.stderr)
253 cmd_deps = cmd_eval.Deps()
254 cmd_deps.mutable_opts = mutable_opts
255
256 search_path = state.SearchPath(mem)
257
258 ext_prog = \
259 ext_prog or process.ExternalProgram('', fd_state, errfmt, debug_f)
260
261 cmd_deps.dumper = dev.CrashDumper('', fd_state)
262 cmd_deps.debug_f = debug_f
263
264 splitter = split.SplitContext(mem)
265
266 arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, mutable_opts,
267 parse_ctx, errfmt)
268 bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, mutable_opts,
269 parse_ctx, errfmt)
270 expr_ev = expr_eval.ExprEvaluator(mem, mutable_opts, methods, splitter,
271 errfmt)
272 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
273 word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, mutable_opts,
274 tilde_ev, splitter, errfmt)
275 signal_safe = pyos.InitSignalSafe()
276 trap_state = trap_osh.TrapState(signal_safe)
277 cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs,
278 assign_builtins, arena, cmd_deps,
279 trap_state, signal_safe)
280
281 multi_trace = dev.MultiTracer(posix.getpid(), '', '', '', fd_state)
282 tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, debug_f,
283 multi_trace)
284 waiter = process.Waiter(job_list, exec_opts, trap_state, tracer)
285
286 hay_state = hay_ysh.HayState()
287 shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
288 hay_state, builtins, search_path,
289 ext_prog, waiter, tracer, job_control,
290 job_list, fd_state, trap_state, errfmt)
291
292 assert cmd_ev.mutable_opts is not None, cmd_ev
293 prompt_ev = prompt.Evaluator('osh', '0.0.0', parse_ctx, mem)
294
295 global_io = value.IO(cmd_ev, prompt_ev)
296 vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
297 prompt_ev, global_io, tracer)
298
299 try:
300 from _devbuild.gen.help_meta import TOPICS
301 except ImportError:
302 TOPICS = None # minimal dev build
303 spec_builder = completion_osh.SpecBuilder(cmd_ev, parse_ctx, word_ev,
304 splitter, comp_lookup, TOPICS,
305 errfmt)
306
307 # Add some builtins that depend on the executor!
308 complete_builtin = completion_osh.Complete(spec_builder, comp_lookup)
309 builtins[builtin_i.complete] = complete_builtin
310 builtins[builtin_i.compgen] = completion_osh.CompGen(spec_builder)
311
312 return cmd_ev
313
314
315def EvalCode(code_str, parse_ctx, comp_lookup=None, mem=None, aliases=None):
316 """Unit tests can evaluate code strings and then use the resulting
317 CommandEvaluator."""
318 arena = parse_ctx.arena
319 errfmt = ui.ErrorFormatter()
320
321 comp_lookup = comp_lookup or completion.Lookup()
322 mem = mem or state.Mem('', [], arena, [])
323 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
324 mem.exec_opts = exec_opts
325
326 state.InitMem(mem, {}, '0.1')
327 mutable_opts.Init()
328
329 line_reader, _ = InitLexer(code_str, arena)
330 c_parser = parse_ctx.MakeOshParser(line_reader)
331
332 cmd_ev = InitCommandEvaluator(parse_ctx=parse_ctx,
333 comp_lookup=comp_lookup,
334 arena=arena,
335 mem=mem,
336 aliases=aliases)
337
338 main_loop.Batch(cmd_ev, c_parser, errfmt) # Parse and execute!
339 return cmd_ev
340
341
342def InitParseContext(arena=None,
343 ysh_grammar=None,
344 aliases=None,
345 parse_opts=None,
346 do_lossless=False):
347 arena = arena or MakeArena('<test_lib>')
348
349 if aliases is None:
350 aliases = {}
351
352 mem = state.Mem('', [], arena, [])
353 if parse_opts is None:
354 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
355
356 parse_ctx = parse_lib.ParseContext(arena,
357 parse_opts,
358 aliases,
359 ysh_grammar,
360 do_lossless=do_lossless)
361
362 return parse_ctx
363
364
365def InitWordParser(word_str, oil_at=False, arena=None):
366 arena = arena or MakeArena('<test_lib>')
367
368 mem = state.Mem('', [], arena, [])
369 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
370
371 # CUSTOM SETTING
372 mutable_opts.opt0_array[option_i.parse_at] = oil_at
373
374 loader = pyutil.GetResourceLoader()
375 ysh_grammar = pyutil.LoadYshGrammar(loader)
376 parse_ctx = parse_lib.ParseContext(arena, parse_opts, {}, ysh_grammar)
377 line_reader, _ = InitLexer(word_str, arena)
378 c_parser = parse_ctx.MakeOshParser(line_reader)
379 # Hack
380 return c_parser.w_parser
381
382
383def InitCommandParser(code_str, arena=None):
384 arena = arena or MakeArena('<test_lib>')
385
386 loader = pyutil.GetResourceLoader()
387 ysh_grammar = pyutil.LoadYshGrammar(loader)
388
389 parse_ctx = InitParseContext(arena=arena, ysh_grammar=ysh_grammar)
390 line_reader, _ = InitLexer(code_str, arena)
391 c_parser = parse_ctx.MakeOshParser(line_reader)
392 return c_parser
393
394
395def SetLocalString(mem, name, s):
396 # type: (state.Mem, str, str) -> None
397 """Bind a local string."""
398 assert isinstance(s, str)
399 mem.SetNamed(location.LName(name), value.Str(s), scope_e.LocalOnly)