1 | #!/usr/bin/env python2
|
2 | from __future__ import print_function
|
3 | """
|
4 | sh_spec.py -- Test framework to compare shells.
|
5 |
|
6 | Assertion help:
|
7 | stdout: A single line of expected stdout. Newline is implicit.
|
8 | stdout-json: JSON-encoded string. Use for the empty string (no newline),
|
9 | for unicode chars, etc.
|
10 |
|
11 | stderr: Ditto for stderr stream.
|
12 | status: Expected shell return code. If not specified, the case must exit 0.
|
13 |
|
14 | Results:
|
15 | PASS - we got the ideal, expected value
|
16 | OK - we got a value that was not ideal, but expected
|
17 | For OSH this is behavior that was defined to be different?
|
18 | N-I - Not implemented (e.g. $''). Assertions still checked (in case it
|
19 | starts working)
|
20 | BUG - we verified the value of a known bug
|
21 | FAIL - we got an unexpected value. If the implementation can't be changed,
|
22 | it should be converted to BUG or OK. Otherwise it should be made to
|
23 | PASS.
|
24 |
|
25 | NOTE: The difference between OK and BUG is a matter of judgement. If the ideal
|
26 | behavior is a compile time error (code 2), a runtime error is generally OK.
|
27 |
|
28 | If ALL shells agree on a broken behavior, they are all marked OK (but our
|
29 | implementation will be PASS.) But if the behavior is NOT POSIX compliant, then
|
30 | it will be a BUG.
|
31 |
|
32 | If one shell disagrees with others, that is generally a BUG.
|
33 |
|
34 | Example test case:
|
35 |
|
36 | #### hello and fail
|
37 | echo hello
|
38 | echo world
|
39 | exit 1
|
40 | ## status: 1
|
41 | #
|
42 | # ignored comment
|
43 | #
|
44 | ## STDOUT:
|
45 | hello
|
46 | world
|
47 | ## END
|
48 |
|
49 | """
|
50 |
|
51 | import collections
|
52 | import cgi
|
53 | import cStringIO
|
54 | import errno
|
55 | import json
|
56 | import optparse
|
57 | import os
|
58 | import pprint
|
59 | import re
|
60 | import shutil
|
61 | import subprocess
|
62 | import sys
|
63 |
|
64 | from test import spec_lib
|
65 | from doctools import html_head
|
66 |
|
67 | log = spec_lib.log
|
68 |
|
69 |
|
70 | # Magic strings for other variants of OSH.
|
71 |
|
72 | # NOTE: osh_ALT is usually _bin/osh -- the release binary.
|
73 | # It would be better to rename these osh-cpython and osh-ovm. Have the concept
|
74 | # of a suffix?
|
75 |
|
76 | OSH_CPYTHON = ('osh', 'osh-dbg')
|
77 | OTHER_OSH = ('osh_ALT',)
|
78 |
|
79 | YSH_CPYTHON = ('ysh', 'ysh-dbg')
|
80 | OTHER_YSH = ('oil_ALT',)
|
81 |
|
82 |
|
83 | class ParseError(Exception):
|
84 | pass
|
85 |
|
86 |
|
87 | # EXAMPLES:
|
88 | ## stdout: foo
|
89 | ## stdout-json: ""
|
90 | #
|
91 | # In other words, it could be (name, value) or (qualifier, name, value)
|
92 |
|
93 | KEY_VALUE_RE = re.compile(r'''
|
94 | [#][#] \s+
|
95 | # optional prefix with qualifier and shells
|
96 | (?: (OK|BUG|N-I) \s+ ([\w+/]+) \s+ )?
|
97 | ([\w\-]+) # key
|
98 | :
|
99 | \s* (.*) # value
|
100 | ''', re.VERBOSE)
|
101 |
|
102 | END_MULTILINE_RE = re.compile(r'''
|
103 | [#][#] \s+ END
|
104 | ''', re.VERBOSE)
|
105 |
|
106 | # Line types
|
107 | TEST_CASE_BEGIN = 0 # Starts with ####
|
108 | KEY_VALUE = 1 # Metadata
|
109 | KEY_VALUE_MULTILINE = 2 # STDOUT STDERR
|
110 | END_MULTILINE = 3 # STDOUT STDERR
|
111 | PLAIN_LINE = 4 # Uncommented
|
112 | EOF = 5
|
113 |
|
114 | LEX_OUTER = 0 # Ignore blank lines, e.g. for separating cases
|
115 | LEX_RAW = 1 # Blank lines are significant
|
116 |
|
117 |
|
118 | class Tokenizer(object):
|
119 | """Modal lexer!"""
|
120 |
|
121 | def __init__(self, f):
|
122 | self.f = f
|
123 |
|
124 | self.cursor = None
|
125 | self.line_num = 0
|
126 |
|
127 | self.next()
|
128 |
|
129 | def _ClassifyLine(self, line, lex_mode):
|
130 | if not line: # empty
|
131 | return self.line_num, EOF, ''
|
132 |
|
133 | if lex_mode == LEX_OUTER and not line.strip():
|
134 | return None
|
135 |
|
136 | if line.startswith('####'):
|
137 | desc = line[4:].strip()
|
138 | return self.line_num, TEST_CASE_BEGIN, desc
|
139 |
|
140 | m = KEY_VALUE_RE.match(line)
|
141 | if m:
|
142 | qualifier, shells, name, value = m.groups()
|
143 | # HACK: Expected data should have the newline.
|
144 | if name in ('stdout', 'stderr'):
|
145 | value += '\n'
|
146 |
|
147 | if name in ('STDOUT', 'STDERR'):
|
148 | token_type = KEY_VALUE_MULTILINE
|
149 | else:
|
150 | token_type = KEY_VALUE
|
151 | return self.line_num, token_type, (qualifier, shells, name, value)
|
152 |
|
153 | m = END_MULTILINE_RE.match(line)
|
154 | if m:
|
155 | return self.line_num, END_MULTILINE, None
|
156 |
|
157 | # If it starts with ##, it should be metadata. This finds some typos.
|
158 | if line.lstrip().startswith('##'):
|
159 | raise RuntimeError('Invalid ## line %r' % line)
|
160 |
|
161 | if line.lstrip().startswith('#'): # Ignore comments
|
162 | return None # try again
|
163 |
|
164 | # Non-empty line that doesn't start with '#'
|
165 | # NOTE: We need the original line to test the whitespace sensitive <<-.
|
166 | # And we need rstrip because we add newlines back below.
|
167 | return self.line_num, PLAIN_LINE, line
|
168 |
|
169 | def next(self, lex_mode=LEX_OUTER):
|
170 | """Raises StopIteration when exhausted."""
|
171 | while True:
|
172 | line = self.f.readline()
|
173 | self.line_num += 1
|
174 |
|
175 | tok = self._ClassifyLine(line, lex_mode)
|
176 | if tok is not None:
|
177 | break
|
178 |
|
179 | self.cursor = tok
|
180 | return self.cursor
|
181 |
|
182 | def peek(self):
|
183 | return self.cursor
|
184 |
|
185 |
|
186 | def AddMetadataToCase(case, qualifier, shells, name, value):
|
187 | shells = shells.split('/') # bash/dash/mksh
|
188 | for shell in shells:
|
189 | if shell not in case:
|
190 | case[shell] = {}
|
191 | case[shell][name] = value
|
192 | case[shell]['qualifier'] = qualifier
|
193 |
|
194 |
|
195 | # Format of a test script.
|
196 | #
|
197 | # -- Code is either literal lines, or a commented out code: value.
|
198 | # code = PLAIN_LINE*
|
199 | # | '## code:' VALUE
|
200 | #
|
201 | # -- Key value pairs can be single- or multi-line
|
202 | # key_value = '##' KEY ':' VALUE
|
203 | # | KEY_VALUE_MULTILINE PLAIN_LINE* END_MULTILINE
|
204 | #
|
205 | # -- Description, then key-value pairs surrounding code.
|
206 | # test_case = '####' DESC
|
207 | # key_value*
|
208 | # code
|
209 | # key_value*
|
210 | #
|
211 | # -- Should be a blank line after each test case. Leading comments and code
|
212 | # -- are OK.
|
213 | #
|
214 | # test_file =
|
215 | # key_value* -- file level metadata
|
216 | # (test_case '\n')*
|
217 |
|
218 |
|
219 | def ParseKeyValue(tokens, case):
|
220 | """Parse commented-out metadata in a test case.
|
221 |
|
222 | The metadata must be contiguous.
|
223 |
|
224 | Args:
|
225 | tokens: Tokenizer
|
226 | case: dictionary to add to
|
227 | """
|
228 | while True:
|
229 | line_num, kind, item = tokens.peek()
|
230 |
|
231 | if kind == KEY_VALUE_MULTILINE:
|
232 | qualifier, shells, name, empty_value = item
|
233 | if empty_value:
|
234 | raise ParseError(
|
235 | 'Line %d: got value %r for %r, but the value should be on the '
|
236 | 'following lines' % (line_num, empty_value, name))
|
237 |
|
238 | value_lines = []
|
239 | while True:
|
240 | tokens.next(lex_mode=LEX_RAW) # empty lines aren't skipped
|
241 | _, kind2, item2 = tokens.peek()
|
242 | if kind2 != PLAIN_LINE:
|
243 | break
|
244 | value_lines.append(item2)
|
245 |
|
246 | value = ''.join(value_lines)
|
247 |
|
248 | name = name.lower() # STDOUT -> stdout
|
249 | if qualifier:
|
250 | AddMetadataToCase(case, qualifier, shells, name, value)
|
251 | else:
|
252 | case[name] = value
|
253 |
|
254 | # END token is optional.
|
255 | if kind2 == END_MULTILINE:
|
256 | tokens.next()
|
257 |
|
258 | elif kind == KEY_VALUE:
|
259 | qualifier, shells, name, value = item
|
260 |
|
261 | if qualifier:
|
262 | AddMetadataToCase(case, qualifier, shells, name, value)
|
263 | else:
|
264 | case[name] = value
|
265 |
|
266 | tokens.next()
|
267 |
|
268 | else: # Unknown token type
|
269 | break
|
270 |
|
271 |
|
272 | def ParseCodeLines(tokens, case):
|
273 | """Parse uncommented code in a test case."""
|
274 | _, kind, item = tokens.peek()
|
275 | if kind != PLAIN_LINE:
|
276 | raise ParseError('Expected a line of code (got %r, %r)' % (kind, item))
|
277 | code_lines = []
|
278 | while True:
|
279 | _, kind, item = tokens.peek()
|
280 | if kind != PLAIN_LINE:
|
281 | case['code'] = ''.join(code_lines)
|
282 | return
|
283 | code_lines.append(item)
|
284 | tokens.next(lex_mode=LEX_RAW)
|
285 |
|
286 |
|
287 | def ParseTestCase(tokens):
|
288 | """Parse a single test case and return it.
|
289 |
|
290 | If at EOF, return None.
|
291 | """
|
292 | line_num, kind, item = tokens.peek()
|
293 | if kind == EOF:
|
294 | return None
|
295 |
|
296 | if kind != TEST_CASE_BEGIN:
|
297 | raise RuntimeError(
|
298 | "line %d: Expected TEST_CASE_BEGIN, got %r" % (line_num, [kind, item]))
|
299 |
|
300 | tokens.next()
|
301 |
|
302 | case = {'desc': item, 'line_num': line_num}
|
303 |
|
304 | ParseKeyValue(tokens, case)
|
305 |
|
306 | # For broken code
|
307 | if 'code' in case: # Got it through a key value pair
|
308 | return case
|
309 |
|
310 | ParseCodeLines(tokens, case)
|
311 | ParseKeyValue(tokens, case)
|
312 |
|
313 | return case
|
314 |
|
315 |
|
316 | _META_FIELDS = [
|
317 | 'our_shell',
|
318 | 'compare_shells',
|
319 | 'suite',
|
320 | 'tags',
|
321 | 'oils_failures_allowed',
|
322 | ]
|
323 |
|
324 |
|
325 | def ParseTestFile(test_file, tokens):
|
326 | """
|
327 | test_file: Only for error message
|
328 | """
|
329 | file_metadata = {}
|
330 | test_cases = []
|
331 |
|
332 | try:
|
333 | # Skip over the header. Setup code can go here, although would we have to
|
334 | # execute it on every case?
|
335 | while True:
|
336 | line_num, kind, item = tokens.peek()
|
337 | if kind != KEY_VALUE:
|
338 | break
|
339 |
|
340 | qualifier, shells, name, value = item
|
341 | if qualifier is not None:
|
342 | raise RuntimeError('Invalid qualifier in spec file metadata')
|
343 | if shells is not None:
|
344 | raise RuntimeError('Invalid shells in spec file metadata')
|
345 |
|
346 | file_metadata[name] = value
|
347 |
|
348 | tokens.next()
|
349 |
|
350 | while True: # Loop over cases
|
351 | test_case = ParseTestCase(tokens)
|
352 | if test_case is None:
|
353 | break
|
354 | test_cases.append(test_case)
|
355 |
|
356 | except StopIteration:
|
357 | raise RuntimeError('Unexpected EOF parsing test cases')
|
358 |
|
359 | for name in file_metadata:
|
360 | if name not in _META_FIELDS:
|
361 | raise RuntimeError('Invalid file metadata %r in %r' % (name, test_file))
|
362 |
|
363 | return file_metadata, test_cases
|
364 |
|
365 |
|
366 | def CreateStringAssertion(d, key, assertions, qualifier=False):
|
367 | found = False
|
368 |
|
369 | exp = d.get(key)
|
370 | if exp is not None:
|
371 | a = EqualAssertion(key, exp, qualifier=qualifier)
|
372 | assertions.append(a)
|
373 | found = True
|
374 |
|
375 | exp_json = d.get(key + '-json')
|
376 | if exp_json is not None:
|
377 | exp = json.loads(exp_json, encoding='utf-8')
|
378 | a = EqualAssertion(key, exp, qualifier=qualifier)
|
379 | assertions.append(a)
|
380 | found = True
|
381 |
|
382 | # For testing invalid unicode
|
383 | exp_repr = d.get(key + '-repr')
|
384 | if exp_repr is not None:
|
385 | exp = eval(exp_repr)
|
386 | a = EqualAssertion(key, exp, qualifier=qualifier)
|
387 | assertions.append(a)
|
388 | found = True
|
389 |
|
390 | return found
|
391 |
|
392 |
|
393 | def CreateIntAssertion(d, key, assertions, qualifier=False):
|
394 | exp = d.get(key) # expected
|
395 | if exp is not None:
|
396 | # For now, turn it into int
|
397 | a = EqualAssertion(key, int(exp), qualifier=qualifier)
|
398 | assertions.append(a)
|
399 | return True
|
400 | return False
|
401 |
|
402 |
|
403 | def CreateAssertions(case, sh_label):
|
404 | """
|
405 | Given a raw test case and a shell label, create EqualAssertion instances to
|
406 | run.
|
407 | """
|
408 | assertions = []
|
409 |
|
410 | # Whether we found assertions
|
411 | stdout = False
|
412 | stderr = False
|
413 | status = False
|
414 |
|
415 | # So the assertion are exactly the same for osh and osh_ALT
|
416 |
|
417 | if sh_label.startswith('osh'):
|
418 | case_sh = 'osh'
|
419 | elif sh_label.startswith('bash'):
|
420 | case_sh = 'bash'
|
421 | else:
|
422 | case_sh = sh_label
|
423 |
|
424 | if case_sh in case:
|
425 | q = case[case_sh]['qualifier']
|
426 | if CreateStringAssertion(case[case_sh], 'stdout', assertions, qualifier=q):
|
427 | stdout = True
|
428 | if CreateStringAssertion(case[case_sh], 'stderr', assertions, qualifier=q):
|
429 | stderr = True
|
430 | if CreateIntAssertion(case[case_sh], 'status', assertions, qualifier=q):
|
431 | status = True
|
432 |
|
433 | if not stdout:
|
434 | CreateStringAssertion(case, 'stdout', assertions)
|
435 | if not stderr:
|
436 | CreateStringAssertion(case, 'stderr', assertions)
|
437 | if not status:
|
438 | if 'status' in case:
|
439 | CreateIntAssertion(case, 'status', assertions)
|
440 | else:
|
441 | # If the user didn't specify a 'status' assertion, assert that the exit
|
442 | # code is 0.
|
443 | a = EqualAssertion('status', 0)
|
444 | assertions.append(a)
|
445 |
|
446 | no_traceback = SubstringAssertion('stderr', 'Traceback (most recent')
|
447 | assertions.append(no_traceback)
|
448 |
|
449 | #print 'SHELL', shell
|
450 | #pprint.pprint(case)
|
451 | #print(assertions)
|
452 | return assertions
|
453 |
|
454 |
|
455 | class Result(object):
|
456 | """Result of an stdout/stderr/status assertion or of a (case, shell) cell.
|
457 |
|
458 | Order is important: the result of a cell is the minimum of the results of
|
459 | each assertion.
|
460 | """
|
461 | TIMEOUT = 0 # ONLY a cell result, not an assertion result
|
462 | FAIL = 1
|
463 | BUG = 2
|
464 | NI = 3
|
465 | OK = 4
|
466 | PASS = 5
|
467 |
|
468 | length = 6 # for loops
|
469 |
|
470 |
|
471 | class EqualAssertion(object):
|
472 | """Check that two values are equal."""
|
473 |
|
474 | def __init__(self, key, expected, qualifier=None):
|
475 | self.key = key
|
476 | self.expected = expected # expected value
|
477 | self.qualifier = qualifier # whether this was a special case?
|
478 |
|
479 | def __repr__(self):
|
480 | return '<EqualAssertion %s == %r>' % (self.key, self.expected)
|
481 |
|
482 | def Check(self, shell, record):
|
483 | actual = record[self.key]
|
484 | if actual != self.expected:
|
485 | if len(str(self.expected)) < 40:
|
486 | msg = '[%s %s] Expected %r, got %r' % (shell, self.key, self.expected,
|
487 | actual)
|
488 | else:
|
489 | msg = '''
|
490 | [%s %s]
|
491 | Expected %r
|
492 | Got %r
|
493 | ''' % (shell, self.key, self.expected, actual)
|
494 |
|
495 | # TODO: Make this better and add a flag for it.
|
496 | if 0:
|
497 | import difflib
|
498 | for line in difflib.unified_diff(
|
499 | self.expected, actual, fromfile='expected', tofile='actual'):
|
500 | print(repr(line))
|
501 |
|
502 | return Result.FAIL, msg
|
503 | if self.qualifier == 'BUG': # equal, but known bad
|
504 | return Result.BUG, ''
|
505 | if self.qualifier == 'N-I': # equal, and known UNIMPLEMENTED
|
506 | return Result.NI, ''
|
507 | if self.qualifier == 'OK': # equal, but ok (not ideal)
|
508 | return Result.OK, ''
|
509 | return Result.PASS, '' # ideal behavior
|
510 |
|
511 |
|
512 | class SubstringAssertion(object):
|
513 | """Check that a string like stderr doesn't have a substring."""
|
514 |
|
515 | def __init__(self, key, substring):
|
516 | self.key = key
|
517 | self.substring = substring
|
518 |
|
519 | def __repr__(self):
|
520 | return '<SubstringAssertion %s == %r>' % (self.key, self.substring)
|
521 |
|
522 | def Check(self, shell, record):
|
523 | actual = record[self.key]
|
524 | if self.substring in actual:
|
525 | msg = '[%s %s] Found %r' % (shell, self.key, self.substring)
|
526 | return Result.FAIL, msg
|
527 | return Result.PASS, ''
|
528 |
|
529 |
|
530 | class Stats(object):
|
531 | def __init__(self, num_cases, sh_labels):
|
532 | self.counters = collections.defaultdict(int)
|
533 | c = self.counters
|
534 | c['num_cases'] = num_cases
|
535 | c['oils_num_passed'] = 0
|
536 | c['oils_num_failed'] = 0
|
537 | # Number of osh_ALT results that differed from osh.
|
538 | c['oils_ALT_delta'] = 0
|
539 |
|
540 | self.by_shell = {}
|
541 | for sh in sh_labels:
|
542 | self.by_shell[sh] = collections.defaultdict(int)
|
543 | self.nonzero_results = collections.defaultdict(int)
|
544 |
|
545 | self.tsv_rows = []
|
546 |
|
547 | def Inc(self, counter_name):
|
548 | self.counters[counter_name] += 1
|
549 |
|
550 | def Get(self, counter_name):
|
551 | return self.counters[counter_name]
|
552 |
|
553 | def Set(self, counter_name, val):
|
554 | self.counters[counter_name] = val
|
555 |
|
556 | def ReportCell(self, case_num, cell_result, sh_label):
|
557 | self.tsv_rows.append((str(case_num), sh_label, TEXT_CELLS[cell_result]))
|
558 |
|
559 | self.by_shell[sh_label][cell_result] += 1
|
560 | self.nonzero_results[cell_result] += 1
|
561 |
|
562 | c = self.counters
|
563 | if cell_result == Result.TIMEOUT:
|
564 | c['num_timeout'] += 1
|
565 | elif cell_result == Result.FAIL:
|
566 | # Special logic: don't count osh_ALT because its failures will be
|
567 | # counted in the delta.
|
568 | if sh_label not in OTHER_OSH + OTHER_YSH:
|
569 | c['num_failed'] += 1
|
570 |
|
571 | if sh_label in OSH_CPYTHON + YSH_CPYTHON:
|
572 | c['oils_num_failed'] += 1
|
573 | elif cell_result == Result.BUG:
|
574 | c['num_bug'] += 1
|
575 | elif cell_result == Result.NI:
|
576 | c['num_ni'] += 1
|
577 | elif cell_result == Result.OK:
|
578 | c['num_ok'] += 1
|
579 | elif cell_result == Result.PASS:
|
580 | c['num_passed'] += 1
|
581 | if sh_label in OSH_CPYTHON + YSH_CPYTHON:
|
582 | c['oils_num_passed'] += 1
|
583 | else:
|
584 | raise AssertionError()
|
585 |
|
586 | def WriteTsv(self, f):
|
587 | f.write('case\tshell\tresult\n')
|
588 | for row in self.tsv_rows:
|
589 | f.write('\t'.join(row))
|
590 | f.write('\n')
|
591 |
|
592 |
|
593 | PIPE = subprocess.PIPE
|
594 |
|
595 | def RunCases(cases, case_predicate, shells, env, out, opts):
|
596 | """
|
597 | Run a list of test 'cases' for all 'shells' and write output to 'out'.
|
598 | """
|
599 | if opts.trace:
|
600 | for _, sh in shells:
|
601 | log('\tshell: %s', sh)
|
602 | print('\twhich $SH: ', end='', file=sys.stderr)
|
603 | subprocess.call(['which', sh])
|
604 |
|
605 | #pprint.pprint(cases)
|
606 |
|
607 | sh_labels = [sh_label for sh_label, _ in shells]
|
608 |
|
609 | out.WriteHeader(sh_labels)
|
610 | stats = Stats(len(cases), sh_labels)
|
611 |
|
612 | # Make an environment for each shell. $SH is the path to the shell, so we
|
613 | # can test flags, etc.
|
614 | sh_env = []
|
615 | for _, sh_path in shells:
|
616 | e = dict(env)
|
617 | e[opts.sh_env_var_name] = sh_path
|
618 | sh_env.append(e)
|
619 |
|
620 | # Determine which one (if any) is osh-cpython, for comparison against other
|
621 | # shells.
|
622 | osh_cpython_index = -1
|
623 | for i, (sh_label, _) in enumerate(shells):
|
624 | if sh_label in OSH_CPYTHON:
|
625 | osh_cpython_index = i
|
626 | break
|
627 |
|
628 | timeout_dir = os.path.abspath('_tmp/spec/timeouts')
|
629 | try:
|
630 | shutil.rmtree(timeout_dir)
|
631 | os.mkdir(timeout_dir)
|
632 | except OSError:
|
633 | pass
|
634 |
|
635 | # Now run each case, and print a table.
|
636 | for i, case in enumerate(cases):
|
637 | line_num = case['line_num']
|
638 | desc = case['desc']
|
639 | code = case['code']
|
640 |
|
641 | if opts.trace:
|
642 | log('case %d: %s', i, desc)
|
643 |
|
644 | if not case_predicate(i, case):
|
645 | stats.Inc('num_skipped')
|
646 | continue
|
647 |
|
648 | if opts.do_print:
|
649 | print('#### %s' % case['desc'])
|
650 | print(case['code'])
|
651 | print()
|
652 | continue
|
653 |
|
654 | stats.Inc('num_cases_run')
|
655 |
|
656 | result_row = []
|
657 |
|
658 | for shell_index, (sh_label, sh_path) in enumerate(shells):
|
659 | timeout_file = os.path.join(timeout_dir, '%02d-%s' % (i, sh_label))
|
660 | if opts.timeout:
|
661 | if opts.timeout_bin:
|
662 | # This is what smoosh itself uses. See smoosh/tests/shell_tests.sh
|
663 | # QUIRK: interval can only be a whole number
|
664 | argv = [
|
665 | opts.timeout_bin,
|
666 | '-t', opts.timeout,
|
667 | # Somehow I'm not able to get this timeout file working? I think
|
668 | # it has a bug when using stdin. It waits for the background
|
669 | # process too.
|
670 |
|
671 | #'-i', '1',
|
672 | #'-l', timeout_file
|
673 | ]
|
674 | else:
|
675 | # This kills hanging tests properly, but somehow they fail with code
|
676 | # -9?
|
677 | #argv = ['timeout', '-s', 'KILL', opts.timeout]
|
678 |
|
679 | # s suffix for seconds
|
680 | argv = ['timeout', opts.timeout + 's']
|
681 | else:
|
682 | argv = []
|
683 | argv.append(sh_path)
|
684 |
|
685 | # dash doesn't support -o posix
|
686 | if opts.posix and sh_label != 'dash':
|
687 | argv.extend(['-o', 'posix'])
|
688 |
|
689 | if opts.trace:
|
690 | log('\targv: %s', ' '.join(argv))
|
691 |
|
692 | case_env = sh_env[shell_index]
|
693 |
|
694 | # Unique dir for every test case and shell
|
695 | tmp_base = os.path.normpath(opts.tmp_env) # no . or ..
|
696 | case_tmp_dir = os.path.join(tmp_base, '%02d-%s' % (i, sh_label))
|
697 |
|
698 | try:
|
699 | os.makedirs(case_tmp_dir)
|
700 | except OSError as e:
|
701 | if e.errno != errno.EEXIST:
|
702 | raise
|
703 |
|
704 | # Some tests assume _tmp exists
|
705 | try:
|
706 | os.mkdir(os.path.join(case_tmp_dir, '_tmp'))
|
707 | except OSError as e:
|
708 | if e.errno != errno.EEXIST:
|
709 | raise
|
710 |
|
711 | case_env['TMP'] = case_tmp_dir
|
712 |
|
713 | if opts.pyann_out_dir:
|
714 | case_env = dict(case_env)
|
715 | case_env['PYANN_OUT'] = os.path.join(opts.pyann_out_dir, '%d.json' % i)
|
716 |
|
717 | try:
|
718 | p = subprocess.Popen(argv, env=case_env, cwd=case_tmp_dir,
|
719 | stdin=PIPE, stdout=PIPE, stderr=PIPE)
|
720 | except OSError as e:
|
721 | print('Error running %r: %s' % (sh_path, e), file=sys.stderr)
|
722 | sys.exit(1)
|
723 |
|
724 | p.stdin.write(code)
|
725 | p.stdin.close()
|
726 |
|
727 | actual = {}
|
728 | actual['stdout'] = p.stdout.read()
|
729 | actual['stderr'] = p.stderr.read()
|
730 | p.stdout.close()
|
731 | p.stderr.close()
|
732 |
|
733 | actual['status'] = p.wait()
|
734 |
|
735 | if opts.timeout_bin and os.path.exists(timeout_file):
|
736 | cell_result = Result.TIMEOUT
|
737 | elif not opts.timeout_bin and actual['status'] == 124:
|
738 | cell_result = Result.TIMEOUT
|
739 | else:
|
740 | messages = []
|
741 | cell_result = Result.PASS
|
742 |
|
743 | # TODO: Warn about no assertions? Well it will always test the error
|
744 | # code.
|
745 | assertions = CreateAssertions(case, sh_label)
|
746 | for a in assertions:
|
747 | result, msg = a.Check(sh_label, actual)
|
748 | # The minimum one wins.
|
749 | # If any failed, then the result is FAIL.
|
750 | # If any are OK, but none are FAIL, the result is OK.
|
751 | cell_result = min(cell_result, result)
|
752 | if msg:
|
753 | messages.append(msg)
|
754 |
|
755 | if cell_result != Result.PASS or opts.details:
|
756 | d = (i, sh_label, actual['stdout'], actual['stderr'], messages)
|
757 | out.AddDetails(d)
|
758 |
|
759 | result_row.append(cell_result)
|
760 |
|
761 | stats.ReportCell(i, cell_result, sh_label)
|
762 |
|
763 | if sh_label in OTHER_OSH:
|
764 | # This is only an error if we tried to run ANY OSH.
|
765 | if osh_cpython_index == -1:
|
766 | raise RuntimeError("Couldn't determine index of osh-cpython")
|
767 |
|
768 | other_result = result_row[shell_index]
|
769 | cpython_result = result_row[osh_cpython_index]
|
770 | if other_result != cpython_result:
|
771 | stats.Inc('oils_ALT_delta')
|
772 |
|
773 | out.WriteRow(i, line_num, result_row, desc)
|
774 |
|
775 | return stats
|
776 |
|
777 |
|
778 | # ANSI color constants
|
779 | _RESET = '\033[0;0m'
|
780 | _BOLD = '\033[1m'
|
781 |
|
782 | _RED = '\033[31m'
|
783 | _GREEN = '\033[32m'
|
784 | _YELLOW = '\033[33m'
|
785 | _PURPLE = '\033[35m'
|
786 |
|
787 |
|
788 | TEXT_CELLS = {
|
789 | Result.TIMEOUT: 'TIME',
|
790 | Result.FAIL: 'FAIL',
|
791 | Result.BUG: 'BUG',
|
792 | Result.NI: 'N-I',
|
793 | Result.OK: 'ok',
|
794 | Result.PASS: 'pass',
|
795 | }
|
796 |
|
797 | ANSI_COLORS = {
|
798 | Result.TIMEOUT: _PURPLE,
|
799 | Result.FAIL: _RED,
|
800 | Result.BUG: _YELLOW,
|
801 | Result.NI: _YELLOW,
|
802 | Result.OK: _YELLOW,
|
803 | Result.PASS: _GREEN,
|
804 | }
|
805 |
|
806 | def _AnsiCells():
|
807 | lookup = {}
|
808 | for i in xrange(Result.length):
|
809 | lookup[i] = ''.join([ANSI_COLORS[i], _BOLD, TEXT_CELLS[i], _RESET])
|
810 | return lookup
|
811 |
|
812 | ANSI_CELLS = _AnsiCells()
|
813 |
|
814 |
|
815 | HTML_CELLS = {
|
816 | Result.TIMEOUT: '<td class="timeout">TIME',
|
817 | Result.FAIL: '<td class="fail">FAIL',
|
818 | Result.BUG: '<td class="bug">BUG',
|
819 | Result.NI: '<td class="n-i">N-I',
|
820 | Result.OK: '<td class="ok">ok',
|
821 | Result.PASS: '<td class="pass">pass',
|
822 | }
|
823 |
|
824 |
|
825 | def _ValidUtf8String(s):
|
826 | """Return an arbitrary string as a readable utf-8 string.
|
827 |
|
828 | We output utf-8 to either HTML or the console. If we get invalid utf-8 as
|
829 | stdout/stderr (which is very possible), then show the ASCII repr().
|
830 | """
|
831 | try:
|
832 | s.decode('utf-8')
|
833 | return s # it decoded OK
|
834 | except UnicodeDecodeError:
|
835 | return repr(s) # ASCII representation
|
836 |
|
837 |
|
838 | class Output(object):
|
839 |
|
840 | def __init__(self, f, verbose):
|
841 | self.f = f
|
842 | self.verbose = verbose
|
843 | self.details = []
|
844 |
|
845 | def BeginCases(self, test_file):
|
846 | pass
|
847 |
|
848 | def WriteHeader(self, sh_labels):
|
849 | pass
|
850 |
|
851 | def WriteRow(self, i, line_num, row, desc):
|
852 | pass
|
853 |
|
854 | def EndCases(self, sh_labels, stats):
|
855 | pass
|
856 |
|
857 | def AddDetails(self, entry):
|
858 | self.details.append(entry)
|
859 |
|
860 | # Helper function
|
861 | def _WriteDetailsAsText(self, details):
|
862 | for case_index, shell, stdout, stderr, messages in details:
|
863 | print('case: %d' % case_index, file=self.f)
|
864 | for m in messages:
|
865 | print(m, file=self.f)
|
866 |
|
867 | # Assume the terminal can show utf-8, but we don't want random binary.
|
868 | print('%s stdout:' % shell, file=self.f)
|
869 | print(_ValidUtf8String(stdout), file=self.f)
|
870 |
|
871 | print('%s stderr:' % shell, file=self.f)
|
872 | print(_ValidUtf8String(stderr), file=self.f)
|
873 |
|
874 | print('', file=self.f)
|
875 |
|
876 |
|
877 | class TeeOutput(object):
|
878 | """For multiple outputs in one run, e.g. HTML and TSV.
|
879 |
|
880 | UNUSED
|
881 | """
|
882 |
|
883 | def __init__(self, outs):
|
884 | self.outs = outs
|
885 |
|
886 | def BeginCases(self, test_file):
|
887 | for out in self.outs:
|
888 | out.BeginCases(test_file)
|
889 |
|
890 | def WriteHeader(self, sh_labels):
|
891 | for out in self.outs:
|
892 | out.WriteHeader(sh_labels)
|
893 |
|
894 | def WriteRow(self, i, line_num, row, desc):
|
895 | for out in self.outs:
|
896 | out.WriteRow(i, line_num, row, desc)
|
897 |
|
898 | def EndCases(self, sh_labels, stats):
|
899 | for out in self.outs:
|
900 | out.EndCases(sh_labels, stats)
|
901 |
|
902 | def AddDetails(self, entry):
|
903 | for out in self.outs:
|
904 | out.AddDetails(entry)
|
905 |
|
906 |
|
907 | class TsvOutput(Output):
|
908 | """Write a plain-text TSV file.
|
909 |
|
910 | UNUSED since we are outputting LONG format with --tsv-output.
|
911 | """
|
912 |
|
913 | def WriteHeader(self, sh_labels):
|
914 | self.f.write('case\tline\t') # case number and line number
|
915 | for sh_label in sh_labels:
|
916 | self.f.write(sh_label)
|
917 | self.f.write('\t')
|
918 | self.f.write('\n')
|
919 |
|
920 | def WriteRow(self, i, line_num, row, desc):
|
921 | self.f.write('%3d\t%3d\t' % (i, line_num))
|
922 |
|
923 | for result in row:
|
924 | c = TEXT_CELLS[result]
|
925 | self.f.write(c)
|
926 | self.f.write('\t')
|
927 |
|
928 | # note: 'desc' could use TSV8, but just ignore it for now
|
929 | #self.f.write(desc)
|
930 | self.f.write('\n')
|
931 |
|
932 |
|
933 | class AnsiOutput(Output):
|
934 |
|
935 | def BeginCases(self, test_file):
|
936 | self.f.write('%s\n' % test_file)
|
937 |
|
938 | def WriteHeader(self, sh_labels):
|
939 | self.f.write(_BOLD)
|
940 | self.f.write('case\tline\t') # case number and line number
|
941 | for sh_label in sh_labels:
|
942 | self.f.write(sh_label)
|
943 | self.f.write('\t')
|
944 | self.f.write(_RESET)
|
945 | self.f.write('\n')
|
946 |
|
947 | def WriteRow(self, i, line_num, row, desc):
|
948 | self.f.write('%3d\t%3d\t' % (i, line_num))
|
949 |
|
950 | for result in row:
|
951 | c = ANSI_CELLS[result]
|
952 | self.f.write(c)
|
953 | self.f.write('\t')
|
954 |
|
955 | self.f.write(desc)
|
956 | self.f.write('\n')
|
957 |
|
958 | if self.verbose:
|
959 | self._WriteDetailsAsText(self.details)
|
960 | self.details = []
|
961 |
|
962 | def _WriteShellSummary(self, sh_labels, stats):
|
963 | if len(stats.nonzero_results) <= 1: # Skip trivial summaries
|
964 | return
|
965 |
|
966 | # Reiterate header
|
967 | self.f.write(_BOLD)
|
968 | self.f.write('\t\t')
|
969 | for sh_label in sh_labels:
|
970 | self.f.write(sh_label)
|
971 | self.f.write('\t')
|
972 | self.f.write(_RESET)
|
973 | self.f.write('\n')
|
974 |
|
975 | # Write totals by cell.
|
976 | for result in sorted(stats.nonzero_results, reverse=True):
|
977 | self.f.write('\t%s' % ANSI_CELLS[result])
|
978 | for sh_label in sh_labels:
|
979 | self.f.write('\t%d' % stats.by_shell[sh_label][result])
|
980 | self.f.write('\n')
|
981 |
|
982 | # The bottom row is all the same, but it helps readability.
|
983 | self.f.write('\ttotal')
|
984 | for sh_label in sh_labels:
|
985 | self.f.write('\t%d' % stats.counters['num_cases_run'])
|
986 | self.f.write('\n')
|
987 |
|
988 | def EndCases(self, sh_labels, stats):
|
989 | print()
|
990 | self._WriteShellSummary(sh_labels, stats)
|
991 |
|
992 |
|
993 | class HtmlOutput(Output):
|
994 |
|
995 | def __init__(self, f, verbose, spec_name, sh_labels, cases):
|
996 | Output.__init__(self, f, verbose)
|
997 | self.spec_name = spec_name
|
998 | self.sh_labels = sh_labels # saved from header
|
999 | self.cases = cases # for linking to code
|
1000 | self.row_html = [] # buffered
|
1001 |
|
1002 | def _SourceLink(self, line_num, desc):
|
1003 | return '<a href="%s.test.html#L%d">%s</a>' % (
|
1004 | self.spec_name, line_num, cgi.escape(desc))
|
1005 |
|
1006 | def BeginCases(self, test_file):
|
1007 | css_urls = [ '../../../web/base.css', '../../../web/spec-tests.css' ]
|
1008 | title = '%s: spec test case results' % self.spec_name
|
1009 | html_head.Write(self.f, title, css_urls=css_urls)
|
1010 |
|
1011 | self.f.write('''\
|
1012 | <body class="width60">
|
1013 | <p id="home-link">
|
1014 | <a href=".">spec test index</a>
|
1015 | /
|
1016 | <a href="/">oilshell.org</a>
|
1017 | </p>
|
1018 | <h1>Results for %s</h1>
|
1019 | <table>
|
1020 | ''' % test_file)
|
1021 |
|
1022 | def _WriteShellSummary(self, sh_labels, stats):
|
1023 | # NOTE: This table has multiple <thead>, which seems OK.
|
1024 | self.f.write('''
|
1025 | <thead>
|
1026 | <tr class="table-header">
|
1027 | ''')
|
1028 |
|
1029 | columns = ['status'] + sh_labels + ['']
|
1030 | for c in columns:
|
1031 | self.f.write('<td>%s</td>' % c)
|
1032 |
|
1033 | self.f.write('''
|
1034 | </tr>
|
1035 | </thead>
|
1036 | ''')
|
1037 |
|
1038 | # Write totals by cell.
|
1039 | for result in sorted(stats.nonzero_results, reverse=True):
|
1040 | self.f.write('<tr>')
|
1041 |
|
1042 | self.f.write(HTML_CELLS[result])
|
1043 | self.f.write('</td> ')
|
1044 |
|
1045 | for sh_label in sh_labels:
|
1046 | self.f.write('<td>%d</td>' % stats.by_shell[sh_label][result])
|
1047 |
|
1048 | self.f.write('<td></td>')
|
1049 | self.f.write('</tr>\n')
|
1050 |
|
1051 | # The bottom row is all the same, but it helps readability.
|
1052 | self.f.write('<tr>')
|
1053 | self.f.write('<td>total</td>')
|
1054 | for sh_label in sh_labels:
|
1055 | self.f.write('<td>%d</td>' % stats.counters['num_cases_run'])
|
1056 | self.f.write('<td></td>')
|
1057 | self.f.write('</tr>\n')
|
1058 |
|
1059 | # Blank row for space.
|
1060 | self.f.write('<tr>')
|
1061 | for i in xrange(len(sh_labels) + 2):
|
1062 | self.f.write('<td style="height: 2em"></td>')
|
1063 | self.f.write('</tr>\n')
|
1064 |
|
1065 | def WriteHeader(self, sh_labels):
|
1066 | f = cStringIO.StringIO()
|
1067 |
|
1068 | f.write('''
|
1069 | <thead>
|
1070 | <tr class="table-header">
|
1071 | ''')
|
1072 |
|
1073 | columns = ['case'] + sh_labels
|
1074 | for c in columns:
|
1075 | f.write('<td>%s</td>' % c)
|
1076 | f.write('<td class="case-desc">description</td>')
|
1077 |
|
1078 | f.write('''
|
1079 | </tr>
|
1080 | </thead>
|
1081 | ''')
|
1082 |
|
1083 | self.row_html.append(f.getvalue())
|
1084 |
|
1085 | def WriteRow(self, i, line_num, row, desc):
|
1086 | f = cStringIO.StringIO()
|
1087 | f.write('<tr>')
|
1088 | f.write('<td>%3d</td>' % i)
|
1089 |
|
1090 | show_details = False
|
1091 |
|
1092 | for result in row:
|
1093 | c = HTML_CELLS[result]
|
1094 | if result not in (Result.PASS, Result.TIMEOUT): # nothing to show
|
1095 | show_details = True
|
1096 |
|
1097 | f.write(c)
|
1098 | f.write('</td>')
|
1099 | f.write('\t')
|
1100 |
|
1101 | f.write('<td class="case-desc">')
|
1102 | f.write(self._SourceLink(line_num, desc))
|
1103 | f.write('</td>')
|
1104 | f.write('</tr>\n')
|
1105 |
|
1106 | # Show row with details link.
|
1107 | if show_details:
|
1108 | f.write('<tr>')
|
1109 | f.write('<td class="details-row"></td>') # for the number
|
1110 |
|
1111 | for col_index, result in enumerate(row):
|
1112 | f.write('<td class="details-row">')
|
1113 | if result != Result.PASS:
|
1114 | sh_label = self.sh_labels[col_index]
|
1115 | f.write('<a href="#details-%s-%s">details</a>' % (i, sh_label))
|
1116 | f.write('</td>')
|
1117 |
|
1118 | f.write('<td class="details-row"></td>') # for the description
|
1119 | f.write('</tr>\n')
|
1120 |
|
1121 | self.row_html.append(f.getvalue()) # buffer it
|
1122 |
|
1123 | def _WriteStats(self, stats):
|
1124 | self.f.write(
|
1125 | '%(num_passed)d passed, %(num_ok)d OK, '
|
1126 | '%(num_ni)d not implemented, %(num_bug)d BUG, '
|
1127 | '%(num_failed)d failed, %(num_timeout)d timeouts, '
|
1128 | '%(num_skipped)d cases skipped\n' % stats.counters)
|
1129 |
|
1130 | def EndCases(self, sh_labels, stats):
|
1131 | self._WriteShellSummary(sh_labels, stats)
|
1132 |
|
1133 | # Write all the buffered rows
|
1134 | for h in self.row_html:
|
1135 | self.f.write(h)
|
1136 |
|
1137 | self.f.write('</table>\n')
|
1138 | self.f.write('<pre>')
|
1139 | self._WriteStats(stats)
|
1140 | if stats.Get('oils_num_failed'):
|
1141 | self.f.write('%(oils_num_failed)d failed under osh\n' % stats.counters)
|
1142 | self.f.write('</pre>')
|
1143 |
|
1144 | if self.details:
|
1145 | self._WriteDetails()
|
1146 |
|
1147 | self.f.write('</body></html>')
|
1148 |
|
1149 | def _WriteDetails(self):
|
1150 | self.f.write("<h2>Details on runs that didn't PASS</h2>")
|
1151 | self.f.write('<table id="details">')
|
1152 |
|
1153 | for case_index, sh_label, stdout, stderr, messages in self.details:
|
1154 | self.f.write('<tr>')
|
1155 | self.f.write('<td><a name="details-%s-%s"></a><b>%s</b></td>' % (
|
1156 | case_index, sh_label, sh_label))
|
1157 |
|
1158 | self.f.write('<td>')
|
1159 |
|
1160 | # Write description and link to the code
|
1161 | case = self.cases[case_index]
|
1162 | line_num = case['line_num']
|
1163 | desc = case['desc']
|
1164 | self.f.write('%d ' % case_index)
|
1165 | self.f.write(self._SourceLink(line_num, desc))
|
1166 | self.f.write('<br/><br/>\n')
|
1167 |
|
1168 | for m in messages:
|
1169 | self.f.write('<span class="assertion">%s</span><br/>\n' % cgi.escape(m))
|
1170 | if messages:
|
1171 | self.f.write('<br/>\n')
|
1172 |
|
1173 | def _WriteRaw(s):
|
1174 | self.f.write('<pre>')
|
1175 |
|
1176 | # stdout might contain invalid utf-8; make it valid;
|
1177 | valid_utf8 = _ValidUtf8String(s)
|
1178 |
|
1179 | self.f.write(cgi.escape(valid_utf8))
|
1180 | self.f.write('</pre>')
|
1181 |
|
1182 | self.f.write('<i>stdout:</i> <br/>\n')
|
1183 | _WriteRaw(stdout)
|
1184 |
|
1185 | self.f.write('<i>stderr:</i> <br/>\n')
|
1186 | _WriteRaw(stderr)
|
1187 |
|
1188 | self.f.write('</td>')
|
1189 | self.f.write('</tr>')
|
1190 |
|
1191 | self.f.write('</table>')
|
1192 |
|
1193 |
|
1194 | def MakeTestEnv(opts):
|
1195 | if not opts.tmp_env:
|
1196 | raise RuntimeError('--tmp-env required')
|
1197 | if not opts.path_env:
|
1198 | raise RuntimeError('--path-env required')
|
1199 | env = {
|
1200 | 'PATH': opts.path_env,
|
1201 | #'LANG': opts.lang_env,
|
1202 | }
|
1203 | for p in opts.env_pair:
|
1204 | name, value = p.split('=', 1)
|
1205 | env[name] = value
|
1206 |
|
1207 | return env
|
1208 |
|
1209 |
|
1210 | def _DefaultSuite(spec_name):
|
1211 | if spec_name.startswith('ysh-'):
|
1212 | suite = 'ysh'
|
1213 | elif spec_name.startswith('hay'): # hay.test.sh is ysh
|
1214 | suite = 'ysh'
|
1215 |
|
1216 | elif spec_name.startswith('tea-'):
|
1217 | suite = 'tea'
|
1218 | else:
|
1219 | suite = 'osh'
|
1220 |
|
1221 | return suite
|
1222 |
|
1223 |
|
1224 | def ParseTestList(test_files):
|
1225 | for test_file in test_files:
|
1226 | with open(test_file) as f:
|
1227 | tokens = Tokenizer(f)
|
1228 | try:
|
1229 | file_metadata, cases = ParseTestFile(test_file, tokens)
|
1230 | except RuntimeError as e:
|
1231 | log('ERROR in %r', test_file)
|
1232 | raise
|
1233 |
|
1234 | tmp = os.path.basename(test_file)
|
1235 | spec_name = tmp.split('.')[0] # foo.test.sh -> foo
|
1236 |
|
1237 | suite = file_metadata.get('suite') or _DefaultSuite(spec_name)
|
1238 |
|
1239 | tmp = file_metadata.get('tags')
|
1240 | tags = tmp.split() if tmp else []
|
1241 |
|
1242 | # Don't need compare_shells, etc. to decide what to run
|
1243 |
|
1244 | row = {'spec_name': spec_name, 'suite': suite, 'tags': tags}
|
1245 | #print(row)
|
1246 | yield row
|
1247 |
|
1248 |
|
1249 | def main(argv):
|
1250 | # First check if bash is polluting the environment. Tests rely on the
|
1251 | # environment.
|
1252 | v = os.getenv('RANDOM')
|
1253 | if v is not None:
|
1254 | raise AssertionError('got $RANDOM = %s' % v)
|
1255 | v = os.getenv('PPID')
|
1256 | if v is not None:
|
1257 | raise AssertionError('got $PPID = %s' % v)
|
1258 |
|
1259 | p = optparse.OptionParser('%s [options] TEST_FILE shell...' % sys.argv[0])
|
1260 | spec_lib.DefineCommon(p)
|
1261 | spec_lib.DefineShSpec(p)
|
1262 | opts, argv = p.parse_args(argv)
|
1263 |
|
1264 | # --print-tagged to figure out what to run
|
1265 | if opts.print_tagged:
|
1266 | to_find = opts.print_tagged
|
1267 | for row in ParseTestList(argv[1:]):
|
1268 | if to_find in row['tags']:
|
1269 | print(row['spec_name'])
|
1270 | return 0
|
1271 |
|
1272 | # --print-table to figure out what to run
|
1273 | if opts.print_table:
|
1274 | for row in ParseTestList(argv[1:]):
|
1275 | print('%(suite)s\t%(spec_name)s' % row)
|
1276 | #print(row)
|
1277 | return 0
|
1278 |
|
1279 | #
|
1280 | # Now deal with a single file
|
1281 | #
|
1282 |
|
1283 | try:
|
1284 | test_file = argv[1]
|
1285 | except IndexError:
|
1286 | p.print_usage()
|
1287 | return 1
|
1288 |
|
1289 | with open(test_file) as f:
|
1290 | tokens = Tokenizer(f)
|
1291 | file_metadata, cases = ParseTestFile(test_file, tokens)
|
1292 |
|
1293 | # List test cases and return
|
1294 | if opts.do_list:
|
1295 | for i, case in enumerate(cases):
|
1296 | if opts.verbose: # print the raw dictionary for debugging
|
1297 | print(pprint.pformat(case))
|
1298 | else:
|
1299 | print('%d\t%s' % (i, case['desc']))
|
1300 | return 0
|
1301 |
|
1302 | # for test/spec-cpp.sh
|
1303 | if opts.print_spec_suite:
|
1304 | tmp = os.path.basename(test_file)
|
1305 | spec_name = tmp.split('.')[0] # foo.test.sh -> foo
|
1306 |
|
1307 | suite = file_metadata.get('suite') or _DefaultSuite(spec_name)
|
1308 | print(suite)
|
1309 | return 0
|
1310 |
|
1311 | if opts.verbose:
|
1312 | for k, v in file_metadata.items():
|
1313 | print('\t%-20s: %s' % (k, v), file=sys.stderr)
|
1314 | print('', file=sys.stderr)
|
1315 |
|
1316 | if opts.oils_bin_dir:
|
1317 |
|
1318 | shells = []
|
1319 |
|
1320 | if opts.compare_shells:
|
1321 | comp = file_metadata.get('compare_shells')
|
1322 | # Compare 'compare_shells' and Python
|
1323 | shells.extend(comp.split() if comp else [])
|
1324 |
|
1325 | # Always run with the Python version
|
1326 | our_shell = file_metadata.get('our_shell', 'osh') # default is OSH
|
1327 | shells.append(os.path.join(opts.oils_bin_dir, our_shell))
|
1328 |
|
1329 | # Legacy OVM/CPython build
|
1330 | if opts.ovm_bin_dir:
|
1331 | shells.append(os.path.join(opts.ovm_bin_dir, our_shell))
|
1332 |
|
1333 | # New C++ build
|
1334 | if opts.oils_cpp_bin_dir:
|
1335 | shells.append(os.path.join(opts.oils_cpp_bin_dir, our_shell))
|
1336 |
|
1337 | # Overwrite it when --oils-bin-dir is set
|
1338 | # It's no longer a flag
|
1339 | opts.oils_failures_allowed = \
|
1340 | int(file_metadata.get('oils_failures_allowed', 0))
|
1341 |
|
1342 | else:
|
1343 | # TODO: remove this mode?
|
1344 | shells = argv[2:]
|
1345 |
|
1346 | shell_pairs = spec_lib.MakeShellPairs(shells)
|
1347 |
|
1348 | if opts.range:
|
1349 | begin, end = spec_lib.ParseRange(opts.range)
|
1350 | case_predicate = spec_lib.RangePredicate(begin, end)
|
1351 | elif opts.regex:
|
1352 | desc_re = re.compile(opts.regex, re.IGNORECASE)
|
1353 | case_predicate = spec_lib.RegexPredicate(desc_re)
|
1354 | else:
|
1355 | case_predicate = lambda i, case: True
|
1356 |
|
1357 | out_f = sys.stderr if opts.do_print else sys.stdout
|
1358 |
|
1359 | # Set up output style. Also see asdl/format.py
|
1360 | if opts.format == 'ansi':
|
1361 | out = AnsiOutput(out_f, opts.verbose)
|
1362 |
|
1363 | elif opts.format == 'html':
|
1364 | spec_name = os.path.basename(test_file)
|
1365 | spec_name = spec_name.split('.')[0]
|
1366 |
|
1367 | sh_labels = [label for label, _ in shell_pairs]
|
1368 |
|
1369 | out = HtmlOutput(out_f, opts.verbose, spec_name, sh_labels, cases)
|
1370 |
|
1371 | else:
|
1372 | raise AssertionError()
|
1373 |
|
1374 | out.BeginCases(os.path.basename(test_file))
|
1375 |
|
1376 | env = MakeTestEnv(opts)
|
1377 | stats = RunCases(cases, case_predicate, shell_pairs, env, out, opts)
|
1378 |
|
1379 | out.EndCases([sh_label for sh_label, _ in shell_pairs], stats)
|
1380 |
|
1381 | if opts.tsv_output:
|
1382 | with open(opts.tsv_output, 'w') as f:
|
1383 | stats.WriteTsv(f)
|
1384 |
|
1385 | # TODO: Could --stats-{file,template} be a separate awk step on .tsv files?
|
1386 | stats.Set('oils_failures_allowed', opts.oils_failures_allowed)
|
1387 | if opts.stats_file:
|
1388 | with open(opts.stats_file, 'w') as f:
|
1389 | f.write(opts.stats_template % stats.counters)
|
1390 | f.write('\n') # bash 'read' requires a newline
|
1391 |
|
1392 | if stats.Get('num_failed') == 0:
|
1393 | return 0
|
1394 |
|
1395 | # spec/smoke.test.sh -> smoke
|
1396 | test_name = os.path.basename(test_file).split('.')[0]
|
1397 |
|
1398 | allowed = opts.oils_failures_allowed
|
1399 | all_count = stats.Get('num_failed')
|
1400 | oils_count = stats.Get('oils_num_failed')
|
1401 | if allowed == 0:
|
1402 | log('')
|
1403 | log('%s: FATAL: %d tests failed (%d oils failures)', test_name, all_count,
|
1404 | oils_count)
|
1405 | log('')
|
1406 | else:
|
1407 | # If we got EXACTLY the allowed number of failures, exit 0.
|
1408 | if allowed == all_count and all_count == oils_count:
|
1409 | log('%s: note: Got %d allowed oils failures (exit with code 0)',
|
1410 | test_name, allowed)
|
1411 | return 0
|
1412 | else:
|
1413 | log('')
|
1414 | log('%s: FATAL: Got %d failures (%d oils failures), but %d are allowed',
|
1415 | test_name, all_count, oils_count, allowed)
|
1416 | log('')
|
1417 |
|
1418 | return 1
|
1419 |
|
1420 |
|
1421 | if __name__ == '__main__':
|
1422 | try:
|
1423 | sys.exit(main(sys.argv))
|
1424 | except KeyboardInterrupt as e:
|
1425 | print('%s: interrupted with Ctrl-C' % sys.argv[0], file=sys.stderr)
|
1426 | sys.exit(1)
|
1427 | except RuntimeError as e:
|
1428 | print('FATAL: %s' % e, file=sys.stderr)
|
1429 | sys.exit(1)
|