OILS / test / sh_spec.py View on Github | oilshell.org

1429 lines, 880 significant
1#!/usr/bin/env python2
2from __future__ import print_function
3"""
4sh_spec.py -- Test framework to compare shells.
5
6Assertion help:
7 stdout: A single line of expected stdout. Newline is implicit.
8 stdout-json: JSON-encoded string. Use for the empty string (no newline),
9 for unicode chars, etc.
10
11 stderr: Ditto for stderr stream.
12 status: Expected shell return code. If not specified, the case must exit 0.
13
14Results:
15 PASS - we got the ideal, expected value
16 OK - we got a value that was not ideal, but expected
17 For OSH this is behavior that was defined to be different?
18 N-I - Not implemented (e.g. $''). Assertions still checked (in case it
19 starts working)
20 BUG - we verified the value of a known bug
21 FAIL - we got an unexpected value. If the implementation can't be changed,
22 it should be converted to BUG or OK. Otherwise it should be made to
23 PASS.
24
25NOTE: The difference between OK and BUG is a matter of judgement. If the ideal
26behavior is a compile time error (code 2), a runtime error is generally OK.
27
28If ALL shells agree on a broken behavior, they are all marked OK (but our
29implementation will be PASS.) But if the behavior is NOT POSIX compliant, then
30it will be a BUG.
31
32If one shell disagrees with others, that is generally a BUG.
33
34Example test case:
35
36#### hello and fail
37echo hello
38echo world
39exit 1
40## status: 1
41#
42# ignored comment
43#
44## STDOUT:
45hello
46world
47## END
48
49"""
50
51import collections
52import cgi
53import cStringIO
54import errno
55import json
56import optparse
57import os
58import pprint
59import re
60import shutil
61import subprocess
62import sys
63
64from test import spec_lib
65from doctools import html_head
66
67log = spec_lib.log
68
69
70# Magic strings for other variants of OSH.
71
72# NOTE: osh_ALT is usually _bin/osh -- the release binary.
73# It would be better to rename these osh-cpython and osh-ovm. Have the concept
74# of a suffix?
75
76OSH_CPYTHON = ('osh', 'osh-dbg')
77OTHER_OSH = ('osh_ALT',)
78
79YSH_CPYTHON = ('ysh', 'ysh-dbg')
80OTHER_YSH = ('oil_ALT',)
81
82
83class ParseError(Exception):
84 pass
85
86
87# EXAMPLES:
88## stdout: foo
89## stdout-json: ""
90#
91# In other words, it could be (name, value) or (qualifier, name, value)
92
93KEY_VALUE_RE = re.compile(r'''
94 [#][#] \s+
95 # optional prefix with qualifier and shells
96 (?: (OK|BUG|N-I) \s+ ([\w+/]+) \s+ )?
97 ([\w\-]+) # key
98 :
99 \s* (.*) # value
100''', re.VERBOSE)
101
102END_MULTILINE_RE = re.compile(r'''
103 [#][#] \s+ END
104''', re.VERBOSE)
105
106# Line types
107TEST_CASE_BEGIN = 0 # Starts with ####
108KEY_VALUE = 1 # Metadata
109KEY_VALUE_MULTILINE = 2 # STDOUT STDERR
110END_MULTILINE = 3 # STDOUT STDERR
111PLAIN_LINE = 4 # Uncommented
112EOF = 5
113
114LEX_OUTER = 0 # Ignore blank lines, e.g. for separating cases
115LEX_RAW = 1 # Blank lines are significant
116
117
118class Tokenizer(object):
119 """Modal lexer!"""
120
121 def __init__(self, f):
122 self.f = f
123
124 self.cursor = None
125 self.line_num = 0
126
127 self.next()
128
129 def _ClassifyLine(self, line, lex_mode):
130 if not line: # empty
131 return self.line_num, EOF, ''
132
133 if lex_mode == LEX_OUTER and not line.strip():
134 return None
135
136 if line.startswith('####'):
137 desc = line[4:].strip()
138 return self.line_num, TEST_CASE_BEGIN, desc
139
140 m = KEY_VALUE_RE.match(line)
141 if m:
142 qualifier, shells, name, value = m.groups()
143 # HACK: Expected data should have the newline.
144 if name in ('stdout', 'stderr'):
145 value += '\n'
146
147 if name in ('STDOUT', 'STDERR'):
148 token_type = KEY_VALUE_MULTILINE
149 else:
150 token_type = KEY_VALUE
151 return self.line_num, token_type, (qualifier, shells, name, value)
152
153 m = END_MULTILINE_RE.match(line)
154 if m:
155 return self.line_num, END_MULTILINE, None
156
157 # If it starts with ##, it should be metadata. This finds some typos.
158 if line.lstrip().startswith('##'):
159 raise RuntimeError('Invalid ## line %r' % line)
160
161 if line.lstrip().startswith('#'): # Ignore comments
162 return None # try again
163
164 # Non-empty line that doesn't start with '#'
165 # NOTE: We need the original line to test the whitespace sensitive <<-.
166 # And we need rstrip because we add newlines back below.
167 return self.line_num, PLAIN_LINE, line
168
169 def next(self, lex_mode=LEX_OUTER):
170 """Raises StopIteration when exhausted."""
171 while True:
172 line = self.f.readline()
173 self.line_num += 1
174
175 tok = self._ClassifyLine(line, lex_mode)
176 if tok is not None:
177 break
178
179 self.cursor = tok
180 return self.cursor
181
182 def peek(self):
183 return self.cursor
184
185
186def AddMetadataToCase(case, qualifier, shells, name, value):
187 shells = shells.split('/') # bash/dash/mksh
188 for shell in shells:
189 if shell not in case:
190 case[shell] = {}
191 case[shell][name] = value
192 case[shell]['qualifier'] = qualifier
193
194
195# Format of a test script.
196#
197# -- Code is either literal lines, or a commented out code: value.
198# code = PLAIN_LINE*
199# | '## code:' VALUE
200#
201# -- Key value pairs can be single- or multi-line
202# key_value = '##' KEY ':' VALUE
203# | KEY_VALUE_MULTILINE PLAIN_LINE* END_MULTILINE
204#
205# -- Description, then key-value pairs surrounding code.
206# test_case = '####' DESC
207# key_value*
208# code
209# key_value*
210#
211# -- Should be a blank line after each test case. Leading comments and code
212# -- are OK.
213#
214# test_file =
215# key_value* -- file level metadata
216# (test_case '\n')*
217
218
219def ParseKeyValue(tokens, case):
220 """Parse commented-out metadata in a test case.
221
222 The metadata must be contiguous.
223
224 Args:
225 tokens: Tokenizer
226 case: dictionary to add to
227 """
228 while True:
229 line_num, kind, item = tokens.peek()
230
231 if kind == KEY_VALUE_MULTILINE:
232 qualifier, shells, name, empty_value = item
233 if empty_value:
234 raise ParseError(
235 'Line %d: got value %r for %r, but the value should be on the '
236 'following lines' % (line_num, empty_value, name))
237
238 value_lines = []
239 while True:
240 tokens.next(lex_mode=LEX_RAW) # empty lines aren't skipped
241 _, kind2, item2 = tokens.peek()
242 if kind2 != PLAIN_LINE:
243 break
244 value_lines.append(item2)
245
246 value = ''.join(value_lines)
247
248 name = name.lower() # STDOUT -> stdout
249 if qualifier:
250 AddMetadataToCase(case, qualifier, shells, name, value)
251 else:
252 case[name] = value
253
254 # END token is optional.
255 if kind2 == END_MULTILINE:
256 tokens.next()
257
258 elif kind == KEY_VALUE:
259 qualifier, shells, name, value = item
260
261 if qualifier:
262 AddMetadataToCase(case, qualifier, shells, name, value)
263 else:
264 case[name] = value
265
266 tokens.next()
267
268 else: # Unknown token type
269 break
270
271
272def ParseCodeLines(tokens, case):
273 """Parse uncommented code in a test case."""
274 _, kind, item = tokens.peek()
275 if kind != PLAIN_LINE:
276 raise ParseError('Expected a line of code (got %r, %r)' % (kind, item))
277 code_lines = []
278 while True:
279 _, kind, item = tokens.peek()
280 if kind != PLAIN_LINE:
281 case['code'] = ''.join(code_lines)
282 return
283 code_lines.append(item)
284 tokens.next(lex_mode=LEX_RAW)
285
286
287def ParseTestCase(tokens):
288 """Parse a single test case and return it.
289
290 If at EOF, return None.
291 """
292 line_num, kind, item = tokens.peek()
293 if kind == EOF:
294 return None
295
296 if kind != TEST_CASE_BEGIN:
297 raise RuntimeError(
298 "line %d: Expected TEST_CASE_BEGIN, got %r" % (line_num, [kind, item]))
299
300 tokens.next()
301
302 case = {'desc': item, 'line_num': line_num}
303
304 ParseKeyValue(tokens, case)
305
306 # For broken code
307 if 'code' in case: # Got it through a key value pair
308 return case
309
310 ParseCodeLines(tokens, case)
311 ParseKeyValue(tokens, case)
312
313 return case
314
315
316_META_FIELDS = [
317 'our_shell',
318 'compare_shells',
319 'suite',
320 'tags',
321 'oils_failures_allowed',
322 ]
323
324
325def ParseTestFile(test_file, tokens):
326 """
327 test_file: Only for error message
328 """
329 file_metadata = {}
330 test_cases = []
331
332 try:
333 # Skip over the header. Setup code can go here, although would we have to
334 # execute it on every case?
335 while True:
336 line_num, kind, item = tokens.peek()
337 if kind != KEY_VALUE:
338 break
339
340 qualifier, shells, name, value = item
341 if qualifier is not None:
342 raise RuntimeError('Invalid qualifier in spec file metadata')
343 if shells is not None:
344 raise RuntimeError('Invalid shells in spec file metadata')
345
346 file_metadata[name] = value
347
348 tokens.next()
349
350 while True: # Loop over cases
351 test_case = ParseTestCase(tokens)
352 if test_case is None:
353 break
354 test_cases.append(test_case)
355
356 except StopIteration:
357 raise RuntimeError('Unexpected EOF parsing test cases')
358
359 for name in file_metadata:
360 if name not in _META_FIELDS:
361 raise RuntimeError('Invalid file metadata %r in %r' % (name, test_file))
362
363 return file_metadata, test_cases
364
365
366def CreateStringAssertion(d, key, assertions, qualifier=False):
367 found = False
368
369 exp = d.get(key)
370 if exp is not None:
371 a = EqualAssertion(key, exp, qualifier=qualifier)
372 assertions.append(a)
373 found = True
374
375 exp_json = d.get(key + '-json')
376 if exp_json is not None:
377 exp = json.loads(exp_json, encoding='utf-8')
378 a = EqualAssertion(key, exp, qualifier=qualifier)
379 assertions.append(a)
380 found = True
381
382 # For testing invalid unicode
383 exp_repr = d.get(key + '-repr')
384 if exp_repr is not None:
385 exp = eval(exp_repr)
386 a = EqualAssertion(key, exp, qualifier=qualifier)
387 assertions.append(a)
388 found = True
389
390 return found
391
392
393def CreateIntAssertion(d, key, assertions, qualifier=False):
394 exp = d.get(key) # expected
395 if exp is not None:
396 # For now, turn it into int
397 a = EqualAssertion(key, int(exp), qualifier=qualifier)
398 assertions.append(a)
399 return True
400 return False
401
402
403def CreateAssertions(case, sh_label):
404 """
405 Given a raw test case and a shell label, create EqualAssertion instances to
406 run.
407 """
408 assertions = []
409
410 # Whether we found assertions
411 stdout = False
412 stderr = False
413 status = False
414
415 # So the assertion are exactly the same for osh and osh_ALT
416
417 if sh_label.startswith('osh'):
418 case_sh = 'osh'
419 elif sh_label.startswith('bash'):
420 case_sh = 'bash'
421 else:
422 case_sh = sh_label
423
424 if case_sh in case:
425 q = case[case_sh]['qualifier']
426 if CreateStringAssertion(case[case_sh], 'stdout', assertions, qualifier=q):
427 stdout = True
428 if CreateStringAssertion(case[case_sh], 'stderr', assertions, qualifier=q):
429 stderr = True
430 if CreateIntAssertion(case[case_sh], 'status', assertions, qualifier=q):
431 status = True
432
433 if not stdout:
434 CreateStringAssertion(case, 'stdout', assertions)
435 if not stderr:
436 CreateStringAssertion(case, 'stderr', assertions)
437 if not status:
438 if 'status' in case:
439 CreateIntAssertion(case, 'status', assertions)
440 else:
441 # If the user didn't specify a 'status' assertion, assert that the exit
442 # code is 0.
443 a = EqualAssertion('status', 0)
444 assertions.append(a)
445
446 no_traceback = SubstringAssertion('stderr', 'Traceback (most recent')
447 assertions.append(no_traceback)
448
449 #print 'SHELL', shell
450 #pprint.pprint(case)
451 #print(assertions)
452 return assertions
453
454
455class Result(object):
456 """Result of an stdout/stderr/status assertion or of a (case, shell) cell.
457
458 Order is important: the result of a cell is the minimum of the results of
459 each assertion.
460 """
461 TIMEOUT = 0 # ONLY a cell result, not an assertion result
462 FAIL = 1
463 BUG = 2
464 NI = 3
465 OK = 4
466 PASS = 5
467
468 length = 6 # for loops
469
470
471class EqualAssertion(object):
472 """Check that two values are equal."""
473
474 def __init__(self, key, expected, qualifier=None):
475 self.key = key
476 self.expected = expected # expected value
477 self.qualifier = qualifier # whether this was a special case?
478
479 def __repr__(self):
480 return '<EqualAssertion %s == %r>' % (self.key, self.expected)
481
482 def Check(self, shell, record):
483 actual = record[self.key]
484 if actual != self.expected:
485 if len(str(self.expected)) < 40:
486 msg = '[%s %s] Expected %r, got %r' % (shell, self.key, self.expected,
487 actual)
488 else:
489 msg = '''
490[%s %s]
491Expected %r
492Got %r
493''' % (shell, self.key, self.expected, actual)
494
495 # TODO: Make this better and add a flag for it.
496 if 0:
497 import difflib
498 for line in difflib.unified_diff(
499 self.expected, actual, fromfile='expected', tofile='actual'):
500 print(repr(line))
501
502 return Result.FAIL, msg
503 if self.qualifier == 'BUG': # equal, but known bad
504 return Result.BUG, ''
505 if self.qualifier == 'N-I': # equal, and known UNIMPLEMENTED
506 return Result.NI, ''
507 if self.qualifier == 'OK': # equal, but ok (not ideal)
508 return Result.OK, ''
509 return Result.PASS, '' # ideal behavior
510
511
512class SubstringAssertion(object):
513 """Check that a string like stderr doesn't have a substring."""
514
515 def __init__(self, key, substring):
516 self.key = key
517 self.substring = substring
518
519 def __repr__(self):
520 return '<SubstringAssertion %s == %r>' % (self.key, self.substring)
521
522 def Check(self, shell, record):
523 actual = record[self.key]
524 if self.substring in actual:
525 msg = '[%s %s] Found %r' % (shell, self.key, self.substring)
526 return Result.FAIL, msg
527 return Result.PASS, ''
528
529
530class Stats(object):
531 def __init__(self, num_cases, sh_labels):
532 self.counters = collections.defaultdict(int)
533 c = self.counters
534 c['num_cases'] = num_cases
535 c['oils_num_passed'] = 0
536 c['oils_num_failed'] = 0
537 # Number of osh_ALT results that differed from osh.
538 c['oils_ALT_delta'] = 0
539
540 self.by_shell = {}
541 for sh in sh_labels:
542 self.by_shell[sh] = collections.defaultdict(int)
543 self.nonzero_results = collections.defaultdict(int)
544
545 self.tsv_rows = []
546
547 def Inc(self, counter_name):
548 self.counters[counter_name] += 1
549
550 def Get(self, counter_name):
551 return self.counters[counter_name]
552
553 def Set(self, counter_name, val):
554 self.counters[counter_name] = val
555
556 def ReportCell(self, case_num, cell_result, sh_label):
557 self.tsv_rows.append((str(case_num), sh_label, TEXT_CELLS[cell_result]))
558
559 self.by_shell[sh_label][cell_result] += 1
560 self.nonzero_results[cell_result] += 1
561
562 c = self.counters
563 if cell_result == Result.TIMEOUT:
564 c['num_timeout'] += 1
565 elif cell_result == Result.FAIL:
566 # Special logic: don't count osh_ALT because its failures will be
567 # counted in the delta.
568 if sh_label not in OTHER_OSH + OTHER_YSH:
569 c['num_failed'] += 1
570
571 if sh_label in OSH_CPYTHON + YSH_CPYTHON:
572 c['oils_num_failed'] += 1
573 elif cell_result == Result.BUG:
574 c['num_bug'] += 1
575 elif cell_result == Result.NI:
576 c['num_ni'] += 1
577 elif cell_result == Result.OK:
578 c['num_ok'] += 1
579 elif cell_result == Result.PASS:
580 c['num_passed'] += 1
581 if sh_label in OSH_CPYTHON + YSH_CPYTHON:
582 c['oils_num_passed'] += 1
583 else:
584 raise AssertionError()
585
586 def WriteTsv(self, f):
587 f.write('case\tshell\tresult\n')
588 for row in self.tsv_rows:
589 f.write('\t'.join(row))
590 f.write('\n')
591
592
593PIPE = subprocess.PIPE
594
595def RunCases(cases, case_predicate, shells, env, out, opts):
596 """
597 Run a list of test 'cases' for all 'shells' and write output to 'out'.
598 """
599 if opts.trace:
600 for _, sh in shells:
601 log('\tshell: %s', sh)
602 print('\twhich $SH: ', end='', file=sys.stderr)
603 subprocess.call(['which', sh])
604
605 #pprint.pprint(cases)
606
607 sh_labels = [sh_label for sh_label, _ in shells]
608
609 out.WriteHeader(sh_labels)
610 stats = Stats(len(cases), sh_labels)
611
612 # Make an environment for each shell. $SH is the path to the shell, so we
613 # can test flags, etc.
614 sh_env = []
615 for _, sh_path in shells:
616 e = dict(env)
617 e[opts.sh_env_var_name] = sh_path
618 sh_env.append(e)
619
620 # Determine which one (if any) is osh-cpython, for comparison against other
621 # shells.
622 osh_cpython_index = -1
623 for i, (sh_label, _) in enumerate(shells):
624 if sh_label in OSH_CPYTHON:
625 osh_cpython_index = i
626 break
627
628 timeout_dir = os.path.abspath('_tmp/spec/timeouts')
629 try:
630 shutil.rmtree(timeout_dir)
631 os.mkdir(timeout_dir)
632 except OSError:
633 pass
634
635 # Now run each case, and print a table.
636 for i, case in enumerate(cases):
637 line_num = case['line_num']
638 desc = case['desc']
639 code = case['code']
640
641 if opts.trace:
642 log('case %d: %s', i, desc)
643
644 if not case_predicate(i, case):
645 stats.Inc('num_skipped')
646 continue
647
648 if opts.do_print:
649 print('#### %s' % case['desc'])
650 print(case['code'])
651 print()
652 continue
653
654 stats.Inc('num_cases_run')
655
656 result_row = []
657
658 for shell_index, (sh_label, sh_path) in enumerate(shells):
659 timeout_file = os.path.join(timeout_dir, '%02d-%s' % (i, sh_label))
660 if opts.timeout:
661 if opts.timeout_bin:
662 # This is what smoosh itself uses. See smoosh/tests/shell_tests.sh
663 # QUIRK: interval can only be a whole number
664 argv = [
665 opts.timeout_bin,
666 '-t', opts.timeout,
667 # Somehow I'm not able to get this timeout file working? I think
668 # it has a bug when using stdin. It waits for the background
669 # process too.
670
671 #'-i', '1',
672 #'-l', timeout_file
673 ]
674 else:
675 # This kills hanging tests properly, but somehow they fail with code
676 # -9?
677 #argv = ['timeout', '-s', 'KILL', opts.timeout]
678
679 # s suffix for seconds
680 argv = ['timeout', opts.timeout + 's']
681 else:
682 argv = []
683 argv.append(sh_path)
684
685 # dash doesn't support -o posix
686 if opts.posix and sh_label != 'dash':
687 argv.extend(['-o', 'posix'])
688
689 if opts.trace:
690 log('\targv: %s', ' '.join(argv))
691
692 case_env = sh_env[shell_index]
693
694 # Unique dir for every test case and shell
695 tmp_base = os.path.normpath(opts.tmp_env) # no . or ..
696 case_tmp_dir = os.path.join(tmp_base, '%02d-%s' % (i, sh_label))
697
698 try:
699 os.makedirs(case_tmp_dir)
700 except OSError as e:
701 if e.errno != errno.EEXIST:
702 raise
703
704 # Some tests assume _tmp exists
705 try:
706 os.mkdir(os.path.join(case_tmp_dir, '_tmp'))
707 except OSError as e:
708 if e.errno != errno.EEXIST:
709 raise
710
711 case_env['TMP'] = case_tmp_dir
712
713 if opts.pyann_out_dir:
714 case_env = dict(case_env)
715 case_env['PYANN_OUT'] = os.path.join(opts.pyann_out_dir, '%d.json' % i)
716
717 try:
718 p = subprocess.Popen(argv, env=case_env, cwd=case_tmp_dir,
719 stdin=PIPE, stdout=PIPE, stderr=PIPE)
720 except OSError as e:
721 print('Error running %r: %s' % (sh_path, e), file=sys.stderr)
722 sys.exit(1)
723
724 p.stdin.write(code)
725 p.stdin.close()
726
727 actual = {}
728 actual['stdout'] = p.stdout.read()
729 actual['stderr'] = p.stderr.read()
730 p.stdout.close()
731 p.stderr.close()
732
733 actual['status'] = p.wait()
734
735 if opts.timeout_bin and os.path.exists(timeout_file):
736 cell_result = Result.TIMEOUT
737 elif not opts.timeout_bin and actual['status'] == 124:
738 cell_result = Result.TIMEOUT
739 else:
740 messages = []
741 cell_result = Result.PASS
742
743 # TODO: Warn about no assertions? Well it will always test the error
744 # code.
745 assertions = CreateAssertions(case, sh_label)
746 for a in assertions:
747 result, msg = a.Check(sh_label, actual)
748 # The minimum one wins.
749 # If any failed, then the result is FAIL.
750 # If any are OK, but none are FAIL, the result is OK.
751 cell_result = min(cell_result, result)
752 if msg:
753 messages.append(msg)
754
755 if cell_result != Result.PASS or opts.details:
756 d = (i, sh_label, actual['stdout'], actual['stderr'], messages)
757 out.AddDetails(d)
758
759 result_row.append(cell_result)
760
761 stats.ReportCell(i, cell_result, sh_label)
762
763 if sh_label in OTHER_OSH:
764 # This is only an error if we tried to run ANY OSH.
765 if osh_cpython_index == -1:
766 raise RuntimeError("Couldn't determine index of osh-cpython")
767
768 other_result = result_row[shell_index]
769 cpython_result = result_row[osh_cpython_index]
770 if other_result != cpython_result:
771 stats.Inc('oils_ALT_delta')
772
773 out.WriteRow(i, line_num, result_row, desc)
774
775 return stats
776
777
778# ANSI color constants
779_RESET = '\033[0;0m'
780_BOLD = '\033[1m'
781
782_RED = '\033[31m'
783_GREEN = '\033[32m'
784_YELLOW = '\033[33m'
785_PURPLE = '\033[35m'
786
787
788TEXT_CELLS = {
789 Result.TIMEOUT: 'TIME',
790 Result.FAIL: 'FAIL',
791 Result.BUG: 'BUG',
792 Result.NI: 'N-I',
793 Result.OK: 'ok',
794 Result.PASS: 'pass',
795}
796
797ANSI_COLORS = {
798 Result.TIMEOUT: _PURPLE,
799 Result.FAIL: _RED,
800 Result.BUG: _YELLOW,
801 Result.NI: _YELLOW,
802 Result.OK: _YELLOW,
803 Result.PASS: _GREEN,
804}
805
806def _AnsiCells():
807 lookup = {}
808 for i in xrange(Result.length):
809 lookup[i] = ''.join([ANSI_COLORS[i], _BOLD, TEXT_CELLS[i], _RESET])
810 return lookup
811
812ANSI_CELLS = _AnsiCells()
813
814
815HTML_CELLS = {
816 Result.TIMEOUT: '<td class="timeout">TIME',
817 Result.FAIL: '<td class="fail">FAIL',
818 Result.BUG: '<td class="bug">BUG',
819 Result.NI: '<td class="n-i">N-I',
820 Result.OK: '<td class="ok">ok',
821 Result.PASS: '<td class="pass">pass',
822}
823
824
825def _ValidUtf8String(s):
826 """Return an arbitrary string as a readable utf-8 string.
827
828 We output utf-8 to either HTML or the console. If we get invalid utf-8 as
829 stdout/stderr (which is very possible), then show the ASCII repr().
830 """
831 try:
832 s.decode('utf-8')
833 return s # it decoded OK
834 except UnicodeDecodeError:
835 return repr(s) # ASCII representation
836
837
838class Output(object):
839
840 def __init__(self, f, verbose):
841 self.f = f
842 self.verbose = verbose
843 self.details = []
844
845 def BeginCases(self, test_file):
846 pass
847
848 def WriteHeader(self, sh_labels):
849 pass
850
851 def WriteRow(self, i, line_num, row, desc):
852 pass
853
854 def EndCases(self, sh_labels, stats):
855 pass
856
857 def AddDetails(self, entry):
858 self.details.append(entry)
859
860 # Helper function
861 def _WriteDetailsAsText(self, details):
862 for case_index, shell, stdout, stderr, messages in details:
863 print('case: %d' % case_index, file=self.f)
864 for m in messages:
865 print(m, file=self.f)
866
867 # Assume the terminal can show utf-8, but we don't want random binary.
868 print('%s stdout:' % shell, file=self.f)
869 print(_ValidUtf8String(stdout), file=self.f)
870
871 print('%s stderr:' % shell, file=self.f)
872 print(_ValidUtf8String(stderr), file=self.f)
873
874 print('', file=self.f)
875
876
877class TeeOutput(object):
878 """For multiple outputs in one run, e.g. HTML and TSV.
879
880 UNUSED
881 """
882
883 def __init__(self, outs):
884 self.outs = outs
885
886 def BeginCases(self, test_file):
887 for out in self.outs:
888 out.BeginCases(test_file)
889
890 def WriteHeader(self, sh_labels):
891 for out in self.outs:
892 out.WriteHeader(sh_labels)
893
894 def WriteRow(self, i, line_num, row, desc):
895 for out in self.outs:
896 out.WriteRow(i, line_num, row, desc)
897
898 def EndCases(self, sh_labels, stats):
899 for out in self.outs:
900 out.EndCases(sh_labels, stats)
901
902 def AddDetails(self, entry):
903 for out in self.outs:
904 out.AddDetails(entry)
905
906
907class TsvOutput(Output):
908 """Write a plain-text TSV file.
909
910 UNUSED since we are outputting LONG format with --tsv-output.
911 """
912
913 def WriteHeader(self, sh_labels):
914 self.f.write('case\tline\t') # case number and line number
915 for sh_label in sh_labels:
916 self.f.write(sh_label)
917 self.f.write('\t')
918 self.f.write('\n')
919
920 def WriteRow(self, i, line_num, row, desc):
921 self.f.write('%3d\t%3d\t' % (i, line_num))
922
923 for result in row:
924 c = TEXT_CELLS[result]
925 self.f.write(c)
926 self.f.write('\t')
927
928 # note: 'desc' could use TSV8, but just ignore it for now
929 #self.f.write(desc)
930 self.f.write('\n')
931
932
933class AnsiOutput(Output):
934
935 def BeginCases(self, test_file):
936 self.f.write('%s\n' % test_file)
937
938 def WriteHeader(self, sh_labels):
939 self.f.write(_BOLD)
940 self.f.write('case\tline\t') # case number and line number
941 for sh_label in sh_labels:
942 self.f.write(sh_label)
943 self.f.write('\t')
944 self.f.write(_RESET)
945 self.f.write('\n')
946
947 def WriteRow(self, i, line_num, row, desc):
948 self.f.write('%3d\t%3d\t' % (i, line_num))
949
950 for result in row:
951 c = ANSI_CELLS[result]
952 self.f.write(c)
953 self.f.write('\t')
954
955 self.f.write(desc)
956 self.f.write('\n')
957
958 if self.verbose:
959 self._WriteDetailsAsText(self.details)
960 self.details = []
961
962 def _WriteShellSummary(self, sh_labels, stats):
963 if len(stats.nonzero_results) <= 1: # Skip trivial summaries
964 return
965
966 # Reiterate header
967 self.f.write(_BOLD)
968 self.f.write('\t\t')
969 for sh_label in sh_labels:
970 self.f.write(sh_label)
971 self.f.write('\t')
972 self.f.write(_RESET)
973 self.f.write('\n')
974
975 # Write totals by cell.
976 for result in sorted(stats.nonzero_results, reverse=True):
977 self.f.write('\t%s' % ANSI_CELLS[result])
978 for sh_label in sh_labels:
979 self.f.write('\t%d' % stats.by_shell[sh_label][result])
980 self.f.write('\n')
981
982 # The bottom row is all the same, but it helps readability.
983 self.f.write('\ttotal')
984 for sh_label in sh_labels:
985 self.f.write('\t%d' % stats.counters['num_cases_run'])
986 self.f.write('\n')
987
988 def EndCases(self, sh_labels, stats):
989 print()
990 self._WriteShellSummary(sh_labels, stats)
991
992
993class HtmlOutput(Output):
994
995 def __init__(self, f, verbose, spec_name, sh_labels, cases):
996 Output.__init__(self, f, verbose)
997 self.spec_name = spec_name
998 self.sh_labels = sh_labels # saved from header
999 self.cases = cases # for linking to code
1000 self.row_html = [] # buffered
1001
1002 def _SourceLink(self, line_num, desc):
1003 return '<a href="%s.test.html#L%d">%s</a>' % (
1004 self.spec_name, line_num, cgi.escape(desc))
1005
1006 def BeginCases(self, test_file):
1007 css_urls = [ '../../../web/base.css', '../../../web/spec-tests.css' ]
1008 title = '%s: spec test case results' % self.spec_name
1009 html_head.Write(self.f, title, css_urls=css_urls)
1010
1011 self.f.write('''\
1012 <body class="width60">
1013 <p id="home-link">
1014 <a href=".">spec test index</a>
1015 /
1016 <a href="/">oilshell.org</a>
1017 </p>
1018 <h1>Results for %s</h1>
1019 <table>
1020 ''' % test_file)
1021
1022 def _WriteShellSummary(self, sh_labels, stats):
1023 # NOTE: This table has multiple <thead>, which seems OK.
1024 self.f.write('''
1025<thead>
1026 <tr class="table-header">
1027 ''')
1028
1029 columns = ['status'] + sh_labels + ['']
1030 for c in columns:
1031 self.f.write('<td>%s</td>' % c)
1032
1033 self.f.write('''
1034 </tr>
1035</thead>
1036''')
1037
1038 # Write totals by cell.
1039 for result in sorted(stats.nonzero_results, reverse=True):
1040 self.f.write('<tr>')
1041
1042 self.f.write(HTML_CELLS[result])
1043 self.f.write('</td> ')
1044
1045 for sh_label in sh_labels:
1046 self.f.write('<td>%d</td>' % stats.by_shell[sh_label][result])
1047
1048 self.f.write('<td></td>')
1049 self.f.write('</tr>\n')
1050
1051 # The bottom row is all the same, but it helps readability.
1052 self.f.write('<tr>')
1053 self.f.write('<td>total</td>')
1054 for sh_label in sh_labels:
1055 self.f.write('<td>%d</td>' % stats.counters['num_cases_run'])
1056 self.f.write('<td></td>')
1057 self.f.write('</tr>\n')
1058
1059 # Blank row for space.
1060 self.f.write('<tr>')
1061 for i in xrange(len(sh_labels) + 2):
1062 self.f.write('<td style="height: 2em"></td>')
1063 self.f.write('</tr>\n')
1064
1065 def WriteHeader(self, sh_labels):
1066 f = cStringIO.StringIO()
1067
1068 f.write('''
1069<thead>
1070 <tr class="table-header">
1071 ''')
1072
1073 columns = ['case'] + sh_labels
1074 for c in columns:
1075 f.write('<td>%s</td>' % c)
1076 f.write('<td class="case-desc">description</td>')
1077
1078 f.write('''
1079 </tr>
1080</thead>
1081''')
1082
1083 self.row_html.append(f.getvalue())
1084
1085 def WriteRow(self, i, line_num, row, desc):
1086 f = cStringIO.StringIO()
1087 f.write('<tr>')
1088 f.write('<td>%3d</td>' % i)
1089
1090 show_details = False
1091
1092 for result in row:
1093 c = HTML_CELLS[result]
1094 if result not in (Result.PASS, Result.TIMEOUT): # nothing to show
1095 show_details = True
1096
1097 f.write(c)
1098 f.write('</td>')
1099 f.write('\t')
1100
1101 f.write('<td class="case-desc">')
1102 f.write(self._SourceLink(line_num, desc))
1103 f.write('</td>')
1104 f.write('</tr>\n')
1105
1106 # Show row with details link.
1107 if show_details:
1108 f.write('<tr>')
1109 f.write('<td class="details-row"></td>') # for the number
1110
1111 for col_index, result in enumerate(row):
1112 f.write('<td class="details-row">')
1113 if result != Result.PASS:
1114 sh_label = self.sh_labels[col_index]
1115 f.write('<a href="#details-%s-%s">details</a>' % (i, sh_label))
1116 f.write('</td>')
1117
1118 f.write('<td class="details-row"></td>') # for the description
1119 f.write('</tr>\n')
1120
1121 self.row_html.append(f.getvalue()) # buffer it
1122
1123 def _WriteStats(self, stats):
1124 self.f.write(
1125 '%(num_passed)d passed, %(num_ok)d OK, '
1126 '%(num_ni)d not implemented, %(num_bug)d BUG, '
1127 '%(num_failed)d failed, %(num_timeout)d timeouts, '
1128 '%(num_skipped)d cases skipped\n' % stats.counters)
1129
1130 def EndCases(self, sh_labels, stats):
1131 self._WriteShellSummary(sh_labels, stats)
1132
1133 # Write all the buffered rows
1134 for h in self.row_html:
1135 self.f.write(h)
1136
1137 self.f.write('</table>\n')
1138 self.f.write('<pre>')
1139 self._WriteStats(stats)
1140 if stats.Get('oils_num_failed'):
1141 self.f.write('%(oils_num_failed)d failed under osh\n' % stats.counters)
1142 self.f.write('</pre>')
1143
1144 if self.details:
1145 self._WriteDetails()
1146
1147 self.f.write('</body></html>')
1148
1149 def _WriteDetails(self):
1150 self.f.write("<h2>Details on runs that didn't PASS</h2>")
1151 self.f.write('<table id="details">')
1152
1153 for case_index, sh_label, stdout, stderr, messages in self.details:
1154 self.f.write('<tr>')
1155 self.f.write('<td><a name="details-%s-%s"></a><b>%s</b></td>' % (
1156 case_index, sh_label, sh_label))
1157
1158 self.f.write('<td>')
1159
1160 # Write description and link to the code
1161 case = self.cases[case_index]
1162 line_num = case['line_num']
1163 desc = case['desc']
1164 self.f.write('%d ' % case_index)
1165 self.f.write(self._SourceLink(line_num, desc))
1166 self.f.write('<br/><br/>\n')
1167
1168 for m in messages:
1169 self.f.write('<span class="assertion">%s</span><br/>\n' % cgi.escape(m))
1170 if messages:
1171 self.f.write('<br/>\n')
1172
1173 def _WriteRaw(s):
1174 self.f.write('<pre>')
1175
1176 # stdout might contain invalid utf-8; make it valid;
1177 valid_utf8 = _ValidUtf8String(s)
1178
1179 self.f.write(cgi.escape(valid_utf8))
1180 self.f.write('</pre>')
1181
1182 self.f.write('<i>stdout:</i> <br/>\n')
1183 _WriteRaw(stdout)
1184
1185 self.f.write('<i>stderr:</i> <br/>\n')
1186 _WriteRaw(stderr)
1187
1188 self.f.write('</td>')
1189 self.f.write('</tr>')
1190
1191 self.f.write('</table>')
1192
1193
1194def MakeTestEnv(opts):
1195 if not opts.tmp_env:
1196 raise RuntimeError('--tmp-env required')
1197 if not opts.path_env:
1198 raise RuntimeError('--path-env required')
1199 env = {
1200 'PATH': opts.path_env,
1201 #'LANG': opts.lang_env,
1202 }
1203 for p in opts.env_pair:
1204 name, value = p.split('=', 1)
1205 env[name] = value
1206
1207 return env
1208
1209
1210def _DefaultSuite(spec_name):
1211 if spec_name.startswith('ysh-'):
1212 suite = 'ysh'
1213 elif spec_name.startswith('hay'): # hay.test.sh is ysh
1214 suite = 'ysh'
1215
1216 elif spec_name.startswith('tea-'):
1217 suite = 'tea'
1218 else:
1219 suite = 'osh'
1220
1221 return suite
1222
1223
1224def ParseTestList(test_files):
1225 for test_file in test_files:
1226 with open(test_file) as f:
1227 tokens = Tokenizer(f)
1228 try:
1229 file_metadata, cases = ParseTestFile(test_file, tokens)
1230 except RuntimeError as e:
1231 log('ERROR in %r', test_file)
1232 raise
1233
1234 tmp = os.path.basename(test_file)
1235 spec_name = tmp.split('.')[0] # foo.test.sh -> foo
1236
1237 suite = file_metadata.get('suite') or _DefaultSuite(spec_name)
1238
1239 tmp = file_metadata.get('tags')
1240 tags = tmp.split() if tmp else []
1241
1242 # Don't need compare_shells, etc. to decide what to run
1243
1244 row = {'spec_name': spec_name, 'suite': suite, 'tags': tags}
1245 #print(row)
1246 yield row
1247
1248
1249def main(argv):
1250 # First check if bash is polluting the environment. Tests rely on the
1251 # environment.
1252 v = os.getenv('RANDOM')
1253 if v is not None:
1254 raise AssertionError('got $RANDOM = %s' % v)
1255 v = os.getenv('PPID')
1256 if v is not None:
1257 raise AssertionError('got $PPID = %s' % v)
1258
1259 p = optparse.OptionParser('%s [options] TEST_FILE shell...' % sys.argv[0])
1260 spec_lib.DefineCommon(p)
1261 spec_lib.DefineShSpec(p)
1262 opts, argv = p.parse_args(argv)
1263
1264 # --print-tagged to figure out what to run
1265 if opts.print_tagged:
1266 to_find = opts.print_tagged
1267 for row in ParseTestList(argv[1:]):
1268 if to_find in row['tags']:
1269 print(row['spec_name'])
1270 return 0
1271
1272 # --print-table to figure out what to run
1273 if opts.print_table:
1274 for row in ParseTestList(argv[1:]):
1275 print('%(suite)s\t%(spec_name)s' % row)
1276 #print(row)
1277 return 0
1278
1279 #
1280 # Now deal with a single file
1281 #
1282
1283 try:
1284 test_file = argv[1]
1285 except IndexError:
1286 p.print_usage()
1287 return 1
1288
1289 with open(test_file) as f:
1290 tokens = Tokenizer(f)
1291 file_metadata, cases = ParseTestFile(test_file, tokens)
1292
1293 # List test cases and return
1294 if opts.do_list:
1295 for i, case in enumerate(cases):
1296 if opts.verbose: # print the raw dictionary for debugging
1297 print(pprint.pformat(case))
1298 else:
1299 print('%d\t%s' % (i, case['desc']))
1300 return 0
1301
1302 # for test/spec-cpp.sh
1303 if opts.print_spec_suite:
1304 tmp = os.path.basename(test_file)
1305 spec_name = tmp.split('.')[0] # foo.test.sh -> foo
1306
1307 suite = file_metadata.get('suite') or _DefaultSuite(spec_name)
1308 print(suite)
1309 return 0
1310
1311 if opts.verbose:
1312 for k, v in file_metadata.items():
1313 print('\t%-20s: %s' % (k, v), file=sys.stderr)
1314 print('', file=sys.stderr)
1315
1316 if opts.oils_bin_dir:
1317
1318 shells = []
1319
1320 if opts.compare_shells:
1321 comp = file_metadata.get('compare_shells')
1322 # Compare 'compare_shells' and Python
1323 shells.extend(comp.split() if comp else [])
1324
1325 # Always run with the Python version
1326 our_shell = file_metadata.get('our_shell', 'osh') # default is OSH
1327 shells.append(os.path.join(opts.oils_bin_dir, our_shell))
1328
1329 # Legacy OVM/CPython build
1330 if opts.ovm_bin_dir:
1331 shells.append(os.path.join(opts.ovm_bin_dir, our_shell))
1332
1333 # New C++ build
1334 if opts.oils_cpp_bin_dir:
1335 shells.append(os.path.join(opts.oils_cpp_bin_dir, our_shell))
1336
1337 # Overwrite it when --oils-bin-dir is set
1338 # It's no longer a flag
1339 opts.oils_failures_allowed = \
1340 int(file_metadata.get('oils_failures_allowed', 0))
1341
1342 else:
1343 # TODO: remove this mode?
1344 shells = argv[2:]
1345
1346 shell_pairs = spec_lib.MakeShellPairs(shells)
1347
1348 if opts.range:
1349 begin, end = spec_lib.ParseRange(opts.range)
1350 case_predicate = spec_lib.RangePredicate(begin, end)
1351 elif opts.regex:
1352 desc_re = re.compile(opts.regex, re.IGNORECASE)
1353 case_predicate = spec_lib.RegexPredicate(desc_re)
1354 else:
1355 case_predicate = lambda i, case: True
1356
1357 out_f = sys.stderr if opts.do_print else sys.stdout
1358
1359 # Set up output style. Also see asdl/format.py
1360 if opts.format == 'ansi':
1361 out = AnsiOutput(out_f, opts.verbose)
1362
1363 elif opts.format == 'html':
1364 spec_name = os.path.basename(test_file)
1365 spec_name = spec_name.split('.')[0]
1366
1367 sh_labels = [label for label, _ in shell_pairs]
1368
1369 out = HtmlOutput(out_f, opts.verbose, spec_name, sh_labels, cases)
1370
1371 else:
1372 raise AssertionError()
1373
1374 out.BeginCases(os.path.basename(test_file))
1375
1376 env = MakeTestEnv(opts)
1377 stats = RunCases(cases, case_predicate, shell_pairs, env, out, opts)
1378
1379 out.EndCases([sh_label for sh_label, _ in shell_pairs], stats)
1380
1381 if opts.tsv_output:
1382 with open(opts.tsv_output, 'w') as f:
1383 stats.WriteTsv(f)
1384
1385 # TODO: Could --stats-{file,template} be a separate awk step on .tsv files?
1386 stats.Set('oils_failures_allowed', opts.oils_failures_allowed)
1387 if opts.stats_file:
1388 with open(opts.stats_file, 'w') as f:
1389 f.write(opts.stats_template % stats.counters)
1390 f.write('\n') # bash 'read' requires a newline
1391
1392 if stats.Get('num_failed') == 0:
1393 return 0
1394
1395 # spec/smoke.test.sh -> smoke
1396 test_name = os.path.basename(test_file).split('.')[0]
1397
1398 allowed = opts.oils_failures_allowed
1399 all_count = stats.Get('num_failed')
1400 oils_count = stats.Get('oils_num_failed')
1401 if allowed == 0:
1402 log('')
1403 log('%s: FATAL: %d tests failed (%d oils failures)', test_name, all_count,
1404 oils_count)
1405 log('')
1406 else:
1407 # If we got EXACTLY the allowed number of failures, exit 0.
1408 if allowed == all_count and all_count == oils_count:
1409 log('%s: note: Got %d allowed oils failures (exit with code 0)',
1410 test_name, allowed)
1411 return 0
1412 else:
1413 log('')
1414 log('%s: FATAL: Got %d failures (%d oils failures), but %d are allowed',
1415 test_name, all_count, oils_count, allowed)
1416 log('')
1417
1418 return 1
1419
1420
1421if __name__ == '__main__':
1422 try:
1423 sys.exit(main(sys.argv))
1424 except KeyboardInterrupt as e:
1425 print('%s: interrupted with Ctrl-C' % sys.argv[0], file=sys.stderr)
1426 sys.exit(1)
1427 except RuntimeError as e:
1428 print('FATAL: %s' % e, file=sys.stderr)
1429 sys.exit(1)