| 1 | #!/usr/bin/env python
|
| 2 | from __future__ import print_function
|
| 3 | """
|
| 4 | sh_spec.py -- Test framework to compare shells.
|
| 5 |
|
| 6 | Assertion help:
|
| 7 | stdout: A single line of expected stdout. Newline is implicit.
|
| 8 | stdout-json: JSON-encoded string. Use for the empty string (no newline),
|
| 9 | for unicode chars, etc.
|
| 10 |
|
| 11 | stderr: Ditto for stderr stream.
|
| 12 | status: Expected shell return code. If not specified, the case must exit 0.
|
| 13 |
|
| 14 | Results:
|
| 15 | PASS - we got the ideal, expected value
|
| 16 | OK - we got a value that was not ideal, but expected
|
| 17 | N-I - Not implemented (e.g. $''). Assertions still checked (in case it
|
| 18 | starts working)
|
| 19 | BUG - we verified the value of a known bug
|
| 20 | FAIL - we got an unexpected value. If the implementation can't be changed,
|
| 21 | it should be converted to BUG or OK. Otherwise it should be made to
|
| 22 | PASS.
|
| 23 |
|
| 24 | TODO: maybe have KBUG and BUG? KBUG is known bug, or intentional
|
| 25 | incompatibility. Like dash interpreting escapes in 'foo\n'. An unintentional
|
| 26 | bug is something else, like bash parsing errors.
|
| 27 | IBUG / BUG / N-I are all variants of the same thing.
|
| 28 |
|
| 29 | NOTE: The difference between OK and BUG is a matter of judgement. If the ideal
|
| 30 | behavior is a compile time error (code 2), a runtime error is generally OK.
|
| 31 |
|
| 32 | If ALL shells agree on a broken behavior, they are all marked OK (but our
|
| 33 | implementation will be PASS.) But if the behavior is NOT POSIX compliant, then
|
| 34 | it will be a BUG.
|
| 35 |
|
| 36 | If one shell disagrees with others, that is generally a BUG.
|
| 37 |
|
| 38 | Example test case:
|
| 39 |
|
| 40 | ### hello and fail
|
| 41 | echo hello
|
| 42 | echo world
|
| 43 | exit 1
|
| 44 | ## status: 1
|
| 45 | #
|
| 46 | # ignored comment
|
| 47 | #
|
| 48 | ## STDOUT
|
| 49 | hello
|
| 50 | world
|
| 51 | ## END
|
| 52 |
|
| 53 | """
|
| 54 |
|
| 55 | import collections
|
| 56 | import cgi
|
| 57 | import json
|
| 58 | import optparse
|
| 59 | import os
|
| 60 | import pprint
|
| 61 | import re
|
| 62 | import subprocess
|
| 63 | import sys
|
| 64 | import time
|
| 65 |
|
| 66 |
|
| 67 | class ParseError(Exception):
|
| 68 | pass
|
| 69 |
|
| 70 |
|
| 71 | def log(msg, *args):
|
| 72 | if args:
|
| 73 | msg = msg % args
|
| 74 | print(msg, file=sys.stderr)
|
| 75 |
|
| 76 |
|
| 77 | # EXAMPLES:
|
| 78 | # stdout: foo
|
| 79 | # stdout-json: ""
|
| 80 | #
|
| 81 | # In other words, it could be (name, value) or (qualifier, name, value)
|
| 82 |
|
| 83 | KEY_VALUE_RE = re.compile(r'''
|
| 84 | [#][#]? \s+
|
| 85 | (?: (OK|BUG|N-I) \s+ ([\w+/]+) \s+ )? # optional prefix
|
| 86 | ([\w\-]+) # key
|
| 87 | :
|
| 88 | \s* (.*) # value
|
| 89 | ''', re.VERBOSE)
|
| 90 |
|
| 91 | END_MULTILINE_RE = re.compile(r'''
|
| 92 | [#][#]? \s+ END
|
| 93 | ''', re.VERBOSE)
|
| 94 |
|
| 95 | # Line types
|
| 96 | TEST_CASE_BEGIN = 0 # Starts with ###
|
| 97 | KEY_VALUE = 1 # Metadata
|
| 98 | KEY_VALUE_MULTILINE = 2 # STDOUT STDERR
|
| 99 | END_MULTILINE = 3 # STDOUT STDERR
|
| 100 | PLAIN_LINE = 4 # Uncommented
|
| 101 | EOF = 5
|
| 102 |
|
| 103 |
|
| 104 | def LineIter(f):
|
| 105 | """Iterate over lines, classify them by token type, and parse token value."""
|
| 106 | for i, line in enumerate(f):
|
| 107 | if not line.strip():
|
| 108 | continue
|
| 109 |
|
| 110 | line_num = i+1 # 1-based
|
| 111 |
|
| 112 | if line.startswith('###'):
|
| 113 | desc = line[3:].strip()
|
| 114 | yield line_num, TEST_CASE_BEGIN, desc
|
| 115 | continue
|
| 116 |
|
| 117 | m = KEY_VALUE_RE.match(line)
|
| 118 | if m:
|
| 119 | qualifier, shells, name, value = m.groups()
|
| 120 | # HACK: Expected data should have the newline.
|
| 121 | if name in ('stdout', 'stderr'):
|
| 122 | value += '\n'
|
| 123 |
|
| 124 | if name in ('STDOUT', 'STDERR'):
|
| 125 | token_type = KEY_VALUE_MULTILINE
|
| 126 | else:
|
| 127 | token_type = KEY_VALUE
|
| 128 | yield line_num, token_type, (qualifier, shells, name, value)
|
| 129 | continue
|
| 130 |
|
| 131 | m = END_MULTILINE_RE.match(line)
|
| 132 | if m:
|
| 133 | yield line_num, END_MULTILINE, None
|
| 134 | continue
|
| 135 |
|
| 136 | if line.lstrip().startswith('#'):
|
| 137 | # Ignore comments
|
| 138 | #yield COMMENT, line
|
| 139 | continue
|
| 140 |
|
| 141 | # Non-empty line that doesn't start with '#'
|
| 142 | # NOTE: We need the original line to test the whitespace sensitive <<-.
|
| 143 | # And we need rstrip because we add newlines back below.
|
| 144 | yield line_num, PLAIN_LINE, line.rstrip('\n')
|
| 145 |
|
| 146 | yield line_num, EOF, None
|
| 147 |
|
| 148 |
|
| 149 | class Tokenizer(object):
|
| 150 | """Wrap a token iterator in a Tokenizer interface."""
|
| 151 |
|
| 152 | def __init__(self, it):
|
| 153 | self.it = it
|
| 154 | self.cursor = None
|
| 155 | self.next()
|
| 156 |
|
| 157 | def next(self):
|
| 158 | """Raises StopIteration when exhausted."""
|
| 159 | self.cursor = self.it.next()
|
| 160 | return self.cursor
|
| 161 |
|
| 162 | def peek(self):
|
| 163 | return self.cursor
|
| 164 |
|
| 165 |
|
| 166 | # Format of a test script.
|
| 167 | #
|
| 168 | # -- Code is either literal lines, or a commented out code: value.
|
| 169 | # code = (? line of code ?)*
|
| 170 | # | '# code:' VALUE
|
| 171 | #
|
| 172 | # -- Description, then key-value pairs surrounding code.
|
| 173 | # test_case = '###' DESC
|
| 174 | # ( '#' KEY ':' VALUE )*
|
| 175 | # code
|
| 176 | # ( '#' KEY ':' VALUE )*
|
| 177 | #
|
| 178 | # -- Should be a blank line after each test case. Leading comments and code
|
| 179 | # -- are OK.
|
| 180 | # test_file = (COMMENT | PLAIN_LINE)* (test_case '\n')*
|
| 181 |
|
| 182 |
|
| 183 | def AddMetadataToCase(case, qualifier, shells, name, value):
|
| 184 | shells = shells.split('/') # bash/dash/mksh
|
| 185 | for shell in shells:
|
| 186 | if shell not in case:
|
| 187 | case[shell] = {}
|
| 188 | case[shell][name] = value
|
| 189 | case[shell]['qualifier'] = qualifier
|
| 190 |
|
| 191 |
|
| 192 | def ParseKeyValue(tokens, case):
|
| 193 | """Parse commented-out metadata in a test case.
|
| 194 |
|
| 195 | The metadata must be contiguous.
|
| 196 |
|
| 197 | Args:
|
| 198 | tokens: Tokenizer
|
| 199 | case: dictionary to add to
|
| 200 | """
|
| 201 | while True:
|
| 202 | line_num, kind, item = tokens.peek()
|
| 203 |
|
| 204 | if kind == KEY_VALUE_MULTILINE:
|
| 205 | qualifier, shells, name, empty_value = item
|
| 206 | if empty_value:
|
| 207 | raise ParseError(
|
| 208 | 'Line %d: got value %r for %r, but the value should be on the '
|
| 209 | 'following lines' % (line_num, empty_value, name))
|
| 210 |
|
| 211 | value_lines = []
|
| 212 | while True:
|
| 213 | tokens.next()
|
| 214 | _, kind2, item2 = tokens.peek()
|
| 215 | if kind2 != PLAIN_LINE:
|
| 216 | break
|
| 217 | value_lines.append(item2)
|
| 218 |
|
| 219 | value = '\n'.join(value_lines) + '\n'
|
| 220 |
|
| 221 | name = name.lower() # STDOUT -> stdout
|
| 222 | if qualifier:
|
| 223 | AddMetadataToCase(case, qualifier, shells, name, value)
|
| 224 | else:
|
| 225 | case[name] = value
|
| 226 |
|
| 227 | # END token is optional.
|
| 228 | if kind2 == END_MULTILINE:
|
| 229 | tokens.next()
|
| 230 |
|
| 231 | elif kind == KEY_VALUE:
|
| 232 | qualifier, shells, name, value = item
|
| 233 |
|
| 234 | if qualifier:
|
| 235 | AddMetadataToCase(case, qualifier, shells, name, value)
|
| 236 | else:
|
| 237 | case[name] = value
|
| 238 |
|
| 239 | tokens.next()
|
| 240 |
|
| 241 | else: # Unknown token type
|
| 242 | break
|
| 243 |
|
| 244 |
|
| 245 |
|
| 246 | def ParseCodeLines(tokens, case):
|
| 247 | """Parse uncommented code in a test case."""
|
| 248 | _, kind, item = tokens.peek()
|
| 249 | if kind != PLAIN_LINE:
|
| 250 | raise ParseError('Expected a line of code (got %r, %r)' % (kind, item))
|
| 251 | code_lines = []
|
| 252 | while True:
|
| 253 | _, kind, item = tokens.peek()
|
| 254 | if kind != PLAIN_LINE:
|
| 255 | case['code'] = '\n'.join(code_lines) + '\n'
|
| 256 | return
|
| 257 | code_lines.append(item)
|
| 258 | tokens.next()
|
| 259 |
|
| 260 |
|
| 261 | def ParseTestCase(tokens):
|
| 262 | """Parse a single test case and return it.
|
| 263 |
|
| 264 | If at EOF, return None.
|
| 265 | """
|
| 266 | line_num, kind, item = tokens.peek()
|
| 267 | if kind == EOF:
|
| 268 | return None
|
| 269 |
|
| 270 | assert kind == TEST_CASE_BEGIN, (line_num, kind, item) # Invariant
|
| 271 | tokens.next()
|
| 272 |
|
| 273 | case = {'desc': item, 'line_num': line_num}
|
| 274 | #print case
|
| 275 |
|
| 276 | ParseKeyValue(tokens, case)
|
| 277 |
|
| 278 | #print 'KV1', case
|
| 279 | # For broken code
|
| 280 | if 'code' in case: # Got it through a key value pair
|
| 281 | return case
|
| 282 |
|
| 283 | ParseCodeLines(tokens, case)
|
| 284 | #print 'AFTER CODE', case
|
| 285 | ParseKeyValue(tokens, case)
|
| 286 | #print 'KV2', case
|
| 287 |
|
| 288 | return case
|
| 289 |
|
| 290 |
|
| 291 | def ParseTestFile(tokens):
|
| 292 | #pprint.pprint(list(lines))
|
| 293 | #return
|
| 294 | test_cases = []
|
| 295 | try:
|
| 296 | # Skip over the header. Setup code can go here, although would we have to
|
| 297 | # execute it on every case?
|
| 298 | while True:
|
| 299 | line_num, kind, item = tokens.peek()
|
| 300 | if kind == TEST_CASE_BEGIN:
|
| 301 | break
|
| 302 | tokens.next()
|
| 303 |
|
| 304 | while True: # Loop over cases
|
| 305 | test_case = ParseTestCase(tokens)
|
| 306 | if test_case is None:
|
| 307 | break
|
| 308 | test_cases.append(test_case)
|
| 309 |
|
| 310 | except StopIteration:
|
| 311 | raise RuntimeError('Unexpected EOF parsing test cases')
|
| 312 |
|
| 313 | return test_cases
|
| 314 |
|
| 315 |
|
| 316 | def CreateStringAssertion(d, key, assertions, qualifier=False):
|
| 317 | found = False
|
| 318 |
|
| 319 | exp = d.get(key)
|
| 320 | if exp is not None:
|
| 321 | a = EqualAssertion(key, exp, qualifier=qualifier)
|
| 322 | assertions.append(a)
|
| 323 | found = True
|
| 324 |
|
| 325 | exp_json = d.get(key + '-json')
|
| 326 | if exp_json is not None:
|
| 327 | exp = json.loads(exp_json, encoding='utf-8')
|
| 328 | a = EqualAssertion(key, exp, qualifier=qualifier)
|
| 329 | assertions.append(a)
|
| 330 | found = True
|
| 331 |
|
| 332 | # For testing invalid unicode
|
| 333 | exp_repr = d.get(key + '-repr')
|
| 334 | if exp_repr is not None:
|
| 335 | exp = eval(exp_repr)
|
| 336 | a = EqualAssertion(key, exp, qualifier=qualifier)
|
| 337 | assertions.append(a)
|
| 338 | found = True
|
| 339 |
|
| 340 | return found
|
| 341 |
|
| 342 |
|
| 343 | def CreateIntAssertion(d, key, assertions, qualifier=False):
|
| 344 | exp = d.get(key) # expected
|
| 345 | if exp is not None:
|
| 346 | # For now, turn it into int
|
| 347 | a = EqualAssertion(key, int(exp), qualifier=qualifier)
|
| 348 | assertions.append(a)
|
| 349 | return True
|
| 350 | return False
|
| 351 |
|
| 352 |
|
| 353 | def CreateAssertions(case, sh_label):
|
| 354 | """
|
| 355 | Given a raw test case and a shell label, create EqualAssertion instances to
|
| 356 | run.
|
| 357 | """
|
| 358 | assertions = []
|
| 359 |
|
| 360 | # Whether we found assertions
|
| 361 | stdout = False
|
| 362 | stderr = False
|
| 363 | status = False
|
| 364 |
|
| 365 | # So the assertion are exactly the same for osh and osh_ALT
|
| 366 | if sh_label == 'osh_ALT':
|
| 367 | sh_label = 'osh'
|
| 368 |
|
| 369 | if sh_label in case:
|
| 370 | q = case[sh_label]['qualifier']
|
| 371 | if CreateStringAssertion(case[sh_label], 'stdout', assertions, qualifier=q):
|
| 372 | stdout = True
|
| 373 | if CreateStringAssertion(case[sh_label], 'stderr', assertions, qualifier=q):
|
| 374 | stderr = True
|
| 375 | if CreateIntAssertion(case[sh_label], 'status', assertions, qualifier=q):
|
| 376 | status = True
|
| 377 |
|
| 378 | if not stdout:
|
| 379 | CreateStringAssertion(case, 'stdout', assertions)
|
| 380 | if not stderr:
|
| 381 | CreateStringAssertion(case, 'stderr', assertions)
|
| 382 | if not status:
|
| 383 | if 'status' in case:
|
| 384 | CreateIntAssertion(case, 'status', assertions)
|
| 385 | else:
|
| 386 | # If the user didn't specify a 'status' assertion, assert that the exit
|
| 387 | # code is 0.
|
| 388 | a = EqualAssertion('status', 0)
|
| 389 | assertions.append(a)
|
| 390 |
|
| 391 | #print 'SHELL', shell
|
| 392 | #pprint.pprint(case)
|
| 393 | #print(assertions)
|
| 394 | return assertions
|
| 395 |
|
| 396 |
|
| 397 | class Result(object):
|
| 398 | """Possible test results.
|
| 399 |
|
| 400 | Order is important: the result of a cell is the minimum of the results of
|
| 401 | each assertions.
|
| 402 | """
|
| 403 | FAIL = 0
|
| 404 | BUG = 1
|
| 405 | NI = 2
|
| 406 | OK = 3
|
| 407 | PASS = 4
|
| 408 |
|
| 409 |
|
| 410 | class EqualAssertion(object):
|
| 411 | """An expected value in a record."""
|
| 412 | def __init__(self, key, expected, qualifier=None):
|
| 413 | self.key = key
|
| 414 | self.expected = expected # expected value
|
| 415 | self.qualifier = qualifier # whether this was a special case?
|
| 416 |
|
| 417 | def __repr__(self):
|
| 418 | return '<EqualAssertion %s == %r>' % (self.key, self.expected)
|
| 419 |
|
| 420 | def Check(self, shell, record):
|
| 421 | actual = record[self.key]
|
| 422 | if actual != self.expected:
|
| 423 | msg = '[%s %s] Expected %r, got %r' % (shell, self.key, self.expected,
|
| 424 | actual)
|
| 425 | return Result.FAIL, msg
|
| 426 | if self.qualifier == 'BUG': # equal, but known bad
|
| 427 | return Result.BUG, ''
|
| 428 | if self.qualifier == 'N-I': # equal, and known UNIMPLEMENTED
|
| 429 | return Result.NI, ''
|
| 430 | if self.qualifier == 'OK': # equal, but ok (not ideal)
|
| 431 | return Result.OK, ''
|
| 432 | return Result.PASS, '' # ideal behavior
|
| 433 |
|
| 434 |
|
| 435 | PIPE = subprocess.PIPE
|
| 436 |
|
| 437 | def RunCases(cases, case_predicate, shells, env, out):
|
| 438 | """
|
| 439 | Run a list of test 'cases' for all 'shells' and write output to 'out'.
|
| 440 | """
|
| 441 | #pprint.pprint(cases)
|
| 442 |
|
| 443 | out.WriteHeader(shells)
|
| 444 |
|
| 445 | stats = collections.defaultdict(int)
|
| 446 | stats['num_cases'] = len(cases)
|
| 447 | stats['osh_num_passed'] = 0
|
| 448 | stats['osh_num_failed'] = 0
|
| 449 | # Number of osh_ALT results that differed from osh.
|
| 450 | stats['osh_ALT_delta'] = 0
|
| 451 |
|
| 452 | # Make an environment for each shell. $SH is the path to the shell, so we
|
| 453 | # can test flags, etc.
|
| 454 | sh_env = []
|
| 455 | for _, sh_path in shells:
|
| 456 | e = dict(env)
|
| 457 | e['SH'] = sh_path
|
| 458 | sh_env.append(e)
|
| 459 |
|
| 460 | for i, case in enumerate(cases):
|
| 461 | line_num = case['line_num']
|
| 462 | desc = case['desc']
|
| 463 | code = case['code']
|
| 464 |
|
| 465 | if not case_predicate(i, case):
|
| 466 | stats['num_skipped'] += 1
|
| 467 | continue
|
| 468 |
|
| 469 | #print code
|
| 470 |
|
| 471 | result_row = []
|
| 472 |
|
| 473 | for shell_index, (sh_label, sh_path) in enumerate(shells):
|
| 474 | argv = [sh_path] # TODO: Be able to test shell flags?
|
| 475 | try:
|
| 476 | p = subprocess.Popen(argv, env=sh_env[shell_index],
|
| 477 | stdin=PIPE, stdout=PIPE, stderr=PIPE)
|
| 478 | except OSError as e:
|
| 479 | print('Error running %r: %s' % (sh_path, e), file=sys.stderr)
|
| 480 | sys.exit(1)
|
| 481 |
|
| 482 | p.stdin.write(code)
|
| 483 | p.stdin.close()
|
| 484 |
|
| 485 | actual = {}
|
| 486 | actual['stdout'] = p.stdout.read()
|
| 487 | actual['stderr'] = p.stderr.read()
|
| 488 | p.stdout.close()
|
| 489 | p.stderr.close()
|
| 490 |
|
| 491 | actual['status'] = p.wait()
|
| 492 |
|
| 493 | messages = []
|
| 494 | cell_result = Result.PASS
|
| 495 |
|
| 496 | # TODO: Warn about no assertions? Well it will always test the error
|
| 497 | # code.
|
| 498 | assertions = CreateAssertions(case, sh_label)
|
| 499 | for a in assertions:
|
| 500 | result, msg = a.Check(sh_label, actual)
|
| 501 | # The minimum one wins.
|
| 502 | # If any failed, then the result is FAIL.
|
| 503 | # If any are OK, but none are FAIL, the result is OK.
|
| 504 | cell_result = min(cell_result, result)
|
| 505 | if msg:
|
| 506 | messages.append(msg)
|
| 507 |
|
| 508 | if cell_result != Result.PASS:
|
| 509 | d = (i, sh_label, actual['stdout'], actual['stderr'], messages)
|
| 510 | out.AddDetails(d)
|
| 511 |
|
| 512 | result_row.append(cell_result)
|
| 513 |
|
| 514 | if cell_result == Result.FAIL:
|
| 515 | # Special logic: don't count osh_ALT because its failures will be
|
| 516 | # counted in the delta.
|
| 517 | if sh_label != 'osh_ALT':
|
| 518 | stats['num_failed'] += 1
|
| 519 |
|
| 520 | if sh_label == 'osh':
|
| 521 | stats['osh_num_failed'] += 1
|
| 522 | elif cell_result == Result.BUG:
|
| 523 | stats['num_bug'] += 1
|
| 524 | elif cell_result == Result.NI:
|
| 525 | stats['num_ni'] += 1
|
| 526 | elif cell_result == Result.OK:
|
| 527 | stats['num_ok'] += 1
|
| 528 | elif cell_result == Result.PASS:
|
| 529 | stats['num_passed'] += 1
|
| 530 | if sh_label == 'osh':
|
| 531 | stats['osh_num_passed'] += 1
|
| 532 | else:
|
| 533 | raise AssertionError
|
| 534 |
|
| 535 | if sh_label == 'osh_ALT':
|
| 536 | osh_alt_result = result_row[-1]
|
| 537 | cpython_result = result_row[-2]
|
| 538 | if osh_alt_result != cpython_result:
|
| 539 | stats['osh_ALT_delta'] += 1
|
| 540 |
|
| 541 | out.WriteRow(i, line_num, result_row, desc)
|
| 542 |
|
| 543 | return stats
|
| 544 |
|
| 545 |
|
| 546 | RANGE_RE = re.compile('(\d+) \s* - \s* (\d+)', re.VERBOSE)
|
| 547 |
|
| 548 |
|
| 549 | def ParseRange(range_str):
|
| 550 | try:
|
| 551 | d = int(range_str)
|
| 552 | return d, d # singleton range
|
| 553 | except ValueError:
|
| 554 | m = RANGE_RE.match(range_str)
|
| 555 | if not m:
|
| 556 | raise RuntimeError('Invalid range %r' % range_str)
|
| 557 | b, e = m.groups()
|
| 558 | return int(b), int(e)
|
| 559 |
|
| 560 |
|
| 561 | class RangePredicate(object):
|
| 562 | """Zero-based indexing, inclusive ranges."""
|
| 563 |
|
| 564 | def __init__(self, begin, end):
|
| 565 | self.begin = begin
|
| 566 | self.end = end
|
| 567 |
|
| 568 | def __call__(self, i, case):
|
| 569 | return self.begin <= i <= self.end
|
| 570 |
|
| 571 |
|
| 572 | class RegexPredicate(object):
|
| 573 | """Filter by name."""
|
| 574 |
|
| 575 | def __init__(self, desc_re):
|
| 576 | self.desc_re = desc_re
|
| 577 |
|
| 578 | def __call__(self, i, case):
|
| 579 | return bool(self.desc_re.search(case['desc']))
|
| 580 |
|
| 581 |
|
| 582 | # ANSI color constants
|
| 583 | _RESET = '\033[0;0m'
|
| 584 | _BOLD = '\033[1m'
|
| 585 |
|
| 586 | _RED = '\033[31m'
|
| 587 | _GREEN = '\033[32m'
|
| 588 | _YELLOW = '\033[33m'
|
| 589 |
|
| 590 |
|
| 591 | COLOR_FAIL = ''.join([_RED, _BOLD, 'FAIL', _RESET])
|
| 592 | COLOR_BUG = ''.join([_YELLOW, _BOLD, 'BUG', _RESET])
|
| 593 | COLOR_NI = ''.join([_YELLOW, _BOLD, 'N-I', _RESET])
|
| 594 | COLOR_OK = ''.join([_YELLOW, _BOLD, 'ok', _RESET])
|
| 595 | COLOR_PASS = ''.join([_GREEN, _BOLD, 'pass', _RESET])
|
| 596 |
|
| 597 |
|
| 598 | ANSI_CELLS = {
|
| 599 | Result.FAIL: COLOR_FAIL,
|
| 600 | Result.BUG: COLOR_BUG,
|
| 601 | Result.NI: COLOR_NI,
|
| 602 | Result.OK: COLOR_OK,
|
| 603 | Result.PASS: COLOR_PASS,
|
| 604 | }
|
| 605 |
|
| 606 | HTML_CELLS = {
|
| 607 | Result.FAIL: '<td class="fail">FAIL',
|
| 608 | Result.BUG: '<td class="bug">BUG',
|
| 609 | Result.NI: '<td class="n-i">N-I',
|
| 610 | Result.OK: '<td class="ok">ok',
|
| 611 | Result.PASS: '<td class="pass">pass',
|
| 612 | }
|
| 613 |
|
| 614 |
|
| 615 | class ColorOutput(object):
|
| 616 |
|
| 617 | def __init__(self, f, verbose):
|
| 618 | self.f = f
|
| 619 | self.verbose = verbose
|
| 620 | self.details = []
|
| 621 |
|
| 622 | def AddDetails(self, entry):
|
| 623 | self.details.append(entry)
|
| 624 |
|
| 625 | def BeginCases(self, test_file):
|
| 626 | self.f.write('%s\n' % test_file)
|
| 627 |
|
| 628 | def WriteHeader(self, shells):
|
| 629 | self.f.write(_BOLD)
|
| 630 | self.f.write('case\tline\t') # for line number and test number
|
| 631 | for sh_label, _ in shells:
|
| 632 | self.f.write(sh_label)
|
| 633 | self.f.write('\t')
|
| 634 | self.f.write(_RESET)
|
| 635 | self.f.write('\n')
|
| 636 |
|
| 637 | def WriteRow(self, i, line_num, row, desc):
|
| 638 | self.f.write('%3d\t%3d\t' % (i, line_num))
|
| 639 |
|
| 640 | for result in row:
|
| 641 | c = ANSI_CELLS[result]
|
| 642 | self.f.write(c)
|
| 643 | self.f.write('\t')
|
| 644 |
|
| 645 | self.f.write(desc)
|
| 646 | self.f.write('\n')
|
| 647 |
|
| 648 | if self.verbose:
|
| 649 | self._WriteDetailsAsText(self.details)
|
| 650 | self.details = []
|
| 651 |
|
| 652 | def _WriteDetailsAsText(self, details):
|
| 653 | for case_index, shell, stdout, stderr, messages in details:
|
| 654 | print('case: %d' % case_index, file=self.f)
|
| 655 | for m in messages:
|
| 656 | print(m, file=self.f)
|
| 657 | print('%s stdout:' % shell, file=self.f)
|
| 658 | try:
|
| 659 | print(stdout.decode('utf-8'), file=self.f)
|
| 660 | except UnicodeDecodeError:
|
| 661 | print(stdout, file=self.f)
|
| 662 | print('%s stderr:' % shell, file=self.f)
|
| 663 | try:
|
| 664 | print(stderr.decode('utf-8'), file=self.f)
|
| 665 | except UnicodeDecodeError:
|
| 666 | print(stderr, file=self.f)
|
| 667 | print('', file=self.f)
|
| 668 |
|
| 669 | def _WriteStats(self, stats):
|
| 670 | self.f.write(
|
| 671 | '%(num_passed)d passed, %(num_ok)d ok, '
|
| 672 | '%(num_ni)d known unimplemented, %(num_bug)d known bugs, '
|
| 673 | '%(num_failed)d failed, %(num_skipped)d skipped\n' % stats)
|
| 674 |
|
| 675 | def EndCases(self, stats):
|
| 676 | self._WriteStats(stats)
|
| 677 |
|
| 678 |
|
| 679 | class AnsiOutput(ColorOutput):
|
| 680 | pass
|
| 681 |
|
| 682 |
|
| 683 | class HtmlOutput(ColorOutput):
|
| 684 |
|
| 685 | def __init__(self, f, verbose, spec_name, sh_labels, cases):
|
| 686 | ColorOutput.__init__(self, f, verbose)
|
| 687 | self.spec_name = spec_name
|
| 688 | self.sh_labels = sh_labels # saved from header
|
| 689 | self.cases = cases # for linking to code
|
| 690 |
|
| 691 | def _SourceLink(self, line_num, desc):
|
| 692 | return '<a href="%s.test.html#L%d">%s</a>' % (
|
| 693 | self.spec_name, line_num, cgi.escape(desc))
|
| 694 |
|
| 695 | def BeginCases(self, test_file):
|
| 696 | self.f.write('''\
|
| 697 | <!DOCTYPE html>
|
| 698 | <html>
|
| 699 | <head>
|
| 700 | <link href="../../web/spec-tests.css" rel="stylesheet">
|
| 701 | </head>
|
| 702 | <body>
|
| 703 | <p id="home-link">
|
| 704 | <a href=".">spec test index</a>
|
| 705 | /
|
| 706 | <a href="/">oilshell.org</a>
|
| 707 | </p>
|
| 708 | <h1>Results for %s</h1>
|
| 709 | <table>
|
| 710 | ''' % test_file)
|
| 711 |
|
| 712 | def EndCases(self, stats):
|
| 713 | self.f.write('</table>\n')
|
| 714 | self.f.write('<p>')
|
| 715 | self._WriteStats(stats)
|
| 716 | self.f.write('</p>')
|
| 717 |
|
| 718 | if self.details:
|
| 719 | self._WriteDetails()
|
| 720 |
|
| 721 | self.f.write('</body></html>')
|
| 722 |
|
| 723 | def _WriteDetails(self):
|
| 724 | self.f.write("<h2>Details on runs that didn't PASS</h2>")
|
| 725 | self.f.write('<table id="details">')
|
| 726 |
|
| 727 | for case_index, sh_label, stdout, stderr, messages in self.details:
|
| 728 | self.f.write('<tr>')
|
| 729 | self.f.write('<td><a name="details-%s-%s"></a><b>%s</b></td>' % (
|
| 730 | case_index, sh_label, sh_label))
|
| 731 |
|
| 732 | self.f.write('<td>')
|
| 733 |
|
| 734 | # Write description and link to the code
|
| 735 | case = self.cases[case_index]
|
| 736 | line_num = case['line_num']
|
| 737 | desc = case['desc']
|
| 738 | self.f.write('%d ' % case_index)
|
| 739 | self.f.write(self._SourceLink(line_num, desc))
|
| 740 | self.f.write('<br/><br/>\n')
|
| 741 |
|
| 742 | for m in messages:
|
| 743 | self.f.write('<span class="assertion">%s</span><br/>\n' % cgi.escape(m))
|
| 744 | if messages:
|
| 745 | self.f.write('<br/>\n')
|
| 746 |
|
| 747 | def _WriteRaw(s):
|
| 748 | self.f.write('<pre>')
|
| 749 | # We output utf-8-encoded HTML. If we get invalid utf-8 as stdout
|
| 750 | # (which is very possible), then show the ASCII repr().
|
| 751 | try:
|
| 752 | s.decode('utf-8')
|
| 753 | except UnicodeDecodeError:
|
| 754 | valid_utf8 = repr(s) # ASCII representation
|
| 755 | else:
|
| 756 | valid_utf8 = s
|
| 757 | self.f.write(cgi.escape(valid_utf8))
|
| 758 | self.f.write('</pre>')
|
| 759 |
|
| 760 | self.f.write('<i>stdout:</i> <br/>\n')
|
| 761 | _WriteRaw(stdout)
|
| 762 |
|
| 763 | self.f.write('<i>stderr:</i> <br/>\n')
|
| 764 | _WriteRaw(stderr)
|
| 765 |
|
| 766 | self.f.write('</td>')
|
| 767 | self.f.write('</tr>')
|
| 768 |
|
| 769 | self.f.write('</table>')
|
| 770 |
|
| 771 | def WriteHeader(self, shells):
|
| 772 | # TODO: Use oil template language for this...
|
| 773 | self.f.write('''
|
| 774 | <thead>
|
| 775 | <tr>
|
| 776 | ''')
|
| 777 |
|
| 778 | columns = ['case'] + [sh_label for sh_label, _ in shells]
|
| 779 | for c in columns:
|
| 780 | self.f.write('<td>%s</td>' % c)
|
| 781 | self.f.write('<td class="case-desc">description</td>')
|
| 782 |
|
| 783 | self.f.write('''
|
| 784 | </tr>
|
| 785 | </thead>
|
| 786 | ''')
|
| 787 |
|
| 788 | def WriteRow(self, i, line_num, row, desc):
|
| 789 | self.f.write('<tr>')
|
| 790 | self.f.write('<td>%3d</td>' % i)
|
| 791 |
|
| 792 | non_passing = False
|
| 793 |
|
| 794 | for result in row:
|
| 795 | c = HTML_CELLS[result]
|
| 796 | if result != Result.PASS:
|
| 797 | non_passing = True
|
| 798 |
|
| 799 | self.f.write(c)
|
| 800 | self.f.write('</td>')
|
| 801 | self.f.write('\t')
|
| 802 |
|
| 803 | self.f.write('<td class="case-desc">')
|
| 804 | self.f.write(self._SourceLink(line_num, desc))
|
| 805 | self.f.write('</td>')
|
| 806 | self.f.write('</tr>\n')
|
| 807 |
|
| 808 | # Show row with details link.
|
| 809 | if non_passing:
|
| 810 | self.f.write('<tr>')
|
| 811 | self.f.write('<td class="details-row"></td>') # for the number
|
| 812 |
|
| 813 | for col_index, result in enumerate(row):
|
| 814 | self.f.write('<td class="details-row">')
|
| 815 | if result != Result.PASS:
|
| 816 | sh_label = self.sh_labels[col_index]
|
| 817 | self.f.write('<a href="#details-%s-%s">details</a>' % (i, sh_label))
|
| 818 | self.f.write('</td>')
|
| 819 |
|
| 820 | self.f.write('<td class="details-row"></td>') # for the description
|
| 821 | self.f.write('</tr>\n')
|
| 822 |
|
| 823 |
|
| 824 | def Options():
|
| 825 | """Returns an option parser instance."""
|
| 826 | p = optparse.OptionParser('sh_spec.py [options] TEST_FILE shell...')
|
| 827 | p.add_option(
|
| 828 | '-v', '--verbose', dest='verbose', action='store_true', default=False,
|
| 829 | help='Show details about test execution')
|
| 830 | p.add_option(
|
| 831 | '--range', dest='range', default=None,
|
| 832 | help='Execute only a given test range, e.g. 5-10, 5-, -10, or 5')
|
| 833 | p.add_option(
|
| 834 | '--regex', dest='regex', default=None,
|
| 835 | help='Execute only tests whose description matches a given regex '
|
| 836 | '(case-insensitive)')
|
| 837 | p.add_option(
|
| 838 | '--list', dest='do_list', action='store_true', default=None,
|
| 839 | help='Just list tests')
|
| 840 | p.add_option(
|
| 841 | '--format', dest='format', choices=['ansi', 'html'], default='ansi',
|
| 842 | help="Output format (default 'ansi')")
|
| 843 | p.add_option(
|
| 844 | '--stats-file', dest='stats_file', default=None,
|
| 845 | help="File to write stats to")
|
| 846 | p.add_option(
|
| 847 | '--stats-template', dest='stats_template', default='',
|
| 848 | help="Python format string for stats")
|
| 849 | p.add_option(
|
| 850 | '--osh-failures-allowed', dest='osh_failures_allowed', type='int',
|
| 851 | default=0, help="Allow this number of osh failures")
|
| 852 | p.add_option(
|
| 853 | '--path-env', dest='path_env', default='',
|
| 854 | help="The full PATH, for finding binaries used in tests.")
|
| 855 | p.add_option(
|
| 856 | '--tmp-env', dest='tmp_env', default='',
|
| 857 | help="A temporary directory that the tests can use.")
|
| 858 |
|
| 859 | return p
|
| 860 |
|
| 861 |
|
| 862 | def main(argv):
|
| 863 | # First check if bash is polluting the environment. Tests rely on the
|
| 864 | # environment.
|
| 865 | v = os.getenv('RANDOM')
|
| 866 | if v is not None:
|
| 867 | raise AssertionError('got $RANDOM = %s' % v)
|
| 868 | v = os.getenv('PPID')
|
| 869 | if v is not None:
|
| 870 | raise AssertionError('got $PPID = %s' % v)
|
| 871 |
|
| 872 | o = Options()
|
| 873 | (opts, argv) = o.parse_args(argv)
|
| 874 |
|
| 875 | try:
|
| 876 | test_file = argv[1]
|
| 877 | except IndexError:
|
| 878 | o.print_usage()
|
| 879 | return 1
|
| 880 |
|
| 881 | shells = argv[2:]
|
| 882 |
|
| 883 | shell_pairs = []
|
| 884 | saw_osh = False
|
| 885 | for path in shells:
|
| 886 | name, _ = os.path.splitext(path)
|
| 887 | label = os.path.basename(name)
|
| 888 | if label == 'osh':
|
| 889 | if saw_osh:
|
| 890 | label = 'osh_ALT' # distinct label
|
| 891 | else:
|
| 892 | saw_osh = True
|
| 893 | shell_pairs.append((label, path))
|
| 894 |
|
| 895 | with open(test_file) as f:
|
| 896 | tokens = Tokenizer(LineIter(f))
|
| 897 | cases = ParseTestFile(tokens)
|
| 898 |
|
| 899 | # List test cases and return
|
| 900 | if opts.do_list:
|
| 901 | for i, case in enumerate(cases):
|
| 902 | if opts.verbose: # print the raw dictionary for debugging
|
| 903 | print(pprint.pformat(case))
|
| 904 | else:
|
| 905 | print('%d\t%s' % (i, case['desc']))
|
| 906 | return
|
| 907 |
|
| 908 | if opts.range:
|
| 909 | begin, end = ParseRange(opts.range)
|
| 910 | case_predicate = RangePredicate(begin, end)
|
| 911 | elif opts.regex:
|
| 912 | desc_re = re.compile(opts.regex, re.IGNORECASE)
|
| 913 | case_predicate = RegexPredicate(desc_re)
|
| 914 | else:
|
| 915 | case_predicate = lambda i, case: True
|
| 916 |
|
| 917 | # Set up output style. Also see asdl/format.py
|
| 918 | if opts.format == 'ansi':
|
| 919 | out = AnsiOutput(sys.stdout, opts.verbose)
|
| 920 | elif opts.format == 'html':
|
| 921 | spec_name = os.path.basename(test_file)
|
| 922 | spec_name = spec_name.split('.')[0]
|
| 923 |
|
| 924 | sh_labels = [label for label, _ in shell_pairs]
|
| 925 |
|
| 926 | out = HtmlOutput(sys.stdout, opts.verbose, spec_name, sh_labels, cases)
|
| 927 | else:
|
| 928 | raise AssertionError
|
| 929 |
|
| 930 | out.BeginCases(os.path.basename(test_file))
|
| 931 |
|
| 932 | if not opts.tmp_env:
|
| 933 | raise RuntimeError('--tmp-env required')
|
| 934 | if not opts.path_env:
|
| 935 | raise RuntimeError('--path-env required')
|
| 936 | env = {
|
| 937 | 'TMP': os.path.normpath(opts.tmp_env), # no .. or .
|
| 938 | 'PATH': opts.path_env,
|
| 939 | # Copied from my own environment. For now, we want to test bash and other
|
| 940 | # shells in utf-8 mode.
|
| 941 | 'LANG': 'en_US.UTF-8',
|
| 942 | }
|
| 943 | stats = RunCases(cases, case_predicate, shell_pairs, env, out)
|
| 944 | out.EndCases(stats)
|
| 945 |
|
| 946 | stats['osh_failures_allowed'] = opts.osh_failures_allowed
|
| 947 | if opts.stats_file:
|
| 948 | with open(opts.stats_file, 'w') as f:
|
| 949 | f.write(opts.stats_template % stats)
|
| 950 | f.write('\n') # bash 'read' requires a newline
|
| 951 |
|
| 952 | if stats['num_failed'] == 0:
|
| 953 | return 0
|
| 954 |
|
| 955 | allowed = opts.osh_failures_allowed
|
| 956 | all_count = stats['num_failed']
|
| 957 | osh_count = stats['osh_num_failed']
|
| 958 | if allowed == 0:
|
| 959 | log('')
|
| 960 | log('FATAL: %d tests failed (%d osh failures)', all_count, osh_count)
|
| 961 | log('')
|
| 962 | else:
|
| 963 | # If we got EXACTLY the allowed number of failures, exit 0.
|
| 964 | if allowed == all_count and all_count == osh_count:
|
| 965 | log('note: Got %d allowed osh failures (exit with code 0)', allowed)
|
| 966 | return 0
|
| 967 | else:
|
| 968 | log('')
|
| 969 | log('FATAL: Got %d failures (%d osh failures), but %d are allowed',
|
| 970 | all_count, osh_count, allowed)
|
| 971 | log('')
|
| 972 |
|
| 973 | return 1
|
| 974 |
|
| 975 |
|
| 976 | if __name__ == '__main__':
|
| 977 | try:
|
| 978 | sys.exit(main(sys.argv))
|
| 979 | except RuntimeError as e:
|
| 980 | print('FATAL: %s' % e, file=sys.stderr)
|
| 981 | sys.exit(1)
|