| 1 | #!/usr/bin/env python
|
| 2 | # Copyright 2016 Andy Chu. All rights reserved.
|
| 3 | # Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 | # you may not use this file except in compliance with the License.
|
| 5 | # You may obtain a copy of the License at
|
| 6 | #
|
| 7 | # http://www.apache.org/licenses/LICENSE-2.0
|
| 8 | from __future__ import print_function
|
| 9 | """
|
| 10 | libc_test.py: Tests for libc.py
|
| 11 | """
|
| 12 |
|
| 13 | import unittest
|
| 14 |
|
| 15 | from osh.meta import Id, IdInstance, types
|
| 16 |
|
| 17 | import fastlex # module under test
|
| 18 |
|
| 19 | lex_mode_e = types.lex_mode_e
|
| 20 |
|
| 21 |
|
| 22 | def MatchToken(lex_mode, line, start_pos):
|
| 23 | tok_type, end_pos = fastlex.MatchToken(lex_mode.enum_id, line, start_pos)
|
| 24 | return IdInstance(tok_type), end_pos
|
| 25 |
|
| 26 |
|
| 27 | def TokenizeLineOuter(line):
|
| 28 | start_pos = 0
|
| 29 | while True:
|
| 30 | tok_type, end_pos = MatchToken(lex_mode_e.OUTER, line, start_pos)
|
| 31 | tok_val = line[start_pos:end_pos]
|
| 32 | print('TOK: %s %r\n' % (tok_type, tok_val))
|
| 33 | start_pos = end_pos
|
| 34 |
|
| 35 | if tok_type == Id.Eol_Tok:
|
| 36 | break
|
| 37 |
|
| 38 |
|
| 39 | class LexTest(unittest.TestCase):
|
| 40 |
|
| 41 | def testMatchToken(self):
|
| 42 | print(dir(fastlex))
|
| 43 | print(MatchToken(lex_mode_e.COMMENT, 'line', 3))
|
| 44 | print()
|
| 45 |
|
| 46 | # Need to be able to pass NUL bytes for EOF.
|
| 47 | line = 'end of line\n'
|
| 48 | TokenizeLineOuter(line)
|
| 49 | line = 'end of file\0'
|
| 50 | TokenizeLineOuter(line)
|
| 51 |
|
| 52 | def testOutOfBounds(self):
|
| 53 | print(MatchToken(lex_mode_e.OUTER, 'line', 3))
|
| 54 | # It's an error to point to the end of the buffer! Have to be one behind
|
| 55 | # it.
|
| 56 | return
|
| 57 | print(MatchToken(lex_mode_e.OUTER, 'line', 4))
|
| 58 | print(MatchToken(lex_mode_e.OUTER, 'line', 5))
|
| 59 |
|
| 60 | def testBug(self):
|
| 61 | code_str = '-n'
|
| 62 | expected = Id.BoolUnary_n
|
| 63 |
|
| 64 | tok_type, end_pos = MatchToken(lex_mode_e.DBRACKET, code_str, 0)
|
| 65 | print('---', 'expected', expected.enum_value, 'got', tok_type.enum_value)
|
| 66 |
|
| 67 | self.assertEqual(expected, tok_type)
|
| 68 |
|
| 69 |
|
| 70 | if __name__ == '__main__':
|
| 71 | unittest.main()
|