1 | #!/usr/bin/env python
|
2 | # Copyright 2016 Andy Chu. All rights reserved.
|
3 | # Licensed under the Apache License, Version 2.0 (the "License");
|
4 | # you may not use this file except in compliance with the License.
|
5 | # You may obtain a copy of the License at
|
6 | #
|
7 | # http://www.apache.org/licenses/LICENSE-2.0
|
8 | from __future__ import print_function
|
9 | """
|
10 | libc_test.py: Tests for libc.py
|
11 | """
|
12 |
|
13 | import unittest
|
14 |
|
15 | from osh.meta import Id, IdInstance, types
|
16 |
|
17 | import fastlex # module under test
|
18 |
|
19 | lex_mode_e = types.lex_mode_e
|
20 |
|
21 |
|
22 | def MatchToken(lex_mode, line, start_pos):
|
23 | tok_type, end_pos = fastlex.MatchToken(lex_mode.enum_id, line, start_pos)
|
24 | return IdInstance(tok_type), end_pos
|
25 |
|
26 |
|
27 | def TokenizeLineOuter(line):
|
28 | start_pos = 0
|
29 | while True:
|
30 | tok_type, end_pos = MatchToken(lex_mode_e.OUTER, line, start_pos)
|
31 | tok_val = line[start_pos:end_pos]
|
32 | print('TOK: %s %r\n' % (tok_type, tok_val))
|
33 | start_pos = end_pos
|
34 |
|
35 | if tok_type == Id.Eol_Tok:
|
36 | break
|
37 |
|
38 |
|
39 | class LexTest(unittest.TestCase):
|
40 |
|
41 | def testMatchToken(self):
|
42 | print(dir(fastlex))
|
43 | print(MatchToken(lex_mode_e.COMMENT, 'line', 3))
|
44 | print()
|
45 |
|
46 | # Need to be able to pass NUL bytes for EOF.
|
47 | line = 'end of line\n'
|
48 | TokenizeLineOuter(line)
|
49 | line = 'end of file\0'
|
50 | TokenizeLineOuter(line)
|
51 |
|
52 | def testOutOfBounds(self):
|
53 | print(MatchToken(lex_mode_e.OUTER, 'line', 3))
|
54 | # It's an error to point to the end of the buffer! Have to be one behind
|
55 | # it.
|
56 | return
|
57 | print(MatchToken(lex_mode_e.OUTER, 'line', 4))
|
58 | print(MatchToken(lex_mode_e.OUTER, 'line', 5))
|
59 |
|
60 | def testBug(self):
|
61 | code_str = '-n'
|
62 | expected = Id.BoolUnary_n
|
63 |
|
64 | tok_type, end_pos = MatchToken(lex_mode_e.DBRACKET, code_str, 0)
|
65 | print('---', 'expected', expected.enum_value, 'got', tok_type.enum_value)
|
66 |
|
67 | self.assertEqual(expected, tok_type)
|
68 |
|
69 |
|
70 | if __name__ == '__main__':
|
71 | unittest.main()
|