| 1 | #!/usr/bin/env python2
|
| 2 | # Copyright 2016 Andy Chu. All rights reserved.
|
| 3 | # Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 | # you may not use this file except in compliance with the License.
|
| 5 | # You may obtain a copy of the License at
|
| 6 | #
|
| 7 | # http://www.apache.org/licenses/LICENSE-2.0
|
| 8 | """
|
| 9 | fastlex_test.py: Tests for fastlex
|
| 10 | """
|
| 11 | from __future__ import print_function
|
| 12 |
|
| 13 | import unittest
|
| 14 |
|
| 15 | from mycpp.mylib import log
|
| 16 | from _devbuild.gen.id_kind_asdl import Id
|
| 17 | from _devbuild.gen.types_asdl import lex_mode_e
|
| 18 |
|
| 19 | import fastlex # module under test
|
| 20 |
|
| 21 | _ = log
|
| 22 |
|
| 23 |
|
| 24 | # NOTE: This is just like _MatchOshToken_Fast in frontend/match.py
|
| 25 | def MatchOshToken(lex_mode, line, start_pos):
|
| 26 | tok_type, end_pos = fastlex.MatchOshToken(lex_mode, line, start_pos)
|
| 27 | #log('tok_type = %d, id = %s', tok_type, tok_type)
|
| 28 | return tok_type, end_pos
|
| 29 |
|
| 30 |
|
| 31 | def TokenizeLineOuter(line):
|
| 32 | start_pos = 0
|
| 33 | while True:
|
| 34 | tok_type, end_pos = MatchOshToken(lex_mode_e.ShCommand, line, start_pos)
|
| 35 | tok_val = line[start_pos:end_pos]
|
| 36 | print('TOK: %s %r\n' % (tok_type, tok_val))
|
| 37 | start_pos = end_pos
|
| 38 |
|
| 39 | if tok_type == Id.Eol_Tok:
|
| 40 | break
|
| 41 |
|
| 42 |
|
| 43 | class LexTest(unittest.TestCase):
|
| 44 |
|
| 45 | def testMatchOshToken(self):
|
| 46 | print(dir(fastlex))
|
| 47 | print(MatchOshToken(lex_mode_e.Comment, 'line', 3))
|
| 48 | print()
|
| 49 |
|
| 50 | # Need to be able to pass NUL bytes for EOF.
|
| 51 | line = 'end of line\n'
|
| 52 | TokenizeLineOuter(line)
|
| 53 | line = 'end of file\0'
|
| 54 | TokenizeLineOuter(line)
|
| 55 |
|
| 56 | def testOutOfBounds(self):
|
| 57 | print(MatchOshToken(lex_mode_e.ShCommand, 'line', 3))
|
| 58 | # It's an error to point to the end of the buffer! Have to be one behind
|
| 59 | # it.
|
| 60 | return
|
| 61 | print(MatchOshToken(lex_mode_e.ShCommand, 'line', 4))
|
| 62 | print(MatchOshToken(lex_mode_e.ShCommand, 'line', 5))
|
| 63 |
|
| 64 | def testBug(self):
|
| 65 | code_str = '-n'
|
| 66 | expected = Id.BoolUnary_n
|
| 67 |
|
| 68 | tok_type, end_pos = MatchOshToken(lex_mode_e.DBracket, code_str, 0)
|
| 69 | print('--- %s expected, got %s' % (expected, tok_type))
|
| 70 |
|
| 71 | self.assertEqual(expected, tok_type)
|
| 72 |
|
| 73 | def testIsValidVarName(self):
|
| 74 | self.assertEqual(True, fastlex.IsValidVarName('abc'))
|
| 75 | self.assertEqual(True, fastlex.IsValidVarName('foo_bar'))
|
| 76 | self.assertEqual(True, fastlex.IsValidVarName('_'))
|
| 77 |
|
| 78 | self.assertEqual(False, fastlex.IsValidVarName(''))
|
| 79 | self.assertEqual(False, fastlex.IsValidVarName('-x'))
|
| 80 | self.assertEqual(False, fastlex.IsValidVarName('x-'))
|
| 81 | self.assertEqual(False, fastlex.IsValidVarName('var_name-foo'))
|
| 82 |
|
| 83 |
|
| 84 | if __name__ == '__main__':
|
| 85 | unittest.main()
|