OILS / pyext / fastlex_test.py View on Github | oilshell.org

85 lines, 49 significant
1#!/usr/bin/env python2
2# Copyright 2016 Andy Chu. All rights reserved.
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8"""
9fastlex_test.py: Tests for fastlex
10"""
11from __future__ import print_function
12
13import unittest
14
15from mycpp.mylib import log
16from _devbuild.gen.id_kind_asdl import Id
17from _devbuild.gen.types_asdl import lex_mode_e
18
19import fastlex # module under test
20
21_ = log
22
23
24# NOTE: This is just like _MatchOshToken_Fast in frontend/match.py
25def MatchOshToken(lex_mode, line, start_pos):
26 tok_type, end_pos = fastlex.MatchOshToken(lex_mode, line, start_pos)
27 #log('tok_type = %d, id = %s', tok_type, tok_type)
28 return tok_type, end_pos
29
30
31def TokenizeLineOuter(line):
32 start_pos = 0
33 while True:
34 tok_type, end_pos = MatchOshToken(lex_mode_e.ShCommand, line, start_pos)
35 tok_val = line[start_pos:end_pos]
36 print('TOK: %s %r\n' % (tok_type, tok_val))
37 start_pos = end_pos
38
39 if tok_type == Id.Eol_Tok:
40 break
41
42
43class LexTest(unittest.TestCase):
44
45 def testMatchOshToken(self):
46 print(dir(fastlex))
47 print(MatchOshToken(lex_mode_e.Comment, 'line', 3))
48 print()
49
50 # Need to be able to pass NUL bytes for EOF.
51 line = 'end of line\n'
52 TokenizeLineOuter(line)
53 line = 'end of file\0'
54 TokenizeLineOuter(line)
55
56 def testOutOfBounds(self):
57 print(MatchOshToken(lex_mode_e.ShCommand, 'line', 3))
58 # It's an error to point to the end of the buffer! Have to be one behind
59 # it.
60 return
61 print(MatchOshToken(lex_mode_e.ShCommand, 'line', 4))
62 print(MatchOshToken(lex_mode_e.ShCommand, 'line', 5))
63
64 def testBug(self):
65 code_str = '-n'
66 expected = Id.BoolUnary_n
67
68 tok_type, end_pos = MatchOshToken(lex_mode_e.DBracket, code_str, 0)
69 print('--- %s expected, got %s' % (expected, tok_type))
70
71 self.assertEqual(expected, tok_type)
72
73 def testIsValidVarName(self):
74 self.assertEqual(True, fastlex.IsValidVarName('abc'))
75 self.assertEqual(True, fastlex.IsValidVarName('foo_bar'))
76 self.assertEqual(True, fastlex.IsValidVarName('_'))
77
78 self.assertEqual(False, fastlex.IsValidVarName(''))
79 self.assertEqual(False, fastlex.IsValidVarName('-x'))
80 self.assertEqual(False, fastlex.IsValidVarName('x-'))
81 self.assertEqual(False, fastlex.IsValidVarName('var_name-foo'))
82
83
84if __name__ == '__main__':
85 unittest.main()