1 | #!/usr/bin/env python2
|
2 | from __future__ import print_function
|
3 |
|
4 | import unittest
|
5 |
|
6 | from _devbuild.gen.syntax_asdl import Id, Id_str
|
7 | from core import error
|
8 | from data_lang import j8
|
9 | from mycpp.mylib import log
|
10 |
|
11 |
|
12 | class FunctionsTest(unittest.TestCase):
|
13 |
|
14 | def testUtf8Encode(self):
|
15 | CASES = [
|
16 | (u'\u0065'.encode('utf-8'), 0x0065),
|
17 | (u'\u0100'.encode('utf-8'), 0x0100),
|
18 | (u'\u1234'.encode('utf-8'), 0x1234),
|
19 | (u'\U00020000'.encode('utf-8'), 0x00020000),
|
20 | # Out of range is checked elsewhere
|
21 | #('\xef\xbf\xbd', 0x10020000),
|
22 | ]
|
23 |
|
24 | for expected, code_point in CASES:
|
25 | print('')
|
26 | print('Utf8Encode case %r %r' % (expected, code_point))
|
27 | self.assertEqual(expected, j8.Utf8Encode(code_point))
|
28 |
|
29 |
|
30 | def _PrintTokens(lex):
|
31 | log('---')
|
32 | log('J8 Case %r', lex.s)
|
33 | pos = 0
|
34 | while True:
|
35 | id_, end_pos, decoded = lex.Next()
|
36 | val = lex.s[pos:end_pos]
|
37 | d = repr(decoded) if decoded is not None else '-'
|
38 | log(' %20s %15r %15s', Id_str(id_), val, d)
|
39 | if id_ == Id.Eol_Tok:
|
40 | break
|
41 | pos = end_pos
|
42 | log('')
|
43 |
|
44 |
|
45 | class Nil8Test(unittest.TestCase):
|
46 |
|
47 | def testNil8Errors(self):
|
48 | cases = [
|
49 | #'()',
|
50 | #'(42)',
|
51 | #'(:)',
|
52 | # extra input
|
53 | '(command.Simple))',
|
54 | '(',
|
55 | '(obj.field',
|
56 | '(obj.',
|
57 |
|
58 | # Expected a value afterward
|
59 | '(obj.)',
|
60 | '(obj.+',
|
61 | '(obj.+)',
|
62 | '(obj.[)',
|
63 | ]
|
64 | for s in cases:
|
65 | p = j8.Nil8Parser(s, True)
|
66 | try:
|
67 | obj = p.ParseNil8()
|
68 | except error.Decode as e:
|
69 | print(e)
|
70 | else:
|
71 | self.fail('Expected error.Decode when parsing %r' % s)
|
72 |
|
73 | def testNil8Operator(self):
|
74 | # Should be equivalent!
|
75 |
|
76 | cases = [
|
77 | # These are equivalent, note that parens like (key: "value") add another layer
|
78 | ('(: key "value")', 'key: "value"'),
|
79 | ('((: k "v") (: k2 "v2"))', '(k: "v" k2: "v2")'),
|
80 | ('(@ "str" x123)', '"str" @ x123'),
|
81 | ('((! a b) c)', '( a ! b c)'),
|
82 | ('(c (! a b))', '( c a ! b )'),
|
83 | ('(. (. obj field1) field2)', 'obj.field1.field2'),
|
84 | ('((-> obj method) (. obj field))', '(obj->method obj.field1)'),
|
85 | ]
|
86 | for prefix, infix in cases:
|
87 | print()
|
88 | print('PREFIX %s' % prefix)
|
89 | p = j8.Nil8Parser(prefix, True)
|
90 | obj1 = p.ParseNil8()
|
91 | print(obj1)
|
92 | log('len %d', len(obj1.items))
|
93 |
|
94 | print()
|
95 | print('INFIX %s' % infix)
|
96 | p = j8.Nil8Parser(infix, True)
|
97 | obj2 = p.ParseNil8()
|
98 | print(obj2)
|
99 | log('len %d', len(obj2.items))
|
100 |
|
101 | self.assertEqual(obj1.tag(), obj2.tag(),
|
102 | '%s != %s' % (obj1.tag(), obj2.tag()))
|
103 | self.assertEqual(len(obj1.items), len(obj2.items))
|
104 |
|
105 | def testNil8(self):
|
106 | cases = [
|
107 | '(unquoted)',
|
108 | '(command.Simple)',
|
109 | '(f x)',
|
110 | '(f 42 "hi")',
|
111 | '((-> obj method) (. obj field))',
|
112 |
|
113 | # address
|
114 | '(@ x123 (Token "foo"))',
|
115 | '(: key "value")',
|
116 | '(. x123)', # dereference, could be @
|
117 |
|
118 | #'(Token "foo") @ x123',
|
119 |
|
120 | # TODO: parse like infix
|
121 | '(Token key:"value" k2:"v2")',
|
122 |
|
123 | # Should be parsed like infix operator
|
124 | '(key !x123)',
|
125 | '(<-)', # symbol
|
126 | "(<- 1 b'hi')", # any kinds of args
|
127 | "(<- 1 'hi' (f [1 2 3]))", # symbol
|
128 | '[]',
|
129 | '[42]',
|
130 | '[42 43]',
|
131 | '[42 ["a" "b"]]',
|
132 | '42',
|
133 | '"hi"',
|
134 | ]
|
135 | for s in cases:
|
136 | p = j8.Nil8Parser(s, True)
|
137 | obj = p.ParseNil8()
|
138 | print(s)
|
139 | print(' %s' % obj)
|
140 | print()
|
141 |
|
142 |
|
143 | class J8Test(unittest.TestCase):
|
144 |
|
145 | def testJ8(self):
|
146 | s = '{}'
|
147 | p = j8.Parser(s, True)
|
148 | obj = p.ParseValue()
|
149 | print(obj)
|
150 |
|
151 | def testLexerDecoder(self):
|
152 | lex = j8.LexerDecoder(r'{"hi": "bye \n"}', True, 'J8')
|
153 | _PrintTokens(lex)
|
154 |
|
155 | lex = j8.LexerDecoder(r"{u'unicode': b'bytes \y1f \yff'}", True, 'J8')
|
156 | _PrintTokens(lex)
|
157 |
|
158 | lex = j8.LexerDecoder(
|
159 | r'{"mu \u03BC \u0001":' + r"b'mu \u{03bc} \u{2620}'", True, 'J8')
|
160 | _PrintTokens(lex)
|
161 |
|
162 | lex = j8.LexerDecoder(r'{"x": [1, 2, 3.14, true]}', True, 'J8')
|
163 | _PrintTokens(lex)
|
164 |
|
165 | lex = j8.LexerDecoder(
|
166 | r'''
|
167 | [
|
168 | 1e9, 1e-9, -1e9, -1E-9, 42
|
169 | ]
|
170 | ''', True, 'J8')
|
171 | _PrintTokens(lex)
|
172 |
|
173 | try:
|
174 | lex = j8.LexerDecoder('"\x01"', True, 'J8')
|
175 | _PrintTokens(lex)
|
176 | except error.Decode as e:
|
177 | print(e)
|
178 | else:
|
179 | self.fail('Expected failure')
|
180 |
|
181 | try:
|
182 | lex = j8.LexerDecoder('"\x1f"', True, 'J8')
|
183 | _PrintTokens(lex)
|
184 | except error.Decode as e:
|
185 | print(e)
|
186 | else:
|
187 | self.fail('Expected failure')
|
188 |
|
189 | def testMoreTokens(self):
|
190 | cases = [
|
191 | 'true # comment',
|
192 | 'truez8 # identifier',
|
193 | 'truez8> # symbol',
|
194 | '(<= 1 3) # less than',
|
195 | # Can allow hex identifiers here like 123
|
196 | '(<- 123 (Token))',
|
197 | '(Node left:(-> 123))',
|
198 | ]
|
199 | for s in cases:
|
200 | lex = j8.LexerDecoder(s, True, 'J8')
|
201 | _PrintTokens(lex)
|
202 |
|
203 | def testErrorMessagePosition(self):
|
204 | lex = j8.LexerDecoder("[ u'hi']", False, 'J8')
|
205 | try:
|
206 | _PrintTokens(lex)
|
207 | except error.Decode as e:
|
208 | print(e)
|
209 | self.assertEquals(2, e.start_pos)
|
210 | self.assertEquals(4, e.end_pos)
|
211 | else:
|
212 | self.fail('Expected failure')
|
213 |
|
214 |
|
215 | class YajlTest(unittest.TestCase):
|
216 | """
|
217 | Note on old tests for YAJL. Differences
|
218 |
|
219 | - It decoded to Python 2 str() type, not unicode()
|
220 | - Bug in emitting \\xff literally, which is not valid JSON
|
221 | - turns out there is a C level option for this
|
222 | """
|
223 | pass
|
224 |
|
225 |
|
226 | if __name__ == '__main__':
|
227 | unittest.main()
|
228 |
|
229 | # vim: sw=4
|