OILS / data_lang / j8_test.py View on Github | oilshell.org

229 lines, 164 significant
1#!/usr/bin/env python2
2from __future__ import print_function
3
4import unittest
5
6from _devbuild.gen.syntax_asdl import Id, Id_str
7from core import error
8from data_lang import j8
9from mycpp.mylib import log
10
11
12class FunctionsTest(unittest.TestCase):
13
14 def testUtf8Encode(self):
15 CASES = [
16 (u'\u0065'.encode('utf-8'), 0x0065),
17 (u'\u0100'.encode('utf-8'), 0x0100),
18 (u'\u1234'.encode('utf-8'), 0x1234),
19 (u'\U00020000'.encode('utf-8'), 0x00020000),
20 # Out of range is checked elsewhere
21 #('\xef\xbf\xbd', 0x10020000),
22 ]
23
24 for expected, code_point in CASES:
25 print('')
26 print('Utf8Encode case %r %r' % (expected, code_point))
27 self.assertEqual(expected, j8.Utf8Encode(code_point))
28
29
30def _PrintTokens(lex):
31 log('---')
32 log('J8 Case %r', lex.s)
33 pos = 0
34 while True:
35 id_, end_pos, decoded = lex.Next()
36 val = lex.s[pos:end_pos]
37 d = repr(decoded) if decoded is not None else '-'
38 log(' %20s %15r %15s', Id_str(id_), val, d)
39 if id_ == Id.Eol_Tok:
40 break
41 pos = end_pos
42 log('')
43
44
45class Nil8Test(unittest.TestCase):
46
47 def testNil8Errors(self):
48 cases = [
49 #'()',
50 #'(42)',
51 #'(:)',
52 # extra input
53 '(command.Simple))',
54 '(',
55 '(obj.field',
56 '(obj.',
57
58 # Expected a value afterward
59 '(obj.)',
60 '(obj.+',
61 '(obj.+)',
62 '(obj.[)',
63 ]
64 for s in cases:
65 p = j8.Nil8Parser(s, True)
66 try:
67 obj = p.ParseNil8()
68 except error.Decode as e:
69 print(e)
70 else:
71 self.fail('Expected error.Decode when parsing %r' % s)
72
73 def testNil8Operator(self):
74 # Should be equivalent!
75
76 cases = [
77 # These are equivalent, note that parens like (key: "value") add another layer
78 ('(: key "value")', 'key: "value"'),
79 ('((: k "v") (: k2 "v2"))', '(k: "v" k2: "v2")'),
80 ('(@ "str" x123)', '"str" @ x123'),
81 ('((! a b) c)', '( a ! b c)'),
82 ('(c (! a b))', '( c a ! b )'),
83 ('(. (. obj field1) field2)', 'obj.field1.field2'),
84 ('((-> obj method) (. obj field))', '(obj->method obj.field1)'),
85 ]
86 for prefix, infix in cases:
87 print()
88 print('PREFIX %s' % prefix)
89 p = j8.Nil8Parser(prefix, True)
90 obj1 = p.ParseNil8()
91 print(obj1)
92 log('len %d', len(obj1.items))
93
94 print()
95 print('INFIX %s' % infix)
96 p = j8.Nil8Parser(infix, True)
97 obj2 = p.ParseNil8()
98 print(obj2)
99 log('len %d', len(obj2.items))
100
101 self.assertEqual(obj1.tag(), obj2.tag(),
102 '%s != %s' % (obj1.tag(), obj2.tag()))
103 self.assertEqual(len(obj1.items), len(obj2.items))
104
105 def testNil8(self):
106 cases = [
107 '(unquoted)',
108 '(command.Simple)',
109 '(f x)',
110 '(f 42 "hi")',
111 '((-> obj method) (. obj field))',
112
113 # address
114 '(@ x123 (Token "foo"))',
115 '(: key "value")',
116 '(. x123)', # dereference, could be @
117
118 #'(Token "foo") @ x123',
119
120 # TODO: parse like infix
121 '(Token key:"value" k2:"v2")',
122
123 # Should be parsed like infix operator
124 '(key !x123)',
125 '(<-)', # symbol
126 "(<- 1 b'hi')", # any kinds of args
127 "(<- 1 'hi' (f [1 2 3]))", # symbol
128 '[]',
129 '[42]',
130 '[42 43]',
131 '[42 ["a" "b"]]',
132 '42',
133 '"hi"',
134 ]
135 for s in cases:
136 p = j8.Nil8Parser(s, True)
137 obj = p.ParseNil8()
138 print(s)
139 print(' %s' % obj)
140 print()
141
142
143class J8Test(unittest.TestCase):
144
145 def testJ8(self):
146 s = '{}'
147 p = j8.Parser(s, True)
148 obj = p.ParseValue()
149 print(obj)
150
151 def testLexerDecoder(self):
152 lex = j8.LexerDecoder(r'{"hi": "bye \n"}', True, 'J8')
153 _PrintTokens(lex)
154
155 lex = j8.LexerDecoder(r"{u'unicode': b'bytes \y1f \yff'}", True, 'J8')
156 _PrintTokens(lex)
157
158 lex = j8.LexerDecoder(
159 r'{"mu \u03BC \u0001":' + r"b'mu \u{03bc} \u{2620}'", True, 'J8')
160 _PrintTokens(lex)
161
162 lex = j8.LexerDecoder(r'{"x": [1, 2, 3.14, true]}', True, 'J8')
163 _PrintTokens(lex)
164
165 lex = j8.LexerDecoder(
166 r'''
167 [
168 1e9, 1e-9, -1e9, -1E-9, 42
169 ]
170 ''', True, 'J8')
171 _PrintTokens(lex)
172
173 try:
174 lex = j8.LexerDecoder('"\x01"', True, 'J8')
175 _PrintTokens(lex)
176 except error.Decode as e:
177 print(e)
178 else:
179 self.fail('Expected failure')
180
181 try:
182 lex = j8.LexerDecoder('"\x1f"', True, 'J8')
183 _PrintTokens(lex)
184 except error.Decode as e:
185 print(e)
186 else:
187 self.fail('Expected failure')
188
189 def testMoreTokens(self):
190 cases = [
191 'true # comment',
192 'truez8 # identifier',
193 'truez8> # symbol',
194 '(<= 1 3) # less than',
195 # Can allow hex identifiers here like 123
196 '(<- 123 (Token))',
197 '(Node left:(-> 123))',
198 ]
199 for s in cases:
200 lex = j8.LexerDecoder(s, True, 'J8')
201 _PrintTokens(lex)
202
203 def testErrorMessagePosition(self):
204 lex = j8.LexerDecoder("[ u'hi']", False, 'J8')
205 try:
206 _PrintTokens(lex)
207 except error.Decode as e:
208 print(e)
209 self.assertEquals(2, e.start_pos)
210 self.assertEquals(4, e.end_pos)
211 else:
212 self.fail('Expected failure')
213
214
215class YajlTest(unittest.TestCase):
216 """
217 Note on old tests for YAJL. Differences
218
219 - It decoded to Python 2 str() type, not unicode()
220 - Bug in emitting \\xff literally, which is not valid JSON
221 - turns out there is a C level option for this
222 """
223 pass
224
225
226if __name__ == '__main__':
227 unittest.main()
228
229# vim: sw=4