1 | #!/usr/bin/env python
|
2 | """
|
3 | py_meta.py
|
4 |
|
5 | Parse an ASDL file, and generate Python classes using metaprogramming.
|
6 | All objects descends from Obj, which allows them to be dynamically type-checked
|
7 | and serialized. Objects hold type descriptors, which are defined in asdl.py.
|
8 |
|
9 | Usage:
|
10 | from osh import ast_ as ast
|
11 |
|
12 | n1 = ast.ArithVar()
|
13 | n2 = ast.ArrayLiteralPart()
|
14 |
|
15 | API Notes:
|
16 |
|
17 | The Python AST module doesn't make any distinction between simple and compound
|
18 | sum types. (Simple types have no constructors with fields.)
|
19 |
|
20 | C++ has to make this distinction for reasons of representation. It's more
|
21 | efficient to hold an enum value than a pointer to a class with an enum value.
|
22 | In Python I guess that's not quite true.
|
23 |
|
24 | So in order to serialize the correct bytes for C++, our Python metaclass
|
25 | implementation has to differ from what's generated by asdl_c.py. More simply
|
26 | put: an op is Add() and not Add, an instance of a class, not an integer value.
|
27 | """
|
28 |
|
29 | from asdl import asdl_ as asdl
|
30 | from asdl import const
|
31 | from asdl import format as fmt
|
32 | from core import util
|
33 |
|
34 | log = util.log
|
35 |
|
36 |
|
37 | def _CheckType(value, expected_desc):
|
38 | """Is value of type expected_desc?
|
39 |
|
40 | Args:
|
41 | value: Obj or primitive type
|
42 | expected_desc: instance of asdl.Product, asl.Sum, asdl.StrType,
|
43 | asdl.IntType, ArrayType, MaybeType, etc.
|
44 | """
|
45 | if isinstance(expected_desc, asdl.Constructor):
|
46 | # This doesn't make sense because the descriptors are derived from the
|
47 | # declared types. You can declare a field as arith_expr_e but not
|
48 | # ArithBinary.
|
49 | raise AssertionError("Invalid Constructor descriptor")
|
50 |
|
51 | if isinstance(expected_desc, asdl.MaybeType):
|
52 | if value is None:
|
53 | return True
|
54 | return _CheckType(value, expected_desc.desc)
|
55 |
|
56 | if isinstance(expected_desc, asdl.ArrayType):
|
57 | if not isinstance(value, list):
|
58 | return False
|
59 | # Now check all entries
|
60 | for item in value:
|
61 | if not _CheckType(item, expected_desc.desc):
|
62 | return False
|
63 | return True
|
64 |
|
65 | if isinstance(expected_desc, asdl.StrType):
|
66 | return isinstance(value, str)
|
67 |
|
68 | if isinstance(expected_desc, asdl.IntType):
|
69 | return isinstance(value, int)
|
70 |
|
71 | if isinstance(expected_desc, asdl.BoolType):
|
72 | return isinstance(value, bool)
|
73 |
|
74 | if isinstance(expected_desc, asdl.UserType):
|
75 | return isinstance(value, expected_desc.typ)
|
76 |
|
77 | try:
|
78 | actual_desc = value.__class__.ASDL_TYPE
|
79 | except AttributeError:
|
80 | return False # it's not of the right type
|
81 |
|
82 | if isinstance(expected_desc, asdl.Product):
|
83 | return actual_desc is expected_desc
|
84 |
|
85 | if isinstance(expected_desc, asdl.Sum):
|
86 | if asdl.is_simple(expected_desc):
|
87 | return actual_desc is expected_desc
|
88 | else:
|
89 | for cons in expected_desc.types: # It has to be one of the alternatives
|
90 | #log("CHECKING desc %s against %s" % (desc, cons))
|
91 | if actual_desc is cons:
|
92 | return True
|
93 | return False
|
94 |
|
95 | raise AssertionError(
|
96 | 'Invalid descriptor %r: %r' % (expected_desc.__class__, expected_desc))
|
97 |
|
98 |
|
99 | class Obj(object):
|
100 | # NOTE: We're using CAPS for these static fields, since they are constant at
|
101 | # runtime after metaprogramming.
|
102 | ASDL_TYPE = None # Used for type checking
|
103 |
|
104 |
|
105 | class SimpleObj(Obj):
|
106 | """An enum value.
|
107 |
|
108 | Other simple objects: int, str, maybe later a float.
|
109 | """
|
110 | def __init__(self, enum_id, name):
|
111 | self.enum_id = enum_id
|
112 | self.name = name
|
113 |
|
114 | # TODO: Why is __hash__ needed? Otherwise native/fastlex_test.py fails.
|
115 | # util.Enum required it too. I thought that instances would hash by
|
116 | # identity?
|
117 | #
|
118 | # Example:
|
119 | # class bool_arg_type_e(py_meta.SimpleObj):
|
120 | # ASDL_TYPE = TYPE_LOOKUP.ByTypeName('bool_arg_type')
|
121 | # bool_arg_type_e.Undefined = bool_arg_type_e(1, 'Undefined')
|
122 |
|
123 | def __hash__(self):
|
124 | # Could it be the integer self.enum_id?
|
125 | return hash(self.__class__.__name__ + self.name)
|
126 |
|
127 | def __repr__(self):
|
128 | return '<%s %s %s>' % (self.__class__.__name__, self.name, self.enum_id)
|
129 |
|
130 |
|
131 | class CompoundObj(Obj):
|
132 | # TODO: Remove tag?
|
133 | # The tag is always set for constructor types, which are subclasses of sum
|
134 | # types. Never set for product types.
|
135 | tag = None
|
136 |
|
137 | # NOTE: SimpleObj could share this.
|
138 | def __repr__(self):
|
139 | ast_f = fmt.TextOutput(util.Buffer()) # No color by default.
|
140 | tree = fmt.MakeTree(self)
|
141 | fmt.PrintTree(tree, ast_f)
|
142 | s, _ = ast_f.GetRaw()
|
143 | return s
|
144 |
|
145 |
|
146 | class DebugCompoundObj(CompoundObj):
|
147 | """A CompoundObj that does dynamic type checks.
|
148 |
|
149 | Used by MakeTypes().
|
150 | """
|
151 | # Always set for constructor types, which are subclasses of sum types. Never
|
152 | # set for product types.
|
153 | tag = None
|
154 |
|
155 | def __init__(self, *args, **kwargs):
|
156 | # The user must specify ALL required fields or NONE.
|
157 | self._assigned = {f: False for f in self.ASDL_TYPE.GetFieldNames()}
|
158 | self._SetDefaults()
|
159 | if args or kwargs:
|
160 | self._Init(args, kwargs)
|
161 |
|
162 | def _SetDefaults(self):
|
163 | for name, desc in self.ASDL_TYPE.GetFields():
|
164 |
|
165 | if isinstance(desc, asdl.MaybeType):
|
166 | child = desc.desc
|
167 | if isinstance(child, asdl.IntType):
|
168 | value = const.NO_INTEGER
|
169 | elif isinstance(child, asdl.StrType):
|
170 | value = ''
|
171 | else:
|
172 | value = None
|
173 | self.__setattr__(name, value) # Maybe values can be None
|
174 |
|
175 | elif isinstance(desc, asdl.ArrayType):
|
176 | self.__setattr__(name, [])
|
177 |
|
178 | def _Init(self, args, kwargs):
|
179 | field_names = list(self.ASDL_TYPE.GetFieldNames())
|
180 | for i, val in enumerate(args):
|
181 | name = field_names[i]
|
182 | self.__setattr__(name, val)
|
183 |
|
184 | for name, val in kwargs.items():
|
185 | if self._assigned[name]:
|
186 | raise TypeError('Duplicate assignment of field %r' % name)
|
187 | self.__setattr__(name, val)
|
188 |
|
189 | # Disable type checking here
|
190 | #return
|
191 | for name in field_names:
|
192 | if not self._assigned[name]:
|
193 | # If anything was set, then required fields raise an error.
|
194 | raise ValueError("Field %r is required and wasn't initialized" % name)
|
195 |
|
196 | def CheckUnassigned(self):
|
197 | """See if there are unassigned fields, for later encoding.
|
198 |
|
199 | This is currently only used in unit tests.
|
200 | """
|
201 | unassigned = []
|
202 | for name in self.ASDL_TYPE.GetFieldNames():
|
203 | if not self._assigned[name]:
|
204 | desc = self.ASDL_TYPE.LookupFieldType(name)
|
205 | if not isinstance(desc, asdl.MaybeType):
|
206 | unassigned.append(name)
|
207 | if unassigned:
|
208 | raise ValueError("Fields %r were't be assigned" % unassigned)
|
209 |
|
210 | if 1: # Disable type checking here
|
211 | def __setattr__(self, name, value):
|
212 | if name == '_assigned':
|
213 | self.__dict__[name] = value
|
214 | return
|
215 | try:
|
216 | desc = self.ASDL_TYPE.LookupFieldType(name)
|
217 | except KeyError:
|
218 | raise AttributeError('Object of type %r has no attribute %r' %
|
219 | (self.__class__.__name__, name))
|
220 |
|
221 | if not _CheckType(value, desc):
|
222 | raise AssertionError("Field %r should be of type %s, got %r (%s)" %
|
223 | (name, desc, value, value.__class__))
|
224 |
|
225 | self._assigned[name] = True # check this later when encoding
|
226 | self.__dict__[name] = value
|
227 |
|
228 |
|
229 | def MakeTypes(module, root, type_lookup):
|
230 | """
|
231 | Args:
|
232 | module: asdl.Module
|
233 | root: an object/package to add types to
|
234 | """
|
235 | for defn in module.dfns:
|
236 | typ = defn.value
|
237 |
|
238 | #print('TYPE', defn.name, typ)
|
239 | if isinstance(typ, asdl.Sum):
|
240 | sum_type = typ
|
241 | if asdl.is_simple(sum_type):
|
242 | # An object without fields, which can be stored inline.
|
243 |
|
244 | # Create a class called foo_e. Unlike the CompoundObj case, it doesn't
|
245 | # have subtypes. Instead if has attributes foo_e.Bar, which Bar is an
|
246 | # instance of foo_e.
|
247 | #
|
248 | # Problem: This means you have a dichotomy between:
|
249 | # cflow_e.Break vs. cflow_e.Break()
|
250 | # If you add a non-simple type like cflow_e.Return(5), the usage will
|
251 | # change. I haven't run into this problem in practice yet.
|
252 |
|
253 | class_name = defn.name + '_e'
|
254 | class_attr = {'ASDL_TYPE': sum_type} # asdl.Sum
|
255 | cls = type(class_name, (SimpleObj, ), class_attr)
|
256 | setattr(root, class_name, cls)
|
257 |
|
258 | # NOTE: Right now the ASDL_TYPE for for an enum value is the Sum type,
|
259 | # not the Constructor type. We may want to change this if we need
|
260 | # reflection.
|
261 | for i, cons in enumerate(sum_type.types):
|
262 | enum_id = i + 1
|
263 | name = cons.name
|
264 | val = cls(enum_id, cons.name) # Instantiate SimpleObj subtype
|
265 |
|
266 | # Set a static attribute like op_id.Plus, op_id.Minus.
|
267 | setattr(cls, name, val)
|
268 | else:
|
269 | tag_num = {}
|
270 |
|
271 | # e.g. for arith_expr
|
272 | # Should this be arith_expr_t? It is in C++.
|
273 | base_class = type(defn.name, (DebugCompoundObj, ), {})
|
274 | setattr(root, defn.name, base_class)
|
275 |
|
276 | # Make a type and a enum tag for each alternative.
|
277 | for i, cons in enumerate(sum_type.types):
|
278 | tag = i + 1 # zero reserved?
|
279 | tag_num[cons.name] = tag # for enum
|
280 |
|
281 | class_attr = {
|
282 | 'ASDL_TYPE': cons, # asdl.Constructor
|
283 | 'tag': tag, # Does this API change?
|
284 | }
|
285 |
|
286 | cls = type(cons.name, (base_class, ), class_attr)
|
287 | setattr(root, cons.name, cls)
|
288 |
|
289 | # e.g. arith_expr_e.Const == 1
|
290 | enum_name = defn.name + '_e'
|
291 | tag_enum = type(enum_name, (), tag_num)
|
292 | setattr(root, enum_name, tag_enum)
|
293 |
|
294 | elif isinstance(typ, asdl.Product):
|
295 | class_attr = {'ASDL_TYPE': typ}
|
296 | cls = type(defn.name, (DebugCompoundObj, ), class_attr)
|
297 | setattr(root, defn.name, cls)
|
298 |
|
299 | else:
|
300 | raise AssertionError(typ)
|
301 |
|
302 |
|
303 | def AssignTypes(src_module, dest_module):
|
304 | """For generated code."""
|
305 | for name in dir(src_module):
|
306 | if not name.startswith('__'):
|
307 | v = getattr(src_module, name)
|
308 | setattr(dest_module, name, v)
|
309 |
|