| 1 | #!/usr/bin/env python
|
| 2 | """
|
| 3 | py_meta.py
|
| 4 |
|
| 5 | Parse an ASDL file, and generate Python classes using metaprogramming.
|
| 6 | All objects descends from Obj, which allows them to be dynamically type-checked
|
| 7 | and serialized. Objects hold type descriptors, which are defined in asdl.py.
|
| 8 |
|
| 9 | Usage:
|
| 10 | from osh import ast_ as ast
|
| 11 |
|
| 12 | n1 = ast.ArithVar()
|
| 13 | n2 = ast.ArrayLiteralPart()
|
| 14 |
|
| 15 | API Notes:
|
| 16 |
|
| 17 | The Python AST module doesn't make any distinction between simple and compound
|
| 18 | sum types. (Simple types have no constructors with fields.)
|
| 19 |
|
| 20 | C++ has to make this distinction for reasons of representation. It's more
|
| 21 | efficient to hold an enum value than a pointer to a class with an enum value.
|
| 22 | In Python I guess that's not quite true.
|
| 23 |
|
| 24 | So in order to serialize the correct bytes for C++, our Python metaclass
|
| 25 | implementation has to differ from what's generated by asdl_c.py. More simply
|
| 26 | put: an op is Add() and not Add, an instance of a class, not an integer value.
|
| 27 | """
|
| 28 |
|
| 29 | from asdl import asdl_ as asdl
|
| 30 | from asdl import const
|
| 31 | from asdl import format as fmt
|
| 32 | from core import util
|
| 33 |
|
| 34 | log = util.log
|
| 35 |
|
| 36 |
|
| 37 | def _CheckType(value, expected_desc):
|
| 38 | """Is value of type expected_desc?
|
| 39 |
|
| 40 | Args:
|
| 41 | value: Obj or primitive type
|
| 42 | expected_desc: instance of asdl.Product, asl.Sum, asdl.StrType,
|
| 43 | asdl.IntType, ArrayType, MaybeType, etc.
|
| 44 | """
|
| 45 | if isinstance(expected_desc, asdl.Constructor):
|
| 46 | # This doesn't make sense because the descriptors are derived from the
|
| 47 | # declared types. You can declare a field as arith_expr_e but not
|
| 48 | # ArithBinary.
|
| 49 | raise AssertionError("Invalid Constructor descriptor")
|
| 50 |
|
| 51 | if isinstance(expected_desc, asdl.MaybeType):
|
| 52 | if value is None:
|
| 53 | return True
|
| 54 | return _CheckType(value, expected_desc.desc)
|
| 55 |
|
| 56 | if isinstance(expected_desc, asdl.ArrayType):
|
| 57 | if not isinstance(value, list):
|
| 58 | return False
|
| 59 | # Now check all entries
|
| 60 | for item in value:
|
| 61 | if not _CheckType(item, expected_desc.desc):
|
| 62 | return False
|
| 63 | return True
|
| 64 |
|
| 65 | if isinstance(expected_desc, asdl.StrType):
|
| 66 | return isinstance(value, str)
|
| 67 |
|
| 68 | if isinstance(expected_desc, asdl.IntType):
|
| 69 | return isinstance(value, int)
|
| 70 |
|
| 71 | if isinstance(expected_desc, asdl.BoolType):
|
| 72 | return isinstance(value, bool)
|
| 73 |
|
| 74 | if isinstance(expected_desc, asdl.UserType):
|
| 75 | return isinstance(value, expected_desc.typ)
|
| 76 |
|
| 77 | try:
|
| 78 | actual_desc = value.__class__.ASDL_TYPE
|
| 79 | except AttributeError:
|
| 80 | return False # it's not of the right type
|
| 81 |
|
| 82 | if isinstance(expected_desc, asdl.Product):
|
| 83 | return actual_desc is expected_desc
|
| 84 |
|
| 85 | if isinstance(expected_desc, asdl.Sum):
|
| 86 | if asdl.is_simple(expected_desc):
|
| 87 | return actual_desc is expected_desc
|
| 88 | else:
|
| 89 | for cons in expected_desc.types: # It has to be one of the alternatives
|
| 90 | #log("CHECKING desc %s against %s" % (desc, cons))
|
| 91 | if actual_desc is cons:
|
| 92 | return True
|
| 93 | return False
|
| 94 |
|
| 95 | raise AssertionError(
|
| 96 | 'Invalid descriptor %r: %r' % (expected_desc.__class__, expected_desc))
|
| 97 |
|
| 98 |
|
| 99 | class Obj(object):
|
| 100 | # NOTE: We're using CAPS for these static fields, since they are constant at
|
| 101 | # runtime after metaprogramming.
|
| 102 | ASDL_TYPE = None # Used for type checking
|
| 103 |
|
| 104 |
|
| 105 | class SimpleObj(Obj):
|
| 106 | """An enum value.
|
| 107 |
|
| 108 | Other simple objects: int, str, maybe later a float.
|
| 109 | """
|
| 110 | def __init__(self, enum_id, name):
|
| 111 | self.enum_id = enum_id
|
| 112 | self.name = name
|
| 113 |
|
| 114 | # TODO: Why is __hash__ needed? Otherwise native/fastlex_test.py fails.
|
| 115 | # util.Enum required it too. I thought that instances would hash by
|
| 116 | # identity?
|
| 117 | #
|
| 118 | # Example:
|
| 119 | # class bool_arg_type_e(py_meta.SimpleObj):
|
| 120 | # ASDL_TYPE = TYPE_LOOKUP.ByTypeName('bool_arg_type')
|
| 121 | # bool_arg_type_e.Undefined = bool_arg_type_e(1, 'Undefined')
|
| 122 |
|
| 123 | def __hash__(self):
|
| 124 | # Could it be the integer self.enum_id?
|
| 125 | return hash(self.__class__.__name__ + self.name)
|
| 126 |
|
| 127 | def __repr__(self):
|
| 128 | return '<%s %s %s>' % (self.__class__.__name__, self.name, self.enum_id)
|
| 129 |
|
| 130 |
|
| 131 | class CompoundObj(Obj):
|
| 132 | # TODO: Remove tag?
|
| 133 | # The tag is always set for constructor types, which are subclasses of sum
|
| 134 | # types. Never set for product types.
|
| 135 | tag = None
|
| 136 |
|
| 137 | # NOTE: SimpleObj could share this.
|
| 138 | def __repr__(self):
|
| 139 | ast_f = fmt.TextOutput(util.Buffer()) # No color by default.
|
| 140 | tree = fmt.MakeTree(self)
|
| 141 | fmt.PrintTree(tree, ast_f)
|
| 142 | s, _ = ast_f.GetRaw()
|
| 143 | return s
|
| 144 |
|
| 145 |
|
| 146 | class DebugCompoundObj(CompoundObj):
|
| 147 | """A CompoundObj that does dynamic type checks.
|
| 148 |
|
| 149 | Used by MakeTypes().
|
| 150 | """
|
| 151 | # Always set for constructor types, which are subclasses of sum types. Never
|
| 152 | # set for product types.
|
| 153 | tag = None
|
| 154 |
|
| 155 | def __init__(self, *args, **kwargs):
|
| 156 | # The user must specify ALL required fields or NONE.
|
| 157 | self._assigned = {f: False for f in self.ASDL_TYPE.GetFieldNames()}
|
| 158 | self._SetDefaults()
|
| 159 | if args or kwargs:
|
| 160 | self._Init(args, kwargs)
|
| 161 |
|
| 162 | def _SetDefaults(self):
|
| 163 | for name, desc in self.ASDL_TYPE.GetFields():
|
| 164 |
|
| 165 | if isinstance(desc, asdl.MaybeType):
|
| 166 | child = desc.desc
|
| 167 | if isinstance(child, asdl.IntType):
|
| 168 | value = const.NO_INTEGER
|
| 169 | elif isinstance(child, asdl.StrType):
|
| 170 | value = ''
|
| 171 | else:
|
| 172 | value = None
|
| 173 | self.__setattr__(name, value) # Maybe values can be None
|
| 174 |
|
| 175 | elif isinstance(desc, asdl.ArrayType):
|
| 176 | self.__setattr__(name, [])
|
| 177 |
|
| 178 | def _Init(self, args, kwargs):
|
| 179 | field_names = list(self.ASDL_TYPE.GetFieldNames())
|
| 180 | for i, val in enumerate(args):
|
| 181 | name = field_names[i]
|
| 182 | self.__setattr__(name, val)
|
| 183 |
|
| 184 | for name, val in kwargs.items():
|
| 185 | if self._assigned[name]:
|
| 186 | raise TypeError('Duplicate assignment of field %r' % name)
|
| 187 | self.__setattr__(name, val)
|
| 188 |
|
| 189 | # Disable type checking here
|
| 190 | #return
|
| 191 | for name in field_names:
|
| 192 | if not self._assigned[name]:
|
| 193 | # If anything was set, then required fields raise an error.
|
| 194 | raise ValueError("Field %r is required and wasn't initialized" % name)
|
| 195 |
|
| 196 | def CheckUnassigned(self):
|
| 197 | """See if there are unassigned fields, for later encoding.
|
| 198 |
|
| 199 | This is currently only used in unit tests.
|
| 200 | """
|
| 201 | unassigned = []
|
| 202 | for name in self.ASDL_TYPE.GetFieldNames():
|
| 203 | if not self._assigned[name]:
|
| 204 | desc = self.ASDL_TYPE.LookupFieldType(name)
|
| 205 | if not isinstance(desc, asdl.MaybeType):
|
| 206 | unassigned.append(name)
|
| 207 | if unassigned:
|
| 208 | raise ValueError("Fields %r were't be assigned" % unassigned)
|
| 209 |
|
| 210 | if 1: # Disable type checking here
|
| 211 | def __setattr__(self, name, value):
|
| 212 | if name == '_assigned':
|
| 213 | self.__dict__[name] = value
|
| 214 | return
|
| 215 | try:
|
| 216 | desc = self.ASDL_TYPE.LookupFieldType(name)
|
| 217 | except KeyError:
|
| 218 | raise AttributeError('Object of type %r has no attribute %r' %
|
| 219 | (self.__class__.__name__, name))
|
| 220 |
|
| 221 | if not _CheckType(value, desc):
|
| 222 | raise AssertionError("Field %r should be of type %s, got %r (%s)" %
|
| 223 | (name, desc, value, value.__class__))
|
| 224 |
|
| 225 | self._assigned[name] = True # check this later when encoding
|
| 226 | self.__dict__[name] = value
|
| 227 |
|
| 228 |
|
| 229 | def MakeTypes(module, root, type_lookup):
|
| 230 | """
|
| 231 | Args:
|
| 232 | module: asdl.Module
|
| 233 | root: an object/package to add types to
|
| 234 | """
|
| 235 | for defn in module.dfns:
|
| 236 | typ = defn.value
|
| 237 |
|
| 238 | #print('TYPE', defn.name, typ)
|
| 239 | if isinstance(typ, asdl.Sum):
|
| 240 | sum_type = typ
|
| 241 | if asdl.is_simple(sum_type):
|
| 242 | # An object without fields, which can be stored inline.
|
| 243 |
|
| 244 | # Create a class called foo_e. Unlike the CompoundObj case, it doesn't
|
| 245 | # have subtypes. Instead if has attributes foo_e.Bar, which Bar is an
|
| 246 | # instance of foo_e.
|
| 247 | #
|
| 248 | # Problem: This means you have a dichotomy between:
|
| 249 | # cflow_e.Break vs. cflow_e.Break()
|
| 250 | # If you add a non-simple type like cflow_e.Return(5), the usage will
|
| 251 | # change. I haven't run into this problem in practice yet.
|
| 252 |
|
| 253 | class_name = defn.name + '_e'
|
| 254 | class_attr = {'ASDL_TYPE': sum_type} # asdl.Sum
|
| 255 | cls = type(class_name, (SimpleObj, ), class_attr)
|
| 256 | setattr(root, class_name, cls)
|
| 257 |
|
| 258 | # NOTE: Right now the ASDL_TYPE for for an enum value is the Sum type,
|
| 259 | # not the Constructor type. We may want to change this if we need
|
| 260 | # reflection.
|
| 261 | for i, cons in enumerate(sum_type.types):
|
| 262 | enum_id = i + 1
|
| 263 | name = cons.name
|
| 264 | val = cls(enum_id, cons.name) # Instantiate SimpleObj subtype
|
| 265 |
|
| 266 | # Set a static attribute like op_id.Plus, op_id.Minus.
|
| 267 | setattr(cls, name, val)
|
| 268 | else:
|
| 269 | tag_num = {}
|
| 270 |
|
| 271 | # e.g. for arith_expr
|
| 272 | # Should this be arith_expr_t? It is in C++.
|
| 273 | base_class = type(defn.name, (DebugCompoundObj, ), {})
|
| 274 | setattr(root, defn.name, base_class)
|
| 275 |
|
| 276 | # Make a type and a enum tag for each alternative.
|
| 277 | for i, cons in enumerate(sum_type.types):
|
| 278 | tag = i + 1 # zero reserved?
|
| 279 | tag_num[cons.name] = tag # for enum
|
| 280 |
|
| 281 | class_attr = {
|
| 282 | 'ASDL_TYPE': cons, # asdl.Constructor
|
| 283 | 'tag': tag, # Does this API change?
|
| 284 | }
|
| 285 |
|
| 286 | cls = type(cons.name, (base_class, ), class_attr)
|
| 287 | setattr(root, cons.name, cls)
|
| 288 |
|
| 289 | # e.g. arith_expr_e.Const == 1
|
| 290 | enum_name = defn.name + '_e'
|
| 291 | tag_enum = type(enum_name, (), tag_num)
|
| 292 | setattr(root, enum_name, tag_enum)
|
| 293 |
|
| 294 | elif isinstance(typ, asdl.Product):
|
| 295 | class_attr = {'ASDL_TYPE': typ}
|
| 296 | cls = type(defn.name, (DebugCompoundObj, ), class_attr)
|
| 297 | setattr(root, defn.name, cls)
|
| 298 |
|
| 299 | else:
|
| 300 | raise AssertionError(typ)
|
| 301 |
|
| 302 |
|
| 303 | def AssignTypes(src_module, dest_module):
|
| 304 | """For generated code."""
|
| 305 | for name in dir(src_module):
|
| 306 | if not name.startswith('__'):
|
| 307 | v = getattr(src_module, name)
|
| 308 | setattr(dest_module, name, v)
|
| 309 |
|