1 | """Token constants (from "token.h")."""
|
2 |
|
3 | # Taken from Python (r53757) and modified to include some tokens
|
4 | # originally monkeypatched in by pgen2.tokenize
|
5 |
|
6 | #--start constants--
|
7 | ENDMARKER = 0
|
8 | NAME = 1
|
9 | NUMBER = 2
|
10 | STRING = 3
|
11 | NEWLINE = 4
|
12 | INDENT = 5
|
13 | DEDENT = 6
|
14 | LPAR = 7
|
15 | RPAR = 8
|
16 | LSQB = 9
|
17 | RSQB = 10
|
18 | COLON = 11
|
19 | COMMA = 12
|
20 | SEMI = 13
|
21 | PLUS = 14
|
22 | MINUS = 15
|
23 | STAR = 16
|
24 | SLASH = 17
|
25 | VBAR = 18
|
26 | AMPER = 19
|
27 | LESS = 20
|
28 | GREATER = 21
|
29 | EQUAL = 22
|
30 | DOT = 23
|
31 | PERCENT = 24
|
32 | BACKQUOTE = 25
|
33 | LBRACE = 26
|
34 | RBRACE = 27
|
35 | EQEQUAL = 28
|
36 | NOTEQUAL = 29
|
37 | LESSEQUAL = 30
|
38 | GREATEREQUAL = 31
|
39 | TILDE = 32
|
40 | CIRCUMFLEX = 33
|
41 | LEFTSHIFT = 34
|
42 | RIGHTSHIFT = 35
|
43 | DOUBLESTAR = 36
|
44 | PLUSEQUAL = 37
|
45 | MINEQUAL = 38
|
46 | STAREQUAL = 39
|
47 | SLASHEQUAL = 40
|
48 | PERCENTEQUAL = 41
|
49 | AMPEREQUAL = 42
|
50 | VBAREQUAL = 43
|
51 | CIRCUMFLEXEQUAL = 44
|
52 | LEFTSHIFTEQUAL = 45
|
53 | RIGHTSHIFTEQUAL = 46
|
54 | DOUBLESTAREQUAL = 47
|
55 | DOUBLESLASH = 48
|
56 | DOUBLESLASHEQUAL = 49
|
57 | AT = 50
|
58 | ATEQUAL = 51
|
59 | OP = 52
|
60 | COMMENT = 53
|
61 | NL = 54
|
62 | RARROW = 55
|
63 | AWAIT = 56
|
64 | ASYNC = 57
|
65 | ERRORTOKEN = 58
|
66 | N_TOKENS = 59
|
67 | NT_OFFSET = 256
|
68 | #--end constants--
|
69 |
|
70 | tok_name = {}
|
71 | for _name, _value in list(globals().items()):
|
72 | if type(_value) is type(0):
|
73 | tok_name[_value] = _name
|
74 |
|
75 |
|
76 | def ISTERMINAL(x):
|
77 | # type: (int) -> bool
|
78 | return x < NT_OFFSET
|
79 |
|
80 | def ISNONTERMINAL(x):
|
81 | # type: (int) -> bool
|
82 | return x >= NT_OFFSET
|
83 |
|
84 | def ISEOF(x):
|
85 | # type: (int) -> bool
|
86 | return x == ENDMARKER
|
87 |
|
88 |
|
89 | # Map from operator to number. NOTE: This was moved here from
|
90 | # pgen2/grammar.py. Oil doesn't need this; only parsers that use Python's
|
91 | # tokenizer (like pgen itself) need it.
|
92 |
|
93 | opmap_raw = """
|
94 | ( LPAR
|
95 | ) RPAR
|
96 | [ LSQB
|
97 | ] RSQB
|
98 | : COLON
|
99 | , COMMA
|
100 | ; SEMI
|
101 | + PLUS
|
102 | - MINUS
|
103 | * STAR
|
104 | / SLASH
|
105 | | VBAR
|
106 | & AMPER
|
107 | < LESS
|
108 | > GREATER
|
109 | = EQUAL
|
110 | . DOT
|
111 | % PERCENT
|
112 | ` BACKQUOTE
|
113 | { LBRACE
|
114 | } RBRACE
|
115 | @ AT
|
116 | @= ATEQUAL
|
117 | == EQEQUAL
|
118 | != NOTEQUAL
|
119 | <> NOTEQUAL
|
120 | <= LESSEQUAL
|
121 | >= GREATEREQUAL
|
122 | ~ TILDE
|
123 | ^ CIRCUMFLEX
|
124 | << LEFTSHIFT
|
125 | >> RIGHTSHIFT
|
126 | ** DOUBLESTAR
|
127 | += PLUSEQUAL
|
128 | -= MINEQUAL
|
129 | *= STAREQUAL
|
130 | /= SLASHEQUAL
|
131 | %= PERCENTEQUAL
|
132 | &= AMPEREQUAL
|
133 | |= VBAREQUAL
|
134 | ^= CIRCUMFLEXEQUAL
|
135 | <<= LEFTSHIFTEQUAL
|
136 | >>= RIGHTSHIFTEQUAL
|
137 | **= DOUBLESTAREQUAL
|
138 | // DOUBLESLASH
|
139 | //= DOUBLESLASHEQUAL
|
140 | -> RARROW
|
141 | """
|
142 |
|
143 | import sys
|
144 | this_module = sys.modules[__name__]
|
145 | opmap = {}
|
146 | for line in opmap_raw.splitlines():
|
147 | if line:
|
148 | op, name = line.split()
|
149 | opmap[op] = getattr(this_module, name)
|