| 1 | """Token constants (from "token.h")."""
|
| 2 |
|
| 3 | # Taken from Python (r53757) and modified to include some tokens
|
| 4 | # originally monkeypatched in by pgen2.tokenize
|
| 5 |
|
| 6 | #--start constants--
|
| 7 | ENDMARKER = 0
|
| 8 | NAME = 1
|
| 9 | NUMBER = 2
|
| 10 | STRING = 3
|
| 11 | NEWLINE = 4
|
| 12 | INDENT = 5
|
| 13 | DEDENT = 6
|
| 14 | LPAR = 7
|
| 15 | RPAR = 8
|
| 16 | LSQB = 9
|
| 17 | RSQB = 10
|
| 18 | COLON = 11
|
| 19 | COMMA = 12
|
| 20 | SEMI = 13
|
| 21 | PLUS = 14
|
| 22 | MINUS = 15
|
| 23 | STAR = 16
|
| 24 | SLASH = 17
|
| 25 | VBAR = 18
|
| 26 | AMPER = 19
|
| 27 | LESS = 20
|
| 28 | GREATER = 21
|
| 29 | EQUAL = 22
|
| 30 | DOT = 23
|
| 31 | PERCENT = 24
|
| 32 | BACKQUOTE = 25
|
| 33 | LBRACE = 26
|
| 34 | RBRACE = 27
|
| 35 | EQEQUAL = 28
|
| 36 | NOTEQUAL = 29
|
| 37 | LESSEQUAL = 30
|
| 38 | GREATEREQUAL = 31
|
| 39 | TILDE = 32
|
| 40 | CIRCUMFLEX = 33
|
| 41 | LEFTSHIFT = 34
|
| 42 | RIGHTSHIFT = 35
|
| 43 | DOUBLESTAR = 36
|
| 44 | PLUSEQUAL = 37
|
| 45 | MINEQUAL = 38
|
| 46 | STAREQUAL = 39
|
| 47 | SLASHEQUAL = 40
|
| 48 | PERCENTEQUAL = 41
|
| 49 | AMPEREQUAL = 42
|
| 50 | VBAREQUAL = 43
|
| 51 | CIRCUMFLEXEQUAL = 44
|
| 52 | LEFTSHIFTEQUAL = 45
|
| 53 | RIGHTSHIFTEQUAL = 46
|
| 54 | DOUBLESTAREQUAL = 47
|
| 55 | DOUBLESLASH = 48
|
| 56 | DOUBLESLASHEQUAL = 49
|
| 57 | AT = 50
|
| 58 | ATEQUAL = 51
|
| 59 | OP = 52
|
| 60 | COMMENT = 53
|
| 61 | NL = 54
|
| 62 | RARROW = 55
|
| 63 | AWAIT = 56
|
| 64 | ASYNC = 57
|
| 65 | ERRORTOKEN = 58
|
| 66 | N_TOKENS = 59
|
| 67 | NT_OFFSET = 256
|
| 68 | #--end constants--
|
| 69 |
|
| 70 | tok_name = {}
|
| 71 | for _name, _value in list(globals().items()):
|
| 72 | if type(_value) is type(0):
|
| 73 | tok_name[_value] = _name
|
| 74 |
|
| 75 |
|
| 76 | def ISTERMINAL(x):
|
| 77 | # type: (int) -> bool
|
| 78 | return x < NT_OFFSET
|
| 79 |
|
| 80 | def ISNONTERMINAL(x):
|
| 81 | # type: (int) -> bool
|
| 82 | return x >= NT_OFFSET
|
| 83 |
|
| 84 | def ISEOF(x):
|
| 85 | # type: (int) -> bool
|
| 86 | return x == ENDMARKER
|
| 87 |
|
| 88 |
|
| 89 | # Map from operator to number. NOTE: This was moved here from
|
| 90 | # pgen2/grammar.py. Oil doesn't need this; only parsers that use Python's
|
| 91 | # tokenizer (like pgen itself) need it.
|
| 92 |
|
| 93 | opmap_raw = """
|
| 94 | ( LPAR
|
| 95 | ) RPAR
|
| 96 | [ LSQB
|
| 97 | ] RSQB
|
| 98 | : COLON
|
| 99 | , COMMA
|
| 100 | ; SEMI
|
| 101 | + PLUS
|
| 102 | - MINUS
|
| 103 | * STAR
|
| 104 | / SLASH
|
| 105 | | VBAR
|
| 106 | & AMPER
|
| 107 | < LESS
|
| 108 | > GREATER
|
| 109 | = EQUAL
|
| 110 | . DOT
|
| 111 | % PERCENT
|
| 112 | ` BACKQUOTE
|
| 113 | { LBRACE
|
| 114 | } RBRACE
|
| 115 | @ AT
|
| 116 | @= ATEQUAL
|
| 117 | == EQEQUAL
|
| 118 | != NOTEQUAL
|
| 119 | <> NOTEQUAL
|
| 120 | <= LESSEQUAL
|
| 121 | >= GREATEREQUAL
|
| 122 | ~ TILDE
|
| 123 | ^ CIRCUMFLEX
|
| 124 | << LEFTSHIFT
|
| 125 | >> RIGHTSHIFT
|
| 126 | ** DOUBLESTAR
|
| 127 | += PLUSEQUAL
|
| 128 | -= MINEQUAL
|
| 129 | *= STAREQUAL
|
| 130 | /= SLASHEQUAL
|
| 131 | %= PERCENTEQUAL
|
| 132 | &= AMPEREQUAL
|
| 133 | |= VBAREQUAL
|
| 134 | ^= CIRCUMFLEXEQUAL
|
| 135 | <<= LEFTSHIFTEQUAL
|
| 136 | >>= RIGHTSHIFTEQUAL
|
| 137 | **= DOUBLESTAREQUAL
|
| 138 | // DOUBLESLASH
|
| 139 | //= DOUBLESLASHEQUAL
|
| 140 | -> RARROW
|
| 141 | """
|
| 142 |
|
| 143 | import sys
|
| 144 | this_module = sys.modules[__name__]
|
| 145 | opmap = {}
|
| 146 | for line in opmap_raw.splitlines():
|
| 147 | if line:
|
| 148 | op, name = line.split()
|
| 149 | opmap[op] = getattr(this_module, name)
|