OILS / frontend / lexer_def.py View on Github | oilshell.org

1119 lines, 566 significant
1"""
2lexer_def.py - Lexing for OSH, YSH, and J8 Notation.
3
4The OSH/YSH lexer has lexer modes, each with a regex -> Id mapping.
5
6After changing this file, run:
7
8 build/py.sh all
9
10or at least:
11
12 build/py.sh fastlex
13
14Input Handling
15--------------
16
17Every line is NUL terminated:
18
19 'one\n\0' 'last line\0'
20
21which means that no regexes below should match \0.
22
23For example, use [^'\0]+ instead of [^']+ .
24
25If this rule isn't followed, we would read uninitialized memory past the
26sentinel. Python's regex engine knows where the end of the input string is, so
27it doesn't require need a sentinel like \0.
28
29The frontend/lexer_gen.py generator adds a pattern mapping \0 to Id.Eol_Tok.
30"""
31
32from _devbuild.gen.id_kind_asdl import Id, Id_t, Kind
33from _devbuild.gen.types_asdl import lex_mode_e
34
35from frontend import id_kind_def
36
37from typing import Tuple
38
39# Initialize spec that the lexer depends on.
40ID_SPEC = id_kind_def.IdSpec({}, {})
41
42id_kind_def.AddKinds(ID_SPEC)
43id_kind_def.AddBoolKinds(ID_SPEC) # must come second
44id_kind_def.SetupTestBuiltin(ID_SPEC, {}, {}, {})
45
46
47def C(pat, tok_type):
48 # type: (str, Id_t) -> Tuple[bool, str, Id_t]
49 """Lexer rule with a constant string, e.g. C('$*', VSub_Star)"""
50 return (False, pat, tok_type)
51
52
53def R(pat, tok_type):
54 # type: (str, Id_t) -> Tuple[bool, str, Id_t]
55 """Lexer rule with a regex string, e.g. R('\$[0-9]', VSub_Number)"""
56 return (True, pat, tok_type)
57
58
59# See unit tests in frontend/match_test.py.
60# We need the [^\0]* because the re2c translation assumes it's anchored like $.
61SHOULD_HIJACK_RE = r'#![^\0]*sh[ \t\r\n][^\0]*'
62
63# Separates words (\r it not whitespace here)
64_SIGNIFICANT_SPACE = R(r'[ \t]+', Id.WS_Space)
65
66_BACKSLASH = [
67 # To be conservative, we could deny a set of chars similar to
68 # _LITERAL_WHITELIST_REGEX, rather than allowing all the operator characters
69 # like \( and \;.
70 #
71 # strict_backslash makes this stricter.
72 R(r'\\[^\n\0]', Id.Lit_EscapedChar),
73 C('\\\n', Id.Ignored_LineCont),
74]
75
76# Only 4 characters are backslash escaped inside "".
77# https://www.gnu.org/software/bash/manual/bash.html#Double-Quotes
78_DQ_BACKSLASH = [
79 R(r'\\[$`"\\]', Id.Lit_EscapedChar),
80 C('\\', Id.Lit_BadBackslash), # syntax error in YSH, but NOT in OSH
81]
82
83VAR_NAME_RE = r'[a-zA-Z_][a-zA-Z0-9_]*'
84
85# All Kind.VSub
86_VARS = [
87 # Unbraced variables
88 R(r'\$' + VAR_NAME_RE, Id.VSub_DollarName),
89 R(r'\$[0-9]', Id.VSub_Number),
90 C(r'$!', Id.VSub_Bang),
91 C(r'$@', Id.VSub_At),
92 C(r'$#', Id.VSub_Pound),
93 C(r'$$', Id.VSub_Dollar),
94 C(r'$*', Id.VSub_Star),
95 C(r'$-', Id.VSub_Hyphen),
96 C(r'$?', Id.VSub_QMark),
97]
98
99# Kind.Left that are valid in double-quoted modes.
100
101_LEFT_SUBS = [
102 C('`', Id.Left_Backtick),
103 C('$(', Id.Left_DollarParen),
104 C('${', Id.Left_DollarBrace),
105 # Parse zsh syntax, but don't execute it.
106 # The examples we've seen so far are like ${(%):-} and ${(m)
107 R(r'\$\{\([^)\0]+\)', Id.Left_DollarBraceZsh),
108 C('$((', Id.Left_DollarDParen),
109 C('$[', Id.Left_DollarBracket),
110]
111
112# Additional Kind.Left that are valid in unquoted modes.
113_LEFT_UNQUOTED = [
114 C('"', Id.Left_DoubleQuote),
115 C("'", Id.Left_SingleQuote),
116 C('$"', Id.Left_DollarDoubleQuote),
117 C("$'", Id.Left_DollarSingleQuote),
118]
119
120_LEFT_PROCSUB = [
121 C('<(', Id.Left_ProcSubIn),
122 C('>(', Id.Left_ProcSubOut),
123]
124
125# The regexes below are in Python syntax, but are translate to re2c syntax by
126# frontend/lexer_gen.py.
127#
128# http://re2c.org/manual/syntax/syntax.html
129# https://docs.python.org/2/library/re.html
130#
131# We use a limited set of constructs:
132# - + and * for repetition
133# - Character classes [] with simple ranges and negation
134# - Escapes like \n \0
135
136LEXER_DEF = {} # TODO: Should be a list so we enforce order.
137
138# Anything until the end of the line is a comment. Does not match the newline
139# itself. We want to switch modes and possibly process Op_Newline for here
140# docs, etc.
141LEXER_DEF[lex_mode_e.Comment] = [R(r'[^\n\0]*', Id.Ignored_Comment)]
142
143# A whitelist to make bigger Lit_Chars tokens. We don't want one byte at a time.
144#
145# The shell language says that "anything other byte" is a literal character --
146# for example, unquoted $ \ ! are literal, not a syntax error.
147#
148# That is, a literal is defined NEGATIVELY, for a single characters. But here
149# we define a SUBSET of literal chars POSITIVELY.
150
151# The range \x80-\xff makes sure that UTF-8 sequences are a single token.
152_LITERAL_WHITELIST_REGEX = r'[\x80-\xffa-zA-Z0-9_.\-]+'
153
154_UNQUOTED = _BACKSLASH + _LEFT_SUBS + _LEFT_UNQUOTED + _LEFT_PROCSUB + _VARS + [
155 # NOTE: We could add anything 128 and above to this character class? So
156 # utf-8 characters don't get split?
157 R(_LITERAL_WHITELIST_REGEX, Id.Lit_Chars),
158 C('~', Id.Lit_Tilde), # for tilde sub
159 C('/', Id.Lit_Slash), # also for tilde sub
160 C(':', Id.Lit_Colon), # for special PATH=a:~foo tilde detection
161 C('$', Id.Lit_Dollar), # shopt -u parse_dollar
162 C('#', Id.Lit_Pound), # For comments
163 _SIGNIFICANT_SPACE,
164 C('\n', Id.Op_Newline),
165 C('&', Id.Op_Amp),
166 C('|', Id.Op_Pipe),
167 C('|&', Id.Op_PipeAmp),
168 C('&&', Id.Op_DAmp),
169 C('||', Id.Op_DPipe),
170 C(';', Id.Op_Semi),
171 # Case terminators
172 C(';;', Id.Op_DSemi),
173 C(';&', Id.Op_SemiAmp),
174 C(';;&', Id.Op_DSemiAmp),
175 C('(', Id.Op_LParen),
176 C(')', Id.Op_RParen),
177 R(r'[^\0]', Id.Lit_Other), # any other single char is a literal
178]
179
180# In ShCommand and DBracket states.
181_EXTGLOB_BEGIN = [
182 C(',(', Id.ExtGlob_Comma), # YSH synonym for @(...)
183 C('@(', Id.ExtGlob_At),
184 C('*(', Id.ExtGlob_Star),
185 C('+(', Id.ExtGlob_Plus),
186 C('?(', Id.ExtGlob_QMark),
187 C('!(', Id.ExtGlob_Bang),
188]
189
190KEYWORDS = [
191 # NOTE: { is matched elsewhere
192 C('[[', Id.KW_DLeftBracket),
193 C('!', Id.KW_Bang),
194 C('for', Id.KW_For),
195 C('while', Id.KW_While),
196 C('until', Id.KW_Until),
197 C('do', Id.KW_Do),
198 C('done', Id.KW_Done),
199 C('in', Id.KW_In),
200 C('case', Id.KW_Case),
201 C('esac', Id.KW_Esac),
202 C('if', Id.KW_If),
203 C('fi', Id.KW_Fi),
204 C('then', Id.KW_Then),
205 C('else', Id.KW_Else),
206 C('elif', Id.KW_Elif),
207 C('function', Id.KW_Function),
208 C('time', Id.KW_Time),
209
210 # YSH
211 C('const', Id.KW_Const), # maybe remove this
212 C('var', Id.KW_Var),
213 C('setvar', Id.KW_SetVar),
214 C('setglobal', Id.KW_SetGlobal),
215 C('call', Id.KW_Call),
216 C('proc', Id.KW_Proc),
217 C('typed', Id.KW_Typed),
218 C('func', Id.KW_Func),
219]
220
221# These are treated like builtins in bash, but keywords in OSH. However, we
222# maintain compatibility with bash for the 'type' builtin.
223CONTROL_FLOW = [
224 C('break', Id.ControlFlow_Break),
225 C('continue', Id.ControlFlow_Continue),
226 C('return', Id.ControlFlow_Return),
227 C('exit', Id.ControlFlow_Exit),
228]
229
230# Used by ysh/grammar_gen.py too
231EXPR_WORDS = [
232 C('null', Id.Expr_Null),
233 C('true', Id.Expr_True),
234 C('false', Id.Expr_False),
235 C('and', Id.Expr_And),
236 C('or', Id.Expr_Or),
237 C('not', Id.Expr_Not),
238 C('for', Id.Expr_For),
239 C('is', Id.Expr_Is),
240 C('in', Id.Expr_In),
241 C('if', Id.Expr_If),
242 C('else', Id.Expr_Else),
243
244 # Unused: could be for function literals, although we also have
245 # |x| x+1 lambdas
246 C('func', Id.Expr_Func),
247
248 # / <capture d+/
249 C('capture', Id.Expr_Capture),
250 # / <capture d+ as date> /
251 C('as', Id.Expr_As),
252]
253
254FD_VAR_NAME = r'\{' + VAR_NAME_RE + r'\}'
255
256# file descriptors can only have two digits, like mksh
257# dash/zsh/etc. can have one
258FD_NUM = r'[0-9]?[0-9]?'
259
260# These two can must be recognized in the ShCommand state, but can't nested
261# within [[.
262# Keywords have to be checked before _UNQUOTED so we get <KW_If "if"> instead
263# of <Lit_Chars "if">.
264LEXER_DEF[lex_mode_e.ShCommand] = [
265 # These four are not allowed within [[, so they are in ShCommand but not
266 # _UNQUOTED.
267
268 # e.g. beginning of NAME=val, which will always be longer than
269 # _LITERAL_WHITELIST_REGEX.
270 R(VAR_NAME_RE + '\+?=', Id.Lit_VarLike),
271 R(VAR_NAME_RE + '\[', Id.Lit_ArrayLhsOpen),
272 R(r'\]\+?=', Id.Lit_ArrayLhsClose),
273 C('((', Id.Op_DLeftParen),
274
275 # For static globbing, and [] for array literals
276 C('[', Id.Lit_LBracket), # e.g. A=(['x']=1)
277 C(']', Id.Lit_RBracket), # e.g. *.[ch]
278 # NOTE: Glob_Star and Glob_QMark are for dynamic parsing
279 C('*', Id.Lit_Star),
280 C('?', Id.Lit_QMark),
281 C('###', Id.Lit_TPound), # like Lit_Pound, for doc comments
282 C('...', Id.Lit_TDot), # ... for multiline commands
283
284 # For brace expansion {a,b}
285 C('{', Id.Lit_LBrace),
286 C('}', Id.Lit_RBrace), # Also for var sub ${a}
287 C(',', Id.Lit_Comma),
288 C('=', Id.Lit_Equals), # for = f(x) and x = 1+2*3
289 C('@', Id.Lit_At), # for detecting @[, @' etc. shopt -s parse_at_all
290
291 # @array and @func(1, c)
292 R('@' + VAR_NAME_RE, Id.Lit_Splice), # for YSH splicing
293 C('@[', Id.Lit_AtLBracket), # @[split(x)]
294 C('@{.', Id.Lit_AtLBraceDot), # for split builtin sub @{.myproc arg1}
295 R(FD_NUM + r'<', Id.Redir_Less),
296 R(FD_NUM + r'>', Id.Redir_Great),
297 R(FD_NUM + r'<<', Id.Redir_DLess),
298 R(FD_NUM + r'<<<', Id.Redir_TLess),
299 R(FD_NUM + r'>>', Id.Redir_DGreat),
300 R(FD_NUM + r'<<-', Id.Redir_DLessDash),
301 R(FD_NUM + r'>&', Id.Redir_GreatAnd),
302 R(FD_NUM + r'<&', Id.Redir_LessAnd),
303 R(FD_NUM + r'<>', Id.Redir_LessGreat),
304 R(FD_NUM + r'>\|', Id.Redir_Clobber),
305 R(FD_VAR_NAME + r'<', Id.Redir_Less),
306 R(FD_VAR_NAME + r'>', Id.Redir_Great),
307 R(FD_VAR_NAME + r'<<', Id.Redir_DLess),
308 R(FD_VAR_NAME + r'<<<', Id.Redir_TLess),
309 R(FD_VAR_NAME + r'>>', Id.Redir_DGreat),
310 R(FD_VAR_NAME + r'<<-', Id.Redir_DLessDash),
311 R(FD_VAR_NAME + r'>&', Id.Redir_GreatAnd),
312 R(FD_VAR_NAME + r'<&', Id.Redir_LessAnd),
313 R(FD_VAR_NAME + r'<>', Id.Redir_LessGreat),
314 R(FD_VAR_NAME + r'>\|', Id.Redir_Clobber),
315
316 # No leading descriptor (2 is implied)
317 C(r'&>', Id.Redir_AndGreat),
318 C(r'&>>', Id.Redir_AndDGreat),
319] + KEYWORDS + CONTROL_FLOW + _UNQUOTED + _EXTGLOB_BEGIN
320
321# Preprocessing before ShCommand
322LEXER_DEF[lex_mode_e.Backtick] = [
323 C(r'`', Id.Backtick_Right),
324 # A backslash, and then $ or ` or \
325 R(r'\\[$`\\]', Id.Backtick_Quoted),
326 # \" treated specially, depending on whether bacticks are double-quoted!
327 R(r'\\"', Id.Backtick_DoubleQuote),
328 R(r'[^`\\\0]+', Id.Backtick_Other), # contiguous run of literals
329 R(r'[^\0]', Id.Backtick_Other), # anything else
330]
331
332# DBRACKET: can be like ShCommand, except:
333# - Don't really need redirects either... Redir_Less could be Op_Less
334# - Id.Op_DLeftParen can't be nested inside.
335LEXER_DEF[lex_mode_e.DBracket] = [
336 C(']]', Id.Lit_DRightBracket),
337 # Must be KW and not Op, because we can have stuff like [[ $foo == !* ]]
338 # in addition to [[ ! a && b ]]
339 C('!', Id.KW_Bang),
340 C('<', Id.Op_Less),
341 C('>', Id.Op_Great),
342] + ID_SPEC.LexerPairs(Kind.BoolUnary) + \
343 ID_SPEC.LexerPairs(Kind.BoolBinary) + \
344 _UNQUOTED + _EXTGLOB_BEGIN
345
346# Inside an extended glob, most characters are literals, including spaces and
347# punctuation. We also accept \, $var, ${var}, "", etc. They can also be
348# nested, so _EXTGLOB_BEGIN appears here.
349#
350# Example: echo @(<> <>|&&|'foo'|$bar)
351LEXER_DEF[lex_mode_e.ExtGlob] = \
352 _BACKSLASH + _LEFT_SUBS + _LEFT_UNQUOTED + _VARS + _EXTGLOB_BEGIN + [
353 R(r'[^\\$`"\'|)@*+!?\0]+', Id.Lit_Chars),
354 C('|', Id.Op_Pipe),
355 C(')', Id.Op_RParen), # maybe be translated to Id.ExtGlob_RParen
356 R(r'[^\0]', Id.Lit_Other), # everything else is literal
357]
358
359# Notes on BASH_REGEX states
360#
361# From bash manual:
362#
363# - Any part of the pattern may be quoted to force the quoted portion to be
364# matched as a string.
365# - Bracket expressions in regular expressions must be treated carefully, since
366# normal quoting characters lose their meanings between brackets.
367# - If the pattern is stored in a shell variable, quoting the variable
368# expansion forces the entire pattern to be matched as a string.
369#
370# Is there a re.escape function? It's just like EscapeGlob and UnescapeGlob.
371#
372# TODO: For testing, write a script to extract and save regexes... and compile
373# them with regcomp. I've only seen constant regexes.
374#
375# bash code: ( | ) are special
376
377LEXER_DEF[lex_mode_e.BashRegex] = _LEFT_SUBS + _LEFT_UNQUOTED + _VARS + [
378 # Like lex_mode_e.ShCommand
379 R(_LITERAL_WHITELIST_REGEX, Id.Lit_Chars),
380
381 # Tokens for Tilde sub. bash weirdness: RHS of [[ x =~ ~ ]] is expanded
382 C('~', Id.Lit_Tilde),
383 C('/', Id.Lit_Slash),
384
385 # Id.WS_Space delimits words. In lex_mode_e.BashRegexFakeInner, we
386 # translate them to Id.Lit_Chars.
387 _SIGNIFICANT_SPACE,
388
389 # Analogous to Id.ExtGlob_* - we need to change lexer modes when we hit this
390 C('(', Id.BashRegex_LParen),
391
392 # Not special, this is like lex_mode_e.Outer
393 C(')', Id.Op_RParen),
394
395 # Copied and adapted from _UNQUOTED
396 # \n & ; < > are parse errors OUTSIDE a group [[ s =~ ; ]]
397 # but become allowed INSIDE a group [[ s =~ (;) ]]
398 C('\n', Id.BashRegex_AllowedInParens),
399 C('&', Id.BashRegex_AllowedInParens),
400 C(';', Id.BashRegex_AllowedInParens),
401 C('>', Id.BashRegex_AllowedInParens),
402 C('<', Id.BashRegex_AllowedInParens),
403
404 # e.g. | is Id.Lit_Other, not pipe operator
405 R(r'[^\0]', Id.Lit_Other), # like _UNQUOTED, any other byte is literal
406] + _BACKSLASH # These have to come after RegexMeta
407
408LEXER_DEF[lex_mode_e.DQ] = _DQ_BACKSLASH + [
409 C('\\\n', Id.Ignored_LineCont),
410] + _LEFT_SUBS + _VARS + [
411 R(r'[^$`"\0\\]+', Id.Lit_Chars), # matches a line at most
412 C('$', Id.Lit_Dollar), # completion of var names relies on this
413 # NOTE: When parsing here doc line, this token doesn't end it.
414 C('"', Id.Right_DoubleQuote),
415]
416
417_VS_ARG_COMMON = [
418 C('/', Id.Lit_Slash), # for patsub (not Id.VOp2_Slash)
419 C('#', Id.Lit_Pound), # for patsub prefix (not Id.VOp1_Pound)
420 C('%', Id.Lit_Percent), # for patsdub suffix (not Id.VOp1_Percent)
421 C('}', Id.Right_DollarBrace), # For var sub "${a}"
422 C('$', Id.Lit_Dollar), # completion of var names relies on this
423]
424
425# We don't execute zsh var subs, but to find the closing } properly, we need to
426# to recognize \} and '}' and "}" $'}' etc.
427LEXER_DEF[lex_mode_e.VSub_Zsh] = \
428 _BACKSLASH + _LEFT_SUBS + _LEFT_UNQUOTED + _LEFT_PROCSUB + \
429 [
430 C('}', Id.Right_DollarBrace), # For var sub "${a}"
431 R(r'[^\0]', Id.Lit_Other), # e.g. "$", must be last
432]
433
434# Kind.{Lit,Ignored,VSub,Left,Right,Eof}
435LEXER_DEF[lex_mode_e.VSub_ArgUnquoted] = \
436 _BACKSLASH + _VS_ARG_COMMON + _LEFT_SUBS + _LEFT_UNQUOTED + _LEFT_PROCSUB + \
437 _VARS + _EXTGLOB_BEGIN + [
438
439 # Token for Tilde sub
440 C('~', Id.Lit_Tilde),
441
442 # - doesn't match ~ for tilde sub
443 # - doesn't match < and > so it doesn't eat <()
444 # - doesn't match @ ! ? + * so it doesn't eat _EXTGLOB_BEGIN -- ( alone it
445 # not enough
446 R(r'[^$`~/}"\'\0\\#%<>@!?+*]+', Id.Lit_Chars),
447 R(r'[^\0]', Id.Lit_Other), # e.g. "$", must be last
448]
449
450# Kind.{Lit,Ignored,VSub,Left,Right,Eof}
451LEXER_DEF[lex_mode_e.VSub_ArgDQ] = \
452 _DQ_BACKSLASH + _VS_ARG_COMMON + _LEFT_SUBS + _VARS + [
453
454 C(r'\}', Id.Lit_EscapedChar), # For "${var-\}}"
455
456 R(r'[^$`/}"\0\\#%]+', Id.Lit_Chars), # matches a line at most
457
458 # Weird wart: even in double quoted state, double quotes are allowed
459 C('"', Id.Left_DoubleQuote),
460
461 # Another weird wart of bash/mksh: $'' is recognized but NOT ''!
462 C("$'", Id.Left_DollarSingleQuote),
463]
464
465# NOTE: Id.Ignored_LineCont is NOT supported in SQ state, as opposed to DQ
466# state.
467LEXER_DEF[lex_mode_e.SQ_Raw] = [
468 R(r"[^'\0]+", Id.Lit_Chars), # matches a line at most
469 C("'", Id.Right_SingleQuote),
470]
471
472# The main purpose for EXPR_CHARS is in regex literals, e.g. [a-z \t \n].
473#
474# In YSH expressions, Chars are code point integers, so \u{1234} is the same as
475# 0x1234. And \0 is 0x0.
476
477# In Python:
478# chr(0x00012345) == u'\U00012345'
479#
480# In YSH:
481# 0x00012345 == \u{12345}
482# chr(0x00012345) == chr(\u{12345}) == $'\u{012345}'
483
484_U_BRACED_CHAR = R(r'\\[uU]\{[0-9a-fA-F]{1,6}\}', Id.Char_UBraced)
485
486_X_CHAR_LOOSE = R(r'\\x[0-9a-fA-F]{1,2}', Id.Char_Hex) # bash
487_CHAR_YHEX = R(r'\\y[0-9a-fA-F]{2}', Id.Char_YHex) # \yff - J8 only
488
489_U4_CHAR_LOOSE = R(r'\\u[0-9a-fA-F]{1,4}', Id.Char_Unicode4) # bash
490
491_U4_CHAR_STRICT = R(r'\\u[0-9a-fA-F]{4}', Id.Char_Unicode4) # JSON-only
492
493#_JSON_ONE_CHAR = R(r'\\[\\"/bfnrt]', Id.Char_OneChar)
494EXPR_CHARS = [
495 # Allow same backslash escapes as J8 strings, except;
496 # - legacy \b \f
497 # - unnecessary \/
498 #
499 # Note that \0 should be written \y00.
500 R(r'''\\[\\"'nrt]''', Id.Char_OneChar),
501 _CHAR_YHEX,
502
503 # Eggex. This is a LITERAL translation to \xff in ERE? So it's not \yff
504 # It doesn't have semantics; it's just syntax.
505 R(r'\\x[0-9a-fA-F]{2}', Id.Char_Hex),
506
507 _U_BRACED_CHAR,
508]
509
510# Shared between echo -e and $''.
511_C_STRING_COMMON = [
512
513 # \x6 is valid in bash
514 _X_CHAR_LOOSE,
515 _U4_CHAR_LOOSE,
516 R(r'\\U[0-9a-fA-F]{1,8}', Id.Char_Unicode8),
517 R(r'\\[0abeEfrtnv\\]', Id.Char_OneChar),
518
519 # e.g. \A is not an escape, and \x doesn't match a hex escape. We allow it,
520 # but a lint tool could warn about it.
521 C('\\', Id.Unknown_Backslash),
522]
523
524ECHO_E_DEF = _C_STRING_COMMON + [
525 # Note: tokens above \0377 can either be truncated or be flagged a syntax
526 # error in strict mode.
527 R(r'\\0[0-7]{1,3}', Id.Char_Octal4),
528 C(r'\c', Id.Char_Stop),
529
530 # e.g. 'foo', anything that's not a backslash escape
531 R(r'[^\\\0]+', Id.Lit_Chars),
532]
533
534# https://json.org/
535
536# Note that [0-9] has to come second, because Python chooses the first match.
537_JSON_INT = r'-?([1-9][0-9]*|[0-9])' # Numbers can't start with leading 0
538_JSON_FRACTION = r'(\.[0-9]+)?'
539_JSON_EXP = r'([eE][-+]?[0-9]+)?'
540
541# R5RS extended alphabetic characters
542# https://groups.csail.mit.edu/mac/ftpdir/scheme-reports/r5rs-html/r5rs_4.html
543#
544# ! $ % & * + - . / : < = > ? @ ^ _ ~
545
546# Description from Guile Scheme - https://www.gnu.org/software/guile/manual/html_node/Symbol-Read-Syntax.html
547#
548# "The read syntax for a symbol is a sequence of letters, digits, and extended
549# alphabetic characters, beginning with a character that cannot begin a
550# number. In addition, the special cases of +, -, and ... are read as symbols
551# even though numbers can begin with +, - or ."
552#
553# (They should have used regular languages!)
554
555# We take out $ and @ for our splicing syntax, i.e. $unquote and
556# @unquote-splicing. And : for now because we use it for name:value.
557
558# Also note Scheme allows |a b| for symbols with funny chars, and Guile scheme
559# allows #{a b}#. We could use `a b` or (symbol "a b").
560
561J8_SYMBOL_CHARS = r'!%&*+./<=>?^_~-' # - is last for regex char class
562
563# yapf: disable
564J8_SYMBOL_RE = (
565 r'[a-zA-Z' + J8_SYMBOL_CHARS + ']' +
566 r'[a-zA-Z0-9' + J8_SYMBOL_CHARS + ']*')
567# yapf: enable
568
569_J8_LEFT = [
570 C('"', Id.Left_DoubleQuote), # JSON string
571 C('j"', Id.Left_JDoubleQuote), # JSON string with explicit J8 prefix
572 # Three left quotes that are J8 only
573 C("u'", Id.Left_USingleQuote), # unicode string
574 C("'", Id.Left_USingleQuote), # '' is alias for u'' in data, not in code
575 C("b'", Id.Left_BSingleQuote), # byte string
576]
577
578J8_DEF = _J8_LEFT + [
579 C('[', Id.J8_LBracket),
580 C(']', Id.J8_RBracket),
581 C('{', Id.J8_LBrace),
582 C('}', Id.J8_RBrace),
583 C('(', Id.J8_LParen), # NIL8 only
584 C(')', Id.J8_RParen), # NIL8 only
585 C(',', Id.J8_Comma),
586 C(':', Id.J8_Colon),
587 C('null', Id.J8_Null),
588 C('true', Id.J8_Bool),
589 C('false', Id.J8_Bool),
590 R(_JSON_INT, Id.J8_Int),
591 R(_JSON_INT + _JSON_FRACTION + _JSON_EXP, Id.J8_Float),
592
593 # Identifier names come AFTER null true false.
594 # - Happens to be the same as shell identifier # names.
595 # - Note that JS allows $ as an identifier, but we don't.
596 # - Used for dict keys / NIL8 field names.
597 R(VAR_NAME_RE, Id.J8_Identifier),
598
599 # Symbol is a SUPERSET of Identifier. The first word in NIL8 can be can
600 # be either Symbol or plain Identifier, but field names can only be
601 # Identifier. JSON8 only has Identifier.
602 #R(J8_SYMBOL_RE, Id.J8_Symbol), # NIL8 only
603 R(r'[~!@$%^&*+=|;./<>?-]+', Id.J8_Operator), # NIL8 only
604 R(r'[ \r\t]+', Id.Ignored_Space),
605 # A separate token, to count lines for error messages
606 C('\n', Id.Ignored_Newline),
607 # comment is # until end of line
608 # // comments are JavaScript style, but right now we might want them as
609 # symbols?
610 R(r'#[^\n\0]*', Id.Ignored_Comment), # J8 only (JSON8, NIL8)
611
612 # This will reject ASCII control chars
613 R(r'[^\0]', Id.Unknown_Tok),
614]
615
616# Exclude control characters 0x00-0x1f, aka 0-31 in J8 data only (not YSH code)
617_ASCII_CONTROL = R(r'[\x01-\x1F]', Id.Char_AsciiControl)
618
619J8_LINES_DEF = _J8_LEFT + [
620 # not sure if we want \r here - same with lex_mode_e.Expr
621 R(r'[ \r\t]+', Id.WS_Space),
622 R(r'[\n]', Id.J8_Newline),
623
624 # doesn't match \t, which means tabs are allowed in the middle of unquoted
625 # lines
626 _ASCII_CONTROL,
627
628 # not space or ' or " or ASCII control or EOF
629 R(r'''[^ \t\r\n'"\x00-\x1F]+''', Id.Lit_Chars),
630]
631
632# https://json.org list of chars, plus '
633_JSON_ONE_CHAR = R(r'\\[\\"/bfnrt]', Id.Char_OneChar)
634
635# b'' u'' strings - what's common between code and data.
636_J8_STR_COMMON = [
637 C("'", Id.Right_SingleQuote), # end for J8
638 _JSON_ONE_CHAR,
639 C("\\'", Id.Char_OneChar), # since ' ends, allow \'
640 _CHAR_YHEX,
641 _U_BRACED_CHAR, # \u{123456} - J8 only
642
643 # osh/word_parse.py relies on this. It has to be consistent with $''
644 # lexing, which uses _C_STRING_COMMON
645 C('\\', Id.Unknown_Backslash),
646]
647
648# Lexer for J8 strings in CODE.
649LEXER_DEF[lex_mode_e.J8_Str] = _J8_STR_COMMON + [
650 # Don't produce Char_AsciiControl tokens - that's only for data
651
652 # will match invalid UTF-8 - we have a separate validation step
653 R(r"[^\\'\0]+", Id.Lit_Chars),
654]
655
656# Lexer for J8 string data.
657# ASCII control characters are disallowed in DATA, but not CODE!
658J8_STR_DEF = _J8_STR_COMMON + [
659 _ASCII_CONTROL,
660 # will match invalid UTF-8 - we have a separate validation step
661 R(r"[^\\'\x00-\x1F]+", Id.Lit_Chars),
662]
663
664# Lexer for JSON string data - e.g. "json \" \u1234"
665JSON_STR_DEF = [
666 C('"', Id.Right_DoubleQuote), # end for JSON
667 _JSON_ONE_CHAR,
668 _U4_CHAR_STRICT, # \u1234 - JSON only
669
670 # High surrogate [\uD800, \uDC00)
671 # Low surrogate [\uDC00, \uE000)
672 # This pattern makes it easier to decode. Unpaired surrogates because Id.Char_Unicode4.
673 R(
674 r'\\u[dD][89aAbB][0-9a-fA-F][0-9a-fA-F]\\u[dD][cCdDeEfF][0-9a-fA-F][0-9a-fA-F]',
675 Id.Char_SurrogatePair),
676 C('\\', Id.Unknown_Backslash), # e.g. the \ before bad \z
677 _ASCII_CONTROL,
678
679 # Note: This will match INVALID UTF-8. UTF-8 validation is another step.
680 R(r'[^\\"\x00-\x1F]+', Id.Lit_Chars),
681]
682
683OCTAL3_RE = r'\\[0-7]{1,3}'
684
685# https://www.gnu.org/software/bash/manual/html_node/Controlling-the-PromptEvaluator.html#Controlling-the-PromptEvaluator
686PS1_DEF = [
687 R(OCTAL3_RE, Id.PS_Octal3),
688 R(r'\\[adehHjlnrstT@AuvVwW!#$\\]', Id.PS_Subst),
689 # \D{%H:%M} strftime format
690 R(r'\\D\{[^}\0]*\}', Id.PS_Subst),
691 C(r'\[', Id.PS_LBrace), # non-printing
692 C(r'\]', Id.PS_RBrace),
693 R(r'[^\\\0]+', Id.PS_Literals),
694 # e.g. \x is not a valid escape.
695 C('\\', Id.PS_BadBackslash),
696]
697
698# NOTE: Id.Ignored_LineCont is also not supported here, even though the whole
699# point of it is that supports other backslash escapes like \n! It just
700# becomes a regular backslash.
701LEXER_DEF[lex_mode_e.SQ_C] = _C_STRING_COMMON + [
702 # Weird special case matching bash: backslash that ends a line. We emit
703 # this token literally in OSH, but disable it in YSH.
704 C('\\\n', Id.Unknown_Backslash),
705
706 # Silly difference! In echo -e, the syntax is \0377, but here it's $'\377',
707 # with no leading 0.
708 R(OCTAL3_RE, Id.Char_Octal3),
709
710 # ' and " are escaped in $'' mode, but not echo -e.
711 C(r"\'", Id.Char_OneChar),
712 C(r'\"', Id.Char_OneChar),
713
714 # e.g. 'foo', anything that's not a backslash escape or '
715 R(r"[^\\'\0]+", Id.Lit_Chars),
716 C("'", Id.Right_SingleQuote),
717]
718
719LEXER_DEF[lex_mode_e.PrintfOuter] = _C_STRING_COMMON + [
720 R(OCTAL3_RE, Id.Char_Octal3),
721 R(r"[^%\\\0]+", Id.Lit_Chars),
722 C('%%', Id.Format_EscapedPercent),
723 C('%', Id.Format_Percent),
724]
725
726# Maybe: bash also supports %(strftime)T
727LEXER_DEF[lex_mode_e.PrintfPercent] = [
728 # Flags
729 R('[- +#]', Id.Format_Flag),
730 C('0', Id.Format_Zero),
731 R('[1-9][0-9]*', Id.Format_Num),
732 C('*', Id.Format_Star),
733 C('.', Id.Format_Dot),
734 # We support dsq. The others we parse to display an error message.
735 R('[disqbcouxXeEfFgG]', Id.Format_Type),
736 R('\([^()\0]*\)T', Id.Format_Time),
737 R(r'[^\0]', Id.Unknown_Tok), # any other char
738]
739
740LEXER_DEF[lex_mode_e.VSub_1] = [
741 R(VAR_NAME_RE, Id.VSub_Name),
742 # ${11} is valid, compared to $11 which is $1 and then literal 1.
743 R(r'[0-9]+', Id.VSub_Number),
744 C('!', Id.VSub_Bang),
745 C('@', Id.VSub_At),
746 C('#', Id.VSub_Pound),
747 C('$', Id.VSub_Dollar),
748 C('*', Id.VSub_Star),
749 C('-', Id.VSub_Hyphen),
750 C('?', Id.VSub_QMark),
751 C('.', Id.VSub_Dot), # ${.myproc builtin sub}
752 C('}', Id.Right_DollarBrace),
753 C('\\\n', Id.Ignored_LineCont),
754 C('\n', Id.Unknown_Tok), # newline not allowed inside ${}
755 R(r'[^\0]', Id.Unknown_Tok), # any char except newline
756]
757
758LEXER_DEF[lex_mode_e.VSub_2] = \
759 ID_SPEC.LexerPairs(Kind.VTest) + \
760 ID_SPEC.LexerPairs(Kind.VOp0) + \
761 ID_SPEC.LexerPairs(Kind.VOpYsh) + \
762 ID_SPEC.LexerPairs(Kind.VOp1) + \
763 ID_SPEC.LexerPairs(Kind.VOp2) + \
764 ID_SPEC.LexerPairs(Kind.VOp3) + [
765 C('}', Id.Right_DollarBrace),
766
767 C('\\\n', Id.Ignored_LineCont),
768 C('\n', Id.Unknown_Tok), # newline not allowed inside ${}
769 R(r'[^\0]', Id.Unknown_Tok), # any char except newline
770]
771
772_EXPR_ARITH_SHARED = [
773 C('\\\n', Id.Ignored_LineCont),
774 R(r'[^\0]', Id.Unknown_Tok) # any char. This should be a syntax error.
775]
776
777# https://www.gnu.org/software/bash/manual/html_node/Shell-Arithmetic.html#Shell-Arithmetic
778LEXER_DEF[lex_mode_e.Arith] = \
779 _LEFT_SUBS + _VARS + _LEFT_UNQUOTED + [
780
781 # Arithmetic expressions can cross newlines.
782 R(r'[ \t\r\n]+', Id.Ignored_Space),
783
784 # Examples of arith constants:
785 # 64#azAZ
786 # 0xabc 0xABC
787 # 0123
788 # A separate digits token makes this easier to parse STATICALLY. But this
789 # doesn't help with DYNAMIC parsing.
790 R(VAR_NAME_RE, Id.Lit_ArithVarLike), # for variable names or 64#_
791 R(r'[0-9]+', Id.Lit_Digits),
792 C('@', Id.Lit_At), # for 64#@ or ${a[@]}
793 C('#', Id.Lit_Pound), # for 64#a
794
795 # TODO: 64#@ interferes with VS_AT. Hm.
796] + ID_SPEC.LexerPairs(Kind.Arith) + _EXPR_ARITH_SHARED
797
798# A lexer for the parser that converts globs to extended regexes. Since we're
799# only parsing character classes ([^[:space:][:alpha:]]) as opaque blobs, we
800# don't need lexer modes here.
801GLOB_DEF = [
802 # These could be operators in the glob, or just literals in a char class,
803 # e.g. touch '?'; echo [?].
804 C('*', Id.Glob_Star),
805 C('?', Id.Glob_QMark),
806
807 # For negation. Treated as operators inside [], but literals outside.
808 C('!', Id.Glob_Bang),
809 C('^', Id.Glob_Caret),
810
811 # Character classes.
812 C('[', Id.Glob_LBracket),
813 C(']', Id.Glob_RBracket),
814
815 # There is no whitelist of characters; backslashes are unconditionally
816 # removed. With libc.fnmatch(), the pattern r'\f' matches 'f' but not '\\f'.
817 # See libc_test.py.
818 R(r'\\[^\0]', Id.Glob_EscapedChar),
819 C('\\', Id.Glob_BadBackslash), # Trailing single backslash
820
821 # For efficiency, combine other characters into a single token, e.g. 'py' in
822 # '*.py' or 'alpha' in '[[:alpha:]]'.
823 R(r'[a-zA-Z0-9_]+', Id.Glob_CleanLiterals), # no regex escaping
824 R(r'[^\0]', Id.Glob_OtherLiteral), # anything else -- examine the char
825]
826
827# History expansion. We're doing this as "pre-lexing" since that's what bash
828# and zsh seem to do. Example:
829#
830# $ foo=x
831# $ echo $
832# $ !!foo # expands to echo $foo and prints x
833#
834# We can also reuse this in the RootCompleter to expand history interactively.
835#
836# bash note: handled in lib/readline/histexpand.c. Quite messy and handles
837# quotes AGAIN.
838#
839# Note: \! gets expanded to literal \! for the real lexer, but no history
840# expansion occurs.
841
842HISTORY_DEF = [
843 # Common operators.
844 R(r'![!*^$]', Id.History_Op),
845
846 # By command number.
847 R(r'!-?[0-9]+', Id.History_Num),
848
849 # Search by prefix of substring (optional '?').
850 # NOTE: there are no numbers allowed here! Bash doesn't seem to support it.
851 # No hyphen since it conflits with $-1 too.
852 #
853 # Required trailing whitespace is there to avoid conflict with [!charclass]
854 # and ${!indirect}. This is a simpler hack than the one bash has. See
855 # frontend/lex_test.py.
856 R(r'!\??[a-zA-Z_/.][0-9a-zA-Z_/.]+[ \t\r\n]', Id.History_Search),
857
858 # Comment is until end of line
859 R(r"#[^\0]*", Id.History_Other),
860
861 # Single quoted, e.g. 'a' or $'\n'. Terminated by another single quote or
862 # end of string.
863 R(r"'[^'\0]*'?", Id.History_Other),
864
865 # Runs of chars that are definitely not special
866 R(r"[^!\\'#\0]+", Id.History_Other),
867
868 # Escaped characters. \! disables history
869 R(r'\\[^\0]', Id.History_Other),
870 # Other single chars, like a trailing \ or !
871 R(r'[^\0]', Id.History_Other),
872]
873
874BRACE_RANGE_DEF = [
875 R(r'-?[0-9]+', Id.Range_Int),
876 R(r'[a-zA-Z]', Id.Range_Char), # just a single character
877 R(r'\.\.', Id.Range_Dots),
878 R(r'[^\0]', Id.Range_Other), # invalid
879]
880
881#
882# YSH lexing
883#
884
885# Valid in lex_mode_e.{Expr,DQ}
886# Used by ysh/grammar_gen.py
887YSH_LEFT_SUBS = [
888 C('$(', Id.Left_DollarParen),
889 C('${', Id.Left_DollarBrace),
890 C('$[', Id.Left_DollarBracket), # TODO: Implement $[x]
891]
892
893# Valid in lex_mode_e.Expr, but not valid in DQ
894# Used by ysh/grammar_gen.py
895
896YSH_LEFT_UNQUOTED = [
897 # Double quoted
898 C('"', Id.Left_DoubleQuote),
899 C('$"', Id.Left_DollarDoubleQuote), # $"" is synonym for ""
900 C('j"', Id.Left_JDoubleQuote), # for printing ERROR
901 # Single quoted
902 C("'", Id.Left_SingleQuote),
903 C("r'", Id.Left_RSingleQuote),
904 C("u'", Id.Left_USingleQuote),
905 C("b'", Id.Left_BSingleQuote),
906 C("$'", Id.Left_DollarSingleQuote), # legacy
907 C('^"', Id.Left_CaretDoubleQuote),
908 C('"""', Id.Left_TDoubleQuote),
909 C('$"""', Id.Left_DollarTDoubleQuote),
910 # In expression mode, we add the r'' and c'' prefixes for '' and $''.
911 C("'''", Id.Left_TSingleQuote),
912 C("r'''", Id.Left_RTSingleQuote),
913 C("u'''", Id.Left_UTSingleQuote),
914 C("b'''", Id.Left_BTSingleQuote),
915 C('@(', Id.Left_AtParen), # Split Command Sub
916 C('^(', Id.Left_CaretParen), # Block literals in expression mode
917 C('^[', Id.Left_CaretBracket), # Expr literals
918 C('^{', Id.Left_CaretBrace), # Unused
919 C(':|', Id.Left_ColonPipe), # shell-like word arrays.
920 C('%(', Id.Left_PercentParen), # old syntax for shell-like word arrays.
921 C('%[', Id.Expr_Reserved), # Maybe: like %() without unquoted [], {}
922 C('%{', Id.Expr_Reserved), # Table literals
923 # t = %{
924 # name:Str age:Int
925 # 'andy c' 10
926 # }
927 # Significant newlines. No unquoted [], {}
928
929 # Not sure if we'll use these
930 C('@{', Id.Expr_Reserved),
931 C('@[', Id.Expr_Reserved),
932
933 # Idea: Set literals are #{a, b} like Clojure
934]
935
936# Used by ysh/grammar_gen.py
937EXPR_OPS = [
938 # Terminator
939 C(';', Id.Op_Semi),
940 C('(', Id.Op_LParen),
941 C(')', Id.Op_RParen),
942 # NOTE: type expressions are expressions, e.g. Dict[Str, Int]
943 C('[', Id.Op_LBracket),
944 C(']', Id.Op_RBracket),
945 C('{', Id.Op_LBrace),
946 C('}', Id.Op_RBrace),
947]
948
949# Newline is significant, but sometimes elided by expr_parse.py.
950_EXPR_NEWLINE_COMMENT = [
951 C('\n', Id.Op_Newline),
952 R(r'#[^\n\0]*', Id.Ignored_Comment),
953 # Like lex_mode_e.Arith, \r is whitespace even without \n
954 R(r'[ \t\r]+', Id.Ignored_Space),
955]
956
957_WHITESPACE = r'[ \t\r\n]*' # ASCII whitespace doesn't have legacy \f \v
958
959# Python allows 0 to be written 00 or 0_0_0, which is weird. But let's be
960# consistent, and avoid '00' turning into a float!
961_DECIMAL_INT_RE = r'[0-9](_?[0-9])*'
962
963# Used for YSH comparison operators > >= < <=
964LOOKS_LIKE_INTEGER = _WHITESPACE + '-?' + _DECIMAL_INT_RE + _WHITESPACE
965
966_FLOAT_RE = (
967 _DECIMAL_INT_RE +
968 # Unlike Python, exponent can't be like 42e5_000. There's no use because
969 # 1e309 is already inf. Let's keep our code simple.
970 r'(\.' + _DECIMAL_INT_RE + ')?([eE][+\-]?[0-9]+)?')
971
972# Ditto, used for comparison operators
973# Added optional Optional -?
974# Example: -3_000_000.000_001e12
975LOOKS_LIKE_FLOAT = _WHITESPACE + '-?' + _FLOAT_RE + _WHITESPACE
976
977# Python 3 float literals:
978
979# digitpart ::= digit (["_"] digit)*
980# fraction ::= "." digitpart
981# exponent ::= ("e" | "E") ["+" | "-"] digitpart
982# pointfloat ::= [digitpart] fraction | digitpart "."
983# exponentfloat ::= (digitpart | pointfloat) exponent
984# floatnumber ::= pointfloat | exponentfloat
985
986# NOTE: Borrowing tokens from Arith (i.e. $(( )) ), but not using LexerPairs().
987LEXER_DEF[lex_mode_e.Expr] = \
988 _VARS + YSH_LEFT_SUBS + YSH_LEFT_UNQUOTED + EXPR_OPS + EXPR_WORDS + \
989 EXPR_CHARS + [
990
991 # https://docs.python.org/3/reference/lexical_analysis.html#integer-literals
992 #
993 # integer ::= decinteger | bininteger | octinteger | hexinteger
994 # decinteger ::= nonzerodigit (["_"] digit)* | "0"+ (["_"] "0")*
995 # bininteger ::= "0" ("b" | "B") (["_"] bindigit)+
996 # octinteger ::= "0" ("o" | "O") (["_"] octdigit)+
997 # hexinteger ::= "0" ("x" | "X") (["_"] hexdigit)+
998 # nonzerodigit ::= "1"..."9"
999 # digit ::= "0"..."9"
1000 # bindigit ::= "0" | "1"
1001 # octdigit ::= "0"..."7"
1002 # hexdigit ::= digit | "a"..."f" | "A"..."F"
1003
1004 R(_DECIMAL_INT_RE, Id.Expr_DecInt),
1005
1006 R(r'0[bB](_?[01])+', Id.Expr_BinInt),
1007 R(r'0[oO](_?[0-7])+', Id.Expr_OctInt),
1008 R(r'0[xX](_?[0-9a-fA-F])+', Id.Expr_HexInt),
1009
1010 R(_FLOAT_RE, Id.Expr_Float),
1011
1012 # These can be looked up as keywords separately, so you enforce that they have
1013 # space around them?
1014 R(VAR_NAME_RE, Id.Expr_Name),
1015
1016 R('%' + VAR_NAME_RE, Id.Expr_Symbol),
1017
1018 #
1019 # Arith
1020 #
1021
1022 C(',', Id.Arith_Comma),
1023 C(':', Id.Arith_Colon), # for slicing a[1:2], and mylist:pop()
1024
1025 C('?', Id.Arith_QMark), # regex postfix
1026
1027 C('+', Id.Arith_Plus), # arith infix, regex postfix
1028 C('-', Id.Arith_Minus), # arith infix, regex postfix
1029 C('*', Id.Arith_Star),
1030 C('^', Id.Arith_Caret), # xor
1031 C('/', Id.Arith_Slash),
1032 C('%', Id.Arith_Percent),
1033
1034 C('**', Id.Arith_DStar), # exponentiation
1035 C('++', Id.Arith_DPlus), # Option for string/list concatenation
1036
1037 C('<', Id.Arith_Less),
1038 C('>', Id.Arith_Great),
1039 C('<=', Id.Arith_LessEqual),
1040 C('>=', Id.Arith_GreatEqual),
1041 C('===', Id.Expr_TEqual),
1042 C('!==', Id.Expr_NotDEqual),
1043
1044 C('==', Id.Unknown_DEqual), # user must choose === or ~==
1045
1046 # Bitwise operators
1047 C('&', Id.Arith_Amp),
1048 C('|', Id.Arith_Pipe),
1049 C('>>', Id.Arith_DGreat),
1050 C('<<', Id.Arith_DLess), # Doesn't Java also have <<< ?
1051
1052 # Bitwise complement, as well as infix pattern matching
1053 C('~', Id.Arith_Tilde),
1054 C('!~', Id.Expr_NotTilde),
1055 C('~~', Id.Expr_DTilde),
1056 C('!~~', Id.Expr_NotDTilde),
1057
1058 # Left out for now:
1059 # ++ -- -- needed for loops, awk?
1060 # ! && || -- needed for find dialect
1061 # = += etc.
1062
1063 C('=', Id.Arith_Equal),
1064
1065 C('+=', Id.Arith_PlusEqual),
1066 C('-=', Id.Arith_MinusEqual),
1067 C('*=', Id.Arith_StarEqual),
1068 C('/=', Id.Arith_SlashEqual),
1069 C('%=', Id.Arith_PercentEqual),
1070
1071 C('>>=', Id.Arith_DGreatEqual),
1072 C('<<=', Id.Arith_DLessEqual),
1073 C('&=', Id.Arith_AmpEqual),
1074 C('|=', Id.Arith_PipeEqual),
1075 C('^=', Id.Arith_CaretEqual), # Exponentiation
1076
1077 # Augmented assignment that YSH has, but sh and OSH don't have
1078 C('**=', Id.Expr_DStarEqual),
1079 C('//=', Id.Expr_DSlashEqual),
1080
1081 #
1082 # Expr
1083 #
1084
1085 C('!', Id.Expr_Bang), # For eggex negation
1086
1087 C('//', Id.Expr_DSlash), # For YSH integer division
1088 C('~==', Id.Expr_TildeDEqual), # approximate equality
1089
1090 C('.', Id.Expr_Dot), # d.key is alias for d['key']
1091 C('..', Id.Expr_DDot), # range 1..5
1092 C('->', Id.Expr_RArrow), # s->startswith()
1093 C('$', Id.Expr_Dollar), # legacy regex end: /d+ $/ (better written /d+ >/
1094
1095 # Reserved this. Go uses it for channels, etc.
1096 # I guess it conflicts with -4<-3, but that's OK -- spaces suffices.
1097 C('<-', Id.Expr_Reserved),
1098 C('=>', Id.Expr_RDArrow), # for df => filter(age > 10)
1099 # and match (x) { 1 => "one" }
1100 # note: other languages use |>
1101 # R/dplyr uses %>%
1102
1103 C('...', Id.Expr_Ellipsis), # f(...args) and maybe a[:, ...]
1104
1105 # For multiline regex literals?
1106 C('///', Id.Expr_Reserved),
1107
1108 # Splat operators
1109 C('@', Id.Expr_At),
1110 # NOTE: Unused
1111 C('@@', Id.Expr_DoubleAt),
1112] + _EXPR_NEWLINE_COMMENT + _EXPR_ARITH_SHARED
1113
1114LEXER_DEF[lex_mode_e.FuncParens] = [
1115 # () with spaces
1116 R(r'[ \t]*\([ \t]*\)', Id.LookAhead_FuncParens),
1117 # anything else
1118 R(r'[^\0]', Id.Unknown_Tok)
1119]