OILS / frontend / args.py View on Github | oilshell.org

669 lines, 345 significant
1"""
2args.py - Flag, option, and arg parsing for the shell.
3
4All existing shells have their own flag parsing, rather than using libc.
5
6We have 3 types of flag parsing here:
7
8 FlagSpecAndMore() -- e.g. for 'sh +u -o errexit' and 'set +u -o errexit'
9 FlagSpec() -- for echo -en, read -t1.0, etc.
10
11Examples:
12 set -opipefail # not allowed, space required
13 read -t1.0 # allowed
14
15Things that getopt/optparse don't support:
16
17- accepts +o +n for 'set' and bin/osh
18 - pushd and popd also uses +, although it's not an arg.
19- parses args -- well argparse is supposed to do this
20- maybe: integrate with usage
21- maybe: integrate with flags
22
23optparse:
24 - has option groups (Go flag package has flagset)
25
26NOTES about builtins:
27- eval and echo implicitly join their args. We don't want that.
28 - option strict-eval and strict-echo
29- bash is inconsistent about checking for extra args
30 - exit 1 2 complains, but pushd /lib /bin just ignores second argument
31 - it has a no_args() function that isn't called everywhere. It's not
32 declarative.
33
34TODO:
35 - Autogenerate help from help='' fields. Usage line like FlagSpec('echo [-en]')
36
37GNU notes:
38
39- Consider adding GNU-style option to interleave flags and args?
40 - Not sure I like this.
41- GNU getopt has fuzzy matching for long flags. I think we should rely
42 on good completion instead.
43
44Bash notes:
45
46bashgetopt.c codes:
47 leading +: allow options
48 : requires argument
49 ; argument may be missing
50 # numeric argument
51
52However I don't see these used anywhere! I only see ':' used.
53"""
54from __future__ import print_function
55
56from _devbuild.gen.syntax_asdl import loc, loc_t, CompoundWord
57from _devbuild.gen.value_asdl import (value, value_e, value_t)
58
59from core.error import e_usage
60from mycpp import mops
61from mycpp.mylib import log, tagswitch, iteritems
62
63_ = log
64
65from typing import (cast, Tuple, Optional, Dict, List, Any, TYPE_CHECKING)
66if TYPE_CHECKING:
67 from frontend import flag_spec
68 OptChange = Tuple[str, bool]
69
70# TODO: Move to flag_spec? We use flag_type_t
71String = 1
72Int = 2
73Float = 3 # e.g. for read -t timeout value
74Bool = 4
75
76
77class _Attributes(object):
78 """Object to hold flags.
79
80 TODO: FlagSpec doesn't need this; only FlagSpecAndMore.
81 """
82
83 def __init__(self, defaults):
84 # type: (Dict[str, value_t]) -> None
85
86 # New style
87 self.attrs = {} # type: Dict[str, value_t]
88
89 self.opt_changes = [] # type: List[OptChange] # -o errexit +o nounset
90 self.shopt_changes = [
91 ] # type: List[OptChange] # -O nullglob +O nullglob
92 self.show_options = False # 'set -o' without an argument
93 self.actions = [] # type: List[str] # for compgen -A
94 self.saw_double_dash = False # for set --
95 for name, v in iteritems(defaults):
96 self.Set(name, v)
97
98 def SetTrue(self, name):
99 # type: (str) -> None
100 self.Set(name, value.Bool(True))
101
102 def Set(self, name, val):
103 # type: (str, value_t) -> None
104
105 # debug-completion -> debug_completion
106 name = name.replace('-', '_')
107 self.attrs[name] = val
108
109 if 0:
110 # Backward compatibility!
111 with tagswitch(val) as case:
112 if case(value_e.Undef):
113 py_val = None # type: Any
114 elif case(value_e.Bool):
115 py_val = cast(value.Bool, val).b
116 elif case(value_e.Int):
117 py_val = cast(value.Int, val).i
118 elif case(value_e.Float):
119 py_val = cast(value.Float, val).f
120 elif case(value_e.Str):
121 py_val = cast(value.Str, val).s
122 else:
123 raise AssertionError(val)
124
125 setattr(self, name, py_val)
126
127 def __repr__(self):
128 # type: () -> str
129 return '<_Attributes %s>' % self.__dict__
130
131
132class Reader(object):
133 """Wrapper for argv.
134
135 Modified by both the parsing loop and various actions.
136
137 The caller of the flags parser can continue to use it after flag parsing is
138 done to get args.
139 """
140
141 def __init__(self, argv, locs=None):
142 # type: (List[str], Optional[List[CompoundWord]]) -> None
143 self.argv = argv
144 self.locs = locs
145 self.n = len(argv)
146 self.i = 0
147
148 def __repr__(self):
149 # type: () -> str
150 return '<args.Reader %r %d>' % (self.argv, self.i)
151
152 def Next(self):
153 # type: () -> None
154 """Advance."""
155 self.i += 1
156
157 def Peek(self):
158 # type: () -> Optional[str]
159 """Return the next token, or None if there are no more.
160
161 None is your SENTINEL for parsing.
162 """
163 if self.i >= self.n:
164 return None
165 else:
166 return self.argv[self.i]
167
168 def Peek2(self):
169 # type: () -> Tuple[Optional[str], loc_t]
170 """Return the next token, or None if there are no more.
171
172 None is your SENTINEL for parsing.
173 """
174 if self.i >= self.n:
175 return None, loc.Missing
176 else:
177 return self.argv[self.i], self.locs[self.i]
178
179 def ReadRequired(self, error_msg):
180 # type: (str) -> str
181 arg = self.Peek()
182 if arg is None:
183 # point at argv[0]
184 e_usage(error_msg, self._FirstLocation())
185 self.Next()
186 return arg
187
188 def ReadRequired2(self, error_msg):
189 # type: (str) -> Tuple[str, loc_t]
190 arg = self.Peek()
191 if arg is None:
192 # point at argv[0]
193 e_usage(error_msg, self._FirstLocation())
194 location = self.locs[self.i]
195 self.Next()
196 return arg, location
197
198 def Rest(self):
199 # type: () -> List[str]
200 """Return the rest of the arguments."""
201 return self.argv[self.i:]
202
203 def Rest2(self):
204 # type: () -> Tuple[List[str], List[CompoundWord]]
205 """Return the rest of the arguments."""
206 return self.argv[self.i:], self.locs[self.i:]
207
208 def AtEnd(self):
209 # type: () -> bool
210 return self.i >= self.n # must be >= and not ==
211
212 def Done(self):
213 # type: () -> None
214 if not self.AtEnd():
215 e_usage('got too many arguments', self.Location())
216
217 def _FirstLocation(self):
218 # type: () -> loc_t
219 if self.locs is not None and self.locs[0] is not None:
220 return self.locs[0]
221 else:
222 return loc.Missing
223
224 def Location(self):
225 # type: () -> loc_t
226 if self.locs is not None:
227 if self.i == self.n:
228 i = self.n - 1 # if the last arg is missing, point at the one before
229 else:
230 i = self.i
231 if self.locs[i] is not None:
232 return self.locs[i]
233 else:
234 return loc.Missing
235 else:
236 return loc.Missing
237
238
239class _Action(object):
240 """What is done when a flag or option is detected."""
241
242 def __init__(self):
243 # type: () -> None
244 """Empty constructor for mycpp."""
245 pass
246
247 def OnMatch(self, attached_arg, arg_r, out):
248 # type: (Optional[str], Reader, _Attributes) -> bool
249 """Called when the flag matches.
250
251 Args:
252 prefix: '-' or '+'
253 suffix: ',' for -d,
254 arg_r: Reader() (rename to Input or InputReader?)
255 out: _Attributes() -- the thing we want to set
256
257 Returns:
258 True if flag parsing should be aborted.
259 """
260 raise NotImplementedError()
261
262
263class _ArgAction(_Action):
264
265 def __init__(self, name, quit_parsing_flags, valid=None):
266 # type: (str, bool, Optional[List[str]]) -> None
267 """
268 Args:
269 quit_parsing_flags: Stop parsing args after this one. for sh -c.
270 python -c behaves the same way.
271 """
272 self.name = name
273 self.quit_parsing_flags = quit_parsing_flags
274 self.valid = valid
275
276 def _Value(self, arg, location):
277 # type: (str, loc_t) -> value_t
278 raise NotImplementedError()
279
280 def OnMatch(self, attached_arg, arg_r, out):
281 # type: (Optional[str], Reader, _Attributes) -> bool
282 """Called when the flag matches."""
283 if attached_arg is not None: # for the ',' in -d,
284 arg = attached_arg
285 else:
286 arg_r.Next()
287 arg = arg_r.Peek()
288 if arg is None:
289 e_usage('expected argument to %r' % ('-' + self.name),
290 arg_r.Location())
291
292 val = self._Value(arg, arg_r.Location())
293 out.Set(self.name, val)
294 return self.quit_parsing_flags
295
296
297class SetToInt(_ArgAction):
298
299 def __init__(self, name):
300 # type: (str) -> None
301 # repeat defaults for C++ translation
302 _ArgAction.__init__(self, name, False, valid=None)
303
304 def _Value(self, arg, location):
305 # type: (str, loc_t) -> value_t
306 try:
307 i = mops.FromStr(arg)
308 except ValueError:
309 e_usage(
310 'expected integer after %s, got %r' % ('-' + self.name, arg),
311 location)
312
313 # So far all our int values are > 0, so use -1 as the 'unset' value
314 # corner case: this treats -0 as 0!
315 if mops.Greater(mops.BigInt(0), i):
316 e_usage('got invalid integer for %s: %s' % ('-' + self.name, arg),
317 location)
318 return value.Int(i)
319
320
321class SetToFloat(_ArgAction):
322
323 def __init__(self, name):
324 # type: (str) -> None
325 # repeat defaults for C++ translation
326 _ArgAction.__init__(self, name, False, valid=None)
327
328 def _Value(self, arg, location):
329 # type: (str, loc_t) -> value_t
330 try:
331 f = float(arg)
332 except ValueError:
333 e_usage(
334 'expected number after %r, got %r' % ('-' + self.name, arg),
335 location)
336 # So far all our float values are > 0, so use -1.0 as the 'unset' value
337 # corner case: this treats -0.0 as 0.0!
338 if f < 0:
339 e_usage('got invalid float for %s: %s' % ('-' + self.name, arg),
340 location)
341 return value.Float(f)
342
343
344class SetToString(_ArgAction):
345
346 def __init__(self, name, quit_parsing_flags, valid=None):
347 # type: (str, bool, Optional[List[str]]) -> None
348 _ArgAction.__init__(self, name, quit_parsing_flags, valid=valid)
349
350 def _Value(self, arg, location):
351 # type: (str, loc_t) -> value_t
352 if self.valid is not None and arg not in self.valid:
353 e_usage(
354 'got invalid argument %r to %r, expected one of: %s' %
355 (arg, ('-' + self.name), '|'.join(self.valid)), location)
356 return value.Str(arg)
357
358
359class SetAttachedBool(_Action):
360
361 def __init__(self, name):
362 # type: (str) -> None
363 self.name = name
364
365 def OnMatch(self, attached_arg, arg_r, out):
366 # type: (Optional[str], Reader, _Attributes) -> bool
367 """Called when the flag matches."""
368
369 # TODO: Delete this part? Is this eqvuivalent to SetToTrue?
370 #
371 # We're not using Go-like --verbose=1, --verbose, or --verbose=0
372 #
373 # 'attached_arg' is also used for -t0 though, which is weird
374
375 if attached_arg is not None: # '0' in --verbose=0
376 if attached_arg in ('0', 'F', 'false',
377 'False'): # TODO: incorrect translation
378 b = False
379 elif attached_arg in ('1', 'T', 'true', 'Talse'):
380 b = True
381 else:
382 e_usage(
383 'got invalid argument to boolean flag: %r' % attached_arg,
384 loc.Missing)
385 else:
386 b = True
387
388 out.Set(self.name, value.Bool(b))
389 return False
390
391
392class SetToTrue(_Action):
393
394 def __init__(self, name):
395 # type: (str) -> None
396 self.name = name
397
398 def OnMatch(self, attached_arg, arg_r, out):
399 # type: (Optional[str], Reader, _Attributes) -> bool
400 """Called when the flag matches."""
401 out.SetTrue(self.name)
402 return False
403
404
405class SetOption(_Action):
406 """Set an option to a boolean, for 'set +e'."""
407
408 def __init__(self, name):
409 # type: (str) -> None
410 self.name = name
411
412 def OnMatch(self, attached_arg, arg_r, out):
413 # type: (Optional[str], Reader, _Attributes) -> bool
414 """Called when the flag matches."""
415 b = (attached_arg == '-')
416 out.opt_changes.append((self.name, b))
417 return False
418
419
420class SetNamedOption(_Action):
421 """Set a named option to a boolean, for 'set +o errexit'."""
422
423 def __init__(self, shopt=False):
424 # type: (bool) -> None
425 self.names = [] # type: List[str]
426 self.shopt = shopt # is it sh -o (set) or sh -O (shopt)?
427
428 def ArgName(self, name):
429 # type: (str) -> None
430 self.names.append(name)
431
432 def OnMatch(self, attached_arg, arg_r, out):
433 # type: (Optional[str], Reader, _Attributes) -> bool
434 """Called when the flag matches."""
435 b = (attached_arg == '-')
436 #log('SetNamedOption %r %r %r', prefix, suffix, arg_r)
437 arg_r.Next() # always advance
438 arg = arg_r.Peek()
439 if arg is None:
440 # triggers on 'set -O' in addition to 'set -o' (meh OK)
441 out.show_options = True
442 return True # quit parsing
443
444 attr_name = arg # Note: validation is done elsewhere
445 if len(self.names) and attr_name not in self.names:
446 e_usage('Invalid option %r' % arg, loc.Missing)
447 changes = out.shopt_changes if self.shopt else out.opt_changes
448 changes.append((attr_name, b))
449 return False
450
451
452class SetAction(_Action):
453 """For compgen -f."""
454
455 def __init__(self, name):
456 # type: (str) -> None
457 self.name = name
458
459 def OnMatch(self, attached_arg, arg_r, out):
460 # type: (Optional[str], Reader, _Attributes) -> bool
461 out.actions.append(self.name)
462 return False
463
464
465class SetNamedAction(_Action):
466 """For compgen -A file."""
467
468 def __init__(self):
469 # type: () -> None
470 self.names = [] # type: List[str]
471
472 def ArgName(self, name):
473 # type: (str) -> None
474 self.names.append(name)
475
476 def OnMatch(self, attached_arg, arg_r, out):
477 # type: (Optional[str], Reader, _Attributes) -> bool
478 """Called when the flag matches."""
479 arg_r.Next() # always advance
480 arg = arg_r.Peek()
481 if arg is None:
482 e_usage('Expected argument for action', loc.Missing)
483
484 attr_name = arg
485 # Validate the option name against a list of valid names.
486 if len(self.names) and attr_name not in self.names:
487 e_usage('Invalid action name %r' % arg, loc.Missing)
488 out.actions.append(attr_name)
489 return False
490
491
492def Parse(spec, arg_r):
493 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
494
495 # NOTE about -:
496 # 'set -' ignores it, vs set
497 # 'unset -' or 'export -' seems to treat it as a variable name
498 out = _Attributes(spec.defaults)
499
500 while not arg_r.AtEnd():
501 arg = arg_r.Peek()
502 if arg == '--':
503 out.saw_double_dash = True
504 arg_r.Next()
505 break
506
507 # Only accept -- if there are any long flags defined
508 if len(spec.actions_long) and arg.startswith('--'):
509 pos = arg.find('=', 2)
510 if pos == -1:
511 suffix = None # type: Optional[str]
512 flag_name = arg[2:] # strip off --
513 else:
514 suffix = arg[pos + 1:]
515 flag_name = arg[2:pos]
516
517 action = spec.actions_long.get(flag_name)
518 if action is None:
519 e_usage('got invalid flag %r' % arg, arg_r.Location())
520
521 action.OnMatch(suffix, arg_r, out)
522 arg_r.Next()
523 continue
524
525 elif arg.startswith('-') and len(arg) > 1:
526 n = len(arg)
527 for i in xrange(1, n): # parse flag combos like -rx
528 ch = arg[i]
529
530 if ch == '0':
531 ch = 'Z' # hack for read -0
532
533 if ch in spec.plus_flags:
534 out.Set(ch, value.Str('-'))
535 continue
536
537 if ch in spec.arity0: # e.g. read -r
538 out.SetTrue(ch)
539 continue
540
541 if ch in spec.arity1: # e.g. read -t1.0
542 action = spec.arity1[ch]
543 # make sure we don't pass empty string for read -t
544 attached_arg = arg[i + 1:] if i < n - 1 else None
545 action.OnMatch(attached_arg, arg_r, out)
546 break
547
548 e_usage("doesn't accept flag %s" % ('-' + ch),
549 arg_r.Location())
550
551 arg_r.Next() # next arg
552
553 # Only accept + if there are ANY options defined, e.g. for declare +rx.
554 elif len(spec.plus_flags) and arg.startswith('+') and len(arg) > 1:
555 n = len(arg)
556 for i in xrange(1, n): # parse flag combos like -rx
557 ch = arg[i]
558 if ch in spec.plus_flags:
559 out.Set(ch, value.Str('+'))
560 continue
561
562 e_usage("doesn't accept option %s" % ('+' + ch),
563 arg_r.Location())
564
565 arg_r.Next() # next arg
566
567 else: # a regular arg
568 break
569
570 return out
571
572
573def ParseLikeEcho(spec, arg_r):
574 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
575 """Echo is a special case. These work: echo -n echo -en.
576
577 - But don't respect --
578 - doesn't fail when an invalid flag is passed
579 """
580 out = _Attributes(spec.defaults)
581
582 while not arg_r.AtEnd():
583 arg = arg_r.Peek()
584 chars = arg[1:]
585 if arg.startswith('-') and len(chars):
586 # Check if it looks like -en or not. TODO: could optimize this.
587 done = False
588 for c in chars:
589 if c not in spec.arity0:
590 done = True
591 break
592 if done:
593 break
594
595 for ch in chars:
596 out.SetTrue(ch)
597
598 else:
599 break # Looks like an arg
600
601 arg_r.Next() # next arg
602
603 return out
604
605
606def ParseMore(spec, arg_r):
607 # type: (flag_spec._FlagSpecAndMore, Reader) -> _Attributes
608 """Return attributes and an index.
609
610 Respects +, like set +eu
611
612 We do NOT respect:
613
614 WRONG: sh -cecho OK: sh -c echo
615 WRONG: set -opipefail OK: set -o pipefail
616
617 But we do accept these
618
619 set -euo pipefail
620 set -oeu pipefail
621 set -oo pipefail errexit
622 """
623 out = _Attributes(spec.defaults)
624
625 quit = False
626 while not arg_r.AtEnd():
627 arg = arg_r.Peek()
628 if arg == '--':
629 out.saw_double_dash = True
630 arg_r.Next()
631 break
632
633 if arg.startswith('--'):
634 action = spec.actions_long.get(arg[2:])
635 if action is None:
636 e_usage('got invalid flag %r' % arg, arg_r.Location())
637
638 # Note: not parsing --foo=bar as attached_arg, as above
639 action.OnMatch(None, arg_r, out)
640 arg_r.Next()
641 continue
642
643 # corner case: sh +c is also accepted!
644 if (arg.startswith('-') or arg.startswith('+')) and len(arg) > 1:
645 # note: we're not handling sh -cecho (no space) as an argument
646 # It complains about a missing argument
647
648 char0 = arg[0]
649
650 # TODO: set - - empty
651 for ch in arg[1:]:
652 #log('ch %r arg_r %s', ch, arg_r)
653 action = spec.actions_short.get(ch)
654 if action is None:
655 e_usage('got invalid flag %r' % ('-' + ch),
656 arg_r.Location())
657
658 attached_arg = char0 if ch in spec.plus_flags else None
659 quit = action.OnMatch(attached_arg, arg_r, out)
660 arg_r.Next() # process the next flag
661
662 if quit:
663 break
664 else:
665 continue
666
667 break # it's a regular arg
668
669 return out