OILS / frontend / args.py View on Github | oilshell.org

664 lines, 342 significant
1"""
2args.py - Flag, option, and arg parsing for the shell.
3
4All existing shells have their own flag parsing, rather than using libc.
5
6We have 3 types of flag parsing here:
7
8 FlagSpecAndMore() -- e.g. for 'sh +u -o errexit' and 'set +u -o errexit'
9 FlagSpec() -- for echo -en, read -t1.0, etc.
10
11Examples:
12 set -opipefail # not allowed, space required
13 read -t1.0 # allowed
14
15Things that getopt/optparse don't support:
16
17- accepts +o +n for 'set' and bin/osh
18 - pushd and popd also uses +, although it's not an arg.
19- parses args -- well argparse is supposed to do this
20- maybe: integrate with usage
21- maybe: integrate with flags
22
23optparse:
24 - has option groups (Go flag package has flagset)
25
26NOTES about builtins:
27- eval and echo implicitly join their args. We don't want that.
28 - option strict-eval and strict-echo
29- bash is inconsistent about checking for extra args
30 - exit 1 2 complains, but pushd /lib /bin just ignores second argument
31 - it has a no_args() function that isn't called everywhere. It's not
32 declarative.
33
34TODO:
35 - Autogenerate help from help='' fields. Usage line like FlagSpec('echo [-en]')
36
37GNU notes:
38
39- Consider adding GNU-style option to interleave flags and args?
40 - Not sure I like this.
41- GNU getopt has fuzzy matching for long flags. I think we should rely
42 on good completion instead.
43
44Bash notes:
45
46bashgetopt.c codes:
47 leading +: allow options
48 : requires argument
49 ; argument may be missing
50 # numeric argument
51
52However I don't see these used anywhere! I only see ':' used.
53"""
54from __future__ import print_function
55
56from _devbuild.gen.syntax_asdl import loc, loc_t, CompoundWord
57from _devbuild.gen.value_asdl import (value, value_e, value_t)
58
59from core.error import e_usage
60from mycpp import mops
61from mycpp.mylib import log, tagswitch, iteritems
62
63_ = log
64
65from typing import (cast, Tuple, Optional, Dict, List, Any, TYPE_CHECKING)
66if TYPE_CHECKING:
67 from frontend import flag_spec
68 OptChange = Tuple[str, bool]
69
70# TODO: Move to flag_spec? We use flag_type_t
71String = 1
72Int = 2
73Float = 3 # e.g. for read -t timeout value
74Bool = 4
75
76
77class _Attributes(object):
78 """Object to hold flags.
79
80 TODO: FlagSpec doesn't need this; only FlagSpecAndMore.
81 """
82
83 def __init__(self, defaults):
84 # type: (Dict[str, value_t]) -> None
85
86 # New style
87 self.attrs = {} # type: Dict[str, value_t]
88
89 self.opt_changes = [] # type: List[OptChange] # -o errexit +o nounset
90 self.shopt_changes = [
91 ] # type: List[OptChange] # -O nullglob +O nullglob
92 self.show_options = False # 'set -o' without an argument
93 self.actions = [] # type: List[str] # for compgen -A
94 self.saw_double_dash = False # for set --
95 for name, v in iteritems(defaults):
96 self.Set(name, v)
97
98 def SetTrue(self, name):
99 # type: (str) -> None
100 self.Set(name, value.Bool(True))
101
102 def Set(self, name, val):
103 # type: (str, value_t) -> None
104
105 # debug-completion -> debug_completion
106 name = name.replace('-', '_')
107 self.attrs[name] = val
108
109 if 0:
110 # Backward compatibility!
111 with tagswitch(val) as case:
112 if case(value_e.Undef):
113 py_val = None # type: Any
114 elif case(value_e.Bool):
115 py_val = cast(value.Bool, val).b
116 elif case(value_e.Int):
117 py_val = cast(value.Int, val).i
118 elif case(value_e.Float):
119 py_val = cast(value.Float, val).f
120 elif case(value_e.Str):
121 py_val = cast(value.Str, val).s
122 else:
123 raise AssertionError(val)
124
125 setattr(self, name, py_val)
126
127 def __repr__(self):
128 # type: () -> str
129 return '<_Attributes %s>' % self.__dict__
130
131
132class Reader(object):
133 """Wrapper for argv.
134
135 Modified by both the parsing loop and various actions.
136
137 The caller of the flags parser can continue to use it after flag parsing is
138 done to get args.
139 """
140
141 def __init__(self, argv, locs=None):
142 # type: (List[str], Optional[List[CompoundWord]]) -> None
143 self.argv = argv
144 self.locs = locs
145 self.n = len(argv)
146 self.i = 0
147
148 def __repr__(self):
149 # type: () -> str
150 return '<args.Reader %r %d>' % (self.argv, self.i)
151
152 def Next(self):
153 # type: () -> None
154 """Advance."""
155 self.i += 1
156
157 def Peek(self):
158 # type: () -> Optional[str]
159 """Return the next token, or None if there are no more.
160
161 None is your SENTINEL for parsing.
162 """
163 if self.i >= self.n:
164 return None
165 else:
166 return self.argv[self.i]
167
168 def Peek2(self):
169 # type: () -> Tuple[Optional[str], loc_t]
170 """Return the next token, or None if there are no more.
171
172 None is your SENTINEL for parsing.
173 """
174 if self.i >= self.n:
175 return None, loc.Missing
176 else:
177 return self.argv[self.i], self.locs[self.i]
178
179 def ReadRequired(self, error_msg):
180 # type: (str) -> str
181 arg = self.Peek()
182 if arg is None:
183 # point at argv[0]
184 e_usage(error_msg, self._FirstLocation())
185 self.Next()
186 return arg
187
188 def ReadRequired2(self, error_msg):
189 # type: (str) -> Tuple[str, loc_t]
190 arg = self.Peek()
191 if arg is None:
192 # point at argv[0]
193 e_usage(error_msg, self._FirstLocation())
194 location = self.locs[self.i]
195 self.Next()
196 return arg, location
197
198 def Rest(self):
199 # type: () -> List[str]
200 """Return the rest of the arguments."""
201 return self.argv[self.i:]
202
203 def Rest2(self):
204 # type: () -> Tuple[List[str], List[CompoundWord]]
205 """Return the rest of the arguments."""
206 return self.argv[self.i:], self.locs[self.i:]
207
208 def AtEnd(self):
209 # type: () -> bool
210 return self.i >= self.n # must be >= and not ==
211
212 def _FirstLocation(self):
213 # type: () -> loc_t
214 if self.locs is not None and self.locs[0] is not None:
215 return self.locs[0]
216 else:
217 return loc.Missing
218
219 def Location(self):
220 # type: () -> loc_t
221 if self.locs is not None:
222 if self.i == self.n:
223 i = self.n - 1 # if the last arg is missing, point at the one before
224 else:
225 i = self.i
226 if self.locs[i] is not None:
227 return self.locs[i]
228 else:
229 return loc.Missing
230 else:
231 return loc.Missing
232
233
234class _Action(object):
235 """What is done when a flag or option is detected."""
236
237 def __init__(self):
238 # type: () -> None
239 """Empty constructor for mycpp."""
240 pass
241
242 def OnMatch(self, attached_arg, arg_r, out):
243 # type: (Optional[str], Reader, _Attributes) -> bool
244 """Called when the flag matches.
245
246 Args:
247 prefix: '-' or '+'
248 suffix: ',' for -d,
249 arg_r: Reader() (rename to Input or InputReader?)
250 out: _Attributes() -- the thing we want to set
251
252 Returns:
253 True if flag parsing should be aborted.
254 """
255 raise NotImplementedError()
256
257
258class _ArgAction(_Action):
259
260 def __init__(self, name, quit_parsing_flags, valid=None):
261 # type: (str, bool, Optional[List[str]]) -> None
262 """
263 Args:
264 quit_parsing_flags: Stop parsing args after this one. for sh -c.
265 python -c behaves the same way.
266 """
267 self.name = name
268 self.quit_parsing_flags = quit_parsing_flags
269 self.valid = valid
270
271 def _Value(self, arg, location):
272 # type: (str, loc_t) -> value_t
273 raise NotImplementedError()
274
275 def OnMatch(self, attached_arg, arg_r, out):
276 # type: (Optional[str], Reader, _Attributes) -> bool
277 """Called when the flag matches."""
278 if attached_arg is not None: # for the ',' in -d,
279 arg = attached_arg
280 else:
281 arg_r.Next()
282 arg = arg_r.Peek()
283 if arg is None:
284 e_usage('expected argument to %r' % ('-' + self.name),
285 arg_r.Location())
286
287 val = self._Value(arg, arg_r.Location())
288 out.Set(self.name, val)
289 return self.quit_parsing_flags
290
291
292class SetToInt(_ArgAction):
293
294 def __init__(self, name):
295 # type: (str) -> None
296 # repeat defaults for C++ translation
297 _ArgAction.__init__(self, name, False, valid=None)
298
299 def _Value(self, arg, location):
300 # type: (str, loc_t) -> value_t
301 try:
302 i = mops.FromStr(arg)
303 except ValueError:
304 e_usage(
305 'expected integer after %s, got %r' % ('-' + self.name, arg),
306 location)
307
308 # So far all our int values are > 0, so use -1 as the 'unset' value
309 # corner case: this treats -0 as 0!
310 if mops.Greater(mops.BigInt(0), i):
311 e_usage('got invalid integer for %s: %s' % ('-' + self.name, arg),
312 location)
313 return value.Int(i)
314
315
316class SetToFloat(_ArgAction):
317
318 def __init__(self, name):
319 # type: (str) -> None
320 # repeat defaults for C++ translation
321 _ArgAction.__init__(self, name, False, valid=None)
322
323 def _Value(self, arg, location):
324 # type: (str, loc_t) -> value_t
325 try:
326 f = float(arg)
327 except ValueError:
328 e_usage(
329 'expected number after %r, got %r' % ('-' + self.name, arg),
330 location)
331 # So far all our float values are > 0, so use -1.0 as the 'unset' value
332 # corner case: this treats -0.0 as 0.0!
333 if f < 0:
334 e_usage('got invalid float for %s: %s' % ('-' + self.name, arg),
335 location)
336 return value.Float(f)
337
338
339class SetToString(_ArgAction):
340
341 def __init__(self, name, quit_parsing_flags, valid=None):
342 # type: (str, bool, Optional[List[str]]) -> None
343 _ArgAction.__init__(self, name, quit_parsing_flags, valid=valid)
344
345 def _Value(self, arg, location):
346 # type: (str, loc_t) -> value_t
347 if self.valid is not None and arg not in self.valid:
348 e_usage(
349 'got invalid argument %r to %r, expected one of: %s' %
350 (arg, ('-' + self.name), '|'.join(self.valid)), location)
351 return value.Str(arg)
352
353
354class SetAttachedBool(_Action):
355
356 def __init__(self, name):
357 # type: (str) -> None
358 self.name = name
359
360 def OnMatch(self, attached_arg, arg_r, out):
361 # type: (Optional[str], Reader, _Attributes) -> bool
362 """Called when the flag matches."""
363
364 # TODO: Delete this part? Is this eqvuivalent to SetToTrue?
365 #
366 # We're not using Go-like --verbose=1, --verbose, or --verbose=0
367 #
368 # 'attached_arg' is also used for -t0 though, which is weird
369
370 if attached_arg is not None: # '0' in --verbose=0
371 if attached_arg in ('0', 'F', 'false',
372 'False'): # TODO: incorrect translation
373 b = False
374 elif attached_arg in ('1', 'T', 'true', 'Talse'):
375 b = True
376 else:
377 e_usage(
378 'got invalid argument to boolean flag: %r' % attached_arg,
379 loc.Missing)
380 else:
381 b = True
382
383 out.Set(self.name, value.Bool(b))
384 return False
385
386
387class SetToTrue(_Action):
388
389 def __init__(self, name):
390 # type: (str) -> None
391 self.name = name
392
393 def OnMatch(self, attached_arg, arg_r, out):
394 # type: (Optional[str], Reader, _Attributes) -> bool
395 """Called when the flag matches."""
396 out.SetTrue(self.name)
397 return False
398
399
400class SetOption(_Action):
401 """Set an option to a boolean, for 'set +e'."""
402
403 def __init__(self, name):
404 # type: (str) -> None
405 self.name = name
406
407 def OnMatch(self, attached_arg, arg_r, out):
408 # type: (Optional[str], Reader, _Attributes) -> bool
409 """Called when the flag matches."""
410 b = (attached_arg == '-')
411 out.opt_changes.append((self.name, b))
412 return False
413
414
415class SetNamedOption(_Action):
416 """Set a named option to a boolean, for 'set +o errexit'."""
417
418 def __init__(self, shopt=False):
419 # type: (bool) -> None
420 self.names = [] # type: List[str]
421 self.shopt = shopt # is it sh -o (set) or sh -O (shopt)?
422
423 def ArgName(self, name):
424 # type: (str) -> None
425 self.names.append(name)
426
427 def OnMatch(self, attached_arg, arg_r, out):
428 # type: (Optional[str], Reader, _Attributes) -> bool
429 """Called when the flag matches."""
430 b = (attached_arg == '-')
431 #log('SetNamedOption %r %r %r', prefix, suffix, arg_r)
432 arg_r.Next() # always advance
433 arg = arg_r.Peek()
434 if arg is None:
435 # triggers on 'set -O' in addition to 'set -o' (meh OK)
436 out.show_options = True
437 return True # quit parsing
438
439 attr_name = arg # Note: validation is done elsewhere
440 if len(self.names) and attr_name not in self.names:
441 e_usage('Invalid option %r' % arg, loc.Missing)
442 changes = out.shopt_changes if self.shopt else out.opt_changes
443 changes.append((attr_name, b))
444 return False
445
446
447class SetAction(_Action):
448 """For compgen -f."""
449
450 def __init__(self, name):
451 # type: (str) -> None
452 self.name = name
453
454 def OnMatch(self, attached_arg, arg_r, out):
455 # type: (Optional[str], Reader, _Attributes) -> bool
456 out.actions.append(self.name)
457 return False
458
459
460class SetNamedAction(_Action):
461 """For compgen -A file."""
462
463 def __init__(self):
464 # type: () -> None
465 self.names = [] # type: List[str]
466
467 def ArgName(self, name):
468 # type: (str) -> None
469 self.names.append(name)
470
471 def OnMatch(self, attached_arg, arg_r, out):
472 # type: (Optional[str], Reader, _Attributes) -> bool
473 """Called when the flag matches."""
474 arg_r.Next() # always advance
475 arg = arg_r.Peek()
476 if arg is None:
477 e_usage('Expected argument for action', loc.Missing)
478
479 attr_name = arg
480 # Validate the option name against a list of valid names.
481 if len(self.names) and attr_name not in self.names:
482 e_usage('Invalid action name %r' % arg, loc.Missing)
483 out.actions.append(attr_name)
484 return False
485
486
487def Parse(spec, arg_r):
488 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
489
490 # NOTE about -:
491 # 'set -' ignores it, vs set
492 # 'unset -' or 'export -' seems to treat it as a variable name
493 out = _Attributes(spec.defaults)
494
495 while not arg_r.AtEnd():
496 arg = arg_r.Peek()
497 if arg == '--':
498 out.saw_double_dash = True
499 arg_r.Next()
500 break
501
502 # Only accept -- if there are any long flags defined
503 if len(spec.actions_long) and arg.startswith('--'):
504 pos = arg.find('=', 2)
505 if pos == -1:
506 suffix = None # type: Optional[str]
507 flag_name = arg[2:] # strip off --
508 else:
509 suffix = arg[pos + 1:]
510 flag_name = arg[2:pos]
511
512 action = spec.actions_long.get(flag_name)
513 if action is None:
514 e_usage('got invalid flag %r' % arg, arg_r.Location())
515
516 action.OnMatch(suffix, arg_r, out)
517 arg_r.Next()
518 continue
519
520 elif arg.startswith('-') and len(arg) > 1:
521 n = len(arg)
522 for i in xrange(1, n): # parse flag combos like -rx
523 ch = arg[i]
524
525 if ch == '0':
526 ch = 'Z' # hack for read -0
527
528 if ch in spec.plus_flags:
529 out.Set(ch, value.Str('-'))
530 continue
531
532 if ch in spec.arity0: # e.g. read -r
533 out.SetTrue(ch)
534 continue
535
536 if ch in spec.arity1: # e.g. read -t1.0
537 action = spec.arity1[ch]
538 # make sure we don't pass empty string for read -t
539 attached_arg = arg[i + 1:] if i < n - 1 else None
540 action.OnMatch(attached_arg, arg_r, out)
541 break
542
543 e_usage("doesn't accept flag %s" % ('-' + ch),
544 arg_r.Location())
545
546 arg_r.Next() # next arg
547
548 # Only accept + if there are ANY options defined, e.g. for declare +rx.
549 elif len(spec.plus_flags) and arg.startswith('+') and len(arg) > 1:
550 n = len(arg)
551 for i in xrange(1, n): # parse flag combos like -rx
552 ch = arg[i]
553 if ch in spec.plus_flags:
554 out.Set(ch, value.Str('+'))
555 continue
556
557 e_usage("doesn't accept option %s" % ('+' + ch),
558 arg_r.Location())
559
560 arg_r.Next() # next arg
561
562 else: # a regular arg
563 break
564
565 return out
566
567
568def ParseLikeEcho(spec, arg_r):
569 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
570 """Echo is a special case. These work: echo -n echo -en.
571
572 - But don't respect --
573 - doesn't fail when an invalid flag is passed
574 """
575 out = _Attributes(spec.defaults)
576
577 while not arg_r.AtEnd():
578 arg = arg_r.Peek()
579 chars = arg[1:]
580 if arg.startswith('-') and len(chars):
581 # Check if it looks like -en or not. TODO: could optimize this.
582 done = False
583 for c in chars:
584 if c not in spec.arity0:
585 done = True
586 break
587 if done:
588 break
589
590 for ch in chars:
591 out.SetTrue(ch)
592
593 else:
594 break # Looks like an arg
595
596 arg_r.Next() # next arg
597
598 return out
599
600
601def ParseMore(spec, arg_r):
602 # type: (flag_spec._FlagSpecAndMore, Reader) -> _Attributes
603 """Return attributes and an index.
604
605 Respects +, like set +eu
606
607 We do NOT respect:
608
609 WRONG: sh -cecho OK: sh -c echo
610 WRONG: set -opipefail OK: set -o pipefail
611
612 But we do accept these
613
614 set -euo pipefail
615 set -oeu pipefail
616 set -oo pipefail errexit
617 """
618 out = _Attributes(spec.defaults)
619
620 quit = False
621 while not arg_r.AtEnd():
622 arg = arg_r.Peek()
623 if arg == '--':
624 out.saw_double_dash = True
625 arg_r.Next()
626 break
627
628 if arg.startswith('--'):
629 action = spec.actions_long.get(arg[2:])
630 if action is None:
631 e_usage('got invalid flag %r' % arg, arg_r.Location())
632
633 # Note: not parsing --foo=bar as attached_arg, as above
634 action.OnMatch(None, arg_r, out)
635 arg_r.Next()
636 continue
637
638 # corner case: sh +c is also accepted!
639 if (arg.startswith('-') or arg.startswith('+')) and len(arg) > 1:
640 # note: we're not handling sh -cecho (no space) as an argument
641 # It complains about a missing argument
642
643 char0 = arg[0]
644
645 # TODO: set - - empty
646 for ch in arg[1:]:
647 #log('ch %r arg_r %s', ch, arg_r)
648 action = spec.actions_short.get(ch)
649 if action is None:
650 e_usage('got invalid flag %r' % ('-' + ch),
651 arg_r.Location())
652
653 attached_arg = char0 if ch in spec.plus_flags else None
654 quit = action.OnMatch(attached_arg, arg_r, out)
655 arg_r.Next() # process the next flag
656
657 if quit:
658 break
659 else:
660 continue
661
662 break # it's a regular arg
663
664 return out