OILS / osh / word_eval.py View on Github | oilshell.org

2407 lines, 1452 significant
1"""
2word_eval.py - Evaluator for the word language.
3"""
4
5from _devbuild.gen.id_kind_asdl import Id, Kind, Kind_str
6from _devbuild.gen.syntax_asdl import (
7 Token,
8 SimpleVarSub,
9 loc,
10 loc_t,
11 BracedVarSub,
12 CommandSub,
13 bracket_op,
14 bracket_op_e,
15 suffix_op,
16 suffix_op_e,
17 ShArrayLiteral,
18 SingleQuoted,
19 DoubleQuoted,
20 word_e,
21 word_t,
22 CompoundWord,
23 rhs_word,
24 rhs_word_e,
25 rhs_word_t,
26 word_part,
27 word_part_e,
28)
29from _devbuild.gen.runtime_asdl import (
30 part_value,
31 part_value_e,
32 part_value_t,
33 cmd_value,
34 cmd_value_e,
35 cmd_value_t,
36 AssignArg,
37 a_index,
38 a_index_e,
39 VTestPlace,
40 VarSubState,
41 Piece,
42)
43from _devbuild.gen.option_asdl import option_i
44from _devbuild.gen.value_asdl import (
45 value,
46 value_e,
47 value_t,
48 sh_lvalue,
49 sh_lvalue_t,
50)
51from core import error
52from core import pyos
53from core import pyutil
54from core import state
55from core import ui
56from core import util
57from data_lang import j8
58from data_lang import j8_lite
59from core.error import e_die
60from frontend import consts
61from frontend import lexer
62from frontend import location
63from mycpp import mops
64from mycpp.mylib import log, tagswitch, NewDict
65from osh import braces
66from osh import glob_
67from osh import string_ops
68from osh import word_
69from ysh import expr_eval
70from ysh import val_ops
71
72from typing import Optional, Tuple, List, Dict, cast, TYPE_CHECKING
73
74if TYPE_CHECKING:
75 from _devbuild.gen.syntax_asdl import word_part_t
76 from _devbuild.gen.option_asdl import builtin_t
77 from core import optview
78 from core.state import Mem
79 from core.ui import ErrorFormatter
80 from core.vm import _Executor
81 from osh.split import SplitContext
82 from osh import prompt
83 from osh import sh_expr_eval
84
85# Flags for _EvalWordToParts and _EvalWordPart (not all are used for both)
86QUOTED = 1 << 0
87IS_SUBST = 1 << 1
88
89EXTGLOB_FILES = 1 << 2 # allow @(cc) from file system?
90EXTGLOB_MATCH = 1 << 3 # allow @(cc) in pattern matching?
91EXTGLOB_NESTED = 1 << 4 # for @(one|!(two|three))
92
93# For EvalWordToString
94QUOTE_FNMATCH = 1 << 5
95QUOTE_ERE = 1 << 6
96
97# For compatibility, ${BASH_SOURCE} and ${BASH_SOURCE[@]} are both valid.
98# Ditto for ${FUNCNAME} and ${BASH_LINENO}.
99_STRING_AND_ARRAY = ['BASH_SOURCE', 'FUNCNAME', 'BASH_LINENO']
100
101
102def ShouldArrayDecay(var_name, exec_opts, is_plain_var_sub=True):
103 # type: (str, optview.Exec, bool) -> bool
104 """Return whether we should allow ${a} to mean ${a[0]}."""
105 return (not exec_opts.strict_array() or
106 is_plain_var_sub and var_name in _STRING_AND_ARRAY)
107
108
109def DecayArray(val):
110 # type: (value_t) -> value_t
111 """Resolve ${array} to ${array[0]}."""
112 if val.tag() == value_e.BashArray:
113 array_val = cast(value.BashArray, val)
114 s = array_val.strs[0] if len(array_val.strs) else None
115 elif val.tag() == value_e.BashAssoc:
116 assoc_val = cast(value.BashAssoc, val)
117 s = assoc_val.d['0'] if '0' in assoc_val.d else None
118 else:
119 raise AssertionError(val.tag())
120
121 if s is None:
122 return value.Undef
123 else:
124 return value.Str(s)
125
126
127def GetArrayItem(strs, index):
128 # type: (List[str], int) -> Optional[str]
129
130 n = len(strs)
131 if index < 0:
132 index += n
133
134 if 0 <= index and index < n:
135 # TODO: strs->index() has a redundant check for (i < 0)
136 s = strs[index]
137 # note: s could be None because representation is sparse
138 else:
139 s = None
140 return s
141
142
143# Use libc to parse NAME, NAME=value, and NAME+=value. We want submatch
144# extraction, but I haven't used that in re2c, and we would need a new kind of
145# binding.
146#
147ASSIGN_ARG_RE = '^([a-zA-Z_][a-zA-Z0-9_]*)((=|\+=)(.*))?$'
148
149# Eggex equivalent:
150#
151# VarName = /
152# [a-z A-Z _ ]
153# [a-z A-Z 0-9 _ ]*
154# /
155#
156# SplitArg = /
157# %begin
158# < VarName >
159# < < '=' | '+=' > < dot* > > ?
160# %end
161# /
162# Note: must use < > for grouping because there is no non-capturing group.
163
164
165def _SplitAssignArg(arg, blame_word):
166 # type: (str, CompoundWord) -> AssignArg
167 """Dynamically parse argument to declare, export, etc.
168
169 This is a fallback to the static parsing done below.
170 """
171 # Note: it would be better to cache regcomp(), but we don't have an API for
172 # that, and it probably isn't a bottleneck now
173 m = util.RegexSearch(ASSIGN_ARG_RE, arg)
174 if m is None:
175 e_die("Assignment builtin expected NAME=value, got %r" % arg,
176 blame_word)
177
178 var_name = m[1]
179 # m[2] is used for grouping; ERE doesn't have non-capturing groups
180
181 op = m[3]
182 assert op is not None, op
183 if len(op): # declare NAME=
184 val = value.Str(m[4]) # type: Optional[value_t]
185 append = op[0] == '+'
186 else: # declare NAME
187 val = None # no operator
188 append = False
189
190 return AssignArg(var_name, val, append, blame_word)
191
192
193# NOTE: Could be done with util.BackslashEscape like glob_.GlobEscape().
194def _BackslashEscape(s):
195 # type: (str) -> str
196 """Double up backslashes.
197
198 Useful for strings about to be globbed and strings about to be IFS
199 escaped.
200 """
201 return s.replace('\\', '\\\\')
202
203
204def _ValueToPartValue(val, quoted, part_loc):
205 # type: (value_t, bool, word_part_t) -> part_value_t
206 """Helper for VarSub evaluation.
207
208 Called by _EvalBracedVarSub and _EvalWordPart for SimpleVarSub.
209 """
210 UP_val = val
211
212 with tagswitch(val) as case:
213 if case(value_e.Undef):
214 # This happens in the case of ${undef+foo}. We skipped _EmptyStrOrError,
215 # but we have to append to the empty string.
216 return Piece('', quoted, not quoted)
217
218 elif case(value_e.Str):
219 val = cast(value.Str, UP_val)
220 return Piece(val.s, quoted, not quoted)
221
222 elif case(value_e.BashArray):
223 val = cast(value.BashArray, UP_val)
224 return part_value.Array(val.strs)
225
226 elif case(value_e.BashAssoc):
227 val = cast(value.BashAssoc, UP_val)
228 # bash behavior: splice values!
229 return part_value.Array(val.d.values())
230
231 # Cases added for YSH
232 # value_e.List is also here - we use val_ops.stringify()s err message
233 elif case(value_e.Null, value_e.Bool, value_e.Int, value_e.Float,
234 value_e.Eggex, value_e.List):
235 s = val_ops.Stringify(val, loc.Missing)
236 return Piece(s, quoted, not quoted)
237
238 else:
239 raise error.TypeErr(val, "Can't substitute into word",
240 loc.WordPart(part_loc))
241
242 raise AssertionError('for -Wreturn-type in C++')
243
244
245def _MakeWordFrames(part_vals):
246 # type: (List[part_value_t]) -> List[List[Piece]]
247 """A word evaluates to a flat list of part_value (String or Array). frame
248 is a portion that results in zero or more args. It can never be joined.
249 This idea exists because of arrays like "$@" and "${a[@]}".
250
251 Example:
252
253 a=(1 '2 3' 4)
254 x=x
255 y=y
256
257 # This word
258 $x"${a[@]}"$y
259
260 # Results in Three frames:
261 [ ('x', False, True), ('1', True, False) ]
262 [ ('2 3', True, False) ]
263 [ ('4', True, False), ('y', False, True) ]
264
265 Note: A frame is a 3-tuple that's identical to Piece()? Maybe we
266 should make that top level type.
267
268 TODO:
269 - Instead of List[List[Piece]], where List[Piece] is a Frame
270 - Change this representation to
271 Frames = (List[Piece] pieces, List[int] break_indices)
272 # where break_indices are the end
273
274 Consider a common case like "$x" or "${x}" - I think this a lot more
275 efficient?
276
277 And then change _EvalWordFrame(pieces: List[Piece], start: int, end: int)
278 """
279 current = [] # type: List[Piece]
280 frames = [current]
281
282 for p in part_vals:
283 UP_p = p
284
285 with tagswitch(p) as case:
286 if case(part_value_e.String):
287 p = cast(Piece, UP_p)
288 current.append(p)
289
290 elif case(part_value_e.Array):
291 p = cast(part_value.Array, UP_p)
292
293 is_first = True
294 for s in p.strs:
295 if s is None:
296 continue # ignore undefined array entries
297
298 # Arrays parts are always quoted; otherwise they would have decayed to
299 # a string.
300 piece = Piece(s, True, False)
301 if is_first:
302 current.append(piece)
303 is_first = False
304 else:
305 current = [piece]
306 frames.append(current) # singleton frame
307
308 else:
309 raise AssertionError()
310
311 return frames
312
313
314# TODO: This could be _MakeWordFrames and then sep.join(). It's redundant.
315def _DecayPartValuesToString(part_vals, join_char):
316 # type: (List[part_value_t], str) -> str
317 # Decay ${a=x"$@"x} to string.
318 out = [] # type: List[str]
319 for p in part_vals:
320 UP_p = p
321 with tagswitch(p) as case:
322 if case(part_value_e.String):
323 p = cast(Piece, UP_p)
324 out.append(p.s)
325 elif case(part_value_e.Array):
326 p = cast(part_value.Array, UP_p)
327 # TODO: Eliminate double join for speed?
328 tmp = [s for s in p.strs if s is not None]
329 out.append(join_char.join(tmp))
330 else:
331 raise AssertionError()
332 return ''.join(out)
333
334
335def _PerformSlice(
336 val, # type: value_t
337 begin, # type: int
338 length, # type: int
339 has_length, # type: bool
340 part, # type: BracedVarSub
341 arg0_val, # type: value.Str
342):
343 # type: (...) -> value_t
344 UP_val = val
345 with tagswitch(val) as case:
346 if case(value_e.Str): # Slice UTF-8 characters in a string.
347 val = cast(value.Str, UP_val)
348 s = val.s
349 n = len(s)
350
351 if begin < 0: # Compute offset with unicode
352 byte_begin = n
353 num_iters = -begin
354 for _ in xrange(num_iters):
355 byte_begin = string_ops.PreviousUtf8Char(s, byte_begin)
356 else:
357 byte_begin = string_ops.AdvanceUtf8Chars(s, begin, 0)
358
359 if has_length:
360 if length < 0: # Compute offset with unicode
361 # Confusing: this is a POSITION
362 byte_end = n
363 num_iters = -length
364 for _ in xrange(num_iters):
365 byte_end = string_ops.PreviousUtf8Char(s, byte_end)
366 else:
367 byte_end = string_ops.AdvanceUtf8Chars(
368 s, length, byte_begin)
369 else:
370 byte_end = len(s)
371
372 substr = s[byte_begin:byte_end]
373 result = value.Str(substr) # type: value_t
374
375 elif case(value_e.BashArray): # Slice array entries.
376 val = cast(value.BashArray, UP_val)
377 # NOTE: This error is ALWAYS fatal in bash. It's inconsistent with
378 # strings.
379 if has_length and length < 0:
380 e_die(
381 "Array slice can't have negative length: %d" %
382 length, loc.WordPart(part))
383
384 # Quirk: "begin" for positional arguments ($@ and $*) counts $0.
385 if arg0_val is not None:
386 orig = [arg0_val.s]
387 orig.extend(val.strs)
388 else:
389 orig = val.strs
390
391 n = len(orig)
392 if begin < 0:
393 i = n + begin # ${@:-3} starts counts from the end
394 else:
395 i = begin
396 strs = [] # type: List[str]
397 count = 0
398 while i < n:
399 if has_length and count == length: # length could be 0
400 break
401 s = orig[i]
402 if s is not None: # Unset elements don't count towards the length
403 strs.append(s)
404 count += 1
405 i += 1
406
407 result = value.BashArray(strs)
408
409 elif case(value_e.BashAssoc):
410 e_die("Can't slice associative arrays", loc.WordPart(part))
411
412 else:
413 raise error.TypeErr(val, 'Slice op expected Str or BashArray',
414 loc.WordPart(part))
415
416 return result
417
418
419class StringWordEvaluator(object):
420 """Interface used by ArithEvaluator / BoolEvaluator"""
421
422 def __init__(self):
423 # type: () -> None
424 """Empty constructor for mycpp."""
425 pass
426
427 def EvalWordToString(self, w, eval_flags=0):
428 # type: (word_t, int) -> value.Str
429 raise NotImplementedError()
430
431
432def _GetDollarHyphen(exec_opts):
433 # type: (optview.Exec) -> str
434 chars = [] # type: List[str]
435 if exec_opts.interactive():
436 chars.append('i')
437
438 if exec_opts.errexit():
439 chars.append('e')
440 if exec_opts.noglob():
441 chars.append('f')
442 if exec_opts.noexec():
443 chars.append('n')
444 if exec_opts.nounset():
445 chars.append('u')
446 # NO letter for pipefail?
447 if exec_opts.xtrace():
448 chars.append('x')
449 if exec_opts.noclobber():
450 chars.append('C')
451
452 # bash has:
453 # - c for sh -c, i for sh -i (mksh also has this)
454 # - h for hashing (mksh also has this)
455 # - B for brace expansion
456 return ''.join(chars)
457
458
459class TildeEvaluator(object):
460
461 def __init__(self, mem, exec_opts):
462 # type: (Mem, optview.Exec) -> None
463 self.mem = mem
464 self.exec_opts = exec_opts
465
466 def GetMyHomeDir(self):
467 # type: () -> Optional[str]
468 """Consult $HOME first, and then make a libc call.
469
470 Important: the libc call can FAIL, which is why we prefer $HOME. See issue
471 #1578.
472 """
473 # First look up the HOME var, then ask the OS. This is what bash does.
474 val = self.mem.GetValue('HOME')
475 UP_val = val
476 if val.tag() == value_e.Str:
477 val = cast(value.Str, UP_val)
478 return val.s
479 return pyos.GetMyHomeDir()
480
481 def Eval(self, part):
482 # type: (word_part.TildeSub) -> str
483 """Evaluates ~ and ~user, given a Lit_TildeLike token."""
484
485 if part.user_name is None:
486 result = self.GetMyHomeDir()
487 else:
488 result = pyos.GetHomeDir(part.user_name)
489
490 if result is None:
491 if self.exec_opts.strict_tilde():
492 e_die("Error expanding tilde (e.g. invalid user)", part.left)
493 else:
494 # Return ~ or ~user literally
495 result = '~'
496 if part.user_name is not None:
497 result = result + part.user_name # mycpp doesn't have +=
498
499 return result
500
501
502class AbstractWordEvaluator(StringWordEvaluator):
503 """Abstract base class for word evaluators.
504
505 Public entry points:
506 EvalWordToString EvalForPlugin EvalRhsWord
507 EvalWordSequence EvalWordSequence2
508 """
509
510 def __init__(
511 self,
512 mem, # type: state.Mem
513 exec_opts, # type: optview.Exec
514 mutable_opts, # type: state.MutableOpts
515 tilde_ev, # type: TildeEvaluator
516 splitter, # type: SplitContext
517 errfmt, # type: ui.ErrorFormatter
518 ):
519 # type: (...) -> None
520 self.arith_ev = None # type: sh_expr_eval.ArithEvaluator
521 self.expr_ev = None # type: expr_eval.ExprEvaluator
522 self.prompt_ev = None # type: prompt.Evaluator
523
524 self.unsafe_arith = None # type: sh_expr_eval.UnsafeArith
525
526 self.tilde_ev = tilde_ev
527
528 self.mem = mem # for $HOME, $1, etc.
529 self.exec_opts = exec_opts # for nounset
530 self.mutable_opts = mutable_opts # for _allow_command_sub
531 self.splitter = splitter
532 self.errfmt = errfmt
533
534 self.globber = glob_.Globber(exec_opts)
535
536 def CheckCircularDeps(self):
537 # type: () -> None
538 raise NotImplementedError()
539
540 def _EvalCommandSub(self, cs_part, quoted):
541 # type: (CommandSub, bool) -> part_value_t
542 """Abstract since it has a side effect."""
543 raise NotImplementedError()
544
545 def _EvalProcessSub(self, cs_part):
546 # type: (CommandSub) -> part_value_t
547 """Abstract since it has a side effect."""
548 raise NotImplementedError()
549
550 def _EvalVarNum(self, var_num):
551 # type: (int) -> value_t
552 assert var_num >= 0
553 return self.mem.GetArgNum(var_num)
554
555 def _EvalSpecialVar(self, op_id, quoted, vsub_state):
556 # type: (int, bool, VarSubState) -> value_t
557 """Evaluate $?
558
559 and so forth
560 """
561 # $@ is special -- it need to know whether it is in a double quoted
562 # context.
563 #
564 # - If it's $@ in a double quoted context, return an ARRAY.
565 # - If it's $@ in a normal context, return a STRING, which then will be
566 # subject to splitting.
567
568 if op_id in (Id.VSub_At, Id.VSub_Star):
569 argv = self.mem.GetArgv()
570 val = value.BashArray(argv) # type: value_t
571 if op_id == Id.VSub_At:
572 # "$@" evaluates to an array, $@ should be decayed
573 vsub_state.join_array = not quoted
574 else: # $* "$*" are both decayed
575 vsub_state.join_array = True
576
577 elif op_id == Id.VSub_Hyphen:
578 val = value.Str(_GetDollarHyphen(self.exec_opts))
579
580 else:
581 val = self.mem.GetSpecialVar(op_id)
582
583 return val
584
585 def _ApplyTestOp(
586 self,
587 val, # type: value_t
588 op, # type: suffix_op.Unary
589 quoted, # type: bool
590 part_vals, # type: Optional[List[part_value_t]]
591 vtest_place, # type: VTestPlace
592 blame_token, # type: Token
593 ):
594 # type: (...) -> bool
595 """
596 Returns:
597 Whether part_vals was mutated
598
599 ${a:-} returns part_value[]
600 ${a:+} returns part_value[]
601 ${a:?error} returns error word?
602 ${a:=} returns part_value[] but also needs self.mem for side effects.
603
604 So I guess it should return part_value[], and then a flag for raising an
605 error, and then a flag for assigning it?
606 The original BracedVarSub will have the name.
607
608 Example of needing multiple part_value[]
609
610 echo X-${a:-'def'"ault"}-X
611
612 We return two part values from the BracedVarSub. Also consider:
613
614 echo ${a:-x"$@"x}
615 """
616 eval_flags = IS_SUBST
617 if quoted:
618 eval_flags |= QUOTED
619
620 tok = op.op
621 # NOTE: Splicing part_values is necessary because of code like
622 # ${undef:-'a b' c 'd # e'}. Each part_value can have a different
623 # do_glob/do_elide setting.
624 UP_val = val
625 with tagswitch(val) as case:
626 if case(value_e.Undef):
627 is_falsey = True
628
629 elif case(value_e.Str):
630 val = cast(value.Str, UP_val)
631 if tok.id in (Id.VTest_ColonHyphen, Id.VTest_ColonEquals,
632 Id.VTest_ColonQMark, Id.VTest_ColonPlus):
633 is_falsey = len(val.s) == 0
634 else:
635 is_falsey = False
636
637 elif case(value_e.BashArray):
638 val = cast(value.BashArray, UP_val)
639 # TODO: allow undefined
640 is_falsey = len(val.strs) == 0
641
642 elif case(value_e.BashAssoc):
643 val = cast(value.BashAssoc, UP_val)
644 is_falsey = len(val.d) == 0
645
646 else:
647 # value.Eggex, etc. are all false
648 is_falsey = False
649
650 if tok.id in (Id.VTest_ColonHyphen, Id.VTest_Hyphen):
651 if is_falsey:
652 self._EvalRhsWordToParts(op.arg_word, part_vals, eval_flags)
653 return True
654 else:
655 return False
656
657 # Inverse of the above.
658 elif tok.id in (Id.VTest_ColonPlus, Id.VTest_Plus):
659 if is_falsey:
660 return False
661 else:
662 self._EvalRhsWordToParts(op.arg_word, part_vals, eval_flags)
663 return True
664
665 # Splice and assign
666 elif tok.id in (Id.VTest_ColonEquals, Id.VTest_Equals):
667 if is_falsey:
668 # Collect new part vals.
669 assign_part_vals = [] # type: List[part_value_t]
670 self._EvalRhsWordToParts(op.arg_word, assign_part_vals,
671 eval_flags)
672 # Append them to out param AND return them.
673 part_vals.extend(assign_part_vals)
674
675 if vtest_place.name is None:
676 # TODO: error context
677 e_die("Can't assign to special variable")
678 else:
679 # NOTE: This decays arrays too! 'shopt -s strict_array' could
680 # avoid it.
681 rhs_str = _DecayPartValuesToString(
682 assign_part_vals, self.splitter.GetJoinChar())
683 if vtest_place.index is None: # using None when no index
684 lval = location.LName(
685 vtest_place.name) # type: sh_lvalue_t
686 else:
687 var_name = vtest_place.name
688 var_index = vtest_place.index
689 UP_var_index = var_index
690
691 with tagswitch(var_index) as case:
692 if case(a_index_e.Int):
693 var_index = cast(a_index.Int, UP_var_index)
694 lval = sh_lvalue.Indexed(
695 var_name, var_index.i, loc.Missing)
696 elif case(a_index_e.Str):
697 var_index = cast(a_index.Str, UP_var_index)
698 lval = sh_lvalue.Keyed(var_name, var_index.s,
699 loc.Missing)
700 else:
701 raise AssertionError()
702
703 state.OshLanguageSetValue(self.mem, lval,
704 value.Str(rhs_str))
705 return True
706
707 else:
708 return False
709
710 elif tok.id in (Id.VTest_ColonQMark, Id.VTest_QMark):
711 if is_falsey:
712 # The arg is the error message
713 error_part_vals = [] # type: List[part_value_t]
714 self._EvalRhsWordToParts(op.arg_word, error_part_vals,
715 eval_flags)
716 error_str = _DecayPartValuesToString(
717 error_part_vals, self.splitter.GetJoinChar())
718
719 #
720 # Display fancy/helpful error
721 #
722 if vtest_place.name is None:
723 var_name = '???'
724 else:
725 var_name = vtest_place.name
726
727 if 0:
728 # This hint is nice, but looks too noisy for now
729 op_str = lexer.LazyStr(tok)
730 if tok.id == Id.VTest_ColonQMark:
731 why = 'empty or unset'
732 else:
733 why = 'unset'
734
735 self.errfmt.Print_(
736 "Hint: operator %s means a variable can't be %s" %
737 (op_str, why), tok)
738
739 if val.tag() == value_e.Undef:
740 actual = 'unset'
741 else:
742 actual = 'empty'
743
744 if len(error_str):
745 suffix = ': %r' % error_str
746 else:
747 suffix = ''
748 e_die("Var %s is %s%s" % (var_name, actual, suffix),
749 blame_token)
750
751 else:
752 return False
753
754 else:
755 raise AssertionError(tok.id)
756
757 def _Length(self, val, token):
758 # type: (value_t, Token) -> int
759 """Returns the length of the value, for ${#var}"""
760 UP_val = val
761 with tagswitch(val) as case:
762 if case(value_e.Str):
763 val = cast(value.Str, UP_val)
764 # NOTE: Whether bash counts bytes or chars is affected by LANG
765 # environment variables.
766 # Should we respect that, or another way to select? set -o
767 # count-bytes?
768
769 # https://stackoverflow.com/questions/17368067/length-of-string-in-bash
770 try:
771 length = string_ops.CountUtf8Chars(val.s)
772 except error.Strict as e:
773 # Add this here so we don't have to add it so far down the stack.
774 # TODO: It's better to show BOTH this CODE an the actual DATA
775 # somehow.
776 e.location = token
777
778 if self.exec_opts.strict_word_eval():
779 raise
780 else:
781 # NOTE: Doesn't make the command exit with 1; it just returns a
782 # length of -1.
783 self.errfmt.PrettyPrintError(e, prefix='warning: ')
784 return -1
785
786 elif case(value_e.BashArray):
787 val = cast(value.BashArray, UP_val)
788 # There can be empty placeholder values in the array.
789 length = 0
790 for s in val.strs:
791 if s is not None:
792 length += 1
793
794 elif case(value_e.BashAssoc):
795 val = cast(value.BashAssoc, UP_val)
796 length = len(val.d)
797
798 else:
799 raise error.TypeErr(
800 val, "Length op expected Str, BashArray, BashAssoc", token)
801
802 return length
803
804 def _Keys(self, val, token):
805 # type: (value_t, Token) -> value_t
806 """Return keys of a container, for ${!array[@]}"""
807
808 UP_val = val
809 with tagswitch(val) as case:
810 if case(value_e.BashArray):
811 val = cast(value.BashArray, UP_val)
812 # translation issue: tuple indices not supported in list comprehensions
813 #indices = [str(i) for i, s in enumerate(val.strs) if s is not None]
814 indices = [] # type: List[str]
815 for i, s in enumerate(val.strs):
816 if s is not None:
817 indices.append(str(i))
818 return value.BashArray(indices)
819
820 elif case(value_e.BashAssoc):
821 val = cast(value.BashAssoc, UP_val)
822 assert val.d is not None # for MyPy, so it's not Optional[]
823
824 # BUG: Keys aren't ordered according to insertion!
825 return value.BashArray(val.d.keys())
826
827 else:
828 raise error.TypeErr(val, 'Keys op expected Str', token)
829
830 def _EvalVarRef(self, val, blame_tok, quoted, vsub_state, vtest_place):
831 # type: (value_t, Token, bool, VarSubState, VTestPlace) -> value_t
832 """Handles indirect expansion like ${!var} and ${!a[0]}.
833
834 Args:
835 blame_tok: 'foo' for ${!foo}
836 """
837 UP_val = val
838 with tagswitch(val) as case:
839 if case(value_e.Undef):
840 return value.Undef # ${!undef} is just weird bash behavior
841
842 elif case(value_e.Str):
843 val = cast(value.Str, UP_val)
844 bvs_part = self.unsafe_arith.ParseVarRef(val.s, blame_tok)
845 return self._VarRefValue(bvs_part, quoted, vsub_state,
846 vtest_place)
847
848 elif case(value_e.BashArray): # caught earlier but OK
849 e_die('Indirect expansion of array')
850
851 elif case(value_e.BashAssoc): # caught earlier but OK
852 e_die('Indirect expansion of assoc array')
853
854 else:
855 raise error.TypeErr(val, 'Var Ref op expected Str', blame_tok)
856
857 def _ApplyUnarySuffixOp(self, val, op):
858 # type: (value_t, suffix_op.Unary) -> value_t
859 assert val.tag() != value_e.Undef
860
861 op_kind = consts.GetKind(op.op.id)
862
863 if op_kind == Kind.VOp1:
864 # NOTE: glob syntax is supported in ^ ^^ , ,, ! As well as % %% # ##.
865 # Detect has_extglob so that DoUnarySuffixOp doesn't use the fast
866 # shortcut for constant strings.
867 arg_val, has_extglob = self.EvalWordToPattern(op.arg_word)
868 assert arg_val.tag() == value_e.Str
869
870 UP_val = val
871 with tagswitch(val) as case:
872 if case(value_e.Str):
873 val = cast(value.Str, UP_val)
874 s = string_ops.DoUnarySuffixOp(val.s, op.op, arg_val.s,
875 has_extglob)
876 #log('%r %r -> %r', val.s, arg_val.s, s)
877 new_val = value.Str(s) # type: value_t
878
879 elif case(value_e.BashArray):
880 val = cast(value.BashArray, UP_val)
881 # ${a[@]#prefix} is VECTORIZED on arrays. YSH should have this too.
882 strs = [] # type: List[str]
883 for s in val.strs:
884 if s is not None:
885 strs.append(
886 string_ops.DoUnarySuffixOp(
887 s, op.op, arg_val.s, has_extglob))
888 new_val = value.BashArray(strs)
889
890 elif case(value_e.BashAssoc):
891 val = cast(value.BashAssoc, UP_val)
892 strs = []
893 for s in val.d.values():
894 strs.append(
895 string_ops.DoUnarySuffixOp(s, op.op, arg_val.s,
896 has_extglob))
897 new_val = value.BashArray(strs)
898
899 else:
900 raise error.TypeErr(
901 val, 'Unary op expected Str, BashArray, BashAssoc',
902 op.op)
903
904 else:
905 raise AssertionError(Kind_str(op_kind))
906
907 return new_val
908
909 def _PatSub(self, val, op):
910 # type: (value_t, suffix_op.PatSub) -> value_t
911
912 pat_val, has_extglob = self.EvalWordToPattern(op.pat)
913 # Extended globs aren't supported because we only translate * ? etc. to
914 # ERE. I don't think there's a straightforward translation from !(*.py) to
915 # ERE! You would need an engine that supports negation? (Derivatives?)
916 if has_extglob:
917 e_die('extended globs not supported in ${x//GLOB/}', op.pat)
918
919 if op.replace:
920 replace_val = self.EvalRhsWord(op.replace)
921 # Can't have an array, so must be a string
922 assert replace_val.tag() == value_e.Str, replace_val
923 replace_str = cast(value.Str, replace_val).s
924 else:
925 replace_str = ''
926
927 # note: doesn't support self.exec_opts.extglob()!
928 regex, warnings = glob_.GlobToERE(pat_val.s)
929 if len(warnings):
930 # TODO:
931 # - Add 'shopt -s strict_glob' mode and expose warnings.
932 # "Glob is not in CANONICAL FORM".
933 # - Propagate location info back to the 'op.pat' word.
934 pass
935 #log('regex %r', regex)
936 replacer = string_ops.GlobReplacer(regex, replace_str, op.slash_tok)
937
938 with tagswitch(val) as case2:
939 if case2(value_e.Str):
940 str_val = cast(value.Str, val)
941 s = replacer.Replace(str_val.s, op)
942 val = value.Str(s)
943
944 elif case2(value_e.BashArray):
945 array_val = cast(value.BashArray, val)
946 strs = [] # type: List[str]
947 for s in array_val.strs:
948 if s is not None:
949 strs.append(replacer.Replace(s, op))
950 val = value.BashArray(strs)
951
952 elif case2(value_e.BashAssoc):
953 assoc_val = cast(value.BashAssoc, val)
954 strs = []
955 for s in assoc_val.d.values():
956 strs.append(replacer.Replace(s, op))
957 val = value.BashArray(strs)
958
959 else:
960 raise error.TypeErr(
961 val, 'Pat Sub op expected Str, BashArray, BashAssoc',
962 op.slash_tok)
963
964 return val
965
966 def _Slice(self, val, op, var_name, part):
967 # type: (value_t, suffix_op.Slice, Optional[str], BracedVarSub) -> value_t
968
969 begin = self.arith_ev.EvalToInt(op.begin)
970
971 # Note: bash allows lengths to be negative (with odd semantics), but
972 # we don't allow that right now.
973 has_length = False
974 length = -1
975 if op.length:
976 has_length = True
977 length = self.arith_ev.EvalToInt(op.length)
978
979 try:
980 arg0_val = None # type: value.Str
981 if var_name is None: # $* or $@
982 arg0_val = self.mem.GetArg0()
983 val = _PerformSlice(val, begin, length, has_length, part, arg0_val)
984 except error.Strict as e:
985 if self.exec_opts.strict_word_eval():
986 raise
987 else:
988 self.errfmt.PrettyPrintError(e, prefix='warning: ')
989 with tagswitch(val) as case2:
990 if case2(value_e.Str):
991 val = value.Str('')
992 elif case2(value_e.BashArray):
993 val = value.BashArray([])
994 else:
995 raise NotImplementedError()
996 return val
997
998 def _Nullary(self, val, op, var_name):
999 # type: (value_t, Token, Optional[str]) -> Tuple[value.Str, bool]
1000
1001 UP_val = val
1002 quoted2 = False
1003 op_id = op.id
1004 if op_id == Id.VOp0_P:
1005 with tagswitch(val) as case:
1006 if case(value_e.Str):
1007 str_val = cast(value.Str, UP_val)
1008 prompt = self.prompt_ev.EvalPrompt(str_val)
1009 # readline gets rid of these, so we should too.
1010 p = prompt.replace('\x01', '').replace('\x02', '')
1011 result = value.Str(p)
1012 else:
1013 e_die("Can't use @P on %s" % ui.ValType(val), op)
1014
1015 elif op_id == Id.VOp0_Q:
1016 with tagswitch(val) as case:
1017 if case(value_e.Str):
1018 str_val = cast(value.Str, UP_val)
1019
1020 # TODO: use fastfunc.ShellEncode or
1021 # fastfunc.PosixShellEncode()
1022 result = value.Str(j8_lite.MaybeShellEncode(str_val.s))
1023 # oddly, 'echo ${x@Q}' is equivalent to 'echo "${x@Q}"' in bash
1024 quoted2 = True
1025 elif case(value_e.BashArray):
1026 array_val = cast(value.BashArray, UP_val)
1027
1028 # TODO: should use fastfunc.ShellEncode
1029 tmp = [j8_lite.MaybeShellEncode(s) for s in array_val.strs]
1030 result = value.Str(' '.join(tmp))
1031 else:
1032 e_die("Can't use @Q on %s" % ui.ValType(val), op)
1033
1034 elif op_id == Id.VOp0_a:
1035 # We're ONLY simluating -a and -A, not -r -x -n for now. See
1036 # spec/ble-idioms.test.sh.
1037 chars = [] # type: List[str]
1038 with tagswitch(val) as case:
1039 if case(value_e.BashArray):
1040 chars.append('a')
1041 elif case(value_e.BashAssoc):
1042 chars.append('A')
1043
1044 if var_name is not None: # e.g. ${?@a} is allowed
1045 cell = self.mem.GetCell(var_name)
1046 if cell:
1047 if cell.readonly:
1048 chars.append('r')
1049 if cell.exported:
1050 chars.append('x')
1051 if cell.nameref:
1052 chars.append('n')
1053
1054 result = value.Str(''.join(chars))
1055
1056 else:
1057 e_die('Var op %r not implemented' % lexer.TokenVal(op), op)
1058
1059 return result, quoted2
1060
1061 def _WholeArray(self, val, part, quoted, vsub_state):
1062 # type: (value_t, BracedVarSub, bool, VarSubState) -> value_t
1063 op_id = cast(bracket_op.WholeArray, part.bracket_op).op_id
1064
1065 if op_id == Id.Lit_At:
1066 vsub_state.join_array = not quoted # ${a[@]} decays but "${a[@]}" doesn't
1067 UP_val = val
1068 with tagswitch(val) as case2:
1069 if case2(value_e.Undef):
1070 if not vsub_state.has_test_op:
1071 val = self._EmptyBashArrayOrError(part.token)
1072 elif case2(value_e.Str):
1073 if self.exec_opts.strict_array():
1074 e_die("Can't index string with @", loc.WordPart(part))
1075 elif case2(value_e.BashArray):
1076 pass # no-op
1077
1078 elif op_id == Id.Arith_Star:
1079 vsub_state.join_array = True # both ${a[*]} and "${a[*]}" decay
1080 UP_val = val
1081 with tagswitch(val) as case2:
1082 if case2(value_e.Undef):
1083 if not vsub_state.has_test_op:
1084 val = self._EmptyBashArrayOrError(part.token)
1085 elif case2(value_e.Str):
1086 if self.exec_opts.strict_array():
1087 e_die("Can't index string with *", loc.WordPart(part))
1088 elif case2(value_e.BashArray):
1089 pass # no-op
1090
1091 else:
1092 raise AssertionError(op_id) # unknown
1093
1094 return val
1095
1096 def _ArrayIndex(self, val, part, vtest_place):
1097 # type: (value_t, BracedVarSub, VTestPlace) -> value_t
1098 """Process a numeric array index like ${a[i+1]}"""
1099 anode = cast(bracket_op.ArrayIndex, part.bracket_op).expr
1100
1101 UP_val = val
1102 with tagswitch(val) as case2:
1103 if case2(value_e.Undef):
1104 pass # it will be checked later
1105
1106 elif case2(value_e.Str):
1107 # Bash treats any string as an array, so we can't add our own
1108 # behavior here without making valid OSH invalid bash.
1109 e_die("Can't index string %r with integer" % part.var_name,
1110 part.token)
1111
1112 elif case2(value_e.BashArray):
1113 array_val = cast(value.BashArray, UP_val)
1114 index = self.arith_ev.EvalToInt(anode)
1115 vtest_place.index = a_index.Int(index)
1116
1117 s = GetArrayItem(array_val.strs, index)
1118
1119 if s is None:
1120 val = value.Undef
1121 else:
1122 val = value.Str(s)
1123
1124 elif case2(value_e.BashAssoc):
1125 assoc_val = cast(value.BashAssoc, UP_val)
1126 # Location could also be attached to bracket_op? But
1127 # arith_expr.VarSub works OK too
1128 key = self.arith_ev.EvalWordToString(
1129 anode, blame_loc=location.TokenForArith(anode))
1130
1131 vtest_place.index = a_index.Str(key) # out param
1132 s = assoc_val.d.get(key)
1133
1134 if s is None:
1135 val = value.Undef
1136 else:
1137 val = value.Str(s)
1138
1139 else:
1140 raise error.TypeErr(val,
1141 'Index op expected BashArray, BashAssoc',
1142 loc.WordPart(part))
1143
1144 return val
1145
1146 def _EvalDoubleQuoted(self, parts, part_vals):
1147 # type: (List[word_part_t], List[part_value_t]) -> None
1148 """Evaluate parts of a DoubleQuoted part.
1149
1150 Args:
1151 part_vals: output param to append to.
1152 """
1153 # Example of returning array:
1154 # $ a=(1 2); b=(3); $ c=(4 5)
1155 # $ argv "${a[@]}${b[@]}${c[@]}"
1156 # ['1', '234', '5']
1157 #
1158 # Example of multiple parts
1159 # $ argv "${a[@]}${undef[@]:-${c[@]}}"
1160 # ['1', '24', '5']
1161
1162 # Special case for "". The parser outputs (DoubleQuoted []), instead
1163 # of (DoubleQuoted [Literal '']). This is better but it means we
1164 # have to check for it.
1165 if len(parts) == 0:
1166 v = Piece('', True, False)
1167 part_vals.append(v)
1168 return
1169
1170 for p in parts:
1171 self._EvalWordPart(p, part_vals, QUOTED)
1172
1173 def EvalDoubleQuotedToString(self, dq_part):
1174 # type: (DoubleQuoted) -> str
1175 """For double quoted strings in YSH expressions.
1176
1177 Example: var x = "$foo-${foo}"
1178 """
1179 part_vals = [] # type: List[part_value_t]
1180 self._EvalDoubleQuoted(dq_part.parts, part_vals)
1181 return self._ConcatPartVals(part_vals, dq_part.left)
1182
1183 def _DecayArray(self, val):
1184 # type: (value.BashArray) -> value.Str
1185 """Decay $* to a string."""
1186 assert val.tag() == value_e.BashArray, val
1187 sep = self.splitter.GetJoinChar()
1188 tmp = [s for s in val.strs if s is not None]
1189 return value.Str(sep.join(tmp))
1190
1191 def _EmptyStrOrError(self, val, token):
1192 # type: (value_t, Token) -> value_t
1193 if val.tag() != value_e.Undef:
1194 return val
1195
1196 if not self.exec_opts.nounset():
1197 return value.Str('')
1198
1199 tok_str = lexer.TokenVal(token)
1200 name = tok_str[1:] if tok_str.startswith('$') else tok_str
1201 e_die('Undefined variable %r' % name, token)
1202
1203 def _EmptyBashArrayOrError(self, token):
1204 # type: (Token) -> value_t
1205 assert token is not None
1206 if self.exec_opts.nounset():
1207 e_die('Undefined array %r' % lexer.TokenVal(token), token)
1208 else:
1209 return value.BashArray([])
1210
1211 def _EvalBracketOp(self, val, part, quoted, vsub_state, vtest_place):
1212 # type: (value_t, BracedVarSub, bool, VarSubState, VTestPlace) -> value_t
1213
1214 if part.bracket_op:
1215 with tagswitch(part.bracket_op) as case:
1216 if case(bracket_op_e.WholeArray):
1217 val = self._WholeArray(val, part, quoted, vsub_state)
1218
1219 elif case(bracket_op_e.ArrayIndex):
1220 val = self._ArrayIndex(val, part, vtest_place)
1221
1222 else:
1223 raise AssertionError(part.bracket_op.tag())
1224
1225 else: # no bracket op
1226 var_name = vtest_place.name
1227 if (var_name is not None and
1228 val.tag() in (value_e.BashArray, value_e.BashAssoc) and
1229 not vsub_state.is_type_query):
1230 if ShouldArrayDecay(var_name, self.exec_opts,
1231 not (part.prefix_op or part.suffix_op)):
1232 # for ${BASH_SOURCE}, etc.
1233 val = DecayArray(val)
1234 else:
1235 e_die(
1236 "Array %r can't be referred to as a scalar (without @ or *)"
1237 % var_name, loc.WordPart(part))
1238
1239 return val
1240
1241 def _VarRefValue(self, part, quoted, vsub_state, vtest_place):
1242 # type: (BracedVarSub, bool, VarSubState, VTestPlace) -> value_t
1243 """Duplicates some logic from _EvalBracedVarSub, but returns a
1244 value_t."""
1245
1246 # 1. Evaluate from (var_name, var_num, token Id) -> value
1247 if part.token.id == Id.VSub_Name:
1248 vtest_place.name = part.var_name
1249 val = self.mem.GetValue(part.var_name)
1250
1251 elif part.token.id == Id.VSub_Number:
1252 var_num = int(part.var_name)
1253 val = self._EvalVarNum(var_num)
1254
1255 else:
1256 # $* decays
1257 val = self._EvalSpecialVar(part.token.id, quoted, vsub_state)
1258
1259 # We don't need var_index because it's only for L-Values of test ops?
1260 if self.exec_opts.eval_unsafe_arith():
1261 val = self._EvalBracketOp(val, part, quoted, vsub_state,
1262 vtest_place)
1263 else:
1264 with state.ctx_Option(self.mutable_opts,
1265 [option_i._allow_command_sub], False):
1266 val = self._EvalBracketOp(val, part, quoted, vsub_state,
1267 vtest_place)
1268
1269 return val
1270
1271 def _EvalBracedVarSub(self, part, part_vals, quoted):
1272 # type: (BracedVarSub, List[part_value_t], bool) -> None
1273 """
1274 Args:
1275 part_vals: output param to append to.
1276 """
1277 # We have different operators that interact in a non-obvious order.
1278 #
1279 # 1. bracket_op: value -> value, with side effect on vsub_state
1280 #
1281 # 2. prefix_op
1282 # a. length ${#x}: value -> value
1283 # b. var ref ${!ref}: can expand to an array
1284 #
1285 # 3. suffix_op:
1286 # a. no operator: you have a value
1287 # b. Test: value -> part_value[]
1288 # c. Other Suffix: value -> value
1289 #
1290 # 4. Process vsub_state.join_array here before returning.
1291 #
1292 # These cases are hard to distinguish:
1293 # - ${!prefix@} prefix query
1294 # - ${!array[@]} keys
1295 # - ${!ref} named reference
1296 # - ${!ref[0]} named reference
1297 #
1298 # I think we need several stages:
1299 #
1300 # 1. value: name, number, special, prefix query
1301 # 2. bracket_op
1302 # 3. prefix length -- this is TERMINAL
1303 # 4. indirection? Only for some of the ! cases
1304 # 5. string transformation suffix ops like ##
1305 # 6. test op
1306 # 7. vsub_state.join_array
1307
1308 # vsub_state.join_array is for joining "${a[*]}" and unquoted ${a[@]} AFTER
1309 # suffix ops are applied. If we take the length with a prefix op, the
1310 # distinction is ignored.
1311
1312 var_name = None # type: Optional[str] # used throughout the function
1313 vtest_place = VTestPlace(var_name, None) # For ${foo=default}
1314 vsub_state = VarSubState.CreateNull() # for $*, ${a[*]}, etc.
1315
1316 # 1. Evaluate from (var_name, var_num, token Id) -> value
1317 if part.token.id == Id.VSub_Name:
1318 # Handle ${!prefix@} first, since that looks at names and not values
1319 # Do NOT handle ${!A[@]@a} here!
1320 if (part.prefix_op is not None and part.bracket_op is None and
1321 part.suffix_op is not None and
1322 part.suffix_op.tag() == suffix_op_e.Nullary):
1323 nullary_op = cast(Token, part.suffix_op)
1324 # ${!x@} but not ${!x@P}
1325 if consts.GetKind(nullary_op.id) == Kind.VOp3:
1326 names = self.mem.VarNamesStartingWith(part.var_name)
1327 names.sort()
1328
1329 if quoted and nullary_op.id == Id.VOp3_At:
1330 part_vals.append(part_value.Array(names))
1331 else:
1332 sep = self.splitter.GetJoinChar()
1333 part_vals.append(Piece(sep.join(names), quoted, True))
1334 return # EARLY RETURN
1335
1336 var_name = part.var_name
1337 vtest_place.name = var_name # for _ApplyTestOp
1338
1339 val = self.mem.GetValue(var_name)
1340
1341 elif part.token.id == Id.VSub_Number:
1342 var_num = int(part.var_name)
1343 val = self._EvalVarNum(var_num)
1344 else:
1345 # $* decays
1346 val = self._EvalSpecialVar(part.token.id, quoted, vsub_state)
1347
1348 suffix_op_ = part.suffix_op
1349 if suffix_op_:
1350 UP_op = suffix_op_
1351 with tagswitch(suffix_op_) as case:
1352 if case(suffix_op_e.Nullary):
1353 suffix_op_ = cast(Token, UP_op)
1354
1355 # Type query ${array@a} is a STRING, not an array
1356 # NOTE: ${array@Q} is ${array[0]@Q} in bash, which is different than
1357 # ${array[@]@Q}
1358 if suffix_op_.id == Id.VOp0_a:
1359 vsub_state.is_type_query = True
1360
1361 elif case(suffix_op_e.Unary):
1362 suffix_op_ = cast(suffix_op.Unary, UP_op)
1363
1364 # Do the _EmptyStrOrError/_EmptyBashArrayOrError up front, EXCEPT in
1365 # the case of Kind.VTest
1366 if consts.GetKind(suffix_op_.op.id) == Kind.VTest:
1367 vsub_state.has_test_op = True
1368
1369 # 2. Bracket Op
1370 val = self._EvalBracketOp(val, part, quoted, vsub_state, vtest_place)
1371
1372 if part.prefix_op:
1373 if part.prefix_op.id == Id.VSub_Pound: # ${#var} for length
1374 if not vsub_state.has_test_op: # undef -> '' BEFORE length
1375 val = self._EmptyStrOrError(val, part.token)
1376
1377 n = self._Length(val, part.token)
1378 part_vals.append(Piece(str(n), quoted, False))
1379 return # EARLY EXIT: nothing else can come after length
1380
1381 elif part.prefix_op.id == Id.VSub_Bang:
1382 if (part.bracket_op and
1383 part.bracket_op.tag() == bracket_op_e.WholeArray):
1384 if vsub_state.has_test_op:
1385 # ${!a[@]-'default'} is a non-fatal runtime error in bash. Here
1386 # it's fatal.
1387 op_tok = cast(suffix_op.Unary, UP_op).op
1388 e_die('Test operation not allowed with ${!array[@]}',
1389 op_tok)
1390
1391 # ${!array[@]} to get indices/keys
1392 val = self._Keys(val, part.token)
1393 # already set vsub_State.join_array ABOVE
1394 else:
1395 # Process ${!ref}. SURPRISE: ${!a[0]} is an indirect expansion unlike
1396 # ${!a[@]} !
1397 # ${!ref} can expand into an array if ref='array[@]'
1398
1399 # Clear it now that we have a var ref
1400 vtest_place.name = None
1401 vtest_place.index = None
1402
1403 val = self._EvalVarRef(val, part.token, quoted, vsub_state,
1404 vtest_place)
1405
1406 if not vsub_state.has_test_op: # undef -> '' AFTER indirection
1407 val = self._EmptyStrOrError(val, part.token)
1408
1409 else:
1410 raise AssertionError(part.prefix_op)
1411
1412 else:
1413 if not vsub_state.has_test_op: # undef -> '' if no prefix op
1414 val = self._EmptyStrOrError(val, part.token)
1415
1416 quoted2 = False # another bit for @Q
1417 if suffix_op_:
1418 op = suffix_op_ # could get rid of this alias
1419
1420 with tagswitch(suffix_op_) as case:
1421 if case(suffix_op_e.Nullary):
1422 op = cast(Token, UP_op)
1423 val, quoted2 = self._Nullary(val, op, var_name)
1424
1425 elif case(suffix_op_e.Unary):
1426 op = cast(suffix_op.Unary, UP_op)
1427 if consts.GetKind(op.op.id) == Kind.VTest:
1428 if self._ApplyTestOp(val, op, quoted, part_vals,
1429 vtest_place, part.token):
1430 # e.g. to evaluate ${undef:-'default'}, we already appended
1431 # what we need
1432 return
1433
1434 else:
1435 # Other suffix: value -> value
1436 val = self._ApplyUnarySuffixOp(val, op)
1437
1438 elif case(suffix_op_e.PatSub): # PatSub, vectorized
1439 op = cast(suffix_op.PatSub, UP_op)
1440 val = self._PatSub(val, op)
1441
1442 elif case(suffix_op_e.Slice):
1443 op = cast(suffix_op.Slice, UP_op)
1444 val = self._Slice(val, op, var_name, part)
1445
1446 elif case(suffix_op_e.Static):
1447 op = cast(suffix_op.Static, UP_op)
1448 e_die('Not implemented', op.tok)
1449
1450 else:
1451 raise AssertionError()
1452
1453 # After applying suffixes, process join_array here.
1454 UP_val = val
1455 if val.tag() == value_e.BashArray:
1456 array_val = cast(value.BashArray, UP_val)
1457 if vsub_state.join_array:
1458 val = self._DecayArray(array_val)
1459 else:
1460 val = array_val
1461
1462 # For example, ${a} evaluates to value.Str(), but we want a
1463 # Piece().
1464 part_val = _ValueToPartValue(val, quoted or quoted2, part)
1465 part_vals.append(part_val)
1466
1467 def _ConcatPartVals(self, part_vals, location):
1468 # type: (List[part_value_t], loc_t) -> str
1469
1470 strs = [] # type: List[str]
1471 for part_val in part_vals:
1472 UP_part_val = part_val
1473 with tagswitch(part_val) as case:
1474 if case(part_value_e.String):
1475 part_val = cast(Piece, UP_part_val)
1476 s = part_val.s
1477
1478 elif case(part_value_e.Array):
1479 part_val = cast(part_value.Array, UP_part_val)
1480 if self.exec_opts.strict_array():
1481 # Examples: echo f > "$@"; local foo="$@"
1482 e_die("Illegal array word part (strict_array)",
1483 location)
1484 else:
1485 # It appears to not respect IFS
1486 # TODO: eliminate double join()?
1487 tmp = [s for s in part_val.strs if s is not None]
1488 s = ' '.join(tmp)
1489
1490 else:
1491 raise AssertionError()
1492
1493 strs.append(s)
1494
1495 return ''.join(strs)
1496
1497 def EvalBracedVarSubToString(self, part):
1498 # type: (BracedVarSub) -> str
1499 """For double quoted strings in YSH expressions.
1500
1501 Example: var x = "$foo-${foo}"
1502 """
1503 part_vals = [] # type: List[part_value_t]
1504 self._EvalBracedVarSub(part, part_vals, False)
1505 # blame ${ location
1506 return self._ConcatPartVals(part_vals, part.left)
1507
1508 def _EvalSimpleVarSub(self, part, part_vals, quoted):
1509 # type: (SimpleVarSub, List[part_value_t], bool) -> None
1510
1511 token = part.tok
1512
1513 vsub_state = VarSubState.CreateNull()
1514
1515 # 1. Evaluate from (var_name, var_num, Token) -> defined, value
1516 if token.id == Id.VSub_DollarName:
1517 var_name = lexer.LazyStr(token)
1518 # TODO: Special case for LINENO
1519 val = self.mem.GetValue(var_name)
1520 if val.tag() in (value_e.BashArray, value_e.BashAssoc):
1521 if ShouldArrayDecay(var_name, self.exec_opts):
1522 # for $BASH_SOURCE, etc.
1523 val = DecayArray(val)
1524 else:
1525 e_die(
1526 "Array %r can't be referred to as a scalar (without @ or *)"
1527 % var_name, token)
1528
1529 elif token.id == Id.VSub_Number:
1530 var_num = int(lexer.LazyStr(token))
1531 val = self._EvalVarNum(var_num)
1532
1533 else:
1534 val = self._EvalSpecialVar(token.id, quoted, vsub_state)
1535
1536 #log('SIMPLE %s', part)
1537 val = self._EmptyStrOrError(val, token)
1538 UP_val = val
1539 if val.tag() == value_e.BashArray:
1540 array_val = cast(value.BashArray, UP_val)
1541 if vsub_state.join_array:
1542 val = self._DecayArray(array_val)
1543 else:
1544 val = array_val
1545
1546 v = _ValueToPartValue(val, quoted, part)
1547 part_vals.append(v)
1548
1549 def EvalSimpleVarSubToString(self, node):
1550 # type: (SimpleVarSub) -> str
1551 """For double quoted strings in YSH expressions.
1552
1553 Example: var x = "$foo-${foo}"
1554 """
1555 part_vals = [] # type: List[part_value_t]
1556 self._EvalSimpleVarSub(node, part_vals, False)
1557 return self._ConcatPartVals(part_vals, node.tok)
1558
1559 def _EvalExtGlob(self, part, part_vals):
1560 # type: (word_part.ExtGlob, List[part_value_t]) -> None
1561 """Evaluate @($x|'foo'|$(hostname)) and flatten it."""
1562 op = part.op
1563 if op.id == Id.ExtGlob_Comma:
1564 op_str = '@('
1565 else:
1566 op_str = lexer.LazyStr(op)
1567 # Do NOT split these.
1568 part_vals.append(Piece(op_str, False, False))
1569
1570 for i, w in enumerate(part.arms):
1571 if i != 0:
1572 part_vals.append(Piece('|', False, False)) # separator
1573 # FLATTEN the tree of extglob "arms".
1574 self._EvalWordToParts(w, part_vals, EXTGLOB_NESTED)
1575 part_vals.append(Piece(')', False, False)) # closing )
1576
1577 def _TranslateExtGlob(self, part_vals, w, glob_parts, fnmatch_parts):
1578 # type: (List[part_value_t], CompoundWord, List[str], List[str]) -> None
1579 """Translate a flattened WORD with an ExtGlob part to string patterns.
1580
1581 We need both glob and fnmatch patterns. _EvalExtGlob does the
1582 flattening.
1583 """
1584 for i, part_val in enumerate(part_vals):
1585 UP_part_val = part_val
1586 with tagswitch(part_val) as case:
1587 if case(part_value_e.String):
1588 part_val = cast(Piece, UP_part_val)
1589 if part_val.quoted and not self.exec_opts.noglob():
1590 s = glob_.GlobEscape(part_val.s)
1591 else:
1592 # e.g. the @( and | in @(foo|bar) aren't quoted
1593 s = part_val.s
1594 glob_parts.append(s)
1595 fnmatch_parts.append(s) # from _EvalExtGlob()
1596
1597 elif case(part_value_e.Array):
1598 # Disallow array
1599 e_die(
1600 "Extended globs and arrays can't appear in the same word",
1601 w)
1602
1603 elif case(part_value_e.ExtGlob):
1604 part_val = cast(part_value.ExtGlob, UP_part_val)
1605 # keep appending fnmatch_parts, but repplace glob_parts with '*'
1606 self._TranslateExtGlob(part_val.part_vals, w, [],
1607 fnmatch_parts)
1608 glob_parts.append('*')
1609
1610 else:
1611 raise AssertionError()
1612
1613 def _EvalWordPart(self, part, part_vals, flags):
1614 # type: (word_part_t, List[part_value_t], int) -> None
1615 """Evaluate a word part, appending to part_vals
1616
1617 Called by _EvalWordToParts, EvalWordToString, and _EvalDoubleQuoted.
1618 """
1619 quoted = bool(flags & QUOTED)
1620 is_subst = bool(flags & IS_SUBST)
1621
1622 UP_part = part
1623 with tagswitch(part) as case:
1624 if case(word_part_e.ShArrayLiteral):
1625 part = cast(ShArrayLiteral, UP_part)
1626 e_die("Unexpected array literal", loc.WordPart(part))
1627 elif case(word_part_e.BashAssocLiteral):
1628 part = cast(word_part.BashAssocLiteral, UP_part)
1629 e_die("Unexpected associative array literal",
1630 loc.WordPart(part))
1631
1632 elif case(word_part_e.Literal):
1633 part = cast(Token, UP_part)
1634 # Split if it's in a substitution.
1635 # That is: echo is not split, but ${foo:-echo} is split
1636 v = Piece(lexer.LazyStr(part), quoted, is_subst)
1637 part_vals.append(v)
1638
1639 elif case(word_part_e.EscapedLiteral):
1640 part = cast(word_part.EscapedLiteral, UP_part)
1641 v = Piece(part.ch, True, False)
1642 part_vals.append(v)
1643
1644 elif case(word_part_e.SingleQuoted):
1645 part = cast(SingleQuoted, UP_part)
1646 v = Piece(part.sval, True, False)
1647 part_vals.append(v)
1648
1649 elif case(word_part_e.DoubleQuoted):
1650 part = cast(DoubleQuoted, UP_part)
1651 self._EvalDoubleQuoted(part.parts, part_vals)
1652
1653 elif case(word_part_e.CommandSub):
1654 part = cast(CommandSub, UP_part)
1655 id_ = part.left_token.id
1656 if id_ in (Id.Left_DollarParen, Id.Left_AtParen,
1657 Id.Left_Backtick):
1658 sv = self._EvalCommandSub(part,
1659 quoted) # type: part_value_t
1660
1661 elif id_ in (Id.Left_ProcSubIn, Id.Left_ProcSubOut):
1662 sv = self._EvalProcessSub(part)
1663
1664 else:
1665 raise AssertionError(id_)
1666
1667 part_vals.append(sv)
1668
1669 elif case(word_part_e.SimpleVarSub):
1670 part = cast(SimpleVarSub, UP_part)
1671 self._EvalSimpleVarSub(part, part_vals, quoted)
1672
1673 elif case(word_part_e.BracedVarSub):
1674 part = cast(BracedVarSub, UP_part)
1675 self._EvalBracedVarSub(part, part_vals, quoted)
1676
1677 elif case(word_part_e.TildeSub):
1678 part = cast(word_part.TildeSub, UP_part)
1679 # We never parse a quoted string into a TildeSub.
1680 assert not quoted
1681 s = self.tilde_ev.Eval(part)
1682 v = Piece(s, True, False) # NOT split even when unquoted!
1683 part_vals.append(v)
1684
1685 elif case(word_part_e.ArithSub):
1686 part = cast(word_part.ArithSub, UP_part)
1687 num = self.arith_ev.EvalToBigInt(part.anode)
1688 v = Piece(mops.ToStr(num), quoted, not quoted)
1689 part_vals.append(v)
1690
1691 elif case(word_part_e.ExtGlob):
1692 part = cast(word_part.ExtGlob, UP_part)
1693 #if not self.exec_opts.extglob():
1694 # die() # disallow at runtime? Don't just decay
1695
1696 # Create a node to hold the flattened tree. The caller decides whether
1697 # to pass it to fnmatch() or replace it with '*' and pass it to glob().
1698 part_vals2 = [] # type: List[part_value_t]
1699 self._EvalExtGlob(part, part_vals2) # flattens tree
1700 part_vals.append(part_value.ExtGlob(part_vals2))
1701
1702 elif case(word_part_e.BashRegexGroup):
1703 part = cast(word_part.BashRegexGroup, UP_part)
1704
1705 part_vals.append(Piece('(', False, False)) # not quoted
1706 if part.child:
1707 self._EvalWordToParts(part.child, part_vals, 0)
1708 part_vals.append(Piece(')', False, False))
1709
1710 elif case(word_part_e.Splice):
1711 part = cast(word_part.Splice, UP_part)
1712 val = self.mem.GetValue(part.var_name)
1713
1714 strs = self.expr_ev.SpliceValue(val, part)
1715 part_vals.append(part_value.Array(strs))
1716
1717 elif case(word_part_e.ExprSub):
1718 part = cast(word_part.ExprSub, UP_part)
1719 part_val = self.expr_ev.EvalExprSub(part)
1720 part_vals.append(part_val)
1721
1722 elif case(word_part_e.ZshVarSub):
1723 part = cast(word_part.ZshVarSub, UP_part)
1724 e_die("ZSH var subs are parsed, but can't be evaluated",
1725 part.left)
1726
1727 else:
1728 raise AssertionError(part.tag())
1729
1730 def _EvalRhsWordToParts(self, w, part_vals, eval_flags=0):
1731 # type: (rhs_word_t, List[part_value_t], int) -> None
1732 quoted = bool(eval_flags & QUOTED)
1733
1734 UP_w = w
1735 with tagswitch(w) as case:
1736 if case(rhs_word_e.Empty):
1737 part_vals.append(Piece('', quoted, not quoted))
1738
1739 elif case(rhs_word_e.Compound):
1740 w = cast(CompoundWord, UP_w)
1741 self._EvalWordToParts(w, part_vals, eval_flags=eval_flags)
1742
1743 else:
1744 raise AssertionError()
1745
1746 def _EvalWordToParts(self, w, part_vals, eval_flags=0):
1747 # type: (CompoundWord, List[part_value_t], int) -> None
1748 """Helper for EvalRhsWord, EvalWordSequence, etc.
1749
1750 Returns:
1751 Appends to part_vals. Note that this is a TREE.
1752 """
1753 # Does the word have an extended glob? This is a special case because
1754 # of the way we use glob() and then fnmatch(..., FNM_EXTMATCH) to
1755 # implement extended globs. It's hard to carry that extra information
1756 # all the way past the word splitting stage.
1757
1758 # OSH semantic limitations: If a word has an extended glob part, then
1759 # 1. It can't have an array
1760 # 2. Word splitting of unquoted words isn't respected
1761
1762 word_part_vals = [] # type: List[part_value_t]
1763 has_extglob = False
1764 for p in w.parts:
1765 if p.tag() == word_part_e.ExtGlob:
1766 has_extglob = True
1767 self._EvalWordPart(p, word_part_vals, eval_flags)
1768
1769 # Caller REQUESTED extglob evaluation, AND we parsed word_part.ExtGlob()
1770 if has_extglob:
1771 if bool(eval_flags & EXTGLOB_FILES):
1772 # Treat the WHOLE word as a pattern. We need to TWO VARIANTS of the
1773 # word because of the way we use libc:
1774 # 1. With '*' for extglob parts
1775 # 2. With _EvalExtGlob() for extglob parts
1776
1777 glob_parts = [] # type: List[str]
1778 fnmatch_parts = [] # type: List[str]
1779 self._TranslateExtGlob(word_part_vals, w, glob_parts,
1780 fnmatch_parts)
1781
1782 #log('word_part_vals %s', word_part_vals)
1783 glob_pat = ''.join(glob_parts)
1784 fnmatch_pat = ''.join(fnmatch_parts)
1785 #log("glob %s fnmatch %s", glob_pat, fnmatch_pat)
1786
1787 results = [] # type: List[str]
1788 n = self.globber.ExpandExtended(glob_pat, fnmatch_pat, results)
1789 if n < 0:
1790 raise error.FailGlob(
1791 'Extended glob %r matched no files' % fnmatch_pat, w)
1792
1793 part_vals.append(part_value.Array(results))
1794 elif bool(eval_flags & EXTGLOB_NESTED):
1795 # We only glob at the TOP level of @(nested|@(pattern))
1796 part_vals.extend(word_part_vals)
1797 else:
1798 # e.g. simple_word_eval, assignment builtin
1799 e_die('Extended glob not allowed in this word', w)
1800 else:
1801 part_vals.extend(word_part_vals)
1802
1803 def _PartValsToString(self, part_vals, w, eval_flags, strs):
1804 # type: (List[part_value_t], CompoundWord, int, List[str]) -> None
1805 """Helper for EvalWordToString, similar to _ConcatPartVals() above.
1806
1807 Note: arg 'w' could just be a span ID
1808 """
1809 for part_val in part_vals:
1810 UP_part_val = part_val
1811 with tagswitch(part_val) as case:
1812 if case(part_value_e.String):
1813 part_val = cast(Piece, UP_part_val)
1814 s = part_val.s
1815 if part_val.quoted:
1816 if eval_flags & QUOTE_FNMATCH:
1817 # [[ foo == */"*".py ]] or case (*.py) or ${x%*.py} or ${x//*.py/}
1818 s = glob_.GlobEscape(s)
1819 elif eval_flags & QUOTE_ERE:
1820 s = glob_.ExtendedRegexEscape(s)
1821 strs.append(s)
1822
1823 elif case(part_value_e.Array):
1824 part_val = cast(part_value.Array, UP_part_val)
1825 if self.exec_opts.strict_array():
1826 # Examples: echo f > "$@"; local foo="$@"
1827
1828 # TODO: This attributes too coarsely, to the word rather than the
1829 # parts. Problem: the word is a TREE of parts, but we only have a
1830 # flat list of part_vals. The only case where we really get arrays
1831 # is "$@", "${a[@]}", "${a[@]//pat/replace}", etc.
1832 e_die(
1833 "This word should yield a string, but it contains an array",
1834 w)
1835
1836 # TODO: Maybe add detail like this.
1837 #e_die('RHS of assignment should only have strings. '
1838 # 'To assign arrays, use b=( "${a[@]}" )')
1839 else:
1840 # It appears to not respect IFS
1841 tmp = [s for s in part_val.strs if s is not None]
1842 s = ' '.join(tmp) # TODO: eliminate double join()?
1843 strs.append(s)
1844
1845 elif case(part_value_e.ExtGlob):
1846 part_val = cast(part_value.ExtGlob, UP_part_val)
1847
1848 # Extended globs are only allowed where we expect them!
1849 if not bool(eval_flags & QUOTE_FNMATCH):
1850 e_die('extended glob not allowed in this word', w)
1851
1852 # recursive call
1853 self._PartValsToString(part_val.part_vals, w, eval_flags,
1854 strs)
1855
1856 else:
1857 raise AssertionError()
1858
1859 def EvalWordToString(self, UP_w, eval_flags=0):
1860 # type: (word_t, int) -> value.Str
1861 """Given a word, return a string.
1862
1863 Flags can contain a quoting algorithm.
1864 """
1865 assert UP_w.tag() == word_e.Compound, UP_w
1866 w = cast(CompoundWord, UP_w)
1867
1868 if eval_flags == 0: # QUOTE_FNMATCH etc. breaks optimization
1869 fast_str = word_.FastStrEval(w)
1870 if fast_str is not None:
1871 return value.Str(fast_str)
1872
1873 # Could we additionally optimize a=$b, if we know $b isn't an array
1874 # etc.?
1875
1876 # Note: these empty lists are hot in fib benchmark
1877
1878 part_vals = [] # type: List[part_value_t]
1879 for p in w.parts:
1880 # this doesn't use eval_flags, which is slightly confusing
1881 self._EvalWordPart(p, part_vals, 0)
1882
1883 strs = [] # type: List[str]
1884 self._PartValsToString(part_vals, w, eval_flags, strs)
1885 return value.Str(''.join(strs))
1886
1887 def EvalWordToPattern(self, UP_w):
1888 # type: (rhs_word_t) -> Tuple[value.Str, bool]
1889 """Like EvalWordToString, but returns whether we got ExtGlob."""
1890 if UP_w.tag() == rhs_word_e.Empty:
1891 return value.Str(''), False
1892
1893 assert UP_w.tag() == rhs_word_e.Compound, UP_w
1894 w = cast(CompoundWord, UP_w)
1895
1896 has_extglob = False
1897 part_vals = [] # type: List[part_value_t]
1898 for p in w.parts:
1899 # this doesn't use eval_flags, which is slightly confusing
1900 self._EvalWordPart(p, part_vals, 0)
1901 if p.tag() == word_part_e.ExtGlob:
1902 has_extglob = True
1903
1904 strs = [] # type: List[str]
1905 self._PartValsToString(part_vals, w, QUOTE_FNMATCH, strs)
1906 return value.Str(''.join(strs)), has_extglob
1907
1908 def EvalForPlugin(self, w):
1909 # type: (CompoundWord) -> value.Str
1910 """Wrapper around EvalWordToString that prevents errors.
1911
1912 Runtime errors like $(( 1 / 0 )) and mutating $? like $(exit 42)
1913 are handled here.
1914
1915 Similar to ExprEvaluator.PluginCall().
1916 """
1917 with state.ctx_Registers(self.mem): # to "sandbox" $? and $PIPESTATUS
1918 try:
1919 val = self.EvalWordToString(w)
1920 except error.FatalRuntime as e:
1921 val = value.Str('<Runtime error: %s>' % e.UserErrorString())
1922
1923 except (IOError, OSError) as e:
1924 val = value.Str('<I/O error: %s>' % pyutil.strerror(e))
1925
1926 except KeyboardInterrupt:
1927 val = value.Str('<Ctrl-C>')
1928
1929 return val
1930
1931 def EvalRhsWord(self, UP_w):
1932 # type: (rhs_word_t) -> value_t
1933 """Used for RHS of assignment.
1934
1935 There is no splitting.
1936 """
1937 if UP_w.tag() == rhs_word_e.Empty:
1938 return value.Str('')
1939
1940 assert UP_w.tag() == word_e.Compound, UP_w
1941 w = cast(CompoundWord, UP_w)
1942
1943 if len(w.parts) == 1:
1944 part0 = w.parts[0]
1945 UP_part0 = part0
1946 tag = part0.tag()
1947 # Special case for a=(1 2). ShArrayLiteral won't appear in words that
1948 # don't look like assignments.
1949 if tag == word_part_e.ShArrayLiteral:
1950 part0 = cast(ShArrayLiteral, UP_part0)
1951 array_words = part0.words
1952 words = braces.BraceExpandWords(array_words)
1953 strs = self.EvalWordSequence(words)
1954 return value.BashArray(strs)
1955
1956 if tag == word_part_e.BashAssocLiteral:
1957 part0 = cast(word_part.BashAssocLiteral, UP_part0)
1958 d = NewDict() # type: Dict[str, str]
1959 for pair in part0.pairs:
1960 k = self.EvalWordToString(pair.key)
1961 v = self.EvalWordToString(pair.value)
1962 d[k.s] = v.s
1963 return value.BashAssoc(d)
1964
1965 # If RHS doesn't look like a=( ... ), then it must be a string.
1966 return self.EvalWordToString(w)
1967
1968 def _EvalWordFrame(self, frame, argv):
1969 # type: (List[Piece], List[str]) -> None
1970 all_empty = True
1971 all_quoted = True
1972 any_quoted = False
1973
1974 #log('--- frame %s', frame)
1975
1976 for piece in frame:
1977 if len(piece.s):
1978 all_empty = False
1979
1980 if piece.quoted:
1981 any_quoted = True
1982 else:
1983 all_quoted = False
1984
1985 # Elision of ${empty}${empty} but not $empty"$empty" or $empty""
1986 if all_empty and not any_quoted:
1987 return
1988
1989 # If every frag is quoted, e.g. "$a$b" or any part in "${a[@]}"x, then
1990 # don't do word splitting or globbing.
1991 if all_quoted:
1992 tmp = [piece.s for piece in frame]
1993 a = ''.join(tmp)
1994 argv.append(a)
1995 return
1996
1997 will_glob = not self.exec_opts.noglob()
1998
1999 # Array of strings, some of which are BOTH IFS-escaped and GLOB escaped!
2000 frags = [] # type: List[str]
2001 for piece in frame:
2002 if will_glob and piece.quoted:
2003 frag = glob_.GlobEscape(piece.s)
2004 else:
2005 # If we have a literal \, then we turn it into \\\\.
2006 # Splitting takes \\\\ -> \\
2007 # Globbing takes \\ to \ if it doesn't match
2008 frag = _BackslashEscape(piece.s)
2009
2010 if piece.do_split:
2011 frag = _BackslashEscape(frag)
2012 else:
2013 frag = self.splitter.Escape(frag)
2014
2015 frags.append(frag)
2016
2017 flat = ''.join(frags)
2018 #log('flat: %r', flat)
2019
2020 args = self.splitter.SplitForWordEval(flat)
2021
2022 # space=' '; argv $space"". We have a quoted part, but we CANNOT elide.
2023 # Add it back and don't bother globbing.
2024 if len(args) == 0 and any_quoted:
2025 argv.append('')
2026 return
2027
2028 #log('split args: %r', args)
2029 for a in args:
2030 if glob_.LooksLikeGlob(a):
2031 n = self.globber.Expand(a, argv)
2032 if n < 0:
2033 # TODO: location info, with span IDs carried through the frame
2034 raise error.FailGlob('Pattern %r matched no files' % a,
2035 loc.Missing)
2036 else:
2037 argv.append(glob_.GlobUnescape(a))
2038
2039 def _EvalWordToArgv(self, w):
2040 # type: (CompoundWord) -> List[str]
2041 """Helper for _EvalAssignBuiltin.
2042
2043 Splitting and globbing are disabled for assignment builtins.
2044
2045 Example: declare -"${a[@]}" b=(1 2)
2046 where a is [x b=a d=a]
2047 """
2048 part_vals = [] # type: List[part_value_t]
2049 self._EvalWordToParts(w, part_vals, 0) # not double quoted
2050 frames = _MakeWordFrames(part_vals)
2051 argv = [] # type: List[str]
2052 for frame in frames:
2053 if len(frame): # empty array gives empty frame!
2054 tmp = [piece.s for piece in frame]
2055 argv.append(''.join(tmp)) # no split or glob
2056 #log('argv: %s', argv)
2057 return argv
2058
2059 def _EvalAssignBuiltin(self, builtin_id, arg0, words):
2060 # type: (builtin_t, str, List[CompoundWord]) -> cmd_value.Assign
2061 """Handles both static and dynamic assignment, e.g.
2062
2063 x='foo=bar' local a=(1 2) $x
2064 """
2065 # Grammar:
2066 #
2067 # ('builtin' | 'command')* keyword flag* pair*
2068 # flag = [-+].*
2069 #
2070 # There is also command -p, but we haven't implemented it. Maybe just punt
2071 # on it. Punted on 'builtin' and 'command' for now too.
2072
2073 eval_to_pairs = True # except for -f and -F
2074 started_pairs = False
2075
2076 flags = [arg0] # initial flags like -p, and -f -F name1 name2
2077 flag_locs = [words[0]]
2078 assign_args = [] # type: List[AssignArg]
2079
2080 n = len(words)
2081 for i in xrange(1, n): # skip first word
2082 w = words[i]
2083
2084 if word_.IsVarLike(w):
2085 started_pairs = True # Everything from now on is an assign_pair
2086
2087 if started_pairs:
2088 left_token, close_token, part_offset = word_.DetectShAssignment(
2089 w)
2090 if left_token: # Detected statically
2091 if left_token.id != Id.Lit_VarLike:
2092 # (not guaranteed since started_pairs is set twice)
2093 e_die('LHS array not allowed in assignment builtin', w)
2094
2095 if lexer.IsPlusEquals(left_token):
2096 var_name = lexer.TokenSliceRight(left_token, -2)
2097 append = True
2098 else:
2099 var_name = lexer.TokenSliceRight(left_token, -1)
2100 append = False
2101
2102 if part_offset == len(w.parts):
2103 rhs = rhs_word.Empty # type: rhs_word_t
2104 else:
2105 # tmp is for intersection of C++/MyPy type systems
2106 tmp = CompoundWord(w.parts[part_offset:])
2107 word_.TildeDetectAssign(tmp)
2108 rhs = tmp
2109
2110 with state.ctx_AssignBuiltin(self.mutable_opts):
2111 right = self.EvalRhsWord(rhs)
2112
2113 arg2 = AssignArg(var_name, right, append, w)
2114 assign_args.append(arg2)
2115
2116 else: # e.g. export $dynamic
2117 argv = self._EvalWordToArgv(w)
2118 for arg in argv:
2119 arg2 = _SplitAssignArg(arg, w)
2120 assign_args.append(arg2)
2121
2122 else:
2123 argv = self._EvalWordToArgv(w)
2124 for arg in argv:
2125 if arg.startswith('-') or arg.startswith('+'):
2126 # e.g. declare -r +r
2127 flags.append(arg)
2128 flag_locs.append(w)
2129
2130 # Shortcut that relies on -f and -F always meaning "function" for
2131 # all assignment builtins
2132 if 'f' in arg or 'F' in arg:
2133 eval_to_pairs = False
2134
2135 else: # e.g. export $dynamic
2136 if eval_to_pairs:
2137 arg2 = _SplitAssignArg(arg, w)
2138 assign_args.append(arg2)
2139 started_pairs = True
2140 else:
2141 flags.append(arg)
2142
2143 return cmd_value.Assign(builtin_id, flags, flag_locs, assign_args)
2144
2145 def SimpleEvalWordSequence2(self, words, allow_assign):
2146 # type: (List[CompoundWord], bool) -> cmd_value_t
2147 """Simple word evaluation for YSH."""
2148 strs = [] # type: List[str]
2149 locs = [] # type: List[CompoundWord]
2150
2151 for i, w in enumerate(words):
2152 # No globbing in the first arg for command.Simple.
2153 if i == 0 and allow_assign:
2154 strs0 = self._EvalWordToArgv(w) # respects strict-array
2155 if len(strs0) == 1:
2156 arg0 = strs0[0]
2157 builtin_id = consts.LookupAssignBuiltin(arg0)
2158 if builtin_id != consts.NO_INDEX:
2159 # Same logic as legacy word eval, with no splitting
2160 return self._EvalAssignBuiltin(builtin_id, arg0, words)
2161
2162 strs.extend(strs0)
2163 for _ in strs0:
2164 locs.append(w)
2165 continue
2166
2167 if glob_.LooksLikeStaticGlob(w):
2168 val = self.EvalWordToString(w) # respects strict-array
2169 num_appended = self.globber.Expand(val.s, strs)
2170 if num_appended < 0:
2171 raise error.FailGlob('Pattern %r matched no files' % val.s,
2172 w)
2173 for _ in xrange(num_appended):
2174 locs.append(w)
2175 continue
2176
2177 part_vals = [] # type: List[part_value_t]
2178 self._EvalWordToParts(w, part_vals, 0) # not quoted
2179
2180 if 0:
2181 log('')
2182 log('Static: part_vals after _EvalWordToParts:')
2183 for entry in part_vals:
2184 log(' %s', entry)
2185
2186 # Still need to process
2187 frames = _MakeWordFrames(part_vals)
2188
2189 if 0:
2190 log('')
2191 log('Static: frames after _MakeWordFrames:')
2192 for entry in frames:
2193 log(' %s', entry)
2194
2195 # We will still allow x"${a[@]"x, though it's deprecated by @a, which
2196 # disallows such expressions at parse time.
2197 for frame in frames:
2198 if len(frame): # empty array gives empty frame!
2199 tmp = [piece.s for piece in frame]
2200 strs.append(''.join(tmp)) # no split or glob
2201 locs.append(w)
2202
2203 return cmd_value.Argv(strs, locs, None, None, None, None)
2204
2205 def EvalWordSequence2(self, words, allow_assign=False):
2206 # type: (List[CompoundWord], bool) -> cmd_value_t
2207 """Turns a list of Words into a list of strings.
2208
2209 Unlike the EvalWord*() methods, it does globbing.
2210
2211 Args:
2212 words: list of Word instances
2213
2214 Returns:
2215 argv: list of string arguments, or None if there was an eval error
2216 """
2217 if self.exec_opts.simple_word_eval():
2218 return self.SimpleEvalWordSequence2(words, allow_assign)
2219
2220 # Parse time:
2221 # 1. brace expansion. TODO: Do at parse time.
2222 # 2. Tilde detection. DONE at parse time. Only if Id.Lit_Tilde is the
2223 # first WordPart.
2224 #
2225 # Run time:
2226 # 3. tilde sub, var sub, command sub, arith sub. These are all
2227 # "concurrent" on WordParts. (optional process sub with <() )
2228 # 4. word splitting. Can turn this off with a shell option? Definitely
2229 # off for oil.
2230 # 5. globbing -- several exec_opts affect this: nullglob, safeglob, etc.
2231
2232 #log('W %s', words)
2233 strs = [] # type: List[str]
2234 locs = [] # type: List[CompoundWord]
2235
2236 n = 0
2237 for i, w in enumerate(words):
2238 fast_str = word_.FastStrEval(w)
2239 if fast_str is not None:
2240 strs.append(fast_str)
2241 locs.append(w)
2242
2243 # e.g. the 'local' in 'local a=b c=d' will be here
2244 if allow_assign and i == 0:
2245 builtin_id = consts.LookupAssignBuiltin(fast_str)
2246 if builtin_id != consts.NO_INDEX:
2247 return self._EvalAssignBuiltin(builtin_id, fast_str,
2248 words)
2249 continue
2250
2251 part_vals = [] # type: List[part_value_t]
2252 self._EvalWordToParts(w, part_vals, EXTGLOB_FILES)
2253
2254 # DYNAMICALLY detect if we're going to run an assignment builtin, and
2255 # change the rest of the evaluation algorithm if so.
2256 #
2257 # We want to allow:
2258 # e=export
2259 # $e foo=bar
2260 #
2261 # But we don't want to evaluate the first word twice in the case of:
2262 # $(some-command) --flag
2263 if allow_assign and i == 0 and len(part_vals) == 1:
2264 val0 = part_vals[0]
2265 UP_val0 = val0
2266 if val0.tag() == part_value_e.String:
2267 val0 = cast(Piece, UP_val0)
2268 if not val0.quoted:
2269 builtin_id = consts.LookupAssignBuiltin(val0.s)
2270 if builtin_id != consts.NO_INDEX:
2271 return self._EvalAssignBuiltin(
2272 builtin_id, val0.s, words)
2273
2274 if 0:
2275 log('')
2276 log('part_vals after _EvalWordToParts:')
2277 for entry in part_vals:
2278 log(' %s', entry)
2279
2280 frames = _MakeWordFrames(part_vals)
2281 if 0:
2282 log('')
2283 log('frames after _MakeWordFrames:')
2284 for entry in frames:
2285 log(' %s', entry)
2286
2287 # Do splitting and globbing. Each frame will append zero or more args.
2288 for frame in frames:
2289 self._EvalWordFrame(frame, strs)
2290
2291 # Fill in locations parallel to strs.
2292 n_next = len(strs)
2293 for _ in xrange(n_next - n):
2294 locs.append(w)
2295 n = n_next
2296
2297 # A non-assignment command.
2298 # NOTE: Can't look up builtins here like we did for assignment, because
2299 # functions can override builtins.
2300 return cmd_value.Argv(strs, locs, None, None, None, None)
2301
2302 def EvalWordSequence(self, words):
2303 # type: (List[CompoundWord]) -> List[str]
2304 """For arrays and for loops.
2305
2306 They don't allow assignment builtins.
2307 """
2308 UP_cmd_val = self.EvalWordSequence2(words)
2309
2310 assert UP_cmd_val.tag() == cmd_value_e.Argv
2311 cmd_val = cast(cmd_value.Argv, UP_cmd_val)
2312 return cmd_val.argv
2313
2314
2315class NormalWordEvaluator(AbstractWordEvaluator):
2316
2317 def __init__(
2318 self,
2319 mem, # type: state.Mem
2320 exec_opts, # type: optview.Exec
2321 mutable_opts, # type: state.MutableOpts
2322 tilde_ev, # type: TildeEvaluator
2323 splitter, # type: SplitContext
2324 errfmt, # type: ErrorFormatter
2325 ):
2326 # type: (...) -> None
2327 AbstractWordEvaluator.__init__(self, mem, exec_opts, mutable_opts,
2328 tilde_ev, splitter, errfmt)
2329 self.shell_ex = None # type: _Executor
2330
2331 def CheckCircularDeps(self):
2332 # type: () -> None
2333 assert self.arith_ev is not None
2334 # Disabled for pure OSH
2335 #assert self.expr_ev is not None
2336 assert self.shell_ex is not None
2337 assert self.prompt_ev is not None
2338
2339 def _EvalCommandSub(self, cs_part, quoted):
2340 # type: (CommandSub, bool) -> part_value_t
2341 stdout_str = self.shell_ex.RunCommandSub(cs_part)
2342
2343 if cs_part.left_token.id == Id.Left_AtParen:
2344 # YSH splitting algorithm: does not depend on IFS
2345 try:
2346 strs = j8.SplitJ8Lines(stdout_str)
2347 except error.Decode as e:
2348 # status code 4 is special, for encode/decode errors.
2349 raise error.Structured(4, e.Message(), cs_part.left_token)
2350
2351 #strs = self.splitter.SplitForWordEval(stdout_str)
2352 return part_value.Array(strs)
2353 else:
2354 return Piece(stdout_str, quoted, not quoted)
2355
2356 def _EvalProcessSub(self, cs_part):
2357 # type: (CommandSub) -> Piece
2358 dev_path = self.shell_ex.RunProcessSub(cs_part)
2359 # pretend it's quoted; no split or glob
2360 return Piece(dev_path, True, False)
2361
2362
2363_DUMMY = '__NO_COMMAND_SUB__'
2364
2365
2366class CompletionWordEvaluator(AbstractWordEvaluator):
2367 """An evaluator that has no access to an executor.
2368
2369 NOTE: core/completion.py doesn't actually try to use these strings to
2370 complete. If you have something like 'echo $(echo hi)/f<TAB>', it sees the
2371 inner command as the last one, and knows that it is not at the end of the
2372 line.
2373 """
2374
2375 def __init__(
2376 self,
2377 mem, # type: state.Mem
2378 exec_opts, # type: optview.Exec
2379 mutable_opts, # type: state.MutableOpts
2380 tilde_ev, # type: TildeEvaluator
2381 splitter, # type: SplitContext
2382 errfmt, # type: ErrorFormatter
2383 ):
2384 # type: (...) -> None
2385 AbstractWordEvaluator.__init__(self, mem, exec_opts, mutable_opts,
2386 tilde_ev, splitter, errfmt)
2387
2388 def CheckCircularDeps(self):
2389 # type: () -> None
2390 assert self.prompt_ev is not None
2391 assert self.arith_ev is not None
2392 assert self.expr_ev is not None
2393
2394 def _EvalCommandSub(self, cs_part, quoted):
2395 # type: (CommandSub, bool) -> part_value_t
2396 if cs_part.left_token.id == Id.Left_AtParen:
2397 return part_value.Array([_DUMMY])
2398 else:
2399 return Piece(_DUMMY, quoted, not quoted)
2400
2401 def _EvalProcessSub(self, cs_part):
2402 # type: (CommandSub) -> Piece
2403 # pretend it's quoted; no split or glob
2404 return Piece('__NO_PROCESS_SUB__', True, False)
2405
2406
2407# vim: sw=4