osh/word_eval.py

OILS / osh / word_eval.py View on Github | oilshell.org

2451 lines, 1473 significant

1	"""
2	word_eval.py - Evaluator for the word language.
3	"""
4
5	from _devbuild.gen.id_kind_asdl import Id, Kind, Kind_str
6	from _devbuild.gen.syntax_asdl import (
7	Token,
8	SimpleVarSub,
9	loc,
10	loc_t,
11	BracedVarSub,
12	CommandSub,
13	bracket_op,
14	bracket_op_e,
15	suffix_op,
16	suffix_op_e,
17	ShArrayLiteral,
18	SingleQuoted,
19	DoubleQuoted,
20	word_e,
21	word_t,
22	CompoundWord,
23	rhs_word,
24	rhs_word_e,
25	rhs_word_t,
26	word_part,
27	word_part_e,
28	)
29	from _devbuild.gen.runtime_asdl import (
30	part_value,
31	part_value_e,
32	part_value_t,
33	cmd_value,
34	cmd_value_e,
35	cmd_value_t,
36	AssignArg,
37	a_index,
38	a_index_e,
39	VTestPlace,
40	VarSubState,
41	Piece,
42	)
43	from _devbuild.gen.option_asdl import option_i, builtin_i
44	from _devbuild.gen.value_asdl import (
45	value,
46	value_e,
47	value_t,
48	sh_lvalue,
49	sh_lvalue_t,
50	)
51	from core import error
52	from core import pyos
53	from core import pyutil
54	from core import state
55	from core import ui
56	from core import util
57	from data_lang import j8
58	from data_lang import j8_lite
59	from core.error import e_die
60	from frontend import consts
61	from frontend import lexer
62	from frontend import location
63	from mycpp import mops
64	from mycpp.mylib import log, tagswitch, NewDict
65	from osh import braces
66	from osh import glob_
67	from osh import string_ops
68	from osh import word_
69	from ysh import expr_eval
70	from ysh import val_ops
71
72	from typing import Optional, Tuple, List, Dict, cast, TYPE_CHECKING
73
74	if TYPE_CHECKING:
75	from _devbuild.gen.syntax_asdl import word_part_t
76	from _devbuild.gen.option_asdl import builtin_t
77	from core import optview
78	from core.state import Mem
79	from core.ui import ErrorFormatter
80	from core.vm import _Executor
81	from osh.split import SplitContext
82	from osh import prompt
83	from osh import sh_expr_eval
84
85	# Flags for _EvalWordToParts and _EvalWordPart (not all are used for both)
86	QUOTED = 1 << 0
87	IS_SUBST = 1 << 1
88
89	EXTGLOB_FILES = 1 << 2 # allow @(cc) from file system?
90	EXTGLOB_MATCH = 1 << 3 # allow @(cc) in pattern matching?
91	EXTGLOB_NESTED = 1 << 4 # for @(one\|!(two\|three))
92
93	# For EvalWordToString
94	QUOTE_FNMATCH = 1 << 5
95	QUOTE_ERE = 1 << 6
96
97	# For compatibility, ${BASH_SOURCE} and ${BASH_SOURCE[@]} are both valid.
98	# Ditto for ${FUNCNAME} and ${BASH_LINENO}.
99	_STRING_AND_ARRAY = ['BASH_SOURCE', 'FUNCNAME', 'BASH_LINENO']
100
101
102	def ShouldArrayDecay(var_name, exec_opts, is_plain_var_sub=True):
103	# type: (str, optview.Exec, bool) -> bool
104	"""Return whether we should allow ${a} to mean ${a[0]}."""
105	return (not exec_opts.strict_array() or
106	is_plain_var_sub and var_name in _STRING_AND_ARRAY)
107
108
109	def DecayArray(val):
110	# type: (value_t) -> value_t
111	"""Resolve ${array} to ${array[0]}."""
112	if val.tag() == value_e.BashArray:
113	array_val = cast(value.BashArray, val)
114	s = array_val.strs[0] if len(array_val.strs) else None
115	elif val.tag() == value_e.BashAssoc:
116	assoc_val = cast(value.BashAssoc, val)
117	s = assoc_val.d['0'] if '0' in assoc_val.d else None
118	else:
119	raise AssertionError(val.tag())
120
121	if s is None:
122	return value.Undef
123	else:
124	return value.Str(s)
125
126
127	def GetArrayItem(strs, index):
128	# type: (List[str], int) -> Optional[str]
129
130	n = len(strs)
131	if index < 0:
132	index += n
133
134	if 0 <= index and index < n:
135	# TODO: strs->index() has a redundant check for (i < 0)
136	s = strs[index]
137	# note: s could be None because representation is sparse
138	else:
139	s = None
140	return s
141
142
143	# Use libc to parse NAME, NAME=value, and NAME+=value. We want submatch
144	# extraction, but I haven't used that in re2c, and we would need a new kind of
145	# binding.
146	#
147	ASSIGN_ARG_RE = '^([a-zA-Z_][a-zA-Z0-9_])((=\|\+=)(.))?$'
148
149	# Eggex equivalent:
150	#
151	# VarName = /
152	# [a-z A-Z _ ]
153	# [a-z A-Z 0-9 _ ]*
154	# /
155	#
156	# SplitArg = /
157	# %begin
158	# < VarName >
159	# < < '=' \| '+=' > < dot* > > ?
160	# %end
161	# /
162	# Note: must use < > for grouping because there is no non-capturing group.
163
164
165	def _SplitAssignArg(arg, blame_word):
166	# type: (str, CompoundWord) -> AssignArg
167	"""Dynamically parse argument to declare, export, etc.
168
169	This is a fallback to the static parsing done below.
170	"""
171	# Note: it would be better to cache regcomp(), but we don't have an API for
172	# that, and it probably isn't a bottleneck now
173	m = util.RegexSearch(ASSIGN_ARG_RE, arg)
174	if m is None:
175	e_die("Assignment builtin expected NAME=value, got %r" % arg,
176	blame_word)
177
178	var_name = m[1]
179	# m[2] is used for grouping; ERE doesn't have non-capturing groups
180
181	op = m[3]
182	assert op is not None, op
183	if len(op): # declare NAME=
184	val = value.Str(m[4]) # type: Optional[value_t]
185	append = op[0] == '+'
186	else: # declare NAME
187	val = None # no operator
188	append = False
189
190	return AssignArg(var_name, val, append, blame_word)
191
192
193	# NOTE: Could be done with util.BackslashEscape like glob_.GlobEscape().
194	def _BackslashEscape(s):
195	# type: (str) -> str
196	"""Double up backslashes.
197
198	Useful for strings about to be globbed and strings about to be IFS
199	escaped.
200	"""
201	return s.replace('\\', '\\\\')
202
203
204	def _ValueToPartValue(val, quoted, part_loc):
205	# type: (value_t, bool, word_part_t) -> part_value_t
206	"""Helper for VarSub evaluation.
207
208	Called by _EvalBracedVarSub and _EvalWordPart for SimpleVarSub.
209	"""
210	UP_val = val
211
212	with tagswitch(val) as case:
213	if case(value_e.Undef):
214	# This happens in the case of ${undef+foo}. We skipped _EmptyStrOrError,
215	# but we have to append to the empty string.
216	return Piece('', quoted, not quoted)
217
218	elif case(value_e.Str):
219	val = cast(value.Str, UP_val)
220	return Piece(val.s, quoted, not quoted)
221
222	elif case(value_e.BashArray):
223	val = cast(value.BashArray, UP_val)
224	return part_value.Array(val.strs)
225
226	elif case(value_e.BashAssoc):
227	val = cast(value.BashAssoc, UP_val)
228	# bash behavior: splice values!
229	return part_value.Array(val.d.values())
230
231	# Cases added for YSH
232	# value_e.List is also here - we use val_ops.stringify()s err message
233	elif case(value_e.Null, value_e.Bool, value_e.Int, value_e.Float,
234	value_e.Eggex, value_e.List):
235	s = val_ops.Stringify(val, loc.Missing)
236	return Piece(s, quoted, not quoted)
237
238	else:
239	raise error.TypeErr(val, "Can't substitute into word",
240	loc.WordPart(part_loc))
241
242	raise AssertionError('for -Wreturn-type in C++')
243
244
245	def _MakeWordFrames(part_vals):
246	# type: (List[part_value_t]) -> List[List[Piece]]
247	"""A word evaluates to a flat list of part_value (String or Array). frame
248	is a portion that results in zero or more args. It can never be joined.
249	This idea exists because of arrays like "$@" and "${a[@]}".
250
251	Example:
252
253	a=(1 '2 3' 4)
254	x=x
255	y=y
256
257	# This word
258	$x"${a[@]}"$y
259
260	# Results in Three frames:
261	[ ('x', False, True), ('1', True, False) ]
262	[ ('2 3', True, False) ]
263	[ ('4', True, False), ('y', False, True) ]
264
265	Note: A frame is a 3-tuple that's identical to Piece()? Maybe we
266	should make that top level type.
267
268	TODO:
269	- Instead of List[List[Piece]], where List[Piece] is a Frame
270	- Change this representation to
271	Frames = (List[Piece] pieces, List[int] break_indices)
272	# where break_indices are the end
273
274	Consider a common case like "$x" or "${x}" - I think this a lot more
275	efficient?
276
277	And then change _EvalWordFrame(pieces: List[Piece], start: int, end: int)
278	"""
279	current = [] # type: List[Piece]
280	frames = [current]
281
282	for p in part_vals:
283	UP_p = p
284
285	with tagswitch(p) as case:
286	if case(part_value_e.String):
287	p = cast(Piece, UP_p)
288	current.append(p)
289
290	elif case(part_value_e.Array):
291	p = cast(part_value.Array, UP_p)
292
293	is_first = True
294	for s in p.strs:
295	if s is None:
296	continue # ignore undefined array entries
297
298	# Arrays parts are always quoted; otherwise they would have decayed to
299	# a string.
300	piece = Piece(s, True, False)
301	if is_first:
302	current.append(piece)
303	is_first = False
304	else:
305	current = [piece]
306	frames.append(current) # singleton frame
307
308	else:
309	raise AssertionError()
310
311	return frames
312
313
314	# TODO: This could be _MakeWordFrames and then sep.join(). It's redundant.
315	def _DecayPartValuesToString(part_vals, join_char):
316	# type: (List[part_value_t], str) -> str
317	# Decay ${a=x"$@"x} to string.
318	out = [] # type: List[str]
319	for p in part_vals:
320	UP_p = p
321	with tagswitch(p) as case:
322	if case(part_value_e.String):
323	p = cast(Piece, UP_p)
324	out.append(p.s)
325	elif case(part_value_e.Array):
326	p = cast(part_value.Array, UP_p)
327	# TODO: Eliminate double join for speed?
328	tmp = [s for s in p.strs if s is not None]
329	out.append(join_char.join(tmp))
330	else:
331	raise AssertionError()
332	return ''.join(out)
333
334
335	def _PerformSlice(
336	val, # type: value_t
337	begin, # type: int
338	length, # type: int
339	has_length, # type: bool
340	part, # type: BracedVarSub
341	arg0_val, # type: value.Str
342	):
343	# type: (...) -> value_t
344	UP_val = val
345	with tagswitch(val) as case:
346	if case(value_e.Str): # Slice UTF-8 characters in a string.
347	val = cast(value.Str, UP_val)
348	s = val.s
349	n = len(s)
350
351	if begin < 0: # Compute offset with unicode
352	byte_begin = n
353	num_iters = -begin
354	for _ in xrange(num_iters):
355	byte_begin = string_ops.PreviousUtf8Char(s, byte_begin)
356	else:
357	byte_begin = string_ops.AdvanceUtf8Chars(s, begin, 0)
358
359	if has_length:
360	if length < 0: # Compute offset with unicode
361	# Confusing: this is a POSITION
362	byte_end = n
363	num_iters = -length
364	for _ in xrange(num_iters):
365	byte_end = string_ops.PreviousUtf8Char(s, byte_end)
366	else:
367	byte_end = string_ops.AdvanceUtf8Chars(
368	s, length, byte_begin)
369	else:
370	byte_end = len(s)
371
372	substr = s[byte_begin:byte_end]
373	result = value.Str(substr) # type: value_t
374
375	elif case(value_e.BashArray): # Slice array entries.
376	val = cast(value.BashArray, UP_val)
377	# NOTE: This error is ALWAYS fatal in bash. It's inconsistent with
378	# strings.
379	if has_length and length < 0:
380	e_die("Array slice can't have negative length: %d" % length,
381	loc.WordPart(part))
382
383	# Quirk: "begin" for positional arguments ($@ and $*) counts $0.
384	if arg0_val is not None:
385	orig = [arg0_val.s]
386	orig.extend(val.strs)
387	else:
388	orig = val.strs
389
390	n = len(orig)
391	if begin < 0:
392	i = n + begin # ${@:-3} starts counts from the end
393	else:
394	i = begin
395	strs = [] # type: List[str]
396	count = 0
397	while i < n:
398	if has_length and count == length: # length could be 0
399	break
400	s = orig[i]
401	if s is not None: # Unset elements don't count towards the length
402	strs.append(s)
403	count += 1
404	i += 1
405
406	result = value.BashArray(strs)
407
408	elif case(value_e.BashAssoc):
409	e_die("Can't slice associative arrays", loc.WordPart(part))
410
411	else:
412	raise error.TypeErr(val, 'Slice op expected Str or BashArray',
413	loc.WordPart(part))
414
415	return result
416
417
418	class StringWordEvaluator(object):
419	"""Interface used by ArithEvaluator / BoolEvaluator"""
420
421	def __init__(self):
422	# type: () -> None
423	"""Empty constructor for mycpp."""
424	pass
425
426	def EvalWordToString(self, w, eval_flags=0):
427	# type: (word_t, int) -> value.Str
428	raise NotImplementedError()
429
430
431	def _GetDollarHyphen(exec_opts):
432	# type: (optview.Exec) -> str
433	chars = [] # type: List[str]
434	if exec_opts.interactive():
435	chars.append('i')
436
437	if exec_opts.errexit():
438	chars.append('e')
439	if exec_opts.noglob():
440	chars.append('f')
441	if exec_opts.noexec():
442	chars.append('n')
443	if exec_opts.nounset():
444	chars.append('u')
445	# NO letter for pipefail?
446	if exec_opts.xtrace():
447	chars.append('x')
448	if exec_opts.noclobber():
449	chars.append('C')
450
451	# bash has:
452	# - c for sh -c, i for sh -i (mksh also has this)
453	# - h for hashing (mksh also has this)
454	# - B for brace expansion
455	return ''.join(chars)
456
457
458	class TildeEvaluator(object):
459
460	def __init__(self, mem, exec_opts):
461	# type: (Mem, optview.Exec) -> None
462	self.mem = mem
463	self.exec_opts = exec_opts
464
465	def GetMyHomeDir(self):
466	# type: () -> Optional[str]
467	"""Consult $HOME first, and then make a libc call.
468
469	Important: the libc call can FAIL, which is why we prefer $HOME. See issue
470	#1578.
471	"""
472	# First look up the HOME var, then ask the OS. This is what bash does.
473	val = self.mem.GetValue('HOME')
474	UP_val = val
475	if val.tag() == value_e.Str:
476	val = cast(value.Str, UP_val)
477	return val.s
478	return pyos.GetMyHomeDir()
479
480	def Eval(self, part):
481	# type: (word_part.TildeSub) -> str
482	"""Evaluates ~ and ~user, given a Lit_TildeLike token."""
483
484	if part.user_name is None:
485	result = self.GetMyHomeDir()
486	else:
487	result = pyos.GetHomeDir(part.user_name)
488
489	if result is None:
490	if self.exec_opts.strict_tilde():
491	e_die("Error expanding tilde (e.g. invalid user)", part.left)
492	else:
493	# Return ~ or ~user literally
494	result = '~'
495	if part.user_name is not None:
496	result = result + part.user_name # mycpp doesn't have +=
497
498	return result
499
500
501	class AbstractWordEvaluator(StringWordEvaluator):
502	"""Abstract base class for word evaluators.
503
504	Public entry points:
505	EvalWordToString EvalForPlugin EvalRhsWord
506	EvalWordSequence EvalWordSequence2
507	"""
508
509	def __init__(
510	self,
511	mem, # type: state.Mem
512	exec_opts, # type: optview.Exec
513	mutable_opts, # type: state.MutableOpts
514	tilde_ev, # type: TildeEvaluator
515	splitter, # type: SplitContext
516	errfmt, # type: ui.ErrorFormatter
517	):
518	# type: (...) -> None
519	self.arith_ev = None # type: sh_expr_eval.ArithEvaluator
520	self.expr_ev = None # type: expr_eval.ExprEvaluator
521	self.prompt_ev = None # type: prompt.Evaluator
522
523	self.unsafe_arith = None # type: sh_expr_eval.UnsafeArith
524
525	self.tilde_ev = tilde_ev
526
527	self.mem = mem # for $HOME, $1, etc.
528	self.exec_opts = exec_opts # for nounset
529	self.mutable_opts = mutable_opts # for _allow_command_sub
530	self.splitter = splitter
531	self.errfmt = errfmt
532
533	self.globber = glob_.Globber(exec_opts)
534
535	def CheckCircularDeps(self):
536	# type: () -> None
537	raise NotImplementedError()
538
539	def _EvalCommandSub(self, cs_part, quoted):
540	# type: (CommandSub, bool) -> part_value_t
541	"""Abstract since it has a side effect."""
542	raise NotImplementedError()
543
544	def _EvalProcessSub(self, cs_part):
545	# type: (CommandSub) -> part_value_t
546	"""Abstract since it has a side effect."""
547	raise NotImplementedError()
548
549	def _EvalVarNum(self, var_num):
550	# type: (int) -> value_t
551	assert var_num >= 0
552	return self.mem.GetArgNum(var_num)
553
554	def _EvalSpecialVar(self, op_id, quoted, vsub_state):
555	# type: (int, bool, VarSubState) -> value_t
556	"""Evaluate $?
557
558	and so forth
559	"""
560	# $@ is special -- it need to know whether it is in a double quoted
561	# context.
562	#
563	# - If it's $@ in a double quoted context, return an ARRAY.
564	# - If it's $@ in a normal context, return a STRING, which then will be
565	# subject to splitting.
566
567	if op_id in (Id.VSub_At, Id.VSub_Star):
568	argv = self.mem.GetArgv()
569	val = value.BashArray(argv) # type: value_t
570	if op_id == Id.VSub_At:
571	# "$@" evaluates to an array, $@ should be decayed
572	vsub_state.join_array = not quoted
573	else: # $* "$*" are both decayed
574	vsub_state.join_array = True
575
576	elif op_id == Id.VSub_Hyphen:
577	val = value.Str(_GetDollarHyphen(self.exec_opts))
578
579	else:
580	val = self.mem.GetSpecialVar(op_id)
581
582	return val
583
584	def _ApplyTestOp(
585	self,
586	val, # type: value_t
587	op, # type: suffix_op.Unary
588	quoted, # type: bool
589	part_vals, # type: Optional[List[part_value_t]]
590	vtest_place, # type: VTestPlace
591	blame_token, # type: Token
592	):
593	# type: (...) -> bool
594	"""
595	Returns:
596	Whether part_vals was mutated
597
598	${a:-} returns part_value[]
599	${a:+} returns part_value[]
600	${a:?error} returns error word?
601	${a:=} returns part_value[] but also needs self.mem for side effects.
602
603	So I guess it should return part_value[], and then a flag for raising an
604	error, and then a flag for assigning it?
605	The original BracedVarSub will have the name.
606
607	Example of needing multiple part_value[]
608
609	echo X-${a:-'def'"ault"}-X
610
611	We return two part values from the BracedVarSub. Also consider:
612
613	echo ${a:-x"$@"x}
614	"""
615	eval_flags = IS_SUBST
616	if quoted:
617	eval_flags \|= QUOTED
618
619	tok = op.op
620	# NOTE: Splicing part_values is necessary because of code like
621	# ${undef:-'a b' c 'd # e'}. Each part_value can have a different
622	# do_glob/do_elide setting.
623	UP_val = val
624	with tagswitch(val) as case:
625	if case(value_e.Undef):
626	is_falsey = True
627
628	elif case(value_e.Str):
629	val = cast(value.Str, UP_val)
630	if tok.id in (Id.VTest_ColonHyphen, Id.VTest_ColonEquals,
631	Id.VTest_ColonQMark, Id.VTest_ColonPlus):
632	is_falsey = len(val.s) == 0
633	else:
634	is_falsey = False
635
636	elif case(value_e.BashArray):
637	val = cast(value.BashArray, UP_val)
638	# TODO: allow undefined
639	is_falsey = len(val.strs) == 0
640
641	elif case(value_e.BashAssoc):
642	val = cast(value.BashAssoc, UP_val)
643	is_falsey = len(val.d) == 0
644
645	else:
646	# value.Eggex, etc. are all false
647	is_falsey = False
648
649	if tok.id in (Id.VTest_ColonHyphen, Id.VTest_Hyphen):
650	if is_falsey:
651	self._EvalRhsWordToParts(op.arg_word, part_vals, eval_flags)
652	return True
653	else:
654	return False
655
656	# Inverse of the above.
657	elif tok.id in (Id.VTest_ColonPlus, Id.VTest_Plus):
658	if is_falsey:
659	return False
660	else:
661	self._EvalRhsWordToParts(op.arg_word, part_vals, eval_flags)
662	return True
663
664	# Splice and assign
665	elif tok.id in (Id.VTest_ColonEquals, Id.VTest_Equals):
666	if is_falsey:
667	# Collect new part vals.
668	assign_part_vals = [] # type: List[part_value_t]
669	self._EvalRhsWordToParts(op.arg_word, assign_part_vals,
670	eval_flags)
671	# Append them to out param AND return them.
672	part_vals.extend(assign_part_vals)
673
674	if vtest_place.name is None:
675	# TODO: error context
676	e_die("Can't assign to special variable")
677	else:
678	# NOTE: This decays arrays too! 'shopt -s strict_array' could
679	# avoid it.
680	rhs_str = _DecayPartValuesToString(
681	assign_part_vals, self.splitter.GetJoinChar())
682	if vtest_place.index is None: # using None when no index
683	lval = location.LName(
684	vtest_place.name) # type: sh_lvalue_t
685	else:
686	var_name = vtest_place.name
687	var_index = vtest_place.index
688	UP_var_index = var_index
689
690	with tagswitch(var_index) as case:
691	if case(a_index_e.Int):
692	var_index = cast(a_index.Int, UP_var_index)
693	lval = sh_lvalue.Indexed(
694	var_name, var_index.i, loc.Missing)
695	elif case(a_index_e.Str):
696	var_index = cast(a_index.Str, UP_var_index)
697	lval = sh_lvalue.Keyed(var_name, var_index.s,
698	loc.Missing)
699	else:
700	raise AssertionError()
701
702	state.OshLanguageSetValue(self.mem, lval,
703	value.Str(rhs_str))
704	return True
705
706	else:
707	return False
708
709	elif tok.id in (Id.VTest_ColonQMark, Id.VTest_QMark):
710	if is_falsey:
711	# The arg is the error message
712	error_part_vals = [] # type: List[part_value_t]
713	self._EvalRhsWordToParts(op.arg_word, error_part_vals,
714	eval_flags)
715	error_str = _DecayPartValuesToString(
716	error_part_vals, self.splitter.GetJoinChar())
717
718	#
719	# Display fancy/helpful error
720	#
721	if vtest_place.name is None:
722	var_name = '???'
723	else:
724	var_name = vtest_place.name
725
726	if 0:
727	# This hint is nice, but looks too noisy for now
728	op_str = lexer.LazyStr(tok)
729	if tok.id == Id.VTest_ColonQMark:
730	why = 'empty or unset'
731	else:
732	why = 'unset'
733
734	self.errfmt.Print_(
735	"Hint: operator %s means a variable can't be %s" %
736	(op_str, why), tok)
737
738	if val.tag() == value_e.Undef:
739	actual = 'unset'
740	else:
741	actual = 'empty'
742
743	if len(error_str):
744	suffix = ': %r' % error_str
745	else:
746	suffix = ''
747	e_die("Var %s is %s%s" % (var_name, actual, suffix),
748	blame_token)
749
750	else:
751	return False
752
753	else:
754	raise AssertionError(tok.id)
755
756	def _Length(self, val, token):
757	# type: (value_t, Token) -> int
758	"""Returns the length of the value, for ${#var}"""
759	UP_val = val
760	with tagswitch(val) as case:
761	if case(value_e.Str):
762	val = cast(value.Str, UP_val)
763	# NOTE: Whether bash counts bytes or chars is affected by LANG
764	# environment variables.
765	# Should we respect that, or another way to select? set -o
766	# count-bytes?
767
768	# https://stackoverflow.com/questions/17368067/length-of-string-in-bash
769	try:
770	length = string_ops.CountUtf8Chars(val.s)
771	except error.Strict as e:
772	# Add this here so we don't have to add it so far down the stack.
773	# TODO: It's better to show BOTH this CODE an the actual DATA
774	# somehow.
775	e.location = token
776
777	if self.exec_opts.strict_word_eval():
778	raise
779	else:
780	# NOTE: Doesn't make the command exit with 1; it just returns a
781	# length of -1.
782	self.errfmt.PrettyPrintError(e, prefix='warning: ')
783	return -1
784
785	elif case(value_e.BashArray):
786	val = cast(value.BashArray, UP_val)
787	# There can be empty placeholder values in the array.
788	length = 0
789	for s in val.strs:
790	if s is not None:
791	length += 1
792
793	elif case(value_e.BashAssoc):
794	val = cast(value.BashAssoc, UP_val)
795	length = len(val.d)
796
797	else:
798	raise error.TypeErr(
799	val, "Length op expected Str, BashArray, BashAssoc", token)
800
801	return length
802
803	def _Keys(self, val, token):
804	# type: (value_t, Token) -> value_t
805	"""Return keys of a container, for ${!array[@]}"""
806
807	UP_val = val
808	with tagswitch(val) as case:
809	if case(value_e.BashArray):
810	val = cast(value.BashArray, UP_val)
811	# translation issue: tuple indices not supported in list comprehensions
812	#indices = [str(i) for i, s in enumerate(val.strs) if s is not None]
813	indices = [] # type: List[str]
814	for i, s in enumerate(val.strs):
815	if s is not None:
816	indices.append(str(i))
817	return value.BashArray(indices)
818
819	elif case(value_e.BashAssoc):
820	val = cast(value.BashAssoc, UP_val)
821	assert val.d is not None # for MyPy, so it's not Optional[]
822
823	# BUG: Keys aren't ordered according to insertion!
824	return value.BashArray(val.d.keys())
825
826	else:
827	raise error.TypeErr(val, 'Keys op expected Str', token)
828
829	def _EvalVarRef(self, val, blame_tok, quoted, vsub_state, vtest_place):
830	# type: (value_t, Token, bool, VarSubState, VTestPlace) -> value_t
831	"""Handles indirect expansion like ${!var} and ${!a[0]}.
832
833	Args:
834	blame_tok: 'foo' for ${!foo}
835	"""
836	UP_val = val
837	with tagswitch(val) as case:
838	if case(value_e.Undef):
839	return value.Undef # ${!undef} is just weird bash behavior
840
841	elif case(value_e.Str):
842	val = cast(value.Str, UP_val)
843	bvs_part = self.unsafe_arith.ParseVarRef(val.s, blame_tok)
844	return self._VarRefValue(bvs_part, quoted, vsub_state,
845	vtest_place)
846
847	elif case(value_e.BashArray): # caught earlier but OK
848	e_die('Indirect expansion of array')
849
850	elif case(value_e.BashAssoc): # caught earlier but OK
851	e_die('Indirect expansion of assoc array')
852
853	else:
854	raise error.TypeErr(val, 'Var Ref op expected Str', blame_tok)
855
856	def _ApplyUnarySuffixOp(self, val, op):
857	# type: (value_t, suffix_op.Unary) -> value_t
858	assert val.tag() != value_e.Undef
859
860	op_kind = consts.GetKind(op.op.id)
861
862	if op_kind == Kind.VOp1:
863	# NOTE: glob syntax is supported in ^ ^^ , ,, ! As well as % %% # ##.
864	# Detect has_extglob so that DoUnarySuffixOp doesn't use the fast
865	# shortcut for constant strings.
866	arg_val, has_extglob = self.EvalWordToPattern(op.arg_word)
867	assert arg_val.tag() == value_e.Str
868
869	UP_val = val
870	with tagswitch(val) as case:
871	if case(value_e.Str):
872	val = cast(value.Str, UP_val)
873	s = string_ops.DoUnarySuffixOp(val.s, op.op, arg_val.s,
874	has_extglob)
875	#log('%r %r -> %r', val.s, arg_val.s, s)
876	new_val = value.Str(s) # type: value_t
877
878	elif case(value_e.BashArray):
879	val = cast(value.BashArray, UP_val)
880	# ${a[@]#prefix} is VECTORIZED on arrays. YSH should have this too.
881	strs = [] # type: List[str]
882	for s in val.strs:
883	if s is not None:
884	strs.append(
885	string_ops.DoUnarySuffixOp(
886	s, op.op, arg_val.s, has_extglob))
887	new_val = value.BashArray(strs)
888
889	elif case(value_e.BashAssoc):
890	val = cast(value.BashAssoc, UP_val)
891	strs = []
892	for s in val.d.values():
893	strs.append(
894	string_ops.DoUnarySuffixOp(s, op.op, arg_val.s,
895	has_extglob))
896	new_val = value.BashArray(strs)
897
898	else:
899	raise error.TypeErr(
900	val, 'Unary op expected Str, BashArray, BashAssoc',
901	op.op)
902
903	else:
904	raise AssertionError(Kind_str(op_kind))
905
906	return new_val
907
908	def _PatSub(self, val, op):
909	# type: (value_t, suffix_op.PatSub) -> value_t
910
911	pat_val, has_extglob = self.EvalWordToPattern(op.pat)
912	# Extended globs aren't supported because we only translate * ? etc. to
913	# ERE. I don't think there's a straightforward translation from !(*.py) to
914	# ERE! You would need an engine that supports negation? (Derivatives?)
915	if has_extglob:
916	e_die('extended globs not supported in ${x//GLOB/}', op.pat)
917
918	if op.replace:
919	replace_val = self.EvalRhsWord(op.replace)
920	# Can't have an array, so must be a string
921	assert replace_val.tag() == value_e.Str, replace_val
922	replace_str = cast(value.Str, replace_val).s
923	else:
924	replace_str = ''
925
926	# note: doesn't support self.exec_opts.extglob()!
927	regex, warnings = glob_.GlobToERE(pat_val.s)
928	if len(warnings):
929	# TODO:
930	# - Add 'shopt -s strict_glob' mode and expose warnings.
931	# "Glob is not in CANONICAL FORM".
932	# - Propagate location info back to the 'op.pat' word.
933	pass
934	#log('regex %r', regex)
935	replacer = string_ops.GlobReplacer(regex, replace_str, op.slash_tok)
936
937	with tagswitch(val) as case2:
938	if case2(value_e.Str):
939	str_val = cast(value.Str, val)
940	s = replacer.Replace(str_val.s, op)
941	val = value.Str(s)
942
943	elif case2(value_e.BashArray):
944	array_val = cast(value.BashArray, val)
945	strs = [] # type: List[str]
946	for s in array_val.strs:
947	if s is not None:
948	strs.append(replacer.Replace(s, op))
949	val = value.BashArray(strs)
950
951	elif case2(value_e.BashAssoc):
952	assoc_val = cast(value.BashAssoc, val)
953	strs = []
954	for s in assoc_val.d.values():
955	strs.append(replacer.Replace(s, op))
956	val = value.BashArray(strs)
957
958	else:
959	raise error.TypeErr(
960	val, 'Pat Sub op expected Str, BashArray, BashAssoc',
961	op.slash_tok)
962
963	return val
964
965	def _Slice(self, val, op, var_name, part):
966	# type: (value_t, suffix_op.Slice, Optional[str], BracedVarSub) -> value_t
967
968	begin = self.arith_ev.EvalToInt(op.begin)
969
970	# Note: bash allows lengths to be negative (with odd semantics), but
971	# we don't allow that right now.
972	has_length = False
973	length = -1
974	if op.length:
975	has_length = True
976	length = self.arith_ev.EvalToInt(op.length)
977
978	try:
979	arg0_val = None # type: value.Str
980	if var_name is None: # $* or $@
981	arg0_val = self.mem.GetArg0()
982	val = _PerformSlice(val, begin, length, has_length, part, arg0_val)
983	except error.Strict as e:
984	if self.exec_opts.strict_word_eval():
985	raise
986	else:
987	self.errfmt.PrettyPrintError(e, prefix='warning: ')
988	with tagswitch(val) as case2:
989	if case2(value_e.Str):
990	val = value.Str('')
991	elif case2(value_e.BashArray):
992	val = value.BashArray([])
993	else:
994	raise NotImplementedError()
995	return val
996
997	def _Nullary(self, val, op, var_name):
998	# type: (value_t, Token, Optional[str]) -> Tuple[value.Str, bool]
999
1000	UP_val = val
1001	quoted2 = False
1002	op_id = op.id
1003	if op_id == Id.VOp0_P:
1004	with tagswitch(val) as case:
1005	if case(value_e.Str):
1006	str_val = cast(value.Str, UP_val)
1007	prompt = self.prompt_ev.EvalPrompt(str_val)
1008	# readline gets rid of these, so we should too.
1009	p = prompt.replace('\x01', '').replace('\x02', '')
1010	result = value.Str(p)
1011	else:
1012	e_die("Can't use @P on %s" % ui.ValType(val), op)
1013
1014	elif op_id == Id.VOp0_Q:
1015	with tagswitch(val) as case:
1016	if case(value_e.Str):
1017	str_val = cast(value.Str, UP_val)
1018
1019	# TODO: use fastfunc.ShellEncode or
1020	# fastfunc.PosixShellEncode()
1021	result = value.Str(j8_lite.MaybeShellEncode(str_val.s))
1022	# oddly, 'echo ${x@Q}' is equivalent to 'echo "${x@Q}"' in bash
1023	quoted2 = True
1024	elif case(value_e.BashArray):
1025	array_val = cast(value.BashArray, UP_val)
1026
1027	# TODO: should use fastfunc.ShellEncode
1028	tmp = [j8_lite.MaybeShellEncode(s) for s in array_val.strs]
1029	result = value.Str(' '.join(tmp))
1030	else:
1031	e_die("Can't use @Q on %s" % ui.ValType(val), op)
1032
1033	elif op_id == Id.VOp0_a:
1034	# We're ONLY simluating -a and -A, not -r -x -n for now. See
1035	# spec/ble-idioms.test.sh.
1036	chars = [] # type: List[str]
1037	with tagswitch(val) as case:
1038	if case(value_e.BashArray):
1039	chars.append('a')
1040	elif case(value_e.BashAssoc):
1041	chars.append('A')
1042
1043	if var_name is not None: # e.g. ${?@a} is allowed
1044	cell = self.mem.GetCell(var_name)
1045	if cell:
1046	if cell.readonly:
1047	chars.append('r')
1048	if cell.exported:
1049	chars.append('x')
1050	if cell.nameref:
1051	chars.append('n')
1052
1053	result = value.Str(''.join(chars))
1054
1055	else:
1056	e_die('Var op %r not implemented' % lexer.TokenVal(op), op)
1057
1058	return result, quoted2
1059
1060	def _WholeArray(self, val, part, quoted, vsub_state):
1061	# type: (value_t, BracedVarSub, bool, VarSubState) -> value_t
1062	op_id = cast(bracket_op.WholeArray, part.bracket_op).op_id
1063
1064	if op_id == Id.Lit_At:
1065	vsub_state.join_array = not quoted # ${a[@]} decays but "${a[@]}" doesn't
1066	UP_val = val
1067	with tagswitch(val) as case2:
1068	if case2(value_e.Undef):
1069	if not vsub_state.has_test_op:
1070	val = self._EmptyBashArrayOrError(part.token)
1071	elif case2(value_e.Str):
1072	if self.exec_opts.strict_array():
1073	e_die("Can't index string with @", loc.WordPart(part))
1074	elif case2(value_e.BashArray):
1075	pass # no-op
1076
1077	elif op_id == Id.Arith_Star:
1078	vsub_state.join_array = True # both ${a[]} and "${a[]}" decay
1079	UP_val = val
1080	with tagswitch(val) as case2:
1081	if case2(value_e.Undef):
1082	if not vsub_state.has_test_op:
1083	val = self._EmptyBashArrayOrError(part.token)
1084	elif case2(value_e.Str):
1085	if self.exec_opts.strict_array():
1086	e_die("Can't index string with *", loc.WordPart(part))
1087	elif case2(value_e.BashArray):
1088	pass # no-op
1089
1090	else:
1091	raise AssertionError(op_id) # unknown
1092
1093	return val
1094
1095	def _ArrayIndex(self, val, part, vtest_place):
1096	# type: (value_t, BracedVarSub, VTestPlace) -> value_t
1097	"""Process a numeric array index like ${a[i+1]}"""
1098	anode = cast(bracket_op.ArrayIndex, part.bracket_op).expr
1099
1100	UP_val = val
1101	with tagswitch(val) as case2:
1102	if case2(value_e.Undef):
1103	pass # it will be checked later
1104
1105	elif case2(value_e.Str):
1106	# Bash treats any string as an array, so we can't add our own
1107	# behavior here without making valid OSH invalid bash.
1108	e_die("Can't index string %r with integer" % part.var_name,
1109	part.token)
1110
1111	elif case2(value_e.BashArray):
1112	array_val = cast(value.BashArray, UP_val)
1113	index = self.arith_ev.EvalToInt(anode)
1114	vtest_place.index = a_index.Int(index)
1115
1116	s = GetArrayItem(array_val.strs, index)
1117
1118	if s is None:
1119	val = value.Undef
1120	else:
1121	val = value.Str(s)
1122
1123	elif case2(value_e.BashAssoc):
1124	assoc_val = cast(value.BashAssoc, UP_val)
1125	# Location could also be attached to bracket_op? But
1126	# arith_expr.VarSub works OK too
1127	key = self.arith_ev.EvalWordToString(
1128	anode, blame_loc=location.TokenForArith(anode))
1129
1130	vtest_place.index = a_index.Str(key) # out param
1131	s = assoc_val.d.get(key)
1132
1133	if s is None:
1134	val = value.Undef
1135	else:
1136	val = value.Str(s)
1137
1138	else:
1139	raise error.TypeErr(val,
1140	'Index op expected BashArray, BashAssoc',
1141	loc.WordPart(part))
1142
1143	return val
1144
1145	def _EvalDoubleQuoted(self, parts, part_vals):
1146	# type: (List[word_part_t], List[part_value_t]) -> None
1147	"""Evaluate parts of a DoubleQuoted part.
1148
1149	Args:
1150	part_vals: output param to append to.
1151	"""
1152	# Example of returning array:
1153	# $ a=(1 2); b=(3); $ c=(4 5)
1154	# $ argv "${a[@]}${b[@]}${c[@]}"
1155	# ['1', '234', '5']
1156	#
1157	# Example of multiple parts
1158	# $ argv "${a[@]}${undef[@]:-${c[@]}}"
1159	# ['1', '24', '5']
1160
1161	# Special case for "". The parser outputs (DoubleQuoted []), instead
1162	# of (DoubleQuoted [Literal '']). This is better but it means we
1163	# have to check for it.
1164	if len(parts) == 0:
1165	v = Piece('', True, False)
1166	part_vals.append(v)
1167	return
1168
1169	for p in parts:
1170	self._EvalWordPart(p, part_vals, QUOTED)
1171
1172	def EvalDoubleQuotedToString(self, dq_part):
1173	# type: (DoubleQuoted) -> str
1174	"""For double quoted strings in YSH expressions.
1175
1176	Example: var x = "$foo-${foo}"
1177	"""
1178	part_vals = [] # type: List[part_value_t]
1179	self._EvalDoubleQuoted(dq_part.parts, part_vals)
1180	return self._ConcatPartVals(part_vals, dq_part.left)
1181
1182	def _DecayArray(self, val):
1183	# type: (value.BashArray) -> value.Str
1184	"""Decay $* to a string."""
1185	assert val.tag() == value_e.BashArray, val
1186	sep = self.splitter.GetJoinChar()
1187	tmp = [s for s in val.strs if s is not None]
1188	return value.Str(sep.join(tmp))
1189
1190	def _EmptyStrOrError(self, val, token):
1191	# type: (value_t, Token) -> value_t
1192	if val.tag() != value_e.Undef:
1193	return val
1194
1195	if not self.exec_opts.nounset():
1196	return value.Str('')
1197
1198	tok_str = lexer.TokenVal(token)
1199	name = tok_str[1:] if tok_str.startswith('$') else tok_str
1200	e_die('Undefined variable %r' % name, token)
1201
1202	def _EmptyBashArrayOrError(self, token):
1203	# type: (Token) -> value_t
1204	assert token is not None
1205	if self.exec_opts.nounset():
1206	e_die('Undefined array %r' % lexer.TokenVal(token), token)
1207	else:
1208	return value.BashArray([])
1209
1210	def _EvalBracketOp(self, val, part, quoted, vsub_state, vtest_place):
1211	# type: (value_t, BracedVarSub, bool, VarSubState, VTestPlace) -> value_t
1212
1213	if part.bracket_op:
1214	with tagswitch(part.bracket_op) as case:
1215	if case(bracket_op_e.WholeArray):
1216	val = self._WholeArray(val, part, quoted, vsub_state)
1217
1218	elif case(bracket_op_e.ArrayIndex):
1219	val = self._ArrayIndex(val, part, vtest_place)
1220
1221	else:
1222	raise AssertionError(part.bracket_op.tag())
1223
1224	else: # no bracket op
1225	var_name = vtest_place.name
1226	if (var_name is not None and
1227	val.tag() in (value_e.BashArray, value_e.BashAssoc) and
1228	not vsub_state.is_type_query):
1229	if ShouldArrayDecay(var_name, self.exec_opts,
1230	not (part.prefix_op or part.suffix_op)):
1231	# for ${BASH_SOURCE}, etc.
1232	val = DecayArray(val)
1233	else:
1234	e_die(
1235	"Array %r can't be referred to as a scalar (without @ or *)"
1236	% var_name, loc.WordPart(part))
1237
1238	return val
1239
1240	def _VarRefValue(self, part, quoted, vsub_state, vtest_place):
1241	# type: (BracedVarSub, bool, VarSubState, VTestPlace) -> value_t
1242	"""Duplicates some logic from _EvalBracedVarSub, but returns a
1243	value_t."""
1244
1245	# 1. Evaluate from (var_name, var_num, token Id) -> value
1246	if part.token.id == Id.VSub_Name:
1247	vtest_place.name = part.var_name
1248	val = self.mem.GetValue(part.var_name)
1249
1250	elif part.token.id == Id.VSub_Number:
1251	var_num = int(part.var_name)
1252	val = self._EvalVarNum(var_num)
1253
1254	else:
1255	# $* decays
1256	val = self._EvalSpecialVar(part.token.id, quoted, vsub_state)
1257
1258	# We don't need var_index because it's only for L-Values of test ops?
1259	if self.exec_opts.eval_unsafe_arith():
1260	val = self._EvalBracketOp(val, part, quoted, vsub_state,
1261	vtest_place)
1262	else:
1263	with state.ctx_Option(self.mutable_opts,
1264	[option_i._allow_command_sub], False):
1265	val = self._EvalBracketOp(val, part, quoted, vsub_state,
1266	vtest_place)
1267
1268	return val
1269
1270	def _EvalBracedVarSub(self, part, part_vals, quoted):
1271	# type: (BracedVarSub, List[part_value_t], bool) -> None
1272	"""
1273	Args:
1274	part_vals: output param to append to.
1275	"""
1276	# We have different operators that interact in a non-obvious order.
1277	#
1278	# 1. bracket_op: value -> value, with side effect on vsub_state
1279	#
1280	# 2. prefix_op
1281	# a. length ${#x}: value -> value
1282	# b. var ref ${!ref}: can expand to an array
1283	#
1284	# 3. suffix_op:
1285	# a. no operator: you have a value
1286	# b. Test: value -> part_value[]
1287	# c. Other Suffix: value -> value
1288	#
1289	# 4. Process vsub_state.join_array here before returning.
1290	#
1291	# These cases are hard to distinguish:
1292	# - ${!prefix@} prefix query
1293	# - ${!array[@]} keys
1294	# - ${!ref} named reference
1295	# - ${!ref[0]} named reference
1296	#
1297	# I think we need several stages:
1298	#
1299	# 1. value: name, number, special, prefix query
1300	# 2. bracket_op
1301	# 3. prefix length -- this is TERMINAL
1302	# 4. indirection? Only for some of the ! cases
1303	# 5. string transformation suffix ops like ##
1304	# 6. test op
1305	# 7. vsub_state.join_array
1306
1307	# vsub_state.join_array is for joining "${a[*]}" and unquoted ${a[@]} AFTER
1308	# suffix ops are applied. If we take the length with a prefix op, the
1309	# distinction is ignored.
1310
1311	var_name = None # type: Optional[str] # used throughout the function
1312	vtest_place = VTestPlace(var_name, None) # For ${foo=default}
1313	vsub_state = VarSubState.CreateNull() # for $, ${a[]}, etc.
1314
1315	# 1. Evaluate from (var_name, var_num, token Id) -> value
1316	if part.token.id == Id.VSub_Name:
1317	# Handle ${!prefix@} first, since that looks at names and not values
1318	# Do NOT handle ${!A[@]@a} here!
1319	if (part.prefix_op is not None and part.bracket_op is None and
1320	part.suffix_op is not None and
1321	part.suffix_op.tag() == suffix_op_e.Nullary):
1322	nullary_op = cast(Token, part.suffix_op)
1323	# ${!x@} but not ${!x@P}
1324	if consts.GetKind(nullary_op.id) == Kind.VOp3:
1325	names = self.mem.VarNamesStartingWith(part.var_name)
1326	names.sort()
1327
1328	if quoted and nullary_op.id == Id.VOp3_At:
1329	part_vals.append(part_value.Array(names))
1330	else:
1331	sep = self.splitter.GetJoinChar()
1332	part_vals.append(Piece(sep.join(names), quoted, True))
1333	return # EARLY RETURN
1334
1335	var_name = part.var_name
1336	vtest_place.name = var_name # for _ApplyTestOp
1337
1338	val = self.mem.GetValue(var_name)
1339
1340	elif part.token.id == Id.VSub_Number:
1341	var_num = int(part.var_name)
1342	val = self._EvalVarNum(var_num)
1343	else:
1344	# $* decays
1345	val = self._EvalSpecialVar(part.token.id, quoted, vsub_state)
1346
1347	suffix_op_ = part.suffix_op
1348	if suffix_op_:
1349	UP_op = suffix_op_
1350	with tagswitch(suffix_op_) as case:
1351	if case(suffix_op_e.Nullary):
1352	suffix_op_ = cast(Token, UP_op)
1353
1354	# Type query ${array@a} is a STRING, not an array
1355	# NOTE: ${array@Q} is ${array[0]@Q} in bash, which is different than
1356	# ${array[@]@Q}
1357	if suffix_op_.id == Id.VOp0_a:
1358	vsub_state.is_type_query = True
1359
1360	elif case(suffix_op_e.Unary):
1361	suffix_op_ = cast(suffix_op.Unary, UP_op)
1362
1363	# Do the _EmptyStrOrError/_EmptyBashArrayOrError up front, EXCEPT in
1364	# the case of Kind.VTest
1365	if consts.GetKind(suffix_op_.op.id) == Kind.VTest:
1366	vsub_state.has_test_op = True
1367
1368	# 2. Bracket Op
1369	val = self._EvalBracketOp(val, part, quoted, vsub_state, vtest_place)
1370
1371	if part.prefix_op:
1372	if part.prefix_op.id == Id.VSub_Pound: # ${#var} for length
1373	if not vsub_state.has_test_op: # undef -> '' BEFORE length
1374	val = self._EmptyStrOrError(val, part.token)
1375
1376	n = self._Length(val, part.token)
1377	part_vals.append(Piece(str(n), quoted, False))
1378	return # EARLY EXIT: nothing else can come after length
1379
1380	elif part.prefix_op.id == Id.VSub_Bang:
1381	if (part.bracket_op and
1382	part.bracket_op.tag() == bracket_op_e.WholeArray):
1383	if vsub_state.has_test_op:
1384	# ${!a[@]-'default'} is a non-fatal runtime error in bash. Here
1385	# it's fatal.
1386	op_tok = cast(suffix_op.Unary, UP_op).op
1387	e_die('Test operation not allowed with ${!array[@]}',
1388	op_tok)
1389
1390	# ${!array[@]} to get indices/keys
1391	val = self._Keys(val, part.token)
1392	# already set vsub_State.join_array ABOVE
1393	else:
1394	# Process ${!ref}. SURPRISE: ${!a[0]} is an indirect expansion unlike
1395	# ${!a[@]} !
1396	# ${!ref} can expand into an array if ref='array[@]'
1397
1398	# Clear it now that we have a var ref
1399	vtest_place.name = None
1400	vtest_place.index = None
1401
1402	val = self._EvalVarRef(val, part.token, quoted, vsub_state,
1403	vtest_place)
1404
1405	if not vsub_state.has_test_op: # undef -> '' AFTER indirection
1406	val = self._EmptyStrOrError(val, part.token)
1407
1408	else:
1409	raise AssertionError(part.prefix_op)
1410
1411	else:
1412	if not vsub_state.has_test_op: # undef -> '' if no prefix op
1413	val = self._EmptyStrOrError(val, part.token)
1414
1415	quoted2 = False # another bit for @Q
1416	if suffix_op_:
1417	op = suffix_op_ # could get rid of this alias
1418
1419	with tagswitch(suffix_op_) as case:
1420	if case(suffix_op_e.Nullary):
1421	op = cast(Token, UP_op)
1422	val, quoted2 = self._Nullary(val, op, var_name)
1423
1424	elif case(suffix_op_e.Unary):
1425	op = cast(suffix_op.Unary, UP_op)
1426	if consts.GetKind(op.op.id) == Kind.VTest:
1427	if self._ApplyTestOp(val, op, quoted, part_vals,
1428	vtest_place, part.token):
1429	# e.g. to evaluate ${undef:-'default'}, we already appended
1430	# what we need
1431	return
1432
1433	else:
1434	# Other suffix: value -> value
1435	val = self._ApplyUnarySuffixOp(val, op)
1436
1437	elif case(suffix_op_e.PatSub): # PatSub, vectorized
1438	op = cast(suffix_op.PatSub, UP_op)
1439	val = self._PatSub(val, op)
1440
1441	elif case(suffix_op_e.Slice):
1442	op = cast(suffix_op.Slice, UP_op)
1443	val = self._Slice(val, op, var_name, part)
1444
1445	elif case(suffix_op_e.Static):
1446	op = cast(suffix_op.Static, UP_op)
1447	e_die('Not implemented', op.tok)
1448
1449	else:
1450	raise AssertionError()
1451
1452	# After applying suffixes, process join_array here.
1453	UP_val = val
1454	if val.tag() == value_e.BashArray:
1455	array_val = cast(value.BashArray, UP_val)
1456	if vsub_state.join_array:
1457	val = self._DecayArray(array_val)
1458	else:
1459	val = array_val
1460
1461	# For example, ${a} evaluates to value.Str(), but we want a
1462	# Piece().
1463	part_val = _ValueToPartValue(val, quoted or quoted2, part)
1464	part_vals.append(part_val)
1465
1466	def _ConcatPartVals(self, part_vals, location):
1467	# type: (List[part_value_t], loc_t) -> str
1468
1469	strs = [] # type: List[str]
1470	for part_val in part_vals:
1471	UP_part_val = part_val
1472	with tagswitch(part_val) as case:
1473	if case(part_value_e.String):
1474	part_val = cast(Piece, UP_part_val)
1475	s = part_val.s
1476
1477	elif case(part_value_e.Array):
1478	part_val = cast(part_value.Array, UP_part_val)
1479	if self.exec_opts.strict_array():
1480	# Examples: echo f > "$@"; local foo="$@"
1481	e_die("Illegal array word part (strict_array)",
1482	location)
1483	else:
1484	# It appears to not respect IFS
1485	# TODO: eliminate double join()?
1486	tmp = [s for s in part_val.strs if s is not None]
1487	s = ' '.join(tmp)
1488
1489	else:
1490	raise AssertionError()
1491
1492	strs.append(s)
1493
1494	return ''.join(strs)
1495
1496	def EvalBracedVarSubToString(self, part):
1497	# type: (BracedVarSub) -> str
1498	"""For double quoted strings in YSH expressions.
1499
1500	Example: var x = "$foo-${foo}"
1501	"""
1502	part_vals = [] # type: List[part_value_t]
1503	self._EvalBracedVarSub(part, part_vals, False)
1504	# blame ${ location
1505	return self._ConcatPartVals(part_vals, part.left)
1506
1507	def _EvalSimpleVarSub(self, part, part_vals, quoted):
1508	# type: (SimpleVarSub, List[part_value_t], bool) -> None
1509
1510	token = part.tok
1511
1512	vsub_state = VarSubState.CreateNull()
1513
1514	# 1. Evaluate from (var_name, var_num, Token) -> defined, value
1515	if token.id == Id.VSub_DollarName:
1516	var_name = lexer.LazyStr(token)
1517	# TODO: Special case for LINENO
1518	val = self.mem.GetValue(var_name)
1519	if val.tag() in (value_e.BashArray, value_e.BashAssoc):
1520	if ShouldArrayDecay(var_name, self.exec_opts):
1521	# for $BASH_SOURCE, etc.
1522	val = DecayArray(val)
1523	else:
1524	e_die(
1525	"Array %r can't be referred to as a scalar (without @ or *)"
1526	% var_name, token)
1527
1528	elif token.id == Id.VSub_Number:
1529	var_num = int(lexer.LazyStr(token))
1530	val = self._EvalVarNum(var_num)
1531
1532	else:
1533	val = self._EvalSpecialVar(token.id, quoted, vsub_state)
1534
1535	#log('SIMPLE %s', part)
1536	val = self._EmptyStrOrError(val, token)
1537	UP_val = val
1538	if val.tag() == value_e.BashArray:
1539	array_val = cast(value.BashArray, UP_val)
1540	if vsub_state.join_array:
1541	val = self._DecayArray(array_val)
1542	else:
1543	val = array_val
1544
1545	v = _ValueToPartValue(val, quoted, part)
1546	part_vals.append(v)
1547
1548	def EvalSimpleVarSubToString(self, node):
1549	# type: (SimpleVarSub) -> str
1550	"""For double quoted strings in YSH expressions.
1551
1552	Example: var x = "$foo-${foo}"
1553	"""
1554	part_vals = [] # type: List[part_value_t]
1555	self._EvalSimpleVarSub(node, part_vals, False)
1556	return self._ConcatPartVals(part_vals, node.tok)
1557
1558	def _EvalExtGlob(self, part, part_vals):
1559	# type: (word_part.ExtGlob, List[part_value_t]) -> None
1560	"""Evaluate @($x\|'foo'\|$(hostname)) and flatten it."""
1561	op = part.op
1562	if op.id == Id.ExtGlob_Comma:
1563	op_str = '@('
1564	else:
1565	op_str = lexer.LazyStr(op)
1566	# Do NOT split these.
1567	part_vals.append(Piece(op_str, False, False))
1568
1569	for i, w in enumerate(part.arms):
1570	if i != 0:
1571	part_vals.append(Piece('\|', False, False)) # separator
1572	# FLATTEN the tree of extglob "arms".
1573	self._EvalWordToParts(w, part_vals, EXTGLOB_NESTED)
1574	part_vals.append(Piece(')', False, False)) # closing )
1575
1576	def _TranslateExtGlob(self, part_vals, w, glob_parts, fnmatch_parts):
1577	# type: (List[part_value_t], CompoundWord, List[str], List[str]) -> None
1578	"""Translate a flattened WORD with an ExtGlob part to string patterns.
1579
1580	We need both glob and fnmatch patterns. _EvalExtGlob does the
1581	flattening.
1582	"""
1583	for i, part_val in enumerate(part_vals):
1584	UP_part_val = part_val
1585	with tagswitch(part_val) as case:
1586	if case(part_value_e.String):
1587	part_val = cast(Piece, UP_part_val)
1588	if part_val.quoted and not self.exec_opts.noglob():
1589	s = glob_.GlobEscape(part_val.s)
1590	else:
1591	# e.g. the @( and \| in @(foo\|bar) aren't quoted
1592	s = part_val.s
1593	glob_parts.append(s)
1594	fnmatch_parts.append(s) # from _EvalExtGlob()
1595
1596	elif case(part_value_e.Array):
1597	# Disallow array
1598	e_die(
1599	"Extended globs and arrays can't appear in the same word",
1600	w)
1601
1602	elif case(part_value_e.ExtGlob):
1603	part_val = cast(part_value.ExtGlob, UP_part_val)
1604	# keep appending fnmatch_parts, but repplace glob_parts with '*'
1605	self._TranslateExtGlob(part_val.part_vals, w, [],
1606	fnmatch_parts)
1607	glob_parts.append('*')
1608
1609	else:
1610	raise AssertionError()
1611
1612	def _EvalWordPart(self, part, part_vals, flags):
1613	# type: (word_part_t, List[part_value_t], int) -> None
1614	"""Evaluate a word part, appending to part_vals
1615
1616	Called by _EvalWordToParts, EvalWordToString, and _EvalDoubleQuoted.
1617	"""
1618	quoted = bool(flags & QUOTED)
1619	is_subst = bool(flags & IS_SUBST)
1620
1621	UP_part = part
1622	with tagswitch(part) as case:
1623	if case(word_part_e.ShArrayLiteral):
1624	part = cast(ShArrayLiteral, UP_part)
1625	e_die("Unexpected array literal", loc.WordPart(part))
1626	elif case(word_part_e.BashAssocLiteral):
1627	part = cast(word_part.BashAssocLiteral, UP_part)
1628	e_die("Unexpected associative array literal",
1629	loc.WordPart(part))
1630
1631	elif case(word_part_e.Literal):
1632	part = cast(Token, UP_part)
1633	# Split if it's in a substitution.
1634	# That is: echo is not split, but ${foo:-echo} is split
1635	v = Piece(lexer.LazyStr(part), quoted, is_subst)
1636	part_vals.append(v)
1637
1638	elif case(word_part_e.EscapedLiteral):
1639	part = cast(word_part.EscapedLiteral, UP_part)
1640	v = Piece(part.ch, True, False)
1641	part_vals.append(v)
1642
1643	elif case(word_part_e.SingleQuoted):
1644	part = cast(SingleQuoted, UP_part)
1645	v = Piece(part.sval, True, False)
1646	part_vals.append(v)
1647
1648	elif case(word_part_e.DoubleQuoted):
1649	part = cast(DoubleQuoted, UP_part)
1650	self._EvalDoubleQuoted(part.parts, part_vals)
1651
1652	elif case(word_part_e.CommandSub):
1653	part = cast(CommandSub, UP_part)
1654	id_ = part.left_token.id
1655	if id_ in (Id.Left_DollarParen, Id.Left_AtParen,
1656	Id.Left_Backtick):
1657	sv = self._EvalCommandSub(part,
1658	quoted) # type: part_value_t
1659
1660	elif id_ in (Id.Left_ProcSubIn, Id.Left_ProcSubOut):
1661	sv = self._EvalProcessSub(part)
1662
1663	else:
1664	raise AssertionError(id_)
1665
1666	part_vals.append(sv)
1667
1668	elif case(word_part_e.SimpleVarSub):
1669	part = cast(SimpleVarSub, UP_part)
1670	self._EvalSimpleVarSub(part, part_vals, quoted)
1671
1672	elif case(word_part_e.BracedVarSub):
1673	part = cast(BracedVarSub, UP_part)
1674	self._EvalBracedVarSub(part, part_vals, quoted)
1675
1676	elif case(word_part_e.TildeSub):
1677	part = cast(word_part.TildeSub, UP_part)
1678	# We never parse a quoted string into a TildeSub.
1679	assert not quoted
1680	s = self.tilde_ev.Eval(part)
1681	v = Piece(s, True, False) # NOT split even when unquoted!
1682	part_vals.append(v)
1683
1684	elif case(word_part_e.ArithSub):
1685	part = cast(word_part.ArithSub, UP_part)
1686	num = self.arith_ev.EvalToBigInt(part.anode)
1687	v = Piece(mops.ToStr(num), quoted, not quoted)
1688	part_vals.append(v)
1689
1690	elif case(word_part_e.ExtGlob):
1691	part = cast(word_part.ExtGlob, UP_part)
1692	#if not self.exec_opts.extglob():
1693	# die() # disallow at runtime? Don't just decay
1694
1695	# Create a node to hold the flattened tree. The caller decides whether
1696	# to pass it to fnmatch() or replace it with '*' and pass it to glob().
1697	part_vals2 = [] # type: List[part_value_t]
1698	self._EvalExtGlob(part, part_vals2) # flattens tree
1699	part_vals.append(part_value.ExtGlob(part_vals2))
1700
1701	elif case(word_part_e.BashRegexGroup):
1702	part = cast(word_part.BashRegexGroup, UP_part)
1703
1704	part_vals.append(Piece('(', False, False)) # not quoted
1705	if part.child:
1706	self._EvalWordToParts(part.child, part_vals, 0)
1707	part_vals.append(Piece(')', False, False))
1708
1709	elif case(word_part_e.Splice):
1710	part = cast(word_part.Splice, UP_part)
1711	val = self.mem.GetValue(part.var_name)
1712
1713	strs = self.expr_ev.SpliceValue(val, part)
1714	part_vals.append(part_value.Array(strs))
1715
1716	elif case(word_part_e.ExprSub):
1717	part = cast(word_part.ExprSub, UP_part)
1718	part_val = self.expr_ev.EvalExprSub(part)
1719	part_vals.append(part_val)
1720
1721	elif case(word_part_e.ZshVarSub):
1722	part = cast(word_part.ZshVarSub, UP_part)
1723	e_die("ZSH var subs are parsed, but can't be evaluated",
1724	part.left)
1725
1726	else:
1727	raise AssertionError(part.tag())
1728
1729	def _EvalRhsWordToParts(self, w, part_vals, eval_flags=0):
1730	# type: (rhs_word_t, List[part_value_t], int) -> None
1731	quoted = bool(eval_flags & QUOTED)
1732
1733	UP_w = w
1734	with tagswitch(w) as case:
1735	if case(rhs_word_e.Empty):
1736	part_vals.append(Piece('', quoted, not quoted))
1737
1738	elif case(rhs_word_e.Compound):
1739	w = cast(CompoundWord, UP_w)
1740	self._EvalWordToParts(w, part_vals, eval_flags=eval_flags)
1741
1742	else:
1743	raise AssertionError()
1744
1745	def _EvalWordToParts(self, w, part_vals, eval_flags=0):
1746	# type: (CompoundWord, List[part_value_t], int) -> None
1747	"""Helper for EvalRhsWord, EvalWordSequence, etc.
1748
1749	Returns:
1750	Appends to part_vals. Note that this is a TREE.
1751	"""
1752	# Does the word have an extended glob? This is a special case because
1753	# of the way we use glob() and then fnmatch(..., FNM_EXTMATCH) to
1754	# implement extended globs. It's hard to carry that extra information
1755	# all the way past the word splitting stage.
1756
1757	# OSH semantic limitations: If a word has an extended glob part, then
1758	# 1. It can't have an array
1759	# 2. Word splitting of unquoted words isn't respected
1760
1761	word_part_vals = [] # type: List[part_value_t]
1762	has_extglob = False
1763	for p in w.parts:
1764	if p.tag() == word_part_e.ExtGlob:
1765	has_extglob = True
1766	self._EvalWordPart(p, word_part_vals, eval_flags)
1767
1768	# Caller REQUESTED extglob evaluation, AND we parsed word_part.ExtGlob()
1769	if has_extglob:
1770	if bool(eval_flags & EXTGLOB_FILES):
1771	# Treat the WHOLE word as a pattern. We need to TWO VARIANTS of the
1772	# word because of the way we use libc:
1773	# 1. With '*' for extglob parts
1774	# 2. With _EvalExtGlob() for extglob parts
1775
1776	glob_parts = [] # type: List[str]
1777	fnmatch_parts = [] # type: List[str]
1778	self._TranslateExtGlob(word_part_vals, w, glob_parts,
1779	fnmatch_parts)
1780
1781	#log('word_part_vals %s', word_part_vals)
1782	glob_pat = ''.join(glob_parts)
1783	fnmatch_pat = ''.join(fnmatch_parts)
1784	#log("glob %s fnmatch %s", glob_pat, fnmatch_pat)
1785
1786	results = [] # type: List[str]
1787	n = self.globber.ExpandExtended(glob_pat, fnmatch_pat, results)
1788	if n < 0:
1789	raise error.FailGlob(
1790	'Extended glob %r matched no files' % fnmatch_pat, w)
1791
1792	part_vals.append(part_value.Array(results))
1793	elif bool(eval_flags & EXTGLOB_NESTED):
1794	# We only glob at the TOP level of @(nested\|@(pattern))
1795	part_vals.extend(word_part_vals)
1796	else:
1797	# e.g. simple_word_eval, assignment builtin
1798	e_die('Extended glob not allowed in this word', w)
1799	else:
1800	part_vals.extend(word_part_vals)
1801
1802	def _PartValsToString(self, part_vals, w, eval_flags, strs):
1803	# type: (List[part_value_t], CompoundWord, int, List[str]) -> None
1804	"""Helper for EvalWordToString, similar to _ConcatPartVals() above.
1805
1806	Note: arg 'w' could just be a span ID
1807	"""
1808	for part_val in part_vals:
1809	UP_part_val = part_val
1810	with tagswitch(part_val) as case:
1811	if case(part_value_e.String):
1812	part_val = cast(Piece, UP_part_val)
1813	s = part_val.s
1814	if part_val.quoted:
1815	if eval_flags & QUOTE_FNMATCH:
1816	# [[ foo == /"".py ]] or case (.py) or ${x%.py} or ${x//*.py/}
1817	s = glob_.GlobEscape(s)
1818	elif eval_flags & QUOTE_ERE:
1819	s = glob_.ExtendedRegexEscape(s)
1820	strs.append(s)
1821
1822	elif case(part_value_e.Array):
1823	part_val = cast(part_value.Array, UP_part_val)
1824	if self.exec_opts.strict_array():
1825	# Examples: echo f > "$@"; local foo="$@"
1826
1827	# TODO: This attributes too coarsely, to the word rather than the
1828	# parts. Problem: the word is a TREE of parts, but we only have a
1829	# flat list of part_vals. The only case where we really get arrays
1830	# is "$@", "${a[@]}", "${a[@]//pat/replace}", etc.
1831	e_die(
1832	"This word should yield a string, but it contains an array",
1833	w)
1834
1835	# TODO: Maybe add detail like this.
1836	#e_die('RHS of assignment should only have strings. '
1837	# 'To assign arrays, use b=( "${a[@]}" )')
1838	else:
1839	# It appears to not respect IFS
1840	tmp = [s for s in part_val.strs if s is not None]
1841	s = ' '.join(tmp) # TODO: eliminate double join()?
1842	strs.append(s)
1843
1844	elif case(part_value_e.ExtGlob):
1845	part_val = cast(part_value.ExtGlob, UP_part_val)
1846
1847	# Extended globs are only allowed where we expect them!
1848	if not bool(eval_flags & QUOTE_FNMATCH):
1849	e_die('extended glob not allowed in this word', w)
1850
1851	# recursive call
1852	self._PartValsToString(part_val.part_vals, w, eval_flags,
1853	strs)
1854
1855	else:
1856	raise AssertionError()
1857
1858	def EvalWordToString(self, UP_w, eval_flags=0):
1859	# type: (word_t, int) -> value.Str
1860	"""Given a word, return a string.
1861
1862	Flags can contain a quoting algorithm.
1863	"""
1864	assert UP_w.tag() == word_e.Compound, UP_w
1865	w = cast(CompoundWord, UP_w)
1866
1867	if eval_flags == 0: # QUOTE_FNMATCH etc. breaks optimization
1868	fast_str = word_.FastStrEval(w)
1869	if fast_str is not None:
1870	return value.Str(fast_str)
1871
1872	# Could we additionally optimize a=$b, if we know $b isn't an array
1873	# etc.?
1874
1875	# Note: these empty lists are hot in fib benchmark
1876
1877	part_vals = [] # type: List[part_value_t]
1878	for p in w.parts:
1879	# this doesn't use eval_flags, which is slightly confusing
1880	self._EvalWordPart(p, part_vals, 0)
1881
1882	strs = [] # type: List[str]
1883	self._PartValsToString(part_vals, w, eval_flags, strs)
1884	return value.Str(''.join(strs))
1885
1886	def EvalWordToPattern(self, UP_w):
1887	# type: (rhs_word_t) -> Tuple[value.Str, bool]
1888	"""Like EvalWordToString, but returns whether we got ExtGlob."""
1889	if UP_w.tag() == rhs_word_e.Empty:
1890	return value.Str(''), False
1891
1892	assert UP_w.tag() == rhs_word_e.Compound, UP_w
1893	w = cast(CompoundWord, UP_w)
1894
1895	has_extglob = False
1896	part_vals = [] # type: List[part_value_t]
1897	for p in w.parts:
1898	# this doesn't use eval_flags, which is slightly confusing
1899	self._EvalWordPart(p, part_vals, 0)
1900	if p.tag() == word_part_e.ExtGlob:
1901	has_extglob = True
1902
1903	strs = [] # type: List[str]
1904	self._PartValsToString(part_vals, w, QUOTE_FNMATCH, strs)
1905	return value.Str(''.join(strs)), has_extglob
1906
1907	def EvalForPlugin(self, w):
1908	# type: (CompoundWord) -> value.Str
1909	"""Wrapper around EvalWordToString that prevents errors.
1910
1911	Runtime errors like $(( 1 / 0 )) and mutating $? like $(exit 42)
1912	are handled here.
1913
1914	Similar to ExprEvaluator.PluginCall().
1915	"""
1916	with state.ctx_Registers(self.mem): # to "sandbox" $? and $PIPESTATUS
1917	try:
1918	val = self.EvalWordToString(w)
1919	except error.FatalRuntime as e:
1920	val = value.Str('<Runtime error: %s>' % e.UserErrorString())
1921
1922	except (IOError, OSError) as e:
1923	val = value.Str('<I/O error: %s>' % pyutil.strerror(e))
1924
1925	except KeyboardInterrupt:
1926	val = value.Str('<Ctrl-C>')
1927
1928	return val
1929
1930	def EvalRhsWord(self, UP_w):
1931	# type: (rhs_word_t) -> value_t
1932	"""Used for RHS of assignment.
1933
1934	There is no splitting.
1935	"""
1936	if UP_w.tag() == rhs_word_e.Empty:
1937	return value.Str('')
1938
1939	assert UP_w.tag() == word_e.Compound, UP_w
1940	w = cast(CompoundWord, UP_w)
1941
1942	if len(w.parts) == 1:
1943	part0 = w.parts[0]
1944	UP_part0 = part0
1945	tag = part0.tag()
1946	# Special case for a=(1 2). ShArrayLiteral won't appear in words that
1947	# don't look like assignments.
1948	if tag == word_part_e.ShArrayLiteral:
1949	part0 = cast(ShArrayLiteral, UP_part0)
1950	array_words = part0.words
1951	words = braces.BraceExpandWords(array_words)
1952	strs = self.EvalWordSequence(words)
1953	return value.BashArray(strs)
1954
1955	if tag == word_part_e.BashAssocLiteral:
1956	part0 = cast(word_part.BashAssocLiteral, UP_part0)
1957	d = NewDict() # type: Dict[str, str]
1958	for pair in part0.pairs:
1959	k = self.EvalWordToString(pair.key)
1960	v = self.EvalWordToString(pair.value)
1961	d[k.s] = v.s
1962	return value.BashAssoc(d)
1963
1964	# If RHS doesn't look like a=( ... ), then it must be a string.
1965	return self.EvalWordToString(w)
1966
1967	def _EvalWordFrame(self, frame, argv):
1968	# type: (List[Piece], List[str]) -> None
1969	all_empty = True
1970	all_quoted = True
1971	any_quoted = False
1972
1973	#log('--- frame %s', frame)
1974
1975	for piece in frame:
1976	if len(piece.s):
1977	all_empty = False
1978
1979	if piece.quoted:
1980	any_quoted = True
1981	else:
1982	all_quoted = False
1983
1984	# Elision of ${empty}${empty} but not $empty"$empty" or $empty""
1985	if all_empty and not any_quoted:
1986	return
1987
1988	# If every frag is quoted, e.g. "$a$b" or any part in "${a[@]}"x, then
1989	# don't do word splitting or globbing.
1990	if all_quoted:
1991	tmp = [piece.s for piece in frame]
1992	a = ''.join(tmp)
1993	argv.append(a)
1994	return
1995
1996	will_glob = not self.exec_opts.noglob()
1997
1998	# Array of strings, some of which are BOTH IFS-escaped and GLOB escaped!
1999	frags = [] # type: List[str]
2000	for piece in frame:
2001	if will_glob and piece.quoted:
2002	frag = glob_.GlobEscape(piece.s)
2003	else:
2004	# If we have a literal \, then we turn it into \\\\.
2005	# Splitting takes \\\\ -> \\
2006	# Globbing takes \\ to \ if it doesn't match
2007	frag = _BackslashEscape(piece.s)
2008
2009	if piece.do_split:
2010	frag = _BackslashEscape(frag)
2011	else:
2012	frag = self.splitter.Escape(frag)
2013
2014	frags.append(frag)
2015
2016	flat = ''.join(frags)
2017	#log('flat: %r', flat)
2018
2019	args = self.splitter.SplitForWordEval(flat)
2020
2021	# space=' '; argv $space"". We have a quoted part, but we CANNOT elide.
2022	# Add it back and don't bother globbing.
2023	if len(args) == 0 and any_quoted:
2024	argv.append('')
2025	return
2026
2027	#log('split args: %r', args)
2028	for a in args:
2029	if glob_.LooksLikeGlob(a):
2030	n = self.globber.Expand(a, argv)
2031	if n < 0:
2032	# TODO: location info, with span IDs carried through the frame
2033	raise error.FailGlob('Pattern %r matched no files' % a,
2034	loc.Missing)
2035	else:
2036	argv.append(glob_.GlobUnescape(a))
2037
2038	def _EvalWordToArgv(self, w):
2039	# type: (CompoundWord) -> List[str]
2040	"""Helper for _EvalAssignBuiltin.
2041
2042	Splitting and globbing are disabled for assignment builtins.
2043
2044	Example: declare -"${a[@]}" b=(1 2)
2045	where a is [x b=a d=a]
2046	"""
2047	part_vals = [] # type: List[part_value_t]
2048	self._EvalWordToParts(w, part_vals, 0) # not double quoted
2049	frames = _MakeWordFrames(part_vals)
2050	argv = [] # type: List[str]
2051	for frame in frames:
2052	if len(frame): # empty array gives empty frame!
2053	tmp = [piece.s for piece in frame]
2054	argv.append(''.join(tmp)) # no split or glob
2055	#log('argv: %s', argv)
2056	return argv
2057
2058	def _EvalAssignBuiltin(self, builtin_id, arg0, words):
2059	# type: (builtin_t, str, List[CompoundWord]) -> cmd_value.Assign
2060	"""Handles both static and dynamic assignment, e.g.
2061
2062	x='foo=bar' local a=(1 2) $x
2063	"""
2064	# Grammar:
2065	#
2066	# ('builtin' \| 'command')* keyword flag* pair*
2067	# flag = [-+].*
2068	#
2069	# There is also command -p, but we haven't implemented it. Maybe just punt
2070	# on it. Punted on 'builtin' and 'command' for now too.
2071
2072	eval_to_pairs = True # except for -f and -F
2073	started_pairs = False
2074
2075	flags = [arg0] # initial flags like -p, and -f -F name1 name2
2076	flag_locs = [words[0]]
2077	assign_args = [] # type: List[AssignArg]
2078
2079	n = len(words)
2080	for i in xrange(1, n): # skip first word
2081	w = words[i]
2082
2083	if word_.IsVarLike(w):
2084	started_pairs = True # Everything from now on is an assign_pair
2085
2086	if started_pairs:
2087	left_token, close_token, part_offset = word_.DetectShAssignment(
2088	w)
2089	if left_token: # Detected statically
2090	if left_token.id != Id.Lit_VarLike:
2091	# (not guaranteed since started_pairs is set twice)
2092	e_die('LHS array not allowed in assignment builtin', w)
2093
2094	if lexer.IsPlusEquals(left_token):
2095	var_name = lexer.TokenSliceRight(left_token, -2)
2096	append = True
2097	else:
2098	var_name = lexer.TokenSliceRight(left_token, -1)
2099	append = False
2100
2101	if part_offset == len(w.parts):
2102	rhs = rhs_word.Empty # type: rhs_word_t
2103	else:
2104	# tmp is for intersection of C++/MyPy type systems
2105	tmp = CompoundWord(w.parts[part_offset:])
2106	word_.TildeDetectAssign(tmp)
2107	rhs = tmp
2108
2109	with state.ctx_AssignBuiltin(self.mutable_opts):
2110	right = self.EvalRhsWord(rhs)
2111
2112	arg2 = AssignArg(var_name, right, append, w)
2113	assign_args.append(arg2)
2114
2115	else: # e.g. export $dynamic
2116	argv = self._EvalWordToArgv(w)
2117	for arg in argv:
2118	arg2 = _SplitAssignArg(arg, w)
2119	assign_args.append(arg2)
2120
2121	else:
2122	argv = self._EvalWordToArgv(w)
2123	for arg in argv:
2124	if arg.startswith('-') or arg.startswith('+'):
2125	# e.g. declare -r +r
2126	flags.append(arg)
2127	flag_locs.append(w)
2128
2129	# Shortcut that relies on -f and -F always meaning "function" for
2130	# all assignment builtins
2131	if 'f' in arg or 'F' in arg:
2132	eval_to_pairs = False
2133
2134	else: # e.g. export $dynamic
2135	if eval_to_pairs:
2136	arg2 = _SplitAssignArg(arg, w)
2137	assign_args.append(arg2)
2138	started_pairs = True
2139	else:
2140	flags.append(arg)
2141
2142	return cmd_value.Assign(builtin_id, flags, flag_locs, assign_args)
2143
2144	def _DetectAssignBuiltinStr(self, arg0, words):
2145	# type: (str, List[CompoundWord]) -> Optional[cmd_value.Assign]
2146	builtin_id = consts.LookupAssignBuiltin(arg0)
2147	if builtin_id != consts.NO_INDEX:
2148	return self._EvalAssignBuiltin(builtin_id, arg0, words)
2149	return None
2150
2151	def _DetectAssignBuiltin(self, val0, words):
2152	# type: (part_value_t, List[CompoundWord]) -> Optional[cmd_value.Assign]
2153	UP_val0 = val0
2154	if val0.tag() == part_value_e.String:
2155	val0 = cast(Piece, UP_val0)
2156	if not val0.quoted:
2157	return self._DetectAssignBuiltinStr(val0.s, words)
2158	return None
2159
2160	def _DetectMetaBuiltinStr(self, s):
2161	# type: (str) -> bool
2162	"""
2163	We need to detect all of these cases:
2164
2165	builtin local
2166	command local
2167	builtin builtin local
2168	builtin command local
2169
2170	Fundamentally, assignment builtins have different WORD EVALUATION RULES
2171	for a=$x (no word splitting), so it seems hard to do this in
2172	meta_osh.Builtin() or meta_osh.Command()
2173	"""
2174	return (consts.LookupNormalBuiltin(s)
2175	in (builtin_i.builtin, builtin_i.command))
2176
2177	def _DetectMetaBuiltin(self, val0):
2178	# type: (part_value_t) -> bool
2179	UP_val0 = val0
2180	if val0.tag() == part_value_e.String:
2181	val0 = cast(Piece, UP_val0)
2182	if not val0.quoted:
2183	return self._DetectMetaBuiltinStr(val0.s)
2184	return False
2185
2186	def SimpleEvalWordSequence2(self, words, allow_assign):
2187	# type: (List[CompoundWord], bool) -> cmd_value_t
2188	"""Simple word evaluation for YSH."""
2189	strs = [] # type: List[str]
2190	locs = [] # type: List[CompoundWord]
2191
2192	assign_builtin_offset = 0
2193	for i, w in enumerate(words):
2194	# No globbing in the first arg for command.Simple.
2195	if i == assign_builtin_offset and allow_assign:
2196	strs0 = self._EvalWordToArgv(w)
2197	# Same logic as legacy word eval, with no splitting
2198	# TODO: Remove this because we should remove assignment
2199	# builtins? What about export?
2200	if len(strs0) == 1:
2201	cmd_val = self._DetectAssignBuiltinStr(strs0[0], words)
2202	if cmd_val:
2203	return cmd_val
2204
2205	strs.extend(strs0)
2206	for _ in strs0:
2207	locs.append(w)
2208	continue
2209
2210	if glob_.LooksLikeStaticGlob(w):
2211	val = self.EvalWordToString(w) # respects strict-array
2212	num_appended = self.globber.Expand(val.s, strs)
2213	if num_appended < 0:
2214	raise error.FailGlob('Pattern %r matched no files' % val.s,
2215	w)
2216	for _ in xrange(num_appended):
2217	locs.append(w)
2218	continue
2219
2220	part_vals = [] # type: List[part_value_t]
2221	self._EvalWordToParts(w, part_vals, 0) # not quoted
2222
2223	if 0:
2224	log('')
2225	log('Static: part_vals after _EvalWordToParts:')
2226	for entry in part_vals:
2227	log(' %s', entry)
2228
2229	# Still need to process
2230	frames = _MakeWordFrames(part_vals)
2231
2232	if 0:
2233	log('')
2234	log('Static: frames after _MakeWordFrames:')
2235	for entry in frames:
2236	log(' %s', entry)
2237
2238	# We will still allow x"${a[@]"x, though it's deprecated by @a, which
2239	# disallows such expressions at parse time.
2240	for frame in frames:
2241	if len(frame): # empty array gives empty frame!
2242	tmp = [piece.s for piece in frame]
2243	strs.append(''.join(tmp)) # no split or glob
2244	locs.append(w)
2245
2246	return cmd_value.Argv(strs, locs, None, None, None, None)
2247
2248	def EvalWordSequence2(self, words, allow_assign=False):
2249	# type: (List[CompoundWord], bool) -> cmd_value_t
2250	"""Turns a list of Words into a list of strings.
2251
2252	Unlike the EvalWord*() methods, it does globbing.
2253
2254	Args:
2255	allow_assign: True for command.Simple, False for BashArray a=(1 2 3)
2256	"""
2257	if self.exec_opts.simple_word_eval():
2258	return self.SimpleEvalWordSequence2(words, allow_assign)
2259
2260	# Parse time:
2261	# 1. brace expansion. TODO: Do at parse time.
2262	# 2. Tilde detection. DONE at parse time. Only if Id.Lit_Tilde is the
2263	# first WordPart.
2264	#
2265	# Run time:
2266	# 3. tilde sub, var sub, command sub, arith sub. These are all
2267	# "concurrent" on WordParts. (optional process sub with <() )
2268	# 4. word splitting. Can turn this off with a shell option? Definitely
2269	# off for oil.
2270	# 5. globbing -- several exec_opts affect this: nullglob, safeglob, etc.
2271
2272	#log('W %s', words)
2273	strs = [] # type: List[str]
2274	locs = [] # type: List[CompoundWord]
2275
2276	assign_builtin_offset = 0
2277
2278	n = 0
2279	for i, w in enumerate(words):
2280	fast_str = word_.FastStrEval(w)
2281	if fast_str is not None:
2282	strs.append(fast_str)
2283	locs.append(w)
2284
2285	# e.g. the 'local' in 'local a=b c=d' will be here
2286	if allow_assign and i == assign_builtin_offset:
2287	cmd_val = self._DetectAssignBuiltinStr(fast_str, words)
2288	if cmd_val:
2289	return cmd_val
2290	#print('YO %s' % fast_str)
2291	if self._DetectMetaBuiltinStr(fast_str):
2292	#print('META')
2293	assign_builtin_offset += 1
2294	continue
2295
2296	part_vals = [] # type: List[part_value_t]
2297	self._EvalWordToParts(w, part_vals, EXTGLOB_FILES)
2298
2299	# DYNAMICALLY detect if we're going to run an assignment builtin, and
2300	# change the rest of the evaluation algorithm if so.
2301	#
2302	# We want to allow:
2303	# e=export
2304	# $e foo=bar
2305	#
2306	# But we don't want to evaluate the first word twice in the case of:
2307	# $(some-command) --flag
2308	if (allow_assign and i == assign_builtin_offset and
2309	len(part_vals) == 1):
2310	cmd_val = self._DetectAssignBuiltin(part_vals[0], words)
2311	if cmd_val:
2312	return cmd_val
2313
2314	if len(part_vals) == 1:
2315	if self._DetectMetaBuiltin(part_vals[0]):
2316	assign_builtin_offset += 1
2317
2318	if 0:
2319	log('')
2320	log('part_vals after _EvalWordToParts:')
2321	for entry in part_vals:
2322	log(' %s', entry)
2323
2324	frames = _MakeWordFrames(part_vals)
2325	if 0:
2326	log('')
2327	log('frames after _MakeWordFrames:')
2328	for entry in frames:
2329	log(' %s', entry)
2330
2331	# Do splitting and globbing. Each frame will append zero or more args.
2332	for frame in frames:
2333	self._EvalWordFrame(frame, strs)
2334
2335	# Fill in locations parallel to strs.
2336	n_next = len(strs)
2337	for _ in xrange(n_next - n):
2338	locs.append(w)
2339	n = n_next
2340
2341	# A non-assignment command.
2342	# NOTE: Can't look up builtins here like we did for assignment, because
2343	# functions can override builtins.
2344	return cmd_value.Argv(strs, locs, None, None, None, None)
2345
2346	def EvalWordSequence(self, words):
2347	# type: (List[CompoundWord]) -> List[str]
2348	"""For arrays and for loops.
2349
2350	They don't allow assignment builtins.
2351	"""
2352	UP_cmd_val = self.EvalWordSequence2(words)
2353
2354	assert UP_cmd_val.tag() == cmd_value_e.Argv
2355	cmd_val = cast(cmd_value.Argv, UP_cmd_val)
2356	return cmd_val.argv
2357
2358
2359	class NormalWordEvaluator(AbstractWordEvaluator):
2360
2361	def __init__(
2362	self,
2363	mem, # type: state.Mem
2364	exec_opts, # type: optview.Exec
2365	mutable_opts, # type: state.MutableOpts
2366	tilde_ev, # type: TildeEvaluator
2367	splitter, # type: SplitContext
2368	errfmt, # type: ErrorFormatter
2369	):
2370	# type: (...) -> None
2371	AbstractWordEvaluator.__init__(self, mem, exec_opts, mutable_opts,
2372	tilde_ev, splitter, errfmt)
2373	self.shell_ex = None # type: _Executor
2374
2375	def CheckCircularDeps(self):
2376	# type: () -> None
2377	assert self.arith_ev is not None
2378	# Disabled for pure OSH
2379	#assert self.expr_ev is not None
2380	assert self.shell_ex is not None
2381	assert self.prompt_ev is not None
2382
2383	def _EvalCommandSub(self, cs_part, quoted):
2384	# type: (CommandSub, bool) -> part_value_t
2385	stdout_str = self.shell_ex.RunCommandSub(cs_part)
2386
2387	if cs_part.left_token.id == Id.Left_AtParen:
2388	# YSH splitting algorithm: does not depend on IFS
2389	try:
2390	strs = j8.SplitJ8Lines(stdout_str)
2391	except error.Decode as e:
2392	# status code 4 is special, for encode/decode errors.
2393	raise error.Structured(4, e.Message(), cs_part.left_token)
2394
2395	#strs = self.splitter.SplitForWordEval(stdout_str)
2396	return part_value.Array(strs)
2397	else:
2398	return Piece(stdout_str, quoted, not quoted)
2399
2400	def _EvalProcessSub(self, cs_part):
2401	# type: (CommandSub) -> Piece
2402	dev_path = self.shell_ex.RunProcessSub(cs_part)
2403	# pretend it's quoted; no split or glob
2404	return Piece(dev_path, True, False)
2405
2406
2407	_DUMMY = '__NO_COMMAND_SUB__'
2408
2409
2410	class CompletionWordEvaluator(AbstractWordEvaluator):
2411	"""An evaluator that has no access to an executor.
2412
2413	NOTE: core/completion.py doesn't actually try to use these strings to
2414	complete. If you have something like 'echo $(echo hi)/f<TAB>', it sees the
2415	inner command as the last one, and knows that it is not at the end of the
2416	line.
2417	"""
2418
2419	def __init__(
2420	self,
2421	mem, # type: state.Mem
2422	exec_opts, # type: optview.Exec
2423	mutable_opts, # type: state.MutableOpts
2424	tilde_ev, # type: TildeEvaluator
2425	splitter, # type: SplitContext
2426	errfmt, # type: ErrorFormatter
2427	):
2428	# type: (...) -> None
2429	AbstractWordEvaluator.__init__(self, mem, exec_opts, mutable_opts,
2430	tilde_ev, splitter, errfmt)
2431
2432	def CheckCircularDeps(self):
2433	# type: () -> None
2434	assert self.prompt_ev is not None
2435	assert self.arith_ev is not None
2436	assert self.expr_ev is not None
2437
2438	def _EvalCommandSub(self, cs_part, quoted):
2439	# type: (CommandSub, bool) -> part_value_t
2440	if cs_part.left_token.id == Id.Left_AtParen:
2441	return part_value.Array([_DUMMY])
2442	else:
2443	return Piece(_DUMMY, quoted, not quoted)
2444
2445	def _EvalProcessSub(self, cs_part):
2446	# type: (CommandSub) -> Piece
2447	# pretend it's quoted; no split or glob
2448	return Piece('__NO_PROCESS_SUB__', True, False)
2449
2450
2451	# vim: sw=4