OILS / opy / _regtest / src / osh / cmd_parse_test.py View on Github | oilshell.org

1303 lines, 696 significant
1#!/usr/bin/env python
2"""
3cmd_parse_test.py: Tests for cmd_parse.py
4"""
5
6import sys
7import unittest
8
9from core import ui
10from osh.meta import Id
11from core import word
12from core import test_lib
13
14from osh.meta import ast
15from osh import ast_lib
16from osh import parse_lib
17from osh.cmd_parse import CommandParser # module under test
18from osh.word_parse import WordParser
19
20command_e = ast.command_e
21
22
23# TODO: Use parse_lib instead
24def InitCommandParser(code_str):
25 arena = test_lib.MakeArena('<cmd_parse_test.py>')
26 line_reader, lexer = parse_lib.InitLexer(code_str, arena)
27 w_parser = WordParser(lexer, line_reader)
28 c_parser = CommandParser(w_parser, lexer, line_reader, arena)
29 return arena, c_parser # arena is returned for printing errors
30
31
32def _assertParseMethod(test, code_str, method, expect_success=True):
33 arena, c_parser = InitCommandParser(code_str)
34 m = getattr(c_parser, method)
35 node = m()
36
37 if node:
38 ast_lib.PrettyPrint(node)
39 if not expect_success:
40 test.fail('Expected %r to fail ' % code_str)
41 else:
42 # TODO: Could copy PrettyPrintError from pysh.py
43 err = c_parser.Error()
44 print(err)
45 ui.PrintErrorStack(err, arena, sys.stdout)
46 if expect_success:
47 test.fail('%r failed' % code_str)
48 return node
49
50
51def _assertParseCommandListError(test, code_str):
52 arena, c_parser = InitCommandParser(code_str)
53 node = c_parser.ParseCommandLine()
54 if node:
55 print('UNEXPECTED:')
56 ast_lib.PrettyPrint(node)
57 test.fail("Expected %r to fail" % code_str)
58 return
59 err = c_parser.Error()
60 #print(err)
61 ui.PrintErrorStack(err, arena, sys.stdout)
62
63
64#
65# Successes
66#
67# (These differences might not matter, but preserve the diversity for now)
68
69def assertParseSimpleCommand(test, code_str):
70 return _assertParseMethod(test, code_str, 'ParseSimpleCommand')
71
72def assertParsePipeline(test, code_str):
73 return _assertParseMethod(test, code_str, 'ParsePipeline')
74
75def assertParseAndOr(test, code_str):
76 return _assertParseMethod(test, code_str, 'ParseAndOr')
77
78def assertParseCommandLine(test, code_str):
79 node = _assertParseMethod(test, code_str, 'ParseCommandLine')
80 if len(node.children) == 1:
81 return node.children[0]
82 else:
83 return node
84
85def assertParseCommandList(test, code_str):
86 node = _assertParseMethod(test, code_str, 'ParseCommandList')
87 if len(node.children) == 1:
88 return node.children[0]
89 else:
90 return node
91
92def assertParseRedirect(test, code_str):
93 return _assertParseMethod(test, code_str, 'ParseRedirect')
94
95#
96# Failures
97#
98
99#def assertFailSimpleCommand(test, code_str):
100# return _assertParseMethod(test, code_str, 'ParseSimpleCommand',
101# expect_success=False)
102#
103#def assertFailCommandLine(test, code_str):
104# return _assertParseMethod(test, code_str, 'ParseCommandLine',
105# expect_success=False)
106
107def assertFailCommandList(test, code_str):
108 return _assertParseMethod(test, code_str, 'ParseCommandList',
109 expect_success=False)
110
111#def assertFailRedirect(test, code_str):
112# return _assertParseMethod(test, code_str, 'ParseRedirect',
113# expect_success=False)
114
115
116class SimpleCommandTest(unittest.TestCase):
117
118 def testParseSimpleCommand1(self):
119 node = assertParseSimpleCommand(self, 'ls foo')
120 self.assertEqual(2, len(node.words), node.words)
121
122 node = assertParseSimpleCommand(self, 'FOO=bar ls foo')
123 self.assertEqual(2, len(node.words))
124 self.assertEqual(1, len(node.more_env))
125
126 node = assertParseSimpleCommand(self,
127 'FOO=bar >output.txt SPAM=eggs ls foo')
128 self.assertEqual(2, len(node.words))
129 self.assertEqual(2, len(node.more_env))
130 self.assertEqual(1, len(node.redirects))
131
132 node = assertParseSimpleCommand(self,
133 'FOO=bar >output.txt SPAM=eggs ls foo >output2.txt')
134 self.assertEqual(2, len(node.words))
135 self.assertEqual(2, len(node.more_env))
136 self.assertEqual(2, len(node.redirects))
137
138 def testMultipleGlobalAssignments(self):
139 node = assertParseCommandList(self, 'ONE=1 TWO=2')
140 self.assertEqual(command_e.Assignment, node.tag)
141 self.assertEqual(2, len(node.pairs))
142
143 def testExport(self):
144 # This is the old static parsing. Probably need to revisit.
145 return
146 node = assertParseCommandList(self, 'export ONE=1 TWO=2 THREE')
147 self.assertEqual(command_e.Assignment, node.tag)
148 self.assertEqual(3, len(node.pairs))
149
150 def testReadonly(self):
151 node = assertParseCommandList(self, 'readonly ONE=1 TWO=2 THREE')
152 self.assertEqual(command_e.Assignment, node.tag)
153 self.assertEqual(3, len(node.pairs))
154
155 def testOnlyRedirect(self):
156 # This just touches the file
157 node = assertParseCommandList(self, '>out.txt')
158 self.assertEqual(command_e.SimpleCommand, node.tag)
159 self.assertEqual(0, len(node.words))
160 self.assertEqual(1, len(node.redirects))
161
162 def testParseRedirectInTheMiddle(self):
163 node = assertParseCommandList(self, 'echo >out.txt 1 2 3')
164 self.assertEqual(command_e.SimpleCommand, node.tag)
165 self.assertEqual(4, len(node.words))
166 self.assertEqual(1, len(node.redirects))
167
168 def testParseRedirectBeforeAssignment(self):
169 # Write ENV to a file
170 node = assertParseCommandList(self, '>out.txt PYTHONPATH=. env')
171 self.assertEqual(command_e.SimpleCommand, node.tag)
172 self.assertEqual(1, len(node.words))
173 self.assertEqual(1, len(node.redirects))
174 self.assertEqual(1, len(node.more_env))
175
176 def testParseAssignment(self):
177 node = assertParseCommandList(self, 'local foo=bar spam eggs one=1')
178 self.assertEqual(4, len(node.pairs))
179
180 node = assertParseCommandList(self, 'foo=bar')
181 self.assertEqual(1, len(node.pairs))
182
183 # This is not valid since env isn't respected
184 assertFailCommandList(self, 'FOO=bar local foo=$(env)')
185
186 def testParseAdjacentDoubleQuotedWords(self):
187 node = assertParseSimpleCommand(self, 'echo "one"two "three""four" five')
188 self.assertEqual(4, len(node.words))
189
190
191def assertHereDocToken(test, expected_token_val, node):
192 #print(node)
193 test.assertEqual(1, len(node.redirects))
194 h = node.redirects[0]
195 word_parts = h.body.parts
196 test.assertEqual(1, len(word_parts)) # 1 line, one literal part
197 part1 = word_parts[0]
198 test.assertGreater(len(part1.parts), 1, part1)
199 test.assertEqual(expected_token_val, part1.parts[0].token.val)
200
201
202class HereDocTest(unittest.TestCase):
203 """NOTE: These ares come from tests/09-here-doc.sh, but add assertions."""
204
205 def testUnquotedHereDoc(self):
206 # Unquoted here docs use the double quoted context.
207 node = assertParseCommandLine(self, """\
208cat <<EOF
209$v
210"two
211EOF
212""")
213 self.assertEqual(1, len(node.redirects))
214 h = node.redirects[0]
215 self.assertEqual(1, len(h.body.parts)) # 1 double quoted part
216 dq = h.body.parts[0]
217 self.assertTrue(isinstance(dq, ast.DoubleQuotedPart))
218 # 4 literal parts: VarSub, newline, right ", "two\n"
219 self.assertEqual(4, len(dq.parts))
220 self.assertEqual(True, h.do_expansion)
221
222 def testQuotedHereDocs(self):
223 # Quoted here doc
224 node = assertParseCommandLine(self, """\
225cat <<"EOF"
226$v
227"two
228EOF
229""")
230 self.assertEqual(1, len(node.redirects))
231 h = node.redirects[0]
232 self.assertEqual(2, len(h.body.parts)) # 2 literal parts
233 self.assertEqual(False, h.do_expansion)
234
235 node = assertParseCommandLine(self, """\
236cat <<'EOF'
237single-quoted: $var
238EOF
239""")
240 self.assertEqual(1, len(node.redirects))
241 h = node.redirects[0]
242 self.assertEqual(1, len(h.body.parts)) # 1 line, one literal part
243 self.assertEqual(False, h.do_expansion)
244
245 # \ escape
246 node = assertParseCommandLine(self, r"""\
247cat <<EO\F
248single-quoted: $var
249EOF
250""")
251 self.assertEqual(1, len(node.redirects))
252 h = node.redirects[0]
253 self.assertEqual(1, len(h.body.parts)) # 1 line, one literal part
254 self.assertEqual(False, h.do_expansion)
255
256 def testLeadingTabs(self):
257 node = assertParseCommandLine(self, """\
258\tcat <<-EOF
259\tone tab then foo: $foo
260\tEOF
261echo hi
262""")
263 self.assertEqual(node.tag, command_e.SimpleCommand)
264 assertHereDocToken(self, 'one tab then foo: ', node)
265
266 def testHereDocInPipeline(self):
267 # Pipe and command on SAME LINE
268 node = assertParseCommandLine(self, """\
269cat <<EOF | tac
270PIPE 1
271PIPE 2
272EOF
273""")
274 self.assertEqual(2, len(node.children))
275 assertHereDocToken(self, 'PIPE 1\n', node.children[0])
276
277 # Pipe command AFTER here doc
278 node = assertParseCommandLine(self, """\
279cat <<EOF |
280PIPE 1
281PIPE 2
282EOF
283tac
284""")
285 self.assertEqual(2, len(node.children))
286 assertHereDocToken(self, 'PIPE 1\n', node.children[0])
287
288 def testTwoHereDocsInPipeline(self):
289 # Pipeline with two here docs
290 node = assertParseCommandList(self, """\
291cat <<EOF1 | tac <<EOF2
292PIPE A1
293PIPE A2
294EOF1
295PIPE B1
296PIPE B2
297EOF2
298""")
299 self.assertEqual(2, len(node.children))
300 assertHereDocToken(self, 'PIPE A1\n', node.children[0])
301 assertHereDocToken(self, 'PIPE B1\n', node.children[1])
302
303 def testHereDocInAndOrChain(self):
304 # || command AFTER here doc
305 node = assertParseCommandLine(self, """\
306cat <<EOF ||
307PIPE 1
308PIPE 2
309EOF
310echo hi
311""")
312 self.assertEqual(2, len(node.children))
313 assertHereDocToken(self, 'PIPE 1\n', node.children[0])
314
315 # && and command on SAME LINE
316 node = assertParseCommandLine(self, """\
317cat <<EOF && echo hi
318PIPE 1
319PIPE 2
320EOF
321""")
322 self.assertEqual(2, len(node.children))
323 assertHereDocToken(self, 'PIPE 1\n', node.children[0])
324
325 node = assertParseCommandLine(self, """\
326tac <<EOF1 && tac <<EOF2
327PIPE A1
328PIPE A2
329EOF1
330PIPE B1
331PIPE B2
332EOF2
333echo
334""")
335 self.assertEqual(2, len(node.children))
336 assertHereDocToken(self, 'PIPE A1\n', node.children[0])
337 assertHereDocToken(self, 'PIPE B1\n', node.children[1])
338
339 def testHereDocInSequence(self):
340 # PROBLEM: ParseCommandList vs ParseCommandLine
341 # ParseCommandLine only used interactively. ParseCommandList is used by
342 # ParseFile.
343
344 # command AFTER here doc
345 node = assertParseCommandList(self, """\
346cat <<EOF ;
347PIPE 1
348PIPE 2
349EOF
350echo hi
351""")
352 self.assertEqual(node.tag, command_e.CommandList)
353 self.assertEqual(2, len(node.children), repr(node))
354 assertHereDocToken(self, 'PIPE 1\n', node.children[0].child)
355
356 def testHereDocInSequence2(self):
357 # ; and command on SAME LINE
358 node = assertParseCommandList(self, """\
359cat <<EOF ; echo hi
360PIPE 1
361PIPE 2
362EOF
363""")
364 self.assertEqual(node.tag, command_e.CommandList)
365 self.assertEqual(2, len(node.children))
366 assertHereDocToken(self, 'PIPE 1\n', node.children[0].child)
367
368 def testCommandSubInHereDoc(self):
369 node = assertParseCommandLine(self, """\
370cat <<EOF
3711 $(echo 2
372echo 3) 4
373EOF
374""")
375 self.assertEqual(1, len(node.words))
376 self.assertEqual(1, len(node.redirects))
377
378
379class ArrayTest(unittest.TestCase):
380
381 def testArrayLiteral(self):
382 # Empty array
383 node = assertParseCommandList(self,
384 'empty=()')
385 self.assertEqual(['empty'], [p.lhs.name for p in node.pairs])
386 self.assertEqual([], node.pairs[0].rhs.parts[0].words) # No words
387 self.assertEqual(command_e.Assignment, node.tag)
388
389 # Array with 3 elements
390 node = assertParseCommandList(self,
391 'array=(a b c)')
392 self.assertEqual(['array'], [p.lhs.name for p in node.pairs])
393 self.assertEqual(3, len(node.pairs[0].rhs.parts[0].words))
394 self.assertEqual(command_e.Assignment, node.tag)
395
396 # Array literal can't come after word
397 assertFailCommandList(self,
398 'ls array=(a b c)')
399
400 # Word can't come after array literal
401 assertFailCommandList(self,
402 'array=(a b c) ls')
403
404 # Two array literals
405 node = assertParseCommandList(self,
406 'array=(a b c); array2=(d e f)')
407 self.assertEqual(2, len(node.children))
408 a2 = node.children[1]
409 self.assertEqual(['array2'], [p.lhs.name for p in a2.pairs])
410
411
412class RedirectTest(unittest.TestCase):
413
414 def testParseRedirects1(self):
415 node = assertParseSimpleCommand(self, '>out.txt cat 1>&2')
416 self.assertEqual(1, len(node.words))
417 self.assertEqual(2, len(node.redirects))
418
419 node = assertParseSimpleCommand(self, ' cat <&3')
420 self.assertEqual(1, len(node.redirects))
421
422 def testParseFilenameRedirect(self):
423 node = assertParseRedirect(self, '>out.txt cat')
424
425 def testDescriptorRedirect(self):
426 node = assertParseRedirect(self, '1>& 2 cat')
427
428 def testHereRedirect(self):
429 node = assertParseRedirect(self, """\
430<<EOF cat
431hi
432EOF
433""")
434
435 def testHereRedirectStrip(self):
436 node = assertParseRedirect(self, """\
437<<-EOF cat
438hi
439EOF
440""")
441
442 def testParseRedirectList(self):
443 node = assertParseRedirect(self, """\
444<<EOF >out.txt cat
445hi
446EOF
447""")
448
449 def testParseCommandWithLeadingRedirects(self):
450 node = assertParseSimpleCommand(self, """\
451<<EOF >out.txt cat
452hi
453EOF
454""")
455 self.assertEqual(1, len(node.words))
456 self.assertEqual(2, len(node.redirects))
457
458 def testClobberRedirect(self):
459 node = assertParseSimpleCommand(self, 'echo hi >| clobbered.txt')
460
461
462class CommandParserTest(unittest.TestCase):
463
464 def testParsePipeline(self):
465 node = assertParsePipeline(self, 'ls foo')
466 self.assertEqual(2, len(node.words))
467
468 node = assertParsePipeline(self, 'ls foo|wc -l')
469 self.assertEqual(2, len(node.children))
470 self.assertEqual(command_e.Pipeline, node.tag)
471
472 node = assertParsePipeline(self, '! echo foo | grep foo')
473 self.assertEqual(2, len(node.children))
474 self.assertEqual(command_e.Pipeline, node.tag)
475 self.assertTrue(node.negated)
476
477 node = assertParsePipeline(self, 'ls foo|wc -l|less')
478 self.assertEqual(3, len(node.children))
479 self.assertEqual(command_e.Pipeline, node.tag)
480
481 # Should be an error
482 _, c_parser = InitCommandParser('ls foo|')
483 self.assertEqual(None, c_parser.ParsePipeline())
484 print(c_parser.Error())
485
486 def testParsePipelineBash(self):
487 node = assertParseCommandList(self, 'ls | cat |& cat')
488 self.assertEqual(command_e.Pipeline, node.tag)
489 self.assertEqual([1], node.stderr_indices)
490
491 node = assertParseCommandList(self, 'ls |& cat | cat')
492 self.assertEqual(command_e.Pipeline, node.tag)
493 self.assertEqual([0], node.stderr_indices)
494
495 node = assertParseCommandList(self, 'ls |& cat |& cat')
496 self.assertEqual(command_e.Pipeline, node.tag)
497 self.assertEqual([0, 1], node.stderr_indices)
498
499 def testParseAndOr(self):
500 node = assertParseAndOr(self, 'ls foo')
501 self.assertEqual(2, len(node.words))
502
503 node = assertParseAndOr(self, 'ls foo|wc -l')
504 self.assertEqual(2, len(node.children))
505 self.assertEqual(command_e.Pipeline, node.tag)
506
507 node = assertParseAndOr(self, 'ls foo || die')
508 self.assertEqual(2, len(node.children))
509 self.assertEqual(command_e.AndOr, node.tag)
510
511 node = assertParseAndOr(self, 'ls foo|wc -l || die')
512 self.assertEqual(2, len(node.children))
513 self.assertEqual(command_e.AndOr, node.tag)
514
515 def testParseCommand(self):
516 _, c_parser = InitCommandParser('ls foo')
517 node = c_parser.ParseCommand()
518 self.assertEqual(2, len(node.words))
519 print(node)
520
521 _, c_parser = InitCommandParser('func() { echo hi; }')
522 node = c_parser.ParseCommand()
523 print(node)
524 self.assertEqual(command_e.FuncDef, node.tag)
525
526 def testParseCommandLine(self):
527 node = assertParseCommandLine(self, 'ls foo 2>/dev/null')
528 self.assertEqual(2, len(node.words))
529
530 node = assertParseCommandLine(self, 'ls foo|wc -l')
531 self.assertEqual(command_e.Pipeline, node.tag)
532
533 node = assertParseCommandLine(self, 'ls foo|wc -l || die')
534 self.assertEqual(command_e.AndOr, node.tag)
535
536 node = assertParseCommandLine(self, 'ls foo|wc -l || die; ls /')
537 self.assertEqual(command_e.CommandList, node.tag)
538 self.assertEqual(2, len(node.children)) # two top level things
539
540 def testParseCommandList(self):
541 node = assertParseCommandList(self, 'ls foo')
542 self.assertEqual(2, len(node.words))
543
544 node = assertParseCommandList(self, 'ls foo|wc -l || die; ls /')
545 self.assertEqual(command_e.CommandList, node.tag)
546 self.assertEqual(2, len(node.children))
547
548 node = assertParseCommandList(self, """\
549ls foo | wc -l || echo fail ;
550echo bar | wc -c || echo f2
551""")
552 self.assertEqual(command_e.CommandList, node.tag)
553 self.assertEqual(2, len(node.children))
554
555 # TODO: Check that we get (LIST (AND_OR (PIPELINE (COMMAND ...)))) here.
556 # We want all levels.
557
558 def testParseCase(self):
559 # Empty case
560 node = assertParseCommandLine(self, """\
561case foo in
562esac
563""")
564 self.assertEqual(command_e.Case, node.tag)
565 self.assertEqual(0, len(node.arms))
566
567# TODO: Test all these. Probably need to add newlines too.
568# case foo esac # INVALID
569# case foo in esac
570# case foo in foo) esac
571# case foo in foo) ;; esac
572# case foo in foo) echo hi ;; esac
573# case foo in foo) echo hi; ;; esac
574
575 node = assertParseCommandLine(self, """\
576case word in
577 foo|foo2|foo3) echo hi ;;
578esac
579""")
580 self.assertEqual(command_e.Case, node.tag)
581 self.assertEqual(1, len(node.arms))
582
583 node = assertParseCommandLine(self, """\
584case word in foo) echo one-line ;; esac
585""")
586 self.assertEqual(command_e.Case, node.tag)
587 self.assertEqual(1, len(node.arms))
588
589 node = assertParseCommandLine(self, """\
590case word in
591 foo) echo foo ;;
592 bar) echo bar ;;
593esac
594""")
595 self.assertEqual(command_e.Case, node.tag)
596 self.assertEqual(2, len(node.arms))
597
598 node = assertParseCommandLine(self, """\
599case word in
600 foo) echo foo ;; # NO TRAILING ;; but trailing ;
601 bar) echo bar ;
602esac
603""")
604 self.assertEqual(command_e.Case, node.tag)
605 self.assertEqual(2, len(node.arms))
606
607 node = assertParseCommandLine(self, """\
608case word in
609 foo) echo foo ;; # NO TRAILING ;;
610 bar) echo bar
611esac
612""")
613 self.assertEqual(command_e.Case, node.tag)
614 self.assertEqual(2, len(node.arms))
615
616 def testParseWhile(self):
617 node = assertParseCommandList(self, """\
618while true; do
619 echo hi
620 break
621done
622""")
623
624 node = assertParseCommandList(self, """\
625while true # comment
626do # comment
627 echo hi # comment
628 break # comment
629done # comment
630""")
631
632 def testParseUntil(self):
633 node = assertParseCommandList(self, """\
634until false; do
635 echo hi
636 break
637done
638""")
639
640 def testParseFor(self):
641 node = assertParseCommandList(self, """\
642for i in 1 2 3; do
643 echo $i
644done
645""")
646 self.assertEqual(3, len(node.iter_words))
647
648 # Don't iterate over anything!
649 node = assertParseCommandList(self, """\
650for i in ; do
651 echo $i
652done
653""")
654 self.assertEqual(0, len(node.iter_words))
655 self.assertEqual(False, node.do_arg_iter)
656
657 # Iterate over the default
658 node = assertParseCommandList(self, """\
659for i; do echo $i; done
660""")
661 self.assertEqual(True, node.do_arg_iter)
662
663 # Iterate over the default, over multiple lines
664 node = assertParseCommandList(self, """\
665for i
666do
667 echo $i
668done
669""")
670 self.assertEqual(True, node.do_arg_iter)
671
672 def testParseForExpression(self):
673 node = assertParseCommandList(self, """\
674for ((i=0; i<5; ++i)); do
675 echo $i
676done
677""")
678 self.assertEqual(Id.Arith_Equal, node.init.op_id)
679 self.assertEqual(Id.Arith_Less, node.cond.op_id)
680 self.assertEqual(Id.Arith_DPlus, node.update.op_id)
681 self.assertEqual(command_e.DoGroup, node.body.tag)
682
683 # Now without the ; OR a newline
684 node = assertParseCommandList(self, """\
685for ((i=0; i<5; ++i)) do
686 echo $i
687done
688""")
689 self.assertEqual(Id.Arith_Equal, node.init.op_id)
690 self.assertEqual(Id.Arith_Less, node.cond.op_id)
691 self.assertEqual(Id.Arith_DPlus, node.update.op_id)
692 self.assertEqual(command_e.DoGroup, node.body.tag)
693
694 node = assertParseCommandList(self, """\
695for ((;;)); do
696 echo $i
697done
698""")
699 self.assertEqual(command_e.DoGroup, node.body.tag)
700
701 def testParseCommandSub(self):
702 # Two adjacent command subs
703 node = assertParseSimpleCommand(self, 'echo $(echo 12)$(echo 34)')
704 self.assertEqual(2, len(node.words))
705
706 # Two adjacent command subs, quoted
707 node = assertParseSimpleCommand(self, 'echo "$(echo 12)$(echo 34)"')
708 self.assertEqual(2, len(node.words))
709
710 def testParseTildeSub(self):
711 node = assertParseCommandList(self,
712 "ls ~ ~root ~/src ~/src/foo ~root/src ~weird!name/blah!blah ")
713
714 def testParseDBracket(self):
715 node = assertParseCommandList(self, '[[ $# -gt 1 ]]')
716
717 # Bash allows embedded newlines in some places, but not all
718 node = assertParseCommandList(self, """\
719[[ $# -gt 1 &&
720
721foo ]]""")
722
723 # Newline needs to be Id.Op_Newline!
724 node = assertParseCommandList(self, """\
725if [[ $# -gt 1 ]]
726then
727 echo hi
728fi
729""")
730
731 # Doh, technically this works!
732 # [[ =~ =~ =~ ]]; echo $?
733 # 0
734
735 def testParseDParen(self):
736 node = assertParseCommandList(self, '(( 1 + 2 ))')
737
738 def testParseDBracketRegex(self):
739 node = assertParseCommandList(self, '[[ foo =~ foo ]]')
740 self.assertEqual(Id.BoolBinary_EqualTilde, node.expr.op_id)
741
742 node = assertParseCommandList(self, '[[ foo =~ (foo|bar) ]]')
743 self.assertEqual(Id.BoolBinary_EqualTilde, node.expr.op_id)
744 right = node.expr.right
745 self.assertEqual(5, len(right.parts))
746 self.assertEqual('(', right.parts[0].token.val)
747
748 # TODO: Implement BASH_REGEX_CHARS
749 return
750 node = assertParseCommandList(self, '[[ "< >" =~ (< >) ]]')
751 self.assertEqual(Id.BoolBinary_EqualTilde, node.expr.op_id)
752
753 node = assertParseCommandList(self, '[[ "ba ba" =~ ([a b]+) ]]')
754 self.assertEqual(Id.BoolBinary_EqualTilde, node.expr.op_id)
755
756 def testParseIf(self):
757 node = assertParseCommandList(self, 'if true; then echo yes; fi')
758 # Subshell in condition
759 node = assertParseCommandList(self, 'if (true); then echo yes; fi')
760
761 def testParseFunction(self):
762 node = assertParseCommandList(self, 'foo() { echo hi; }')
763
764 node = assertParseCommandList(self,
765 'foo() ( echo hi )')
766 node = assertParseCommandList(self,
767 'foo() for i in x; do echo $i; done')
768
769 # KSH FUNCTION
770 node = assertParseCommandList(self, 'function foo { echo hi; }')
771 node = assertParseCommandList(self, 'function foo () { echo hi; }')
772
773 node = assertParseCommandList(self,
774 'function foo() ( echo hi )')
775 node = assertParseCommandList(self,
776 'function foo() for i in x; do echo $i; done')
777
778 # No () is OK here!
779 node = assertParseCommandList(self,
780 'function foo for i in x; do echo $i; done')
781
782 # Redirects
783 node = assertParseCommandList(self, 'foo() { echo hi; } 1>&2 2>/dev/null')
784 self.assertEqual(2, len(node.redirects))
785 self.assertEqual(command_e.BraceGroup, node.body.tag)
786
787 def testParseKeyword(self):
788 # NOTE: It chooses the longest match, which is Lit_Chars>
789 node = assertParseCommandList(self, 'ifFOO')
790
791
792class NestedParensTest(unittest.TestCase):
793 """Test the hard $() and () nesting.
794
795 Meanings of ):
796
797 ( echo x ) # subshell (cmd_parse)
798 echo $(echo x) # command substitution (word_parse)
799 (( )) # end arith command (cmd_parse)
800 $(( )) # end arith sub (word_parse))
801 a=(1 2 3) # array literal and assoc array literal
802 a[1*(2+3)]=x # grouping in arith context
803 func() { echo x ; } # function def
804
805 case x in x) echo x ;; esac # case, with balanced or unbalanced
806 case x in (x) echo x ;; esac
807 """
808
809 def testParseSubshell(self):
810 node = assertParseCommandLine(self,
811 '(cd /; echo PWD 1); echo PWD 2')
812 self.assertEqual(2, len(node.children))
813 self.assertEqual(command_e.CommandList, node.tag)
814
815 def testParseBraceGroup(self):
816 node = assertParseCommandLine(self,
817 '{ cd /; echo PWD; }')
818 self.assertEqual(2, len(node.children))
819 self.assertEqual(command_e.BraceGroup, node.tag)
820
821 node = assertParseCommandLine(self,
822 '{ cd /; echo PWD; }; echo PWD')
823 self.assertEqual(2, len(node.children))
824 self.assertEqual(command_e.CommandList, node.tag)
825
826 def testUnquotedComSub(self):
827 # CommandSubPart with two LiteralPart instances surrounding it
828 node = assertParseSimpleCommand(self,
829 'echo ab$(echo hi)cd ef')
830 self.assertEqual(3, len(node.words))
831
832 def testNestedComSub(self):
833 node = assertParseSimpleCommand(self,
834 'echo $(one$(echo two)one) three')
835 self.assertEqual(3, len(node.words))
836
837 def testArithSubWithin(self):
838 # Within com sub
839 node = assertParseSimpleCommand(self,
840 'echo $(echo $((1+2)))')
841 self.assertEqual(command_e.SimpleCommand, node.tag)
842 self.assertEqual(2, len(node.words))
843
844 # Within subshell
845 node = assertParseCommandList(self,
846 '(echo $((1+2)))')
847 self.assertEqual(command_e.Subshell, node.tag)
848 self.assertEqual(command_e.SimpleCommand, node.child.tag)
849
850 def testArithGroupingWithin(self):
851 # Within com sub
852 node = assertParseSimpleCommand(self,
853 'echo $(echo $((1*(2+3))) )')
854 self.assertEqual(command_e.SimpleCommand, node.tag)
855 self.assertEqual(2, len(node.words))
856
857 # Within subshell
858 node = assertParseCommandList(self,
859 '(echo $((1*(2+3))) )')
860 self.assertEqual(command_e.Subshell, node.tag)
861 self.assertEqual(command_e.SimpleCommand, node.child.tag)
862
863 def testLhsArithGroupingWithin(self):
864 # Within Arith sub
865 node = assertParseSimpleCommand(self, 'echo $((a[1*(2+3)]=x))')
866 self.assertEqual(2, len(node.words))
867
868 # Within Command Sub -- NOT IMPLEMENTED
869 return
870 node = assertParseSimpleCommand(self, 'echo $(a[1*(2+3)]=x)')
871 self.assertEqual(2, len(node.words))
872
873 def testFuncDefWithin(self):
874 node = assertParseCommandList(self,
875 'echo $(func() { echo hi; }; func)')
876 self.assertEqual(command_e.SimpleCommand, node.tag)
877 self.assertEqual(2, len(node.words))
878
879 node = assertParseCommandList(self,
880 '(func() { echo hi; }; func)')
881 self.assertEqual(command_e.Subshell, node.tag)
882 self.assertEqual(command_e.CommandList, node.child.tag)
883
884 def testArrayLiteralWithin(self):
885 node = assertParseCommandList(self,
886 'echo $(array=(a b c))')
887 self.assertEqual(command_e.SimpleCommand, node.tag)
888 self.assertEqual(2, len(node.words))
889
890 node = assertParseCommandList(self,
891 '(array=(a b c))')
892 self.assertEqual(command_e.Subshell, node.tag)
893 self.assertEqual(command_e.Assignment, node.child.tag)
894
895 def testSubshellWithinComSub(self):
896 node = assertParseCommandList(self,
897 'echo one; echo $( (cd /; echo subshell_PWD); echo comsub_PWD); echo two')
898 self.assertEqual(command_e.CommandList, node.tag)
899 self.assertEqual(3, len(node.children)) # 3 echo statements
900
901 # TODO: Need a way to test the literal value of a word
902 #words = [w.UnquotedLiteralValue() for w in node.children[2].words]
903 #print(words)
904
905 def testCaseWithinComSub(self):
906 node = assertParseCommandList(self,
907 'echo $( case foo in one) echo comsub;; esac)')
908 self.assertEqual(2, len(node.words))
909
910 node = assertParseCommandList(self, """\
911echo $(
912case foo in one) echo comsub1;; esac
913case bar in two) echo comsub2;; esac
914)
915""")
916 self.assertEqual(2, len(node.words))
917
918 def testComsubWithinCaseWithinComSub(self):
919 # Comsub within case within comsub
920 node = assertParseCommandList(self,
921 'echo one; echo $( case one in $(echo one)) echo $(comsub);; esac ); echo two')
922 self.assertEqual(command_e.CommandList, node.tag)
923 # Top level should have 3 echo statements
924 self.assertEqual(3, len(node.children))
925
926 def testComSubWithinDoubleQuotes(self):
927 # CommandSubPart with two LiteralPart instances surrounding it
928 node = assertParseSimpleCommand(self,
929 'echo "double $(echo hi) quoted" two')
930 self.assertEqual(3, len(node.words))
931
932 def testEmptyCaseWithinSubshell(self):
933 node = assertParseCommandList(self, """\
934( case foo in
935 esac
936)
937""")
938 self.assertEqual(command_e.Subshell, node.tag)
939
940 def testBalancedCaseWithin(self):
941 # With leading ( in case. This one doesn't cause problems! We don't need
942 # the MaybeUnreadOne() lexer hack.
943 node = assertParseCommandList(self, """\
944$( case foo in
945 (one) echo hi ;;
946 esac
947)
948""")
949 self.assertEqual(command_e.SimpleCommand, node.tag)
950
951 node = assertParseCommandList(self, """\
952( case foo in
953 (one) echo hi ;;
954 esac
955)
956""")
957 self.assertEqual(command_e.Subshell, node.tag)
958
959 def testUnbalancedCaseWithin(self):
960 # With leading ( in case. This one doesn't cause problems! We don't need
961 # the MaybeUnreadOne() lexer hack.
962 node = assertParseCommandList(self, """\
963$( case foo in
964 one) echo hi ;;
965 esac
966)
967""")
968 self.assertEqual(command_e.SimpleCommand, node.tag)
969
970 node = assertParseCommandList(self, """\
971( case foo in
972 one) echo hi ;;
973 esac
974)
975""")
976 self.assertEqual(command_e.Subshell, node.tag)
977
978 def testForExpressionWithin(self):
979 # With leading ( in case. This one doesn't cause problems! We don't need
980 # the MaybeUnreadOne() lexer hack.
981 node = assertParseCommandList(self, """\
982$( for ((i=0; i<3; ++i)); do
983 echo hi
984 done
985)
986""")
987 self.assertEqual(command_e.SimpleCommand, node.tag)
988
989 node = assertParseCommandList(self, """\
990( for ((i=0; i<3; ++i)); do
991 echo hi
992 done
993)
994""")
995 self.assertEqual(command_e.Subshell, node.tag)
996
997
998class RealBugsTest(unittest.TestCase):
999
1000 def testGitBug(self):
1001 # Original bug from git codebase. Case in subshell.
1002 node = assertParseCommandList(self, """\
1003( cd "$PACKDIR" &&
1004 for e in $existing
1005 do
1006 case " $fullbases " in
1007 *" $e "*) ;;
1008 *) rm -f "$e.pack" "$e.idx" "$e.keep" ;;
1009 esac
1010 done
1011)
1012""")
1013 self.assertEqual(command_e.Subshell, node.tag)
1014
1015 def testParseCase3(self):
1016 # Bug from git codebase. NOT a comment token.
1017 node = assertParseCommandLine(self, """\
1018case "$fd,$command" in
1019 3,#*|3,)
1020 # copy comments
1021 ;;
1022esac
1023""")
1024 self.assertEqual(command_e.Case, node.tag)
1025
1026 def testGitComment(self):
1027 # ;# is a comment! Gah.
1028 # Conclusion: Comments are NOT LEXICAL. They are part of word parsing.
1029
1030 node = assertParseCommandList(self, """\
1031. "$TEST_DIRECTORY"/diff-lib.sh ;# test-lib chdir's into trash
1032""")
1033 self.assertEqual(command_e.Sentence, node.tag)
1034 self.assertEqual(2, len(node.child.words))
1035
1036 # This is NOT a comment
1037 node = assertParseCommandList(self, """\
1038echo foo#bar
1039""")
1040 self.assertEqual(command_e.SimpleCommand, node.tag)
1041 self.assertEqual(2, len(node.words))
1042 _, s, _ = word.StaticEval(node.words[1])
1043 self.assertEqual('foo#bar', s)
1044
1045 # This is a comment
1046 node = assertParseCommandList(self, """\
1047echo foo #comment
1048""")
1049 self.assertEqual(command_e.SimpleCommand, node.tag)
1050 self.assertEqual(2, len(node.words))
1051 _, s, _ = word.StaticEval(node.words[1])
1052 self.assertEqual('foo', s)
1053
1054 # Empty comment
1055 node = assertParseCommandList(self, """\
1056echo foo #
1057""")
1058 self.assertEqual(command_e.SimpleCommand, node.tag)
1059 self.assertEqual(2, len(node.words))
1060 _, s, _ = word.StaticEval(node.words[1])
1061 self.assertEqual('foo', s)
1062
1063 def testChromeIfSubshell(self):
1064 node = assertParseCommandList(self, """\
1065if true; then (
1066 echo hi
1067)
1068fi
1069""")
1070 self.assertEqual(command_e.If, node.tag)
1071
1072 node = assertParseCommandList(self, """\
1073while true; do {
1074 echo hi
1075 break
1076} done
1077""")
1078 self.assertEqual(command_e.While, node.tag)
1079
1080 node = assertParseCommandList(self, """\
1081if true; then (
1082 echo hi
1083) fi
1084""")
1085 self.assertEqual(command_e.If, node.tag)
1086
1087 # Related: two fi's in a row, found in Chrome configure. Compound commands
1088 # are special; don't need newlines.
1089 node = assertParseCommandList(self, """\
1090if true; then
1091 if true; then
1092 echo hi
1093 fi fi
1094echo hi
1095""")
1096 self.assertEqual(command_e.CommandList, node.tag)
1097
1098 def testBackticks(self):
1099 #return
1100
1101 # Another empty command sub
1102 node = assertParseCommandList(self, """\
1103echo $()
1104""")
1105
1106 # Simplest case
1107 node = assertParseCommandList(self, """\
1108echo ``
1109""")
1110
1111 # Found in the wild.
1112 # Just a comment trick found in sandstorm
1113 node = assertParseCommandList(self, """\
1114cmd \
1115 flag `# comment` \
1116 flag2
1117""")
1118
1119 # Empty should be allowed
1120 node = assertParseCommandList(self, """\
1121FOO="bar"`
1122 `"baz"
1123""")
1124
1125 def testQuineDb(self):
1126 # Need to handle the DOLLAR_SQ lex state
1127 node = assertParseCommandList(self, r"""\
1128case foo in
1129$'\'')
1130 ret+="\\"\'
1131 ;;
1132esac
1133""")
1134 self.assertEqual(command_e.Case, node.tag)
1135
1136 node = assertParseCommandList(self, r"""\
1137$'abc\ndef'
1138""")
1139 self.assertEqual(command_e.SimpleCommand, node.tag)
1140 self.assertEqual(1, len(node.words))
1141 w = node.words[0]
1142 self.assertEqual(1, len(w.parts))
1143 p = w.parts[0]
1144 self.assertEqual(3, len(p.tokens))
1145 self.assertEqual(Id.Char_Literals, p.tokens[0].id)
1146 self.assertEqual(Id.Char_OneChar, p.tokens[1].id)
1147 self.assertEqual(Id.Char_Literals, p.tokens[2].id)
1148
1149 def testArithConstants(self):
1150 # Found in Gherkin
1151 node = assertParseCommandList(self, r"""\
1152 [[ -n "${marks[${tag_marker}002${cons_ptr}]}" ]];
1153""")
1154 # Dynamic constant
1155 node = assertParseCommandList(self, r"""\
1156echo $(( 0x$foo ))
1157""")
1158
1159 def testBacktickCommentHack(self):
1160 # Found in sandstorm.
1161 # The problem here is that the comment goes to the end of the line, which
1162 # eats up the closing backtick! We could change the mode of the lexer
1163 # inside a command sub, or possibly just ignore this use case.
1164 return
1165
1166 node = assertParseCommandList(self, r"""\
1167openssl \
1168 -newkey rsa:4096 `# Create a new RSA key of length 4096 bits.` \
1169 `# Sandcats just needs the CN= (common name) in the request.` \
1170 -subj "/CN=*.${SS_HOSTNAME}/"
1171""")
1172
1173 def testArrayLiteralFromSetup(self):
1174 # Found in setup.shl/bin/setup -- this is the "Parsing Bash is
1175 # Undecidable" problem.
1176 err = _assertParseCommandListError(self, """\
1177errcmd=( "${SETUP_STATE[$err.cmd]}" )
1178""")
1179
1180 # Double quotes fix it.
1181 node = assertParseCommandList(self, r"""\
1182errcmd=( "${SETUP_STATE["$err.cmd"]}" )
1183""")
1184
1185
1186class ErrorLocationsTest(unittest.TestCase):
1187
1188 def testCommand(self):
1189 """Enumerating errors in cmd_parse.py."""
1190
1191 err = _assertParseCommandListError(self, 'ls <')
1192
1193 err = _assertParseCommandListError(self, 'ls < <')
1194
1195 # Invalid words as here docs
1196 err = _assertParseCommandListError(self, 'cat << $(invalid here end)')
1197
1198 # TODO: Arith parser doesn't have location information
1199 err = _assertParseCommandListError(self, 'cat << $((1+2))')
1200 err = _assertParseCommandListError(self, 'cat << a=(1 2 3)')
1201 err = _assertParseCommandListError(self, r'cat << \a$(invalid)')
1202
1203 # Actually the $invalid part should be highlighted... yeah an individual
1204 # part is the problem.
1205 err = _assertParseCommandListError(self, r"cat << 'single'$(invalid)")
1206 err = _assertParseCommandListError(self, r'cat << "double"$(invalid)')
1207 err = _assertParseCommandListError(self, r'cat << ~foo/$(invalid)')
1208 err = _assertParseCommandListError(self, r'cat << $var/$(invalid)')
1209
1210 # Word parse error in command parser
1211 err = _assertParseCommandListError(self, r'echo foo$(ls <)bar')
1212
1213 err = _assertParseCommandListError(self, r'BAD_ENV=(1 2 3) ls')
1214 err = _assertParseCommandListError(self, r'ls BAD_ENV=(1 2 3)')
1215 err = _assertParseCommandListError(self, r'ENV1=A ENV2=B local foo=bar')
1216
1217 # This needs more context
1218 err = _assertParseCommandListError(self,
1219 'for ((i=1; i<)); do echo $i; done')
1220
1221 err = _assertParseCommandListError(self,
1222 'for ((i=1; i<5; ++i)) OOPS echo $i; ERR')
1223
1224 # After semi
1225 err = _assertParseCommandListError(self,
1226 'for ((i=1; i<5; ++i)); OOPS echo $i; ERR')
1227
1228 err = _assertParseCommandListError(self,
1229 'for $bad in 1 2; do echo hi; done')
1230
1231 err = _assertParseCommandListError(self, 'for foo BAD')
1232
1233 err = _assertParseCommandListError(self, 'if foo; then echo hi; z')
1234
1235 err = _assertParseCommandListError(self, 'foo$(invalid) () { echo hi; }')
1236
1237 def testErrorInHereDoc(self):
1238 return
1239 # Here doc body. Hm this should be failing. Does it just fail to get
1240 # filled?
1241 err = _assertParseCommandListError(self, """cat <<EOF
1242$(echo <)
1243EOF
1244""")
1245 return
1246
1247 def testBool(self):
1248 """Enumerating errors in bool_parse.py."""
1249 err = _assertParseCommandListError(self, '[[ foo bar ]]')
1250 err = _assertParseCommandListError(self, '[[ foo -eq ]]')
1251
1252 # error in word
1253 err = _assertParseCommandListError(self, '[[ foo$(echo <) -eq foo ]]')
1254
1255 # Invalid regex
1256 err = _assertParseCommandListError(self, '[[ foo =~ \( ]]')
1257
1258 def testArith(self):
1259 """Enumerating errors in arith_parse.py."""
1260 err = _assertParseCommandListError(self, '(( 1 + ))')
1261
1262 def testArraySyntax(self):
1263 err = _assertParseCommandListError(self, 'A= (1 2)')
1264
1265 def testRedirectsInAssignment(self):
1266 err = _assertParseCommandListError(self, 'x=1 >/dev/null')
1267 err = _assertParseCommandListError(self, 'echo hi; x=1 >/dev/null')
1268 err = _assertParseCommandListError(self, 'declare x=1 >/dev/null')
1269
1270 def testEofInDoubleQuoted(self):
1271 err = _assertParseCommandListError(self, 'foo="" echo "bar ')
1272
1273 def testQuotesInFunctionName(self):
1274 err = _assertParseCommandListError(self, """\
1275 foo"bar" () {
1276 echo hi
1277 }
1278 """)
1279
1280 def testForLoopName(self):
1281 err = _assertParseCommandListError(self, """\
1282 for ( i = 1; i < 10; i++ )
1283 """)
1284 err = _assertParseCommandListError(self, """\
1285 for = in a
1286 """)
1287
1288 def testHereDocCommandSub(self):
1289 # Originally from spec/09-here-doc.sh.
1290 err = _assertParseCommandListError(self, """\
1291for x in 1 2 $(cat <<EOF
1292THREE
1293EOF); do
1294 echo for word $x
1295done
1296""")
1297
1298 def testForLoopEof(self):
1299 err = _assertParseCommandListError(self, "for x in 1 2 $(")
1300
1301
1302if __name__ == '__main__':
1303 unittest.main()