OILS / core / process.py View on Github | oilshell.org

1974 lines, 952 significant
1# Copyright 2016 Andy Chu. All rights reserved.
2# Licensed under the Apache License, Version 2.0 (the "License");
3# you may not use this file except in compliance with the License.
4# You may obtain a copy of the License at
5#
6# http://www.apache.org/licenses/LICENSE-2.0
7"""
8process.py - Launch processes and manipulate file descriptors.
9"""
10from __future__ import print_function
11
12from errno import EACCES, EBADF, ECHILD, EINTR, ENOENT, ENOEXEC
13import fcntl as fcntl_
14from fcntl import F_DUPFD, F_GETFD, F_SETFD, FD_CLOEXEC
15from signal import (SIG_DFL, SIG_IGN, SIGINT, SIGPIPE, SIGQUIT, SIGTSTP,
16 SIGTTOU, SIGTTIN, SIGWINCH)
17
18from _devbuild.gen.id_kind_asdl import Id
19from _devbuild.gen.runtime_asdl import (job_state_e, job_state_t,
20 job_state_str, wait_status,
21 wait_status_t, RedirValue,
22 redirect_arg, redirect_arg_e, trace,
23 trace_t)
24from _devbuild.gen.syntax_asdl import (
25 loc_t,
26 redir_loc,
27 redir_loc_e,
28 redir_loc_t,
29)
30from _devbuild.gen.value_asdl import (value, value_e)
31from core import dev
32from core import error
33from core.error import e_die
34from core import pyutil
35from core import pyos
36from core import state
37from core import ui
38from core import util
39from data_lang import j8_lite
40from frontend import location
41from frontend import match
42from mycpp import mylib
43from mycpp.mylib import log, print_stderr, probe, tagswitch, iteritems
44
45import posix_ as posix
46from posix_ import (
47 # translated by mycpp and directly called! No wrapper!
48 WIFSIGNALED,
49 WIFEXITED,
50 WIFSTOPPED,
51 WEXITSTATUS,
52 WSTOPSIG,
53 WTERMSIG,
54 WNOHANG,
55 O_APPEND,
56 O_CREAT,
57 O_NONBLOCK,
58 O_NOCTTY,
59 O_RDONLY,
60 O_RDWR,
61 O_WRONLY,
62 O_TRUNC,
63)
64
65from typing import IO, List, Tuple, Dict, Optional, Any, cast, TYPE_CHECKING
66
67if TYPE_CHECKING:
68 from _devbuild.gen.runtime_asdl import cmd_value
69 from _devbuild.gen.syntax_asdl import command_t
70 from builtin import trap_osh
71 from core import optview
72 from core.ui import ErrorFormatter
73 from core.util import _DebugFile
74 from osh.cmd_eval import CommandEvaluator
75
76NO_FD = -1
77
78# Minimum file descriptor that the shell can use. Other descriptors can be
79# directly used by user programs, e.g. exec 9>&1
80#
81# Oil uses 100 because users are allowed TWO digits in frontend/lexer_def.py.
82# This is a compromise between bash (unlimited, but requires crazy
83# bookkeeping), and dash/zsh (10) and mksh (24)
84_SHELL_MIN_FD = 100
85
86# Style for 'jobs' builtin
87STYLE_DEFAULT = 0
88STYLE_LONG = 1
89STYLE_PID_ONLY = 2
90
91# To save on allocations in JobList::GetJobWithSpec()
92CURRENT_JOB_SPECS = ['', '%', '%%', '%+']
93
94
95class ctx_FileCloser(object):
96
97 def __init__(self, f):
98 # type: (mylib.LineReader) -> None
99 self.f = f
100
101 def __enter__(self):
102 # type: () -> None
103 pass
104
105 def __exit__(self, type, value, traceback):
106 # type: (Any, Any, Any) -> None
107 self.f.close()
108
109
110def InitInteractiveShell():
111 # type: () -> None
112 """Called when initializing an interactive shell."""
113
114 # The shell itself should ignore Ctrl-\.
115 pyos.Sigaction(SIGQUIT, SIG_IGN)
116
117 # This prevents Ctrl-Z from suspending OSH in interactive mode.
118 pyos.Sigaction(SIGTSTP, SIG_IGN)
119
120 # More signals from
121 # https://www.gnu.org/software/libc/manual/html_node/Initializing-the-Shell.html
122 # (but not SIGCHLD)
123 pyos.Sigaction(SIGTTOU, SIG_IGN)
124 pyos.Sigaction(SIGTTIN, SIG_IGN)
125
126 # Register a callback to receive terminal width changes.
127 # NOTE: In line_input.c, we turned off rl_catch_sigwinch.
128
129 # This is ALWAYS on, which means that it can cause EINTR, and wait() and
130 # read() have to handle it
131 pyos.RegisterSignalInterest(SIGWINCH)
132
133
134def SaveFd(fd):
135 # type: (int) -> int
136 saved = fcntl_.fcntl(fd, F_DUPFD, _SHELL_MIN_FD) # type: int
137 return saved
138
139
140class _RedirFrame(object):
141
142 def __init__(self, saved_fd, orig_fd, forget):
143 # type: (int, int, bool) -> None
144 self.saved_fd = saved_fd
145 self.orig_fd = orig_fd
146 self.forget = forget
147
148
149class _FdFrame(object):
150
151 def __init__(self):
152 # type: () -> None
153 self.saved = [] # type: List[_RedirFrame]
154 self.need_wait = [] # type: List[Process]
155
156 def Forget(self):
157 # type: () -> None
158 """For exec 1>&2."""
159 for rf in reversed(self.saved):
160 if rf.saved_fd != NO_FD and rf.forget:
161 posix.close(rf.saved_fd)
162
163 del self.saved[:] # like list.clear() in Python 3.3
164 del self.need_wait[:]
165
166 def __repr__(self):
167 # type: () -> str
168 return '<_FdFrame %s>' % self.saved
169
170
171class FdState(object):
172 """File descriptor state for the current process.
173
174 For example, you can do 'myfunc > out.txt' without forking. Child
175 processes inherit our state.
176 """
177
178 def __init__(
179 self,
180 errfmt, # type: ui.ErrorFormatter
181 job_control, # type: JobControl
182 job_list, # type: JobList
183 mem, #type: state.Mem
184 tracer, # type: Optional[dev.Tracer]
185 waiter, # type: Optional[Waiter]
186 ):
187 # type: (...) -> None
188 """
189 Args:
190 errfmt: for errors
191 job_list: For keeping track of _HereDocWriterThunk
192 """
193 self.errfmt = errfmt
194 self.job_control = job_control
195 self.job_list = job_list
196 self.cur_frame = _FdFrame() # for the top level
197 self.stack = [self.cur_frame]
198 self.mem = mem
199 self.tracer = tracer
200 self.waiter = waiter
201
202 def Open(self, path):
203 # type: (str) -> mylib.LineReader
204 """Opens a path for read, but moves it out of the reserved 3-9 fd
205 range.
206
207 Returns:
208 A Python file object. The caller is responsible for Close().
209
210 Raises:
211 IOError or OSError if the path can't be found. (This is Python-induced wart)
212 """
213 fd_mode = O_RDONLY
214 f = self._Open(path, 'r', fd_mode)
215
216 # Hacky downcast
217 return cast('mylib.LineReader', f)
218
219 # used for util.DebugFile
220 def OpenForWrite(self, path):
221 # type: (str) -> mylib.Writer
222 fd_mode = O_CREAT | O_RDWR
223 f = self._Open(path, 'w', fd_mode)
224
225 # Hacky downcast
226 return cast('mylib.Writer', f)
227
228 def _Open(self, path, c_mode, fd_mode):
229 # type: (str, str, int) -> IO[str]
230 fd = posix.open(path, fd_mode, 0o666) # may raise OSError
231
232 # Immediately move it to a new location
233 new_fd = SaveFd(fd)
234 posix.close(fd)
235
236 # Return a Python file handle
237 f = posix.fdopen(new_fd, c_mode) # may raise IOError
238 return f
239
240 def _WriteFdToMem(self, fd_name, fd):
241 # type: (str, int) -> None
242 if self.mem:
243 # setvar, not setref
244 state.OshLanguageSetValue(self.mem, location.LName(fd_name),
245 value.Str(str(fd)))
246
247 def _ReadFdFromMem(self, fd_name):
248 # type: (str) -> int
249 val = self.mem.GetValue(fd_name)
250 if val.tag() == value_e.Str:
251 try:
252 return int(cast(value.Str, val).s)
253 except ValueError:
254 return NO_FD
255 return NO_FD
256
257 def _PushSave(self, fd):
258 # type: (int) -> bool
259 """Save fd to a new location and remember to restore it later."""
260 #log('---- _PushSave %s', fd)
261 ok = True
262 try:
263 new_fd = SaveFd(fd)
264 except (IOError, OSError) as e:
265 ok = False
266 # Example program that causes this error: exec 4>&1. Descriptor 4 isn't
267 # open.
268 # This seems to be ignored in dash too in savefd()?
269 if e.errno != EBADF:
270 raise
271 if ok:
272 posix.close(fd)
273 fcntl_.fcntl(new_fd, F_SETFD, FD_CLOEXEC)
274 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, True))
275 else:
276 # if we got EBADF, we still need to close the original on Pop()
277 self._PushClose(fd)
278
279 return ok
280
281 def _PushDup(self, fd1, blame_loc):
282 # type: (int, redir_loc_t) -> int
283 """Save fd2 in a higher range, and dup fd1 onto fd2.
284
285 Returns whether F_DUPFD/dup2 succeeded, and the new descriptor.
286 """
287 UP_loc = blame_loc
288 if blame_loc.tag() == redir_loc_e.VarName:
289 fd2_name = cast(redir_loc.VarName, UP_loc).name
290 try:
291 # F_DUPFD: GREATER than range
292 new_fd = fcntl_.fcntl(fd1, F_DUPFD, _SHELL_MIN_FD) # type: int
293 except (IOError, OSError) as e:
294 if e.errno == EBADF:
295 print_stderr('F_DUPFD fd %d: %s' %
296 (fd1, pyutil.strerror(e)))
297 return NO_FD
298 else:
299 raise # this redirect failed
300
301 self._WriteFdToMem(fd2_name, new_fd)
302
303 elif blame_loc.tag() == redir_loc_e.Fd:
304 fd2 = cast(redir_loc.Fd, UP_loc).fd
305
306 if fd1 == fd2:
307 # The user could have asked for it to be open on descriptor 3, but open()
308 # already returned 3, e.g. echo 3>out.txt
309 return NO_FD
310
311 # Check the validity of fd1 before _PushSave(fd2)
312 try:
313 fcntl_.fcntl(fd1, F_GETFD)
314 except (IOError, OSError) as e:
315 print_stderr('F_GETFD fd %d: %s' % (fd1, pyutil.strerror(e)))
316 raise
317
318 need_restore = self._PushSave(fd2)
319
320 #log('==== dup2 %s %s\n' % (fd1, fd2))
321 try:
322 posix.dup2(fd1, fd2)
323 except (IOError, OSError) as e:
324 # bash/dash give this error too, e.g. for 'echo hi 1>&3'
325 print_stderr('dup2(%d, %d): %s' %
326 (fd1, fd2, pyutil.strerror(e)))
327
328 # Restore and return error
329 if need_restore:
330 rf = self.cur_frame.saved.pop()
331 posix.dup2(rf.saved_fd, rf.orig_fd)
332 posix.close(rf.saved_fd)
333
334 raise # this redirect failed
335
336 new_fd = fd2
337
338 else:
339 raise AssertionError()
340
341 return new_fd
342
343 def _PushCloseFd(self, blame_loc):
344 # type: (redir_loc_t) -> bool
345 """For 2>&-"""
346 # exec {fd}>&- means close the named descriptor
347
348 UP_loc = blame_loc
349 if blame_loc.tag() == redir_loc_e.VarName:
350 fd_name = cast(redir_loc.VarName, UP_loc).name
351 fd = self._ReadFdFromMem(fd_name)
352 if fd == NO_FD:
353 return False
354
355 elif blame_loc.tag() == redir_loc_e.Fd:
356 fd = cast(redir_loc.Fd, UP_loc).fd
357
358 else:
359 raise AssertionError()
360
361 self._PushSave(fd)
362
363 return True
364
365 def _PushClose(self, fd):
366 # type: (int) -> None
367 self.cur_frame.saved.append(_RedirFrame(NO_FD, fd, False))
368
369 def _PushWait(self, proc):
370 # type: (Process) -> None
371 self.cur_frame.need_wait.append(proc)
372
373 def _ApplyRedirect(self, r):
374 # type: (RedirValue) -> None
375 arg = r.arg
376 UP_arg = arg
377 with tagswitch(arg) as case:
378
379 if case(redirect_arg_e.Path):
380 arg = cast(redirect_arg.Path, UP_arg)
381
382 if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &>
383 # NOTE: This is different than >| because it respects noclobber, but
384 # that option is almost never used. See test/wild.sh.
385 mode = O_CREAT | O_WRONLY | O_TRUNC
386 elif r.op_id == Id.Redir_Clobber: # >|
387 mode = O_CREAT | O_WRONLY | O_TRUNC
388 elif r.op_id in (Id.Redir_DGreat,
389 Id.Redir_AndDGreat): # >> &>>
390 mode = O_CREAT | O_WRONLY | O_APPEND
391 elif r.op_id == Id.Redir_Less: # <
392 mode = O_RDONLY
393 elif r.op_id == Id.Redir_LessGreat: # <>
394 mode = O_CREAT | O_RDWR
395 else:
396 raise NotImplementedError(r.op_id)
397
398 # NOTE: 0666 is affected by umask, all shells use it.
399 try:
400 open_fd = posix.open(arg.filename, mode, 0o666)
401 except (IOError, OSError) as e:
402 self.errfmt.Print_("Can't open %r: %s" %
403 (arg.filename, pyutil.strerror(e)),
404 blame_loc=r.op_loc)
405 raise # redirect failed
406
407 new_fd = self._PushDup(open_fd, r.loc)
408 if new_fd != NO_FD:
409 posix.close(open_fd)
410
411 # Now handle &> and &>> and their variants. These pairs are the same:
412 #
413 # stdout_stderr.py &> out-err.txt
414 # stdout_stderr.py > out-err.txt 2>&1
415 #
416 # stdout_stderr.py 3&> out-err.txt
417 # stdout_stderr.py 3> out-err.txt 2>&3
418 #
419 # Ditto for {fd}> and {fd}&>
420
421 if r.op_id in (Id.Redir_AndGreat, Id.Redir_AndDGreat):
422 self._PushDup(new_fd, redir_loc.Fd(2))
423
424 elif case(redirect_arg_e.CopyFd): # e.g. echo hi 1>&2
425 arg = cast(redirect_arg.CopyFd, UP_arg)
426
427 if r.op_id == Id.Redir_GreatAnd: # 1>&2
428 self._PushDup(arg.target_fd, r.loc)
429
430 elif r.op_id == Id.Redir_LessAnd: # 0<&5
431 # The only difference between >& and <& is the default file
432 # descriptor argument.
433 self._PushDup(arg.target_fd, r.loc)
434
435 else:
436 raise NotImplementedError()
437
438 elif case(redirect_arg_e.MoveFd): # e.g. echo hi 5>&6-
439 arg = cast(redirect_arg.MoveFd, UP_arg)
440 new_fd = self._PushDup(arg.target_fd, r.loc)
441 if new_fd != NO_FD:
442 posix.close(arg.target_fd)
443
444 UP_loc = r.loc
445 if r.loc.tag() == redir_loc_e.Fd:
446 fd = cast(redir_loc.Fd, UP_loc).fd
447 else:
448 fd = NO_FD
449
450 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, False))
451
452 elif case(redirect_arg_e.CloseFd): # e.g. echo hi 5>&-
453 self._PushCloseFd(r.loc)
454
455 elif case(redirect_arg_e.HereDoc):
456 arg = cast(redirect_arg.HereDoc, UP_arg)
457
458 # NOTE: Do these descriptors have to be moved out of the range 0-9?
459 read_fd, write_fd = posix.pipe()
460
461 self._PushDup(read_fd, r.loc) # stdin is now the pipe
462
463 # We can't close like we do in the filename case above? The writer can
464 # get a "broken pipe".
465 self._PushClose(read_fd)
466
467 thunk = _HereDocWriterThunk(write_fd, arg.body)
468
469 # Use PIPE_SIZE to save a process in the case of small here
470 # docs, which are the common case. (dash does this.)
471
472 # Note: could instrument this to see how often it happens.
473 # Though strace -ff can also work.
474 start_process = len(arg.body) > 4096
475 #start_process = True
476
477 if start_process:
478 here_proc = Process(thunk, self.job_control, self.job_list,
479 self.tracer)
480
481 # NOTE: we could close the read pipe here, but it doesn't really
482 # matter because we control the code.
483 here_proc.StartProcess(trace.HereDoc)
484 #log('Started %s as %d', here_proc, pid)
485 self._PushWait(here_proc)
486
487 # Now that we've started the child, close it in the parent.
488 posix.close(write_fd)
489
490 else:
491 posix.write(write_fd, arg.body)
492 posix.close(write_fd)
493
494 def Push(self, redirects, err_out):
495 # type: (List[RedirValue], List[error.IOError_OSError]) -> None
496 """Apply a group of redirects and remember to undo them."""
497
498 #log('> fd_state.Push %s', redirects)
499 new_frame = _FdFrame()
500 self.stack.append(new_frame)
501 self.cur_frame = new_frame
502
503 for r in redirects:
504 #log('apply %s', r)
505 with ui.ctx_Location(self.errfmt, r.op_loc):
506 try:
507 self._ApplyRedirect(r)
508 except (IOError, OSError) as e:
509 err_out.append(e)
510 # This can fail too
511 self.Pop(err_out)
512 return # for bad descriptor, etc.
513
514 def PushStdinFromPipe(self, r):
515 # type: (int) -> bool
516 """Save the current stdin and make it come from descriptor 'r'.
517
518 'r' is typically the read-end of a pipe. For 'lastpipe'/ZSH
519 semantics of
520
521 echo foo | read line; echo $line
522 """
523 new_frame = _FdFrame()
524 self.stack.append(new_frame)
525 self.cur_frame = new_frame
526
527 self._PushDup(r, redir_loc.Fd(0))
528 return True
529
530 def Pop(self, err_out):
531 # type: (List[error.IOError_OSError]) -> None
532 frame = self.stack.pop()
533 #log('< Pop %s', frame)
534 for rf in reversed(frame.saved):
535 if rf.saved_fd == NO_FD:
536 #log('Close %d', orig)
537 try:
538 posix.close(rf.orig_fd)
539 except (IOError, OSError) as e:
540 err_out.append(e)
541 log('Error closing descriptor %d: %s', rf.orig_fd,
542 pyutil.strerror(e))
543 return
544 else:
545 try:
546 posix.dup2(rf.saved_fd, rf.orig_fd)
547 except (IOError, OSError) as e:
548 err_out.append(e)
549 log('dup2(%d, %d) error: %s', rf.saved_fd, rf.orig_fd,
550 pyutil.strerror(e))
551 #log('fd state:')
552 #posix.system('ls -l /proc/%s/fd' % posix.getpid())
553 return
554 posix.close(rf.saved_fd)
555 #log('dup2 %s %s', saved, orig)
556
557 # Wait for here doc processes to finish.
558 for proc in frame.need_wait:
559 unused_status = proc.Wait(self.waiter)
560
561 def MakePermanent(self):
562 # type: () -> None
563 self.cur_frame.Forget()
564
565
566class ChildStateChange(object):
567
568 def __init__(self):
569 # type: () -> None
570 """Empty constructor for mycpp."""
571 pass
572
573 def Apply(self):
574 # type: () -> None
575 raise NotImplementedError()
576
577 def ApplyFromParent(self, proc):
578 # type: (Process) -> None
579 """Noop for all state changes other than SetPgid for mycpp."""
580 pass
581
582
583class StdinFromPipe(ChildStateChange):
584
585 def __init__(self, pipe_read_fd, w):
586 # type: (int, int) -> None
587 self.r = pipe_read_fd
588 self.w = w
589
590 def __repr__(self):
591 # type: () -> str
592 return '<StdinFromPipe %d %d>' % (self.r, self.w)
593
594 def Apply(self):
595 # type: () -> None
596 posix.dup2(self.r, 0)
597 posix.close(self.r) # close after dup
598
599 posix.close(self.w) # we're reading from the pipe, not writing
600 #log('child CLOSE w %d pid=%d', self.w, posix.getpid())
601
602
603class StdoutToPipe(ChildStateChange):
604
605 def __init__(self, r, pipe_write_fd):
606 # type: (int, int) -> None
607 self.r = r
608 self.w = pipe_write_fd
609
610 def __repr__(self):
611 # type: () -> str
612 return '<StdoutToPipe %d %d>' % (self.r, self.w)
613
614 def Apply(self):
615 # type: () -> None
616 posix.dup2(self.w, 1)
617 posix.close(self.w) # close after dup
618
619 posix.close(self.r) # we're writing to the pipe, not reading
620 #log('child CLOSE r %d pid=%d', self.r, posix.getpid())
621
622
623INVALID_PGID = -1
624# argument to setpgid() that means the process is its own leader
625OWN_LEADER = 0
626
627
628class SetPgid(ChildStateChange):
629
630 def __init__(self, pgid, tracer):
631 # type: (int, dev.Tracer) -> None
632 self.pgid = pgid
633 self.tracer = tracer
634
635 def Apply(self):
636 # type: () -> None
637 try:
638 posix.setpgid(0, self.pgid)
639 except (IOError, OSError) as e:
640 self.tracer.OtherMessage(
641 'osh: child %d failed to set its process group to %d: %s' %
642 (posix.getpid(), self.pgid, pyutil.strerror(e)))
643
644 def ApplyFromParent(self, proc):
645 # type: (Process) -> None
646 try:
647 posix.setpgid(proc.pid, self.pgid)
648 except (IOError, OSError) as e:
649 self.tracer.OtherMessage(
650 'osh: parent failed to set process group for PID %d to %d: %s'
651 % (proc.pid, self.pgid, pyutil.strerror(e)))
652
653
654class ExternalProgram(object):
655 """The capability to execute an external program like 'ls'."""
656
657 def __init__(
658 self,
659 hijack_shebang, # type: str
660 fd_state, # type: FdState
661 errfmt, # type: ErrorFormatter
662 debug_f, # type: _DebugFile
663 ):
664 # type: (...) -> None
665 """
666 Args:
667 hijack_shebang: The path of an interpreter to run instead of the one
668 specified in the shebang line. May be empty.
669 """
670 self.hijack_shebang = hijack_shebang
671 self.fd_state = fd_state
672 self.errfmt = errfmt
673 self.debug_f = debug_f
674
675 def Exec(self, argv0_path, cmd_val, environ):
676 # type: (str, cmd_value.Argv, Dict[str, str]) -> None
677 """Execute a program and exit this process.
678
679 Called by: ls / exec ls / ( ls / )
680 """
681 probe('process', 'ExternalProgram_Exec', argv0_path)
682 self._Exec(argv0_path, cmd_val.argv, cmd_val.arg_locs[0], environ,
683 True)
684 assert False, "This line should never execute" # NO RETURN
685
686 def _Exec(self, argv0_path, argv, argv0_loc, environ, should_retry):
687 # type: (str, List[str], loc_t, Dict[str, str], bool) -> None
688 if len(self.hijack_shebang):
689 opened = True
690 try:
691 f = self.fd_state.Open(argv0_path)
692 except (IOError, OSError) as e:
693 opened = False
694
695 if opened:
696 with ctx_FileCloser(f):
697 # Test if the shebang looks like a shell. TODO: The file might be
698 # binary with no newlines, so read 80 bytes instead of readline().
699
700 #line = f.read(80) # type: ignore # TODO: fix this
701 line = f.readline()
702
703 if match.ShouldHijack(line):
704 h_argv = [self.hijack_shebang, argv0_path]
705 h_argv.extend(argv[1:])
706 argv = h_argv
707 argv0_path = self.hijack_shebang
708 self.debug_f.writeln('Hijacked: %s' % argv0_path)
709 else:
710 #self.debug_f.log('Not hijacking %s (%r)', argv, line)
711 pass
712
713 try:
714 posix.execve(argv0_path, argv, environ)
715 except (IOError, OSError) as e:
716 # Run with /bin/sh when ENOEXEC error (no shebang). All shells do this.
717 if e.errno == ENOEXEC and should_retry:
718 new_argv = ['/bin/sh', argv0_path]
719 new_argv.extend(argv[1:])
720 self._Exec('/bin/sh', new_argv, argv0_loc, environ, False)
721 # NO RETURN
722
723 # Would be nice: when the path is relative and ENOENT: print PWD and do
724 # spelling correction?
725
726 self.errfmt.Print_(
727 "Can't execute %r: %s" % (argv0_path, pyutil.strerror(e)),
728 argv0_loc)
729
730 # POSIX mentions 126 and 127 for two specific errors. The rest are
731 # unspecified.
732 #
733 # http://pubs.opengroup.org/onlinepubs/9699919799.2016edition/utilities/V3_chap02.html#tag_18_08_02
734 if e.errno == EACCES:
735 status = 126
736 elif e.errno == ENOENT:
737 # TODO: most shells print 'command not found', rather than strerror()
738 # == "No such file or directory". That's better because it's at the
739 # end of the path search, and we're never searching for a directory.
740 status = 127
741 else:
742 # dash uses 2, but we use that for parse errors. This seems to be
743 # consistent with mksh and zsh.
744 status = 127
745
746 posix._exit(status)
747 # NO RETURN
748
749
750class Thunk(object):
751 """Abstract base class for things runnable in another process."""
752
753 def __init__(self):
754 # type: () -> None
755 """Empty constructor for mycpp."""
756 pass
757
758 def Run(self):
759 # type: () -> None
760 """Returns a status code."""
761 raise NotImplementedError()
762
763 def UserString(self):
764 # type: () -> str
765 """Display for the 'jobs' list."""
766 raise NotImplementedError()
767
768 def SkipTrace(self):
769 # type: () -> bool
770 """Returns true if the caller can skip the tracer"""
771 raise NotImplementedError()
772
773 def __repr__(self):
774 # type: () -> str
775 return self.UserString()
776
777
778class ExternalThunk(Thunk):
779 """An external executable."""
780
781 def __init__(self, ext_prog, argv0_path, cmd_val, environ):
782 # type: (ExternalProgram, str, cmd_value.Argv, Dict[str, str]) -> None
783 self.ext_prog = ext_prog
784 self.argv0_path = argv0_path
785 self.cmd_val = cmd_val
786 self.environ = environ
787
788 def UserString(self):
789 # type: () -> str
790
791 # NOTE: This is the format the Tracer uses.
792 # bash displays sleep $n & (code)
793 # but OSH displays sleep 1 & (argv array)
794 # We could switch the former but I'm not sure it's necessary.
795 tmp = [j8_lite.MaybeShellEncode(a) for a in self.cmd_val.argv]
796 return '[process] %s' % ' '.join(tmp)
797
798 def SkipTrace(self):
799 # type: () -> bool
800 """Returns true if the caller can skip the tracer"""
801 return True
802
803 def Run(self):
804 # type: () -> None
805 """An ExternalThunk is run in parent for the exec builtin."""
806 self.ext_prog.Exec(self.argv0_path, self.cmd_val, self.environ)
807
808
809class SubProgramThunk(Thunk):
810 """A subprogram that can be executed in another process."""
811
812 def __init__(self,
813 cmd_ev,
814 node,
815 trap_state,
816 multi_trace,
817 inherit_errexit=True):
818 # type: (CommandEvaluator, command_t, trap_osh.TrapState, dev.MultiTracer, bool) -> None
819 self.cmd_ev = cmd_ev
820 self.node = node
821 self.trap_state = trap_state
822 self.multi_trace = multi_trace
823 self.inherit_errexit = inherit_errexit # for bash errexit compatibility
824
825 def UserString(self):
826 # type: () -> str
827
828 # NOTE: These can be pieces of a pipeline, so they're arbitrary nodes.
829 # TODO: Extract SPIDS from node to display source? Note that
830 # CompoundStatus also has locations of each pipeline component; see
831 # Executor.RunPipeline()
832 thunk_str = ui.CommandType(self.node)
833 return '[subprog] %s' % thunk_str
834
835 def SkipTrace(self):
836 # type: () -> bool
837 """Returns true if the caller can skip the tracer"""
838 return False
839
840 def Run(self):
841 # type: () -> None
842 #self.errfmt.OneLineErrExit() # don't quote code in child processes
843 probe('process', 'SubProgramThunk_Run')
844
845 # TODO: break circular dep. Bit flags could go in ASDL or headers.
846 from osh import cmd_eval
847
848 # signal handlers aren't inherited
849 self.trap_state.ClearForSubProgram()
850
851 # NOTE: may NOT return due to exec().
852 if not self.inherit_errexit:
853 self.cmd_ev.mutable_opts.DisableErrExit()
854 try:
855 # optimize to eliminate redundant subshells like ( echo hi ) | wc -l etc.
856 self.cmd_ev.ExecuteAndCatch(self.node, cmd_flags=cmd_eval.Optimize)
857 status = self.cmd_ev.LastStatus()
858 # NOTE: We ignore the is_fatal return value. The user should set -o
859 # errexit so failures in subprocesses cause failures in the parent.
860 except util.UserExit as e:
861 status = e.status
862
863 # Handle errors in a subshell. These two cases are repeated from main()
864 # and the core/completion.py hook.
865 except KeyboardInterrupt:
866 print('')
867 status = 130 # 128 + 2
868 except (IOError, OSError) as e:
869 print_stderr('oils I/O error (subprogram): %s' %
870 pyutil.strerror(e))
871 status = 2
872
873 # If ProcessInit() doesn't turn off buffering, this is needed before
874 # _exit()
875 pyos.FlushStdout()
876
877 self.multi_trace.WriteDumps()
878
879 # We do NOT want to raise SystemExit here. Otherwise dev.Tracer::Pop()
880 # gets called in BOTH processes.
881 # The crash dump seems to be unaffected.
882 posix._exit(status)
883
884
885class _HereDocWriterThunk(Thunk):
886 """Write a here doc to one end of a pipe.
887
888 May be be executed in either a child process or the main shell
889 process.
890 """
891
892 def __init__(self, w, body_str):
893 # type: (int, str) -> None
894 self.w = w
895 self.body_str = body_str
896
897 def UserString(self):
898 # type: () -> str
899
900 # You can hit Ctrl-Z and the here doc writer will be suspended! Other
901 # shells don't have this problem because they use temp files! That's a bit
902 # unfortunate.
903 return '[here doc writer]'
904
905 def SkipTrace(self):
906 # type: () -> bool
907 """Returns true if the caller can skip the tracer"""
908 return True
909
910 def Run(self):
911 # type: () -> None
912 """do_exit: For small pipelines."""
913 probe('process', 'HereDocWriterThunk_Run')
914 #log('Writing %r', self.body_str)
915 posix.write(self.w, self.body_str)
916 #log('Wrote %r', self.body_str)
917 posix.close(self.w)
918 #log('Closed %d', self.w)
919
920 posix._exit(0)
921
922
923class Job(object):
924 """Interface for both Process and Pipeline.
925
926 They both can be put in the background and waited on.
927
928 Confusing thing about pipelines in the background: They have TOO MANY NAMES.
929
930 sleep 1 | sleep 2 &
931
932 - The LAST PID is what's printed at the prompt. This is $!, a PROCESS ID and
933 not a JOB ID.
934 # https://www.gnu.org/software/bash/manual/html_node/Special-Parameters.html#Special-Parameters
935 - The process group leader (setpgid) is the FIRST PID.
936 - It's also %1 or %+. The last job started.
937 """
938
939 def __init__(self):
940 # type: () -> None
941 # Initial state with & or Ctrl-Z is Running.
942 self.state = job_state_e.Running
943 self.job_id = -1
944 self.in_background = False
945
946 def DisplayJob(self, job_id, f, style):
947 # type: (int, mylib.Writer, int) -> None
948 raise NotImplementedError()
949
950 def State(self):
951 # type: () -> job_state_t
952 return self.state
953
954 def ProcessGroupId(self):
955 # type: () -> int
956 """Return the process group ID associated with this job."""
957 raise NotImplementedError()
958
959 def JobWait(self, waiter):
960 # type: (Waiter) -> wait_status_t
961 """Wait for this process/pipeline to be stopped or finished."""
962 raise NotImplementedError()
963
964 def SetBackground(self):
965 # type: () -> None
966 """Record that this job is running in the background."""
967 self.in_background = True
968
969 def SetForeground(self):
970 # type: () -> None
971 """Record that this job is running in the foreground."""
972 self.in_background = False
973
974
975class Process(Job):
976 """A process to run.
977
978 TODO: Should we make it clear that this is a FOREGROUND process? A
979 background process is wrapped in a "job". It is unevaluated.
980
981 It provides an API to manipulate file descriptor state in parent and child.
982 """
983
984 def __init__(self, thunk, job_control, job_list, tracer):
985 # type: (Thunk, JobControl, JobList, dev.Tracer) -> None
986 """
987 Args:
988 thunk: Thunk instance
989 job_list: for process bookkeeping
990 """
991 Job.__init__(self)
992 assert isinstance(thunk, Thunk), thunk
993 self.thunk = thunk
994 self.job_control = job_control
995 self.job_list = job_list
996 self.tracer = tracer
997
998 # For pipelines
999 self.parent_pipeline = None # type: Pipeline
1000 self.state_changes = [] # type: List[ChildStateChange]
1001 self.close_r = -1
1002 self.close_w = -1
1003
1004 self.pid = -1
1005 self.status = -1
1006
1007 def Init_ParentPipeline(self, pi):
1008 # type: (Pipeline) -> None
1009 """For updating PIPESTATUS."""
1010 self.parent_pipeline = pi
1011
1012 def __repr__(self):
1013 # type: () -> str
1014
1015 # note: be wary of infinite mutual recursion
1016 #s = ' %s' % self.parent_pipeline if self.parent_pipeline else ''
1017 #return '<Process %s%s>' % (self.thunk, s)
1018 return '<Process %s %s>' % (_JobStateStr(self.state), self.thunk)
1019
1020 def ProcessGroupId(self):
1021 # type: () -> int
1022 """Returns the group ID of this process."""
1023 # This should only ever be called AFTER the process has started
1024 assert self.pid != -1
1025 if self.parent_pipeline:
1026 # XXX: Maybe we should die here instead? Unclear if this branch
1027 # should even be reachable with the current builtins.
1028 return self.parent_pipeline.ProcessGroupId()
1029
1030 return self.pid
1031
1032 def DisplayJob(self, job_id, f, style):
1033 # type: (int, mylib.Writer, int) -> None
1034 if job_id == -1:
1035 job_id_str = ' '
1036 else:
1037 job_id_str = '%%%d' % job_id
1038 if style == STYLE_PID_ONLY:
1039 f.write('%d\n' % self.pid)
1040 else:
1041 f.write('%s %d %7s ' %
1042 (job_id_str, self.pid, _JobStateStr(self.state)))
1043 f.write(self.thunk.UserString())
1044 f.write('\n')
1045
1046 def AddStateChange(self, s):
1047 # type: (ChildStateChange) -> None
1048 self.state_changes.append(s)
1049
1050 def AddPipeToClose(self, r, w):
1051 # type: (int, int) -> None
1052 self.close_r = r
1053 self.close_w = w
1054
1055 def MaybeClosePipe(self):
1056 # type: () -> None
1057 if self.close_r != -1:
1058 posix.close(self.close_r)
1059 posix.close(self.close_w)
1060
1061 def StartProcess(self, why):
1062 # type: (trace_t) -> int
1063 """Start this process with fork(), handling redirects."""
1064 pid = posix.fork()
1065 if pid < 0:
1066 # When does this happen?
1067 e_die('Fatal error in posix.fork()')
1068
1069 elif pid == 0: # child
1070 # Note: this happens in BOTH interactive and non-interactive shells.
1071 # We technically don't need to do most of it in non-interactive, since we
1072 # did not change state in InitInteractiveShell().
1073
1074 for st in self.state_changes:
1075 st.Apply()
1076
1077 # Python sets SIGPIPE handler to SIG_IGN by default. Child processes
1078 # shouldn't have this.
1079 # https://docs.python.org/2/library/signal.html
1080 # See Python/pythonrun.c.
1081 pyos.Sigaction(SIGPIPE, SIG_DFL)
1082
1083 # Respond to Ctrl-\ (core dump)
1084 pyos.Sigaction(SIGQUIT, SIG_DFL)
1085
1086 # Only standalone children should get Ctrl-Z. Pipelines remain in the
1087 # foreground because suspending them is difficult with our 'lastpipe'
1088 # semantics.
1089 pid = posix.getpid()
1090 if posix.getpgid(0) == pid and self.parent_pipeline is None:
1091 pyos.Sigaction(SIGTSTP, SIG_DFL)
1092
1093 # More signals from
1094 # https://www.gnu.org/software/libc/manual/html_node/Launching-Jobs.html
1095 # (but not SIGCHLD)
1096 pyos.Sigaction(SIGTTOU, SIG_DFL)
1097 pyos.Sigaction(SIGTTIN, SIG_DFL)
1098
1099 if not self.thunk.SkipTrace():
1100 self.tracer.OnNewProcess(pid)
1101
1102 # clear foreground pipeline for subshells
1103 self.thunk.Run()
1104 # Never returns
1105
1106 #log('STARTED process %s, pid = %d', self, pid)
1107 self.tracer.OnProcessStart(pid, why)
1108
1109 # Class invariant: after the process is started, it stores its PID.
1110 self.pid = pid
1111
1112 # SetPgid needs to be applied from the child and the parent to avoid
1113 # racing in calls to tcsetpgrp() in the parent. See APUE sec. 9.2.
1114 for st in self.state_changes:
1115 st.ApplyFromParent(self)
1116
1117 # Program invariant: We keep track of every child process!
1118 self.job_list.AddChildProcess(pid, self)
1119
1120 return pid
1121
1122 def Wait(self, waiter):
1123 # type: (Waiter) -> int
1124 """Wait for this process to finish."""
1125 while self.state == job_state_e.Running:
1126 # Only return if there's nothing to wait for. Keep waiting if we were
1127 # interrupted with a signal.
1128 if waiter.WaitForOne() == W1_ECHILD:
1129 break
1130
1131 assert self.status >= 0, self.status
1132 return self.status
1133
1134 def JobWait(self, waiter):
1135 # type: (Waiter) -> wait_status_t
1136 # wait builtin can be interrupted
1137 while self.state == job_state_e.Running:
1138 result = waiter.WaitForOne()
1139
1140 if result >= 0: # signal
1141 return wait_status.Cancelled(result)
1142
1143 if result == W1_ECHILD:
1144 break
1145
1146 return wait_status.Proc(self.status)
1147
1148 def WhenStopped(self, stop_sig):
1149 # type: (int) -> None
1150
1151 # 128 is a shell thing
1152 # https://www.gnu.org/software/bash/manual/html_node/Exit-Status.html
1153 self.status = 128 + stop_sig
1154 self.state = job_state_e.Stopped
1155
1156 if self.job_id == -1:
1157 # This process was started in the foreground
1158 self.job_list.AddJob(self)
1159
1160 if not self.in_background:
1161 self.job_control.MaybeTakeTerminal()
1162 self.SetBackground()
1163
1164 def WhenDone(self, pid, status):
1165 # type: (int, int) -> None
1166 """Called by the Waiter when this Process finishes."""
1167
1168 #log('WhenDone %d %d', pid, status)
1169 assert pid == self.pid, 'Expected %d, got %d' % (self.pid, pid)
1170 self.status = status
1171 self.state = job_state_e.Done
1172 if self.parent_pipeline:
1173 self.parent_pipeline.WhenDone(pid, status)
1174 else:
1175 if self.job_id != -1:
1176 # Job might have been brought to the foreground after being
1177 # assigned a job ID.
1178 if self.in_background:
1179 print_stderr('[%d] Done PID %d' % (self.job_id, self.pid))
1180
1181 self.job_list.RemoveJob(self.job_id)
1182
1183 self.job_list.RemoveChildProcess(self.pid)
1184
1185 if not self.in_background:
1186 self.job_control.MaybeTakeTerminal()
1187
1188 def RunProcess(self, waiter, why):
1189 # type: (Waiter, trace_t) -> int
1190 """Run this process synchronously."""
1191 self.StartProcess(why)
1192 # ShellExecutor might be calling this for the last part of a pipeline.
1193 if self.parent_pipeline is None:
1194 # QUESTION: Can the PGID of a single process just be the PID? i.e. avoid
1195 # calling getpgid()?
1196 self.job_control.MaybeGiveTerminal(posix.getpgid(self.pid))
1197 return self.Wait(waiter)
1198
1199
1200class ctx_Pipe(object):
1201
1202 def __init__(self, fd_state, fd, err_out):
1203 # type: (FdState, int, List[error.IOError_OSError]) -> None
1204 fd_state.PushStdinFromPipe(fd)
1205 self.fd_state = fd_state
1206 self.err_out = err_out
1207
1208 def __enter__(self):
1209 # type: () -> None
1210 pass
1211
1212 def __exit__(self, type, value, traceback):
1213 # type: (Any, Any, Any) -> None
1214 self.fd_state.Pop(self.err_out)
1215
1216
1217class Pipeline(Job):
1218 """A pipeline of processes to run.
1219
1220 Cases we handle:
1221
1222 foo | bar
1223 $(foo | bar)
1224 foo | bar | read v
1225 """
1226
1227 def __init__(self, sigpipe_status_ok, job_control, job_list, tracer):
1228 # type: (bool, JobControl, JobList, dev.Tracer) -> None
1229 Job.__init__(self)
1230 self.job_control = job_control
1231 self.job_list = job_list
1232 self.tracer = tracer
1233
1234 self.procs = [] # type: List[Process]
1235 self.pids = [] # type: List[int] # pids in order
1236 self.pipe_status = [] # type: List[int] # status in order
1237 self.status = -1 # for 'wait' jobs
1238
1239 self.pgid = INVALID_PGID
1240
1241 # Optional for foreground
1242 self.last_thunk = None # type: Tuple[CommandEvaluator, command_t]
1243 self.last_pipe = None # type: Tuple[int, int]
1244
1245 self.sigpipe_status_ok = sigpipe_status_ok
1246
1247 def ProcessGroupId(self):
1248 # type: () -> int
1249 """Returns the group ID of this pipeline."""
1250 return self.pgid
1251
1252 def DisplayJob(self, job_id, f, style):
1253 # type: (int, mylib.Writer, int) -> None
1254 if style == STYLE_PID_ONLY:
1255 f.write('%d\n' % self.procs[0].pid)
1256 else:
1257 # Note: this is STYLE_LONG.
1258 for i, proc in enumerate(self.procs):
1259 if i == 0: # show job ID for first element in pipeline
1260 job_id_str = '%%%d' % job_id
1261 else:
1262 job_id_str = ' ' # 2 spaces
1263
1264 f.write('%s %d %7s ' %
1265 (job_id_str, proc.pid, _JobStateStr(proc.state)))
1266 f.write(proc.thunk.UserString())
1267 f.write('\n')
1268
1269 def DebugPrint(self):
1270 # type: () -> None
1271 print('Pipeline in state %s' % _JobStateStr(self.state))
1272 if mylib.PYTHON: # %s for Process not allowed in C++
1273 for proc in self.procs:
1274 print(' proc %s' % proc)
1275 _, last_node = self.last_thunk
1276 print(' last %s' % last_node)
1277 print(' pipe_status %s' % self.pipe_status)
1278
1279 def Add(self, p):
1280 # type: (Process) -> None
1281 """Append a process to the pipeline."""
1282 if len(self.procs) == 0:
1283 self.procs.append(p)
1284 return
1285
1286 r, w = posix.pipe()
1287 #log('pipe for %s: %d %d', p, r, w)
1288 prev = self.procs[-1]
1289
1290 prev.AddStateChange(StdoutToPipe(r, w)) # applied on StartPipeline()
1291 p.AddStateChange(StdinFromPipe(r, w)) # applied on StartPipeline()
1292
1293 p.AddPipeToClose(r, w) # MaybeClosePipe() on StartPipeline()
1294
1295 self.procs.append(p)
1296
1297 def AddLast(self, thunk):
1298 # type: (Tuple[CommandEvaluator, command_t]) -> None
1299 """Append the last noden to the pipeline.
1300
1301 This is run in the CURRENT process. It is OPTIONAL, because
1302 pipelines in the background are run uniformly.
1303 """
1304 self.last_thunk = thunk
1305
1306 assert len(self.procs) != 0
1307
1308 r, w = posix.pipe()
1309 prev = self.procs[-1]
1310 prev.AddStateChange(StdoutToPipe(r, w))
1311
1312 self.last_pipe = (r, w) # So we can connect it to last_thunk
1313
1314 def StartPipeline(self, waiter):
1315 # type: (Waiter) -> None
1316
1317 # If we are creating a pipeline in a subshell or we aren't running with job
1318 # control, our children should remain in our inherited process group.
1319 # the pipelines's group ID.
1320 if self.job_control.Enabled():
1321 self.pgid = OWN_LEADER # first process in pipeline is the leader
1322
1323 for i, proc in enumerate(self.procs):
1324 if self.pgid != INVALID_PGID:
1325 proc.AddStateChange(SetPgid(self.pgid, self.tracer))
1326
1327 # Figure out the pid
1328 pid = proc.StartProcess(trace.PipelinePart)
1329 if i == 0 and self.pgid != INVALID_PGID:
1330 # Mimic bash and use the PID of the FIRST process as the group for the
1331 # whole pipeline.
1332 self.pgid = pid
1333
1334 self.pids.append(pid)
1335 self.pipe_status.append(-1) # uninitialized
1336
1337 # NOTE: This is done in the SHELL PROCESS after every fork() call.
1338 # It can't be done at the end; otherwise processes will have descriptors
1339 # from non-adjacent pipes.
1340 proc.MaybeClosePipe()
1341
1342 if self.last_thunk:
1343 self.pipe_status.append(-1) # for self.last_thunk
1344
1345 def LastPid(self):
1346 # type: () -> int
1347 """For the odd $! variable.
1348
1349 It would be better if job IDs or PGIDs were used consistently.
1350 """
1351 return self.pids[-1]
1352
1353 def Wait(self, waiter):
1354 # type: (Waiter) -> List[int]
1355 """Wait for this pipeline to finish."""
1356
1357 assert self.procs, "no procs for Wait()"
1358 # waitpid(-1) zero or more times
1359 while self.state == job_state_e.Running:
1360 # Keep waiting until there's nothing to wait for.
1361 if waiter.WaitForOne() == W1_ECHILD:
1362 break
1363
1364 return self.pipe_status
1365
1366 def JobWait(self, waiter):
1367 # type: (Waiter) -> wait_status_t
1368 """Called by 'wait' builtin, e.g. 'wait %1'."""
1369 # wait builtin can be interrupted
1370 assert self.procs, "no procs for Wait()"
1371 while self.state == job_state_e.Running:
1372 result = waiter.WaitForOne()
1373
1374 if result >= 0: # signal
1375 return wait_status.Cancelled(result)
1376
1377 if result == W1_ECHILD:
1378 break
1379
1380 return wait_status.Pipeline(self.pipe_status)
1381
1382 def RunLastPart(self, waiter, fd_state):
1383 # type: (Waiter, FdState) -> List[int]
1384 """Run this pipeline synchronously (foreground pipeline).
1385
1386 Returns:
1387 pipe_status (list of integers).
1388 """
1389 assert len(self.pids) == len(self.procs)
1390
1391 # TODO: break circular dep. Bit flags could go in ASDL or headers.
1392 from osh import cmd_eval
1393
1394 # This is tcsetpgrp()
1395 # TODO: fix race condition -- I believe the first process could have
1396 # stopped already, and thus getpgid() will fail
1397 self.job_control.MaybeGiveTerminal(self.pgid)
1398
1399 # Run the last part of the pipeline IN PARALLEL with other processes. It
1400 # may or may not fork:
1401 # echo foo | read line # no fork, the builtin runs in THIS shell process
1402 # ls | wc -l # fork for 'wc'
1403
1404 cmd_ev, last_node = self.last_thunk
1405
1406 assert self.last_pipe is not None
1407 r, w = self.last_pipe # set in AddLast()
1408 posix.close(w) # we will not write here
1409
1410 # Fix lastpipe / job control / DEBUG trap interaction
1411 cmd_flags = cmd_eval.NoDebugTrap if self.job_control.Enabled() else 0
1412
1413 # The ERR trap only runs for the WHOLE pipeline, not the COMPONENTS in
1414 # a pipeline.
1415 cmd_flags |= cmd_eval.NoErrTrap
1416
1417 io_errors = [] # type: List[error.IOError_OSError]
1418 with ctx_Pipe(fd_state, r, io_errors):
1419 cmd_ev.ExecuteAndCatch(last_node, cmd_flags)
1420
1421 if len(io_errors):
1422 e_die('Error setting up last part of pipeline: %s' %
1423 pyutil.strerror(io_errors[0]))
1424
1425 # We won't read anymore. If we don't do this, then 'cat' in 'cat
1426 # /dev/urandom | sleep 1' will never get SIGPIPE.
1427 posix.close(r)
1428
1429 self.pipe_status[-1] = cmd_ev.LastStatus()
1430 if self.AllDone():
1431 self.state = job_state_e.Done
1432
1433 #log('pipestatus before all have finished = %s', self.pipe_status)
1434 return self.Wait(waiter)
1435
1436 def AllDone(self):
1437 # type: () -> bool
1438
1439 # mycpp rewrite: all(status != -1 for status in self.pipe_status)
1440 for status in self.pipe_status:
1441 if status == -1:
1442 return False
1443 return True
1444
1445 def WhenDone(self, pid, status):
1446 # type: (int, int) -> None
1447 """Called by Process.WhenDone."""
1448 #log('Pipeline WhenDone %d %d', pid, status)
1449 i = self.pids.index(pid)
1450 assert i != -1, 'Unexpected PID %d' % pid
1451
1452 if status == 141 and self.sigpipe_status_ok:
1453 status = 0
1454
1455 self.job_list.RemoveChildProcess(pid)
1456 self.pipe_status[i] = status
1457 if self.AllDone():
1458 if self.job_id != -1:
1459 # Job might have been brought to the foreground after being
1460 # assigned a job ID.
1461 if self.in_background:
1462 print_stderr('[%d] Done PGID %d' %
1463 (self.job_id, self.pids[0]))
1464
1465 self.job_list.RemoveJob(self.job_id)
1466
1467 # status of pipeline is status of last process
1468 self.status = self.pipe_status[-1]
1469 self.state = job_state_e.Done
1470 if not self.in_background:
1471 self.job_control.MaybeTakeTerminal()
1472
1473
1474def _JobStateStr(i):
1475 # type: (job_state_t) -> str
1476 return job_state_str(i)[10:] # remove 'job_state.'
1477
1478
1479def _GetTtyFd():
1480 # type: () -> int
1481 """Returns -1 if stdio is not a TTY."""
1482 try:
1483 return posix.open("/dev/tty", O_NONBLOCK | O_NOCTTY | O_RDWR, 0o666)
1484 except (IOError, OSError) as e:
1485 return -1
1486
1487
1488class ctx_TerminalControl(object):
1489
1490 def __init__(self, job_control, errfmt):
1491 # type: (JobControl, ui.ErrorFormatter) -> None
1492 job_control.InitJobControl()
1493 self.job_control = job_control
1494 self.errfmt = errfmt
1495
1496 def __enter__(self):
1497 # type: () -> None
1498 pass
1499
1500 def __exit__(self, type, value, traceback):
1501 # type: (Any, Any, Any) -> None
1502
1503 # Return the TTY to the original owner before exiting.
1504 try:
1505 self.job_control.MaybeReturnTerminal()
1506 except error.FatalRuntime as e:
1507 # Don't abort the shell on error, just print a message.
1508 self.errfmt.PrettyPrintError(e)
1509
1510
1511class JobControl(object):
1512 """Interface to setpgid(), tcsetpgrp(), etc."""
1513
1514 def __init__(self):
1515 # type: () -> None
1516
1517 # The main shell's PID and group ID.
1518 self.shell_pid = -1
1519 self.shell_pgid = -1
1520
1521 # The fd of the controlling tty. Set to -1 when job control is disabled.
1522 self.shell_tty_fd = -1
1523
1524 # For giving the terminal back to our parent before exiting (if not a login
1525 # shell).
1526 self.original_tty_pgid = -1
1527
1528 def InitJobControl(self):
1529 # type: () -> None
1530 self.shell_pid = posix.getpid()
1531 orig_shell_pgid = posix.getpgid(0)
1532 self.shell_pgid = orig_shell_pgid
1533 self.shell_tty_fd = _GetTtyFd()
1534
1535 # If we aren't the leader of our process group, create a group and mark
1536 # ourselves as the leader.
1537 if self.shell_pgid != self.shell_pid:
1538 try:
1539 posix.setpgid(self.shell_pid, self.shell_pid)
1540 self.shell_pgid = self.shell_pid
1541 except (IOError, OSError) as e:
1542 self.shell_tty_fd = -1
1543
1544 if self.shell_tty_fd != -1:
1545 self.original_tty_pgid = posix.tcgetpgrp(self.shell_tty_fd)
1546
1547 # If stdio is a TTY, put the shell's process group in the foreground.
1548 try:
1549 posix.tcsetpgrp(self.shell_tty_fd, self.shell_pgid)
1550 except (IOError, OSError) as e:
1551 # We probably aren't in the session leader's process group. Disable job
1552 # control.
1553 self.shell_tty_fd = -1
1554 self.shell_pgid = orig_shell_pgid
1555 posix.setpgid(self.shell_pid, self.shell_pgid)
1556
1557 def Enabled(self):
1558 # type: () -> bool
1559
1560 # TODO: get rid of this syscall? SubProgramThunk should set a flag I
1561 # think.
1562 curr_pid = posix.getpid()
1563 # Only the main shell should bother with job control functions.
1564 return curr_pid == self.shell_pid and self.shell_tty_fd != -1
1565
1566 # TODO: This isn't a PID. This is a process group ID?
1567 #
1568 # What should the table look like?
1569 #
1570 # Do we need the last PID? I don't know why bash prints that. Probably so
1571 # you can do wait $!
1572 # wait -n waits for any node to go from job_state_e.Running to job_state_e.Done?
1573 #
1574 # And it needs a flag for CURRENT, for the implicit arg to 'fg'.
1575 # job_id is just an integer. This is sort of lame.
1576 #
1577 # [job_id, flag, pgid, job_state, node]
1578
1579 def MaybeGiveTerminal(self, pgid):
1580 # type: (int) -> None
1581 """If stdio is a TTY, move the given process group to the
1582 foreground."""
1583 if not self.Enabled():
1584 # Only call tcsetpgrp when job control is enabled.
1585 return
1586
1587 try:
1588 posix.tcsetpgrp(self.shell_tty_fd, pgid)
1589 except (IOError, OSError) as e:
1590 e_die('osh: Failed to move process group %d to foreground: %s' %
1591 (pgid, pyutil.strerror(e)))
1592
1593 def MaybeTakeTerminal(self):
1594 # type: () -> None
1595 """If stdio is a TTY, return the main shell's process group to the
1596 foreground."""
1597 self.MaybeGiveTerminal(self.shell_pgid)
1598
1599 def MaybeReturnTerminal(self):
1600 # type: () -> None
1601 """Called before the shell exits."""
1602 self.MaybeGiveTerminal(self.original_tty_pgid)
1603
1604
1605class JobList(object):
1606 """Global list of jobs, used by a few builtins."""
1607
1608 def __init__(self):
1609 # type: () -> None
1610
1611 # job_id -> Job instance
1612 self.jobs = {} # type: Dict[int, Job]
1613
1614 # pid -> Process. This is for STOP notification.
1615 self.child_procs = {} # type: Dict[int, Process]
1616 self.debug_pipelines = [] # type: List[Pipeline]
1617
1618 # Counter used to assign IDs to jobs. It is incremented every time a job
1619 # is created. Once all active jobs are done it is reset to 1. I'm not
1620 # sure if this reset behavior is mandated by POSIX, but other shells do
1621 # it, so we mimic for the sake of compatibility.
1622 self.job_id = 1
1623
1624 def AddJob(self, job):
1625 # type: (Job) -> int
1626 """Add a background job to the list.
1627
1628 A job is either a Process or Pipeline. You can resume a job with 'fg',
1629 kill it with 'kill', etc.
1630
1631 Two cases:
1632
1633 1. async jobs: sleep 5 | sleep 4 &
1634 2. stopped jobs: sleep 5; then Ctrl-Z
1635 """
1636 job_id = self.job_id
1637 self.jobs[job_id] = job
1638 job.job_id = job_id
1639 self.job_id += 1
1640 return job_id
1641
1642 def RemoveJob(self, job_id):
1643 # type: (int) -> None
1644 """Process and Pipeline can call this."""
1645 mylib.dict_erase(self.jobs, job_id)
1646
1647 if len(self.jobs) == 0:
1648 self.job_id = 1
1649
1650 def AddChildProcess(self, pid, proc):
1651 # type: (int, Process) -> None
1652 """Every child process should be added here as soon as we know its PID.
1653
1654 When the Waiter gets an EXITED or STOPPED notification, we need
1655 to know about it so 'jobs' can work.
1656 """
1657 self.child_procs[pid] = proc
1658
1659 def RemoveChildProcess(self, pid):
1660 # type: (int) -> None
1661 """Remove the child process with the given PID."""
1662 mylib.dict_erase(self.child_procs, pid)
1663
1664 if mylib.PYTHON:
1665
1666 def AddPipeline(self, pi):
1667 # type: (Pipeline) -> None
1668 """For debugging only."""
1669 self.debug_pipelines.append(pi)
1670
1671 def ProcessFromPid(self, pid):
1672 # type: (int) -> Process
1673 """For wait $PID.
1674
1675 There's no way to wait for a pipeline with a PID. That uses job
1676 syntax, e.g. %1. Not a great interface.
1677 """
1678 return self.child_procs.get(pid)
1679
1680 def GetCurrentAndPreviousJobs(self):
1681 # type: () -> Tuple[Optional[Job], Optional[Job]]
1682 """Return the "current" and "previous" jobs (AKA `%+` and `%-`).
1683
1684 See the POSIX specification for the `jobs` builtin for details:
1685 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1686
1687 IMPORTANT NOTE: This method assumes that the jobs list will not change
1688 during its execution! This assumption holds for now because we only ever
1689 update the jobs list from the main loop after WaitPid() informs us of a
1690 change. If we implement `set -b` and install a signal handler for
1691 SIGCHLD we should be careful to synchronize it with this function. The
1692 unsafety of mutating GC data structures from a signal handler should
1693 make this a non-issue, but if bugs related to this appear this note may
1694 be helpful...
1695 """
1696 # Split all active jobs by state and sort each group by decreasing job
1697 # ID to approximate newness.
1698 stopped_jobs = [] # type: List[Job]
1699 running_jobs = [] # type: List[Job]
1700 for i in xrange(0, self.job_id):
1701 job = self.jobs.get(i, None)
1702 if not job:
1703 continue
1704
1705 if job.state == job_state_e.Stopped:
1706 stopped_jobs.append(job)
1707
1708 elif job.state == job_state_e.Running:
1709 running_jobs.append(job)
1710
1711 current = None # type: Optional[Job]
1712 previous = None # type: Optional[Job]
1713 # POSIX says: If there is any suspended job, then the current job shall
1714 # be a suspended job. If there are at least two suspended jobs, then the
1715 # previous job also shall be a suspended job.
1716 #
1717 # So, we will only return running jobs from here if there are no recent
1718 # stopped jobs.
1719 if len(stopped_jobs) > 0:
1720 current = stopped_jobs.pop()
1721
1722 if len(stopped_jobs) > 0:
1723 previous = stopped_jobs.pop()
1724
1725 if len(running_jobs) > 0 and not current:
1726 current = running_jobs.pop()
1727
1728 if len(running_jobs) > 0 and not previous:
1729 previous = running_jobs.pop()
1730
1731 if not previous:
1732 previous = current
1733
1734 return current, previous
1735
1736 def GetJobWithSpec(self, job_spec):
1737 # type: (str) -> Optional[Job]
1738 """Parse the given job spec and return the matching job. If there is no
1739 matching job, this function returns None.
1740
1741 See the POSIX spec for the `jobs` builtin for details about job specs:
1742 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1743 """
1744 if job_spec in CURRENT_JOB_SPECS:
1745 current, _ = self.GetCurrentAndPreviousJobs()
1746 return current
1747
1748 if job_spec == '%-':
1749 _, previous = self.GetCurrentAndPreviousJobs()
1750 return previous
1751
1752 # TODO: Add support for job specs based on prefixes of process argv.
1753 m = util.RegexSearch(r'^%([0-9]+)$', job_spec)
1754 if m is not None:
1755 assert len(m) == 2
1756 job_id = int(m[1])
1757 if job_id in self.jobs:
1758 return self.jobs[job_id]
1759
1760 return None
1761
1762 def DisplayJobs(self, style):
1763 # type: (int) -> None
1764 """Used by the 'jobs' builtin.
1765
1766 https://pubs.opengroup.org/onlinepubs/9699919799/utilities/jobs.html
1767
1768 "By default, the jobs utility shall display the status of all stopped jobs,
1769 running background jobs and all jobs whose status has changed and have not
1770 been reported by the shell."
1771 """
1772 # NOTE: A job is a background process or pipeline.
1773 #
1774 # echo hi | wc -l -- this starts two processes. Wait for TWO
1775 # echo hi | wc -l & -- this starts a process which starts two processes
1776 # Wait for ONE.
1777 #
1778 # 'jobs -l' GROUPS the PIDs by job. It has the job number, + - indicators
1779 # for %% and %-, PID, status, and "command".
1780 #
1781 # Every component of a pipeline is on the same line with 'jobs', but
1782 # they're separated into different lines with 'jobs -l'.
1783 #
1784 # See demo/jobs-builtin.sh
1785
1786 # $ jobs -l
1787 # [1]+ 24414 Stopped sleep 5
1788 # 24415 | sleep 5
1789 # [2] 24502 Running sleep 6
1790 # 24503 | sleep 6
1791 # 24504 | sleep 5 &
1792 # [3]- 24508 Running sleep 6
1793 # 24509 | sleep 6
1794 # 24510 | sleep 5 &
1795
1796 f = mylib.Stdout()
1797 for job_id, job in iteritems(self.jobs):
1798 # Use the %1 syntax
1799 job.DisplayJob(job_id, f, style)
1800
1801 def DebugPrint(self):
1802 # type: () -> None
1803
1804 f = mylib.Stdout()
1805 f.write('\n')
1806 f.write('[process debug info]\n')
1807
1808 for pid, proc in iteritems(self.child_procs):
1809 proc.DisplayJob(-1, f, STYLE_DEFAULT)
1810 #p = ' |' if proc.parent_pipeline else ''
1811 #print('%d %7s %s%s' % (pid, _JobStateStr(proc.state), proc.thunk.UserString(), p))
1812
1813 if len(self.debug_pipelines):
1814 f.write('\n')
1815 f.write('[pipeline debug info]\n')
1816 for pi in self.debug_pipelines:
1817 pi.DebugPrint()
1818
1819 def ListRecent(self):
1820 # type: () -> None
1821 """For jobs -n, which I think is also used in the interactive
1822 prompt."""
1823 pass
1824
1825 def NumRunning(self):
1826 # type: () -> int
1827 """Return the number of running jobs.
1828
1829 Used by 'wait' and 'wait -n'.
1830 """
1831 count = 0
1832 for _, job in iteritems(self.jobs): # mycpp rewrite: from itervalues()
1833 if job.State() == job_state_e.Running:
1834 count += 1
1835 return count
1836
1837
1838# Some WaitForOne() return values
1839W1_OK = -2 # waitpid(-1) returned
1840W1_ECHILD = -3 # no processes to wait for
1841W1_AGAIN = -4 # WNOHANG was passed and there were no state changes
1842
1843
1844class Waiter(object):
1845 """A capability to wait for processes.
1846
1847 This must be a singleton (and is because CommandEvaluator is a singleton).
1848
1849 Invariants:
1850 - Every child process is registered once
1851 - Every child process is waited for
1852
1853 Canonical example of why we need a GLOBAL waiter:
1854
1855 { sleep 3; echo 'done 3'; } &
1856 { sleep 4; echo 'done 4'; } &
1857
1858 # ... do arbitrary stuff ...
1859
1860 { sleep 1; exit 1; } | { sleep 2; exit 2; }
1861
1862 Now when you do wait() after starting the pipeline, you might get a pipeline
1863 process OR a background process! So you have to distinguish between them.
1864 """
1865
1866 def __init__(self, job_list, exec_opts, signal_safe, tracer):
1867 # type: (JobList, optview.Exec, pyos.SignalSafe, dev.Tracer) -> None
1868 self.job_list = job_list
1869 self.exec_opts = exec_opts
1870 self.signal_safe = signal_safe
1871 self.tracer = tracer
1872 self.last_status = 127 # wait -n error code
1873
1874 def WaitForOne(self, waitpid_options=0):
1875 # type: (int) -> int
1876 """Wait until the next process returns (or maybe Ctrl-C).
1877
1878 Returns:
1879 One of these negative numbers:
1880 W1_ECHILD Nothing to wait for
1881 W1_OK Caller should keep waiting
1882 UNTRAPPED_SIGWINCH
1883 Or
1884 result > 0 Signal that waitpid() was interrupted with
1885
1886 In the interactive shell, we return 0 if we get a Ctrl-C, so the caller
1887 will try again.
1888
1889 Callers:
1890 wait -n -- loop until there is one fewer process (TODO)
1891 wait -- loop until there are no processes
1892 wait $! -- loop until job state is Done (process or pipeline)
1893 Process::Wait() -- loop until Process state is done
1894 Pipeline::Wait() -- loop until Pipeline state is done
1895
1896 Comparisons:
1897 bash: jobs.c waitchld() Has a special case macro(!) CHECK_WAIT_INTR for
1898 the wait builtin
1899
1900 dash: jobs.c waitproc() uses sigfillset(), sigprocmask(), etc. Runs in a
1901 loop while (gotsigchld), but that might be a hack for System V!
1902
1903 Should we have a cleaner API like named posix::wait_for_one() ?
1904
1905 wait_result =
1906 ECHILD -- nothing to wait for
1907 | Done(int pid, int status) -- process done
1908 | EINTR(bool sigint) -- may or may not retry
1909 """
1910 pid, status = pyos.WaitPid(waitpid_options)
1911 if pid == 0: # WNOHANG passed, and no state changes
1912 return W1_AGAIN
1913 elif pid < 0: # error case
1914 err_num = status
1915 #log('waitpid() error => %d %s', e.errno, pyutil.strerror(e))
1916 if err_num == ECHILD:
1917 return W1_ECHILD # nothing to wait for caller should stop
1918 elif err_num == EINTR: # Bug #858 fix
1919 #log('WaitForOne() => %d', self.trap_state.GetLastSignal())
1920 return self.signal_safe.LastSignal() # e.g. 1 for SIGHUP
1921 else:
1922 # The signature of waitpid() means this shouldn't happen
1923 raise AssertionError()
1924
1925 # All child processes are supposed to be in this dict. But this may
1926 # legitimately happen if a grandchild outlives the child (its parent).
1927 # Then it is reparented under this process, so we might receive
1928 # notification of its exit, even though we didn't start it. We can't have
1929 # any knowledge of such processes, so print a warning.
1930 if pid not in self.job_list.child_procs:
1931 print_stderr("osh: PID %d stopped, but osh didn't start it" % pid)
1932 return W1_OK
1933
1934 proc = self.job_list.child_procs[pid]
1935 if 0:
1936 self.job_list.DebugPrint()
1937
1938 if WIFSIGNALED(status):
1939 term_sig = WTERMSIG(status)
1940 status = 128 + term_sig
1941
1942 # Print newline after Ctrl-C.
1943 if term_sig == SIGINT:
1944 print('')
1945
1946 proc.WhenDone(pid, status)
1947
1948 elif WIFEXITED(status):
1949 status = WEXITSTATUS(status)
1950 #log('exit status: %s', status)
1951 proc.WhenDone(pid, status)
1952
1953 elif WIFSTOPPED(status):
1954 #status = WEXITSTATUS(status)
1955 stop_sig = WSTOPSIG(status)
1956
1957 print_stderr('')
1958 print_stderr('[PID %d] Stopped with signal %d' % (pid, stop_sig))
1959 proc.WhenStopped(stop_sig)
1960
1961 else:
1962 raise AssertionError(status)
1963
1964 self.last_status = status # for wait -n
1965 self.tracer.OnProcessEnd(pid, status)
1966 return W1_OK
1967
1968 def PollNotifications(self):
1969 # type: () -> None
1970 """
1971 Process all pending state changes.
1972 """
1973 while self.WaitForOne(waitpid_options=WNOHANG) == W1_OK:
1974 continue