OILS / core / process.py View on Github | oilshell.org

1965 lines, 955 significant
1# Copyright 2016 Andy Chu. All rights reserved.
2# Licensed under the Apache License, Version 2.0 (the "License");
3# you may not use this file except in compliance with the License.
4# You may obtain a copy of the License at
5#
6# http://www.apache.org/licenses/LICENSE-2.0
7"""
8process.py - Launch processes and manipulate file descriptors.
9"""
10from __future__ import print_function
11
12from errno import EACCES, EBADF, ECHILD, EINTR, ENOENT, ENOEXEC, EEXIST
13import fcntl as fcntl_
14from fcntl import F_DUPFD, F_GETFD, F_SETFD, FD_CLOEXEC
15from signal import (SIG_DFL, SIG_IGN, SIGINT, SIGPIPE, SIGQUIT, SIGTSTP,
16 SIGTTOU, SIGTTIN, SIGWINCH)
17
18from _devbuild.gen.id_kind_asdl import Id
19from _devbuild.gen.runtime_asdl import (job_state_e, job_state_t,
20 job_state_str, wait_status,
21 wait_status_t, RedirValue,
22 redirect_arg, redirect_arg_e, trace,
23 trace_t)
24from _devbuild.gen.syntax_asdl import (
25 loc_t,
26 redir_loc,
27 redir_loc_e,
28 redir_loc_t,
29)
30from _devbuild.gen.value_asdl import (value, value_e)
31from core import dev
32from core import error
33from core.error import e_die
34from core import pyutil
35from core import pyos
36from core import state
37from core import ui
38from core import util
39from data_lang import j8_lite
40from frontend import location
41from frontend import match
42from mycpp import mylib
43from mycpp.mylib import log, print_stderr, probe, tagswitch, iteritems
44
45import posix_ as posix
46from posix_ import (
47 # translated by mycpp and directly called! No wrapper!
48 WIFSIGNALED,
49 WIFEXITED,
50 WIFSTOPPED,
51 WEXITSTATUS,
52 WSTOPSIG,
53 WTERMSIG,
54 WNOHANG,
55 O_APPEND,
56 O_CREAT,
57 O_EXCL,
58 O_NONBLOCK,
59 O_NOCTTY,
60 O_RDONLY,
61 O_RDWR,
62 O_WRONLY,
63 O_TRUNC,
64)
65
66from typing import IO, List, Tuple, Dict, Optional, Any, cast, TYPE_CHECKING
67
68if TYPE_CHECKING:
69 from _devbuild.gen.runtime_asdl import cmd_value
70 from _devbuild.gen.syntax_asdl import command_t
71 from builtin import trap_osh
72 from core import optview
73 from core.ui import ErrorFormatter
74 from core.util import _DebugFile
75 from osh.cmd_eval import CommandEvaluator
76
77NO_FD = -1
78
79# Minimum file descriptor that the shell can use. Other descriptors can be
80# directly used by user programs, e.g. exec 9>&1
81#
82# Oil uses 100 because users are allowed TWO digits in frontend/lexer_def.py.
83# This is a compromise between bash (unlimited, but requires crazy
84# bookkeeping), and dash/zsh (10) and mksh (24)
85_SHELL_MIN_FD = 100
86
87# Style for 'jobs' builtin
88STYLE_DEFAULT = 0
89STYLE_LONG = 1
90STYLE_PID_ONLY = 2
91
92# To save on allocations in JobList::GetJobWithSpec()
93CURRENT_JOB_SPECS = ['', '%', '%%', '%+']
94
95
96class ctx_FileCloser(object):
97
98 def __init__(self, f):
99 # type: (mylib.LineReader) -> None
100 self.f = f
101
102 def __enter__(self):
103 # type: () -> None
104 pass
105
106 def __exit__(self, type, value, traceback):
107 # type: (Any, Any, Any) -> None
108 self.f.close()
109
110
111def InitInteractiveShell():
112 # type: () -> None
113 """Called when initializing an interactive shell."""
114
115 # The shell itself should ignore Ctrl-\.
116 pyos.Sigaction(SIGQUIT, SIG_IGN)
117
118 # This prevents Ctrl-Z from suspending OSH in interactive mode.
119 pyos.Sigaction(SIGTSTP, SIG_IGN)
120
121 # More signals from
122 # https://www.gnu.org/software/libc/manual/html_node/Initializing-the-Shell.html
123 # (but not SIGCHLD)
124 pyos.Sigaction(SIGTTOU, SIG_IGN)
125 pyos.Sigaction(SIGTTIN, SIG_IGN)
126
127 # Register a callback to receive terminal width changes.
128 # NOTE: In line_input.c, we turned off rl_catch_sigwinch.
129
130 # This is ALWAYS on, which means that it can cause EINTR, and wait() and
131 # read() have to handle it
132 pyos.RegisterSignalInterest(SIGWINCH)
133
134
135def SaveFd(fd):
136 # type: (int) -> int
137 saved = fcntl_.fcntl(fd, F_DUPFD, _SHELL_MIN_FD) # type: int
138 return saved
139
140
141class _RedirFrame(object):
142
143 def __init__(self, saved_fd, orig_fd, forget):
144 # type: (int, int, bool) -> None
145 self.saved_fd = saved_fd
146 self.orig_fd = orig_fd
147 self.forget = forget
148
149
150class _FdFrame(object):
151
152 def __init__(self):
153 # type: () -> None
154 self.saved = [] # type: List[_RedirFrame]
155 self.need_wait = [] # type: List[Process]
156
157 def Forget(self):
158 # type: () -> None
159 """For exec 1>&2."""
160 for rf in reversed(self.saved):
161 if rf.saved_fd != NO_FD and rf.forget:
162 posix.close(rf.saved_fd)
163
164 del self.saved[:] # like list.clear() in Python 3.3
165 del self.need_wait[:]
166
167 def __repr__(self):
168 # type: () -> str
169 return '<_FdFrame %s>' % self.saved
170
171
172class FdState(object):
173 """File descriptor state for the current process.
174
175 For example, you can do 'myfunc > out.txt' without forking. Child
176 processes inherit our state.
177 """
178
179 def __init__(
180 self,
181 errfmt, # type: ui.ErrorFormatter
182 job_control, # type: JobControl
183 job_list, # type: JobList
184 mem, #type: state.Mem
185 tracer, # type: Optional[dev.Tracer]
186 waiter, # type: Optional[Waiter]
187 exec_opts, #type: optview.Exec
188 ):
189 # type: (...) -> None
190 """
191 Args:
192 errfmt: for errors
193 job_list: For keeping track of _HereDocWriterThunk
194 """
195 self.errfmt = errfmt
196 self.job_control = job_control
197 self.job_list = job_list
198 self.cur_frame = _FdFrame() # for the top level
199 self.stack = [self.cur_frame]
200 self.mem = mem
201 self.tracer = tracer
202 self.waiter = waiter
203 self.exec_opts = exec_opts
204
205 def Open(self, path):
206 # type: (str) -> mylib.LineReader
207 """Opens a path for read, but moves it out of the reserved 3-9 fd
208 range.
209
210 Returns:
211 A Python file object. The caller is responsible for Close().
212
213 Raises:
214 IOError or OSError if the path can't be found. (This is Python-induced wart)
215 """
216 fd_mode = O_RDONLY
217 f = self._Open(path, 'r', fd_mode)
218
219 # Hacky downcast
220 return cast('mylib.LineReader', f)
221
222 # used for util.DebugFile
223 def OpenForWrite(self, path):
224 # type: (str) -> mylib.Writer
225 fd_mode = O_CREAT | O_RDWR
226 f = self._Open(path, 'w', fd_mode)
227
228 # Hacky downcast
229 return cast('mylib.Writer', f)
230
231 def _Open(self, path, c_mode, fd_mode):
232 # type: (str, str, int) -> IO[str]
233 fd = posix.open(path, fd_mode, 0o666) # may raise OSError
234
235 # Immediately move it to a new location
236 new_fd = SaveFd(fd)
237 posix.close(fd)
238
239 # Return a Python file handle
240 f = posix.fdopen(new_fd, c_mode) # may raise IOError
241 return f
242
243 def _WriteFdToMem(self, fd_name, fd):
244 # type: (str, int) -> None
245 if self.mem:
246 # setvar, not setref
247 state.OshLanguageSetValue(self.mem, location.LName(fd_name),
248 value.Str(str(fd)))
249
250 def _ReadFdFromMem(self, fd_name):
251 # type: (str) -> int
252 val = self.mem.GetValue(fd_name)
253 if val.tag() == value_e.Str:
254 try:
255 return int(cast(value.Str, val).s)
256 except ValueError:
257 return NO_FD
258 return NO_FD
259
260 def _PushSave(self, fd):
261 # type: (int) -> bool
262 """Save fd to a new location and remember to restore it later."""
263 #log('---- _PushSave %s', fd)
264 ok = True
265 try:
266 new_fd = SaveFd(fd)
267 except (IOError, OSError) as e:
268 ok = False
269 # Example program that causes this error: exec 4>&1. Descriptor 4 isn't
270 # open.
271 # This seems to be ignored in dash too in savefd()?
272 if e.errno != EBADF:
273 raise
274 if ok:
275 posix.close(fd)
276 fcntl_.fcntl(new_fd, F_SETFD, FD_CLOEXEC)
277 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, True))
278 else:
279 # if we got EBADF, we still need to close the original on Pop()
280 self._PushClose(fd)
281
282 return ok
283
284 def _PushDup(self, fd1, blame_loc):
285 # type: (int, redir_loc_t) -> int
286 """Save fd2 in a higher range, and dup fd1 onto fd2.
287
288 Returns whether F_DUPFD/dup2 succeeded, and the new descriptor.
289 """
290 UP_loc = blame_loc
291 if blame_loc.tag() == redir_loc_e.VarName:
292 fd2_name = cast(redir_loc.VarName, UP_loc).name
293 try:
294 # F_DUPFD: GREATER than range
295 new_fd = fcntl_.fcntl(fd1, F_DUPFD, _SHELL_MIN_FD) # type: int
296 except (IOError, OSError) as e:
297 if e.errno == EBADF:
298 print_stderr('F_DUPFD fd %d: %s' %
299 (fd1, pyutil.strerror(e)))
300 return NO_FD
301 else:
302 raise # this redirect failed
303
304 self._WriteFdToMem(fd2_name, new_fd)
305
306 elif blame_loc.tag() == redir_loc_e.Fd:
307 fd2 = cast(redir_loc.Fd, UP_loc).fd
308
309 if fd1 == fd2:
310 # The user could have asked for it to be open on descriptor 3, but open()
311 # already returned 3, e.g. echo 3>out.txt
312 return NO_FD
313
314 # Check the validity of fd1 before _PushSave(fd2)
315 try:
316 fcntl_.fcntl(fd1, F_GETFD)
317 except (IOError, OSError) as e:
318 print_stderr('F_GETFD fd %d: %s' % (fd1, pyutil.strerror(e)))
319 raise
320
321 need_restore = self._PushSave(fd2)
322
323 #log('==== dup2 %s %s\n' % (fd1, fd2))
324 try:
325 posix.dup2(fd1, fd2)
326 except (IOError, OSError) as e:
327 # bash/dash give this error too, e.g. for 'echo hi 1>&3'
328 print_stderr('dup2(%d, %d): %s' %
329 (fd1, fd2, pyutil.strerror(e)))
330
331 # Restore and return error
332 if need_restore:
333 rf = self.cur_frame.saved.pop()
334 posix.dup2(rf.saved_fd, rf.orig_fd)
335 posix.close(rf.saved_fd)
336
337 raise # this redirect failed
338
339 new_fd = fd2
340
341 else:
342 raise AssertionError()
343
344 return new_fd
345
346 def _PushCloseFd(self, blame_loc):
347 # type: (redir_loc_t) -> bool
348 """For 2>&-"""
349 # exec {fd}>&- means close the named descriptor
350
351 UP_loc = blame_loc
352 if blame_loc.tag() == redir_loc_e.VarName:
353 fd_name = cast(redir_loc.VarName, UP_loc).name
354 fd = self._ReadFdFromMem(fd_name)
355 if fd == NO_FD:
356 return False
357
358 elif blame_loc.tag() == redir_loc_e.Fd:
359 fd = cast(redir_loc.Fd, UP_loc).fd
360
361 else:
362 raise AssertionError()
363
364 self._PushSave(fd)
365
366 return True
367
368 def _PushClose(self, fd):
369 # type: (int) -> None
370 self.cur_frame.saved.append(_RedirFrame(NO_FD, fd, False))
371
372 def _PushWait(self, proc):
373 # type: (Process) -> None
374 self.cur_frame.need_wait.append(proc)
375
376 def _ApplyRedirect(self, r):
377 # type: (RedirValue) -> None
378 arg = r.arg
379 UP_arg = arg
380 with tagswitch(arg) as case:
381
382 if case(redirect_arg_e.Path):
383 arg = cast(redirect_arg.Path, UP_arg)
384 # noclobber flag is OR'd with other flags when allowed
385 noclobber_mode = O_EXCL if self.exec_opts.noclobber() else 0
386 if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &>
387 # NOTE: This is different than >| because it respects noclobber, but
388 # that option is almost never used. See test/wild.sh.
389 mode = O_CREAT | O_WRONLY | O_TRUNC | noclobber_mode
390 elif r.op_id == Id.Redir_Clobber: # >|
391 mode = O_CREAT | O_WRONLY | O_TRUNC
392 elif r.op_id in (Id.Redir_DGreat,
393 Id.Redir_AndDGreat): # >> &>>
394 mode = O_CREAT | O_WRONLY | O_APPEND | noclobber_mode
395 elif r.op_id == Id.Redir_Less: # <
396 mode = O_RDONLY
397 elif r.op_id == Id.Redir_LessGreat: # <>
398 mode = O_CREAT | O_RDWR
399 else:
400 raise NotImplementedError(r.op_id)
401
402 # NOTE: 0666 is affected by umask, all shells use it.
403 try:
404 open_fd = posix.open(arg.filename, mode, 0o666)
405 except (IOError, OSError) as e:
406 if noclobber_mode != 0 and e.errno == EEXIST:
407 self.errfmt.PrintMessage(
408 "I/O redirect error: can't overwrite existing file %r: %s"
409 % (arg.filename, pyutil.strerror(e)),
410 r.op_loc)
411 else:
412 self.errfmt.PrintMessage(
413 "I/O redirect error: can't open file %r: %s" %
414 (arg.filename, pyutil.strerror(e)),
415 r.op_loc)
416 raise IOError(
417 0
418 ) # redirect failed, errno=0 to hide error at parent level
419
420 new_fd = self._PushDup(open_fd, r.loc)
421 if new_fd != NO_FD:
422 posix.close(open_fd)
423
424 # Now handle &> and &>> and their variants. These pairs are the same:
425 #
426 # stdout_stderr.py &> out-err.txt
427 # stdout_stderr.py > out-err.txt 2>&1
428 #
429 # stdout_stderr.py 3&> out-err.txt
430 # stdout_stderr.py 3> out-err.txt 2>&3
431 #
432 # Ditto for {fd}> and {fd}&>
433
434 if r.op_id in (Id.Redir_AndGreat, Id.Redir_AndDGreat):
435 self._PushDup(new_fd, redir_loc.Fd(2))
436
437 elif case(redirect_arg_e.CopyFd): # e.g. echo hi 1>&2
438 arg = cast(redirect_arg.CopyFd, UP_arg)
439
440 if r.op_id == Id.Redir_GreatAnd: # 1>&2
441 self._PushDup(arg.target_fd, r.loc)
442
443 elif r.op_id == Id.Redir_LessAnd: # 0<&5
444 # The only difference between >& and <& is the default file
445 # descriptor argument.
446 self._PushDup(arg.target_fd, r.loc)
447
448 else:
449 raise NotImplementedError()
450
451 elif case(redirect_arg_e.MoveFd): # e.g. echo hi 5>&6-
452 arg = cast(redirect_arg.MoveFd, UP_arg)
453 new_fd = self._PushDup(arg.target_fd, r.loc)
454 if new_fd != NO_FD:
455 posix.close(arg.target_fd)
456
457 UP_loc = r.loc
458 if r.loc.tag() == redir_loc_e.Fd:
459 fd = cast(redir_loc.Fd, UP_loc).fd
460 else:
461 fd = NO_FD
462
463 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, False))
464
465 elif case(redirect_arg_e.CloseFd): # e.g. echo hi 5>&-
466 self._PushCloseFd(r.loc)
467
468 elif case(redirect_arg_e.HereDoc):
469 arg = cast(redirect_arg.HereDoc, UP_arg)
470
471 # NOTE: Do these descriptors have to be moved out of the range 0-9?
472 read_fd, write_fd = posix.pipe()
473
474 self._PushDup(read_fd, r.loc) # stdin is now the pipe
475
476 # We can't close like we do in the filename case above? The writer can
477 # get a "broken pipe".
478 self._PushClose(read_fd)
479
480 thunk = _HereDocWriterThunk(write_fd, arg.body)
481
482 # Use PIPE_SIZE to save a process in the case of small here
483 # docs, which are the common case. (dash does this.)
484
485 # Note: could instrument this to see how often it happens.
486 # Though strace -ff can also work.
487 start_process = len(arg.body) > 4096
488 #start_process = True
489
490 if start_process:
491 here_proc = Process(thunk, self.job_control, self.job_list,
492 self.tracer)
493
494 # NOTE: we could close the read pipe here, but it doesn't really
495 # matter because we control the code.
496 here_proc.StartProcess(trace.HereDoc)
497 #log('Started %s as %d', here_proc, pid)
498 self._PushWait(here_proc)
499
500 # Now that we've started the child, close it in the parent.
501 posix.close(write_fd)
502
503 else:
504 posix.write(write_fd, arg.body)
505 posix.close(write_fd)
506
507 def Push(self, redirects, err_out):
508 # type: (List[RedirValue], List[error.IOError_OSError]) -> None
509 """Apply a group of redirects and remember to undo them."""
510
511 #log('> fd_state.Push %s', redirects)
512 new_frame = _FdFrame()
513 self.stack.append(new_frame)
514 self.cur_frame = new_frame
515
516 for r in redirects:
517 #log('apply %s', r)
518 with ui.ctx_Location(self.errfmt, r.op_loc):
519 try:
520 self._ApplyRedirect(r)
521 except (IOError, OSError) as e:
522 err_out.append(e)
523 # This can fail too
524 self.Pop(err_out)
525 return # for bad descriptor, etc.
526
527 def PushStdinFromPipe(self, r):
528 # type: (int) -> bool
529 """Save the current stdin and make it come from descriptor 'r'.
530
531 'r' is typically the read-end of a pipe. For 'lastpipe'/ZSH
532 semantics of
533
534 echo foo | read line; echo $line
535 """
536 new_frame = _FdFrame()
537 self.stack.append(new_frame)
538 self.cur_frame = new_frame
539
540 self._PushDup(r, redir_loc.Fd(0))
541 return True
542
543 def Pop(self, err_out):
544 # type: (List[error.IOError_OSError]) -> None
545 frame = self.stack.pop()
546 #log('< Pop %s', frame)
547 for rf in reversed(frame.saved):
548 if rf.saved_fd == NO_FD:
549 #log('Close %d', orig)
550 try:
551 posix.close(rf.orig_fd)
552 except (IOError, OSError) as e:
553 err_out.append(e)
554 log('Error closing descriptor %d: %s', rf.orig_fd,
555 pyutil.strerror(e))
556 return
557 else:
558 try:
559 posix.dup2(rf.saved_fd, rf.orig_fd)
560 except (IOError, OSError) as e:
561 err_out.append(e)
562 log('dup2(%d, %d) error: %s', rf.saved_fd, rf.orig_fd,
563 pyutil.strerror(e))
564 #log('fd state:')
565 #posix.system('ls -l /proc/%s/fd' % posix.getpid())
566 return
567 posix.close(rf.saved_fd)
568 #log('dup2 %s %s', saved, orig)
569
570 # Wait for here doc processes to finish.
571 for proc in frame.need_wait:
572 unused_status = proc.Wait(self.waiter)
573
574 def MakePermanent(self):
575 # type: () -> None
576 self.cur_frame.Forget()
577
578
579class ChildStateChange(object):
580
581 def __init__(self):
582 # type: () -> None
583 """Empty constructor for mycpp."""
584 pass
585
586 def Apply(self):
587 # type: () -> None
588 raise NotImplementedError()
589
590 def ApplyFromParent(self, proc):
591 # type: (Process) -> None
592 """Noop for all state changes other than SetPgid for mycpp."""
593 pass
594
595
596class StdinFromPipe(ChildStateChange):
597
598 def __init__(self, pipe_read_fd, w):
599 # type: (int, int) -> None
600 self.r = pipe_read_fd
601 self.w = w
602
603 def __repr__(self):
604 # type: () -> str
605 return '<StdinFromPipe %d %d>' % (self.r, self.w)
606
607 def Apply(self):
608 # type: () -> None
609 posix.dup2(self.r, 0)
610 posix.close(self.r) # close after dup
611
612 posix.close(self.w) # we're reading from the pipe, not writing
613 #log('child CLOSE w %d pid=%d', self.w, posix.getpid())
614
615
616class StdoutToPipe(ChildStateChange):
617
618 def __init__(self, r, pipe_write_fd):
619 # type: (int, int) -> None
620 self.r = r
621 self.w = pipe_write_fd
622
623 def __repr__(self):
624 # type: () -> str
625 return '<StdoutToPipe %d %d>' % (self.r, self.w)
626
627 def Apply(self):
628 # type: () -> None
629 posix.dup2(self.w, 1)
630 posix.close(self.w) # close after dup
631
632 posix.close(self.r) # we're writing to the pipe, not reading
633 #log('child CLOSE r %d pid=%d', self.r, posix.getpid())
634
635
636INVALID_PGID = -1
637# argument to setpgid() that means the process is its own leader
638OWN_LEADER = 0
639
640
641class SetPgid(ChildStateChange):
642
643 def __init__(self, pgid, tracer):
644 # type: (int, dev.Tracer) -> None
645 self.pgid = pgid
646 self.tracer = tracer
647
648 def Apply(self):
649 # type: () -> None
650 try:
651 posix.setpgid(0, self.pgid)
652 except (IOError, OSError) as e:
653 self.tracer.OtherMessage(
654 'osh: child %d failed to set its process group to %d: %s' %
655 (posix.getpid(), self.pgid, pyutil.strerror(e)))
656
657 def ApplyFromParent(self, proc):
658 # type: (Process) -> None
659 try:
660 posix.setpgid(proc.pid, self.pgid)
661 except (IOError, OSError) as e:
662 self.tracer.OtherMessage(
663 'osh: parent failed to set process group for PID %d to %d: %s'
664 % (proc.pid, self.pgid, pyutil.strerror(e)))
665
666
667class ExternalProgram(object):
668 """The capability to execute an external program like 'ls'."""
669
670 def __init__(
671 self,
672 hijack_shebang, # type: str
673 fd_state, # type: FdState
674 errfmt, # type: ErrorFormatter
675 debug_f, # type: _DebugFile
676 ):
677 # type: (...) -> None
678 """
679 Args:
680 hijack_shebang: The path of an interpreter to run instead of the one
681 specified in the shebang line. May be empty.
682 """
683 self.hijack_shebang = hijack_shebang
684 self.fd_state = fd_state
685 self.errfmt = errfmt
686 self.debug_f = debug_f
687
688 def Exec(self, argv0_path, cmd_val, environ):
689 # type: (str, cmd_value.Argv, Dict[str, str]) -> None
690 """Execute a program and exit this process.
691
692 Called by: ls / exec ls / ( ls / )
693 """
694 probe('process', 'ExternalProgram_Exec', argv0_path)
695 self._Exec(argv0_path, cmd_val.argv, cmd_val.arg_locs[0], environ,
696 True)
697 assert False, "This line should never execute" # NO RETURN
698
699 def _Exec(self, argv0_path, argv, argv0_loc, environ, should_retry):
700 # type: (str, List[str], loc_t, Dict[str, str], bool) -> None
701 if len(self.hijack_shebang):
702 opened = True
703 try:
704 f = self.fd_state.Open(argv0_path)
705 except (IOError, OSError) as e:
706 opened = False
707
708 if opened:
709 with ctx_FileCloser(f):
710 # Test if the shebang looks like a shell. TODO: The file might be
711 # binary with no newlines, so read 80 bytes instead of readline().
712
713 #line = f.read(80) # type: ignore # TODO: fix this
714 line = f.readline()
715
716 if match.ShouldHijack(line):
717 h_argv = [self.hijack_shebang, argv0_path]
718 h_argv.extend(argv[1:])
719 argv = h_argv
720 argv0_path = self.hijack_shebang
721 self.debug_f.writeln('Hijacked: %s' % argv0_path)
722 else:
723 #self.debug_f.log('Not hijacking %s (%r)', argv, line)
724 pass
725
726 try:
727 posix.execve(argv0_path, argv, environ)
728 except (IOError, OSError) as e:
729 # Run with /bin/sh when ENOEXEC error (no shebang). All shells do this.
730 if e.errno == ENOEXEC and should_retry:
731 new_argv = ['/bin/sh', argv0_path]
732 new_argv.extend(argv[1:])
733 self._Exec('/bin/sh', new_argv, argv0_loc, environ, False)
734 # NO RETURN
735
736 # Would be nice: when the path is relative and ENOENT: print PWD and do
737 # spelling correction?
738
739 self.errfmt.Print_(
740 "Can't execute %r: %s" % (argv0_path, pyutil.strerror(e)),
741 argv0_loc)
742
743 # POSIX mentions 126 and 127 for two specific errors. The rest are
744 # unspecified.
745 #
746 # http://pubs.opengroup.org/onlinepubs/9699919799.2016edition/utilities/V3_chap02.html#tag_18_08_02
747 if e.errno == EACCES:
748 status = 126
749 elif e.errno == ENOENT:
750 # TODO: most shells print 'command not found', rather than strerror()
751 # == "No such file or directory". That's better because it's at the
752 # end of the path search, and we're never searching for a directory.
753 status = 127
754 else:
755 # dash uses 2, but we use that for parse errors. This seems to be
756 # consistent with mksh and zsh.
757 status = 127
758
759 posix._exit(status)
760 # NO RETURN
761
762
763class Thunk(object):
764 """Abstract base class for things runnable in another process."""
765
766 def __init__(self):
767 # type: () -> None
768 """Empty constructor for mycpp."""
769 pass
770
771 def Run(self):
772 # type: () -> None
773 """Returns a status code."""
774 raise NotImplementedError()
775
776 def UserString(self):
777 # type: () -> str
778 """Display for the 'jobs' list."""
779 raise NotImplementedError()
780
781 def __repr__(self):
782 # type: () -> str
783 return self.UserString()
784
785
786class ExternalThunk(Thunk):
787 """An external executable."""
788
789 def __init__(self, ext_prog, argv0_path, cmd_val, environ):
790 # type: (ExternalProgram, str, cmd_value.Argv, Dict[str, str]) -> None
791 self.ext_prog = ext_prog
792 self.argv0_path = argv0_path
793 self.cmd_val = cmd_val
794 self.environ = environ
795
796 def UserString(self):
797 # type: () -> str
798
799 # NOTE: This is the format the Tracer uses.
800 # bash displays sleep $n & (code)
801 # but OSH displays sleep 1 & (argv array)
802 # We could switch the former but I'm not sure it's necessary.
803 tmp = [j8_lite.MaybeShellEncode(a) for a in self.cmd_val.argv]
804 return '[process] %s' % ' '.join(tmp)
805
806 def Run(self):
807 # type: () -> None
808 """An ExternalThunk is run in parent for the exec builtin."""
809 self.ext_prog.Exec(self.argv0_path, self.cmd_val, self.environ)
810
811
812class SubProgramThunk(Thunk):
813 """A subprogram that can be executed in another process."""
814
815 def __init__(self,
816 cmd_ev,
817 node,
818 trap_state,
819 multi_trace,
820 inherit_errexit=True):
821 # type: (CommandEvaluator, command_t, trap_osh.TrapState, dev.MultiTracer, bool) -> None
822 self.cmd_ev = cmd_ev
823 self.node = node
824 self.trap_state = trap_state
825 self.multi_trace = multi_trace
826 self.inherit_errexit = inherit_errexit # for bash errexit compatibility
827
828 def UserString(self):
829 # type: () -> str
830
831 # NOTE: These can be pieces of a pipeline, so they're arbitrary nodes.
832 # TODO: Extract SPIDS from node to display source? Note that
833 # CompoundStatus also has locations of each pipeline component; see
834 # Executor.RunPipeline()
835 thunk_str = ui.CommandType(self.node)
836 return '[subprog] %s' % thunk_str
837
838 def Run(self):
839 # type: () -> None
840 #self.errfmt.OneLineErrExit() # don't quote code in child processes
841 probe('process', 'SubProgramThunk_Run')
842
843 # TODO: break circular dep. Bit flags could go in ASDL or headers.
844 from osh import cmd_eval
845
846 # signal handlers aren't inherited
847 self.trap_state.ClearForSubProgram()
848
849 # NOTE: may NOT return due to exec().
850 if not self.inherit_errexit:
851 self.cmd_ev.mutable_opts.DisableErrExit()
852 try:
853 # optimize to eliminate redundant subshells like ( echo hi ) | wc -l etc.
854 self.cmd_ev.ExecuteAndCatch(self.node, cmd_flags=cmd_eval.Optimize)
855 status = self.cmd_ev.LastStatus()
856 # NOTE: We ignore the is_fatal return value. The user should set -o
857 # errexit so failures in subprocesses cause failures in the parent.
858 except util.UserExit as e:
859 status = e.status
860
861 # Handle errors in a subshell. These two cases are repeated from main()
862 # and the core/completion.py hook.
863 except KeyboardInterrupt:
864 print('')
865 status = 130 # 128 + 2
866 except (IOError, OSError) as e:
867 print_stderr('oils I/O error (subprogram): %s' %
868 pyutil.strerror(e))
869 status = 2
870
871 # If ProcessInit() doesn't turn off buffering, this is needed before
872 # _exit()
873 pyos.FlushStdout()
874
875 self.multi_trace.WriteDumps()
876
877 # We do NOT want to raise SystemExit here. Otherwise dev.Tracer::Pop()
878 # gets called in BOTH processes.
879 # The crash dump seems to be unaffected.
880 posix._exit(status)
881
882
883class _HereDocWriterThunk(Thunk):
884 """Write a here doc to one end of a pipe.
885
886 May be be executed in either a child process or the main shell
887 process.
888 """
889
890 def __init__(self, w, body_str):
891 # type: (int, str) -> None
892 self.w = w
893 self.body_str = body_str
894
895 def UserString(self):
896 # type: () -> str
897
898 # You can hit Ctrl-Z and the here doc writer will be suspended! Other
899 # shells don't have this problem because they use temp files! That's a bit
900 # unfortunate.
901 return '[here doc writer]'
902
903 def Run(self):
904 # type: () -> None
905 """do_exit: For small pipelines."""
906 probe('process', 'HereDocWriterThunk_Run')
907 #log('Writing %r', self.body_str)
908 posix.write(self.w, self.body_str)
909 #log('Wrote %r', self.body_str)
910 posix.close(self.w)
911 #log('Closed %d', self.w)
912
913 posix._exit(0)
914
915
916class Job(object):
917 """Interface for both Process and Pipeline.
918
919 They both can be put in the background and waited on.
920
921 Confusing thing about pipelines in the background: They have TOO MANY NAMES.
922
923 sleep 1 | sleep 2 &
924
925 - The LAST PID is what's printed at the prompt. This is $!, a PROCESS ID and
926 not a JOB ID.
927 # https://www.gnu.org/software/bash/manual/html_node/Special-Parameters.html#Special-Parameters
928 - The process group leader (setpgid) is the FIRST PID.
929 - It's also %1 or %+. The last job started.
930 """
931
932 def __init__(self):
933 # type: () -> None
934 # Initial state with & or Ctrl-Z is Running.
935 self.state = job_state_e.Running
936 self.job_id = -1
937 self.in_background = False
938
939 def DisplayJob(self, job_id, f, style):
940 # type: (int, mylib.Writer, int) -> None
941 raise NotImplementedError()
942
943 def State(self):
944 # type: () -> job_state_t
945 return self.state
946
947 def ProcessGroupId(self):
948 # type: () -> int
949 """Return the process group ID associated with this job."""
950 raise NotImplementedError()
951
952 def JobWait(self, waiter):
953 # type: (Waiter) -> wait_status_t
954 """Wait for this process/pipeline to be stopped or finished."""
955 raise NotImplementedError()
956
957 def SetBackground(self):
958 # type: () -> None
959 """Record that this job is running in the background."""
960 self.in_background = True
961
962 def SetForeground(self):
963 # type: () -> None
964 """Record that this job is running in the foreground."""
965 self.in_background = False
966
967
968class Process(Job):
969 """A process to run.
970
971 TODO: Should we make it clear that this is a FOREGROUND process? A
972 background process is wrapped in a "job". It is unevaluated.
973
974 It provides an API to manipulate file descriptor state in parent and child.
975 """
976
977 def __init__(self, thunk, job_control, job_list, tracer):
978 # type: (Thunk, JobControl, JobList, dev.Tracer) -> None
979 """
980 Args:
981 thunk: Thunk instance
982 job_list: for process bookkeeping
983 """
984 Job.__init__(self)
985 assert isinstance(thunk, Thunk), thunk
986 self.thunk = thunk
987 self.job_control = job_control
988 self.job_list = job_list
989 self.tracer = tracer
990
991 # For pipelines
992 self.parent_pipeline = None # type: Pipeline
993 self.state_changes = [] # type: List[ChildStateChange]
994 self.close_r = -1
995 self.close_w = -1
996
997 self.pid = -1
998 self.status = -1
999
1000 def Init_ParentPipeline(self, pi):
1001 # type: (Pipeline) -> None
1002 """For updating PIPESTATUS."""
1003 self.parent_pipeline = pi
1004
1005 def __repr__(self):
1006 # type: () -> str
1007
1008 # note: be wary of infinite mutual recursion
1009 #s = ' %s' % self.parent_pipeline if self.parent_pipeline else ''
1010 #return '<Process %s%s>' % (self.thunk, s)
1011 return '<Process %s %s>' % (_JobStateStr(self.state), self.thunk)
1012
1013 def ProcessGroupId(self):
1014 # type: () -> int
1015 """Returns the group ID of this process."""
1016 # This should only ever be called AFTER the process has started
1017 assert self.pid != -1
1018 if self.parent_pipeline:
1019 # XXX: Maybe we should die here instead? Unclear if this branch
1020 # should even be reachable with the current builtins.
1021 return self.parent_pipeline.ProcessGroupId()
1022
1023 return self.pid
1024
1025 def DisplayJob(self, job_id, f, style):
1026 # type: (int, mylib.Writer, int) -> None
1027 if job_id == -1:
1028 job_id_str = ' '
1029 else:
1030 job_id_str = '%%%d' % job_id
1031 if style == STYLE_PID_ONLY:
1032 f.write('%d\n' % self.pid)
1033 else:
1034 f.write('%s %d %7s ' %
1035 (job_id_str, self.pid, _JobStateStr(self.state)))
1036 f.write(self.thunk.UserString())
1037 f.write('\n')
1038
1039 def AddStateChange(self, s):
1040 # type: (ChildStateChange) -> None
1041 self.state_changes.append(s)
1042
1043 def AddPipeToClose(self, r, w):
1044 # type: (int, int) -> None
1045 self.close_r = r
1046 self.close_w = w
1047
1048 def MaybeClosePipe(self):
1049 # type: () -> None
1050 if self.close_r != -1:
1051 posix.close(self.close_r)
1052 posix.close(self.close_w)
1053
1054 def StartProcess(self, why):
1055 # type: (trace_t) -> int
1056 """Start this process with fork(), handling redirects."""
1057 pid = posix.fork()
1058 if pid < 0:
1059 # When does this happen?
1060 e_die('Fatal error in posix.fork()')
1061
1062 elif pid == 0: # child
1063 # Note: this happens in BOTH interactive and non-interactive shells.
1064 # We technically don't need to do most of it in non-interactive, since we
1065 # did not change state in InitInteractiveShell().
1066
1067 for st in self.state_changes:
1068 st.Apply()
1069
1070 # Python sets SIGPIPE handler to SIG_IGN by default. Child processes
1071 # shouldn't have this.
1072 # https://docs.python.org/2/library/signal.html
1073 # See Python/pythonrun.c.
1074 pyos.Sigaction(SIGPIPE, SIG_DFL)
1075
1076 # Respond to Ctrl-\ (core dump)
1077 pyos.Sigaction(SIGQUIT, SIG_DFL)
1078
1079 # Only standalone children should get Ctrl-Z. Pipelines remain in the
1080 # foreground because suspending them is difficult with our 'lastpipe'
1081 # semantics.
1082 pid = posix.getpid()
1083 if posix.getpgid(0) == pid and self.parent_pipeline is None:
1084 pyos.Sigaction(SIGTSTP, SIG_DFL)
1085
1086 # More signals from
1087 # https://www.gnu.org/software/libc/manual/html_node/Launching-Jobs.html
1088 # (but not SIGCHLD)
1089 pyos.Sigaction(SIGTTOU, SIG_DFL)
1090 pyos.Sigaction(SIGTTIN, SIG_DFL)
1091
1092 self.tracer.OnNewProcess(pid)
1093 # clear foreground pipeline for subshells
1094 self.thunk.Run()
1095 # Never returns
1096
1097 #log('STARTED process %s, pid = %d', self, pid)
1098 self.tracer.OnProcessStart(pid, why)
1099
1100 # Class invariant: after the process is started, it stores its PID.
1101 self.pid = pid
1102
1103 # SetPgid needs to be applied from the child and the parent to avoid
1104 # racing in calls to tcsetpgrp() in the parent. See APUE sec. 9.2.
1105 for st in self.state_changes:
1106 st.ApplyFromParent(self)
1107
1108 # Program invariant: We keep track of every child process!
1109 self.job_list.AddChildProcess(pid, self)
1110
1111 return pid
1112
1113 def Wait(self, waiter):
1114 # type: (Waiter) -> int
1115 """Wait for this process to finish."""
1116 while self.state == job_state_e.Running:
1117 # Only return if there's nothing to wait for. Keep waiting if we were
1118 # interrupted with a signal.
1119 if waiter.WaitForOne() == W1_ECHILD:
1120 break
1121
1122 assert self.status >= 0, self.status
1123 return self.status
1124
1125 def JobWait(self, waiter):
1126 # type: (Waiter) -> wait_status_t
1127 # wait builtin can be interrupted
1128 while self.state == job_state_e.Running:
1129 result = waiter.WaitForOne()
1130
1131 if result >= 0: # signal
1132 return wait_status.Cancelled(result)
1133
1134 if result == W1_ECHILD:
1135 break
1136
1137 return wait_status.Proc(self.status)
1138
1139 def WhenStopped(self, stop_sig):
1140 # type: (int) -> None
1141
1142 # 128 is a shell thing
1143 # https://www.gnu.org/software/bash/manual/html_node/Exit-Status.html
1144 self.status = 128 + stop_sig
1145 self.state = job_state_e.Stopped
1146
1147 if self.job_id == -1:
1148 # This process was started in the foreground
1149 self.job_list.AddJob(self)
1150
1151 if not self.in_background:
1152 self.job_control.MaybeTakeTerminal()
1153 self.SetBackground()
1154
1155 def WhenDone(self, pid, status):
1156 # type: (int, int) -> None
1157 """Called by the Waiter when this Process finishes."""
1158
1159 #log('WhenDone %d %d', pid, status)
1160 assert pid == self.pid, 'Expected %d, got %d' % (self.pid, pid)
1161 self.status = status
1162 self.state = job_state_e.Done
1163 if self.parent_pipeline:
1164 self.parent_pipeline.WhenDone(pid, status)
1165 else:
1166 if self.job_id != -1:
1167 # Job might have been brought to the foreground after being
1168 # assigned a job ID.
1169 if self.in_background:
1170 print_stderr('[%d] Done PID %d' % (self.job_id, self.pid))
1171
1172 self.job_list.RemoveJob(self.job_id)
1173
1174 self.job_list.RemoveChildProcess(self.pid)
1175
1176 if not self.in_background:
1177 self.job_control.MaybeTakeTerminal()
1178
1179 def RunProcess(self, waiter, why):
1180 # type: (Waiter, trace_t) -> int
1181 """Run this process synchronously."""
1182 self.StartProcess(why)
1183 # ShellExecutor might be calling this for the last part of a pipeline.
1184 if self.parent_pipeline is None:
1185 # QUESTION: Can the PGID of a single process just be the PID? i.e. avoid
1186 # calling getpgid()?
1187 self.job_control.MaybeGiveTerminal(posix.getpgid(self.pid))
1188 return self.Wait(waiter)
1189
1190
1191class ctx_Pipe(object):
1192
1193 def __init__(self, fd_state, fd, err_out):
1194 # type: (FdState, int, List[error.IOError_OSError]) -> None
1195 fd_state.PushStdinFromPipe(fd)
1196 self.fd_state = fd_state
1197 self.err_out = err_out
1198
1199 def __enter__(self):
1200 # type: () -> None
1201 pass
1202
1203 def __exit__(self, type, value, traceback):
1204 # type: (Any, Any, Any) -> None
1205 self.fd_state.Pop(self.err_out)
1206
1207
1208class Pipeline(Job):
1209 """A pipeline of processes to run.
1210
1211 Cases we handle:
1212
1213 foo | bar
1214 $(foo | bar)
1215 foo | bar | read v
1216 """
1217
1218 def __init__(self, sigpipe_status_ok, job_control, job_list, tracer):
1219 # type: (bool, JobControl, JobList, dev.Tracer) -> None
1220 Job.__init__(self)
1221 self.job_control = job_control
1222 self.job_list = job_list
1223 self.tracer = tracer
1224
1225 self.procs = [] # type: List[Process]
1226 self.pids = [] # type: List[int] # pids in order
1227 self.pipe_status = [] # type: List[int] # status in order
1228 self.status = -1 # for 'wait' jobs
1229
1230 self.pgid = INVALID_PGID
1231
1232 # Optional for foreground
1233 self.last_thunk = None # type: Tuple[CommandEvaluator, command_t]
1234 self.last_pipe = None # type: Tuple[int, int]
1235
1236 self.sigpipe_status_ok = sigpipe_status_ok
1237
1238 def ProcessGroupId(self):
1239 # type: () -> int
1240 """Returns the group ID of this pipeline."""
1241 return self.pgid
1242
1243 def DisplayJob(self, job_id, f, style):
1244 # type: (int, mylib.Writer, int) -> None
1245 if style == STYLE_PID_ONLY:
1246 f.write('%d\n' % self.procs[0].pid)
1247 else:
1248 # Note: this is STYLE_LONG.
1249 for i, proc in enumerate(self.procs):
1250 if i == 0: # show job ID for first element in pipeline
1251 job_id_str = '%%%d' % job_id
1252 else:
1253 job_id_str = ' ' # 2 spaces
1254
1255 f.write('%s %d %7s ' %
1256 (job_id_str, proc.pid, _JobStateStr(proc.state)))
1257 f.write(proc.thunk.UserString())
1258 f.write('\n')
1259
1260 def DebugPrint(self):
1261 # type: () -> None
1262 print('Pipeline in state %s' % _JobStateStr(self.state))
1263 if mylib.PYTHON: # %s for Process not allowed in C++
1264 for proc in self.procs:
1265 print(' proc %s' % proc)
1266 _, last_node = self.last_thunk
1267 print(' last %s' % last_node)
1268 print(' pipe_status %s' % self.pipe_status)
1269
1270 def Add(self, p):
1271 # type: (Process) -> None
1272 """Append a process to the pipeline."""
1273 if len(self.procs) == 0:
1274 self.procs.append(p)
1275 return
1276
1277 r, w = posix.pipe()
1278 #log('pipe for %s: %d %d', p, r, w)
1279 prev = self.procs[-1]
1280
1281 prev.AddStateChange(StdoutToPipe(r, w)) # applied on StartPipeline()
1282 p.AddStateChange(StdinFromPipe(r, w)) # applied on StartPipeline()
1283
1284 p.AddPipeToClose(r, w) # MaybeClosePipe() on StartPipeline()
1285
1286 self.procs.append(p)
1287
1288 def AddLast(self, thunk):
1289 # type: (Tuple[CommandEvaluator, command_t]) -> None
1290 """Append the last noden to the pipeline.
1291
1292 This is run in the CURRENT process. It is OPTIONAL, because
1293 pipelines in the background are run uniformly.
1294 """
1295 self.last_thunk = thunk
1296
1297 assert len(self.procs) != 0
1298
1299 r, w = posix.pipe()
1300 prev = self.procs[-1]
1301 prev.AddStateChange(StdoutToPipe(r, w))
1302
1303 self.last_pipe = (r, w) # So we can connect it to last_thunk
1304
1305 def StartPipeline(self, waiter):
1306 # type: (Waiter) -> None
1307
1308 # If we are creating a pipeline in a subshell or we aren't running with job
1309 # control, our children should remain in our inherited process group.
1310 # the pipelines's group ID.
1311 if self.job_control.Enabled():
1312 self.pgid = OWN_LEADER # first process in pipeline is the leader
1313
1314 for i, proc in enumerate(self.procs):
1315 if self.pgid != INVALID_PGID:
1316 proc.AddStateChange(SetPgid(self.pgid, self.tracer))
1317
1318 # Figure out the pid
1319 pid = proc.StartProcess(trace.PipelinePart)
1320 if i == 0 and self.pgid != INVALID_PGID:
1321 # Mimic bash and use the PID of the FIRST process as the group for the
1322 # whole pipeline.
1323 self.pgid = pid
1324
1325 self.pids.append(pid)
1326 self.pipe_status.append(-1) # uninitialized
1327
1328 # NOTE: This is done in the SHELL PROCESS after every fork() call.
1329 # It can't be done at the end; otherwise processes will have descriptors
1330 # from non-adjacent pipes.
1331 proc.MaybeClosePipe()
1332
1333 if self.last_thunk:
1334 self.pipe_status.append(-1) # for self.last_thunk
1335
1336 def LastPid(self):
1337 # type: () -> int
1338 """For the odd $! variable.
1339
1340 It would be better if job IDs or PGIDs were used consistently.
1341 """
1342 return self.pids[-1]
1343
1344 def Wait(self, waiter):
1345 # type: (Waiter) -> List[int]
1346 """Wait for this pipeline to finish."""
1347
1348 assert self.procs, "no procs for Wait()"
1349 # waitpid(-1) zero or more times
1350 while self.state == job_state_e.Running:
1351 # Keep waiting until there's nothing to wait for.
1352 if waiter.WaitForOne() == W1_ECHILD:
1353 break
1354
1355 return self.pipe_status
1356
1357 def JobWait(self, waiter):
1358 # type: (Waiter) -> wait_status_t
1359 """Called by 'wait' builtin, e.g. 'wait %1'."""
1360 # wait builtin can be interrupted
1361 assert self.procs, "no procs for Wait()"
1362 while self.state == job_state_e.Running:
1363 result = waiter.WaitForOne()
1364
1365 if result >= 0: # signal
1366 return wait_status.Cancelled(result)
1367
1368 if result == W1_ECHILD:
1369 break
1370
1371 return wait_status.Pipeline(self.pipe_status)
1372
1373 def RunLastPart(self, waiter, fd_state):
1374 # type: (Waiter, FdState) -> List[int]
1375 """Run this pipeline synchronously (foreground pipeline).
1376
1377 Returns:
1378 pipe_status (list of integers).
1379 """
1380 assert len(self.pids) == len(self.procs)
1381
1382 # TODO: break circular dep. Bit flags could go in ASDL or headers.
1383 from osh import cmd_eval
1384
1385 # This is tcsetpgrp()
1386 # TODO: fix race condition -- I believe the first process could have
1387 # stopped already, and thus getpgid() will fail
1388 self.job_control.MaybeGiveTerminal(self.pgid)
1389
1390 # Run the last part of the pipeline IN PARALLEL with other processes. It
1391 # may or may not fork:
1392 # echo foo | read line # no fork, the builtin runs in THIS shell process
1393 # ls | wc -l # fork for 'wc'
1394
1395 cmd_ev, last_node = self.last_thunk
1396
1397 assert self.last_pipe is not None
1398 r, w = self.last_pipe # set in AddLast()
1399 posix.close(w) # we will not write here
1400
1401 # Fix lastpipe / job control / DEBUG trap interaction
1402 cmd_flags = cmd_eval.NoDebugTrap if self.job_control.Enabled() else 0
1403
1404 # The ERR trap only runs for the WHOLE pipeline, not the COMPONENTS in
1405 # a pipeline.
1406 cmd_flags |= cmd_eval.NoErrTrap
1407
1408 io_errors = [] # type: List[error.IOError_OSError]
1409 with ctx_Pipe(fd_state, r, io_errors):
1410 cmd_ev.ExecuteAndCatch(last_node, cmd_flags)
1411
1412 if len(io_errors):
1413 e_die('Error setting up last part of pipeline: %s' %
1414 pyutil.strerror(io_errors[0]))
1415
1416 # We won't read anymore. If we don't do this, then 'cat' in 'cat
1417 # /dev/urandom | sleep 1' will never get SIGPIPE.
1418 posix.close(r)
1419
1420 self.pipe_status[-1] = cmd_ev.LastStatus()
1421 if self.AllDone():
1422 self.state = job_state_e.Done
1423
1424 #log('pipestatus before all have finished = %s', self.pipe_status)
1425 return self.Wait(waiter)
1426
1427 def AllDone(self):
1428 # type: () -> bool
1429
1430 # mycpp rewrite: all(status != -1 for status in self.pipe_status)
1431 for status in self.pipe_status:
1432 if status == -1:
1433 return False
1434 return True
1435
1436 def WhenDone(self, pid, status):
1437 # type: (int, int) -> None
1438 """Called by Process.WhenDone."""
1439 #log('Pipeline WhenDone %d %d', pid, status)
1440 i = self.pids.index(pid)
1441 assert i != -1, 'Unexpected PID %d' % pid
1442
1443 if status == 141 and self.sigpipe_status_ok:
1444 status = 0
1445
1446 self.job_list.RemoveChildProcess(pid)
1447 self.pipe_status[i] = status
1448 if self.AllDone():
1449 if self.job_id != -1:
1450 # Job might have been brought to the foreground after being
1451 # assigned a job ID.
1452 if self.in_background:
1453 print_stderr('[%d] Done PGID %d' %
1454 (self.job_id, self.pids[0]))
1455
1456 self.job_list.RemoveJob(self.job_id)
1457
1458 # status of pipeline is status of last process
1459 self.status = self.pipe_status[-1]
1460 self.state = job_state_e.Done
1461 if not self.in_background:
1462 self.job_control.MaybeTakeTerminal()
1463
1464
1465def _JobStateStr(i):
1466 # type: (job_state_t) -> str
1467 return job_state_str(i)[10:] # remove 'job_state.'
1468
1469
1470def _GetTtyFd():
1471 # type: () -> int
1472 """Returns -1 if stdio is not a TTY."""
1473 try:
1474 return posix.open("/dev/tty", O_NONBLOCK | O_NOCTTY | O_RDWR, 0o666)
1475 except (IOError, OSError) as e:
1476 return -1
1477
1478
1479class ctx_TerminalControl(object):
1480
1481 def __init__(self, job_control, errfmt):
1482 # type: (JobControl, ui.ErrorFormatter) -> None
1483 job_control.InitJobControl()
1484 self.job_control = job_control
1485 self.errfmt = errfmt
1486
1487 def __enter__(self):
1488 # type: () -> None
1489 pass
1490
1491 def __exit__(self, type, value, traceback):
1492 # type: (Any, Any, Any) -> None
1493
1494 # Return the TTY to the original owner before exiting.
1495 try:
1496 self.job_control.MaybeReturnTerminal()
1497 except error.FatalRuntime as e:
1498 # Don't abort the shell on error, just print a message.
1499 self.errfmt.PrettyPrintError(e)
1500
1501
1502class JobControl(object):
1503 """Interface to setpgid(), tcsetpgrp(), etc."""
1504
1505 def __init__(self):
1506 # type: () -> None
1507
1508 # The main shell's PID and group ID.
1509 self.shell_pid = -1
1510 self.shell_pgid = -1
1511
1512 # The fd of the controlling tty. Set to -1 when job control is disabled.
1513 self.shell_tty_fd = -1
1514
1515 # For giving the terminal back to our parent before exiting (if not a login
1516 # shell).
1517 self.original_tty_pgid = -1
1518
1519 def InitJobControl(self):
1520 # type: () -> None
1521 self.shell_pid = posix.getpid()
1522 orig_shell_pgid = posix.getpgid(0)
1523 self.shell_pgid = orig_shell_pgid
1524 self.shell_tty_fd = _GetTtyFd()
1525
1526 # If we aren't the leader of our process group, create a group and mark
1527 # ourselves as the leader.
1528 if self.shell_pgid != self.shell_pid:
1529 try:
1530 posix.setpgid(self.shell_pid, self.shell_pid)
1531 self.shell_pgid = self.shell_pid
1532 except (IOError, OSError) as e:
1533 self.shell_tty_fd = -1
1534
1535 if self.shell_tty_fd != -1:
1536 self.original_tty_pgid = posix.tcgetpgrp(self.shell_tty_fd)
1537
1538 # If stdio is a TTY, put the shell's process group in the foreground.
1539 try:
1540 posix.tcsetpgrp(self.shell_tty_fd, self.shell_pgid)
1541 except (IOError, OSError) as e:
1542 # We probably aren't in the session leader's process group. Disable job
1543 # control.
1544 self.shell_tty_fd = -1
1545 self.shell_pgid = orig_shell_pgid
1546 posix.setpgid(self.shell_pid, self.shell_pgid)
1547
1548 def Enabled(self):
1549 # type: () -> bool
1550
1551 # TODO: get rid of this syscall? SubProgramThunk should set a flag I
1552 # think.
1553 curr_pid = posix.getpid()
1554 # Only the main shell should bother with job control functions.
1555 return curr_pid == self.shell_pid and self.shell_tty_fd != -1
1556
1557 # TODO: This isn't a PID. This is a process group ID?
1558 #
1559 # What should the table look like?
1560 #
1561 # Do we need the last PID? I don't know why bash prints that. Probably so
1562 # you can do wait $!
1563 # wait -n waits for any node to go from job_state_e.Running to job_state_e.Done?
1564 #
1565 # And it needs a flag for CURRENT, for the implicit arg to 'fg'.
1566 # job_id is just an integer. This is sort of lame.
1567 #
1568 # [job_id, flag, pgid, job_state, node]
1569
1570 def MaybeGiveTerminal(self, pgid):
1571 # type: (int) -> None
1572 """If stdio is a TTY, move the given process group to the
1573 foreground."""
1574 if not self.Enabled():
1575 # Only call tcsetpgrp when job control is enabled.
1576 return
1577
1578 try:
1579 posix.tcsetpgrp(self.shell_tty_fd, pgid)
1580 except (IOError, OSError) as e:
1581 e_die('osh: Failed to move process group %d to foreground: %s' %
1582 (pgid, pyutil.strerror(e)))
1583
1584 def MaybeTakeTerminal(self):
1585 # type: () -> None
1586 """If stdio is a TTY, return the main shell's process group to the
1587 foreground."""
1588 self.MaybeGiveTerminal(self.shell_pgid)
1589
1590 def MaybeReturnTerminal(self):
1591 # type: () -> None
1592 """Called before the shell exits."""
1593 self.MaybeGiveTerminal(self.original_tty_pgid)
1594
1595
1596class JobList(object):
1597 """Global list of jobs, used by a few builtins."""
1598
1599 def __init__(self):
1600 # type: () -> None
1601
1602 # job_id -> Job instance
1603 self.jobs = {} # type: Dict[int, Job]
1604
1605 # pid -> Process. This is for STOP notification.
1606 self.child_procs = {} # type: Dict[int, Process]
1607 self.debug_pipelines = [] # type: List[Pipeline]
1608
1609 # Counter used to assign IDs to jobs. It is incremented every time a job
1610 # is created. Once all active jobs are done it is reset to 1. I'm not
1611 # sure if this reset behavior is mandated by POSIX, but other shells do
1612 # it, so we mimic for the sake of compatibility.
1613 self.job_id = 1
1614
1615 def AddJob(self, job):
1616 # type: (Job) -> int
1617 """Add a background job to the list.
1618
1619 A job is either a Process or Pipeline. You can resume a job with 'fg',
1620 kill it with 'kill', etc.
1621
1622 Two cases:
1623
1624 1. async jobs: sleep 5 | sleep 4 &
1625 2. stopped jobs: sleep 5; then Ctrl-Z
1626 """
1627 job_id = self.job_id
1628 self.jobs[job_id] = job
1629 job.job_id = job_id
1630 self.job_id += 1
1631 return job_id
1632
1633 def RemoveJob(self, job_id):
1634 # type: (int) -> None
1635 """Process and Pipeline can call this."""
1636 mylib.dict_erase(self.jobs, job_id)
1637
1638 if len(self.jobs) == 0:
1639 self.job_id = 1
1640
1641 def AddChildProcess(self, pid, proc):
1642 # type: (int, Process) -> None
1643 """Every child process should be added here as soon as we know its PID.
1644
1645 When the Waiter gets an EXITED or STOPPED notification, we need
1646 to know about it so 'jobs' can work.
1647 """
1648 self.child_procs[pid] = proc
1649
1650 def RemoveChildProcess(self, pid):
1651 # type: (int) -> None
1652 """Remove the child process with the given PID."""
1653 mylib.dict_erase(self.child_procs, pid)
1654
1655 if mylib.PYTHON:
1656
1657 def AddPipeline(self, pi):
1658 # type: (Pipeline) -> None
1659 """For debugging only."""
1660 self.debug_pipelines.append(pi)
1661
1662 def ProcessFromPid(self, pid):
1663 # type: (int) -> Process
1664 """For wait $PID.
1665
1666 There's no way to wait for a pipeline with a PID. That uses job
1667 syntax, e.g. %1. Not a great interface.
1668 """
1669 return self.child_procs.get(pid)
1670
1671 def GetCurrentAndPreviousJobs(self):
1672 # type: () -> Tuple[Optional[Job], Optional[Job]]
1673 """Return the "current" and "previous" jobs (AKA `%+` and `%-`).
1674
1675 See the POSIX specification for the `jobs` builtin for details:
1676 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1677
1678 IMPORTANT NOTE: This method assumes that the jobs list will not change
1679 during its execution! This assumption holds for now because we only ever
1680 update the jobs list from the main loop after WaitPid() informs us of a
1681 change. If we implement `set -b` and install a signal handler for
1682 SIGCHLD we should be careful to synchronize it with this function. The
1683 unsafety of mutating GC data structures from a signal handler should
1684 make this a non-issue, but if bugs related to this appear this note may
1685 be helpful...
1686 """
1687 # Split all active jobs by state and sort each group by decreasing job
1688 # ID to approximate newness.
1689 stopped_jobs = [] # type: List[Job]
1690 running_jobs = [] # type: List[Job]
1691 for i in xrange(0, self.job_id):
1692 job = self.jobs.get(i, None)
1693 if not job:
1694 continue
1695
1696 if job.state == job_state_e.Stopped:
1697 stopped_jobs.append(job)
1698
1699 elif job.state == job_state_e.Running:
1700 running_jobs.append(job)
1701
1702 current = None # type: Optional[Job]
1703 previous = None # type: Optional[Job]
1704 # POSIX says: If there is any suspended job, then the current job shall
1705 # be a suspended job. If there are at least two suspended jobs, then the
1706 # previous job also shall be a suspended job.
1707 #
1708 # So, we will only return running jobs from here if there are no recent
1709 # stopped jobs.
1710 if len(stopped_jobs) > 0:
1711 current = stopped_jobs.pop()
1712
1713 if len(stopped_jobs) > 0:
1714 previous = stopped_jobs.pop()
1715
1716 if len(running_jobs) > 0 and not current:
1717 current = running_jobs.pop()
1718
1719 if len(running_jobs) > 0 and not previous:
1720 previous = running_jobs.pop()
1721
1722 if not previous:
1723 previous = current
1724
1725 return current, previous
1726
1727 def GetJobWithSpec(self, job_spec):
1728 # type: (str) -> Optional[Job]
1729 """Parse the given job spec and return the matching job. If there is no
1730 matching job, this function returns None.
1731
1732 See the POSIX spec for the `jobs` builtin for details about job specs:
1733 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1734 """
1735 if job_spec in CURRENT_JOB_SPECS:
1736 current, _ = self.GetCurrentAndPreviousJobs()
1737 return current
1738
1739 if job_spec == '%-':
1740 _, previous = self.GetCurrentAndPreviousJobs()
1741 return previous
1742
1743 # TODO: Add support for job specs based on prefixes of process argv.
1744 m = util.RegexSearch(r'^%([0-9]+)$', job_spec)
1745 if m is not None:
1746 assert len(m) == 2
1747 job_id = int(m[1])
1748 if job_id in self.jobs:
1749 return self.jobs[job_id]
1750
1751 return None
1752
1753 def DisplayJobs(self, style):
1754 # type: (int) -> None
1755 """Used by the 'jobs' builtin.
1756
1757 https://pubs.opengroup.org/onlinepubs/9699919799/utilities/jobs.html
1758
1759 "By default, the jobs utility shall display the status of all stopped jobs,
1760 running background jobs and all jobs whose status has changed and have not
1761 been reported by the shell."
1762 """
1763 # NOTE: A job is a background process or pipeline.
1764 #
1765 # echo hi | wc -l -- this starts two processes. Wait for TWO
1766 # echo hi | wc -l & -- this starts a process which starts two processes
1767 # Wait for ONE.
1768 #
1769 # 'jobs -l' GROUPS the PIDs by job. It has the job number, + - indicators
1770 # for %% and %-, PID, status, and "command".
1771 #
1772 # Every component of a pipeline is on the same line with 'jobs', but
1773 # they're separated into different lines with 'jobs -l'.
1774 #
1775 # See demo/jobs-builtin.sh
1776
1777 # $ jobs -l
1778 # [1]+ 24414 Stopped sleep 5
1779 # 24415 | sleep 5
1780 # [2] 24502 Running sleep 6
1781 # 24503 | sleep 6
1782 # 24504 | sleep 5 &
1783 # [3]- 24508 Running sleep 6
1784 # 24509 | sleep 6
1785 # 24510 | sleep 5 &
1786
1787 f = mylib.Stdout()
1788 for job_id, job in iteritems(self.jobs):
1789 # Use the %1 syntax
1790 job.DisplayJob(job_id, f, style)
1791
1792 def DebugPrint(self):
1793 # type: () -> None
1794
1795 f = mylib.Stdout()
1796 f.write('\n')
1797 f.write('[process debug info]\n')
1798
1799 for pid, proc in iteritems(self.child_procs):
1800 proc.DisplayJob(-1, f, STYLE_DEFAULT)
1801 #p = ' |' if proc.parent_pipeline else ''
1802 #print('%d %7s %s%s' % (pid, _JobStateStr(proc.state), proc.thunk.UserString(), p))
1803
1804 if len(self.debug_pipelines):
1805 f.write('\n')
1806 f.write('[pipeline debug info]\n')
1807 for pi in self.debug_pipelines:
1808 pi.DebugPrint()
1809
1810 def ListRecent(self):
1811 # type: () -> None
1812 """For jobs -n, which I think is also used in the interactive
1813 prompt."""
1814 pass
1815
1816 def NumRunning(self):
1817 # type: () -> int
1818 """Return the number of running jobs.
1819
1820 Used by 'wait' and 'wait -n'.
1821 """
1822 count = 0
1823 for _, job in iteritems(self.jobs): # mycpp rewrite: from itervalues()
1824 if job.State() == job_state_e.Running:
1825 count += 1
1826 return count
1827
1828
1829# Some WaitForOne() return values
1830W1_OK = -2 # waitpid(-1) returned
1831W1_ECHILD = -3 # no processes to wait for
1832W1_AGAIN = -4 # WNOHANG was passed and there were no state changes
1833
1834
1835class Waiter(object):
1836 """A capability to wait for processes.
1837
1838 This must be a singleton (and is because CommandEvaluator is a singleton).
1839
1840 Invariants:
1841 - Every child process is registered once
1842 - Every child process is waited for
1843
1844 Canonical example of why we need a GLOBAL waiter:
1845
1846 { sleep 3; echo 'done 3'; } &
1847 { sleep 4; echo 'done 4'; } &
1848
1849 # ... do arbitrary stuff ...
1850
1851 { sleep 1; exit 1; } | { sleep 2; exit 2; }
1852
1853 Now when you do wait() after starting the pipeline, you might get a pipeline
1854 process OR a background process! So you have to distinguish between them.
1855 """
1856
1857 def __init__(self, job_list, exec_opts, signal_safe, tracer):
1858 # type: (JobList, optview.Exec, pyos.SignalSafe, dev.Tracer) -> None
1859 self.job_list = job_list
1860 self.exec_opts = exec_opts
1861 self.signal_safe = signal_safe
1862 self.tracer = tracer
1863 self.last_status = 127 # wait -n error code
1864
1865 def WaitForOne(self, waitpid_options=0):
1866 # type: (int) -> int
1867 """Wait until the next process returns (or maybe Ctrl-C).
1868
1869 Returns:
1870 One of these negative numbers:
1871 W1_ECHILD Nothing to wait for
1872 W1_OK Caller should keep waiting
1873 UNTRAPPED_SIGWINCH
1874 Or
1875 result > 0 Signal that waitpid() was interrupted with
1876
1877 In the interactive shell, we return 0 if we get a Ctrl-C, so the caller
1878 will try again.
1879
1880 Callers:
1881 wait -n -- loop until there is one fewer process (TODO)
1882 wait -- loop until there are no processes
1883 wait $! -- loop until job state is Done (process or pipeline)
1884 Process::Wait() -- loop until Process state is done
1885 Pipeline::Wait() -- loop until Pipeline state is done
1886
1887 Comparisons:
1888 bash: jobs.c waitchld() Has a special case macro(!) CHECK_WAIT_INTR for
1889 the wait builtin
1890
1891 dash: jobs.c waitproc() uses sigfillset(), sigprocmask(), etc. Runs in a
1892 loop while (gotsigchld), but that might be a hack for System V!
1893
1894 Should we have a cleaner API like named posix::wait_for_one() ?
1895
1896 wait_result =
1897 ECHILD -- nothing to wait for
1898 | Done(int pid, int status) -- process done
1899 | EINTR(bool sigint) -- may or may not retry
1900 """
1901 pid, status = pyos.WaitPid(waitpid_options)
1902 if pid == 0: # WNOHANG passed, and no state changes
1903 return W1_AGAIN
1904 elif pid < 0: # error case
1905 err_num = status
1906 #log('waitpid() error => %d %s', e.errno, pyutil.strerror(e))
1907 if err_num == ECHILD:
1908 return W1_ECHILD # nothing to wait for caller should stop
1909 elif err_num == EINTR: # Bug #858 fix
1910 #log('WaitForOne() => %d', self.trap_state.GetLastSignal())
1911 return self.signal_safe.LastSignal() # e.g. 1 for SIGHUP
1912 else:
1913 # The signature of waitpid() means this shouldn't happen
1914 raise AssertionError()
1915
1916 # All child processes are supposed to be in this dict. But this may
1917 # legitimately happen if a grandchild outlives the child (its parent).
1918 # Then it is reparented under this process, so we might receive
1919 # notification of its exit, even though we didn't start it. We can't have
1920 # any knowledge of such processes, so print a warning.
1921 if pid not in self.job_list.child_procs:
1922 print_stderr("osh: PID %d stopped, but osh didn't start it" % pid)
1923 return W1_OK
1924
1925 proc = self.job_list.child_procs[pid]
1926 if 0:
1927 self.job_list.DebugPrint()
1928
1929 if WIFSIGNALED(status):
1930 term_sig = WTERMSIG(status)
1931 status = 128 + term_sig
1932
1933 # Print newline after Ctrl-C.
1934 if term_sig == SIGINT:
1935 print('')
1936
1937 proc.WhenDone(pid, status)
1938
1939 elif WIFEXITED(status):
1940 status = WEXITSTATUS(status)
1941 #log('exit status: %s', status)
1942 proc.WhenDone(pid, status)
1943
1944 elif WIFSTOPPED(status):
1945 #status = WEXITSTATUS(status)
1946 stop_sig = WSTOPSIG(status)
1947
1948 print_stderr('')
1949 print_stderr('[PID %d] Stopped with signal %d' % (pid, stop_sig))
1950 proc.WhenStopped(stop_sig)
1951
1952 else:
1953 raise AssertionError(status)
1954
1955 self.last_status = status # for wait -n
1956 self.tracer.OnProcessEnd(pid, status)
1957 return W1_OK
1958
1959 def PollNotifications(self):
1960 # type: () -> None
1961 """
1962 Process all pending state changes.
1963 """
1964 while self.WaitForOne(waitpid_options=WNOHANG) == W1_OK:
1965 continue