OILS / core / process.py View on Github | oilshell.org

1967 lines, 957 significant
1# Copyright 2016 Andy Chu. All rights reserved.
2# Licensed under the Apache License, Version 2.0 (the "License");
3# you may not use this file except in compliance with the License.
4# You may obtain a copy of the License at
5#
6# http://www.apache.org/licenses/LICENSE-2.0
7"""
8process.py - Launch processes and manipulate file descriptors.
9"""
10from __future__ import print_function
11
12from errno import EACCES, EBADF, ECHILD, EINTR, ENOENT, ENOEXEC, EEXIST
13import fcntl as fcntl_
14from fcntl import F_DUPFD, F_GETFD, F_SETFD, FD_CLOEXEC
15from signal import (SIG_DFL, SIG_IGN, SIGINT, SIGPIPE, SIGQUIT, SIGTSTP,
16 SIGTTOU, SIGTTIN, SIGWINCH)
17
18from _devbuild.gen.id_kind_asdl import Id
19from _devbuild.gen.option_asdl import option_i
20from _devbuild.gen.runtime_asdl import (job_state_e, job_state_t,
21 job_state_str, wait_status,
22 wait_status_t, RedirValue,
23 redirect_arg, redirect_arg_e, trace,
24 trace_t)
25from _devbuild.gen.syntax_asdl import (
26 loc_t,
27 redir_loc,
28 redir_loc_e,
29 redir_loc_t,
30)
31from _devbuild.gen.value_asdl import (value, value_e)
32from core import dev
33from core import error
34from core.error import e_die
35from core import pyutil
36from core import pyos
37from core import state
38from core import ui
39from core import util
40from core.state import MutableOpts
41from data_lang import j8_lite
42from frontend import location
43from frontend import match
44from mycpp import mylib
45from mycpp.mylib import log, print_stderr, probe, tagswitch, iteritems
46
47import posix_ as posix
48from posix_ import (
49 # translated by mycpp and directly called! No wrapper!
50 WIFSIGNALED,
51 WIFEXITED,
52 WIFSTOPPED,
53 WEXITSTATUS,
54 WSTOPSIG,
55 WTERMSIG,
56 WNOHANG,
57 O_APPEND,
58 O_CREAT,
59 O_EXCL,
60 O_NONBLOCK,
61 O_NOCTTY,
62 O_RDONLY,
63 O_RDWR,
64 O_WRONLY,
65 O_TRUNC,
66)
67
68from typing import IO, List, Tuple, Dict, Optional, Any, cast, TYPE_CHECKING
69
70if TYPE_CHECKING:
71 from _devbuild.gen.runtime_asdl import cmd_value
72 from _devbuild.gen.syntax_asdl import command_t
73 from builtin import trap_osh
74 from core import optview
75 from core.ui import ErrorFormatter
76 from core.util import _DebugFile
77 from osh.cmd_eval import CommandEvaluator
78
79NO_FD = -1
80
81# Minimum file descriptor that the shell can use. Other descriptors can be
82# directly used by user programs, e.g. exec 9>&1
83#
84# Oil uses 100 because users are allowed TWO digits in frontend/lexer_def.py.
85# This is a compromise between bash (unlimited, but requires crazy
86# bookkeeping), and dash/zsh (10) and mksh (24)
87_SHELL_MIN_FD = 100
88
89# Style for 'jobs' builtin
90STYLE_DEFAULT = 0
91STYLE_LONG = 1
92STYLE_PID_ONLY = 2
93
94# To save on allocations in JobList::GetJobWithSpec()
95CURRENT_JOB_SPECS = ['', '%', '%%', '%+']
96
97
98class ctx_FileCloser(object):
99
100 def __init__(self, f):
101 # type: (mylib.LineReader) -> None
102 self.f = f
103
104 def __enter__(self):
105 # type: () -> None
106 pass
107
108 def __exit__(self, type, value, traceback):
109 # type: (Any, Any, Any) -> None
110 self.f.close()
111
112
113def InitInteractiveShell():
114 # type: () -> None
115 """Called when initializing an interactive shell."""
116
117 # The shell itself should ignore Ctrl-\.
118 pyos.Sigaction(SIGQUIT, SIG_IGN)
119
120 # This prevents Ctrl-Z from suspending OSH in interactive mode.
121 pyos.Sigaction(SIGTSTP, SIG_IGN)
122
123 # More signals from
124 # https://www.gnu.org/software/libc/manual/html_node/Initializing-the-Shell.html
125 # (but not SIGCHLD)
126 pyos.Sigaction(SIGTTOU, SIG_IGN)
127 pyos.Sigaction(SIGTTIN, SIG_IGN)
128
129 # Register a callback to receive terminal width changes.
130 # NOTE: In line_input.c, we turned off rl_catch_sigwinch.
131
132 # This is ALWAYS on, which means that it can cause EINTR, and wait() and
133 # read() have to handle it
134 pyos.RegisterSignalInterest(SIGWINCH)
135
136
137def SaveFd(fd):
138 # type: (int) -> int
139 saved = fcntl_.fcntl(fd, F_DUPFD, _SHELL_MIN_FD) # type: int
140 return saved
141
142
143class _RedirFrame(object):
144
145 def __init__(self, saved_fd, orig_fd, forget):
146 # type: (int, int, bool) -> None
147 self.saved_fd = saved_fd
148 self.orig_fd = orig_fd
149 self.forget = forget
150
151
152class _FdFrame(object):
153
154 def __init__(self):
155 # type: () -> None
156 self.saved = [] # type: List[_RedirFrame]
157 self.need_wait = [] # type: List[Process]
158
159 def Forget(self):
160 # type: () -> None
161 """For exec 1>&2."""
162 for rf in reversed(self.saved):
163 if rf.saved_fd != NO_FD and rf.forget:
164 posix.close(rf.saved_fd)
165
166 del self.saved[:] # like list.clear() in Python 3.3
167 del self.need_wait[:]
168
169 def __repr__(self):
170 # type: () -> str
171 return '<_FdFrame %s>' % self.saved
172
173
174class FdState(object):
175 """File descriptor state for the current process.
176
177 For example, you can do 'myfunc > out.txt' without forking. Child
178 processes inherit our state.
179 """
180
181 def __init__(
182 self,
183 errfmt, # type: ui.ErrorFormatter
184 job_control, # type: JobControl
185 job_list, # type: JobList
186 mem, #type: state.Mem
187 tracer, # type: Optional[dev.Tracer]
188 waiter, # type: Optional[Waiter]
189 exec_opts, #type: optview.Exec
190 ):
191 # type: (...) -> None
192 """
193 Args:
194 errfmt: for errors
195 job_list: For keeping track of _HereDocWriterThunk
196 """
197 self.errfmt = errfmt
198 self.job_control = job_control
199 self.job_list = job_list
200 self.cur_frame = _FdFrame() # for the top level
201 self.stack = [self.cur_frame]
202 self.mem = mem
203 self.tracer = tracer
204 self.waiter = waiter
205 self.exec_opts = exec_opts
206
207 def Open(self, path):
208 # type: (str) -> mylib.LineReader
209 """Opens a path for read, but moves it out of the reserved 3-9 fd
210 range.
211
212 Returns:
213 A Python file object. The caller is responsible for Close().
214
215 Raises:
216 IOError or OSError if the path can't be found. (This is Python-induced wart)
217 """
218 fd_mode = O_RDONLY
219 f = self._Open(path, 'r', fd_mode)
220
221 # Hacky downcast
222 return cast('mylib.LineReader', f)
223
224 # used for util.DebugFile
225 def OpenForWrite(self, path):
226 # type: (str) -> mylib.Writer
227 fd_mode = O_CREAT | O_RDWR
228 f = self._Open(path, 'w', fd_mode)
229
230 # Hacky downcast
231 return cast('mylib.Writer', f)
232
233 def _Open(self, path, c_mode, fd_mode):
234 # type: (str, str, int) -> IO[str]
235 fd = posix.open(path, fd_mode, 0o666) # may raise OSError
236
237 # Immediately move it to a new location
238 new_fd = SaveFd(fd)
239 posix.close(fd)
240
241 # Return a Python file handle
242 f = posix.fdopen(new_fd, c_mode) # may raise IOError
243 return f
244
245 def _WriteFdToMem(self, fd_name, fd):
246 # type: (str, int) -> None
247 if self.mem:
248 # setvar, not setref
249 state.OshLanguageSetValue(self.mem, location.LName(fd_name),
250 value.Str(str(fd)))
251
252 def _ReadFdFromMem(self, fd_name):
253 # type: (str) -> int
254 val = self.mem.GetValue(fd_name)
255 if val.tag() == value_e.Str:
256 try:
257 return int(cast(value.Str, val).s)
258 except ValueError:
259 return NO_FD
260 return NO_FD
261
262 def _PushSave(self, fd):
263 # type: (int) -> bool
264 """Save fd to a new location and remember to restore it later."""
265 #log('---- _PushSave %s', fd)
266 ok = True
267 try:
268 new_fd = SaveFd(fd)
269 except (IOError, OSError) as e:
270 ok = False
271 # Example program that causes this error: exec 4>&1. Descriptor 4 isn't
272 # open.
273 # This seems to be ignored in dash too in savefd()?
274 if e.errno != EBADF:
275 raise
276 if ok:
277 posix.close(fd)
278 fcntl_.fcntl(new_fd, F_SETFD, FD_CLOEXEC)
279 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, True))
280 else:
281 # if we got EBADF, we still need to close the original on Pop()
282 self._PushClose(fd)
283
284 return ok
285
286 def _PushDup(self, fd1, blame_loc):
287 # type: (int, redir_loc_t) -> int
288 """Save fd2 in a higher range, and dup fd1 onto fd2.
289
290 Returns whether F_DUPFD/dup2 succeeded, and the new descriptor.
291 """
292 UP_loc = blame_loc
293 if blame_loc.tag() == redir_loc_e.VarName:
294 fd2_name = cast(redir_loc.VarName, UP_loc).name
295 try:
296 # F_DUPFD: GREATER than range
297 new_fd = fcntl_.fcntl(fd1, F_DUPFD, _SHELL_MIN_FD) # type: int
298 except (IOError, OSError) as e:
299 if e.errno == EBADF:
300 print_stderr('F_DUPFD fd %d: %s' %
301 (fd1, pyutil.strerror(e)))
302 return NO_FD
303 else:
304 raise # this redirect failed
305
306 self._WriteFdToMem(fd2_name, new_fd)
307
308 elif blame_loc.tag() == redir_loc_e.Fd:
309 fd2 = cast(redir_loc.Fd, UP_loc).fd
310
311 if fd1 == fd2:
312 # The user could have asked for it to be open on descriptor 3, but open()
313 # already returned 3, e.g. echo 3>out.txt
314 return NO_FD
315
316 # Check the validity of fd1 before _PushSave(fd2)
317 try:
318 fcntl_.fcntl(fd1, F_GETFD)
319 except (IOError, OSError) as e:
320 print_stderr('F_GETFD fd %d: %s' % (fd1, pyutil.strerror(e)))
321 raise
322
323 need_restore = self._PushSave(fd2)
324
325 #log('==== dup2 %s %s\n' % (fd1, fd2))
326 try:
327 posix.dup2(fd1, fd2)
328 except (IOError, OSError) as e:
329 # bash/dash give this error too, e.g. for 'echo hi 1>&3'
330 print_stderr('dup2(%d, %d): %s' %
331 (fd1, fd2, pyutil.strerror(e)))
332
333 # Restore and return error
334 if need_restore:
335 rf = self.cur_frame.saved.pop()
336 posix.dup2(rf.saved_fd, rf.orig_fd)
337 posix.close(rf.saved_fd)
338
339 raise # this redirect failed
340
341 new_fd = fd2
342
343 else:
344 raise AssertionError()
345
346 return new_fd
347
348 def _PushCloseFd(self, blame_loc):
349 # type: (redir_loc_t) -> bool
350 """For 2>&-"""
351 # exec {fd}>&- means close the named descriptor
352
353 UP_loc = blame_loc
354 if blame_loc.tag() == redir_loc_e.VarName:
355 fd_name = cast(redir_loc.VarName, UP_loc).name
356 fd = self._ReadFdFromMem(fd_name)
357 if fd == NO_FD:
358 return False
359
360 elif blame_loc.tag() == redir_loc_e.Fd:
361 fd = cast(redir_loc.Fd, UP_loc).fd
362
363 else:
364 raise AssertionError()
365
366 self._PushSave(fd)
367
368 return True
369
370 def _PushClose(self, fd):
371 # type: (int) -> None
372 self.cur_frame.saved.append(_RedirFrame(NO_FD, fd, False))
373
374 def _PushWait(self, proc):
375 # type: (Process) -> None
376 self.cur_frame.need_wait.append(proc)
377
378 def _ApplyRedirect(self, r):
379 # type: (RedirValue) -> None
380 arg = r.arg
381 UP_arg = arg
382 with tagswitch(arg) as case:
383
384 if case(redirect_arg_e.Path):
385 arg = cast(redirect_arg.Path, UP_arg)
386 # noclobber flag is OR'd with other flags when allowed
387 noclobber_mode = O_EXCL if self.exec_opts.noclobber() else 0
388 if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &>
389 # NOTE: This is different than >| because it respects noclobber, but
390 # that option is almost never used. See test/wild.sh.
391 mode = O_CREAT | O_WRONLY | O_TRUNC | noclobber_mode
392 elif r.op_id == Id.Redir_Clobber: # >|
393 mode = O_CREAT | O_WRONLY | O_TRUNC
394 elif r.op_id in (Id.Redir_DGreat,
395 Id.Redir_AndDGreat): # >> &>>
396 mode = O_CREAT | O_WRONLY | O_APPEND | noclobber_mode
397 elif r.op_id == Id.Redir_Less: # <
398 mode = O_RDONLY
399 elif r.op_id == Id.Redir_LessGreat: # <>
400 mode = O_CREAT | O_RDWR
401 else:
402 raise NotImplementedError(r.op_id)
403
404 # NOTE: 0666 is affected by umask, all shells use it.
405 try:
406 open_fd = posix.open(arg.filename, mode, 0o666)
407 except (IOError, OSError) as e:
408 if noclobber_mode != 0 and e.errno == EEXIST:
409 self.errfmt.PrintMessage(
410 "I/O redirect error: can't overwrite existing file %r: %s"
411 % (arg.filename, pyutil.strerror(e)),
412 r.op_loc)
413 else:
414 self.errfmt.PrintMessage(
415 "I/O redirect error: can't open file %r: %s" %
416 (arg.filename, pyutil.strerror(e)),
417 r.op_loc)
418 raise IOError(
419 0
420 ) # redirect failed, errno=0 to hide error at parent level
421
422 new_fd = self._PushDup(open_fd, r.loc)
423 if new_fd != NO_FD:
424 posix.close(open_fd)
425
426 # Now handle &> and &>> and their variants. These pairs are the same:
427 #
428 # stdout_stderr.py &> out-err.txt
429 # stdout_stderr.py > out-err.txt 2>&1
430 #
431 # stdout_stderr.py 3&> out-err.txt
432 # stdout_stderr.py 3> out-err.txt 2>&3
433 #
434 # Ditto for {fd}> and {fd}&>
435
436 if r.op_id in (Id.Redir_AndGreat, Id.Redir_AndDGreat):
437 self._PushDup(new_fd, redir_loc.Fd(2))
438
439 elif case(redirect_arg_e.CopyFd): # e.g. echo hi 1>&2
440 arg = cast(redirect_arg.CopyFd, UP_arg)
441
442 if r.op_id == Id.Redir_GreatAnd: # 1>&2
443 self._PushDup(arg.target_fd, r.loc)
444
445 elif r.op_id == Id.Redir_LessAnd: # 0<&5
446 # The only difference between >& and <& is the default file
447 # descriptor argument.
448 self._PushDup(arg.target_fd, r.loc)
449
450 else:
451 raise NotImplementedError()
452
453 elif case(redirect_arg_e.MoveFd): # e.g. echo hi 5>&6-
454 arg = cast(redirect_arg.MoveFd, UP_arg)
455 new_fd = self._PushDup(arg.target_fd, r.loc)
456 if new_fd != NO_FD:
457 posix.close(arg.target_fd)
458
459 UP_loc = r.loc
460 if r.loc.tag() == redir_loc_e.Fd:
461 fd = cast(redir_loc.Fd, UP_loc).fd
462 else:
463 fd = NO_FD
464
465 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, False))
466
467 elif case(redirect_arg_e.CloseFd): # e.g. echo hi 5>&-
468 self._PushCloseFd(r.loc)
469
470 elif case(redirect_arg_e.HereDoc):
471 arg = cast(redirect_arg.HereDoc, UP_arg)
472
473 # NOTE: Do these descriptors have to be moved out of the range 0-9?
474 read_fd, write_fd = posix.pipe()
475
476 self._PushDup(read_fd, r.loc) # stdin is now the pipe
477
478 # We can't close like we do in the filename case above? The writer can
479 # get a "broken pipe".
480 self._PushClose(read_fd)
481
482 thunk = _HereDocWriterThunk(write_fd, arg.body)
483
484 # Use PIPE_SIZE to save a process in the case of small here
485 # docs, which are the common case. (dash does this.)
486
487 # Note: could instrument this to see how often it happens.
488 # Though strace -ff can also work.
489 start_process = len(arg.body) > 4096
490 #start_process = True
491
492 if start_process:
493 here_proc = Process(thunk, self.job_control, self.job_list,
494 self.tracer)
495
496 # NOTE: we could close the read pipe here, but it doesn't really
497 # matter because we control the code.
498 here_proc.StartProcess(trace.HereDoc)
499 #log('Started %s as %d', here_proc, pid)
500 self._PushWait(here_proc)
501
502 # Now that we've started the child, close it in the parent.
503 posix.close(write_fd)
504
505 else:
506 posix.write(write_fd, arg.body)
507 posix.close(write_fd)
508
509 def Push(self, redirects, err_out):
510 # type: (List[RedirValue], List[error.IOError_OSError]) -> None
511 """Apply a group of redirects and remember to undo them."""
512
513 #log('> fd_state.Push %s', redirects)
514 new_frame = _FdFrame()
515 self.stack.append(new_frame)
516 self.cur_frame = new_frame
517
518 for r in redirects:
519 #log('apply %s', r)
520 with ui.ctx_Location(self.errfmt, r.op_loc):
521 try:
522 self._ApplyRedirect(r)
523 except (IOError, OSError) as e:
524 err_out.append(e)
525 # This can fail too
526 self.Pop(err_out)
527 return # for bad descriptor, etc.
528
529 def PushStdinFromPipe(self, r):
530 # type: (int) -> bool
531 """Save the current stdin and make it come from descriptor 'r'.
532
533 'r' is typically the read-end of a pipe. For 'lastpipe'/ZSH
534 semantics of
535
536 echo foo | read line; echo $line
537 """
538 new_frame = _FdFrame()
539 self.stack.append(new_frame)
540 self.cur_frame = new_frame
541
542 self._PushDup(r, redir_loc.Fd(0))
543 return True
544
545 def Pop(self, err_out):
546 # type: (List[error.IOError_OSError]) -> None
547 frame = self.stack.pop()
548 #log('< Pop %s', frame)
549 for rf in reversed(frame.saved):
550 if rf.saved_fd == NO_FD:
551 #log('Close %d', orig)
552 try:
553 posix.close(rf.orig_fd)
554 except (IOError, OSError) as e:
555 err_out.append(e)
556 log('Error closing descriptor %d: %s', rf.orig_fd,
557 pyutil.strerror(e))
558 return
559 else:
560 try:
561 posix.dup2(rf.saved_fd, rf.orig_fd)
562 except (IOError, OSError) as e:
563 err_out.append(e)
564 log('dup2(%d, %d) error: %s', rf.saved_fd, rf.orig_fd,
565 pyutil.strerror(e))
566 #log('fd state:')
567 #posix.system('ls -l /proc/%s/fd' % posix.getpid())
568 return
569 posix.close(rf.saved_fd)
570 #log('dup2 %s %s', saved, orig)
571
572 # Wait for here doc processes to finish.
573 for proc in frame.need_wait:
574 unused_status = proc.Wait(self.waiter)
575
576 def MakePermanent(self):
577 # type: () -> None
578 self.cur_frame.Forget()
579
580
581class ChildStateChange(object):
582
583 def __init__(self):
584 # type: () -> None
585 """Empty constructor for mycpp."""
586 pass
587
588 def Apply(self):
589 # type: () -> None
590 raise NotImplementedError()
591
592 def ApplyFromParent(self, proc):
593 # type: (Process) -> None
594 """Noop for all state changes other than SetPgid for mycpp."""
595 pass
596
597
598class StdinFromPipe(ChildStateChange):
599
600 def __init__(self, pipe_read_fd, w):
601 # type: (int, int) -> None
602 self.r = pipe_read_fd
603 self.w = w
604
605 def __repr__(self):
606 # type: () -> str
607 return '<StdinFromPipe %d %d>' % (self.r, self.w)
608
609 def Apply(self):
610 # type: () -> None
611 posix.dup2(self.r, 0)
612 posix.close(self.r) # close after dup
613
614 posix.close(self.w) # we're reading from the pipe, not writing
615 #log('child CLOSE w %d pid=%d', self.w, posix.getpid())
616
617
618class StdoutToPipe(ChildStateChange):
619
620 def __init__(self, r, pipe_write_fd):
621 # type: (int, int) -> None
622 self.r = r
623 self.w = pipe_write_fd
624
625 def __repr__(self):
626 # type: () -> str
627 return '<StdoutToPipe %d %d>' % (self.r, self.w)
628
629 def Apply(self):
630 # type: () -> None
631 posix.dup2(self.w, 1)
632 posix.close(self.w) # close after dup
633
634 posix.close(self.r) # we're writing to the pipe, not reading
635 #log('child CLOSE r %d pid=%d', self.r, posix.getpid())
636
637
638INVALID_PGID = -1
639# argument to setpgid() that means the process is its own leader
640OWN_LEADER = 0
641
642
643class SetPgid(ChildStateChange):
644
645 def __init__(self, pgid, tracer):
646 # type: (int, dev.Tracer) -> None
647 self.pgid = pgid
648 self.tracer = tracer
649
650 def Apply(self):
651 # type: () -> None
652 try:
653 posix.setpgid(0, self.pgid)
654 except (IOError, OSError) as e:
655 self.tracer.OtherMessage(
656 'osh: child %d failed to set its process group to %d: %s' %
657 (posix.getpid(), self.pgid, pyutil.strerror(e)))
658
659 def ApplyFromParent(self, proc):
660 # type: (Process) -> None
661 try:
662 posix.setpgid(proc.pid, self.pgid)
663 except (IOError, OSError) as e:
664 self.tracer.OtherMessage(
665 'osh: parent failed to set process group for PID %d to %d: %s'
666 % (proc.pid, self.pgid, pyutil.strerror(e)))
667
668
669class ExternalProgram(object):
670 """The capability to execute an external program like 'ls'."""
671
672 def __init__(
673 self,
674 hijack_shebang, # type: str
675 fd_state, # type: FdState
676 errfmt, # type: ErrorFormatter
677 debug_f, # type: _DebugFile
678 ):
679 # type: (...) -> None
680 """
681 Args:
682 hijack_shebang: The path of an interpreter to run instead of the one
683 specified in the shebang line. May be empty.
684 """
685 self.hijack_shebang = hijack_shebang
686 self.fd_state = fd_state
687 self.errfmt = errfmt
688 self.debug_f = debug_f
689
690 def Exec(self, argv0_path, cmd_val, environ):
691 # type: (str, cmd_value.Argv, Dict[str, str]) -> None
692 """Execute a program and exit this process.
693
694 Called by: ls / exec ls / ( ls / )
695 """
696 probe('process', 'ExternalProgram_Exec', argv0_path)
697 self._Exec(argv0_path, cmd_val.argv, cmd_val.arg_locs[0], environ,
698 True)
699 assert False, "This line should never execute" # NO RETURN
700
701 def _Exec(self, argv0_path, argv, argv0_loc, environ, should_retry):
702 # type: (str, List[str], loc_t, Dict[str, str], bool) -> None
703 if len(self.hijack_shebang):
704 opened = True
705 try:
706 f = self.fd_state.Open(argv0_path)
707 except (IOError, OSError) as e:
708 opened = False
709
710 if opened:
711 with ctx_FileCloser(f):
712 # Test if the shebang looks like a shell. TODO: The file might be
713 # binary with no newlines, so read 80 bytes instead of readline().
714
715 #line = f.read(80) # type: ignore # TODO: fix this
716 line = f.readline()
717
718 if match.ShouldHijack(line):
719 h_argv = [self.hijack_shebang, argv0_path]
720 h_argv.extend(argv[1:])
721 argv = h_argv
722 argv0_path = self.hijack_shebang
723 self.debug_f.writeln('Hijacked: %s' % argv0_path)
724 else:
725 #self.debug_f.log('Not hijacking %s (%r)', argv, line)
726 pass
727
728 try:
729 posix.execve(argv0_path, argv, environ)
730 except (IOError, OSError) as e:
731 # Run with /bin/sh when ENOEXEC error (no shebang). All shells do this.
732 if e.errno == ENOEXEC and should_retry:
733 new_argv = ['/bin/sh', argv0_path]
734 new_argv.extend(argv[1:])
735 self._Exec('/bin/sh', new_argv, argv0_loc, environ, False)
736 # NO RETURN
737
738 # Would be nice: when the path is relative and ENOENT: print PWD and do
739 # spelling correction?
740
741 self.errfmt.Print_(
742 "Can't execute %r: %s" % (argv0_path, pyutil.strerror(e)),
743 argv0_loc)
744
745 # POSIX mentions 126 and 127 for two specific errors. The rest are
746 # unspecified.
747 #
748 # http://pubs.opengroup.org/onlinepubs/9699919799.2016edition/utilities/V3_chap02.html#tag_18_08_02
749 if e.errno == EACCES:
750 status = 126
751 elif e.errno == ENOENT:
752 # TODO: most shells print 'command not found', rather than strerror()
753 # == "No such file or directory". That's better because it's at the
754 # end of the path search, and we're never searching for a directory.
755 status = 127
756 else:
757 # dash uses 2, but we use that for parse errors. This seems to be
758 # consistent with mksh and zsh.
759 status = 127
760
761 posix._exit(status)
762 # NO RETURN
763
764
765class Thunk(object):
766 """Abstract base class for things runnable in another process."""
767
768 def __init__(self):
769 # type: () -> None
770 """Empty constructor for mycpp."""
771 pass
772
773 def Run(self):
774 # type: () -> None
775 """Returns a status code."""
776 raise NotImplementedError()
777
778 def UserString(self):
779 # type: () -> str
780 """Display for the 'jobs' list."""
781 raise NotImplementedError()
782
783 def __repr__(self):
784 # type: () -> str
785 return self.UserString()
786
787
788class ExternalThunk(Thunk):
789 """An external executable."""
790
791 def __init__(self, ext_prog, argv0_path, cmd_val, environ):
792 # type: (ExternalProgram, str, cmd_value.Argv, Dict[str, str]) -> None
793 self.ext_prog = ext_prog
794 self.argv0_path = argv0_path
795 self.cmd_val = cmd_val
796 self.environ = environ
797
798 def UserString(self):
799 # type: () -> str
800
801 # NOTE: This is the format the Tracer uses.
802 # bash displays sleep $n & (code)
803 # but OSH displays sleep 1 & (argv array)
804 # We could switch the former but I'm not sure it's necessary.
805 tmp = [j8_lite.MaybeShellEncode(a) for a in self.cmd_val.argv]
806 return '[process] %s' % ' '.join(tmp)
807
808 def Run(self):
809 # type: () -> None
810 """An ExternalThunk is run in parent for the exec builtin."""
811 self.ext_prog.Exec(self.argv0_path, self.cmd_val, self.environ)
812
813
814class SubProgramThunk(Thunk):
815 """A subprogram that can be executed in another process."""
816
817 def __init__(self,
818 cmd_ev,
819 node,
820 trap_state,
821 multi_trace,
822 inherit_errexit=True):
823 # type: (CommandEvaluator, command_t, trap_osh.TrapState, dev.MultiTracer, bool) -> None
824 self.cmd_ev = cmd_ev
825 self.node = node
826 self.trap_state = trap_state
827 self.multi_trace = multi_trace
828 self.inherit_errexit = inherit_errexit # for bash errexit compatibility
829
830 def UserString(self):
831 # type: () -> str
832
833 # NOTE: These can be pieces of a pipeline, so they're arbitrary nodes.
834 # TODO: Extract SPIDS from node to display source? Note that
835 # CompoundStatus also has locations of each pipeline component; see
836 # Executor.RunPipeline()
837 thunk_str = ui.CommandType(self.node)
838 return '[subprog] %s' % thunk_str
839
840 def Run(self):
841 # type: () -> None
842 #self.errfmt.OneLineErrExit() # don't quote code in child processes
843 probe('process', 'SubProgramThunk_Run')
844
845 # TODO: break circular dep. Bit flags could go in ASDL or headers.
846 from osh import cmd_eval
847
848 # signal handlers aren't inherited
849 self.trap_state.ClearForSubProgram()
850
851 # NOTE: may NOT return due to exec().
852 if not self.inherit_errexit:
853 self.cmd_ev.mutable_opts.DisableErrExit()
854 try:
855 # optimize to eliminate redundant subshells like ( echo hi ) | wc -l etc.
856 self.cmd_ev.ExecuteAndCatch(self.node, cmd_flags=cmd_eval.Optimize)
857 status = self.cmd_ev.LastStatus()
858 # NOTE: We ignore the is_fatal return value. The user should set -o
859 # errexit so failures in subprocesses cause failures in the parent.
860 except util.UserExit as e:
861 status = e.status
862
863 # Handle errors in a subshell. These two cases are repeated from main()
864 # and the core/completion.py hook.
865 except KeyboardInterrupt:
866 print('')
867 status = 130 # 128 + 2
868 except (IOError, OSError) as e:
869 print_stderr('oils I/O error (subprogram): %s' %
870 pyutil.strerror(e))
871 status = 2
872
873 # If ProcessInit() doesn't turn off buffering, this is needed before
874 # _exit()
875 pyos.FlushStdout()
876
877 self.multi_trace.WriteDumps()
878
879 # We do NOT want to raise SystemExit here. Otherwise dev.Tracer::Pop()
880 # gets called in BOTH processes.
881 # The crash dump seems to be unaffected.
882 posix._exit(status)
883
884
885class _HereDocWriterThunk(Thunk):
886 """Write a here doc to one end of a pipe.
887
888 May be be executed in either a child process or the main shell
889 process.
890 """
891
892 def __init__(self, w, body_str):
893 # type: (int, str) -> None
894 self.w = w
895 self.body_str = body_str
896
897 def UserString(self):
898 # type: () -> str
899
900 # You can hit Ctrl-Z and the here doc writer will be suspended! Other
901 # shells don't have this problem because they use temp files! That's a bit
902 # unfortunate.
903 return '[here doc writer]'
904
905 def Run(self):
906 # type: () -> None
907 """do_exit: For small pipelines."""
908 probe('process', 'HereDocWriterThunk_Run')
909 #log('Writing %r', self.body_str)
910 posix.write(self.w, self.body_str)
911 #log('Wrote %r', self.body_str)
912 posix.close(self.w)
913 #log('Closed %d', self.w)
914
915 posix._exit(0)
916
917
918class Job(object):
919 """Interface for both Process and Pipeline.
920
921 They both can be put in the background and waited on.
922
923 Confusing thing about pipelines in the background: They have TOO MANY NAMES.
924
925 sleep 1 | sleep 2 &
926
927 - The LAST PID is what's printed at the prompt. This is $!, a PROCESS ID and
928 not a JOB ID.
929 # https://www.gnu.org/software/bash/manual/html_node/Special-Parameters.html#Special-Parameters
930 - The process group leader (setpgid) is the FIRST PID.
931 - It's also %1 or %+. The last job started.
932 """
933
934 def __init__(self):
935 # type: () -> None
936 # Initial state with & or Ctrl-Z is Running.
937 self.state = job_state_e.Running
938 self.job_id = -1
939 self.in_background = False
940
941 def DisplayJob(self, job_id, f, style):
942 # type: (int, mylib.Writer, int) -> None
943 raise NotImplementedError()
944
945 def State(self):
946 # type: () -> job_state_t
947 return self.state
948
949 def ProcessGroupId(self):
950 # type: () -> int
951 """Return the process group ID associated with this job."""
952 raise NotImplementedError()
953
954 def JobWait(self, waiter):
955 # type: (Waiter) -> wait_status_t
956 """Wait for this process/pipeline to be stopped or finished."""
957 raise NotImplementedError()
958
959 def SetBackground(self):
960 # type: () -> None
961 """Record that this job is running in the background."""
962 self.in_background = True
963
964 def SetForeground(self):
965 # type: () -> None
966 """Record that this job is running in the foreground."""
967 self.in_background = False
968
969
970class Process(Job):
971 """A process to run.
972
973 TODO: Should we make it clear that this is a FOREGROUND process? A
974 background process is wrapped in a "job". It is unevaluated.
975
976 It provides an API to manipulate file descriptor state in parent and child.
977 """
978
979 def __init__(self, thunk, job_control, job_list, tracer):
980 # type: (Thunk, JobControl, JobList, dev.Tracer) -> None
981 """
982 Args:
983 thunk: Thunk instance
984 job_list: for process bookkeeping
985 """
986 Job.__init__(self)
987 assert isinstance(thunk, Thunk), thunk
988 self.thunk = thunk
989 self.job_control = job_control
990 self.job_list = job_list
991 self.tracer = tracer
992
993 # For pipelines
994 self.parent_pipeline = None # type: Pipeline
995 self.state_changes = [] # type: List[ChildStateChange]
996 self.close_r = -1
997 self.close_w = -1
998
999 self.pid = -1
1000 self.status = -1
1001
1002 def Init_ParentPipeline(self, pi):
1003 # type: (Pipeline) -> None
1004 """For updating PIPESTATUS."""
1005 self.parent_pipeline = pi
1006
1007 def __repr__(self):
1008 # type: () -> str
1009
1010 # note: be wary of infinite mutual recursion
1011 #s = ' %s' % self.parent_pipeline if self.parent_pipeline else ''
1012 #return '<Process %s%s>' % (self.thunk, s)
1013 return '<Process %s %s>' % (_JobStateStr(self.state), self.thunk)
1014
1015 def ProcessGroupId(self):
1016 # type: () -> int
1017 """Returns the group ID of this process."""
1018 # This should only ever be called AFTER the process has started
1019 assert self.pid != -1
1020 if self.parent_pipeline:
1021 # XXX: Maybe we should die here instead? Unclear if this branch
1022 # should even be reachable with the current builtins.
1023 return self.parent_pipeline.ProcessGroupId()
1024
1025 return self.pid
1026
1027 def DisplayJob(self, job_id, f, style):
1028 # type: (int, mylib.Writer, int) -> None
1029 if job_id == -1:
1030 job_id_str = ' '
1031 else:
1032 job_id_str = '%%%d' % job_id
1033 if style == STYLE_PID_ONLY:
1034 f.write('%d\n' % self.pid)
1035 else:
1036 f.write('%s %d %7s ' %
1037 (job_id_str, self.pid, _JobStateStr(self.state)))
1038 f.write(self.thunk.UserString())
1039 f.write('\n')
1040
1041 def AddStateChange(self, s):
1042 # type: (ChildStateChange) -> None
1043 self.state_changes.append(s)
1044
1045 def AddPipeToClose(self, r, w):
1046 # type: (int, int) -> None
1047 self.close_r = r
1048 self.close_w = w
1049
1050 def MaybeClosePipe(self):
1051 # type: () -> None
1052 if self.close_r != -1:
1053 posix.close(self.close_r)
1054 posix.close(self.close_w)
1055
1056 def StartProcess(self, why):
1057 # type: (trace_t) -> int
1058 """Start this process with fork(), handling redirects."""
1059 pid = posix.fork()
1060 if pid < 0:
1061 # When does this happen?
1062 e_die('Fatal error in posix.fork()')
1063
1064 elif pid == 0: # child
1065 # Note: this happens in BOTH interactive and non-interactive shells.
1066 # We technically don't need to do most of it in non-interactive, since we
1067 # did not change state in InitInteractiveShell().
1068
1069 for st in self.state_changes:
1070 st.Apply()
1071
1072 # Python sets SIGPIPE handler to SIG_IGN by default. Child processes
1073 # shouldn't have this.
1074 # https://docs.python.org/2/library/signal.html
1075 # See Python/pythonrun.c.
1076 pyos.Sigaction(SIGPIPE, SIG_DFL)
1077
1078 # Respond to Ctrl-\ (core dump)
1079 pyos.Sigaction(SIGQUIT, SIG_DFL)
1080
1081 # Only standalone children should get Ctrl-Z. Pipelines remain in the
1082 # foreground because suspending them is difficult with our 'lastpipe'
1083 # semantics.
1084 pid = posix.getpid()
1085 if posix.getpgid(0) == pid and self.parent_pipeline is None:
1086 pyos.Sigaction(SIGTSTP, SIG_DFL)
1087
1088 # More signals from
1089 # https://www.gnu.org/software/libc/manual/html_node/Launching-Jobs.html
1090 # (but not SIGCHLD)
1091 pyos.Sigaction(SIGTTOU, SIG_DFL)
1092 pyos.Sigaction(SIGTTIN, SIG_DFL)
1093
1094 self.tracer.OnNewProcess(pid)
1095 # clear foreground pipeline for subshells
1096 self.thunk.Run()
1097 # Never returns
1098
1099 #log('STARTED process %s, pid = %d', self, pid)
1100 self.tracer.OnProcessStart(pid, why)
1101
1102 # Class invariant: after the process is started, it stores its PID.
1103 self.pid = pid
1104
1105 # SetPgid needs to be applied from the child and the parent to avoid
1106 # racing in calls to tcsetpgrp() in the parent. See APUE sec. 9.2.
1107 for st in self.state_changes:
1108 st.ApplyFromParent(self)
1109
1110 # Program invariant: We keep track of every child process!
1111 self.job_list.AddChildProcess(pid, self)
1112
1113 return pid
1114
1115 def Wait(self, waiter):
1116 # type: (Waiter) -> int
1117 """Wait for this process to finish."""
1118 while self.state == job_state_e.Running:
1119 # Only return if there's nothing to wait for. Keep waiting if we were
1120 # interrupted with a signal.
1121 if waiter.WaitForOne() == W1_ECHILD:
1122 break
1123
1124 assert self.status >= 0, self.status
1125 return self.status
1126
1127 def JobWait(self, waiter):
1128 # type: (Waiter) -> wait_status_t
1129 # wait builtin can be interrupted
1130 while self.state == job_state_e.Running:
1131 result = waiter.WaitForOne()
1132
1133 if result >= 0: # signal
1134 return wait_status.Cancelled(result)
1135
1136 if result == W1_ECHILD:
1137 break
1138
1139 return wait_status.Proc(self.status)
1140
1141 def WhenStopped(self, stop_sig):
1142 # type: (int) -> None
1143
1144 # 128 is a shell thing
1145 # https://www.gnu.org/software/bash/manual/html_node/Exit-Status.html
1146 self.status = 128 + stop_sig
1147 self.state = job_state_e.Stopped
1148
1149 if self.job_id == -1:
1150 # This process was started in the foreground
1151 self.job_list.AddJob(self)
1152
1153 if not self.in_background:
1154 self.job_control.MaybeTakeTerminal()
1155 self.SetBackground()
1156
1157 def WhenDone(self, pid, status):
1158 # type: (int, int) -> None
1159 """Called by the Waiter when this Process finishes."""
1160
1161 #log('WhenDone %d %d', pid, status)
1162 assert pid == self.pid, 'Expected %d, got %d' % (self.pid, pid)
1163 self.status = status
1164 self.state = job_state_e.Done
1165 if self.parent_pipeline:
1166 self.parent_pipeline.WhenDone(pid, status)
1167 else:
1168 if self.job_id != -1:
1169 # Job might have been brought to the foreground after being
1170 # assigned a job ID.
1171 if self.in_background:
1172 print_stderr('[%d] Done PID %d' % (self.job_id, self.pid))
1173
1174 self.job_list.RemoveJob(self.job_id)
1175
1176 self.job_list.RemoveChildProcess(self.pid)
1177
1178 if not self.in_background:
1179 self.job_control.MaybeTakeTerminal()
1180
1181 def RunProcess(self, waiter, why):
1182 # type: (Waiter, trace_t) -> int
1183 """Run this process synchronously."""
1184 self.StartProcess(why)
1185 # ShellExecutor might be calling this for the last part of a pipeline.
1186 if self.parent_pipeline is None:
1187 # QUESTION: Can the PGID of a single process just be the PID? i.e. avoid
1188 # calling getpgid()?
1189 self.job_control.MaybeGiveTerminal(posix.getpgid(self.pid))
1190 return self.Wait(waiter)
1191
1192
1193class ctx_Pipe(object):
1194
1195 def __init__(self, fd_state, fd, err_out):
1196 # type: (FdState, int, List[error.IOError_OSError]) -> None
1197 fd_state.PushStdinFromPipe(fd)
1198 self.fd_state = fd_state
1199 self.err_out = err_out
1200
1201 def __enter__(self):
1202 # type: () -> None
1203 pass
1204
1205 def __exit__(self, type, value, traceback):
1206 # type: (Any, Any, Any) -> None
1207 self.fd_state.Pop(self.err_out)
1208
1209
1210class Pipeline(Job):
1211 """A pipeline of processes to run.
1212
1213 Cases we handle:
1214
1215 foo | bar
1216 $(foo | bar)
1217 foo | bar | read v
1218 """
1219
1220 def __init__(self, sigpipe_status_ok, job_control, job_list, tracer):
1221 # type: (bool, JobControl, JobList, dev.Tracer) -> None
1222 Job.__init__(self)
1223 self.job_control = job_control
1224 self.job_list = job_list
1225 self.tracer = tracer
1226
1227 self.procs = [] # type: List[Process]
1228 self.pids = [] # type: List[int] # pids in order
1229 self.pipe_status = [] # type: List[int] # status in order
1230 self.status = -1 # for 'wait' jobs
1231
1232 self.pgid = INVALID_PGID
1233
1234 # Optional for foreground
1235 self.last_thunk = None # type: Tuple[CommandEvaluator, command_t]
1236 self.last_pipe = None # type: Tuple[int, int]
1237
1238 self.sigpipe_status_ok = sigpipe_status_ok
1239
1240 def ProcessGroupId(self):
1241 # type: () -> int
1242 """Returns the group ID of this pipeline."""
1243 return self.pgid
1244
1245 def DisplayJob(self, job_id, f, style):
1246 # type: (int, mylib.Writer, int) -> None
1247 if style == STYLE_PID_ONLY:
1248 f.write('%d\n' % self.procs[0].pid)
1249 else:
1250 # Note: this is STYLE_LONG.
1251 for i, proc in enumerate(self.procs):
1252 if i == 0: # show job ID for first element in pipeline
1253 job_id_str = '%%%d' % job_id
1254 else:
1255 job_id_str = ' ' # 2 spaces
1256
1257 f.write('%s %d %7s ' %
1258 (job_id_str, proc.pid, _JobStateStr(proc.state)))
1259 f.write(proc.thunk.UserString())
1260 f.write('\n')
1261
1262 def DebugPrint(self):
1263 # type: () -> None
1264 print('Pipeline in state %s' % _JobStateStr(self.state))
1265 if mylib.PYTHON: # %s for Process not allowed in C++
1266 for proc in self.procs:
1267 print(' proc %s' % proc)
1268 _, last_node = self.last_thunk
1269 print(' last %s' % last_node)
1270 print(' pipe_status %s' % self.pipe_status)
1271
1272 def Add(self, p):
1273 # type: (Process) -> None
1274 """Append a process to the pipeline."""
1275 if len(self.procs) == 0:
1276 self.procs.append(p)
1277 return
1278
1279 r, w = posix.pipe()
1280 #log('pipe for %s: %d %d', p, r, w)
1281 prev = self.procs[-1]
1282
1283 prev.AddStateChange(StdoutToPipe(r, w)) # applied on StartPipeline()
1284 p.AddStateChange(StdinFromPipe(r, w)) # applied on StartPipeline()
1285
1286 p.AddPipeToClose(r, w) # MaybeClosePipe() on StartPipeline()
1287
1288 self.procs.append(p)
1289
1290 def AddLast(self, thunk):
1291 # type: (Tuple[CommandEvaluator, command_t]) -> None
1292 """Append the last noden to the pipeline.
1293
1294 This is run in the CURRENT process. It is OPTIONAL, because
1295 pipelines in the background are run uniformly.
1296 """
1297 self.last_thunk = thunk
1298
1299 assert len(self.procs) != 0
1300
1301 r, w = posix.pipe()
1302 prev = self.procs[-1]
1303 prev.AddStateChange(StdoutToPipe(r, w))
1304
1305 self.last_pipe = (r, w) # So we can connect it to last_thunk
1306
1307 def StartPipeline(self, waiter):
1308 # type: (Waiter) -> None
1309
1310 # If we are creating a pipeline in a subshell or we aren't running with job
1311 # control, our children should remain in our inherited process group.
1312 # the pipelines's group ID.
1313 if self.job_control.Enabled():
1314 self.pgid = OWN_LEADER # first process in pipeline is the leader
1315
1316 for i, proc in enumerate(self.procs):
1317 if self.pgid != INVALID_PGID:
1318 proc.AddStateChange(SetPgid(self.pgid, self.tracer))
1319
1320 # Figure out the pid
1321 pid = proc.StartProcess(trace.PipelinePart)
1322 if i == 0 and self.pgid != INVALID_PGID:
1323 # Mimic bash and use the PID of the FIRST process as the group for the
1324 # whole pipeline.
1325 self.pgid = pid
1326
1327 self.pids.append(pid)
1328 self.pipe_status.append(-1) # uninitialized
1329
1330 # NOTE: This is done in the SHELL PROCESS after every fork() call.
1331 # It can't be done at the end; otherwise processes will have descriptors
1332 # from non-adjacent pipes.
1333 proc.MaybeClosePipe()
1334
1335 if self.last_thunk:
1336 self.pipe_status.append(-1) # for self.last_thunk
1337
1338 def LastPid(self):
1339 # type: () -> int
1340 """For the odd $! variable.
1341
1342 It would be better if job IDs or PGIDs were used consistently.
1343 """
1344 return self.pids[-1]
1345
1346 def Wait(self, waiter):
1347 # type: (Waiter) -> List[int]
1348 """Wait for this pipeline to finish."""
1349
1350 assert self.procs, "no procs for Wait()"
1351 # waitpid(-1) zero or more times
1352 while self.state == job_state_e.Running:
1353 # Keep waiting until there's nothing to wait for.
1354 if waiter.WaitForOne() == W1_ECHILD:
1355 break
1356
1357 return self.pipe_status
1358
1359 def JobWait(self, waiter):
1360 # type: (Waiter) -> wait_status_t
1361 """Called by 'wait' builtin, e.g. 'wait %1'."""
1362 # wait builtin can be interrupted
1363 assert self.procs, "no procs for Wait()"
1364 while self.state == job_state_e.Running:
1365 result = waiter.WaitForOne()
1366
1367 if result >= 0: # signal
1368 return wait_status.Cancelled(result)
1369
1370 if result == W1_ECHILD:
1371 break
1372
1373 return wait_status.Pipeline(self.pipe_status)
1374
1375 def RunLastPart(self, waiter, fd_state):
1376 # type: (Waiter, FdState) -> List[int]
1377 """Run this pipeline synchronously (foreground pipeline).
1378
1379 Returns:
1380 pipe_status (list of integers).
1381 """
1382 assert len(self.pids) == len(self.procs)
1383
1384 # TODO: break circular dep. Bit flags could go in ASDL or headers.
1385 from osh import cmd_eval
1386
1387 # This is tcsetpgrp()
1388 # TODO: fix race condition -- I believe the first process could have
1389 # stopped already, and thus getpgid() will fail
1390 self.job_control.MaybeGiveTerminal(self.pgid)
1391
1392 # Run the last part of the pipeline IN PARALLEL with other processes. It
1393 # may or may not fork:
1394 # echo foo | read line # no fork, the builtin runs in THIS shell process
1395 # ls | wc -l # fork for 'wc'
1396
1397 cmd_ev, last_node = self.last_thunk
1398
1399 assert self.last_pipe is not None
1400 r, w = self.last_pipe # set in AddLast()
1401 posix.close(w) # we will not write here
1402
1403 # Fix lastpipe / job control / DEBUG trap interaction
1404 cmd_flags = cmd_eval.NoDebugTrap if self.job_control.Enabled() else 0
1405
1406 # The ERR trap only runs for the WHOLE pipeline, not the COMPONENTS in
1407 # a pipeline.
1408 cmd_flags |= cmd_eval.NoErrTrap
1409
1410 io_errors = [] # type: List[error.IOError_OSError]
1411 with ctx_Pipe(fd_state, r, io_errors):
1412 cmd_ev.ExecuteAndCatch(last_node, cmd_flags)
1413
1414 if len(io_errors):
1415 e_die('Error setting up last part of pipeline: %s' %
1416 pyutil.strerror(io_errors[0]))
1417
1418 # We won't read anymore. If we don't do this, then 'cat' in 'cat
1419 # /dev/urandom | sleep 1' will never get SIGPIPE.
1420 posix.close(r)
1421
1422 self.pipe_status[-1] = cmd_ev.LastStatus()
1423 if self.AllDone():
1424 self.state = job_state_e.Done
1425
1426 #log('pipestatus before all have finished = %s', self.pipe_status)
1427 return self.Wait(waiter)
1428
1429 def AllDone(self):
1430 # type: () -> bool
1431
1432 # mycpp rewrite: all(status != -1 for status in self.pipe_status)
1433 for status in self.pipe_status:
1434 if status == -1:
1435 return False
1436 return True
1437
1438 def WhenDone(self, pid, status):
1439 # type: (int, int) -> None
1440 """Called by Process.WhenDone."""
1441 #log('Pipeline WhenDone %d %d', pid, status)
1442 i = self.pids.index(pid)
1443 assert i != -1, 'Unexpected PID %d' % pid
1444
1445 if status == 141 and self.sigpipe_status_ok:
1446 status = 0
1447
1448 self.job_list.RemoveChildProcess(pid)
1449 self.pipe_status[i] = status
1450 if self.AllDone():
1451 if self.job_id != -1:
1452 # Job might have been brought to the foreground after being
1453 # assigned a job ID.
1454 if self.in_background:
1455 print_stderr('[%d] Done PGID %d' %
1456 (self.job_id, self.pids[0]))
1457
1458 self.job_list.RemoveJob(self.job_id)
1459
1460 # status of pipeline is status of last process
1461 self.status = self.pipe_status[-1]
1462 self.state = job_state_e.Done
1463 if not self.in_background:
1464 self.job_control.MaybeTakeTerminal()
1465
1466
1467def _JobStateStr(i):
1468 # type: (job_state_t) -> str
1469 return job_state_str(i)[10:] # remove 'job_state.'
1470
1471
1472def _GetTtyFd():
1473 # type: () -> int
1474 """Returns -1 if stdio is not a TTY."""
1475 try:
1476 return posix.open("/dev/tty", O_NONBLOCK | O_NOCTTY | O_RDWR, 0o666)
1477 except (IOError, OSError) as e:
1478 return -1
1479
1480
1481class ctx_TerminalControl(object):
1482
1483 def __init__(self, job_control, errfmt):
1484 # type: (JobControl, ui.ErrorFormatter) -> None
1485 job_control.InitJobControl()
1486 self.job_control = job_control
1487 self.errfmt = errfmt
1488
1489 def __enter__(self):
1490 # type: () -> None
1491 pass
1492
1493 def __exit__(self, type, value, traceback):
1494 # type: (Any, Any, Any) -> None
1495
1496 # Return the TTY to the original owner before exiting.
1497 try:
1498 self.job_control.MaybeReturnTerminal()
1499 except error.FatalRuntime as e:
1500 # Don't abort the shell on error, just print a message.
1501 self.errfmt.PrettyPrintError(e)
1502
1503
1504class JobControl(object):
1505 """Interface to setpgid(), tcsetpgrp(), etc."""
1506
1507 def __init__(self):
1508 # type: () -> None
1509
1510 # The main shell's PID and group ID.
1511 self.shell_pid = -1
1512 self.shell_pgid = -1
1513
1514 # The fd of the controlling tty. Set to -1 when job control is disabled.
1515 self.shell_tty_fd = -1
1516
1517 # For giving the terminal back to our parent before exiting (if not a login
1518 # shell).
1519 self.original_tty_pgid = -1
1520
1521 def InitJobControl(self):
1522 # type: () -> None
1523 self.shell_pid = posix.getpid()
1524 orig_shell_pgid = posix.getpgid(0)
1525 self.shell_pgid = orig_shell_pgid
1526 self.shell_tty_fd = _GetTtyFd()
1527
1528 # If we aren't the leader of our process group, create a group and mark
1529 # ourselves as the leader.
1530 if self.shell_pgid != self.shell_pid:
1531 try:
1532 posix.setpgid(self.shell_pid, self.shell_pid)
1533 self.shell_pgid = self.shell_pid
1534 except (IOError, OSError) as e:
1535 self.shell_tty_fd = -1
1536
1537 if self.shell_tty_fd != -1:
1538 self.original_tty_pgid = posix.tcgetpgrp(self.shell_tty_fd)
1539
1540 # If stdio is a TTY, put the shell's process group in the foreground.
1541 try:
1542 posix.tcsetpgrp(self.shell_tty_fd, self.shell_pgid)
1543 except (IOError, OSError) as e:
1544 # We probably aren't in the session leader's process group. Disable job
1545 # control.
1546 self.shell_tty_fd = -1
1547 self.shell_pgid = orig_shell_pgid
1548 posix.setpgid(self.shell_pid, self.shell_pgid)
1549
1550 def Enabled(self):
1551 # type: () -> bool
1552
1553 # TODO: get rid of this syscall? SubProgramThunk should set a flag I
1554 # think.
1555 curr_pid = posix.getpid()
1556 # Only the main shell should bother with job control functions.
1557 return curr_pid == self.shell_pid and self.shell_tty_fd != -1
1558
1559 # TODO: This isn't a PID. This is a process group ID?
1560 #
1561 # What should the table look like?
1562 #
1563 # Do we need the last PID? I don't know why bash prints that. Probably so
1564 # you can do wait $!
1565 # wait -n waits for any node to go from job_state_e.Running to job_state_e.Done?
1566 #
1567 # And it needs a flag for CURRENT, for the implicit arg to 'fg'.
1568 # job_id is just an integer. This is sort of lame.
1569 #
1570 # [job_id, flag, pgid, job_state, node]
1571
1572 def MaybeGiveTerminal(self, pgid):
1573 # type: (int) -> None
1574 """If stdio is a TTY, move the given process group to the
1575 foreground."""
1576 if not self.Enabled():
1577 # Only call tcsetpgrp when job control is enabled.
1578 return
1579
1580 try:
1581 posix.tcsetpgrp(self.shell_tty_fd, pgid)
1582 except (IOError, OSError) as e:
1583 e_die('osh: Failed to move process group %d to foreground: %s' %
1584 (pgid, pyutil.strerror(e)))
1585
1586 def MaybeTakeTerminal(self):
1587 # type: () -> None
1588 """If stdio is a TTY, return the main shell's process group to the
1589 foreground."""
1590 self.MaybeGiveTerminal(self.shell_pgid)
1591
1592 def MaybeReturnTerminal(self):
1593 # type: () -> None
1594 """Called before the shell exits."""
1595 self.MaybeGiveTerminal(self.original_tty_pgid)
1596
1597
1598class JobList(object):
1599 """Global list of jobs, used by a few builtins."""
1600
1601 def __init__(self):
1602 # type: () -> None
1603
1604 # job_id -> Job instance
1605 self.jobs = {} # type: Dict[int, Job]
1606
1607 # pid -> Process. This is for STOP notification.
1608 self.child_procs = {} # type: Dict[int, Process]
1609 self.debug_pipelines = [] # type: List[Pipeline]
1610
1611 # Counter used to assign IDs to jobs. It is incremented every time a job
1612 # is created. Once all active jobs are done it is reset to 1. I'm not
1613 # sure if this reset behavior is mandated by POSIX, but other shells do
1614 # it, so we mimic for the sake of compatibility.
1615 self.job_id = 1
1616
1617 def AddJob(self, job):
1618 # type: (Job) -> int
1619 """Add a background job to the list.
1620
1621 A job is either a Process or Pipeline. You can resume a job with 'fg',
1622 kill it with 'kill', etc.
1623
1624 Two cases:
1625
1626 1. async jobs: sleep 5 | sleep 4 &
1627 2. stopped jobs: sleep 5; then Ctrl-Z
1628 """
1629 job_id = self.job_id
1630 self.jobs[job_id] = job
1631 job.job_id = job_id
1632 self.job_id += 1
1633 return job_id
1634
1635 def RemoveJob(self, job_id):
1636 # type: (int) -> None
1637 """Process and Pipeline can call this."""
1638 mylib.dict_erase(self.jobs, job_id)
1639
1640 if len(self.jobs) == 0:
1641 self.job_id = 1
1642
1643 def AddChildProcess(self, pid, proc):
1644 # type: (int, Process) -> None
1645 """Every child process should be added here as soon as we know its PID.
1646
1647 When the Waiter gets an EXITED or STOPPED notification, we need
1648 to know about it so 'jobs' can work.
1649 """
1650 self.child_procs[pid] = proc
1651
1652 def RemoveChildProcess(self, pid):
1653 # type: (int) -> None
1654 """Remove the child process with the given PID."""
1655 mylib.dict_erase(self.child_procs, pid)
1656
1657 if mylib.PYTHON:
1658
1659 def AddPipeline(self, pi):
1660 # type: (Pipeline) -> None
1661 """For debugging only."""
1662 self.debug_pipelines.append(pi)
1663
1664 def ProcessFromPid(self, pid):
1665 # type: (int) -> Process
1666 """For wait $PID.
1667
1668 There's no way to wait for a pipeline with a PID. That uses job
1669 syntax, e.g. %1. Not a great interface.
1670 """
1671 return self.child_procs.get(pid)
1672
1673 def GetCurrentAndPreviousJobs(self):
1674 # type: () -> Tuple[Optional[Job], Optional[Job]]
1675 """Return the "current" and "previous" jobs (AKA `%+` and `%-`).
1676
1677 See the POSIX specification for the `jobs` builtin for details:
1678 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1679
1680 IMPORTANT NOTE: This method assumes that the jobs list will not change
1681 during its execution! This assumption holds for now because we only ever
1682 update the jobs list from the main loop after WaitPid() informs us of a
1683 change. If we implement `set -b` and install a signal handler for
1684 SIGCHLD we should be careful to synchronize it with this function. The
1685 unsafety of mutating GC data structures from a signal handler should
1686 make this a non-issue, but if bugs related to this appear this note may
1687 be helpful...
1688 """
1689 # Split all active jobs by state and sort each group by decreasing job
1690 # ID to approximate newness.
1691 stopped_jobs = [] # type: List[Job]
1692 running_jobs = [] # type: List[Job]
1693 for i in xrange(0, self.job_id):
1694 job = self.jobs.get(i, None)
1695 if not job:
1696 continue
1697
1698 if job.state == job_state_e.Stopped:
1699 stopped_jobs.append(job)
1700
1701 elif job.state == job_state_e.Running:
1702 running_jobs.append(job)
1703
1704 current = None # type: Optional[Job]
1705 previous = None # type: Optional[Job]
1706 # POSIX says: If there is any suspended job, then the current job shall
1707 # be a suspended job. If there are at least two suspended jobs, then the
1708 # previous job also shall be a suspended job.
1709 #
1710 # So, we will only return running jobs from here if there are no recent
1711 # stopped jobs.
1712 if len(stopped_jobs) > 0:
1713 current = stopped_jobs.pop()
1714
1715 if len(stopped_jobs) > 0:
1716 previous = stopped_jobs.pop()
1717
1718 if len(running_jobs) > 0 and not current:
1719 current = running_jobs.pop()
1720
1721 if len(running_jobs) > 0 and not previous:
1722 previous = running_jobs.pop()
1723
1724 if not previous:
1725 previous = current
1726
1727 return current, previous
1728
1729 def GetJobWithSpec(self, job_spec):
1730 # type: (str) -> Optional[Job]
1731 """Parse the given job spec and return the matching job. If there is no
1732 matching job, this function returns None.
1733
1734 See the POSIX spec for the `jobs` builtin for details about job specs:
1735 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1736 """
1737 if job_spec in CURRENT_JOB_SPECS:
1738 current, _ = self.GetCurrentAndPreviousJobs()
1739 return current
1740
1741 if job_spec == '%-':
1742 _, previous = self.GetCurrentAndPreviousJobs()
1743 return previous
1744
1745 # TODO: Add support for job specs based on prefixes of process argv.
1746 m = util.RegexSearch(r'^%([0-9]+)$', job_spec)
1747 if m is not None:
1748 assert len(m) == 2
1749 job_id = int(m[1])
1750 if job_id in self.jobs:
1751 return self.jobs[job_id]
1752
1753 return None
1754
1755 def DisplayJobs(self, style):
1756 # type: (int) -> None
1757 """Used by the 'jobs' builtin.
1758
1759 https://pubs.opengroup.org/onlinepubs/9699919799/utilities/jobs.html
1760
1761 "By default, the jobs utility shall display the status of all stopped jobs,
1762 running background jobs and all jobs whose status has changed and have not
1763 been reported by the shell."
1764 """
1765 # NOTE: A job is a background process or pipeline.
1766 #
1767 # echo hi | wc -l -- this starts two processes. Wait for TWO
1768 # echo hi | wc -l & -- this starts a process which starts two processes
1769 # Wait for ONE.
1770 #
1771 # 'jobs -l' GROUPS the PIDs by job. It has the job number, + - indicators
1772 # for %% and %-, PID, status, and "command".
1773 #
1774 # Every component of a pipeline is on the same line with 'jobs', but
1775 # they're separated into different lines with 'jobs -l'.
1776 #
1777 # See demo/jobs-builtin.sh
1778
1779 # $ jobs -l
1780 # [1]+ 24414 Stopped sleep 5
1781 # 24415 | sleep 5
1782 # [2] 24502 Running sleep 6
1783 # 24503 | sleep 6
1784 # 24504 | sleep 5 &
1785 # [3]- 24508 Running sleep 6
1786 # 24509 | sleep 6
1787 # 24510 | sleep 5 &
1788
1789 f = mylib.Stdout()
1790 for job_id, job in iteritems(self.jobs):
1791 # Use the %1 syntax
1792 job.DisplayJob(job_id, f, style)
1793
1794 def DebugPrint(self):
1795 # type: () -> None
1796
1797 f = mylib.Stdout()
1798 f.write('\n')
1799 f.write('[process debug info]\n')
1800
1801 for pid, proc in iteritems(self.child_procs):
1802 proc.DisplayJob(-1, f, STYLE_DEFAULT)
1803 #p = ' |' if proc.parent_pipeline else ''
1804 #print('%d %7s %s%s' % (pid, _JobStateStr(proc.state), proc.thunk.UserString(), p))
1805
1806 if len(self.debug_pipelines):
1807 f.write('\n')
1808 f.write('[pipeline debug info]\n')
1809 for pi in self.debug_pipelines:
1810 pi.DebugPrint()
1811
1812 def ListRecent(self):
1813 # type: () -> None
1814 """For jobs -n, which I think is also used in the interactive
1815 prompt."""
1816 pass
1817
1818 def NumRunning(self):
1819 # type: () -> int
1820 """Return the number of running jobs.
1821
1822 Used by 'wait' and 'wait -n'.
1823 """
1824 count = 0
1825 for _, job in iteritems(self.jobs): # mycpp rewrite: from itervalues()
1826 if job.State() == job_state_e.Running:
1827 count += 1
1828 return count
1829
1830
1831# Some WaitForOne() return values
1832W1_OK = -2 # waitpid(-1) returned
1833W1_ECHILD = -3 # no processes to wait for
1834W1_AGAIN = -4 # WNOHANG was passed and there were no state changes
1835
1836
1837class Waiter(object):
1838 """A capability to wait for processes.
1839
1840 This must be a singleton (and is because CommandEvaluator is a singleton).
1841
1842 Invariants:
1843 - Every child process is registered once
1844 - Every child process is waited for
1845
1846 Canonical example of why we need a GLOBAL waiter:
1847
1848 { sleep 3; echo 'done 3'; } &
1849 { sleep 4; echo 'done 4'; } &
1850
1851 # ... do arbitrary stuff ...
1852
1853 { sleep 1; exit 1; } | { sleep 2; exit 2; }
1854
1855 Now when you do wait() after starting the pipeline, you might get a pipeline
1856 process OR a background process! So you have to distinguish between them.
1857 """
1858
1859 def __init__(self, job_list, exec_opts, signal_safe, tracer):
1860 # type: (JobList, optview.Exec, pyos.SignalSafe, dev.Tracer) -> None
1861 self.job_list = job_list
1862 self.exec_opts = exec_opts
1863 self.signal_safe = signal_safe
1864 self.tracer = tracer
1865 self.last_status = 127 # wait -n error code
1866
1867 def WaitForOne(self, waitpid_options=0):
1868 # type: (int) -> int
1869 """Wait until the next process returns (or maybe Ctrl-C).
1870
1871 Returns:
1872 One of these negative numbers:
1873 W1_ECHILD Nothing to wait for
1874 W1_OK Caller should keep waiting
1875 UNTRAPPED_SIGWINCH
1876 Or
1877 result > 0 Signal that waitpid() was interrupted with
1878
1879 In the interactive shell, we return 0 if we get a Ctrl-C, so the caller
1880 will try again.
1881
1882 Callers:
1883 wait -n -- loop until there is one fewer process (TODO)
1884 wait -- loop until there are no processes
1885 wait $! -- loop until job state is Done (process or pipeline)
1886 Process::Wait() -- loop until Process state is done
1887 Pipeline::Wait() -- loop until Pipeline state is done
1888
1889 Comparisons:
1890 bash: jobs.c waitchld() Has a special case macro(!) CHECK_WAIT_INTR for
1891 the wait builtin
1892
1893 dash: jobs.c waitproc() uses sigfillset(), sigprocmask(), etc. Runs in a
1894 loop while (gotsigchld), but that might be a hack for System V!
1895
1896 Should we have a cleaner API like named posix::wait_for_one() ?
1897
1898 wait_result =
1899 ECHILD -- nothing to wait for
1900 | Done(int pid, int status) -- process done
1901 | EINTR(bool sigint) -- may or may not retry
1902 """
1903 pid, status = pyos.WaitPid(waitpid_options)
1904 if pid == 0: # WNOHANG passed, and no state changes
1905 return W1_AGAIN
1906 elif pid < 0: # error case
1907 err_num = status
1908 #log('waitpid() error => %d %s', e.errno, pyutil.strerror(e))
1909 if err_num == ECHILD:
1910 return W1_ECHILD # nothing to wait for caller should stop
1911 elif err_num == EINTR: # Bug #858 fix
1912 #log('WaitForOne() => %d', self.trap_state.GetLastSignal())
1913 return self.signal_safe.LastSignal() # e.g. 1 for SIGHUP
1914 else:
1915 # The signature of waitpid() means this shouldn't happen
1916 raise AssertionError()
1917
1918 # All child processes are supposed to be in this dict. But this may
1919 # legitimately happen if a grandchild outlives the child (its parent).
1920 # Then it is reparented under this process, so we might receive
1921 # notification of its exit, even though we didn't start it. We can't have
1922 # any knowledge of such processes, so print a warning.
1923 if pid not in self.job_list.child_procs:
1924 print_stderr("osh: PID %d stopped, but osh didn't start it" % pid)
1925 return W1_OK
1926
1927 proc = self.job_list.child_procs[pid]
1928 if 0:
1929 self.job_list.DebugPrint()
1930
1931 if WIFSIGNALED(status):
1932 term_sig = WTERMSIG(status)
1933 status = 128 + term_sig
1934
1935 # Print newline after Ctrl-C.
1936 if term_sig == SIGINT:
1937 print('')
1938
1939 proc.WhenDone(pid, status)
1940
1941 elif WIFEXITED(status):
1942 status = WEXITSTATUS(status)
1943 #log('exit status: %s', status)
1944 proc.WhenDone(pid, status)
1945
1946 elif WIFSTOPPED(status):
1947 #status = WEXITSTATUS(status)
1948 stop_sig = WSTOPSIG(status)
1949
1950 print_stderr('')
1951 print_stderr('[PID %d] Stopped with signal %d' % (pid, stop_sig))
1952 proc.WhenStopped(stop_sig)
1953
1954 else:
1955 raise AssertionError(status)
1956
1957 self.last_status = status # for wait -n
1958 self.tracer.OnProcessEnd(pid, status)
1959 return W1_OK
1960
1961 def PollNotifications(self):
1962 # type: () -> None
1963 """
1964 Process all pending state changes.
1965 """
1966 while self.WaitForOne(waitpid_options=WNOHANG) == W1_OK:
1967 continue