1 | # Copyright 2016 Andy Chu. All rights reserved.
|
2 | # Licensed under the Apache License, Version 2.0 (the "License");
|
3 | # you may not use this file except in compliance with the License.
|
4 | # You may obtain a copy of the License at
|
5 | #
|
6 | # http://www.apache.org/licenses/LICENSE-2.0
|
7 | """
|
8 | process.py - Launch processes and manipulate file descriptors.
|
9 | """
|
10 | from __future__ import print_function
|
11 |
|
12 | from errno import EACCES, EBADF, ECHILD, EINTR, ENOENT, ENOEXEC
|
13 | import fcntl as fcntl_
|
14 | from fcntl import F_DUPFD, F_GETFD, F_SETFD, FD_CLOEXEC
|
15 | from signal import (SIG_DFL, SIG_IGN, SIGINT, SIGPIPE, SIGQUIT, SIGTSTP,
|
16 | SIGTTOU, SIGTTIN, SIGWINCH)
|
17 |
|
18 | from _devbuild.gen.id_kind_asdl import Id
|
19 | from _devbuild.gen.runtime_asdl import (job_state_e, job_state_t,
|
20 | job_state_str, wait_status,
|
21 | wait_status_t, RedirValue,
|
22 | redirect_arg, redirect_arg_e, trace,
|
23 | trace_t)
|
24 | from _devbuild.gen.syntax_asdl import (
|
25 | loc_t,
|
26 | redir_loc,
|
27 | redir_loc_e,
|
28 | redir_loc_t,
|
29 | )
|
30 | from _devbuild.gen.value_asdl import (value, value_e)
|
31 | from core import dev
|
32 | from core import error
|
33 | from core.error import e_die
|
34 | from core import pyutil
|
35 | from core import pyos
|
36 | from core import state
|
37 | from core import ui
|
38 | from core import util
|
39 | from data_lang import j8_lite
|
40 | from frontend import location
|
41 | from frontend import match
|
42 | from mycpp import mylib
|
43 | from mycpp.mylib import log, print_stderr, probe, tagswitch, iteritems
|
44 |
|
45 | import posix_ as posix
|
46 | from posix_ import (
|
47 | # translated by mycpp and directly called! No wrapper!
|
48 | WIFSIGNALED,
|
49 | WIFEXITED,
|
50 | WIFSTOPPED,
|
51 | WEXITSTATUS,
|
52 | WSTOPSIG,
|
53 | WTERMSIG,
|
54 | WNOHANG,
|
55 | O_APPEND,
|
56 | O_CREAT,
|
57 | O_NONBLOCK,
|
58 | O_NOCTTY,
|
59 | O_RDONLY,
|
60 | O_RDWR,
|
61 | O_WRONLY,
|
62 | O_TRUNC,
|
63 | )
|
64 |
|
65 | from typing import IO, List, Tuple, Dict, Optional, Any, cast, TYPE_CHECKING
|
66 |
|
67 | if TYPE_CHECKING:
|
68 | from _devbuild.gen.runtime_asdl import cmd_value
|
69 | from _devbuild.gen.syntax_asdl import command_t
|
70 | from builtin import trap_osh
|
71 | from core import optview
|
72 | from core.ui import ErrorFormatter
|
73 | from core.util import _DebugFile
|
74 | from osh.cmd_eval import CommandEvaluator
|
75 |
|
76 | NO_FD = -1
|
77 |
|
78 | # Minimum file descriptor that the shell can use. Other descriptors can be
|
79 | # directly used by user programs, e.g. exec 9>&1
|
80 | #
|
81 | # Oil uses 100 because users are allowed TWO digits in frontend/lexer_def.py.
|
82 | # This is a compromise between bash (unlimited, but requires crazy
|
83 | # bookkeeping), and dash/zsh (10) and mksh (24)
|
84 | _SHELL_MIN_FD = 100
|
85 |
|
86 | # Style for 'jobs' builtin
|
87 | STYLE_DEFAULT = 0
|
88 | STYLE_LONG = 1
|
89 | STYLE_PID_ONLY = 2
|
90 |
|
91 | # To save on allocations in JobList::GetJobWithSpec()
|
92 | CURRENT_JOB_SPECS = ['', '%', '%%', '%+']
|
93 |
|
94 |
|
95 | class ctx_FileCloser(object):
|
96 |
|
97 | def __init__(self, f):
|
98 | # type: (mylib.LineReader) -> None
|
99 | self.f = f
|
100 |
|
101 | def __enter__(self):
|
102 | # type: () -> None
|
103 | pass
|
104 |
|
105 | def __exit__(self, type, value, traceback):
|
106 | # type: (Any, Any, Any) -> None
|
107 | self.f.close()
|
108 |
|
109 |
|
110 | def InitInteractiveShell():
|
111 | # type: () -> None
|
112 | """Called when initializing an interactive shell."""
|
113 |
|
114 | # The shell itself should ignore Ctrl-\.
|
115 | pyos.Sigaction(SIGQUIT, SIG_IGN)
|
116 |
|
117 | # This prevents Ctrl-Z from suspending OSH in interactive mode.
|
118 | pyos.Sigaction(SIGTSTP, SIG_IGN)
|
119 |
|
120 | # More signals from
|
121 | # https://www.gnu.org/software/libc/manual/html_node/Initializing-the-Shell.html
|
122 | # (but not SIGCHLD)
|
123 | pyos.Sigaction(SIGTTOU, SIG_IGN)
|
124 | pyos.Sigaction(SIGTTIN, SIG_IGN)
|
125 |
|
126 | # Register a callback to receive terminal width changes.
|
127 | # NOTE: In line_input.c, we turned off rl_catch_sigwinch.
|
128 |
|
129 | # This is ALWAYS on, which means that it can cause EINTR, and wait() and
|
130 | # read() have to handle it
|
131 | pyos.RegisterSignalInterest(SIGWINCH)
|
132 |
|
133 |
|
134 | def SaveFd(fd):
|
135 | # type: (int) -> int
|
136 | saved = fcntl_.fcntl(fd, F_DUPFD, _SHELL_MIN_FD) # type: int
|
137 | return saved
|
138 |
|
139 |
|
140 | class _RedirFrame(object):
|
141 |
|
142 | def __init__(self, saved_fd, orig_fd, forget):
|
143 | # type: (int, int, bool) -> None
|
144 | self.saved_fd = saved_fd
|
145 | self.orig_fd = orig_fd
|
146 | self.forget = forget
|
147 |
|
148 |
|
149 | class _FdFrame(object):
|
150 |
|
151 | def __init__(self):
|
152 | # type: () -> None
|
153 | self.saved = [] # type: List[_RedirFrame]
|
154 | self.need_wait = [] # type: List[Process]
|
155 |
|
156 | def Forget(self):
|
157 | # type: () -> None
|
158 | """For exec 1>&2."""
|
159 | for rf in reversed(self.saved):
|
160 | if rf.saved_fd != NO_FD and rf.forget:
|
161 | posix.close(rf.saved_fd)
|
162 |
|
163 | del self.saved[:] # like list.clear() in Python 3.3
|
164 | del self.need_wait[:]
|
165 |
|
166 | def __repr__(self):
|
167 | # type: () -> str
|
168 | return '<_FdFrame %s>' % self.saved
|
169 |
|
170 |
|
171 | class FdState(object):
|
172 | """File descriptor state for the current process.
|
173 |
|
174 | For example, you can do 'myfunc > out.txt' without forking. Child
|
175 | processes inherit our state.
|
176 | """
|
177 |
|
178 | def __init__(
|
179 | self,
|
180 | errfmt, # type: ui.ErrorFormatter
|
181 | job_control, # type: JobControl
|
182 | job_list, # type: JobList
|
183 | mem, #type: state.Mem
|
184 | tracer, # type: Optional[dev.Tracer]
|
185 | waiter, # type: Optional[Waiter]
|
186 | ):
|
187 | # type: (...) -> None
|
188 | """
|
189 | Args:
|
190 | errfmt: for errors
|
191 | job_list: For keeping track of _HereDocWriterThunk
|
192 | """
|
193 | self.errfmt = errfmt
|
194 | self.job_control = job_control
|
195 | self.job_list = job_list
|
196 | self.cur_frame = _FdFrame() # for the top level
|
197 | self.stack = [self.cur_frame]
|
198 | self.mem = mem
|
199 | self.tracer = tracer
|
200 | self.waiter = waiter
|
201 |
|
202 | def Open(self, path):
|
203 | # type: (str) -> mylib.LineReader
|
204 | """Opens a path for read, but moves it out of the reserved 3-9 fd
|
205 | range.
|
206 |
|
207 | Returns:
|
208 | A Python file object. The caller is responsible for Close().
|
209 |
|
210 | Raises:
|
211 | IOError or OSError if the path can't be found. (This is Python-induced wart)
|
212 | """
|
213 | fd_mode = O_RDONLY
|
214 | f = self._Open(path, 'r', fd_mode)
|
215 |
|
216 | # Hacky downcast
|
217 | return cast('mylib.LineReader', f)
|
218 |
|
219 | # used for util.DebugFile
|
220 | def OpenForWrite(self, path):
|
221 | # type: (str) -> mylib.Writer
|
222 | fd_mode = O_CREAT | O_RDWR
|
223 | f = self._Open(path, 'w', fd_mode)
|
224 |
|
225 | # Hacky downcast
|
226 | return cast('mylib.Writer', f)
|
227 |
|
228 | def _Open(self, path, c_mode, fd_mode):
|
229 | # type: (str, str, int) -> IO[str]
|
230 | fd = posix.open(path, fd_mode, 0o666) # may raise OSError
|
231 |
|
232 | # Immediately move it to a new location
|
233 | new_fd = SaveFd(fd)
|
234 | posix.close(fd)
|
235 |
|
236 | # Return a Python file handle
|
237 | f = posix.fdopen(new_fd, c_mode) # may raise IOError
|
238 | return f
|
239 |
|
240 | def _WriteFdToMem(self, fd_name, fd):
|
241 | # type: (str, int) -> None
|
242 | if self.mem:
|
243 | # setvar, not setref
|
244 | state.OshLanguageSetValue(self.mem, location.LName(fd_name),
|
245 | value.Str(str(fd)))
|
246 |
|
247 | def _ReadFdFromMem(self, fd_name):
|
248 | # type: (str) -> int
|
249 | val = self.mem.GetValue(fd_name)
|
250 | if val.tag() == value_e.Str:
|
251 | try:
|
252 | return int(cast(value.Str, val).s)
|
253 | except ValueError:
|
254 | return NO_FD
|
255 | return NO_FD
|
256 |
|
257 | def _PushSave(self, fd):
|
258 | # type: (int) -> bool
|
259 | """Save fd to a new location and remember to restore it later."""
|
260 | #log('---- _PushSave %s', fd)
|
261 | ok = True
|
262 | try:
|
263 | new_fd = SaveFd(fd)
|
264 | except (IOError, OSError) as e:
|
265 | ok = False
|
266 | # Example program that causes this error: exec 4>&1. Descriptor 4 isn't
|
267 | # open.
|
268 | # This seems to be ignored in dash too in savefd()?
|
269 | if e.errno != EBADF:
|
270 | raise
|
271 | if ok:
|
272 | posix.close(fd)
|
273 | fcntl_.fcntl(new_fd, F_SETFD, FD_CLOEXEC)
|
274 | self.cur_frame.saved.append(_RedirFrame(new_fd, fd, True))
|
275 | else:
|
276 | # if we got EBADF, we still need to close the original on Pop()
|
277 | self._PushClose(fd)
|
278 |
|
279 | return ok
|
280 |
|
281 | def _PushDup(self, fd1, blame_loc):
|
282 | # type: (int, redir_loc_t) -> int
|
283 | """Save fd2 in a higher range, and dup fd1 onto fd2.
|
284 |
|
285 | Returns whether F_DUPFD/dup2 succeeded, and the new descriptor.
|
286 | """
|
287 | UP_loc = blame_loc
|
288 | if blame_loc.tag() == redir_loc_e.VarName:
|
289 | fd2_name = cast(redir_loc.VarName, UP_loc).name
|
290 | try:
|
291 | # F_DUPFD: GREATER than range
|
292 | new_fd = fcntl_.fcntl(fd1, F_DUPFD, _SHELL_MIN_FD) # type: int
|
293 | except (IOError, OSError) as e:
|
294 | if e.errno == EBADF:
|
295 | print_stderr('F_DUPFD fd %d: %s' %
|
296 | (fd1, pyutil.strerror(e)))
|
297 | return NO_FD
|
298 | else:
|
299 | raise # this redirect failed
|
300 |
|
301 | self._WriteFdToMem(fd2_name, new_fd)
|
302 |
|
303 | elif blame_loc.tag() == redir_loc_e.Fd:
|
304 | fd2 = cast(redir_loc.Fd, UP_loc).fd
|
305 |
|
306 | if fd1 == fd2:
|
307 | # The user could have asked for it to be open on descriptor 3, but open()
|
308 | # already returned 3, e.g. echo 3>out.txt
|
309 | return NO_FD
|
310 |
|
311 | # Check the validity of fd1 before _PushSave(fd2)
|
312 | try:
|
313 | fcntl_.fcntl(fd1, F_GETFD)
|
314 | except (IOError, OSError) as e:
|
315 | print_stderr('F_GETFD fd %d: %s' % (fd1, pyutil.strerror(e)))
|
316 | raise
|
317 |
|
318 | need_restore = self._PushSave(fd2)
|
319 |
|
320 | #log('==== dup2 %s %s\n' % (fd1, fd2))
|
321 | try:
|
322 | posix.dup2(fd1, fd2)
|
323 | except (IOError, OSError) as e:
|
324 | # bash/dash give this error too, e.g. for 'echo hi 1>&3'
|
325 | print_stderr('dup2(%d, %d): %s' %
|
326 | (fd1, fd2, pyutil.strerror(e)))
|
327 |
|
328 | # Restore and return error
|
329 | if need_restore:
|
330 | rf = self.cur_frame.saved.pop()
|
331 | posix.dup2(rf.saved_fd, rf.orig_fd)
|
332 | posix.close(rf.saved_fd)
|
333 |
|
334 | raise # this redirect failed
|
335 |
|
336 | new_fd = fd2
|
337 |
|
338 | else:
|
339 | raise AssertionError()
|
340 |
|
341 | return new_fd
|
342 |
|
343 | def _PushCloseFd(self, blame_loc):
|
344 | # type: (redir_loc_t) -> bool
|
345 | """For 2>&-"""
|
346 | # exec {fd}>&- means close the named descriptor
|
347 |
|
348 | UP_loc = blame_loc
|
349 | if blame_loc.tag() == redir_loc_e.VarName:
|
350 | fd_name = cast(redir_loc.VarName, UP_loc).name
|
351 | fd = self._ReadFdFromMem(fd_name)
|
352 | if fd == NO_FD:
|
353 | return False
|
354 |
|
355 | elif blame_loc.tag() == redir_loc_e.Fd:
|
356 | fd = cast(redir_loc.Fd, UP_loc).fd
|
357 |
|
358 | else:
|
359 | raise AssertionError()
|
360 |
|
361 | self._PushSave(fd)
|
362 |
|
363 | return True
|
364 |
|
365 | def _PushClose(self, fd):
|
366 | # type: (int) -> None
|
367 | self.cur_frame.saved.append(_RedirFrame(NO_FD, fd, False))
|
368 |
|
369 | def _PushWait(self, proc):
|
370 | # type: (Process) -> None
|
371 | self.cur_frame.need_wait.append(proc)
|
372 |
|
373 | def _ApplyRedirect(self, r):
|
374 | # type: (RedirValue) -> None
|
375 | arg = r.arg
|
376 | UP_arg = arg
|
377 | with tagswitch(arg) as case:
|
378 |
|
379 | if case(redirect_arg_e.Path):
|
380 | arg = cast(redirect_arg.Path, UP_arg)
|
381 |
|
382 | if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &>
|
383 | # NOTE: This is different than >| because it respects noclobber, but
|
384 | # that option is almost never used. See test/wild.sh.
|
385 | mode = O_CREAT | O_WRONLY | O_TRUNC
|
386 | elif r.op_id == Id.Redir_Clobber: # >|
|
387 | mode = O_CREAT | O_WRONLY | O_TRUNC
|
388 | elif r.op_id in (Id.Redir_DGreat,
|
389 | Id.Redir_AndDGreat): # >> &>>
|
390 | mode = O_CREAT | O_WRONLY | O_APPEND
|
391 | elif r.op_id == Id.Redir_Less: # <
|
392 | mode = O_RDONLY
|
393 | elif r.op_id == Id.Redir_LessGreat: # <>
|
394 | mode = O_CREAT | O_RDWR
|
395 | else:
|
396 | raise NotImplementedError(r.op_id)
|
397 |
|
398 | # NOTE: 0666 is affected by umask, all shells use it.
|
399 | try:
|
400 | open_fd = posix.open(arg.filename, mode, 0o666)
|
401 | except (IOError, OSError) as e:
|
402 | self.errfmt.Print_("Can't open %r: %s" %
|
403 | (arg.filename, pyutil.strerror(e)),
|
404 | blame_loc=r.op_loc)
|
405 | raise # redirect failed
|
406 |
|
407 | new_fd = self._PushDup(open_fd, r.loc)
|
408 | if new_fd != NO_FD:
|
409 | posix.close(open_fd)
|
410 |
|
411 | # Now handle &> and &>> and their variants. These pairs are the same:
|
412 | #
|
413 | # stdout_stderr.py &> out-err.txt
|
414 | # stdout_stderr.py > out-err.txt 2>&1
|
415 | #
|
416 | # stdout_stderr.py 3&> out-err.txt
|
417 | # stdout_stderr.py 3> out-err.txt 2>&3
|
418 | #
|
419 | # Ditto for {fd}> and {fd}&>
|
420 |
|
421 | if r.op_id in (Id.Redir_AndGreat, Id.Redir_AndDGreat):
|
422 | self._PushDup(new_fd, redir_loc.Fd(2))
|
423 |
|
424 | elif case(redirect_arg_e.CopyFd): # e.g. echo hi 1>&2
|
425 | arg = cast(redirect_arg.CopyFd, UP_arg)
|
426 |
|
427 | if r.op_id == Id.Redir_GreatAnd: # 1>&2
|
428 | self._PushDup(arg.target_fd, r.loc)
|
429 |
|
430 | elif r.op_id == Id.Redir_LessAnd: # 0<&5
|
431 | # The only difference between >& and <& is the default file
|
432 | # descriptor argument.
|
433 | self._PushDup(arg.target_fd, r.loc)
|
434 |
|
435 | else:
|
436 | raise NotImplementedError()
|
437 |
|
438 | elif case(redirect_arg_e.MoveFd): # e.g. echo hi 5>&6-
|
439 | arg = cast(redirect_arg.MoveFd, UP_arg)
|
440 | new_fd = self._PushDup(arg.target_fd, r.loc)
|
441 | if new_fd != NO_FD:
|
442 | posix.close(arg.target_fd)
|
443 |
|
444 | UP_loc = r.loc
|
445 | if r.loc.tag() == redir_loc_e.Fd:
|
446 | fd = cast(redir_loc.Fd, UP_loc).fd
|
447 | else:
|
448 | fd = NO_FD
|
449 |
|
450 | self.cur_frame.saved.append(_RedirFrame(new_fd, fd, False))
|
451 |
|
452 | elif case(redirect_arg_e.CloseFd): # e.g. echo hi 5>&-
|
453 | self._PushCloseFd(r.loc)
|
454 |
|
455 | elif case(redirect_arg_e.HereDoc):
|
456 | arg = cast(redirect_arg.HereDoc, UP_arg)
|
457 |
|
458 | # NOTE: Do these descriptors have to be moved out of the range 0-9?
|
459 | read_fd, write_fd = posix.pipe()
|
460 |
|
461 | self._PushDup(read_fd, r.loc) # stdin is now the pipe
|
462 |
|
463 | # We can't close like we do in the filename case above? The writer can
|
464 | # get a "broken pipe".
|
465 | self._PushClose(read_fd)
|
466 |
|
467 | thunk = _HereDocWriterThunk(write_fd, arg.body)
|
468 |
|
469 | # Use PIPE_SIZE to save a process in the case of small here
|
470 | # docs, which are the common case. (dash does this.)
|
471 |
|
472 | # Note: could instrument this to see how often it happens.
|
473 | # Though strace -ff can also work.
|
474 | start_process = len(arg.body) > 4096
|
475 | #start_process = True
|
476 |
|
477 | if start_process:
|
478 | here_proc = Process(thunk, self.job_control, self.job_list,
|
479 | self.tracer)
|
480 |
|
481 | # NOTE: we could close the read pipe here, but it doesn't really
|
482 | # matter because we control the code.
|
483 | here_proc.StartProcess(trace.HereDoc)
|
484 | #log('Started %s as %d', here_proc, pid)
|
485 | self._PushWait(here_proc)
|
486 |
|
487 | # Now that we've started the child, close it in the parent.
|
488 | posix.close(write_fd)
|
489 |
|
490 | else:
|
491 | posix.write(write_fd, arg.body)
|
492 | posix.close(write_fd)
|
493 |
|
494 | def Push(self, redirects, err_out):
|
495 | # type: (List[RedirValue], List[error.IOError_OSError]) -> None
|
496 | """Apply a group of redirects and remember to undo them."""
|
497 |
|
498 | #log('> fd_state.Push %s', redirects)
|
499 | new_frame = _FdFrame()
|
500 | self.stack.append(new_frame)
|
501 | self.cur_frame = new_frame
|
502 |
|
503 | for r in redirects:
|
504 | #log('apply %s', r)
|
505 | with ui.ctx_Location(self.errfmt, r.op_loc):
|
506 | try:
|
507 | self._ApplyRedirect(r)
|
508 | except (IOError, OSError) as e:
|
509 | err_out.append(e)
|
510 | # This can fail too
|
511 | self.Pop(err_out)
|
512 | return # for bad descriptor, etc.
|
513 |
|
514 | def PushStdinFromPipe(self, r):
|
515 | # type: (int) -> bool
|
516 | """Save the current stdin and make it come from descriptor 'r'.
|
517 |
|
518 | 'r' is typically the read-end of a pipe. For 'lastpipe'/ZSH
|
519 | semantics of
|
520 |
|
521 | echo foo | read line; echo $line
|
522 | """
|
523 | new_frame = _FdFrame()
|
524 | self.stack.append(new_frame)
|
525 | self.cur_frame = new_frame
|
526 |
|
527 | self._PushDup(r, redir_loc.Fd(0))
|
528 | return True
|
529 |
|
530 | def Pop(self, err_out):
|
531 | # type: (List[error.IOError_OSError]) -> None
|
532 | frame = self.stack.pop()
|
533 | #log('< Pop %s', frame)
|
534 | for rf in reversed(frame.saved):
|
535 | if rf.saved_fd == NO_FD:
|
536 | #log('Close %d', orig)
|
537 | try:
|
538 | posix.close(rf.orig_fd)
|
539 | except (IOError, OSError) as e:
|
540 | err_out.append(e)
|
541 | log('Error closing descriptor %d: %s', rf.orig_fd,
|
542 | pyutil.strerror(e))
|
543 | return
|
544 | else:
|
545 | try:
|
546 | posix.dup2(rf.saved_fd, rf.orig_fd)
|
547 | except (IOError, OSError) as e:
|
548 | err_out.append(e)
|
549 | log('dup2(%d, %d) error: %s', rf.saved_fd, rf.orig_fd,
|
550 | pyutil.strerror(e))
|
551 | #log('fd state:')
|
552 | #posix.system('ls -l /proc/%s/fd' % posix.getpid())
|
553 | return
|
554 | posix.close(rf.saved_fd)
|
555 | #log('dup2 %s %s', saved, orig)
|
556 |
|
557 | # Wait for here doc processes to finish.
|
558 | for proc in frame.need_wait:
|
559 | unused_status = proc.Wait(self.waiter)
|
560 |
|
561 | def MakePermanent(self):
|
562 | # type: () -> None
|
563 | self.cur_frame.Forget()
|
564 |
|
565 |
|
566 | class ChildStateChange(object):
|
567 |
|
568 | def __init__(self):
|
569 | # type: () -> None
|
570 | """Empty constructor for mycpp."""
|
571 | pass
|
572 |
|
573 | def Apply(self):
|
574 | # type: () -> None
|
575 | raise NotImplementedError()
|
576 |
|
577 | def ApplyFromParent(self, proc):
|
578 | # type: (Process) -> None
|
579 | """Noop for all state changes other than SetPgid for mycpp."""
|
580 | pass
|
581 |
|
582 |
|
583 | class StdinFromPipe(ChildStateChange):
|
584 |
|
585 | def __init__(self, pipe_read_fd, w):
|
586 | # type: (int, int) -> None
|
587 | self.r = pipe_read_fd
|
588 | self.w = w
|
589 |
|
590 | def __repr__(self):
|
591 | # type: () -> str
|
592 | return '<StdinFromPipe %d %d>' % (self.r, self.w)
|
593 |
|
594 | def Apply(self):
|
595 | # type: () -> None
|
596 | posix.dup2(self.r, 0)
|
597 | posix.close(self.r) # close after dup
|
598 |
|
599 | posix.close(self.w) # we're reading from the pipe, not writing
|
600 | #log('child CLOSE w %d pid=%d', self.w, posix.getpid())
|
601 |
|
602 |
|
603 | class StdoutToPipe(ChildStateChange):
|
604 |
|
605 | def __init__(self, r, pipe_write_fd):
|
606 | # type: (int, int) -> None
|
607 | self.r = r
|
608 | self.w = pipe_write_fd
|
609 |
|
610 | def __repr__(self):
|
611 | # type: () -> str
|
612 | return '<StdoutToPipe %d %d>' % (self.r, self.w)
|
613 |
|
614 | def Apply(self):
|
615 | # type: () -> None
|
616 | posix.dup2(self.w, 1)
|
617 | posix.close(self.w) # close after dup
|
618 |
|
619 | posix.close(self.r) # we're writing to the pipe, not reading
|
620 | #log('child CLOSE r %d pid=%d', self.r, posix.getpid())
|
621 |
|
622 |
|
623 | INVALID_PGID = -1
|
624 | # argument to setpgid() that means the process is its own leader
|
625 | OWN_LEADER = 0
|
626 |
|
627 |
|
628 | class SetPgid(ChildStateChange):
|
629 |
|
630 | def __init__(self, pgid, tracer):
|
631 | # type: (int, dev.Tracer) -> None
|
632 | self.pgid = pgid
|
633 | self.tracer = tracer
|
634 |
|
635 | def Apply(self):
|
636 | # type: () -> None
|
637 | try:
|
638 | posix.setpgid(0, self.pgid)
|
639 | except (IOError, OSError) as e:
|
640 | self.tracer.OtherMessage(
|
641 | 'osh: child %d failed to set its process group to %d: %s' %
|
642 | (posix.getpid(), self.pgid, pyutil.strerror(e)))
|
643 |
|
644 | def ApplyFromParent(self, proc):
|
645 | # type: (Process) -> None
|
646 | try:
|
647 | posix.setpgid(proc.pid, self.pgid)
|
648 | except (IOError, OSError) as e:
|
649 | self.tracer.OtherMessage(
|
650 | 'osh: parent failed to set process group for PID %d to %d: %s'
|
651 | % (proc.pid, self.pgid, pyutil.strerror(e)))
|
652 |
|
653 |
|
654 | class ExternalProgram(object):
|
655 | """The capability to execute an external program like 'ls'."""
|
656 |
|
657 | def __init__(
|
658 | self,
|
659 | hijack_shebang, # type: str
|
660 | fd_state, # type: FdState
|
661 | errfmt, # type: ErrorFormatter
|
662 | debug_f, # type: _DebugFile
|
663 | ):
|
664 | # type: (...) -> None
|
665 | """
|
666 | Args:
|
667 | hijack_shebang: The path of an interpreter to run instead of the one
|
668 | specified in the shebang line. May be empty.
|
669 | """
|
670 | self.hijack_shebang = hijack_shebang
|
671 | self.fd_state = fd_state
|
672 | self.errfmt = errfmt
|
673 | self.debug_f = debug_f
|
674 |
|
675 | def Exec(self, argv0_path, cmd_val, environ):
|
676 | # type: (str, cmd_value.Argv, Dict[str, str]) -> None
|
677 | """Execute a program and exit this process.
|
678 |
|
679 | Called by: ls / exec ls / ( ls / )
|
680 | """
|
681 | probe('process', 'ExternalProgram_Exec', argv0_path)
|
682 | self._Exec(argv0_path, cmd_val.argv, cmd_val.arg_locs[0], environ,
|
683 | True)
|
684 | assert False, "This line should never execute" # NO RETURN
|
685 |
|
686 | def _Exec(self, argv0_path, argv, argv0_loc, environ, should_retry):
|
687 | # type: (str, List[str], loc_t, Dict[str, str], bool) -> None
|
688 | if len(self.hijack_shebang):
|
689 | opened = True
|
690 | try:
|
691 | f = self.fd_state.Open(argv0_path)
|
692 | except (IOError, OSError) as e:
|
693 | opened = False
|
694 |
|
695 | if opened:
|
696 | with ctx_FileCloser(f):
|
697 | # Test if the shebang looks like a shell. TODO: The file might be
|
698 | # binary with no newlines, so read 80 bytes instead of readline().
|
699 |
|
700 | #line = f.read(80) # type: ignore # TODO: fix this
|
701 | line = f.readline()
|
702 |
|
703 | if match.ShouldHijack(line):
|
704 | h_argv = [self.hijack_shebang, argv0_path]
|
705 | h_argv.extend(argv[1:])
|
706 | argv = h_argv
|
707 | argv0_path = self.hijack_shebang
|
708 | self.debug_f.writeln('Hijacked: %s' % argv0_path)
|
709 | else:
|
710 | #self.debug_f.log('Not hijacking %s (%r)', argv, line)
|
711 | pass
|
712 |
|
713 | try:
|
714 | posix.execve(argv0_path, argv, environ)
|
715 | except (IOError, OSError) as e:
|
716 | # Run with /bin/sh when ENOEXEC error (no shebang). All shells do this.
|
717 | if e.errno == ENOEXEC and should_retry:
|
718 | new_argv = ['/bin/sh', argv0_path]
|
719 | new_argv.extend(argv[1:])
|
720 | self._Exec('/bin/sh', new_argv, argv0_loc, environ, False)
|
721 | # NO RETURN
|
722 |
|
723 | # Would be nice: when the path is relative and ENOENT: print PWD and do
|
724 | # spelling correction?
|
725 |
|
726 | self.errfmt.Print_(
|
727 | "Can't execute %r: %s" % (argv0_path, pyutil.strerror(e)),
|
728 | argv0_loc)
|
729 |
|
730 | # POSIX mentions 126 and 127 for two specific errors. The rest are
|
731 | # unspecified.
|
732 | #
|
733 | # http://pubs.opengroup.org/onlinepubs/9699919799.2016edition/utilities/V3_chap02.html#tag_18_08_02
|
734 | if e.errno == EACCES:
|
735 | status = 126
|
736 | elif e.errno == ENOENT:
|
737 | # TODO: most shells print 'command not found', rather than strerror()
|
738 | # == "No such file or directory". That's better because it's at the
|
739 | # end of the path search, and we're never searching for a directory.
|
740 | status = 127
|
741 | else:
|
742 | # dash uses 2, but we use that for parse errors. This seems to be
|
743 | # consistent with mksh and zsh.
|
744 | status = 127
|
745 |
|
746 | posix._exit(status)
|
747 | # NO RETURN
|
748 |
|
749 |
|
750 | class Thunk(object):
|
751 | """Abstract base class for things runnable in another process."""
|
752 |
|
753 | def __init__(self):
|
754 | # type: () -> None
|
755 | """Empty constructor for mycpp."""
|
756 | pass
|
757 |
|
758 | def Run(self):
|
759 | # type: () -> None
|
760 | """Returns a status code."""
|
761 | raise NotImplementedError()
|
762 |
|
763 | def UserString(self):
|
764 | # type: () -> str
|
765 | """Display for the 'jobs' list."""
|
766 | raise NotImplementedError()
|
767 |
|
768 | def __repr__(self):
|
769 | # type: () -> str
|
770 | return self.UserString()
|
771 |
|
772 |
|
773 | class ExternalThunk(Thunk):
|
774 | """An external executable."""
|
775 |
|
776 | def __init__(self, ext_prog, argv0_path, cmd_val, environ):
|
777 | # type: (ExternalProgram, str, cmd_value.Argv, Dict[str, str]) -> None
|
778 | self.ext_prog = ext_prog
|
779 | self.argv0_path = argv0_path
|
780 | self.cmd_val = cmd_val
|
781 | self.environ = environ
|
782 |
|
783 | def UserString(self):
|
784 | # type: () -> str
|
785 |
|
786 | # NOTE: This is the format the Tracer uses.
|
787 | # bash displays sleep $n & (code)
|
788 | # but OSH displays sleep 1 & (argv array)
|
789 | # We could switch the former but I'm not sure it's necessary.
|
790 | tmp = [j8_lite.MaybeShellEncode(a) for a in self.cmd_val.argv]
|
791 | return '[process] %s' % ' '.join(tmp)
|
792 |
|
793 | def Run(self):
|
794 | # type: () -> None
|
795 | """An ExternalThunk is run in parent for the exec builtin."""
|
796 | self.ext_prog.Exec(self.argv0_path, self.cmd_val, self.environ)
|
797 |
|
798 |
|
799 | class SubProgramThunk(Thunk):
|
800 | """A subprogram that can be executed in another process."""
|
801 |
|
802 | def __init__(self,
|
803 | cmd_ev,
|
804 | node,
|
805 | trap_state,
|
806 | multi_trace,
|
807 | inherit_errexit=True):
|
808 | # type: (CommandEvaluator, command_t, trap_osh.TrapState, dev.MultiTracer, bool) -> None
|
809 | self.cmd_ev = cmd_ev
|
810 | self.node = node
|
811 | self.trap_state = trap_state
|
812 | self.multi_trace = multi_trace
|
813 | self.inherit_errexit = inherit_errexit # for bash errexit compatibility
|
814 |
|
815 | def UserString(self):
|
816 | # type: () -> str
|
817 |
|
818 | # NOTE: These can be pieces of a pipeline, so they're arbitrary nodes.
|
819 | # TODO: Extract SPIDS from node to display source? Note that
|
820 | # CompoundStatus also has locations of each pipeline component; see
|
821 | # Executor.RunPipeline()
|
822 | thunk_str = ui.CommandType(self.node)
|
823 | return '[subprog] %s' % thunk_str
|
824 |
|
825 | def Run(self):
|
826 | # type: () -> None
|
827 | #self.errfmt.OneLineErrExit() # don't quote code in child processes
|
828 | probe('process', 'SubProgramThunk_Run')
|
829 |
|
830 | # TODO: break circular dep. Bit flags could go in ASDL or headers.
|
831 | from osh import cmd_eval
|
832 |
|
833 | # signal handlers aren't inherited
|
834 | self.trap_state.ClearForSubProgram()
|
835 |
|
836 | # NOTE: may NOT return due to exec().
|
837 | if not self.inherit_errexit:
|
838 | self.cmd_ev.mutable_opts.DisableErrExit()
|
839 | try:
|
840 | # optimize to eliminate redundant subshells like ( echo hi ) | wc -l etc.
|
841 | self.cmd_ev.ExecuteAndCatch(self.node, cmd_flags=cmd_eval.Optimize)
|
842 | status = self.cmd_ev.LastStatus()
|
843 | # NOTE: We ignore the is_fatal return value. The user should set -o
|
844 | # errexit so failures in subprocesses cause failures in the parent.
|
845 | except util.UserExit as e:
|
846 | status = e.status
|
847 |
|
848 | # Handle errors in a subshell. These two cases are repeated from main()
|
849 | # and the core/completion.py hook.
|
850 | except KeyboardInterrupt:
|
851 | print('')
|
852 | status = 130 # 128 + 2
|
853 | except (IOError, OSError) as e:
|
854 | print_stderr('oils I/O error (subprogram): %s' %
|
855 | pyutil.strerror(e))
|
856 | status = 2
|
857 |
|
858 | # If ProcessInit() doesn't turn off buffering, this is needed before
|
859 | # _exit()
|
860 | pyos.FlushStdout()
|
861 |
|
862 | self.multi_trace.WriteDumps()
|
863 |
|
864 | # We do NOT want to raise SystemExit here. Otherwise dev.Tracer::Pop()
|
865 | # gets called in BOTH processes.
|
866 | # The crash dump seems to be unaffected.
|
867 | posix._exit(status)
|
868 |
|
869 |
|
870 | class _HereDocWriterThunk(Thunk):
|
871 | """Write a here doc to one end of a pipe.
|
872 |
|
873 | May be be executed in either a child process or the main shell
|
874 | process.
|
875 | """
|
876 |
|
877 | def __init__(self, w, body_str):
|
878 | # type: (int, str) -> None
|
879 | self.w = w
|
880 | self.body_str = body_str
|
881 |
|
882 | def UserString(self):
|
883 | # type: () -> str
|
884 |
|
885 | # You can hit Ctrl-Z and the here doc writer will be suspended! Other
|
886 | # shells don't have this problem because they use temp files! That's a bit
|
887 | # unfortunate.
|
888 | return '[here doc writer]'
|
889 |
|
890 | def Run(self):
|
891 | # type: () -> None
|
892 | """do_exit: For small pipelines."""
|
893 | probe('process', 'HereDocWriterThunk_Run')
|
894 | #log('Writing %r', self.body_str)
|
895 | posix.write(self.w, self.body_str)
|
896 | #log('Wrote %r', self.body_str)
|
897 | posix.close(self.w)
|
898 | #log('Closed %d', self.w)
|
899 |
|
900 | posix._exit(0)
|
901 |
|
902 |
|
903 | class Job(object):
|
904 | """Interface for both Process and Pipeline.
|
905 |
|
906 | They both can be put in the background and waited on.
|
907 |
|
908 | Confusing thing about pipelines in the background: They have TOO MANY NAMES.
|
909 |
|
910 | sleep 1 | sleep 2 &
|
911 |
|
912 | - The LAST PID is what's printed at the prompt. This is $!, a PROCESS ID and
|
913 | not a JOB ID.
|
914 | # https://www.gnu.org/software/bash/manual/html_node/Special-Parameters.html#Special-Parameters
|
915 | - The process group leader (setpgid) is the FIRST PID.
|
916 | - It's also %1 or %+. The last job started.
|
917 | """
|
918 |
|
919 | def __init__(self):
|
920 | # type: () -> None
|
921 | # Initial state with & or Ctrl-Z is Running.
|
922 | self.state = job_state_e.Running
|
923 | self.job_id = -1
|
924 | self.in_background = False
|
925 |
|
926 | def DisplayJob(self, job_id, f, style):
|
927 | # type: (int, mylib.Writer, int) -> None
|
928 | raise NotImplementedError()
|
929 |
|
930 | def State(self):
|
931 | # type: () -> job_state_t
|
932 | return self.state
|
933 |
|
934 | def ProcessGroupId(self):
|
935 | # type: () -> int
|
936 | """Return the process group ID associated with this job."""
|
937 | raise NotImplementedError()
|
938 |
|
939 | def JobWait(self, waiter):
|
940 | # type: (Waiter) -> wait_status_t
|
941 | """Wait for this process/pipeline to be stopped or finished."""
|
942 | raise NotImplementedError()
|
943 |
|
944 | def SetBackground(self):
|
945 | # type: () -> None
|
946 | """Record that this job is running in the background."""
|
947 | self.in_background = True
|
948 |
|
949 | def SetForeground(self):
|
950 | # type: () -> None
|
951 | """Record that this job is running in the foreground."""
|
952 | self.in_background = False
|
953 |
|
954 |
|
955 | class Process(Job):
|
956 | """A process to run.
|
957 |
|
958 | TODO: Should we make it clear that this is a FOREGROUND process? A
|
959 | background process is wrapped in a "job". It is unevaluated.
|
960 |
|
961 | It provides an API to manipulate file descriptor state in parent and child.
|
962 | """
|
963 |
|
964 | def __init__(self, thunk, job_control, job_list, tracer):
|
965 | # type: (Thunk, JobControl, JobList, dev.Tracer) -> None
|
966 | """
|
967 | Args:
|
968 | thunk: Thunk instance
|
969 | job_list: for process bookkeeping
|
970 | """
|
971 | Job.__init__(self)
|
972 | assert isinstance(thunk, Thunk), thunk
|
973 | self.thunk = thunk
|
974 | self.job_control = job_control
|
975 | self.job_list = job_list
|
976 | self.tracer = tracer
|
977 |
|
978 | # For pipelines
|
979 | self.parent_pipeline = None # type: Pipeline
|
980 | self.state_changes = [] # type: List[ChildStateChange]
|
981 | self.close_r = -1
|
982 | self.close_w = -1
|
983 |
|
984 | self.pid = -1
|
985 | self.status = -1
|
986 |
|
987 | def Init_ParentPipeline(self, pi):
|
988 | # type: (Pipeline) -> None
|
989 | """For updating PIPESTATUS."""
|
990 | self.parent_pipeline = pi
|
991 |
|
992 | def __repr__(self):
|
993 | # type: () -> str
|
994 |
|
995 | # note: be wary of infinite mutual recursion
|
996 | #s = ' %s' % self.parent_pipeline if self.parent_pipeline else ''
|
997 | #return '<Process %s%s>' % (self.thunk, s)
|
998 | return '<Process %s %s>' % (_JobStateStr(self.state), self.thunk)
|
999 |
|
1000 | def ProcessGroupId(self):
|
1001 | # type: () -> int
|
1002 | """Returns the group ID of this process."""
|
1003 | # This should only ever be called AFTER the process has started
|
1004 | assert self.pid != -1
|
1005 | if self.parent_pipeline:
|
1006 | # XXX: Maybe we should die here instead? Unclear if this branch
|
1007 | # should even be reachable with the current builtins.
|
1008 | return self.parent_pipeline.ProcessGroupId()
|
1009 |
|
1010 | return self.pid
|
1011 |
|
1012 | def DisplayJob(self, job_id, f, style):
|
1013 | # type: (int, mylib.Writer, int) -> None
|
1014 | if job_id == -1:
|
1015 | job_id_str = ' '
|
1016 | else:
|
1017 | job_id_str = '%%%d' % job_id
|
1018 | if style == STYLE_PID_ONLY:
|
1019 | f.write('%d\n' % self.pid)
|
1020 | else:
|
1021 | f.write('%s %d %7s ' %
|
1022 | (job_id_str, self.pid, _JobStateStr(self.state)))
|
1023 | f.write(self.thunk.UserString())
|
1024 | f.write('\n')
|
1025 |
|
1026 | def AddStateChange(self, s):
|
1027 | # type: (ChildStateChange) -> None
|
1028 | self.state_changes.append(s)
|
1029 |
|
1030 | def AddPipeToClose(self, r, w):
|
1031 | # type: (int, int) -> None
|
1032 | self.close_r = r
|
1033 | self.close_w = w
|
1034 |
|
1035 | def MaybeClosePipe(self):
|
1036 | # type: () -> None
|
1037 | if self.close_r != -1:
|
1038 | posix.close(self.close_r)
|
1039 | posix.close(self.close_w)
|
1040 |
|
1041 | def StartProcess(self, why):
|
1042 | # type: (trace_t) -> int
|
1043 | """Start this process with fork(), handling redirects."""
|
1044 | pid = posix.fork()
|
1045 | if pid < 0:
|
1046 | # When does this happen?
|
1047 | e_die('Fatal error in posix.fork()')
|
1048 |
|
1049 | elif pid == 0: # child
|
1050 | # Note: this happens in BOTH interactive and non-interactive shells.
|
1051 | # We technically don't need to do most of it in non-interactive, since we
|
1052 | # did not change state in InitInteractiveShell().
|
1053 |
|
1054 | for st in self.state_changes:
|
1055 | st.Apply()
|
1056 |
|
1057 | # Python sets SIGPIPE handler to SIG_IGN by default. Child processes
|
1058 | # shouldn't have this.
|
1059 | # https://docs.python.org/2/library/signal.html
|
1060 | # See Python/pythonrun.c.
|
1061 | pyos.Sigaction(SIGPIPE, SIG_DFL)
|
1062 |
|
1063 | # Respond to Ctrl-\ (core dump)
|
1064 | pyos.Sigaction(SIGQUIT, SIG_DFL)
|
1065 |
|
1066 | # Only standalone children should get Ctrl-Z. Pipelines remain in the
|
1067 | # foreground because suspending them is difficult with our 'lastpipe'
|
1068 | # semantics.
|
1069 | pid = posix.getpid()
|
1070 | if posix.getpgid(0) == pid and self.parent_pipeline is None:
|
1071 | pyos.Sigaction(SIGTSTP, SIG_DFL)
|
1072 |
|
1073 | # More signals from
|
1074 | # https://www.gnu.org/software/libc/manual/html_node/Launching-Jobs.html
|
1075 | # (but not SIGCHLD)
|
1076 | pyos.Sigaction(SIGTTOU, SIG_DFL)
|
1077 | pyos.Sigaction(SIGTTIN, SIG_DFL)
|
1078 |
|
1079 | self.tracer.OnNewProcess(pid)
|
1080 | # clear foreground pipeline for subshells
|
1081 | self.thunk.Run()
|
1082 | # Never returns
|
1083 |
|
1084 | #log('STARTED process %s, pid = %d', self, pid)
|
1085 | self.tracer.OnProcessStart(pid, why)
|
1086 |
|
1087 | # Class invariant: after the process is started, it stores its PID.
|
1088 | self.pid = pid
|
1089 |
|
1090 | # SetPgid needs to be applied from the child and the parent to avoid
|
1091 | # racing in calls to tcsetpgrp() in the parent. See APUE sec. 9.2.
|
1092 | for st in self.state_changes:
|
1093 | st.ApplyFromParent(self)
|
1094 |
|
1095 | # Program invariant: We keep track of every child process!
|
1096 | self.job_list.AddChildProcess(pid, self)
|
1097 |
|
1098 | return pid
|
1099 |
|
1100 | def Wait(self, waiter):
|
1101 | # type: (Waiter) -> int
|
1102 | """Wait for this process to finish."""
|
1103 | while self.state == job_state_e.Running:
|
1104 | # Only return if there's nothing to wait for. Keep waiting if we were
|
1105 | # interrupted with a signal.
|
1106 | if waiter.WaitForOne() == W1_ECHILD:
|
1107 | break
|
1108 |
|
1109 | assert self.status >= 0, self.status
|
1110 | return self.status
|
1111 |
|
1112 | def JobWait(self, waiter):
|
1113 | # type: (Waiter) -> wait_status_t
|
1114 | # wait builtin can be interrupted
|
1115 | while self.state == job_state_e.Running:
|
1116 | result = waiter.WaitForOne()
|
1117 |
|
1118 | if result >= 0: # signal
|
1119 | return wait_status.Cancelled(result)
|
1120 |
|
1121 | if result == W1_ECHILD:
|
1122 | break
|
1123 |
|
1124 | return wait_status.Proc(self.status)
|
1125 |
|
1126 | def WhenStopped(self, stop_sig):
|
1127 | # type: (int) -> None
|
1128 |
|
1129 | # 128 is a shell thing
|
1130 | # https://www.gnu.org/software/bash/manual/html_node/Exit-Status.html
|
1131 | self.status = 128 + stop_sig
|
1132 | self.state = job_state_e.Stopped
|
1133 |
|
1134 | if self.job_id == -1:
|
1135 | # This process was started in the foreground
|
1136 | self.job_list.AddJob(self)
|
1137 |
|
1138 | if not self.in_background:
|
1139 | self.job_control.MaybeTakeTerminal()
|
1140 | self.SetBackground()
|
1141 |
|
1142 | def WhenDone(self, pid, status):
|
1143 | # type: (int, int) -> None
|
1144 | """Called by the Waiter when this Process finishes."""
|
1145 |
|
1146 | #log('WhenDone %d %d', pid, status)
|
1147 | assert pid == self.pid, 'Expected %d, got %d' % (self.pid, pid)
|
1148 | self.status = status
|
1149 | self.state = job_state_e.Done
|
1150 | if self.parent_pipeline:
|
1151 | self.parent_pipeline.WhenDone(pid, status)
|
1152 | else:
|
1153 | if self.job_id != -1:
|
1154 | # Job might have been brought to the foreground after being
|
1155 | # assigned a job ID.
|
1156 | if self.in_background:
|
1157 | print_stderr('[%d] Done PID %d' % (self.job_id, self.pid))
|
1158 |
|
1159 | self.job_list.RemoveJob(self.job_id)
|
1160 |
|
1161 | self.job_list.RemoveChildProcess(self.pid)
|
1162 |
|
1163 | if not self.in_background:
|
1164 | self.job_control.MaybeTakeTerminal()
|
1165 |
|
1166 | def RunProcess(self, waiter, why):
|
1167 | # type: (Waiter, trace_t) -> int
|
1168 | """Run this process synchronously."""
|
1169 | self.StartProcess(why)
|
1170 | # ShellExecutor might be calling this for the last part of a pipeline.
|
1171 | if self.parent_pipeline is None:
|
1172 | # QUESTION: Can the PGID of a single process just be the PID? i.e. avoid
|
1173 | # calling getpgid()?
|
1174 | self.job_control.MaybeGiveTerminal(posix.getpgid(self.pid))
|
1175 | return self.Wait(waiter)
|
1176 |
|
1177 |
|
1178 | class ctx_Pipe(object):
|
1179 |
|
1180 | def __init__(self, fd_state, fd, err_out):
|
1181 | # type: (FdState, int, List[error.IOError_OSError]) -> None
|
1182 | fd_state.PushStdinFromPipe(fd)
|
1183 | self.fd_state = fd_state
|
1184 | self.err_out = err_out
|
1185 |
|
1186 | def __enter__(self):
|
1187 | # type: () -> None
|
1188 | pass
|
1189 |
|
1190 | def __exit__(self, type, value, traceback):
|
1191 | # type: (Any, Any, Any) -> None
|
1192 | self.fd_state.Pop(self.err_out)
|
1193 |
|
1194 |
|
1195 | class Pipeline(Job):
|
1196 | """A pipeline of processes to run.
|
1197 |
|
1198 | Cases we handle:
|
1199 |
|
1200 | foo | bar
|
1201 | $(foo | bar)
|
1202 | foo | bar | read v
|
1203 | """
|
1204 |
|
1205 | def __init__(self, sigpipe_status_ok, job_control, job_list, tracer):
|
1206 | # type: (bool, JobControl, JobList, dev.Tracer) -> None
|
1207 | Job.__init__(self)
|
1208 | self.job_control = job_control
|
1209 | self.job_list = job_list
|
1210 | self.tracer = tracer
|
1211 |
|
1212 | self.procs = [] # type: List[Process]
|
1213 | self.pids = [] # type: List[int] # pids in order
|
1214 | self.pipe_status = [] # type: List[int] # status in order
|
1215 | self.status = -1 # for 'wait' jobs
|
1216 |
|
1217 | self.pgid = INVALID_PGID
|
1218 |
|
1219 | # Optional for foreground
|
1220 | self.last_thunk = None # type: Tuple[CommandEvaluator, command_t]
|
1221 | self.last_pipe = None # type: Tuple[int, int]
|
1222 |
|
1223 | self.sigpipe_status_ok = sigpipe_status_ok
|
1224 |
|
1225 | def ProcessGroupId(self):
|
1226 | # type: () -> int
|
1227 | """Returns the group ID of this pipeline."""
|
1228 | return self.pgid
|
1229 |
|
1230 | def DisplayJob(self, job_id, f, style):
|
1231 | # type: (int, mylib.Writer, int) -> None
|
1232 | if style == STYLE_PID_ONLY:
|
1233 | f.write('%d\n' % self.procs[0].pid)
|
1234 | else:
|
1235 | # Note: this is STYLE_LONG.
|
1236 | for i, proc in enumerate(self.procs):
|
1237 | if i == 0: # show job ID for first element in pipeline
|
1238 | job_id_str = '%%%d' % job_id
|
1239 | else:
|
1240 | job_id_str = ' ' # 2 spaces
|
1241 |
|
1242 | f.write('%s %d %7s ' %
|
1243 | (job_id_str, proc.pid, _JobStateStr(proc.state)))
|
1244 | f.write(proc.thunk.UserString())
|
1245 | f.write('\n')
|
1246 |
|
1247 | def DebugPrint(self):
|
1248 | # type: () -> None
|
1249 | print('Pipeline in state %s' % _JobStateStr(self.state))
|
1250 | if mylib.PYTHON: # %s for Process not allowed in C++
|
1251 | for proc in self.procs:
|
1252 | print(' proc %s' % proc)
|
1253 | _, last_node = self.last_thunk
|
1254 | print(' last %s' % last_node)
|
1255 | print(' pipe_status %s' % self.pipe_status)
|
1256 |
|
1257 | def Add(self, p):
|
1258 | # type: (Process) -> None
|
1259 | """Append a process to the pipeline."""
|
1260 | if len(self.procs) == 0:
|
1261 | self.procs.append(p)
|
1262 | return
|
1263 |
|
1264 | r, w = posix.pipe()
|
1265 | #log('pipe for %s: %d %d', p, r, w)
|
1266 | prev = self.procs[-1]
|
1267 |
|
1268 | prev.AddStateChange(StdoutToPipe(r, w)) # applied on StartPipeline()
|
1269 | p.AddStateChange(StdinFromPipe(r, w)) # applied on StartPipeline()
|
1270 |
|
1271 | p.AddPipeToClose(r, w) # MaybeClosePipe() on StartPipeline()
|
1272 |
|
1273 | self.procs.append(p)
|
1274 |
|
1275 | def AddLast(self, thunk):
|
1276 | # type: (Tuple[CommandEvaluator, command_t]) -> None
|
1277 | """Append the last noden to the pipeline.
|
1278 |
|
1279 | This is run in the CURRENT process. It is OPTIONAL, because
|
1280 | pipelines in the background are run uniformly.
|
1281 | """
|
1282 | self.last_thunk = thunk
|
1283 |
|
1284 | assert len(self.procs) != 0
|
1285 |
|
1286 | r, w = posix.pipe()
|
1287 | prev = self.procs[-1]
|
1288 | prev.AddStateChange(StdoutToPipe(r, w))
|
1289 |
|
1290 | self.last_pipe = (r, w) # So we can connect it to last_thunk
|
1291 |
|
1292 | def StartPipeline(self, waiter):
|
1293 | # type: (Waiter) -> None
|
1294 |
|
1295 | # If we are creating a pipeline in a subshell or we aren't running with job
|
1296 | # control, our children should remain in our inherited process group.
|
1297 | # the pipelines's group ID.
|
1298 | if self.job_control.Enabled():
|
1299 | self.pgid = OWN_LEADER # first process in pipeline is the leader
|
1300 |
|
1301 | for i, proc in enumerate(self.procs):
|
1302 | if self.pgid != INVALID_PGID:
|
1303 | proc.AddStateChange(SetPgid(self.pgid, self.tracer))
|
1304 |
|
1305 | # Figure out the pid
|
1306 | pid = proc.StartProcess(trace.PipelinePart)
|
1307 | if i == 0 and self.pgid != INVALID_PGID:
|
1308 | # Mimic bash and use the PID of the FIRST process as the group for the
|
1309 | # whole pipeline.
|
1310 | self.pgid = pid
|
1311 |
|
1312 | self.pids.append(pid)
|
1313 | self.pipe_status.append(-1) # uninitialized
|
1314 |
|
1315 | # NOTE: This is done in the SHELL PROCESS after every fork() call.
|
1316 | # It can't be done at the end; otherwise processes will have descriptors
|
1317 | # from non-adjacent pipes.
|
1318 | proc.MaybeClosePipe()
|
1319 |
|
1320 | if self.last_thunk:
|
1321 | self.pipe_status.append(-1) # for self.last_thunk
|
1322 |
|
1323 | def LastPid(self):
|
1324 | # type: () -> int
|
1325 | """For the odd $! variable.
|
1326 |
|
1327 | It would be better if job IDs or PGIDs were used consistently.
|
1328 | """
|
1329 | return self.pids[-1]
|
1330 |
|
1331 | def Wait(self, waiter):
|
1332 | # type: (Waiter) -> List[int]
|
1333 | """Wait for this pipeline to finish."""
|
1334 |
|
1335 | assert self.procs, "no procs for Wait()"
|
1336 | # waitpid(-1) zero or more times
|
1337 | while self.state == job_state_e.Running:
|
1338 | # Keep waiting until there's nothing to wait for.
|
1339 | if waiter.WaitForOne() == W1_ECHILD:
|
1340 | break
|
1341 |
|
1342 | return self.pipe_status
|
1343 |
|
1344 | def JobWait(self, waiter):
|
1345 | # type: (Waiter) -> wait_status_t
|
1346 | """Called by 'wait' builtin, e.g. 'wait %1'."""
|
1347 | # wait builtin can be interrupted
|
1348 | assert self.procs, "no procs for Wait()"
|
1349 | while self.state == job_state_e.Running:
|
1350 | result = waiter.WaitForOne()
|
1351 |
|
1352 | if result >= 0: # signal
|
1353 | return wait_status.Cancelled(result)
|
1354 |
|
1355 | if result == W1_ECHILD:
|
1356 | break
|
1357 |
|
1358 | return wait_status.Pipeline(self.pipe_status)
|
1359 |
|
1360 | def RunLastPart(self, waiter, fd_state):
|
1361 | # type: (Waiter, FdState) -> List[int]
|
1362 | """Run this pipeline synchronously (foreground pipeline).
|
1363 |
|
1364 | Returns:
|
1365 | pipe_status (list of integers).
|
1366 | """
|
1367 | assert len(self.pids) == len(self.procs)
|
1368 |
|
1369 | # TODO: break circular dep. Bit flags could go in ASDL or headers.
|
1370 | from osh import cmd_eval
|
1371 |
|
1372 | # This is tcsetpgrp()
|
1373 | # TODO: fix race condition -- I believe the first process could have
|
1374 | # stopped already, and thus getpgid() will fail
|
1375 | self.job_control.MaybeGiveTerminal(self.pgid)
|
1376 |
|
1377 | # Run the last part of the pipeline IN PARALLEL with other processes. It
|
1378 | # may or may not fork:
|
1379 | # echo foo | read line # no fork, the builtin runs in THIS shell process
|
1380 | # ls | wc -l # fork for 'wc'
|
1381 |
|
1382 | cmd_ev, last_node = self.last_thunk
|
1383 |
|
1384 | assert self.last_pipe is not None
|
1385 | r, w = self.last_pipe # set in AddLast()
|
1386 | posix.close(w) # we will not write here
|
1387 |
|
1388 | # Fix lastpipe / job control / DEBUG trap interaction
|
1389 | cmd_flags = cmd_eval.NoDebugTrap if self.job_control.Enabled() else 0
|
1390 |
|
1391 | # The ERR trap only runs for the WHOLE pipeline, not the COMPONENTS in
|
1392 | # a pipeline.
|
1393 | cmd_flags |= cmd_eval.NoErrTrap
|
1394 |
|
1395 | io_errors = [] # type: List[error.IOError_OSError]
|
1396 | with ctx_Pipe(fd_state, r, io_errors):
|
1397 | cmd_ev.ExecuteAndCatch(last_node, cmd_flags)
|
1398 |
|
1399 | if len(io_errors):
|
1400 | e_die('Error setting up last part of pipeline: %s' %
|
1401 | pyutil.strerror(io_errors[0]))
|
1402 |
|
1403 | # We won't read anymore. If we don't do this, then 'cat' in 'cat
|
1404 | # /dev/urandom | sleep 1' will never get SIGPIPE.
|
1405 | posix.close(r)
|
1406 |
|
1407 | self.pipe_status[-1] = cmd_ev.LastStatus()
|
1408 | if self.AllDone():
|
1409 | self.state = job_state_e.Done
|
1410 |
|
1411 | #log('pipestatus before all have finished = %s', self.pipe_status)
|
1412 | return self.Wait(waiter)
|
1413 |
|
1414 | def AllDone(self):
|
1415 | # type: () -> bool
|
1416 |
|
1417 | # mycpp rewrite: all(status != -1 for status in self.pipe_status)
|
1418 | for status in self.pipe_status:
|
1419 | if status == -1:
|
1420 | return False
|
1421 | return True
|
1422 |
|
1423 | def WhenDone(self, pid, status):
|
1424 | # type: (int, int) -> None
|
1425 | """Called by Process.WhenDone."""
|
1426 | #log('Pipeline WhenDone %d %d', pid, status)
|
1427 | i = self.pids.index(pid)
|
1428 | assert i != -1, 'Unexpected PID %d' % pid
|
1429 |
|
1430 | if status == 141 and self.sigpipe_status_ok:
|
1431 | status = 0
|
1432 |
|
1433 | self.job_list.RemoveChildProcess(pid)
|
1434 | self.pipe_status[i] = status
|
1435 | if self.AllDone():
|
1436 | if self.job_id != -1:
|
1437 | # Job might have been brought to the foreground after being
|
1438 | # assigned a job ID.
|
1439 | if self.in_background:
|
1440 | print_stderr('[%d] Done PGID %d' %
|
1441 | (self.job_id, self.pids[0]))
|
1442 |
|
1443 | self.job_list.RemoveJob(self.job_id)
|
1444 |
|
1445 | # status of pipeline is status of last process
|
1446 | self.status = self.pipe_status[-1]
|
1447 | self.state = job_state_e.Done
|
1448 | if not self.in_background:
|
1449 | self.job_control.MaybeTakeTerminal()
|
1450 |
|
1451 |
|
1452 | def _JobStateStr(i):
|
1453 | # type: (job_state_t) -> str
|
1454 | return job_state_str(i)[10:] # remove 'job_state.'
|
1455 |
|
1456 |
|
1457 | def _GetTtyFd():
|
1458 | # type: () -> int
|
1459 | """Returns -1 if stdio is not a TTY."""
|
1460 | try:
|
1461 | return posix.open("/dev/tty", O_NONBLOCK | O_NOCTTY | O_RDWR, 0o666)
|
1462 | except (IOError, OSError) as e:
|
1463 | return -1
|
1464 |
|
1465 |
|
1466 | class ctx_TerminalControl(object):
|
1467 |
|
1468 | def __init__(self, job_control, errfmt):
|
1469 | # type: (JobControl, ui.ErrorFormatter) -> None
|
1470 | job_control.InitJobControl()
|
1471 | self.job_control = job_control
|
1472 | self.errfmt = errfmt
|
1473 |
|
1474 | def __enter__(self):
|
1475 | # type: () -> None
|
1476 | pass
|
1477 |
|
1478 | def __exit__(self, type, value, traceback):
|
1479 | # type: (Any, Any, Any) -> None
|
1480 |
|
1481 | # Return the TTY to the original owner before exiting.
|
1482 | try:
|
1483 | self.job_control.MaybeReturnTerminal()
|
1484 | except error.FatalRuntime as e:
|
1485 | # Don't abort the shell on error, just print a message.
|
1486 | self.errfmt.PrettyPrintError(e)
|
1487 |
|
1488 |
|
1489 | class JobControl(object):
|
1490 | """Interface to setpgid(), tcsetpgrp(), etc."""
|
1491 |
|
1492 | def __init__(self):
|
1493 | # type: () -> None
|
1494 |
|
1495 | # The main shell's PID and group ID.
|
1496 | self.shell_pid = -1
|
1497 | self.shell_pgid = -1
|
1498 |
|
1499 | # The fd of the controlling tty. Set to -1 when job control is disabled.
|
1500 | self.shell_tty_fd = -1
|
1501 |
|
1502 | # For giving the terminal back to our parent before exiting (if not a login
|
1503 | # shell).
|
1504 | self.original_tty_pgid = -1
|
1505 |
|
1506 | def InitJobControl(self):
|
1507 | # type: () -> None
|
1508 | self.shell_pid = posix.getpid()
|
1509 | orig_shell_pgid = posix.getpgid(0)
|
1510 | self.shell_pgid = orig_shell_pgid
|
1511 | self.shell_tty_fd = _GetTtyFd()
|
1512 |
|
1513 | # If we aren't the leader of our process group, create a group and mark
|
1514 | # ourselves as the leader.
|
1515 | if self.shell_pgid != self.shell_pid:
|
1516 | try:
|
1517 | posix.setpgid(self.shell_pid, self.shell_pid)
|
1518 | self.shell_pgid = self.shell_pid
|
1519 | except (IOError, OSError) as e:
|
1520 | self.shell_tty_fd = -1
|
1521 |
|
1522 | if self.shell_tty_fd != -1:
|
1523 | self.original_tty_pgid = posix.tcgetpgrp(self.shell_tty_fd)
|
1524 |
|
1525 | # If stdio is a TTY, put the shell's process group in the foreground.
|
1526 | try:
|
1527 | posix.tcsetpgrp(self.shell_tty_fd, self.shell_pgid)
|
1528 | except (IOError, OSError) as e:
|
1529 | # We probably aren't in the session leader's process group. Disable job
|
1530 | # control.
|
1531 | self.shell_tty_fd = -1
|
1532 | self.shell_pgid = orig_shell_pgid
|
1533 | posix.setpgid(self.shell_pid, self.shell_pgid)
|
1534 |
|
1535 | def Enabled(self):
|
1536 | # type: () -> bool
|
1537 |
|
1538 | # TODO: get rid of this syscall? SubProgramThunk should set a flag I
|
1539 | # think.
|
1540 | curr_pid = posix.getpid()
|
1541 | # Only the main shell should bother with job control functions.
|
1542 | return curr_pid == self.shell_pid and self.shell_tty_fd != -1
|
1543 |
|
1544 | # TODO: This isn't a PID. This is a process group ID?
|
1545 | #
|
1546 | # What should the table look like?
|
1547 | #
|
1548 | # Do we need the last PID? I don't know why bash prints that. Probably so
|
1549 | # you can do wait $!
|
1550 | # wait -n waits for any node to go from job_state_e.Running to job_state_e.Done?
|
1551 | #
|
1552 | # And it needs a flag for CURRENT, for the implicit arg to 'fg'.
|
1553 | # job_id is just an integer. This is sort of lame.
|
1554 | #
|
1555 | # [job_id, flag, pgid, job_state, node]
|
1556 |
|
1557 | def MaybeGiveTerminal(self, pgid):
|
1558 | # type: (int) -> None
|
1559 | """If stdio is a TTY, move the given process group to the
|
1560 | foreground."""
|
1561 | if not self.Enabled():
|
1562 | # Only call tcsetpgrp when job control is enabled.
|
1563 | return
|
1564 |
|
1565 | try:
|
1566 | posix.tcsetpgrp(self.shell_tty_fd, pgid)
|
1567 | except (IOError, OSError) as e:
|
1568 | e_die('osh: Failed to move process group %d to foreground: %s' %
|
1569 | (pgid, pyutil.strerror(e)))
|
1570 |
|
1571 | def MaybeTakeTerminal(self):
|
1572 | # type: () -> None
|
1573 | """If stdio is a TTY, return the main shell's process group to the
|
1574 | foreground."""
|
1575 | self.MaybeGiveTerminal(self.shell_pgid)
|
1576 |
|
1577 | def MaybeReturnTerminal(self):
|
1578 | # type: () -> None
|
1579 | """Called before the shell exits."""
|
1580 | self.MaybeGiveTerminal(self.original_tty_pgid)
|
1581 |
|
1582 |
|
1583 | class JobList(object):
|
1584 | """Global list of jobs, used by a few builtins."""
|
1585 |
|
1586 | def __init__(self):
|
1587 | # type: () -> None
|
1588 |
|
1589 | # job_id -> Job instance
|
1590 | self.jobs = {} # type: Dict[int, Job]
|
1591 |
|
1592 | # pid -> Process. This is for STOP notification.
|
1593 | self.child_procs = {} # type: Dict[int, Process]
|
1594 | self.debug_pipelines = [] # type: List[Pipeline]
|
1595 |
|
1596 | # Counter used to assign IDs to jobs. It is incremented every time a job
|
1597 | # is created. Once all active jobs are done it is reset to 1. I'm not
|
1598 | # sure if this reset behavior is mandated by POSIX, but other shells do
|
1599 | # it, so we mimic for the sake of compatibility.
|
1600 | self.job_id = 1
|
1601 |
|
1602 | def AddJob(self, job):
|
1603 | # type: (Job) -> int
|
1604 | """Add a background job to the list.
|
1605 |
|
1606 | A job is either a Process or Pipeline. You can resume a job with 'fg',
|
1607 | kill it with 'kill', etc.
|
1608 |
|
1609 | Two cases:
|
1610 |
|
1611 | 1. async jobs: sleep 5 | sleep 4 &
|
1612 | 2. stopped jobs: sleep 5; then Ctrl-Z
|
1613 | """
|
1614 | job_id = self.job_id
|
1615 | self.jobs[job_id] = job
|
1616 | job.job_id = job_id
|
1617 | self.job_id += 1
|
1618 | return job_id
|
1619 |
|
1620 | def RemoveJob(self, job_id):
|
1621 | # type: (int) -> None
|
1622 | """Process and Pipeline can call this."""
|
1623 | mylib.dict_erase(self.jobs, job_id)
|
1624 |
|
1625 | if len(self.jobs) == 0:
|
1626 | self.job_id = 1
|
1627 |
|
1628 | def AddChildProcess(self, pid, proc):
|
1629 | # type: (int, Process) -> None
|
1630 | """Every child process should be added here as soon as we know its PID.
|
1631 |
|
1632 | When the Waiter gets an EXITED or STOPPED notification, we need
|
1633 | to know about it so 'jobs' can work.
|
1634 | """
|
1635 | self.child_procs[pid] = proc
|
1636 |
|
1637 | def RemoveChildProcess(self, pid):
|
1638 | # type: (int) -> None
|
1639 | """Remove the child process with the given PID."""
|
1640 | mylib.dict_erase(self.child_procs, pid)
|
1641 |
|
1642 | if mylib.PYTHON:
|
1643 |
|
1644 | def AddPipeline(self, pi):
|
1645 | # type: (Pipeline) -> None
|
1646 | """For debugging only."""
|
1647 | self.debug_pipelines.append(pi)
|
1648 |
|
1649 | def ProcessFromPid(self, pid):
|
1650 | # type: (int) -> Process
|
1651 | """For wait $PID.
|
1652 |
|
1653 | There's no way to wait for a pipeline with a PID. That uses job
|
1654 | syntax, e.g. %1. Not a great interface.
|
1655 | """
|
1656 | return self.child_procs.get(pid)
|
1657 |
|
1658 | def GetCurrentAndPreviousJobs(self):
|
1659 | # type: () -> Tuple[Optional[Job], Optional[Job]]
|
1660 | """Return the "current" and "previous" jobs (AKA `%+` and `%-`).
|
1661 |
|
1662 | See the POSIX specification for the `jobs` builtin for details:
|
1663 | https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
|
1664 |
|
1665 | IMPORTANT NOTE: This method assumes that the jobs list will not change
|
1666 | during its execution! This assumption holds for now because we only ever
|
1667 | update the jobs list from the main loop after WaitPid() informs us of a
|
1668 | change. If we implement `set -b` and install a signal handler for
|
1669 | SIGCHLD we should be careful to synchronize it with this function. The
|
1670 | unsafety of mutating GC data structures from a signal handler should
|
1671 | make this a non-issue, but if bugs related to this appear this note may
|
1672 | be helpful...
|
1673 | """
|
1674 | # Split all active jobs by state and sort each group by decreasing job
|
1675 | # ID to approximate newness.
|
1676 | stopped_jobs = [] # type: List[Job]
|
1677 | running_jobs = [] # type: List[Job]
|
1678 | for i in xrange(0, self.job_id):
|
1679 | job = self.jobs.get(i, None)
|
1680 | if not job:
|
1681 | continue
|
1682 |
|
1683 | if job.state == job_state_e.Stopped:
|
1684 | stopped_jobs.append(job)
|
1685 |
|
1686 | elif job.state == job_state_e.Running:
|
1687 | running_jobs.append(job)
|
1688 |
|
1689 | current = None # type: Optional[Job]
|
1690 | previous = None # type: Optional[Job]
|
1691 | # POSIX says: If there is any suspended job, then the current job shall
|
1692 | # be a suspended job. If there are at least two suspended jobs, then the
|
1693 | # previous job also shall be a suspended job.
|
1694 | #
|
1695 | # So, we will only return running jobs from here if there are no recent
|
1696 | # stopped jobs.
|
1697 | if len(stopped_jobs) > 0:
|
1698 | current = stopped_jobs.pop()
|
1699 |
|
1700 | if len(stopped_jobs) > 0:
|
1701 | previous = stopped_jobs.pop()
|
1702 |
|
1703 | if len(running_jobs) > 0 and not current:
|
1704 | current = running_jobs.pop()
|
1705 |
|
1706 | if len(running_jobs) > 0 and not previous:
|
1707 | previous = running_jobs.pop()
|
1708 |
|
1709 | if not previous:
|
1710 | previous = current
|
1711 |
|
1712 | return current, previous
|
1713 |
|
1714 | def GetJobWithSpec(self, job_spec):
|
1715 | # type: (str) -> Optional[Job]
|
1716 | """Parse the given job spec and return the matching job. If there is no
|
1717 | matching job, this function returns None.
|
1718 |
|
1719 | See the POSIX spec for the `jobs` builtin for details about job specs:
|
1720 | https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
|
1721 | """
|
1722 | if job_spec in CURRENT_JOB_SPECS:
|
1723 | current, _ = self.GetCurrentAndPreviousJobs()
|
1724 | return current
|
1725 |
|
1726 | if job_spec == '%-':
|
1727 | _, previous = self.GetCurrentAndPreviousJobs()
|
1728 | return previous
|
1729 |
|
1730 | # TODO: Add support for job specs based on prefixes of process argv.
|
1731 | m = util.RegexSearch(r'^%([0-9]+)$', job_spec)
|
1732 | if m is not None:
|
1733 | assert len(m) == 2
|
1734 | job_id = int(m[1])
|
1735 | if job_id in self.jobs:
|
1736 | return self.jobs[job_id]
|
1737 |
|
1738 | return None
|
1739 |
|
1740 | def DisplayJobs(self, style):
|
1741 | # type: (int) -> None
|
1742 | """Used by the 'jobs' builtin.
|
1743 |
|
1744 | https://pubs.opengroup.org/onlinepubs/9699919799/utilities/jobs.html
|
1745 |
|
1746 | "By default, the jobs utility shall display the status of all stopped jobs,
|
1747 | running background jobs and all jobs whose status has changed and have not
|
1748 | been reported by the shell."
|
1749 | """
|
1750 | # NOTE: A job is a background process or pipeline.
|
1751 | #
|
1752 | # echo hi | wc -l -- this starts two processes. Wait for TWO
|
1753 | # echo hi | wc -l & -- this starts a process which starts two processes
|
1754 | # Wait for ONE.
|
1755 | #
|
1756 | # 'jobs -l' GROUPS the PIDs by job. It has the job number, + - indicators
|
1757 | # for %% and %-, PID, status, and "command".
|
1758 | #
|
1759 | # Every component of a pipeline is on the same line with 'jobs', but
|
1760 | # they're separated into different lines with 'jobs -l'.
|
1761 | #
|
1762 | # See demo/jobs-builtin.sh
|
1763 |
|
1764 | # $ jobs -l
|
1765 | # [1]+ 24414 Stopped sleep 5
|
1766 | # 24415 | sleep 5
|
1767 | # [2] 24502 Running sleep 6
|
1768 | # 24503 | sleep 6
|
1769 | # 24504 | sleep 5 &
|
1770 | # [3]- 24508 Running sleep 6
|
1771 | # 24509 | sleep 6
|
1772 | # 24510 | sleep 5 &
|
1773 |
|
1774 | f = mylib.Stdout()
|
1775 | for job_id, job in iteritems(self.jobs):
|
1776 | # Use the %1 syntax
|
1777 | job.DisplayJob(job_id, f, style)
|
1778 |
|
1779 | def DebugPrint(self):
|
1780 | # type: () -> None
|
1781 |
|
1782 | f = mylib.Stdout()
|
1783 | f.write('\n')
|
1784 | f.write('[process debug info]\n')
|
1785 |
|
1786 | for pid, proc in iteritems(self.child_procs):
|
1787 | proc.DisplayJob(-1, f, STYLE_DEFAULT)
|
1788 | #p = ' |' if proc.parent_pipeline else ''
|
1789 | #print('%d %7s %s%s' % (pid, _JobStateStr(proc.state), proc.thunk.UserString(), p))
|
1790 |
|
1791 | if len(self.debug_pipelines):
|
1792 | f.write('\n')
|
1793 | f.write('[pipeline debug info]\n')
|
1794 | for pi in self.debug_pipelines:
|
1795 | pi.DebugPrint()
|
1796 |
|
1797 | def ListRecent(self):
|
1798 | # type: () -> None
|
1799 | """For jobs -n, which I think is also used in the interactive
|
1800 | prompt."""
|
1801 | pass
|
1802 |
|
1803 | def NumRunning(self):
|
1804 | # type: () -> int
|
1805 | """Return the number of running jobs.
|
1806 |
|
1807 | Used by 'wait' and 'wait -n'.
|
1808 | """
|
1809 | count = 0
|
1810 | for _, job in iteritems(self.jobs): # mycpp rewrite: from itervalues()
|
1811 | if job.State() == job_state_e.Running:
|
1812 | count += 1
|
1813 | return count
|
1814 |
|
1815 |
|
1816 | # Some WaitForOne() return values
|
1817 | W1_OK = -2 # waitpid(-1) returned
|
1818 | W1_ECHILD = -3 # no processes to wait for
|
1819 | W1_AGAIN = -4 # WNOHANG was passed and there were no state changes
|
1820 |
|
1821 |
|
1822 | class Waiter(object):
|
1823 | """A capability to wait for processes.
|
1824 |
|
1825 | This must be a singleton (and is because CommandEvaluator is a singleton).
|
1826 |
|
1827 | Invariants:
|
1828 | - Every child process is registered once
|
1829 | - Every child process is waited for
|
1830 |
|
1831 | Canonical example of why we need a GLOBAL waiter:
|
1832 |
|
1833 | { sleep 3; echo 'done 3'; } &
|
1834 | { sleep 4; echo 'done 4'; } &
|
1835 |
|
1836 | # ... do arbitrary stuff ...
|
1837 |
|
1838 | { sleep 1; exit 1; } | { sleep 2; exit 2; }
|
1839 |
|
1840 | Now when you do wait() after starting the pipeline, you might get a pipeline
|
1841 | process OR a background process! So you have to distinguish between them.
|
1842 | """
|
1843 |
|
1844 | def __init__(self, job_list, exec_opts, signal_safe, tracer):
|
1845 | # type: (JobList, optview.Exec, pyos.SignalSafe, dev.Tracer) -> None
|
1846 | self.job_list = job_list
|
1847 | self.exec_opts = exec_opts
|
1848 | self.signal_safe = signal_safe
|
1849 | self.tracer = tracer
|
1850 | self.last_status = 127 # wait -n error code
|
1851 |
|
1852 | def WaitForOne(self, waitpid_options=0):
|
1853 | # type: (int) -> int
|
1854 | """Wait until the next process returns (or maybe Ctrl-C).
|
1855 |
|
1856 | Returns:
|
1857 | One of these negative numbers:
|
1858 | W1_ECHILD Nothing to wait for
|
1859 | W1_OK Caller should keep waiting
|
1860 | UNTRAPPED_SIGWINCH
|
1861 | Or
|
1862 | result > 0 Signal that waitpid() was interrupted with
|
1863 |
|
1864 | In the interactive shell, we return 0 if we get a Ctrl-C, so the caller
|
1865 | will try again.
|
1866 |
|
1867 | Callers:
|
1868 | wait -n -- loop until there is one fewer process (TODO)
|
1869 | wait -- loop until there are no processes
|
1870 | wait $! -- loop until job state is Done (process or pipeline)
|
1871 | Process::Wait() -- loop until Process state is done
|
1872 | Pipeline::Wait() -- loop until Pipeline state is done
|
1873 |
|
1874 | Comparisons:
|
1875 | bash: jobs.c waitchld() Has a special case macro(!) CHECK_WAIT_INTR for
|
1876 | the wait builtin
|
1877 |
|
1878 | dash: jobs.c waitproc() uses sigfillset(), sigprocmask(), etc. Runs in a
|
1879 | loop while (gotsigchld), but that might be a hack for System V!
|
1880 |
|
1881 | Should we have a cleaner API like named posix::wait_for_one() ?
|
1882 |
|
1883 | wait_result =
|
1884 | ECHILD -- nothing to wait for
|
1885 | | Done(int pid, int status) -- process done
|
1886 | | EINTR(bool sigint) -- may or may not retry
|
1887 | """
|
1888 | pid, status = pyos.WaitPid(waitpid_options)
|
1889 | if pid == 0: # WNOHANG passed, and no state changes
|
1890 | return W1_AGAIN
|
1891 | elif pid < 0: # error case
|
1892 | err_num = status
|
1893 | #log('waitpid() error => %d %s', e.errno, pyutil.strerror(e))
|
1894 | if err_num == ECHILD:
|
1895 | return W1_ECHILD # nothing to wait for caller should stop
|
1896 | elif err_num == EINTR: # Bug #858 fix
|
1897 | #log('WaitForOne() => %d', self.trap_state.GetLastSignal())
|
1898 | return self.signal_safe.LastSignal() # e.g. 1 for SIGHUP
|
1899 | else:
|
1900 | # The signature of waitpid() means this shouldn't happen
|
1901 | raise AssertionError()
|
1902 |
|
1903 | # All child processes are supposed to be in this dict. But this may
|
1904 | # legitimately happen if a grandchild outlives the child (its parent).
|
1905 | # Then it is reparented under this process, so we might receive
|
1906 | # notification of its exit, even though we didn't start it. We can't have
|
1907 | # any knowledge of such processes, so print a warning.
|
1908 | if pid not in self.job_list.child_procs:
|
1909 | print_stderr("osh: PID %d stopped, but osh didn't start it" % pid)
|
1910 | return W1_OK
|
1911 |
|
1912 | proc = self.job_list.child_procs[pid]
|
1913 | if 0:
|
1914 | self.job_list.DebugPrint()
|
1915 |
|
1916 | if WIFSIGNALED(status):
|
1917 | term_sig = WTERMSIG(status)
|
1918 | status = 128 + term_sig
|
1919 |
|
1920 | # Print newline after Ctrl-C.
|
1921 | if term_sig == SIGINT:
|
1922 | print('')
|
1923 |
|
1924 | proc.WhenDone(pid, status)
|
1925 |
|
1926 | elif WIFEXITED(status):
|
1927 | status = WEXITSTATUS(status)
|
1928 | #log('exit status: %s', status)
|
1929 | proc.WhenDone(pid, status)
|
1930 |
|
1931 | elif WIFSTOPPED(status):
|
1932 | #status = WEXITSTATUS(status)
|
1933 | stop_sig = WSTOPSIG(status)
|
1934 |
|
1935 | print_stderr('')
|
1936 | print_stderr('[PID %d] Stopped with signal %d' % (pid, stop_sig))
|
1937 | proc.WhenStopped(stop_sig)
|
1938 |
|
1939 | else:
|
1940 | raise AssertionError(status)
|
1941 |
|
1942 | self.last_status = status # for wait -n
|
1943 | self.tracer.OnProcessEnd(pid, status)
|
1944 | return W1_OK
|
1945 |
|
1946 | def PollNotifications(self):
|
1947 | # type: () -> None
|
1948 | """
|
1949 | Process all pending state changes.
|
1950 | """
|
1951 | while self.WaitForOne(waitpid_options=WNOHANG) == W1_OK:
|
1952 | continue
|