OILS / core / process.py View on Github | oilshell.org

1947 lines, 942 significant
1# Copyright 2016 Andy Chu. All rights reserved.
2# Licensed under the Apache License, Version 2.0 (the "License");
3# you may not use this file except in compliance with the License.
4# You may obtain a copy of the License at
5#
6# http://www.apache.org/licenses/LICENSE-2.0
7"""
8process.py - Launch processes and manipulate file descriptors.
9"""
10from __future__ import print_function
11
12from errno import EACCES, EBADF, ECHILD, EINTR, ENOENT, ENOEXEC
13import fcntl as fcntl_
14from fcntl import F_DUPFD, F_GETFD, F_SETFD, FD_CLOEXEC
15from signal import (SIG_DFL, SIG_IGN, SIGINT, SIGPIPE, SIGQUIT, SIGTSTP,
16 SIGTTOU, SIGTTIN, SIGWINCH)
17
18from _devbuild.gen.id_kind_asdl import Id
19from _devbuild.gen.runtime_asdl import (job_state_e, job_state_t,
20 job_state_str, wait_status,
21 wait_status_t, RedirValue,
22 redirect_arg, redirect_arg_e, trace,
23 trace_t)
24from _devbuild.gen.syntax_asdl import (
25 loc_t,
26 redir_loc,
27 redir_loc_e,
28 redir_loc_t,
29)
30from _devbuild.gen.value_asdl import (value, value_e)
31from core import dev
32from core import error
33from core.error import e_die
34from core import pyutil
35from core import pyos
36from core import state
37from core import ui
38from core import util
39from data_lang import j8_lite
40from frontend import location
41from frontend import match
42from mycpp import mylib
43from mycpp.mylib import log, print_stderr, probe, tagswitch, iteritems
44
45import posix_ as posix
46from posix_ import (
47 # translated by mycpp and directly called! No wrapper!
48 WIFSIGNALED,
49 WIFEXITED,
50 WIFSTOPPED,
51 WEXITSTATUS,
52 WSTOPSIG,
53 WTERMSIG,
54 WNOHANG,
55 O_APPEND,
56 O_CREAT,
57 O_NONBLOCK,
58 O_NOCTTY,
59 O_RDONLY,
60 O_RDWR,
61 O_WRONLY,
62 O_TRUNC,
63)
64
65from typing import IO, List, Tuple, Dict, Optional, Any, cast, TYPE_CHECKING
66
67if TYPE_CHECKING:
68 from _devbuild.gen.runtime_asdl import cmd_value
69 from _devbuild.gen.syntax_asdl import command_t
70 from builtin import trap_osh
71 from core import optview
72 from core.ui import ErrorFormatter
73 from core.util import _DebugFile
74 from osh.cmd_eval import CommandEvaluator
75
76NO_FD = -1
77
78# Minimum file descriptor that the shell can use. Other descriptors can be
79# directly used by user programs, e.g. exec 9>&1
80#
81# Oil uses 100 because users are allowed TWO digits in frontend/lexer_def.py.
82# This is a compromise between bash (unlimited, but requires crazy
83# bookkeeping), and dash/zsh (10) and mksh (24)
84_SHELL_MIN_FD = 100
85
86# Style for 'jobs' builtin
87STYLE_DEFAULT = 0
88STYLE_LONG = 1
89STYLE_PID_ONLY = 2
90
91# To save on allocations in JobList::GetJobWithSpec()
92CURRENT_JOB_SPECS = ['', '%', '%%', '%+']
93
94
95class ctx_FileCloser(object):
96
97 def __init__(self, f):
98 # type: (mylib.LineReader) -> None
99 self.f = f
100
101 def __enter__(self):
102 # type: () -> None
103 pass
104
105 def __exit__(self, type, value, traceback):
106 # type: (Any, Any, Any) -> None
107 self.f.close()
108
109
110def InitInteractiveShell():
111 # type: () -> None
112 """Called when initializing an interactive shell."""
113
114 # The shell itself should ignore Ctrl-\.
115 pyos.Sigaction(SIGQUIT, SIG_IGN)
116
117 # This prevents Ctrl-Z from suspending OSH in interactive mode.
118 pyos.Sigaction(SIGTSTP, SIG_IGN)
119
120 # More signals from
121 # https://www.gnu.org/software/libc/manual/html_node/Initializing-the-Shell.html
122 # (but not SIGCHLD)
123 pyos.Sigaction(SIGTTOU, SIG_IGN)
124 pyos.Sigaction(SIGTTIN, SIG_IGN)
125
126 # Register a callback to receive terminal width changes.
127 # NOTE: In line_input.c, we turned off rl_catch_sigwinch.
128
129 # This is ALWAYS on, which means that it can cause EINTR, and wait() and
130 # read() have to handle it
131 pyos.RegisterSignalInterest(SIGWINCH)
132
133
134def SaveFd(fd):
135 # type: (int) -> int
136 saved = fcntl_.fcntl(fd, F_DUPFD, _SHELL_MIN_FD) # type: int
137 return saved
138
139
140class _RedirFrame(object):
141
142 def __init__(self, saved_fd, orig_fd, forget):
143 # type: (int, int, bool) -> None
144 self.saved_fd = saved_fd
145 self.orig_fd = orig_fd
146 self.forget = forget
147
148
149class _FdFrame(object):
150
151 def __init__(self):
152 # type: () -> None
153 self.saved = [] # type: List[_RedirFrame]
154 self.need_wait = [] # type: List[Process]
155
156 def Forget(self):
157 # type: () -> None
158 """For exec 1>&2."""
159 for rf in reversed(self.saved):
160 if rf.saved_fd != NO_FD and rf.forget:
161 posix.close(rf.saved_fd)
162
163 del self.saved[:] # like list.clear() in Python 3.3
164 del self.need_wait[:]
165
166 def __repr__(self):
167 # type: () -> str
168 return '<_FdFrame %s>' % self.saved
169
170
171class FdState(object):
172 """File descriptor state for the current process.
173
174 For example, you can do 'myfunc > out.txt' without forking. Child
175 processes inherit our state.
176 """
177
178 def __init__(
179 self,
180 errfmt, # type: ui.ErrorFormatter
181 job_control, # type: JobControl
182 job_list, # type: JobList
183 mem, #type: state.Mem
184 tracer, # type: Optional[dev.Tracer]
185 waiter, # type: Optional[Waiter]
186 ):
187 # type: (...) -> None
188 """
189 Args:
190 errfmt: for errors
191 job_list: For keeping track of _HereDocWriterThunk
192 """
193 self.errfmt = errfmt
194 self.job_control = job_control
195 self.job_list = job_list
196 self.cur_frame = _FdFrame() # for the top level
197 self.stack = [self.cur_frame]
198 self.mem = mem
199 self.tracer = tracer
200 self.waiter = waiter
201
202 def Open(self, path):
203 # type: (str) -> mylib.LineReader
204 """Opens a path for read, but moves it out of the reserved 3-9 fd
205 range.
206
207 Returns:
208 A Python file object. The caller is responsible for Close().
209
210 Raises:
211 IOError or OSError if the path can't be found. (This is Python-induced wart)
212 """
213 fd_mode = O_RDONLY
214 f = self._Open(path, 'r', fd_mode)
215
216 # Hacky downcast
217 return cast('mylib.LineReader', f)
218
219 # used for util.DebugFile
220 def OpenForWrite(self, path):
221 # type: (str) -> mylib.Writer
222 fd_mode = O_CREAT | O_RDWR
223 f = self._Open(path, 'w', fd_mode)
224
225 # Hacky downcast
226 return cast('mylib.Writer', f)
227
228 def _Open(self, path, c_mode, fd_mode):
229 # type: (str, str, int) -> IO[str]
230 fd = posix.open(path, fd_mode, 0o666) # may raise OSError
231
232 # Immediately move it to a new location
233 new_fd = SaveFd(fd)
234 posix.close(fd)
235
236 # Return a Python file handle
237 f = posix.fdopen(new_fd, c_mode) # may raise IOError
238 return f
239
240 def _WriteFdToMem(self, fd_name, fd):
241 # type: (str, int) -> None
242 if self.mem:
243 # setvar, not setref
244 state.OshLanguageSetValue(self.mem, location.LName(fd_name),
245 value.Str(str(fd)))
246
247 def _ReadFdFromMem(self, fd_name):
248 # type: (str) -> int
249 val = self.mem.GetValue(fd_name)
250 if val.tag() == value_e.Str:
251 try:
252 return int(cast(value.Str, val).s)
253 except ValueError:
254 return NO_FD
255 return NO_FD
256
257 def _PushSave(self, fd):
258 # type: (int) -> bool
259 """Save fd to a new location and remember to restore it later."""
260 #log('---- _PushSave %s', fd)
261 ok = True
262 try:
263 new_fd = SaveFd(fd)
264 except (IOError, OSError) as e:
265 ok = False
266 # Example program that causes this error: exec 4>&1. Descriptor 4 isn't
267 # open.
268 # This seems to be ignored in dash too in savefd()?
269 if e.errno != EBADF:
270 raise
271 if ok:
272 posix.close(fd)
273 fcntl_.fcntl(new_fd, F_SETFD, FD_CLOEXEC)
274 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, True))
275 else:
276 # if we got EBADF, we still need to close the original on Pop()
277 self._PushClose(fd)
278
279 return ok
280
281 def _PushDup(self, fd1, blame_loc):
282 # type: (int, redir_loc_t) -> int
283 """Save fd2 in a higher range, and dup fd1 onto fd2.
284
285 Returns whether F_DUPFD/dup2 succeeded, and the new descriptor.
286 """
287 UP_loc = blame_loc
288 if blame_loc.tag() == redir_loc_e.VarName:
289 fd2_name = cast(redir_loc.VarName, UP_loc).name
290 try:
291 # F_DUPFD: GREATER than range
292 new_fd = fcntl_.fcntl(fd1, F_DUPFD, _SHELL_MIN_FD) # type: int
293 except (IOError, OSError) as e:
294 if e.errno == EBADF:
295 print_stderr('F_DUPFD fd %d: %s' %
296 (fd1, pyutil.strerror(e)))
297 return NO_FD
298 else:
299 raise # this redirect failed
300
301 self._WriteFdToMem(fd2_name, new_fd)
302
303 elif blame_loc.tag() == redir_loc_e.Fd:
304 fd2 = cast(redir_loc.Fd, UP_loc).fd
305
306 if fd1 == fd2:
307 # The user could have asked for it to be open on descriptor 3, but open()
308 # already returned 3, e.g. echo 3>out.txt
309 return NO_FD
310
311 # Check the validity of fd1 before _PushSave(fd2)
312 try:
313 fcntl_.fcntl(fd1, F_GETFD)
314 except (IOError, OSError) as e:
315 print_stderr('F_GETFD fd %d: %s' % (fd1, pyutil.strerror(e)))
316 raise
317
318 need_restore = self._PushSave(fd2)
319
320 #log('==== dup2 %s %s\n' % (fd1, fd2))
321 try:
322 posix.dup2(fd1, fd2)
323 except (IOError, OSError) as e:
324 # bash/dash give this error too, e.g. for 'echo hi 1>&3'
325 print_stderr('dup2(%d, %d): %s' %
326 (fd1, fd2, pyutil.strerror(e)))
327
328 # Restore and return error
329 if need_restore:
330 rf = self.cur_frame.saved.pop()
331 posix.dup2(rf.saved_fd, rf.orig_fd)
332 posix.close(rf.saved_fd)
333
334 raise # this redirect failed
335
336 new_fd = fd2
337
338 else:
339 raise AssertionError()
340
341 return new_fd
342
343 def _PushCloseFd(self, blame_loc):
344 # type: (redir_loc_t) -> bool
345 """For 2>&-"""
346 # exec {fd}>&- means close the named descriptor
347
348 UP_loc = blame_loc
349 if blame_loc.tag() == redir_loc_e.VarName:
350 fd_name = cast(redir_loc.VarName, UP_loc).name
351 fd = self._ReadFdFromMem(fd_name)
352 if fd == NO_FD:
353 return False
354
355 elif blame_loc.tag() == redir_loc_e.Fd:
356 fd = cast(redir_loc.Fd, UP_loc).fd
357
358 else:
359 raise AssertionError()
360
361 self._PushSave(fd)
362
363 return True
364
365 def _PushClose(self, fd):
366 # type: (int) -> None
367 self.cur_frame.saved.append(_RedirFrame(NO_FD, fd, False))
368
369 def _PushWait(self, proc):
370 # type: (Process) -> None
371 self.cur_frame.need_wait.append(proc)
372
373 def _ApplyRedirect(self, r):
374 # type: (RedirValue) -> None
375 arg = r.arg
376 UP_arg = arg
377 with tagswitch(arg) as case:
378
379 if case(redirect_arg_e.Path):
380 arg = cast(redirect_arg.Path, UP_arg)
381
382 if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &>
383 # NOTE: This is different than >| because it respects noclobber, but
384 # that option is almost never used. See test/wild.sh.
385 mode = O_CREAT | O_WRONLY | O_TRUNC
386 elif r.op_id == Id.Redir_Clobber: # >|
387 mode = O_CREAT | O_WRONLY | O_TRUNC
388 elif r.op_id in (Id.Redir_DGreat,
389 Id.Redir_AndDGreat): # >> &>>
390 mode = O_CREAT | O_WRONLY | O_APPEND
391 elif r.op_id == Id.Redir_Less: # <
392 mode = O_RDONLY
393 elif r.op_id == Id.Redir_LessGreat: # <>
394 mode = O_CREAT | O_RDWR
395 else:
396 raise NotImplementedError(r.op_id)
397
398 # NOTE: 0666 is affected by umask, all shells use it.
399 try:
400 open_fd = posix.open(arg.filename, mode, 0o666)
401 except (IOError, OSError) as e:
402 self.errfmt.Print_("Can't open %r: %s" %
403 (arg.filename, pyutil.strerror(e)),
404 blame_loc=r.op_loc)
405 raise # redirect failed
406
407 new_fd = self._PushDup(open_fd, r.loc)
408 if new_fd != NO_FD:
409 posix.close(open_fd)
410
411 # Now handle &> and &>> and their variants. These pairs are the same:
412 #
413 # stdout_stderr.py &> out-err.txt
414 # stdout_stderr.py > out-err.txt 2>&1
415 #
416 # stdout_stderr.py 3&> out-err.txt
417 # stdout_stderr.py 3> out-err.txt 2>&3
418 #
419 # Ditto for {fd}> and {fd}&>
420
421 if r.op_id in (Id.Redir_AndGreat, Id.Redir_AndDGreat):
422 self._PushDup(new_fd, redir_loc.Fd(2))
423
424 elif case(redirect_arg_e.CopyFd): # e.g. echo hi 1>&2
425 arg = cast(redirect_arg.CopyFd, UP_arg)
426
427 if r.op_id == Id.Redir_GreatAnd: # 1>&2
428 self._PushDup(arg.target_fd, r.loc)
429
430 elif r.op_id == Id.Redir_LessAnd: # 0<&5
431 # The only difference between >& and <& is the default file
432 # descriptor argument.
433 self._PushDup(arg.target_fd, r.loc)
434
435 else:
436 raise NotImplementedError()
437
438 elif case(redirect_arg_e.MoveFd): # e.g. echo hi 5>&6-
439 arg = cast(redirect_arg.MoveFd, UP_arg)
440 new_fd = self._PushDup(arg.target_fd, r.loc)
441 if new_fd != NO_FD:
442 posix.close(arg.target_fd)
443
444 UP_loc = r.loc
445 if r.loc.tag() == redir_loc_e.Fd:
446 fd = cast(redir_loc.Fd, UP_loc).fd
447 else:
448 fd = NO_FD
449
450 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, False))
451
452 elif case(redirect_arg_e.CloseFd): # e.g. echo hi 5>&-
453 self._PushCloseFd(r.loc)
454
455 elif case(redirect_arg_e.HereDoc):
456 arg = cast(redirect_arg.HereDoc, UP_arg)
457
458 # NOTE: Do these descriptors have to be moved out of the range 0-9?
459 read_fd, write_fd = posix.pipe()
460
461 self._PushDup(read_fd, r.loc) # stdin is now the pipe
462
463 # We can't close like we do in the filename case above? The writer can
464 # get a "broken pipe".
465 self._PushClose(read_fd)
466
467 thunk = _HereDocWriterThunk(write_fd, arg.body)
468
469 # Use PIPE_SIZE to save a process in the case of small here
470 # docs, which are the common case. (dash does this.)
471
472 # Note: could instrument this to see how often it happens.
473 # Though strace -ff can also work.
474 start_process = len(arg.body) > 4096
475 #start_process = True
476
477 if start_process:
478 here_proc = Process(thunk, self.job_control, self.job_list,
479 self.tracer)
480
481 # NOTE: we could close the read pipe here, but it doesn't really
482 # matter because we control the code.
483 here_proc.StartProcess(trace.HereDoc)
484 #log('Started %s as %d', here_proc, pid)
485 self._PushWait(here_proc)
486
487 # Now that we've started the child, close it in the parent.
488 posix.close(write_fd)
489
490 else:
491 posix.write(write_fd, arg.body)
492 posix.close(write_fd)
493
494 def Push(self, redirects, err_out):
495 # type: (List[RedirValue], List[error.IOError_OSError]) -> None
496 """Apply a group of redirects and remember to undo them."""
497
498 #log('> fd_state.Push %s', redirects)
499 new_frame = _FdFrame()
500 self.stack.append(new_frame)
501 self.cur_frame = new_frame
502
503 for r in redirects:
504 #log('apply %s', r)
505 with ui.ctx_Location(self.errfmt, r.op_loc):
506 try:
507 self._ApplyRedirect(r)
508 except (IOError, OSError) as e:
509 err_out.append(e)
510 # This can fail too
511 self.Pop(err_out)
512 return # for bad descriptor, etc.
513
514 def PushStdinFromPipe(self, r):
515 # type: (int) -> bool
516 """Save the current stdin and make it come from descriptor 'r'.
517
518 'r' is typically the read-end of a pipe. For 'lastpipe'/ZSH
519 semantics of
520
521 echo foo | read line; echo $line
522 """
523 new_frame = _FdFrame()
524 self.stack.append(new_frame)
525 self.cur_frame = new_frame
526
527 self._PushDup(r, redir_loc.Fd(0))
528 return True
529
530 def Pop(self, err_out):
531 # type: (List[error.IOError_OSError]) -> None
532 frame = self.stack.pop()
533 #log('< Pop %s', frame)
534 for rf in reversed(frame.saved):
535 if rf.saved_fd == NO_FD:
536 #log('Close %d', orig)
537 try:
538 posix.close(rf.orig_fd)
539 except (IOError, OSError) as e:
540 err_out.append(e)
541 log('Error closing descriptor %d: %s', rf.orig_fd,
542 pyutil.strerror(e))
543 return
544 else:
545 try:
546 posix.dup2(rf.saved_fd, rf.orig_fd)
547 except (IOError, OSError) as e:
548 err_out.append(e)
549 log('dup2(%d, %d) error: %s', rf.saved_fd, rf.orig_fd,
550 pyutil.strerror(e))
551 #log('fd state:')
552 #posix.system('ls -l /proc/%s/fd' % posix.getpid())
553 return
554 posix.close(rf.saved_fd)
555 #log('dup2 %s %s', saved, orig)
556
557 # Wait for here doc processes to finish.
558 for proc in frame.need_wait:
559 unused_status = proc.Wait(self.waiter)
560
561 def MakePermanent(self):
562 # type: () -> None
563 self.cur_frame.Forget()
564
565
566class ChildStateChange(object):
567
568 def __init__(self):
569 # type: () -> None
570 """Empty constructor for mycpp."""
571 pass
572
573 def Apply(self):
574 # type: () -> None
575 raise NotImplementedError()
576
577 def ApplyFromParent(self, proc):
578 # type: (Process) -> None
579 """Noop for all state changes other than SetPgid for mycpp."""
580 pass
581
582
583class StdinFromPipe(ChildStateChange):
584
585 def __init__(self, pipe_read_fd, w):
586 # type: (int, int) -> None
587 self.r = pipe_read_fd
588 self.w = w
589
590 def __repr__(self):
591 # type: () -> str
592 return '<StdinFromPipe %d %d>' % (self.r, self.w)
593
594 def Apply(self):
595 # type: () -> None
596 posix.dup2(self.r, 0)
597 posix.close(self.r) # close after dup
598
599 posix.close(self.w) # we're reading from the pipe, not writing
600 #log('child CLOSE w %d pid=%d', self.w, posix.getpid())
601
602
603class StdoutToPipe(ChildStateChange):
604
605 def __init__(self, r, pipe_write_fd):
606 # type: (int, int) -> None
607 self.r = r
608 self.w = pipe_write_fd
609
610 def __repr__(self):
611 # type: () -> str
612 return '<StdoutToPipe %d %d>' % (self.r, self.w)
613
614 def Apply(self):
615 # type: () -> None
616 posix.dup2(self.w, 1)
617 posix.close(self.w) # close after dup
618
619 posix.close(self.r) # we're writing to the pipe, not reading
620 #log('child CLOSE r %d pid=%d', self.r, posix.getpid())
621
622
623INVALID_PGID = -1
624# argument to setpgid() that means the process is its own leader
625OWN_LEADER = 0
626
627
628class SetPgid(ChildStateChange):
629
630 def __init__(self, pgid, tracer):
631 # type: (int, dev.Tracer) -> None
632 self.pgid = pgid
633 self.tracer = tracer
634
635 def Apply(self):
636 # type: () -> None
637 try:
638 posix.setpgid(0, self.pgid)
639 except (IOError, OSError) as e:
640 self.tracer.OtherMessage(
641 'osh: child %d failed to set its process group to %d: %s' %
642 (posix.getpid(), self.pgid, pyutil.strerror(e)))
643
644 def ApplyFromParent(self, proc):
645 # type: (Process) -> None
646 try:
647 posix.setpgid(proc.pid, self.pgid)
648 except (IOError, OSError) as e:
649 self.tracer.OtherMessage(
650 'osh: parent failed to set process group for PID %d to %d: %s'
651 % (proc.pid, self.pgid, pyutil.strerror(e)))
652
653
654class ExternalProgram(object):
655 """The capability to execute an external program like 'ls'."""
656
657 def __init__(
658 self,
659 hijack_shebang, # type: str
660 fd_state, # type: FdState
661 errfmt, # type: ErrorFormatter
662 debug_f, # type: _DebugFile
663 ):
664 # type: (...) -> None
665 """
666 Args:
667 hijack_shebang: The path of an interpreter to run instead of the one
668 specified in the shebang line. May be empty.
669 """
670 self.hijack_shebang = hijack_shebang
671 self.fd_state = fd_state
672 self.errfmt = errfmt
673 self.debug_f = debug_f
674
675 def Exec(self, argv0_path, cmd_val, environ):
676 # type: (str, cmd_value.Argv, Dict[str, str]) -> None
677 """Execute a program and exit this process.
678
679 Called by: ls / exec ls / ( ls / )
680 """
681 probe('process', 'ExternalProgram_Exec', argv0_path)
682 self._Exec(argv0_path, cmd_val.argv, cmd_val.arg_locs[0], environ,
683 True)
684 assert False, "This line should never execute" # NO RETURN
685
686 def _Exec(self, argv0_path, argv, argv0_loc, environ, should_retry):
687 # type: (str, List[str], loc_t, Dict[str, str], bool) -> None
688 if len(self.hijack_shebang):
689 opened = True
690 try:
691 f = self.fd_state.Open(argv0_path)
692 except (IOError, OSError) as e:
693 opened = False
694
695 if opened:
696 with ctx_FileCloser(f):
697 # Test if the shebang looks like a shell. TODO: The file might be
698 # binary with no newlines, so read 80 bytes instead of readline().
699
700 #line = f.read(80) # type: ignore # TODO: fix this
701 line = f.readline()
702
703 if match.ShouldHijack(line):
704 h_argv = [self.hijack_shebang, argv0_path]
705 h_argv.extend(argv[1:])
706 argv = h_argv
707 argv0_path = self.hijack_shebang
708 self.debug_f.writeln('Hijacked: %s' % argv0_path)
709 else:
710 #self.debug_f.log('Not hijacking %s (%r)', argv, line)
711 pass
712
713 try:
714 posix.execve(argv0_path, argv, environ)
715 except (IOError, OSError) as e:
716 # Run with /bin/sh when ENOEXEC error (no shebang). All shells do this.
717 if e.errno == ENOEXEC and should_retry:
718 new_argv = ['/bin/sh', argv0_path]
719 new_argv.extend(argv[1:])
720 self._Exec('/bin/sh', new_argv, argv0_loc, environ, False)
721 # NO RETURN
722
723 # Would be nice: when the path is relative and ENOENT: print PWD and do
724 # spelling correction?
725
726 self.errfmt.Print_(
727 "Can't execute %r: %s" % (argv0_path, pyutil.strerror(e)),
728 argv0_loc)
729
730 # POSIX mentions 126 and 127 for two specific errors. The rest are
731 # unspecified.
732 #
733 # http://pubs.opengroup.org/onlinepubs/9699919799.2016edition/utilities/V3_chap02.html#tag_18_08_02
734 if e.errno == EACCES:
735 status = 126
736 elif e.errno == ENOENT:
737 # TODO: most shells print 'command not found', rather than strerror()
738 # == "No such file or directory". That's better because it's at the
739 # end of the path search, and we're never searching for a directory.
740 status = 127
741 else:
742 # dash uses 2, but we use that for parse errors. This seems to be
743 # consistent with mksh and zsh.
744 status = 127
745
746 posix._exit(status)
747 # NO RETURN
748
749
750class Thunk(object):
751 """Abstract base class for things runnable in another process."""
752
753 def __init__(self):
754 # type: () -> None
755 """Empty constructor for mycpp."""
756 pass
757
758 def Run(self):
759 # type: () -> None
760 """Returns a status code."""
761 raise NotImplementedError()
762
763 def UserString(self):
764 # type: () -> str
765 """Display for the 'jobs' list."""
766 raise NotImplementedError()
767
768 def __repr__(self):
769 # type: () -> str
770 return self.UserString()
771
772
773class ExternalThunk(Thunk):
774 """An external executable."""
775
776 def __init__(self, ext_prog, argv0_path, cmd_val, environ):
777 # type: (ExternalProgram, str, cmd_value.Argv, Dict[str, str]) -> None
778 self.ext_prog = ext_prog
779 self.argv0_path = argv0_path
780 self.cmd_val = cmd_val
781 self.environ = environ
782
783 def UserString(self):
784 # type: () -> str
785
786 # NOTE: This is the format the Tracer uses.
787 # bash displays sleep $n & (code)
788 # but OSH displays sleep 1 & (argv array)
789 # We could switch the former but I'm not sure it's necessary.
790 tmp = [j8_lite.MaybeShellEncode(a) for a in self.cmd_val.argv]
791 return '[process] %s' % ' '.join(tmp)
792
793 def Run(self):
794 # type: () -> None
795 """An ExternalThunk is run in parent for the exec builtin."""
796 self.ext_prog.Exec(self.argv0_path, self.cmd_val, self.environ)
797
798
799class SubProgramThunk(Thunk):
800 """A subprogram that can be executed in another process."""
801
802 def __init__(self,
803 cmd_ev,
804 node,
805 trap_state,
806 multi_trace,
807 inherit_errexit=True):
808 # type: (CommandEvaluator, command_t, trap_osh.TrapState, dev.MultiTracer, bool) -> None
809 self.cmd_ev = cmd_ev
810 self.node = node
811 self.trap_state = trap_state
812 self.multi_trace = multi_trace
813 self.inherit_errexit = inherit_errexit # for bash errexit compatibility
814
815 def UserString(self):
816 # type: () -> str
817
818 # NOTE: These can be pieces of a pipeline, so they're arbitrary nodes.
819 # TODO: Extract SPIDS from node to display source? Note that
820 # CompoundStatus also has locations of each pipeline component; see
821 # Executor.RunPipeline()
822 thunk_str = ui.CommandType(self.node)
823 return '[subprog] %s' % thunk_str
824
825 def Run(self):
826 # type: () -> None
827 #self.errfmt.OneLineErrExit() # don't quote code in child processes
828 probe('process', 'SubProgramThunk_Run')
829
830 # TODO: break circular dep. Bit flags could go in ASDL or headers.
831 from osh import cmd_eval
832
833 # signal handlers aren't inherited
834 self.trap_state.ClearForSubProgram()
835
836 # NOTE: may NOT return due to exec().
837 if not self.inherit_errexit:
838 self.cmd_ev.mutable_opts.DisableErrExit()
839 try:
840 # optimize to eliminate redundant subshells like ( echo hi ) | wc -l etc.
841 self.cmd_ev.ExecuteAndCatch(self.node, cmd_flags=cmd_eval.Optimize)
842 status = self.cmd_ev.LastStatus()
843 # NOTE: We ignore the is_fatal return value. The user should set -o
844 # errexit so failures in subprocesses cause failures in the parent.
845 except util.UserExit as e:
846 status = e.status
847
848 # Handle errors in a subshell. These two cases are repeated from main()
849 # and the core/completion.py hook.
850 except KeyboardInterrupt:
851 print('')
852 status = 130 # 128 + 2
853 except (IOError, OSError) as e:
854 print_stderr('oils I/O error (subprogram): %s' %
855 pyutil.strerror(e))
856 status = 2
857
858 # If ProcessInit() doesn't turn off buffering, this is needed before
859 # _exit()
860 pyos.FlushStdout()
861
862 self.multi_trace.WriteDumps()
863
864 # We do NOT want to raise SystemExit here. Otherwise dev.Tracer::Pop()
865 # gets called in BOTH processes.
866 # The crash dump seems to be unaffected.
867 posix._exit(status)
868
869
870class _HereDocWriterThunk(Thunk):
871 """Write a here doc to one end of a pipe.
872
873 May be be executed in either a child process or the main shell
874 process.
875 """
876
877 def __init__(self, w, body_str):
878 # type: (int, str) -> None
879 self.w = w
880 self.body_str = body_str
881
882 def UserString(self):
883 # type: () -> str
884
885 # You can hit Ctrl-Z and the here doc writer will be suspended! Other
886 # shells don't have this problem because they use temp files! That's a bit
887 # unfortunate.
888 return '[here doc writer]'
889
890 def Run(self):
891 # type: () -> None
892 """do_exit: For small pipelines."""
893 probe('process', 'HereDocWriterThunk_Run')
894 #log('Writing %r', self.body_str)
895 posix.write(self.w, self.body_str)
896 #log('Wrote %r', self.body_str)
897 posix.close(self.w)
898 #log('Closed %d', self.w)
899
900 posix._exit(0)
901
902
903class Job(object):
904 """Interface for both Process and Pipeline.
905
906 They both can be put in the background and waited on.
907
908 Confusing thing about pipelines in the background: They have TOO MANY NAMES.
909
910 sleep 1 | sleep 2 &
911
912 - The LAST PID is what's printed at the prompt. This is $!, a PROCESS ID and
913 not a JOB ID.
914 # https://www.gnu.org/software/bash/manual/html_node/Special-Parameters.html#Special-Parameters
915 - The process group leader (setpgid) is the FIRST PID.
916 - It's also %1 or %+. The last job started.
917 """
918
919 def __init__(self):
920 # type: () -> None
921 # Initial state with & or Ctrl-Z is Running.
922 self.state = job_state_e.Running
923 self.job_id = -1
924 self.in_background = False
925
926 def DisplayJob(self, job_id, f, style):
927 # type: (int, mylib.Writer, int) -> None
928 raise NotImplementedError()
929
930 def State(self):
931 # type: () -> job_state_t
932 return self.state
933
934 def ProcessGroupId(self):
935 # type: () -> int
936 """Return the process group ID associated with this job."""
937 raise NotImplementedError()
938
939 def JobWait(self, waiter):
940 # type: (Waiter) -> wait_status_t
941 """Wait for this process/pipeline to be stopped or finished."""
942 raise NotImplementedError()
943
944 def SetBackground(self):
945 # type: () -> None
946 """Record that this job is running in the background."""
947 self.in_background = True
948
949 def SetForeground(self):
950 # type: () -> None
951 """Record that this job is running in the foreground."""
952 self.in_background = False
953
954
955class Process(Job):
956 """A process to run.
957
958 TODO: Should we make it clear that this is a FOREGROUND process? A
959 background process is wrapped in a "job". It is unevaluated.
960
961 It provides an API to manipulate file descriptor state in parent and child.
962 """
963
964 def __init__(self, thunk, job_control, job_list, tracer):
965 # type: (Thunk, JobControl, JobList, dev.Tracer) -> None
966 """
967 Args:
968 thunk: Thunk instance
969 job_list: for process bookkeeping
970 """
971 Job.__init__(self)
972 assert isinstance(thunk, Thunk), thunk
973 self.thunk = thunk
974 self.job_control = job_control
975 self.job_list = job_list
976 self.tracer = tracer
977
978 # For pipelines
979 self.parent_pipeline = None # type: Pipeline
980 self.state_changes = [] # type: List[ChildStateChange]
981 self.close_r = -1
982 self.close_w = -1
983
984 self.pid = -1
985 self.status = -1
986
987 def Init_ParentPipeline(self, pi):
988 # type: (Pipeline) -> None
989 """For updating PIPESTATUS."""
990 self.parent_pipeline = pi
991
992 def __repr__(self):
993 # type: () -> str
994
995 # note: be wary of infinite mutual recursion
996 #s = ' %s' % self.parent_pipeline if self.parent_pipeline else ''
997 #return '<Process %s%s>' % (self.thunk, s)
998 return '<Process %s %s>' % (_JobStateStr(self.state), self.thunk)
999
1000 def ProcessGroupId(self):
1001 # type: () -> int
1002 """Returns the group ID of this process."""
1003 # This should only ever be called AFTER the process has started
1004 assert self.pid != -1
1005 if self.parent_pipeline:
1006 # XXX: Maybe we should die here instead? Unclear if this branch
1007 # should even be reachable with the current builtins.
1008 return self.parent_pipeline.ProcessGroupId()
1009
1010 return self.pid
1011
1012 def DisplayJob(self, job_id, f, style):
1013 # type: (int, mylib.Writer, int) -> None
1014 if job_id == -1:
1015 job_id_str = ' '
1016 else:
1017 job_id_str = '%%%d' % job_id
1018 if style == STYLE_PID_ONLY:
1019 f.write('%d\n' % self.pid)
1020 else:
1021 f.write('%s %d %7s ' %
1022 (job_id_str, self.pid, _JobStateStr(self.state)))
1023 f.write(self.thunk.UserString())
1024 f.write('\n')
1025
1026 def AddStateChange(self, s):
1027 # type: (ChildStateChange) -> None
1028 self.state_changes.append(s)
1029
1030 def AddPipeToClose(self, r, w):
1031 # type: (int, int) -> None
1032 self.close_r = r
1033 self.close_w = w
1034
1035 def MaybeClosePipe(self):
1036 # type: () -> None
1037 if self.close_r != -1:
1038 posix.close(self.close_r)
1039 posix.close(self.close_w)
1040
1041 def StartProcess(self, why):
1042 # type: (trace_t) -> int
1043 """Start this process with fork(), handling redirects."""
1044 pid = posix.fork()
1045 if pid < 0:
1046 # When does this happen?
1047 e_die('Fatal error in posix.fork()')
1048
1049 elif pid == 0: # child
1050 # Note: this happens in BOTH interactive and non-interactive shells.
1051 # We technically don't need to do most of it in non-interactive, since we
1052 # did not change state in InitInteractiveShell().
1053
1054 for st in self.state_changes:
1055 st.Apply()
1056
1057 # Python sets SIGPIPE handler to SIG_IGN by default. Child processes
1058 # shouldn't have this.
1059 # https://docs.python.org/2/library/signal.html
1060 # See Python/pythonrun.c.
1061 pyos.Sigaction(SIGPIPE, SIG_DFL)
1062
1063 # Respond to Ctrl-\ (core dump)
1064 pyos.Sigaction(SIGQUIT, SIG_DFL)
1065
1066 # Only standalone children should get Ctrl-Z. Pipelines remain in the
1067 # foreground because suspending them is difficult with our 'lastpipe'
1068 # semantics.
1069 pid = posix.getpid()
1070 if posix.getpgid(0) == pid and self.parent_pipeline is None:
1071 pyos.Sigaction(SIGTSTP, SIG_DFL)
1072
1073 # More signals from
1074 # https://www.gnu.org/software/libc/manual/html_node/Launching-Jobs.html
1075 # (but not SIGCHLD)
1076 pyos.Sigaction(SIGTTOU, SIG_DFL)
1077 pyos.Sigaction(SIGTTIN, SIG_DFL)
1078
1079 self.tracer.OnNewProcess(pid)
1080 # clear foreground pipeline for subshells
1081 self.thunk.Run()
1082 # Never returns
1083
1084 #log('STARTED process %s, pid = %d', self, pid)
1085 self.tracer.OnProcessStart(pid, why)
1086
1087 # Class invariant: after the process is started, it stores its PID.
1088 self.pid = pid
1089
1090 # SetPgid needs to be applied from the child and the parent to avoid
1091 # racing in calls to tcsetpgrp() in the parent. See APUE sec. 9.2.
1092 for st in self.state_changes:
1093 st.ApplyFromParent(self)
1094
1095 # Program invariant: We keep track of every child process!
1096 self.job_list.AddChildProcess(pid, self)
1097
1098 return pid
1099
1100 def Wait(self, waiter):
1101 # type: (Waiter) -> int
1102 """Wait for this process to finish."""
1103 while self.state == job_state_e.Running:
1104 # Only return if there's nothing to wait for. Keep waiting if we were
1105 # interrupted with a signal.
1106 if waiter.WaitForOne() == W1_ECHILD:
1107 break
1108
1109 assert self.status >= 0, self.status
1110 return self.status
1111
1112 def JobWait(self, waiter):
1113 # type: (Waiter) -> wait_status_t
1114 # wait builtin can be interrupted
1115 while self.state == job_state_e.Running:
1116 result = waiter.WaitForOne()
1117
1118 if result >= 0: # signal
1119 return wait_status.Cancelled(result)
1120
1121 if result == W1_ECHILD:
1122 break
1123
1124 return wait_status.Proc(self.status)
1125
1126 def WhenStopped(self, stop_sig):
1127 # type: (int) -> None
1128
1129 # 128 is a shell thing
1130 # https://www.gnu.org/software/bash/manual/html_node/Exit-Status.html
1131 self.status = 128 + stop_sig
1132 self.state = job_state_e.Stopped
1133
1134 if self.job_id == -1:
1135 # This process was started in the foreground
1136 self.job_list.AddJob(self)
1137
1138 if not self.in_background:
1139 self.job_control.MaybeTakeTerminal()
1140 self.SetBackground()
1141
1142 def WhenDone(self, pid, status):
1143 # type: (int, int) -> None
1144 """Called by the Waiter when this Process finishes."""
1145
1146 #log('WhenDone %d %d', pid, status)
1147 assert pid == self.pid, 'Expected %d, got %d' % (self.pid, pid)
1148 self.status = status
1149 self.state = job_state_e.Done
1150 if self.parent_pipeline:
1151 self.parent_pipeline.WhenDone(pid, status)
1152 else:
1153 if self.job_id != -1:
1154 # Job might have been brought to the foreground after being
1155 # assigned a job ID.
1156 if self.in_background:
1157 print_stderr('[%d] Done PID %d' % (self.job_id, self.pid))
1158
1159 self.job_list.RemoveJob(self.job_id)
1160
1161 self.job_list.RemoveChildProcess(self.pid)
1162
1163 if not self.in_background:
1164 self.job_control.MaybeTakeTerminal()
1165
1166 def RunProcess(self, waiter, why):
1167 # type: (Waiter, trace_t) -> int
1168 """Run this process synchronously."""
1169 self.StartProcess(why)
1170 # ShellExecutor might be calling this for the last part of a pipeline.
1171 if self.parent_pipeline is None:
1172 # QUESTION: Can the PGID of a single process just be the PID? i.e. avoid
1173 # calling getpgid()?
1174 self.job_control.MaybeGiveTerminal(posix.getpgid(self.pid))
1175 return self.Wait(waiter)
1176
1177
1178class ctx_Pipe(object):
1179
1180 def __init__(self, fd_state, fd, err_out):
1181 # type: (FdState, int, List[error.IOError_OSError]) -> None
1182 fd_state.PushStdinFromPipe(fd)
1183 self.fd_state = fd_state
1184 self.err_out = err_out
1185
1186 def __enter__(self):
1187 # type: () -> None
1188 pass
1189
1190 def __exit__(self, type, value, traceback):
1191 # type: (Any, Any, Any) -> None
1192 self.fd_state.Pop(self.err_out)
1193
1194
1195class Pipeline(Job):
1196 """A pipeline of processes to run.
1197
1198 Cases we handle:
1199
1200 foo | bar
1201 $(foo | bar)
1202 foo | bar | read v
1203 """
1204
1205 def __init__(self, sigpipe_status_ok, job_control, job_list, tracer):
1206 # type: (bool, JobControl, JobList, dev.Tracer) -> None
1207 Job.__init__(self)
1208 self.job_control = job_control
1209 self.job_list = job_list
1210 self.tracer = tracer
1211
1212 self.procs = [] # type: List[Process]
1213 self.pids = [] # type: List[int] # pids in order
1214 self.pipe_status = [] # type: List[int] # status in order
1215 self.status = -1 # for 'wait' jobs
1216
1217 self.pgid = INVALID_PGID
1218
1219 # Optional for foreground
1220 self.last_thunk = None # type: Tuple[CommandEvaluator, command_t]
1221 self.last_pipe = None # type: Tuple[int, int]
1222
1223 self.sigpipe_status_ok = sigpipe_status_ok
1224
1225 def ProcessGroupId(self):
1226 # type: () -> int
1227 """Returns the group ID of this pipeline."""
1228 return self.pgid
1229
1230 def DisplayJob(self, job_id, f, style):
1231 # type: (int, mylib.Writer, int) -> None
1232 if style == STYLE_PID_ONLY:
1233 f.write('%d\n' % self.procs[0].pid)
1234 else:
1235 # Note: this is STYLE_LONG.
1236 for i, proc in enumerate(self.procs):
1237 if i == 0: # show job ID for first element in pipeline
1238 job_id_str = '%%%d' % job_id
1239 else:
1240 job_id_str = ' ' # 2 spaces
1241
1242 f.write('%s %d %7s ' %
1243 (job_id_str, proc.pid, _JobStateStr(proc.state)))
1244 f.write(proc.thunk.UserString())
1245 f.write('\n')
1246
1247 def DebugPrint(self):
1248 # type: () -> None
1249 print('Pipeline in state %s' % _JobStateStr(self.state))
1250 if mylib.PYTHON: # %s for Process not allowed in C++
1251 for proc in self.procs:
1252 print(' proc %s' % proc)
1253 _, last_node = self.last_thunk
1254 print(' last %s' % last_node)
1255 print(' pipe_status %s' % self.pipe_status)
1256
1257 def Add(self, p):
1258 # type: (Process) -> None
1259 """Append a process to the pipeline."""
1260 if len(self.procs) == 0:
1261 self.procs.append(p)
1262 return
1263
1264 r, w = posix.pipe()
1265 #log('pipe for %s: %d %d', p, r, w)
1266 prev = self.procs[-1]
1267
1268 prev.AddStateChange(StdoutToPipe(r, w)) # applied on StartPipeline()
1269 p.AddStateChange(StdinFromPipe(r, w)) # applied on StartPipeline()
1270
1271 p.AddPipeToClose(r, w) # MaybeClosePipe() on StartPipeline()
1272
1273 self.procs.append(p)
1274
1275 def AddLast(self, thunk):
1276 # type: (Tuple[CommandEvaluator, command_t]) -> None
1277 """Append the last noden to the pipeline.
1278
1279 This is run in the CURRENT process. It is OPTIONAL, because
1280 pipelines in the background are run uniformly.
1281 """
1282 self.last_thunk = thunk
1283
1284 assert len(self.procs) != 0
1285
1286 r, w = posix.pipe()
1287 prev = self.procs[-1]
1288 prev.AddStateChange(StdoutToPipe(r, w))
1289
1290 self.last_pipe = (r, w) # So we can connect it to last_thunk
1291
1292 def StartPipeline(self, waiter):
1293 # type: (Waiter) -> None
1294
1295 # If we are creating a pipeline in a subshell or we aren't running with job
1296 # control, our children should remain in our inherited process group.
1297 # the pipelines's group ID.
1298 if self.job_control.Enabled():
1299 self.pgid = OWN_LEADER # first process in pipeline is the leader
1300
1301 for i, proc in enumerate(self.procs):
1302 if self.pgid != INVALID_PGID:
1303 proc.AddStateChange(SetPgid(self.pgid, self.tracer))
1304
1305 # Figure out the pid
1306 pid = proc.StartProcess(trace.PipelinePart)
1307 if i == 0 and self.pgid != INVALID_PGID:
1308 # Mimic bash and use the PID of the FIRST process as the group for the
1309 # whole pipeline.
1310 self.pgid = pid
1311
1312 self.pids.append(pid)
1313 self.pipe_status.append(-1) # uninitialized
1314
1315 # NOTE: This is done in the SHELL PROCESS after every fork() call.
1316 # It can't be done at the end; otherwise processes will have descriptors
1317 # from non-adjacent pipes.
1318 proc.MaybeClosePipe()
1319
1320 if self.last_thunk:
1321 self.pipe_status.append(-1) # for self.last_thunk
1322
1323 def LastPid(self):
1324 # type: () -> int
1325 """For the odd $! variable.
1326
1327 It would be better if job IDs or PGIDs were used consistently.
1328 """
1329 return self.pids[-1]
1330
1331 def Wait(self, waiter):
1332 # type: (Waiter) -> List[int]
1333 """Wait for this pipeline to finish."""
1334
1335 assert self.procs, "no procs for Wait()"
1336 # waitpid(-1) zero or more times
1337 while self.state == job_state_e.Running:
1338 # Keep waiting until there's nothing to wait for.
1339 if waiter.WaitForOne() == W1_ECHILD:
1340 break
1341
1342 return self.pipe_status
1343
1344 def JobWait(self, waiter):
1345 # type: (Waiter) -> wait_status_t
1346 """Called by 'wait' builtin, e.g. 'wait %1'."""
1347 # wait builtin can be interrupted
1348 assert self.procs, "no procs for Wait()"
1349 while self.state == job_state_e.Running:
1350 result = waiter.WaitForOne()
1351
1352 if result >= 0: # signal
1353 return wait_status.Cancelled(result)
1354
1355 if result == W1_ECHILD:
1356 break
1357
1358 return wait_status.Pipeline(self.pipe_status)
1359
1360 def RunLastPart(self, waiter, fd_state):
1361 # type: (Waiter, FdState) -> List[int]
1362 """Run this pipeline synchronously (foreground pipeline).
1363
1364 Returns:
1365 pipe_status (list of integers).
1366 """
1367 assert len(self.pids) == len(self.procs)
1368
1369 # TODO: break circular dep. Bit flags could go in ASDL or headers.
1370 from osh import cmd_eval
1371
1372 # This is tcsetpgrp()
1373 # TODO: fix race condition -- I believe the first process could have
1374 # stopped already, and thus getpgid() will fail
1375 self.job_control.MaybeGiveTerminal(self.pgid)
1376
1377 # Run the last part of the pipeline IN PARALLEL with other processes. It
1378 # may or may not fork:
1379 # echo foo | read line # no fork, the builtin runs in THIS shell process
1380 # ls | wc -l # fork for 'wc'
1381
1382 cmd_ev, last_node = self.last_thunk
1383
1384 assert self.last_pipe is not None
1385 r, w = self.last_pipe # set in AddLast()
1386 posix.close(w) # we will not write here
1387
1388 # Fix lastpipe / job control / DEBUG trap interaction
1389 cmd_flags = cmd_eval.NoDebugTrap if self.job_control.Enabled() else 0
1390 io_errors = [] # type: List[error.IOError_OSError]
1391 with ctx_Pipe(fd_state, r, io_errors):
1392 cmd_ev.ExecuteAndCatch(last_node, cmd_flags)
1393
1394 if len(io_errors):
1395 e_die('Error setting up last part of pipeline: %s' %
1396 pyutil.strerror(io_errors[0]))
1397
1398 # We won't read anymore. If we don't do this, then 'cat' in 'cat
1399 # /dev/urandom | sleep 1' will never get SIGPIPE.
1400 posix.close(r)
1401
1402 self.pipe_status[-1] = cmd_ev.LastStatus()
1403 if self.AllDone():
1404 self.state = job_state_e.Done
1405
1406 #log('pipestatus before all have finished = %s', self.pipe_status)
1407 return self.Wait(waiter)
1408
1409 def AllDone(self):
1410 # type: () -> bool
1411
1412 # mycpp rewrite: all(status != -1 for status in self.pipe_status)
1413 for status in self.pipe_status:
1414 if status == -1:
1415 return False
1416 return True
1417
1418 def WhenDone(self, pid, status):
1419 # type: (int, int) -> None
1420 """Called by Process.WhenDone."""
1421 #log('Pipeline WhenDone %d %d', pid, status)
1422 i = self.pids.index(pid)
1423 assert i != -1, 'Unexpected PID %d' % pid
1424
1425 if status == 141 and self.sigpipe_status_ok:
1426 status = 0
1427
1428 self.job_list.RemoveChildProcess(pid)
1429 self.pipe_status[i] = status
1430 if self.AllDone():
1431 if self.job_id != -1:
1432 # Job might have been brought to the foreground after being
1433 # assigned a job ID.
1434 if self.in_background:
1435 print_stderr('[%d] Done PGID %d' %
1436 (self.job_id, self.pids[0]))
1437
1438 self.job_list.RemoveJob(self.job_id)
1439
1440 # status of pipeline is status of last process
1441 self.status = self.pipe_status[-1]
1442 self.state = job_state_e.Done
1443 if not self.in_background:
1444 self.job_control.MaybeTakeTerminal()
1445
1446
1447def _JobStateStr(i):
1448 # type: (job_state_t) -> str
1449 return job_state_str(i)[10:] # remove 'job_state.'
1450
1451
1452def _GetTtyFd():
1453 # type: () -> int
1454 """Returns -1 if stdio is not a TTY."""
1455 try:
1456 return posix.open("/dev/tty", O_NONBLOCK | O_NOCTTY | O_RDWR, 0o666)
1457 except (IOError, OSError) as e:
1458 return -1
1459
1460
1461class ctx_TerminalControl(object):
1462
1463 def __init__(self, job_control, errfmt):
1464 # type: (JobControl, ui.ErrorFormatter) -> None
1465 job_control.InitJobControl()
1466 self.job_control = job_control
1467 self.errfmt = errfmt
1468
1469 def __enter__(self):
1470 # type: () -> None
1471 pass
1472
1473 def __exit__(self, type, value, traceback):
1474 # type: (Any, Any, Any) -> None
1475
1476 # Return the TTY to the original owner before exiting.
1477 try:
1478 self.job_control.MaybeReturnTerminal()
1479 except error.FatalRuntime as e:
1480 # Don't abort the shell on error, just print a message.
1481 self.errfmt.PrettyPrintError(e)
1482
1483
1484class JobControl(object):
1485 """Interface to setpgid(), tcsetpgrp(), etc."""
1486
1487 def __init__(self):
1488 # type: () -> None
1489
1490 # The main shell's PID and group ID.
1491 self.shell_pid = -1
1492 self.shell_pgid = -1
1493
1494 # The fd of the controlling tty. Set to -1 when job control is disabled.
1495 self.shell_tty_fd = -1
1496
1497 # For giving the terminal back to our parent before exiting (if not a login
1498 # shell).
1499 self.original_tty_pgid = -1
1500
1501 def InitJobControl(self):
1502 # type: () -> None
1503 self.shell_pid = posix.getpid()
1504 orig_shell_pgid = posix.getpgid(0)
1505 self.shell_pgid = orig_shell_pgid
1506 self.shell_tty_fd = _GetTtyFd()
1507
1508 # If we aren't the leader of our process group, create a group and mark
1509 # ourselves as the leader.
1510 if self.shell_pgid != self.shell_pid:
1511 try:
1512 posix.setpgid(self.shell_pid, self.shell_pid)
1513 self.shell_pgid = self.shell_pid
1514 except (IOError, OSError) as e:
1515 self.shell_tty_fd = -1
1516
1517 if self.shell_tty_fd != -1:
1518 self.original_tty_pgid = posix.tcgetpgrp(self.shell_tty_fd)
1519
1520 # If stdio is a TTY, put the shell's process group in the foreground.
1521 try:
1522 posix.tcsetpgrp(self.shell_tty_fd, self.shell_pgid)
1523 except (IOError, OSError) as e:
1524 # We probably aren't in the session leader's process group. Disable job
1525 # control.
1526 self.shell_tty_fd = -1
1527 self.shell_pgid = orig_shell_pgid
1528 posix.setpgid(self.shell_pid, self.shell_pgid)
1529
1530 def Enabled(self):
1531 # type: () -> bool
1532
1533 # TODO: get rid of this syscall? SubProgramThunk should set a flag I
1534 # think.
1535 curr_pid = posix.getpid()
1536 # Only the main shell should bother with job control functions.
1537 return curr_pid == self.shell_pid and self.shell_tty_fd != -1
1538
1539 # TODO: This isn't a PID. This is a process group ID?
1540 #
1541 # What should the table look like?
1542 #
1543 # Do we need the last PID? I don't know why bash prints that. Probably so
1544 # you can do wait $!
1545 # wait -n waits for any node to go from job_state_e.Running to job_state_e.Done?
1546 #
1547 # And it needs a flag for CURRENT, for the implicit arg to 'fg'.
1548 # job_id is just an integer. This is sort of lame.
1549 #
1550 # [job_id, flag, pgid, job_state, node]
1551
1552 def MaybeGiveTerminal(self, pgid):
1553 # type: (int) -> None
1554 """If stdio is a TTY, move the given process group to the
1555 foreground."""
1556 if not self.Enabled():
1557 # Only call tcsetpgrp when job control is enabled.
1558 return
1559
1560 try:
1561 posix.tcsetpgrp(self.shell_tty_fd, pgid)
1562 except (IOError, OSError) as e:
1563 e_die('osh: Failed to move process group %d to foreground: %s' %
1564 (pgid, pyutil.strerror(e)))
1565
1566 def MaybeTakeTerminal(self):
1567 # type: () -> None
1568 """If stdio is a TTY, return the main shell's process group to the
1569 foreground."""
1570 self.MaybeGiveTerminal(self.shell_pgid)
1571
1572 def MaybeReturnTerminal(self):
1573 # type: () -> None
1574 """Called before the shell exits."""
1575 self.MaybeGiveTerminal(self.original_tty_pgid)
1576
1577
1578class JobList(object):
1579 """Global list of jobs, used by a few builtins."""
1580
1581 def __init__(self):
1582 # type: () -> None
1583
1584 # job_id -> Job instance
1585 self.jobs = {} # type: Dict[int, Job]
1586
1587 # pid -> Process. This is for STOP notification.
1588 self.child_procs = {} # type: Dict[int, Process]
1589 self.debug_pipelines = [] # type: List[Pipeline]
1590
1591 # Counter used to assign IDs to jobs. It is incremented every time a job
1592 # is created. Once all active jobs are done it is reset to 1. I'm not
1593 # sure if this reset behavior is mandated by POSIX, but other shells do
1594 # it, so we mimic for the sake of compatibility.
1595 self.job_id = 1
1596
1597 def AddJob(self, job):
1598 # type: (Job) -> int
1599 """Add a background job to the list.
1600
1601 A job is either a Process or Pipeline. You can resume a job with 'fg',
1602 kill it with 'kill', etc.
1603
1604 Two cases:
1605
1606 1. async jobs: sleep 5 | sleep 4 &
1607 2. stopped jobs: sleep 5; then Ctrl-Z
1608 """
1609 job_id = self.job_id
1610 self.jobs[job_id] = job
1611 job.job_id = job_id
1612 self.job_id += 1
1613 return job_id
1614
1615 def RemoveJob(self, job_id):
1616 # type: (int) -> None
1617 """Process and Pipeline can call this."""
1618 mylib.dict_erase(self.jobs, job_id)
1619
1620 if len(self.jobs) == 0:
1621 self.job_id = 1
1622
1623 def AddChildProcess(self, pid, proc):
1624 # type: (int, Process) -> None
1625 """Every child process should be added here as soon as we know its PID.
1626
1627 When the Waiter gets an EXITED or STOPPED notification, we need
1628 to know about it so 'jobs' can work.
1629 """
1630 self.child_procs[pid] = proc
1631
1632 def RemoveChildProcess(self, pid):
1633 # type: (int) -> None
1634 """Remove the child process with the given PID."""
1635 mylib.dict_erase(self.child_procs, pid)
1636
1637 if mylib.PYTHON:
1638
1639 def AddPipeline(self, pi):
1640 # type: (Pipeline) -> None
1641 """For debugging only."""
1642 self.debug_pipelines.append(pi)
1643
1644 def ProcessFromPid(self, pid):
1645 # type: (int) -> Process
1646 """For wait $PID.
1647
1648 There's no way to wait for a pipeline with a PID. That uses job
1649 syntax, e.g. %1. Not a great interface.
1650 """
1651 return self.child_procs.get(pid)
1652
1653 def GetCurrentAndPreviousJobs(self):
1654 # type: () -> Tuple[Optional[Job], Optional[Job]]
1655 """Return the "current" and "previous" jobs (AKA `%+` and `%-`).
1656
1657 See the POSIX specification for the `jobs` builtin for details:
1658 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1659
1660 IMPORTANT NOTE: This method assumes that the jobs list will not change
1661 during its execution! This assumption holds for now because we only ever
1662 update the jobs list from the main loop after WaitPid() informs us of a
1663 change. If we implement `set -b` and install a signal handler for
1664 SIGCHLD we should be careful to synchronize it with this function. The
1665 unsafety of mutating GC data structures from a signal handler should
1666 make this a non-issue, but if bugs related to this appear this note may
1667 be helpful...
1668 """
1669 # Split all active jobs by state and sort each group by decreasing job
1670 # ID to approximate newness.
1671 stopped_jobs = [] # type: List[Job]
1672 running_jobs = [] # type: List[Job]
1673 for i in xrange(0, self.job_id):
1674 job = self.jobs.get(i, None)
1675 if not job:
1676 continue
1677
1678 if job.state == job_state_e.Stopped:
1679 stopped_jobs.append(job)
1680
1681 elif job.state == job_state_e.Running:
1682 running_jobs.append(job)
1683
1684 current = None # type: Optional[Job]
1685 previous = None # type: Optional[Job]
1686 # POSIX says: If there is any suspended job, then the current job shall
1687 # be a suspended job. If there are at least two suspended jobs, then the
1688 # previous job also shall be a suspended job.
1689 #
1690 # So, we will only return running jobs from here if there are no recent
1691 # stopped jobs.
1692 if len(stopped_jobs) > 0:
1693 current = stopped_jobs.pop()
1694
1695 if len(stopped_jobs) > 0:
1696 previous = stopped_jobs.pop()
1697
1698 if len(running_jobs) > 0 and not current:
1699 current = running_jobs.pop()
1700
1701 if len(running_jobs) > 0 and not previous:
1702 previous = running_jobs.pop()
1703
1704 if not previous:
1705 previous = current
1706
1707 return current, previous
1708
1709 def GetJobWithSpec(self, job_spec):
1710 # type: (str) -> Optional[Job]
1711 """Parse the given job spec and return the matching job. If there is no
1712 matching job, this function returns None.
1713
1714 See the POSIX spec for the `jobs` builtin for details about job specs:
1715 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1716 """
1717 if job_spec in CURRENT_JOB_SPECS:
1718 current, _ = self.GetCurrentAndPreviousJobs()
1719 return current
1720
1721 if job_spec == '%-':
1722 _, previous = self.GetCurrentAndPreviousJobs()
1723 return previous
1724
1725 # TODO: Add support for job specs based on prefixes of process argv.
1726 m = util.RegexSearch(r'^%([0-9]+)$', job_spec)
1727 if m is not None:
1728 assert len(m) == 2
1729 job_id = int(m[1])
1730 if job_id in self.jobs:
1731 return self.jobs[job_id]
1732
1733 return None
1734
1735 def DisplayJobs(self, style):
1736 # type: (int) -> None
1737 """Used by the 'jobs' builtin.
1738
1739 https://pubs.opengroup.org/onlinepubs/9699919799/utilities/jobs.html
1740
1741 "By default, the jobs utility shall display the status of all stopped jobs,
1742 running background jobs and all jobs whose status has changed and have not
1743 been reported by the shell."
1744 """
1745 # NOTE: A job is a background process or pipeline.
1746 #
1747 # echo hi | wc -l -- this starts two processes. Wait for TWO
1748 # echo hi | wc -l & -- this starts a process which starts two processes
1749 # Wait for ONE.
1750 #
1751 # 'jobs -l' GROUPS the PIDs by job. It has the job number, + - indicators
1752 # for %% and %-, PID, status, and "command".
1753 #
1754 # Every component of a pipeline is on the same line with 'jobs', but
1755 # they're separated into different lines with 'jobs -l'.
1756 #
1757 # See demo/jobs-builtin.sh
1758
1759 # $ jobs -l
1760 # [1]+ 24414 Stopped sleep 5
1761 # 24415 | sleep 5
1762 # [2] 24502 Running sleep 6
1763 # 24503 | sleep 6
1764 # 24504 | sleep 5 &
1765 # [3]- 24508 Running sleep 6
1766 # 24509 | sleep 6
1767 # 24510 | sleep 5 &
1768
1769 f = mylib.Stdout()
1770 for job_id, job in iteritems(self.jobs):
1771 # Use the %1 syntax
1772 job.DisplayJob(job_id, f, style)
1773
1774 def DebugPrint(self):
1775 # type: () -> None
1776
1777 f = mylib.Stdout()
1778 f.write('\n')
1779 f.write('[process debug info]\n')
1780
1781 for pid, proc in iteritems(self.child_procs):
1782 proc.DisplayJob(-1, f, STYLE_DEFAULT)
1783 #p = ' |' if proc.parent_pipeline else ''
1784 #print('%d %7s %s%s' % (pid, _JobStateStr(proc.state), proc.thunk.UserString(), p))
1785
1786 if len(self.debug_pipelines):
1787 f.write('\n')
1788 f.write('[pipeline debug info]\n')
1789 for pi in self.debug_pipelines:
1790 pi.DebugPrint()
1791
1792 def ListRecent(self):
1793 # type: () -> None
1794 """For jobs -n, which I think is also used in the interactive
1795 prompt."""
1796 pass
1797
1798 def NumRunning(self):
1799 # type: () -> int
1800 """Return the number of running jobs.
1801
1802 Used by 'wait' and 'wait -n'.
1803 """
1804 count = 0
1805 for _, job in iteritems(self.jobs): # mycpp rewrite: from itervalues()
1806 if job.State() == job_state_e.Running:
1807 count += 1
1808 return count
1809
1810
1811# Some WaitForOne() return values
1812W1_OK = -2 # waitpid(-1) returned
1813W1_ECHILD = -3 # no processes to wait for
1814W1_AGAIN = -4 # WNOHANG was passed and there were no state changes
1815
1816
1817class Waiter(object):
1818 """A capability to wait for processes.
1819
1820 This must be a singleton (and is because CommandEvaluator is a singleton).
1821
1822 Invariants:
1823 - Every child process is registered once
1824 - Every child process is waited for
1825
1826 Canonical example of why we need a GLOBAL waiter:
1827
1828 { sleep 3; echo 'done 3'; } &
1829 { sleep 4; echo 'done 4'; } &
1830
1831 # ... do arbitrary stuff ...
1832
1833 { sleep 1; exit 1; } | { sleep 2; exit 2; }
1834
1835 Now when you do wait() after starting the pipeline, you might get a pipeline
1836 process OR a background process! So you have to distinguish between them.
1837 """
1838
1839 def __init__(self, job_list, exec_opts, signal_safe, tracer):
1840 # type: (JobList, optview.Exec, pyos.SignalSafe, dev.Tracer) -> None
1841 self.job_list = job_list
1842 self.exec_opts = exec_opts
1843 self.signal_safe = signal_safe
1844 self.tracer = tracer
1845 self.last_status = 127 # wait -n error code
1846
1847 def WaitForOne(self, waitpid_options=0):
1848 # type: (int) -> int
1849 """Wait until the next process returns (or maybe Ctrl-C).
1850
1851 Returns:
1852 One of these negative numbers:
1853 W1_ECHILD Nothing to wait for
1854 W1_OK Caller should keep waiting
1855 UNTRAPPED_SIGWINCH
1856 Or
1857 result > 0 Signal that waitpid() was interrupted with
1858
1859 In the interactive shell, we return 0 if we get a Ctrl-C, so the caller
1860 will try again.
1861
1862 Callers:
1863 wait -n -- loop until there is one fewer process (TODO)
1864 wait -- loop until there are no processes
1865 wait $! -- loop until job state is Done (process or pipeline)
1866 Process::Wait() -- loop until Process state is done
1867 Pipeline::Wait() -- loop until Pipeline state is done
1868
1869 Comparisons:
1870 bash: jobs.c waitchld() Has a special case macro(!) CHECK_WAIT_INTR for
1871 the wait builtin
1872
1873 dash: jobs.c waitproc() uses sigfillset(), sigprocmask(), etc. Runs in a
1874 loop while (gotsigchld), but that might be a hack for System V!
1875
1876 Should we have a cleaner API like named posix::wait_for_one() ?
1877
1878 wait_result =
1879 ECHILD -- nothing to wait for
1880 | Done(int pid, int status) -- process done
1881 | EINTR(bool sigint) -- may or may not retry
1882 """
1883 pid, status = pyos.WaitPid(waitpid_options)
1884 if pid == 0: # WNOHANG passed, and no state changes
1885 return W1_AGAIN
1886 elif pid < 0: # error case
1887 err_num = status
1888 #log('waitpid() error => %d %s', e.errno, pyutil.strerror(e))
1889 if err_num == ECHILD:
1890 return W1_ECHILD # nothing to wait for caller should stop
1891 elif err_num == EINTR: # Bug #858 fix
1892 #log('WaitForOne() => %d', self.trap_state.GetLastSignal())
1893 return self.signal_safe.LastSignal() # e.g. 1 for SIGHUP
1894 else:
1895 # The signature of waitpid() means this shouldn't happen
1896 raise AssertionError()
1897
1898 # All child processes are supposed to be in this dict. But this may
1899 # legitimately happen if a grandchild outlives the child (its parent).
1900 # Then it is reparented under this process, so we might receive
1901 # notification of its exit, even though we didn't start it. We can't have
1902 # any knowledge of such processes, so print a warning.
1903 if pid not in self.job_list.child_procs:
1904 print_stderr("osh: PID %d stopped, but osh didn't start it" % pid)
1905 return W1_OK
1906
1907 proc = self.job_list.child_procs[pid]
1908 if 0:
1909 self.job_list.DebugPrint()
1910
1911 if WIFSIGNALED(status):
1912 term_sig = WTERMSIG(status)
1913 status = 128 + term_sig
1914
1915 # Print newline after Ctrl-C.
1916 if term_sig == SIGINT:
1917 print('')
1918
1919 proc.WhenDone(pid, status)
1920
1921 elif WIFEXITED(status):
1922 status = WEXITSTATUS(status)
1923 #log('exit status: %s', status)
1924 proc.WhenDone(pid, status)
1925
1926 elif WIFSTOPPED(status):
1927 #status = WEXITSTATUS(status)
1928 stop_sig = WSTOPSIG(status)
1929
1930 print_stderr('')
1931 print_stderr('[PID %d] Stopped with signal %d' % (pid, stop_sig))
1932 proc.WhenStopped(stop_sig)
1933
1934 else:
1935 raise AssertionError(status)
1936
1937 self.last_status = status # for wait -n
1938 self.tracer.OnProcessEnd(pid, status)
1939 return W1_OK
1940
1941 def PollNotifications(self):
1942 # type: () -> None
1943 """
1944 Process all pending state changes.
1945 """
1946 while self.WaitForOne(waitpid_options=WNOHANG) == W1_OK:
1947 continue