2 * Copyright 2013 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
18 * Subprocess library, modeled after Python's subprocess module
19 * (http://docs.python.org/2/library/subprocess.html)
21 * This library defines one class (Subprocess) which represents a child
22 * process. Subprocess has two constructors: one that takes a vector<string>
23 * and executes the given executable without using the shell, and one
24 * that takes a string and executes the given command using the shell.
25 * Subprocess allows you to redirect the child's standard input, standard
26 * output, and standard error to/from child descriptors in the parent,
27 * or to create communication pipes between the child and the parent.
29 * The simplest example is a thread-safe version of the system() library
31 * Subprocess(cmd).wait();
32 * which executes the command using the default shell and waits for it
33 * to complete, returning the exit status.
35 * A thread-safe version of popen() (type="r", to read from the child):
36 * Subprocess proc(cmd, Subprocess::pipeStdout());
37 * // read from proc.stdout()
40 * A thread-safe version of popen() (type="w", to write from the child):
41 * Subprocess proc(cmd, Subprocess::pipeStdin());
42 * // write to proc.stdin()
45 * If you want to redirect both stdin and stdout to pipes, you can, but
46 * note that you're subject to a variety of deadlocks. You'll want to use
47 * nonblocking I/O; look at the implementation of communicate() for an example.
49 * communicate() is a way to communicate to a child via its standard input,
50 * standard output, and standard error. It buffers everything in memory,
51 * so it's not great for large amounts of data (or long-running processes),
52 * but it insulates you from the deadlocks mentioned above.
54 #ifndef FOLLY_SUBPROCESS_H_
55 #define FOLLY_SUBPROCESS_H_
57 #include <sys/types.h>
65 #include <boost/container/flat_map.hpp>
66 #include <boost/operators.hpp>
67 #include <boost/noncopyable.hpp>
69 #include "folly/io/IOBufQueue.h"
70 #include "folly/MapUtil.h"
71 #include "folly/Portability.h"
72 #include "folly/Range.h"
77 * Class to wrap a process return code.
80 class ProcessReturnCode {
81 friend class Subprocess;
91 * Process state. One of:
92 * NOT_STARTED: process hasn't been started successfully
93 * RUNNING: process is currently running
94 * EXITED: process exited (successfully or not)
95 * KILLED: process was killed by a signal.
100 * Helper wrappers around state().
102 bool notStarted() const { return state() == NOT_STARTED; }
103 bool running() const { return state() == RUNNING; }
104 bool exited() const { return state() == EXITED; }
105 bool killed() const { return state() == KILLED; }
108 * Exit status. Only valid if state() == EXITED; throws otherwise.
110 int exitStatus() const;
113 * Signal that caused the process's termination. Only valid if
114 * state() == KILLED; throws otherwise.
116 int killSignal() const;
119 * Was a core file generated? Only valid if state() == KILLED; throws
122 bool coreDumped() const;
125 * String representation; one of
128 * "exited with status <status>"
129 * "killed by signal <signal>"
130 * "killed by signal <signal> (core dumped)"
132 std::string str() const;
135 * Helper function to enforce a precondition based on this.
136 * Throws std::logic_error if in an unexpected state.
138 void enforce(State state) const;
140 explicit ProcessReturnCode(int rv) : rawStatus_(rv) { }
141 static constexpr int RV_NOT_STARTED = -2;
142 static constexpr int RV_RUNNING = -1;
148 * Base exception thrown by the Subprocess methods.
150 class SubprocessError : public std::exception {};
153 * Exception thrown by *Checked methods of Subprocess.
155 class CalledProcessError : public SubprocessError {
157 explicit CalledProcessError(ProcessReturnCode rc);
158 ~CalledProcessError() throw() { }
159 const char* what() const throw() FOLLY_OVERRIDE { return what_.c_str(); }
160 ProcessReturnCode returnCode() const { return returnCode_; }
162 ProcessReturnCode returnCode_;
167 * Exception thrown if the subprocess cannot be started.
169 class SubprocessSpawnError : public SubprocessError {
171 SubprocessSpawnError(const char* executable, int errCode, int errnoValue);
172 ~SubprocessSpawnError() throw() {}
173 const char* what() const throw() FOLLY_OVERRIDE { return what_.c_str(); }
174 int errnoValue() const { return errnoValue_; }
184 class Subprocess : private boost::noncopyable {
186 static const int CLOSE = -1;
187 static const int PIPE = -2;
188 static const int PIPE_IN = -3;
189 static const int PIPE_OUT = -4;
192 * Class representing various options: file descriptor behavior, and
193 * whether to use $PATH for searching for the executable,
195 * By default, we don't use $PATH, file descriptors are closed if
196 * the close-on-exec flag is set (fcntl FD_CLOEXEC) and inherited
199 class Options : private boost::orable<Options> {
200 friend class Subprocess;
203 : closeOtherFds_(false),
205 parentDeathSignal_(0) {
209 * Change action for file descriptor fd.
211 * "action" may be another file descriptor number (dup2()ed before the
212 * child execs), or one of CLOSE, PIPE_IN, and PIPE_OUT.
214 * CLOSE: close the file descriptor in the child
215 * PIPE_IN: open a pipe *from* the child
216 * PIPE_OUT: open a pipe *to* the child
218 * PIPE is a shortcut; same as PIPE_IN for stdin (fd 0), same as
219 * PIPE_OUT for stdout (fd 1) or stderr (fd 2), and an error for
220 * other file descriptors.
222 Options& fd(int fd, int action);
225 * Shortcut to change the action for standard input.
227 Options& stdin(int action) { return fd(0, action); }
230 * Shortcut to change the action for standard output.
232 Options& stdout(int action) { return fd(1, action); }
235 * Shortcut to change the action for standard error.
236 * Note that stderr(1) will redirect the standard error to the same
237 * file descriptor as standard output; the equivalent of bash's "2>&1"
239 Options& stderr(int action) { return fd(2, action); }
242 * Close all other fds (other than standard input, output, error,
243 * and file descriptors explicitly specified with fd()).
245 * This is potentially slow; it's generally a better idea to
246 * set the close-on-exec flag on all file descriptors that shouldn't
247 * be inherited by the child.
249 * Even with this option set, standard input, output, and error are
250 * not closed; use stdin(CLOSE), stdout(CLOSE), stderr(CLOSE) if you
253 Options& closeOtherFds() { closeOtherFds_ = true; return *this; }
256 * Use the search path ($PATH) when searching for the executable.
258 Options& usePath() { usePath_ = true; return *this; }
261 * Child will receive a signal when the parent exits.
263 Options& parentDeathSignal(int sig) {
264 parentDeathSignal_ = sig;
269 * Helpful way to combine Options.
271 Options& operator|=(const Options& other);
274 typedef boost::container::flat_map<int, int> FdMap;
278 int parentDeathSignal_;
281 static Options pipeStdin() { return Options().stdin(PIPE); }
282 static Options pipeStdout() { return Options().stdout(PIPE); }
283 static Options pipeStderr() { return Options().stderr(PIPE); }
286 * Create a subprocess from the given arguments. argv[0] must be listed.
287 * If not-null, executable must be the actual executable
288 * being used (otherwise it's the same as argv[0]).
290 * If env is not-null, it must contain name=value strings to be used
291 * as the child's environment; otherwise, we inherit the environment
292 * from the parent. env must be null if options.usePath is set.
295 const std::vector<std::string>& argv,
296 const Options& options = Options(),
297 const char* executable = nullptr,
298 const std::vector<std::string>* env = nullptr);
302 * Create a subprocess run as a shell command (as shell -c 'command')
304 * The shell to use is taken from the environment variable $SHELL,
305 * or /bin/sh if $SHELL is unset.
308 const std::string& cmd,
309 const Options& options = Options(),
310 const std::vector<std::string>* env = nullptr);
313 * Append all data, close the stdin (to-child) fd, and read all data,
314 * except that this is done in a safe manner to prevent deadlocking.
316 * If writeStdin() is given in flags, the process must have been opened with
319 * If readStdout() is given in flags, the first returned value will be the
320 * value read from the child's stdout; the child must have been opened with
323 * If readStderr() is given in flags, the second returned value will be the
324 * value read from the child's stderr; the child must have been opened with
327 * Note that communicate() returns when all pipes to/from the child are
328 * closed; the child might stay alive after that, so you must still wait().
330 * communicateIOBuf uses IOBufQueue for buffering (which has the advantage
331 * that it won't try to allocate all data at once). communicate
332 * uses strings for simplicity.
334 class CommunicateFlags : private boost::orable<CommunicateFlags> {
335 friend class Subprocess;
338 : writeStdin_(false), readStdout_(false), readStderr_(false) { }
339 CommunicateFlags& writeStdin() { writeStdin_ = true; return *this; }
340 CommunicateFlags& readStdout() { readStdout_ = true; return *this; }
341 CommunicateFlags& readStderr() { readStderr_ = true; return *this; }
343 CommunicateFlags& operator|=(const CommunicateFlags& other);
350 static CommunicateFlags writeStdin() {
351 return CommunicateFlags().writeStdin();
353 static CommunicateFlags readStdout() {
354 return CommunicateFlags().readStdout();
356 static CommunicateFlags readStderr() {
357 return CommunicateFlags().readStderr();
360 std::pair<IOBufQueue, IOBufQueue> communicateIOBuf(
361 const CommunicateFlags& flags = readStdout(),
362 IOBufQueue data = IOBufQueue());
364 std::pair<std::string, std::string> communicate(
365 const CommunicateFlags& flags = readStdout(),
366 StringPiece data = StringPiece());
369 * Communicate with the child until all pipes to/from the child are closed.
371 * readCallback(pfd, cfd) will be called whenever there's data available
372 * on any pipe *from* the child (PIPE_OUT). pfd is the file descriptor
373 * in the parent (that you use to read from); cfd is the file descriptor
374 * in the child (used for identifying the stream; 1 = child's standard
375 * output, 2 = child's standard error, etc)
377 * writeCallback(pfd, cfd) will be called whenever a pipe *to* the child is
378 * writable (PIPE_IN). pfd is the file descriptor in the parent (that you
379 * use to write to); cfd is the file descriptor in the child (used for
380 * identifying the stream; 0 = child's standard input, etc)
382 * The read and write callbacks must read from / write to pfd and return
383 * false during normal operation or true at end-of-file;
384 * communicate() will then close the pipe. Note that pfd is
385 * nonblocking, so be prepared for read() / write() to return -1 and
386 * set errno to EAGAIN (in which case you should return false).
388 * NOTE that you MUST consume all data passed to readCallback (or return
389 * true, which will close the pipe, possibly sending SIGPIPE to the child or
390 * making its writes fail with EPIPE), and you MUST write to a writable pipe
391 * (or return true, which will close the pipe). To do otherwise is an
392 * error. You must do this even for pipes you are not interested in.
394 * Note that communicate() returns when all pipes to/from the child are
395 * closed; the child might stay alive after that, so you must still wait().
397 * Most users won't need to use this; the simpler version of communicate
398 * (which buffers data in memory) will probably work fine.
400 typedef std::function<bool(int, int)> FdCallback;
401 void communicate(FdCallback readCallback, FdCallback writeCallback);
404 * Return the child's pid, or -1 if the child wasn't successfully spawned
405 * or has already been wait()ed upon.
410 * Return the child's status (as per wait()) if the process has already
411 * been waited on, -1 if the process is still running, or -2 if the process
412 * hasn't been successfully started. NOTE that this does not poll, but
413 * returns the status stored in the Subprocess object.
415 ProcessReturnCode returnCode() const { return returnCode_; }
418 * Poll the child's status and return it, return -1 if the process
419 * is still running. NOTE that it is illegal to call poll again after
420 * poll indicated that the process has terminated, or to call poll on a
421 * process that hasn't been successfully started (the constructor threw an
424 ProcessReturnCode poll();
427 * Poll the child's status. If the process is still running, return false.
428 * Otherwise, return true if the process exited with status 0 (success),
429 * or throw CalledProcessError if the process exited with a non-zero status.
434 * Wait for the process to terminate and return its status.
435 * Similarly to poll, it is illegal to call wait after the process
436 * has already been reaped or if the process has not successfully started.
438 ProcessReturnCode wait();
441 * Wait for the process to terminate, throw if unsuccessful.
446 * Set all pipes from / to child non-blocking. communicate() does
449 void setAllNonBlocking();
452 * Get parent file descriptor corresponding to the given file descriptor
453 * in the child. Throws if childFd isn't a pipe (PIPE_IN / PIPE_OUT).
454 * Do not close() the return file descriptor; use closeParentFd, below.
456 int parentFd(int childFd) const {
457 return pipes_[findByChildFd(childFd)].parentFd;
459 int stdin() const { return parentFd(0); }
460 int stdout() const { return parentFd(1); }
461 int stderr() const { return parentFd(2); }
464 * Close the parent file descriptor given a file descriptor in the child.
466 void closeParentFd(int childFd);
469 * Send a signal to the child. Shortcuts for the commonly used Unix
472 void sendSignal(int signal);
473 void terminate() { sendSignal(SIGTERM); }
474 void kill() { sendSignal(SIGKILL); }
477 static const int RV_RUNNING = ProcessReturnCode::RV_RUNNING;
478 static const int RV_NOT_STARTED = ProcessReturnCode::RV_NOT_STARTED;
480 // spawn() sets up a pipe to read errors from the child,
481 // then calls spawnInternal() to do the bulk of the work. Once
482 // spawnInternal() returns it reads the error pipe to see if the child
483 // encountered any errors.
485 std::unique_ptr<const char*[]> argv,
486 const char* executable,
487 const Options& options,
488 const std::vector<std::string>* env);
490 std::unique_ptr<const char*[]> argv,
491 const char* executable,
493 const std::vector<std::string>* env,
496 // Actions to run in child.
497 // Note that this runs after vfork(), so tread lightly.
498 // Returns 0 on success, or an errno value on failure.
499 int prepareChild(const Options& options, const sigset_t* sigmask) const;
500 int runChild(const char* executable, char** argv, char** env,
501 const Options& options) const;
504 * Read from the error pipe, and throw SubprocessSpawnError if the child
505 * failed before calling exec().
507 void readChildErrorPipe(int pfd, const char* executable);
510 * Close all file descriptors.
514 // return index in pipes_
515 int findByChildFd(int childFd) const;
518 ProcessReturnCode returnCode_;
520 // The number of pipes between parent and child is assumed to be small,
521 // so we're happy with a vector here, even if it means linear erase.
523 struct PipeInfo : private boost::totally_ordered<PipeInfo> {
526 int direction; // one of PIPE_IN / PIPE_OUT
527 bool operator<(const PipeInfo& other) const {
528 return childFd < other.childFd;
530 bool operator==(const PipeInfo& other) const {
531 return childFd == other.childFd;
534 std::vector<PipeInfo> pipes_;
537 inline Subprocess::Options& Subprocess::Options::operator|=(
538 const Subprocess::Options& other) {
539 if (this == &other) return *this;
541 for (auto& p : other.fdActions_) {
542 fdActions_[p.first] = p.second;
544 closeOtherFds_ |= other.closeOtherFds_;
545 usePath_ |= other.usePath_;
549 inline Subprocess::CommunicateFlags& Subprocess::CommunicateFlags::operator|=(
550 const Subprocess::CommunicateFlags& other) {
551 if (this == &other) return *this;
552 writeStdin_ |= other.writeStdin_;
553 readStdout_ |= other.readStdout_;
554 readStderr_ |= other.readStderr_;
560 #endif /* FOLLY_SUBPROCESS_H_ */