X-Git-Url: http://plrg.eecs.uci.edu/git/?p=folly.git;a=blobdiff_plain;f=folly%2FSubprocess.cpp;h=52dad4350bc91eec3f15eb0a21be3af18e3f5ca6;hp=ac8eab60d7a1423c8db0af85ef0b0998885c870b;hb=9883bc3793fc71b8d4c8d1adf6c6fe796e9131e5;hpb=217e88e6a6f011a6cc4d44490b319f919574a620 diff --git a/folly/Subprocess.cpp b/folly/Subprocess.cpp index ac8eab60..52dad435 100644 --- a/folly/Subprocess.cpp +++ b/folly/Subprocess.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2014 Facebook, Inc. + * Copyright 2017 Facebook, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,9 +24,6 @@ #include #endif #include -#include - -#include #include #include @@ -37,20 +34,35 @@ #include +#include #include #include -#include #include +#include #include #include - -extern char** environ; +#include +#include +#include +#include constexpr int kExecFailure = 127; constexpr int kChildFailure = 126; namespace folly { +ProcessReturnCode::ProcessReturnCode(ProcessReturnCode&& p) noexcept + : rawStatus_(p.rawStatus_) { + p.rawStatus_ = ProcessReturnCode::RV_NOT_STARTED; +} + +ProcessReturnCode& ProcessReturnCode::operator=(ProcessReturnCode&& p) + noexcept { + rawStatus_ = p.rawStatus_; + p.rawStatus_ = ProcessReturnCode::RV_NOT_STARTED; + return *this; +} + ProcessReturnCode::State ProcessReturnCode::state() const { if (rawStatus_ == RV_NOT_STARTED) return NOT_STARTED; if (rawStatus_ == RV_RUNNING) return RUNNING; @@ -96,30 +108,35 @@ std::string ProcessReturnCode::str() const { return to("killed by signal ", killSignal(), (coreDumped() ? " (core dumped)" : "")); } - CHECK(false); // unreached + assume_unreachable(); } CalledProcessError::CalledProcessError(ProcessReturnCode rc) - : returnCode_(rc), - what_(returnCode_.str()) { -} + : SubprocessError(rc.str()), returnCode_(rc) {} -SubprocessSpawnError::SubprocessSpawnError(const char* executable, - int errCode, - int errnoValue) - : errnoValue_(errnoValue), - what_(to(errCode == kExecFailure ? - "failed to execute " : - "error preparing to execute ", - executable, ": ", errnoStr(errnoValue))) { +static inline std::string toSubprocessSpawnErrorMessage( + char const* executable, + int errCode, + int errnoValue) { + auto prefix = errCode == kExecFailure ? "failed to execute " + : "error preparing to execute "; + return to(prefix, executable, ": ", errnoStr(errnoValue)); } +SubprocessSpawnError::SubprocessSpawnError( + const char* executable, + int errCode, + int errnoValue) + : SubprocessError( + toSubprocessSpawnErrorMessage(executable, errCode, errnoValue)), + errnoValue_(errnoValue) {} + namespace { // Copy pointers to the given strings in a format suitable for posix_spawn std::unique_ptr cloneStrings(const std::vector& s) { std::unique_ptr d(new const char*[s.size() + 1]); - for (int i = 0; i < s.size(); i++) { + for (size_t i = 0; i < s.size(); i++) { d[i] = s[i].c_str(); } d[s.size()] = nullptr; @@ -151,13 +168,13 @@ Subprocess::Options& Subprocess::Options::fd(int fd, int action) { return *this; } +Subprocess::Subprocess() {} + Subprocess::Subprocess( const std::vector& argv, const Options& options, const char* executable, - const std::vector* env) - : pid_(-1), - returnCode_(RV_NOT_STARTED) { + const std::vector* env) { if (argv.empty()) { throw std::invalid_argument("argv must not be empty"); } @@ -168,43 +185,28 @@ Subprocess::Subprocess( Subprocess::Subprocess( const std::string& cmd, const Options& options, - const std::vector* env) - : pid_(-1), - returnCode_(RV_NOT_STARTED) { + const std::vector* env) { if (options.usePath_) { throw std::invalid_argument("usePath() not allowed when running in shell"); } - const char* shell = getenv("SHELL"); - if (!shell) { - shell = "/bin/sh"; - } - std::unique_ptr argv(new const char*[4]); - argv[0] = shell; - argv[1] = "-c"; - argv[2] = cmd.c_str(); - argv[3] = nullptr; - spawn(std::move(argv), shell, options, env); + std::vector argv = {"/bin/sh", "-c", cmd}; + spawn(cloneStrings(argv), argv[0].c_str(), options, env); } Subprocess::~Subprocess() { CHECK_NE(returnCode_.state(), ProcessReturnCode::RUNNING) << "Subprocess destroyed without reaping child"; - closeAll(); } namespace { -void closeChecked(int fd) { - checkUnixError(::close(fd), "close"); -} struct ChildErrorInfo { int errCode; int errnoValue; }; -FOLLY_NORETURN void childError(int errFd, int errCode, int errnoValue); -void childError(int errFd, int errCode, int errnoValue) { +[[noreturn]] void childError(int errFd, int errCode, int errnoValue) { ChildErrorInfo info = {errCode, errnoValue}; // Write the error information over the pipe to our parent process. // We can't really do anything else if this write call fails. @@ -215,16 +217,9 @@ void childError(int errFd, int errCode, int errnoValue) { } // namespace -void Subprocess::closeAll() { - for (auto& p : pipes_) { - closeChecked(p.parentFd); - } - pipes_.clear(); -} - void Subprocess::setAllNonBlocking() { for (auto& p : pipes_) { - int fd = p.parentFd; + int fd = p.pipe.fd(); int flags = ::fcntl(fd, F_GETFL); checkUnixError(flags, "fcntl"); int r = ::fcntl(fd, F_SETFL, flags | O_NONBLOCK); @@ -245,40 +240,32 @@ void Subprocess::spawn( // Make a copy, we'll mutate options Options options(optionsIn); - // On error, close all of the pipes_ - auto pipesGuard = makeGuard([&] { - for (auto& p : this->pipes_) { - CHECK_ERR(::close(p.parentFd)); - } - }); + // On error, close all pipes_ (ignoring errors, but that seems fine here). + auto pipesGuard = makeGuard([this] { pipes_.clear(); }); // Create a pipe to use to receive error information from the child, // in case it fails before calling exec() int errFds[2]; #if FOLLY_HAVE_PIPE2 - int r = ::pipe2(errFds, O_CLOEXEC); + checkUnixError(::pipe2(errFds, O_CLOEXEC), "pipe2"); #else - int r = ::pipe(errFds); + checkUnixError(::pipe(errFds), "pipe"); #endif - checkUnixError(r, "pipe"); SCOPE_EXIT { CHECK_ERR(::close(errFds[0])); if (errFds[1] >= 0) { CHECK_ERR(::close(errFds[1])); } }; - // Ask the child to close the read end of the error pipe. - options.fdActions_[errFds[0]] = CLOSE; #if !FOLLY_HAVE_PIPE2 - r = fcntl(errFds[0], F_SETFD, FD_CLOEXEC); - checkUnixError(r, "set FD_CLOEXEC"); + // Ask the child to close the read end of the error pipe. + checkUnixError(fcntl(errFds[0], F_SETFD, FD_CLOEXEC), "set FD_CLOEXEC"); // Set the close-on-exec flag on the write side of the pipe. // This way the pipe will be closed automatically in the child if execve() // succeeds. If the exec fails the child can write error information to the // pipe. - r = fcntl(errFds[1], F_SETFD, FD_CLOEXEC); - checkUnixError(r, "set FD_CLOEXEC"); + checkUnixError(fcntl(errFds[1], F_SETFD, FD_CLOEXEC), "set FD_CLOEXEC"); #endif // Perform the actual work of setting up pipes then forking and @@ -330,6 +317,9 @@ void Subprocess::spawnInternal( // doesn't need to reset the flag on its end, as we always dup2() the fd, // and dup2() fds don't share the close-on-exec flag. #if FOLLY_HAVE_PIPE2 + // If possible, set close-on-exec atomically. Otherwise, a concurrent + // Subprocess invocation can fork() between "pipe" and "fnctl", + // causing FDs to leak. r = ::pipe2(fds, O_CLOEXEC); checkUnixError(r, "pipe2"); #else @@ -340,21 +330,21 @@ void Subprocess::spawnInternal( r = fcntl(fds[1], F_SETFD, FD_CLOEXEC); checkUnixError(r, "set FD_CLOEXEC"); #endif - PipeInfo pinfo; - pinfo.direction = p.second; + pipes_.emplace_back(); + Pipe& pipe = pipes_.back(); + pipe.direction = p.second; int cfd; if (p.second == PIPE_IN) { // Child gets reading end - pinfo.parentFd = fds[1]; + pipe.pipe = folly::File(fds[1], /*owns_fd=*/ true); cfd = fds[0]; } else { - pinfo.parentFd = fds[0]; + pipe.pipe = folly::File(fds[0], /*owns_fd=*/ true); cfd = fds[1]; } p.second = cfd; // ensure it gets dup2()ed - pinfo.childFd = p.first; + pipe.childFd = p.first; childFds.push_back(cfd); - pipes_.push_back(pinfo); } } @@ -405,7 +395,19 @@ void Subprocess::spawnInternal( // Call c_str() here, as it's not necessarily safe after fork. const char* childDir = options.childDir_.empty() ? nullptr : options.childDir_.c_str(); - pid_t pid = vfork(); + + pid_t pid; +#ifdef __linux__ + if (options.cloneFlags_) { + pid = syscall(SYS_clone, *options.cloneFlags_, 0, nullptr, nullptr); + checkUnixError(pid, errno, "clone"); + } else { +#endif + pid = vfork(); + checkUnixError(pid, errno, "vfork"); +#ifdef __linux__ + } +#endif if (pid == 0) { int errnoValue = prepareChild(options, &oldSignals, childDir); if (errnoValue != 0) { @@ -416,8 +418,6 @@ void Subprocess::spawnInternal( // If we get here, exec() failed. childError(errFd, kExecFailure, errnoValue); } - // In parent. Make sure vfork() succeeded. - checkUnixError(pid, errno, "vfork"); // Child is alive. We have to be very careful about throwing after this // point. We are inside the constructor, so if we throw the Subprocess @@ -454,12 +454,9 @@ int Subprocess::prepareChild(const Options& options, } } - // Close parent's ends of all pipes - for (auto& p : pipes_) { - if (::close(p.parentFd) == -1) { - return errno; - } - } + // We don't have to explicitly close the parent's end of all pipes, + // as they all have the FD_CLOEXEC flag set and will be closed at + // exec time. // Close all fds that we're supposed to close. for (auto& p : options.fdActions_) { @@ -488,12 +485,27 @@ int Subprocess::prepareChild(const Options& options, #if __linux__ // Opt to receive signal on parent death, if requested if (options.parentDeathSignal_ != 0) { - if (prctl(PR_SET_PDEATHSIG, options.parentDeathSignal_, 0, 0, 0) == -1) { + const auto parentDeathSignal = + static_cast(options.parentDeathSignal_); + if (prctl(PR_SET_PDEATHSIG, parentDeathSignal, 0, 0, 0) == -1) { return errno; } } #endif + if (options.processGroupLeader_) { + if (setpgrp() == -1) { + return errno; + } + } + + // The user callback comes last, so that the child is otherwise all set up. + if (options.dangerousPostForkPreExecCallback_) { + if (int error = (*options.dangerousPostForkPreExecCallback_)()) { + return error; + } + } + return 0; } @@ -501,7 +513,6 @@ int Subprocess::runChild(const char* executable, char** argv, char** env, const Options& options) const { // Now, finally, exec. - int r; if (options.usePath_) { ::execvp(executable, argv); } else { @@ -539,13 +550,18 @@ void Subprocess::readChildErrorPipe(int pfd, const char* executable) { throw SubprocessSpawnError(executable, info.errCode, info.errnoValue); } -ProcessReturnCode Subprocess::poll() { +ProcessReturnCode Subprocess::poll(struct rusage* ru) { returnCode_.enforce(ProcessReturnCode::RUNNING); DCHECK_GT(pid_, 0); int status; - pid_t found = ::waitpid(pid_, &status, WNOHANG); - checkUnixError(found, "waitpid"); + pid_t found = ::wait4(pid_, &status, WNOHANG, ru); + // The spec guarantees that EINTR does not occur with WNOHANG, so the only + // two remaining errors are ECHILD (other code reaped the child?), or + // EINVAL (cosmic rays?), both of which merit an abort: + PCHECK(found != -1) << "waitpid(" << pid_ << ", &status, WNOHANG)"; if (found != 0) { + // Though the child process had quit, this call does not close the pipes + // since its descendants may still be using them. returnCode_ = ProcessReturnCode(status); pid_ = -1; } @@ -568,7 +584,11 @@ ProcessReturnCode Subprocess::wait() { do { found = ::waitpid(pid_, &status, 0); } while (found == -1 && errno == EINTR); - checkUnixError(found, "waitpid"); + // The only two remaining errors are ECHILD (other code reaped the + // child?), or EINVAL (cosmic rays?), and both merit an abort: + PCHECK(found != -1) << "waitpid(" << pid_ << ", &status, WNOHANG)"; + // Though the child process had quit, this call does not close the pipes + // since its descendants may still be using them. DCHECK_EQ(found, pid_); returnCode_ = ProcessReturnCode(status); pid_ = -1; @@ -592,21 +612,23 @@ pid_t Subprocess::pid() const { namespace { -std::pair queueFront(const IOBufQueue& queue) { +ByteRange queueFront(const IOBufQueue& queue) { auto* p = queue.front(); - if (!p) return std::make_pair(nullptr, 0); - return io::Cursor(p).peek(); + if (!p) { + return ByteRange{}; + } + return io::Cursor(p).peekBytes(); } // fd write bool handleWrite(int fd, IOBufQueue& queue) { for (;;) { - auto p = queueFront(queue); - if (p.second == 0) { + auto b = queueFront(queue); + if (b.empty()) { return true; // EOF } - ssize_t n = writeNoInt(fd, p.first, p.second); + ssize_t n = writeNoInt(fd, b.data(), b.size()); if (n == -1 && errno == EAGAIN) { return false; } @@ -712,12 +734,14 @@ std::pair Subprocess::communicateIOBuf( void Subprocess::communicate(FdCallback readCallback, FdCallback writeCallback) { + // This serves to prevent wait() followed by communicate(), but if you + // legitimately need that, send a patch to delete this line. returnCode_.enforce(ProcessReturnCode::RUNNING); setAllNonBlocking(); std::vector fds; fds.reserve(pipes_.size()); - std::vector toClose; + std::vector toClose; // indexes into pipes_ toClose.reserve(pipes_.size()); while (!pipes_.empty()) { @@ -726,7 +750,7 @@ void Subprocess::communicate(FdCallback readCallback, for (auto& p : pipes_) { pollfd pfd; - pfd.fd = p.parentFd; + pfd.fd = p.pipe.fd(); // Yes, backwards, PIPE_IN / PIPE_OUT are defined from the // child's point of view. if (!p.enabled) { @@ -747,15 +771,16 @@ void Subprocess::communicate(FdCallback readCallback, } while (r == -1 && errno == EINTR); checkUnixError(r, "poll"); - for (int i = 0; i < pipes_.size(); ++i) { + for (size_t i = 0; i < pipes_.size(); ++i) { auto& p = pipes_[i]; - DCHECK_EQ(fds[i].fd, p.parentFd); + auto parentFd = p.pipe.fd(); + DCHECK_EQ(fds[i].fd, parentFd); short events = fds[i].revents; bool closed = false; if (events & POLLOUT) { DCHECK(!(events & POLLIN)); - if (writeCallback(p.parentFd, p.childFd)) { + if (writeCallback(parentFd, p.childFd)) { toClose.push_back(i); closed = true; } @@ -765,7 +790,7 @@ void Subprocess::communicate(FdCallback readCallback, // on) end of file if (events & (POLLIN | POLLHUP)) { DCHECK(!(events & POLLOUT)); - if (readCallback(p.parentFd, p.childFd)) { + if (readCallback(parentFd, p.childFd)) { toClose.push_back(i); closed = true; } @@ -780,7 +805,7 @@ void Subprocess::communicate(FdCallback readCallback, // Close the fds in reverse order so the indexes hold after erase() for (int idx : boost::adaptors::reverse(toClose)) { auto pos = pipes_.begin() + idx; - closeChecked(pos->parentFd); + pos->pipe.close(); // Throws on error pipes_.erase(pos); } } @@ -794,10 +819,10 @@ bool Subprocess::notificationsEnabled(int childFd) const { return pipes_[findByChildFd(childFd)].enabled; } -int Subprocess::findByChildFd(int childFd) const { +size_t Subprocess::findByChildFd(int childFd) const { auto pos = std::lower_bound( pipes_.begin(), pipes_.end(), childFd, - [] (const PipeInfo& info, int fd) { return info.childFd < fd; }); + [] (const Pipe& pipe, int fd) { return pipe.childFd < fd; }); if (pos == pipes_.end() || pos->childFd != childFd) { throw std::invalid_argument(folly::to( "child fd not found ", childFd)); @@ -807,10 +832,20 @@ int Subprocess::findByChildFd(int childFd) const { void Subprocess::closeParentFd(int childFd) { int idx = findByChildFd(childFd); - closeChecked(pipes_[idx].parentFd); + pipes_[idx].pipe.close(); // May throw pipes_.erase(pipes_.begin() + idx); } +std::vector Subprocess::takeOwnershipOfPipes() { + std::vector pipes; + for (auto& p : pipes_) { + pipes.emplace_back(p.childFd, std::move(p.pipe)); + } + // release memory + std::vector().swap(pipes_); + return pipes; +} + namespace { class Initializer { @@ -826,4 +861,3 @@ Initializer initializer; } // namespace } // namespace folly -