X-Git-Url: http://plrg.eecs.uci.edu/git/?p=folly.git;a=blobdiff_plain;f=folly%2FSubprocess.cpp;h=b2d9cca0d21c05385298052f45fb6f56dd2e3494;hp=08d93b4a8fd9ae3af58bafba27fef9aaaabd30da;hb=c1a1a5e6039c13c55597fa942ce1c01ecfe2f6a3;hpb=ce64f0f685111ac24c7a321ea56d0c3524621df1 diff --git a/folly/Subprocess.cpp b/folly/Subprocess.cpp index 08d93b4a..b2d9cca0 100644 --- a/folly/Subprocess.cpp +++ b/folly/Subprocess.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2014 Facebook, Inc. + * Copyright 2017 Facebook, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,18 +14,16 @@ * limitations under the License. */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + #include #if __linux__ #include #endif #include -#include - -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif -#include #include #include @@ -36,20 +34,35 @@ #include +#include #include #include -#include #include +#include #include #include - -extern char** environ; +#include +#include +#include +#include constexpr int kExecFailure = 127; constexpr int kChildFailure = 126; namespace folly { +ProcessReturnCode::ProcessReturnCode(ProcessReturnCode&& p) noexcept + : rawStatus_(p.rawStatus_) { + p.rawStatus_ = ProcessReturnCode::RV_NOT_STARTED; +} + +ProcessReturnCode& ProcessReturnCode::operator=(ProcessReturnCode&& p) + noexcept { + rawStatus_ = p.rawStatus_; + p.rawStatus_ = ProcessReturnCode::RV_NOT_STARTED; + return *this; +} + ProcessReturnCode::State ProcessReturnCode::state() const { if (rawStatus_ == RV_NOT_STARTED) return NOT_STARTED; if (rawStatus_ == RV_RUNNING) return RUNNING; @@ -95,7 +108,7 @@ std::string ProcessReturnCode::str() const { return to("killed by signal ", killSignal(), (coreDumped() ? " (core dumped)" : "")); } - CHECK(false); // unreached + assume_unreachable(); } CalledProcessError::CalledProcessError(ProcessReturnCode rc) @@ -118,7 +131,7 @@ namespace { // Copy pointers to the given strings in a format suitable for posix_spawn std::unique_ptr cloneStrings(const std::vector& s) { std::unique_ptr d(new const char*[s.size() + 1]); - for (int i = 0; i < s.size(); i++) { + for (size_t i = 0; i < s.size(); i++) { d[i] = s[i].c_str(); } d[s.size()] = nullptr; @@ -150,13 +163,13 @@ Subprocess::Options& Subprocess::Options::fd(int fd, int action) { return *this; } +Subprocess::Subprocess() {} + Subprocess::Subprocess( const std::vector& argv, const Options& options, const char* executable, - const std::vector* env) - : pid_(-1), - returnCode_(RV_NOT_STARTED) { + const std::vector* env) { if (argv.empty()) { throw std::invalid_argument("argv must not be empty"); } @@ -167,43 +180,28 @@ Subprocess::Subprocess( Subprocess::Subprocess( const std::string& cmd, const Options& options, - const std::vector* env) - : pid_(-1), - returnCode_(RV_NOT_STARTED) { + const std::vector* env) { if (options.usePath_) { throw std::invalid_argument("usePath() not allowed when running in shell"); } - const char* shell = getenv("SHELL"); - if (!shell) { - shell = "/bin/sh"; - } - std::unique_ptr argv(new const char*[4]); - argv[0] = shell; - argv[1] = "-c"; - argv[2] = cmd.c_str(); - argv[3] = nullptr; - spawn(std::move(argv), shell, options, env); + std::vector argv = {"/bin/sh", "-c", cmd}; + spawn(cloneStrings(argv), argv[0].c_str(), options, env); } Subprocess::~Subprocess() { CHECK_NE(returnCode_.state(), ProcessReturnCode::RUNNING) << "Subprocess destroyed without reaping child"; - closeAll(); } namespace { -void closeChecked(int fd) { - checkUnixError(::close(fd), "close"); -} struct ChildErrorInfo { int errCode; int errnoValue; }; -FOLLY_NORETURN void childError(int errFd, int errCode, int errnoValue); -void childError(int errFd, int errCode, int errnoValue) { +[[noreturn]] void childError(int errFd, int errCode, int errnoValue) { ChildErrorInfo info = {errCode, errnoValue}; // Write the error information over the pipe to our parent process. // We can't really do anything else if this write call fails. @@ -214,16 +212,9 @@ void childError(int errFd, int errCode, int errnoValue) { } // namespace -void Subprocess::closeAll() { - for (auto& p : pipes_) { - closeChecked(p.parentFd); - } - pipes_.clear(); -} - void Subprocess::setAllNonBlocking() { for (auto& p : pipes_) { - int fd = p.parentFd; + int fd = p.pipe.fd(); int flags = ::fcntl(fd, F_GETFL); checkUnixError(flags, "fcntl"); int r = ::fcntl(fd, F_SETFL, flags | O_NONBLOCK); @@ -244,32 +235,33 @@ void Subprocess::spawn( // Make a copy, we'll mutate options Options options(optionsIn); - // On error, close all of the pipes_ - auto pipesGuard = makeGuard([&] { - for (auto& p : this->pipes_) { - CHECK_ERR(::close(p.parentFd)); - } - }); + // On error, close all pipes_ (ignoring errors, but that seems fine here). + auto pipesGuard = makeGuard([this] { pipes_.clear(); }); // Create a pipe to use to receive error information from the child, // in case it fails before calling exec() int errFds[2]; - int r = ::pipe(errFds); - checkUnixError(r, "pipe"); +#if FOLLY_HAVE_PIPE2 + checkUnixError(::pipe2(errFds, O_CLOEXEC), "pipe2"); +#else + checkUnixError(::pipe(errFds), "pipe"); +#endif SCOPE_EXIT { CHECK_ERR(::close(errFds[0])); if (errFds[1] >= 0) { CHECK_ERR(::close(errFds[1])); } }; + +#if !FOLLY_HAVE_PIPE2 // Ask the child to close the read end of the error pipe. - options.fdActions_[errFds[0]] = CLOSE; + checkUnixError(fcntl(errFds[0], F_SETFD, FD_CLOEXEC), "set FD_CLOEXEC"); // Set the close-on-exec flag on the write side of the pipe. // This way the pipe will be closed automatically in the child if execve() // succeeds. If the exec fails the child can write error information to the // pipe. - r = fcntl(errFds[1], F_SETFD, FD_CLOEXEC); - checkUnixError(r, "set FD_CLOEXEC"); + checkUnixError(fcntl(errFds[1], F_SETFD, FD_CLOEXEC), "set FD_CLOEXEC"); +#endif // Perform the actual work of setting up pipes then forking and // executing the child. @@ -316,23 +308,38 @@ void Subprocess::spawnInternal( for (auto& p : options.fdActions_) { if (p.second == PIPE_IN || p.second == PIPE_OUT) { int fds[2]; + // We're setting both ends of the pipe as close-on-exec. The child + // doesn't need to reset the flag on its end, as we always dup2() the fd, + // and dup2() fds don't share the close-on-exec flag. +#if FOLLY_HAVE_PIPE2 + // If possible, set close-on-exec atomically. Otherwise, a concurrent + // Subprocess invocation can fork() between "pipe" and "fnctl", + // causing FDs to leak. + r = ::pipe2(fds, O_CLOEXEC); + checkUnixError(r, "pipe2"); +#else r = ::pipe(fds); checkUnixError(r, "pipe"); - PipeInfo pinfo; - pinfo.direction = p.second; + r = fcntl(fds[0], F_SETFD, FD_CLOEXEC); + checkUnixError(r, "set FD_CLOEXEC"); + r = fcntl(fds[1], F_SETFD, FD_CLOEXEC); + checkUnixError(r, "set FD_CLOEXEC"); +#endif + pipes_.emplace_back(); + Pipe& pipe = pipes_.back(); + pipe.direction = p.second; int cfd; if (p.second == PIPE_IN) { // Child gets reading end - pinfo.parentFd = fds[1]; + pipe.pipe = folly::File(fds[1], /*owns_fd=*/ true); cfd = fds[0]; } else { - pinfo.parentFd = fds[0]; + pipe.pipe = folly::File(fds[0], /*owns_fd=*/ true); cfd = fds[1]; } p.second = cfd; // ensure it gets dup2()ed - pinfo.childFd = p.first; + pipe.childFd = p.first; childFds.push_back(cfd); - pipes_.push_back(pinfo); } } @@ -380,9 +387,24 @@ void Subprocess::spawnInternal( CHECK_EQ(r, 0) << "pthread_sigmask: " << errnoStr(r); // shouldn't fail }; - pid_t pid = vfork(); + // Call c_str() here, as it's not necessarily safe after fork. + const char* childDir = + options.childDir_.empty() ? nullptr : options.childDir_.c_str(); + + pid_t pid; +#ifdef __linux__ + if (options.cloneFlags_) { + pid = syscall(SYS_clone, *options.cloneFlags_, 0, nullptr, nullptr); + checkUnixError(pid, errno, "clone"); + } else { +#endif + pid = vfork(); + checkUnixError(pid, errno, "vfork"); +#ifdef __linux__ + } +#endif if (pid == 0) { - int errnoValue = prepareChild(options, &oldSignals); + int errnoValue = prepareChild(options, &oldSignals, childDir); if (errnoValue != 0) { childError(errFd, kChildFailure, errnoValue); } @@ -391,8 +413,6 @@ void Subprocess::spawnInternal( // If we get here, exec() failed. childError(errFd, kExecFailure, errnoValue); } - // In parent. Make sure vfork() succeeded. - checkUnixError(pid, errno, "vfork"); // Child is alive. We have to be very careful about throwing after this // point. We are inside the constructor, so if we throw the Subprocess @@ -406,43 +426,41 @@ void Subprocess::spawnInternal( } int Subprocess::prepareChild(const Options& options, - const sigset_t* sigmask) const { + const sigset_t* sigmask, + const char* childDir) const { // While all signals are blocked, we must reset their // dispositions to default. for (int sig = 1; sig < NSIG; ++sig) { ::signal(sig, SIG_DFL); } - // Unblock signals; restore signal mask. - int r = pthread_sigmask(SIG_SETMASK, sigmask, nullptr); - if (r != 0) { - return r; // pthread_sigmask() returns an errno value - } - // Change the working directory, if one is given - if (!options.childDir_.empty()) { - r = ::chdir(options.childDir_.c_str()); - if (r == -1) { - return errno; + { + // Unblock signals; restore signal mask. + int r = pthread_sigmask(SIG_SETMASK, sigmask, nullptr); + if (r != 0) { + return r; // pthread_sigmask() returns an errno value } } - // Close parent's ends of all pipes - for (auto& p : pipes_) { - r = ::close(p.parentFd); - if (r == -1) { + // Change the working directory, if one is given + if (childDir) { + if (::chdir(childDir) == -1) { return errno; } } + // We don't have to explicitly close the parent's end of all pipes, + // as they all have the FD_CLOEXEC flag set and will be closed at + // exec time. + // Close all fds that we're supposed to close. - // Note that we're ignoring errors here, in case some of these - // fds were set to close on exec. for (auto& p : options.fdActions_) { if (p.second == CLOSE) { - ::close(p.first); - } else { - r = ::dup2(p.second, p.first); - if (r == -1) { + if (::close(p.first) == -1) { + return errno; + } + } else if (p.second != p.first) { + if (::dup2(p.second, p.first) == -1) { return errno; } } @@ -462,13 +480,27 @@ int Subprocess::prepareChild(const Options& options, #if __linux__ // Opt to receive signal on parent death, if requested if (options.parentDeathSignal_ != 0) { - r = prctl(PR_SET_PDEATHSIG, options.parentDeathSignal_, 0, 0, 0); - if (r == -1) { + const auto parentDeathSignal = + static_cast(options.parentDeathSignal_); + if (prctl(PR_SET_PDEATHSIG, parentDeathSignal, 0, 0, 0) == -1) { return errno; } } #endif + if (options.processGroupLeader_) { + if (setpgrp() == -1) { + return errno; + } + } + + // The user callback comes last, so that the child is otherwise all set up. + if (options.dangerousPostForkPreExecCallback_) { + if (int error = (*options.dangerousPostForkPreExecCallback_)()) { + return error; + } + } + return 0; } @@ -476,7 +508,6 @@ int Subprocess::runChild(const char* executable, char** argv, char** env, const Options& options) const { // Now, finally, exec. - int r; if (options.usePath_) { ::execvp(executable, argv); } else { @@ -514,13 +545,18 @@ void Subprocess::readChildErrorPipe(int pfd, const char* executable) { throw SubprocessSpawnError(executable, info.errCode, info.errnoValue); } -ProcessReturnCode Subprocess::poll() { +ProcessReturnCode Subprocess::poll(struct rusage* ru) { returnCode_.enforce(ProcessReturnCode::RUNNING); DCHECK_GT(pid_, 0); int status; - pid_t found = ::waitpid(pid_, &status, WNOHANG); - checkUnixError(found, "waitpid"); + pid_t found = ::wait4(pid_, &status, WNOHANG, ru); + // The spec guarantees that EINTR does not occur with WNOHANG, so the only + // two remaining errors are ECHILD (other code reaped the child?), or + // EINVAL (cosmic rays?), both of which merit an abort: + PCHECK(found != -1) << "waitpid(" << pid_ << ", &status, WNOHANG)"; if (found != 0) { + // Though the child process had quit, this call does not close the pipes + // since its descendants may still be using them. returnCode_ = ProcessReturnCode(status); pid_ = -1; } @@ -543,7 +579,11 @@ ProcessReturnCode Subprocess::wait() { do { found = ::waitpid(pid_, &status, 0); } while (found == -1 && errno == EINTR); - checkUnixError(found, "waitpid"); + // The only two remaining errors are ECHILD (other code reaped the + // child?), or EINVAL (cosmic rays?), and both merit an abort: + PCHECK(found != -1) << "waitpid(" << pid_ << ", &status, WNOHANG)"; + // Though the child process had quit, this call does not close the pipes + // since its descendants may still be using them. DCHECK_EQ(found, pid_); returnCode_ = ProcessReturnCode(status); pid_ = -1; @@ -567,21 +607,23 @@ pid_t Subprocess::pid() const { namespace { -std::pair queueFront(const IOBufQueue& queue) { +ByteRange queueFront(const IOBufQueue& queue) { auto* p = queue.front(); - if (!p) return std::make_pair(nullptr, 0); - return io::Cursor(p).peek(); + if (!p) { + return ByteRange{}; + } + return io::Cursor(p).peekBytes(); } // fd write bool handleWrite(int fd, IOBufQueue& queue) { for (;;) { - auto p = queueFront(queue); - if (p.second == 0) { + auto b = queueFront(queue); + if (b.empty()) { return true; // EOF } - ssize_t n = writeNoInt(fd, p.first, p.second); + ssize_t n = writeNoInt(fd, b.data(), b.size()); if (n == -1 && errno == EAGAIN) { return false; } @@ -687,12 +729,14 @@ std::pair Subprocess::communicateIOBuf( void Subprocess::communicate(FdCallback readCallback, FdCallback writeCallback) { + // This serves to prevent wait() followed by communicate(), but if you + // legitimately need that, send a patch to delete this line. returnCode_.enforce(ProcessReturnCode::RUNNING); setAllNonBlocking(); std::vector fds; fds.reserve(pipes_.size()); - std::vector toClose; + std::vector toClose; // indexes into pipes_ toClose.reserve(pipes_.size()); while (!pipes_.empty()) { @@ -701,7 +745,7 @@ void Subprocess::communicate(FdCallback readCallback, for (auto& p : pipes_) { pollfd pfd; - pfd.fd = p.parentFd; + pfd.fd = p.pipe.fd(); // Yes, backwards, PIPE_IN / PIPE_OUT are defined from the // child's point of view. if (!p.enabled) { @@ -722,15 +766,16 @@ void Subprocess::communicate(FdCallback readCallback, } while (r == -1 && errno == EINTR); checkUnixError(r, "poll"); - for (int i = 0; i < pipes_.size(); ++i) { + for (size_t i = 0; i < pipes_.size(); ++i) { auto& p = pipes_[i]; - DCHECK_EQ(fds[i].fd, p.parentFd); + auto parentFd = p.pipe.fd(); + DCHECK_EQ(fds[i].fd, parentFd); short events = fds[i].revents; bool closed = false; if (events & POLLOUT) { DCHECK(!(events & POLLIN)); - if (writeCallback(p.parentFd, p.childFd)) { + if (writeCallback(parentFd, p.childFd)) { toClose.push_back(i); closed = true; } @@ -740,7 +785,7 @@ void Subprocess::communicate(FdCallback readCallback, // on) end of file if (events & (POLLIN | POLLHUP)) { DCHECK(!(events & POLLOUT)); - if (readCallback(p.parentFd, p.childFd)) { + if (readCallback(parentFd, p.childFd)) { toClose.push_back(i); closed = true; } @@ -755,7 +800,7 @@ void Subprocess::communicate(FdCallback readCallback, // Close the fds in reverse order so the indexes hold after erase() for (int idx : boost::adaptors::reverse(toClose)) { auto pos = pipes_.begin() + idx; - closeChecked(pos->parentFd); + pos->pipe.close(); // Throws on error pipes_.erase(pos); } } @@ -769,10 +814,10 @@ bool Subprocess::notificationsEnabled(int childFd) const { return pipes_[findByChildFd(childFd)].enabled; } -int Subprocess::findByChildFd(int childFd) const { +size_t Subprocess::findByChildFd(int childFd) const { auto pos = std::lower_bound( pipes_.begin(), pipes_.end(), childFd, - [] (const PipeInfo& info, int fd) { return info.childFd < fd; }); + [] (const Pipe& pipe, int fd) { return pipe.childFd < fd; }); if (pos == pipes_.end() || pos->childFd != childFd) { throw std::invalid_argument(folly::to( "child fd not found ", childFd)); @@ -782,10 +827,20 @@ int Subprocess::findByChildFd(int childFd) const { void Subprocess::closeParentFd(int childFd) { int idx = findByChildFd(childFd); - closeChecked(pipes_[idx].parentFd); + pipes_[idx].pipe.close(); // May throw pipes_.erase(pipes_.begin() + idx); } +std::vector Subprocess::takeOwnershipOfPipes() { + std::vector pipes; + for (auto& p : pipes_) { + pipes.emplace_back(p.childFd, std::move(p.pipe)); + } + // release memory + std::vector().swap(pipes_); + return pipes; +} + namespace { class Initializer { @@ -801,4 +856,3 @@ Initializer initializer; } // namespace } // namespace folly -