From d40182262d41679cab28f6be7366cc5ff901683b Mon Sep 17 00:00:00 2001
From: Dan Melnic <dmm@fb.com>
Date: Thu, 9 Nov 2017 11:28:07 -0800
Subject: [PATCH] Remove the zerocopy write threshold support, add support for
 ENOBUFS

Summary: Remove the zerocopy write threshold support since it is a little bit confusing

Reviewed By: djwatson

Differential Revision: D6256854

fbshipit-source-id: 1c992f93d7b04c4ede2fbefebde7a7ae89de3764
---
 folly/io/async/AsyncSocket.cpp            | 52 ++++++-----------------
 folly/io/async/AsyncSocket.h              | 14 +-----
 folly/io/async/test/ZeroCopyBenchmark.cpp |  5 ---
 3 files changed, 15 insertions(+), 56 deletions(-)

diff --git a/folly/io/async/AsyncSocket.cpp b/folly/io/async/AsyncSocket.cpp
index 0a78e404..67c86bf8 100644
--- a/folly/io/async/AsyncSocket.cpp
+++ b/folly/io/async/AsyncSocket.cpp
@@ -106,7 +106,7 @@ class AsyncSocket::BytesWriteRequest : public AsyncSocket::WriteRequest {
       writeFlags |= WriteFlags::CORK;
     }
 
-    socket_->adjustZeroCopyFlags(getOps(), getOpCount(), writeFlags);
+    socket_->adjustZeroCopyFlags(writeFlags);
 
     auto writeResult = socket_->performWrite(
         getOps(), getOpCount(), writeFlags, &opsWritten_, &partialBytes_);
@@ -854,44 +854,13 @@ bool AsyncSocket::setZeroCopy(bool enable) {
   return false;
 }
 
-void AsyncSocket::setZeroCopyWriteChainThreshold(size_t threshold) {
-  zeroCopyWriteChainThreshold_ = threshold;
-}
-
 bool AsyncSocket::isZeroCopyRequest(WriteFlags flags) {
   return (zeroCopyEnabled_ && isSet(flags, WriteFlags::WRITE_MSG_ZEROCOPY));
 }
 
-void AsyncSocket::adjustZeroCopyFlags(
-    folly::IOBuf* buf,
-    folly::WriteFlags& flags) {
-  if (zeroCopyEnabled_ && zeroCopyWriteChainThreshold_ && buf &&
-      buf->isManaged()) {
-    if (buf->computeChainDataLength() >= zeroCopyWriteChainThreshold_) {
-      flags |= folly::WriteFlags::WRITE_MSG_ZEROCOPY;
-    } else {
-      flags = unSet(flags, folly::WriteFlags::WRITE_MSG_ZEROCOPY);
-    }
-  }
-}
-
-void AsyncSocket::adjustZeroCopyFlags(
-    const iovec* vec,
-    uint32_t count,
-    folly::WriteFlags& flags) {
-  if (zeroCopyEnabled_ && zeroCopyWriteChainThreshold_) {
-    count = std::min<uint32_t>(count, kIovMax);
-    size_t sum = 0;
-    for (uint32_t i = 0; i < count; ++i) {
-      const iovec* v = vec + i;
-      sum += v->iov_len;
-    }
-
-    if (sum >= zeroCopyWriteChainThreshold_) {
-      flags |= folly::WriteFlags::WRITE_MSG_ZEROCOPY;
-    } else {
-      flags = unSet(flags, folly::WriteFlags::WRITE_MSG_ZEROCOPY);
-    }
+void AsyncSocket::adjustZeroCopyFlags(folly::WriteFlags& flags) {
+  if (!zeroCopyEnabled_) {
+    flags = unSet(flags, folly::WriteFlags::WRITE_MSG_ZEROCOPY);
   }
 }
 
@@ -987,7 +956,7 @@ void AsyncSocket::writev(WriteCallback* callback,
 
 void AsyncSocket::writeChain(WriteCallback* callback, unique_ptr<IOBuf>&& buf,
                               WriteFlags flags) {
-  adjustZeroCopyFlags(buf.get(), flags);
+  adjustZeroCopyFlags(flags);
 
   constexpr size_t kSmallSizeMax = 64;
   size_t count = buf->countChainElements();
@@ -1758,8 +1727,7 @@ void AsyncSocket::handleErrMessages() noexcept {
   // supporting per-socket error queues.
   VLOG(5) << "AsyncSocket::handleErrMessages() this=" << this << ", fd=" << fd_
           << ", state=" << state_;
-  if (errMessageCallback_ == nullptr &&
-      (!zeroCopyEnabled_ || idZeroCopyBufPtrMap_.empty())) {
+  if (errMessageCallback_ == nullptr && idZeroCopyBufPtrMap_.empty()) {
     VLOG(7) << "AsyncSocket::handleErrMessages(): "
             << "no callback installed - exiting.";
     return;
@@ -2358,6 +2326,14 @@ AsyncSocket::WriteResult AsyncSocket::performWrite(
     // this bug is fixed.
     tryAgain |= (errno == ENOTCONN);
 #endif
+
+    // workaround for running with zerocopy enabled but without a proper
+    // memlock value - see ulimit -l
+    if (zeroCopyEnabled_ && (errno == ENOBUFS)) {
+      tryAgain = true;
+      zeroCopyEnabled_ = false;
+    }
+
     if (!writeResult.exception && tryAgain) {
       // TCP buffer is full; we can't write any more data right now.
       *countWritten = 0;
diff --git a/folly/io/async/AsyncSocket.h b/folly/io/async/AsyncSocket.h
index 22dbc394..17980bb7 100644
--- a/folly/io/async/AsyncSocket.h
+++ b/folly/io/async/AsyncSocket.h
@@ -505,18 +505,11 @@ class AsyncSocket : virtual public AsyncTransportWrapper {
   void setReadCB(ReadCallback* callback) override;
   ReadCallback* getReadCallback() const override;
 
-  static const size_t kDefaultZeroCopyThreshold = 0;
-
   bool setZeroCopy(bool enable);
   bool getZeroCopy() const {
     return zeroCopyEnabled_;
   }
 
-  void setZeroCopyWriteChainThreshold(size_t threshold);
-  size_t getZeroCopyWriteChainThreshold() const {
-    return zeroCopyWriteChainThreshold_;
-  }
-
   uint32_t getZeroCopyBufId() const {
     return zeroCopyBufId_;
   }
@@ -1164,11 +1157,7 @@ class AsyncSocket : virtual public AsyncTransportWrapper {
   uint32_t getNextZeroCopyBufId() {
     return zeroCopyBufId_++;
   }
-  void adjustZeroCopyFlags(folly::IOBuf* buf, folly::WriteFlags& flags);
-  void adjustZeroCopyFlags(
-      const iovec* vec,
-      uint32_t count,
-      folly::WriteFlags& flags);
+  void adjustZeroCopyFlags(folly::WriteFlags& flags);
   void addZeroCopyBuf(std::unique_ptr<folly::IOBuf>&& buf);
   void addZeroCopyBuf(folly::IOBuf* ptr);
   void setZeroCopyBuf(std::unique_ptr<folly::IOBuf>&& buf);
@@ -1240,7 +1229,6 @@ class AsyncSocket : virtual public AsyncTransportWrapper {
   bool trackEor_{false};
   bool zeroCopyEnabled_{false};
   bool zeroCopyVal_{false};
-  size_t zeroCopyWriteChainThreshold_{kDefaultZeroCopyThreshold};
 };
 #ifdef _MSC_VER
 #pragma vtordisp(pop)
diff --git a/folly/io/async/test/ZeroCopyBenchmark.cpp b/folly/io/async/test/ZeroCopyBenchmark.cpp
index 6ef917ea..f7dd422d 100644
--- a/folly/io/async/test/ZeroCopyBenchmark.cpp
+++ b/folly/io/async/test/ZeroCopyBenchmark.cpp
@@ -27,8 +27,6 @@
 
 using namespace folly;
 
-static constexpr auto const kZeroCopyThreshold = 4096;
-
 class TestAsyncSocket {
  public:
   explicit TestAsyncSocket(
@@ -79,9 +77,6 @@ class TestAsyncSocket {
     zeroCopy_ = enable;
     if (sock_) {
       sock_->setZeroCopy(zeroCopy_);
-      if (zeroCopy_) {
-        sock_->setZeroCopyWriteChainThreshold(kZeroCopyThreshold);
-      }
     }
   }
 
-- 
2.34.1