From: Woo Xie Date: Fri, 26 Jun 2015 03:11:55 +0000 (-0700) Subject: opt proxygen with newly added OpenSSL functions X-Git-Tag: v0.48.0~8 X-Git-Url: http://plrg.eecs.uci.edu/git/?a=commitdiff_plain;h=06557e1f06b3eaea99e0f32406e5a6989354b8bf;p=folly.git opt proxygen with newly added OpenSSL functions Summary: this taks advantage of SSL_write_iovec and SSL_read_buf functions to improve CPU. This eliminates one malloc+memcpy+free operation for all HTTPS traffic, and save up to two for small packets. It saves 10~15% CPU https://fburl.com/99681397 https://fburl.com/99427544 Reviewed By: @djwatson Differential Revision: D1993764 --- diff --git a/folly/io/async/AsyncSSLSocket.cpp b/folly/io/async/AsyncSSLSocket.cpp index 7be1983e..4c9348a7 100644 --- a/folly/io/async/AsyncSSLSocket.cpp +++ b/folly/io/async/AsyncSSLSocket.cpp @@ -1070,6 +1070,22 @@ AsyncSSLSocket::handleConnect() noexcept { AsyncSocket::handleInitialReadWrite(); } +void AsyncSSLSocket::prepareReadBuffer(void** buf, size_t* buflen) noexcept { + CHECK(readCallback_); +#ifdef SSL_MODE_MOVE_BUFFER_OWNERSHIP + // turn on the buffer movable in openssl + if (!isBufferMovable_ && readCallback_->isBufferMovable()) { + SSL_set_mode(ssl_, SSL_get_mode(ssl_) | SSL_MODE_MOVE_BUFFER_OWNERSHIP); + *buf = nullptr; + *buflen = 0; + isBufferMovable_ = true; + return; + } +#endif + // buf is necessary for SSLSocket without SSL_MODE_MOVE_BUFFER_OWNERSHIP + readCallback_->getReadBuffer(buf, buflen); +} + void AsyncSSLSocket::handleRead() noexcept { VLOG(5) << "AsyncSSLSocket::handleRead() this=" << this << ", fd=" << fd_ @@ -1096,13 +1112,25 @@ AsyncSSLSocket::handleRead() noexcept { } ssize_t -AsyncSSLSocket::performRead(void* buf, size_t buflen) { +AsyncSSLSocket::performRead(void** buf, size_t* buflen, size_t* offset) { + VLOG(4) << "AsyncSSLSocket::performRead() this=" << this + << ", buf=" << *buf << ", buflen=" << *buflen; + if (sslState_ == STATE_UNENCRYPTED) { - return AsyncSocket::performRead(buf, buflen); + return AsyncSocket::performRead(buf, buflen, offset); } errno = 0; - ssize_t bytes = SSL_read(ssl_, buf, buflen); + ssize_t bytes = 0; + if (!isBufferMovable_) { + bytes = SSL_read(ssl_, *buf, *buflen); + } +#ifdef SSL_MODE_MOVE_BUFFER_OWNERSHIP + else { + bytes = SSL_read_buf(ssl_, buf, (int *) offset, (int *) buflen); + } +#endif + if (server_ && renegotiateAttempted_) { LOG(ERROR) << "AsyncSSLSocket(fd=" << fd_ << ", state=" << int(state_) << ", sslstate=" << sslState_ << ", events=" << eventFlags_ diff --git a/folly/io/async/AsyncSSLSocket.h b/folly/io/async/AsyncSSLSocket.h index 8523bdb6..85c3536e 100644 --- a/folly/io/async/AsyncSSLSocket.h +++ b/folly/io/async/AsyncSSLSocket.h @@ -673,7 +673,7 @@ class AsyncSSLSocket : public virtual AsyncSocket { // Inherit event notification methods from AsyncSocket except // the following. - + void prepareReadBuffer(void** buf, size_t* buflen) noexcept override; void handleRead() noexcept override; void handleWrite() noexcept override; void handleAccept() noexcept; @@ -687,7 +687,7 @@ class AsyncSSLSocket : public virtual AsyncSocket { void handleInitialReadWrite() noexcept override {} int interpretSSLError(int rc, int error); - ssize_t performRead(void* buf, size_t buflen) override; + ssize_t performRead(void** buf, size_t* buflen, size_t* offset) override; ssize_t performWrite(const iovec* vec, uint32_t count, WriteFlags flags, uint32_t* countWritten, uint32_t* partialWritten) override; diff --git a/folly/io/async/AsyncSocket.cpp b/folly/io/async/AsyncSocket.cpp index 4ab1cdea..9df09eb2 100644 --- a/folly/io/async/AsyncSocket.cpp +++ b/folly/io/async/AsyncSocket.cpp @@ -1231,8 +1231,11 @@ void AsyncSocket::ioReady(uint16_t events) noexcept { } } -ssize_t AsyncSocket::performRead(void* buf, size_t buflen) { - ssize_t bytes = recv(fd_, buf, buflen, MSG_DONTWAIT); +ssize_t AsyncSocket::performRead(void** buf, size_t* buflen, size_t* offset) { + VLOG(5) << "AsyncSocket::performRead() this=" << this + << ", buf=" << *buf << ", buflen=" << *buflen; + + ssize_t bytes = recv(fd_, *buf, *buflen, MSG_DONTWAIT); if (bytes < 0) { if (errno == EAGAIN || errno == EWOULDBLOCK) { // No more data to read right now. @@ -1246,6 +1249,12 @@ ssize_t AsyncSocket::performRead(void* buf, size_t buflen) { } } +void AsyncSocket::prepareReadBuffer(void** buf, size_t* buflen) noexcept { + // no matter what, buffer should be preapared for non-ssl socket + CHECK(readCallback_); + readCallback_->getReadBuffer(buf, buflen); +} + void AsyncSocket::handleRead() noexcept { VLOG(5) << "AsyncSocket::handleRead() this=" << this << ", fd=" << fd_ << ", state=" << state_; @@ -1277,9 +1286,10 @@ void AsyncSocket::handleRead() noexcept { while (readCallback_ && eventBase_ == originalEventBase) { // Get the buffer to read into. void* buf = nullptr; - size_t buflen = 0; + size_t buflen = 0, offset = 0; try { - readCallback_->getReadBuffer(&buf, &buflen); + prepareReadBuffer(&buf, &buflen); + VLOG(5) << "prepareReadBuffer() buf=" << buf << ", buflen=" << buflen; } catch (const AsyncSocketException& ex) { return failRead(__func__, ex); } catch (const std::exception& ex) { @@ -1294,7 +1304,7 @@ void AsyncSocket::handleRead() noexcept { "non-exception type"); return failRead(__func__, ex); } - if (buf == nullptr || buflen == 0) { + if (!isBufferMovable_ && (buf == nullptr || buflen == 0)) { AsyncSocketException ex(AsyncSocketException::BAD_ARGS, "ReadCallback::getReadBuffer() returned " "empty buffer"); @@ -1302,9 +1312,23 @@ void AsyncSocket::handleRead() noexcept { } // Perform the read - ssize_t bytesRead = performRead(buf, buflen); + ssize_t bytesRead = performRead(&buf, &buflen, &offset); + VLOG(4) << "this=" << this << ", AsyncSocket::handleRead() got " + << bytesRead << " bytes"; if (bytesRead > 0) { - readCallback_->readDataAvailable(bytesRead); + if (!isBufferMovable_) { + readCallback_->readDataAvailable(bytesRead); + } else { + CHECK(kOpenSslModeMoveBufferOwnership); + VLOG(5) << "this=" << this << ", AsyncSocket::handleRead() got " + << "buf=" << buf << ", " << bytesRead << "/" << buflen + << ", offset=" << offset; + auto readBuf = folly::IOBuf::takeOwnership(buf, buflen); + readBuf->trimStart(offset); + readBuf->trimEnd(buflen - offset - bytesRead); + readCallback_->readBufferAvailable(std::move(readBuf)); + } + // Fall through and continue around the loop if the read // completely filled the available buffer. // Note that readCallback_ may have been uninstalled or changed inside diff --git a/folly/io/async/AsyncSocket.h b/folly/io/async/AsyncSocket.h index 5ea024ce..c0525bbe 100644 --- a/folly/io/async/AsyncSocket.h +++ b/folly/io/async/AsyncSocket.h @@ -636,6 +636,7 @@ class AsyncSocket : virtual public AsyncTransportWrapper { void ioReady(uint16_t events) noexcept; virtual void checkForImmediateRead() noexcept; virtual void handleInitialReadWrite() noexcept; + virtual void prepareReadBuffer(void** buf, size_t* buflen) noexcept; virtual void handleRead() noexcept; virtual void handleWrite() noexcept; virtual void handleConnect() noexcept; @@ -651,7 +652,7 @@ class AsyncSocket : virtual public AsyncTransportWrapper { * READ_ERROR on error, or READ_BLOCKING if the operation will * block. */ - virtual ssize_t performRead(void* buf, size_t buflen); + virtual ssize_t performRead(void** buf, size_t* buflen, size_t* offset); /** * Populate an iovec array from an IOBuf and attempt to write it. @@ -762,6 +763,7 @@ class AsyncSocket : virtual public AsyncTransportWrapper { ShutdownSocketSet* shutdownSocketSet_; size_t appBytesReceived_; ///< Num of bytes received from socket size_t appBytesWritten_; ///< Num of bytes written to socket + bool isBufferMovable_{false}; }; diff --git a/folly/io/async/AsyncTransport.h b/folly/io/async/AsyncTransport.h index a166db50..b4ab7721 100644 --- a/folly/io/async/AsyncTransport.h +++ b/folly/io/async/AsyncTransport.h @@ -23,6 +23,16 @@ #include #include +#include + +constexpr bool kOpenSslModeMoveBufferOwnership = +#ifdef SSL_MODE_MOVE_BUFFER_OWNERSHIP + true +#else + false +#endif +; + namespace folly { class AsyncSocketException; @@ -376,8 +386,53 @@ class AsyncTransportWrapper : virtual public AsyncTransport { * * @param len The number of bytes placed in the buffer. */ + virtual void readDataAvailable(size_t len) noexcept = 0; + /** + * When data becomes available, isBufferMovable() will be invoked to figure + * out which API will be used, readBufferAvailable() or + * readDataAvailable(). If isBufferMovable() returns true, that means + * ReadCallback supports the IOBuf ownership transfer and + * readBufferAvailable() will be used. Otherwise, not. + + * By default, isBufferMovable() always return false. If + * readBufferAvailable() is implemented and to be invoked, You should + * overwrite isBufferMovable() and return true in the inherited class. + * + * This method allows the AsyncSocket/AsyncSSLSocket do buffer allocation by + * itself until data becomes available. Compared with the pre/post buffer + * allocation in getReadBuffer()/readDataAvailabe(), readBufferAvailable() + * has two advantages. First, this can avoid memcpy. E.g., in + * AsyncSSLSocket, the decrypted data was copied from the openssl internal + * buffer to the readbuf buffer. With the buffer ownership transfer, the + * internal buffer can be directly "moved" to ReadCallback. Second, the + * memory allocation can be more precise. The reason is + * AsyncSocket/AsyncSSLSocket can allocate the memory of precise size + * because they have more context about the available data than + * ReadCallback. Think about the getReadBuffer() pre-allocate 4072 bytes + * buffer, but the available data is always 16KB (max OpenSSL record size). + */ + + virtual bool isBufferMovable() noexcept { + return false; + } + + /** + * readBufferAvailable() will be invoked when data has been successfully + * read. + * + * Note that only either readBufferAvailable() or readDataAvailable() will + * be invoked according to the return value of isBufferMovable(). The timing + * and aftereffect of readBufferAvailable() are the same as + * readDataAvailable() + * + * @param readBuf The unique pointer of read buffer. + */ + + virtual void readBufferAvailable(std::unique_ptr readBuf) + noexcept {}; + /** * readEOF() will be invoked when the transport is closed. *