/*
- * Copyright 2014 Facebook, Inc.
+ * Copyright 2017 Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* limitations under the License.
*/
+#ifndef __STDC_LIMIT_MACROS
#define __STDC_LIMIT_MACROS
+#endif
#include <folly/io/IOBuf.h>
+#include <cassert>
+#include <cstdint>
+#include <cstdlib>
+#include <stdexcept>
+
#include <folly/Conv.h>
#include <folly/Likely.h>
#include <folly/Malloc.h>
#include <folly/Memory.h>
#include <folly/ScopeGuard.h>
-#include <folly/SpookyHashV2.h>
+#include <folly/hash/SpookyHashV2.h>
#include <folly/io/Cursor.h>
-#include <stdexcept>
-#include <assert.h>
-#include <stdint.h>
-#include <stdlib.h>
-
using std::unique_ptr;
namespace {
}
}
-} // unnamed namespace
+} // namespace
namespace folly {
struct IOBuf::HeapPrefix {
- HeapPrefix(uint16_t flg)
- : magic(kHeapMagic),
- flags(flg) {}
+ explicit HeapPrefix(uint16_t flg) : magic(kHeapMagic), flags(flg) {}
~HeapPrefix() {
// Reset magic to 0 on destruction. This is solely for debugging purposes
// to help catch bugs where someone tries to use HeapStorage after it has
HeapStorage hs;
SharedInfo shared;
- MaxAlign align;
+ folly::max_align_t align;
};
IOBuf::SharedInfo::SharedInfo()
void* IOBuf::operator new(size_t size) {
size_t fullSize = offsetof(HeapStorage, buf) + size;
auto* storage = static_cast<HeapStorage*>(malloc(fullSize));
- // operator new is not allowed to return NULL
+ // operator new is not allowed to return nullptr
if (UNLIKELY(storage == nullptr)) {
throw std::bad_alloc();
}
return &(storage->buf);
}
-void* IOBuf::operator new(size_t size, void* ptr) {
- return ptr;
-}
+void* IOBuf::operator new(size_t /* size */, void* ptr) { return ptr; }
void IOBuf::operator delete(void* ptr) {
auto* storageAddr = static_cast<uint8_t*>(ptr) - offsetof(HeapStorage, buf);
DCHECK_EQ((flags & freeFlags), freeFlags);
while (true) {
- uint16_t newFlags = (flags & ~freeFlags);
+ uint16_t newFlags = uint16_t(flags & ~freeFlags);
if (newFlags == 0) {
// The storage space is now unused. Free it.
storage->prefix.HeapPrefix::~HeapPrefix();
}
}
-void IOBuf::freeInternalBuf(void* buf, void* userData) {
+void IOBuf::freeInternalBuf(void* /* buf */, void* userData) {
auto* storage = static_cast<HeapStorage*>(userData);
releaseStorage(storage, kDataInUse);
}
data_ = buf_;
}
-IOBuf::IOBuf(CopyBufferOp op, const void* buf, uint64_t size,
- uint64_t headroom, uint64_t minTailroom)
- : IOBuf(CREATE, headroom + size + minTailroom) {
+IOBuf::IOBuf(CopyBufferOp /* op */,
+ const void* buf,
+ uint64_t size,
+ uint64_t headroom,
+ uint64_t minTailroom)
+ : IOBuf(CREATE, headroom + size + minTailroom) {
advance(headroom);
- memcpy(writableData(), buf, size);
- append(size);
+ if (size > 0) {
+ assert(buf != nullptr);
+ memcpy(writableData(), buf, size);
+ append(size);
+ }
}
IOBuf::IOBuf(CopyBufferOp op, ByteRange br,
uint8_t* bufAddr = reinterpret_cast<uint8_t*>(&storage->align);
uint8_t* storageEnd = reinterpret_cast<uint8_t*>(storage) + mallocSize;
- size_t actualCapacity = storageEnd - bufAddr;
+ size_t actualCapacity = size_t(storageEnd - bufAddr);
unique_ptr<IOBuf> ret(new (&storage->hs.buf) IOBuf(
InternalConstructor(), packFlagsAndSharedInfo(0, &storage->shared),
bufAddr, actualCapacity, bufAddr, 0));
}
unique_ptr<IOBuf> IOBuf::createSeparate(uint64_t capacity) {
- return make_unique<IOBuf>(CREATE, capacity);
+ return std::make_unique<IOBuf>(CREATE, capacity);
}
unique_ptr<IOBuf> IOBuf::createChain(
//
// Note that we always pass freeOnError as false to the constructor.
// If the constructor throws we'll handle it below. (We have to handle
- // allocation failures from make_unique too.)
- return make_unique<IOBuf>(TAKE_OWNERSHIP, buf, capacity, length,
- freeFn, userData, false);
+ // allocation failures from std::make_unique too.)
+ return std::make_unique<IOBuf>(
+ TAKE_OWNERSHIP, buf, capacity, length, freeFn, userData, false);
} catch (...) {
takeOwnershipError(freeOnError, buf, freeFn, userData);
throw;
}
unique_ptr<IOBuf> IOBuf::wrapBuffer(const void* buf, uint64_t capacity) {
- return make_unique<IOBuf>(WRAP_BUFFER, buf, capacity);
+ return std::make_unique<IOBuf>(WRAP_BUFFER, buf, capacity);
+}
+
+IOBuf IOBuf::wrapBufferAsValue(const void* buf, uint64_t capacity) {
+ return IOBuf(WrapBufferOp::WRAP_BUFFER, buf, capacity);
}
IOBuf::IOBuf() noexcept {
}
-IOBuf::IOBuf(IOBuf&& other) noexcept {
- *this = std::move(other);
+IOBuf::IOBuf(IOBuf&& other) noexcept
+ : data_(other.data_),
+ buf_(other.buf_),
+ length_(other.length_),
+ capacity_(other.capacity_),
+ flagsAndSharedInfo_(other.flagsAndSharedInfo_) {
+ // Reset other so it is a clean state to be destroyed.
+ other.data_ = nullptr;
+ other.buf_ = nullptr;
+ other.length_ = 0;
+ other.capacity_ = 0;
+ other.flagsAndSharedInfo_ = 0;
+
+ // If other was part of the chain, assume ownership of the rest of its chain.
+ // (It's only valid to perform move assignment on the head of a chain.)
+ if (other.next_ != &other) {
+ next_ = other.next_;
+ next_->prev_ = this;
+ other.next_ = &other;
+
+ prev_ = other.prev_;
+ prev_->next_ = this;
+ other.prev_ = &other;
+ }
+
+ // Sanity check to make sure that other is in a valid state to be destroyed.
+ DCHECK_EQ(other.prev_, &other);
+ DCHECK_EQ(other.next_, &other);
+}
+
+IOBuf::IOBuf(const IOBuf& other) {
+ *this = other.cloneAsValue();
}
IOBuf::IOBuf(InternalConstructor,
return *this;
}
+IOBuf& IOBuf::operator=(const IOBuf& other) {
+ if (this != &other) {
+ *this = IOBuf(other);
+ }
+ return *this;
+}
+
bool IOBuf::empty() const {
const IOBuf* current = this;
do {
}
unique_ptr<IOBuf> IOBuf::clone() const {
- unique_ptr<IOBuf> ret = make_unique<IOBuf>();
- cloneInto(*ret);
- return ret;
+ return std::make_unique<IOBuf>(cloneAsValue());
}
unique_ptr<IOBuf> IOBuf::cloneOne() const {
- unique_ptr<IOBuf> ret = make_unique<IOBuf>();
- cloneOneInto(*ret);
- return ret;
+ return std::make_unique<IOBuf>(cloneOneAsValue());
+}
+
+unique_ptr<IOBuf> IOBuf::cloneCoalesced() const {
+ return std::make_unique<IOBuf>(cloneCoalescedAsValue());
}
-void IOBuf::cloneInto(IOBuf& other) const {
- IOBuf tmp;
- cloneOneInto(tmp);
+IOBuf IOBuf::cloneAsValue() const {
+ auto tmp = cloneOneAsValue();
for (IOBuf* current = next_; current != this; current = current->next_) {
tmp.prependChain(current->cloneOne());
}
- other = std::move(tmp);
+ return tmp;
}
-void IOBuf::cloneOneInto(IOBuf& other) const {
- SharedInfo* info = sharedInfo();
- if (info) {
+IOBuf IOBuf::cloneOneAsValue() const {
+ if (SharedInfo* info = sharedInfo()) {
setFlags(kFlagMaybeShared);
- }
- other = IOBuf(InternalConstructor(),
- flagsAndSharedInfo_, buf_, capacity_,
- data_, length_);
- if (info) {
info->refcount.fetch_add(1, std::memory_order_acq_rel);
}
+ return IOBuf(
+ InternalConstructor(),
+ flagsAndSharedInfo_,
+ buf_,
+ capacity_,
+ data_,
+ length_);
+}
+
+IOBuf IOBuf::cloneCoalescedAsValue() const {
+ if (!isChained()) {
+ return cloneOneAsValue();
+ }
+ // Coalesce into newBuf
+ const uint64_t newLength = computeChainDataLength();
+ const uint64_t newHeadroom = headroom();
+ const uint64_t newTailroom = prev()->tailroom();
+ const uint64_t newCapacity = newLength + newHeadroom + newTailroom;
+ IOBuf newBuf{CREATE, newCapacity};
+ newBuf.advance(newHeadroom);
+
+ auto current = this;
+ do {
+ if (current->length() > 0) {
+ DCHECK_NOTNULL(current->data());
+ DCHECK_LE(current->length(), newBuf.tailroom());
+ memcpy(newBuf.writableTail(), current->data(), current->length());
+ newBuf.append(current->length());
+ }
+ current = current->next();
+ } while (current != this);
+
+ DCHECK_EQ(newLength, newBuf.length());
+ DCHECK_EQ(newHeadroom, newBuf.headroom());
+ DCHECK_LE(newTailroom, newBuf.tailroom());
+
+ return newBuf;
}
void IOBuf::unshareOneSlow() {
// Maintain the same amount of headroom. Since we maintained the same
// minimum capacity we also maintain at least the same amount of tailroom.
uint64_t headlen = headroom();
- memcpy(buf + headlen, data_, length_);
+ if (length_ > 0) {
+ assert(data_ != nullptr);
+ memcpy(buf + headlen, data_, length_);
+ }
// Release our reference on the old buffer
decrementRefcount();
coalesceSlow();
}
+void IOBuf::markExternallyShared() {
+ IOBuf* current = this;
+ do {
+ current->markExternallySharedOne();
+ current = current->next_;
+ } while (current != this);
+}
+
+void IOBuf::makeManagedChained() {
+ assert(isChained());
+
+ IOBuf* current = this;
+ while (true) {
+ current->makeManagedOne();
+ current = current->next_;
+ if (current == this) {
+ break;
+ }
+ }
+}
+
void IOBuf::coalesceSlow() {
// coalesceSlow() should only be called if we are part of a chain of multiple
// IOBufs. The caller should have already verified this.
IOBuf* current = this;
size_t remaining = newLength;
do {
- assert(current->length_ <= remaining);
- remaining -= current->length_;
- memcpy(p, current->data_, current->length_);
- p += current->length_;
+ if (current->length_ > 0) {
+ assert(current->length_ <= remaining);
+ assert(current->data_ != nullptr);
+ remaining -= current->length_;
+ memcpy(p, current->data_, current->length_);
+ p += current->length_;
+ }
current = current->next_;
} while (current != end);
assert(remaining == 0);
return;
}
- size_t newAllocatedCapacity = goodExtBufferSize(newCapacity);
+ size_t newAllocatedCapacity = 0;
uint8_t* newBuffer = nullptr;
uint64_t newHeadroom = 0;
uint64_t oldHeadroom = headroom();
SharedInfo* info = sharedInfo();
if (info && (info->freeFn == nullptr) && length_ != 0 &&
oldHeadroom >= minHeadroom) {
+ size_t headSlack = oldHeadroom - minHeadroom;
+ newAllocatedCapacity = goodExtBufferSize(newCapacity + headSlack);
if (usingJEMalloc()) {
- size_t headSlack = oldHeadroom - minHeadroom;
// We assume that tailroom is more useful and more important than
// headroom (not least because realloc / xallocx allow us to grow the
// buffer at the tail, but not at the head) So, if we have more headroom
if (xallocx(p, newAllocatedCapacity, 0, 0) == newAllocatedCapacity) {
newBuffer = static_cast<uint8_t*>(p);
newHeadroom = oldHeadroom;
- newAllocatedCapacity = newAllocatedCapacity;
}
// if xallocx failed, do nothing, fall back to malloc/memcpy/free
}
// None of the previous reallocation strategies worked (or we're using
// an internal buffer). malloc/copy/free.
if (newBuffer == nullptr) {
+ newAllocatedCapacity = goodExtBufferSize(newCapacity);
void* p = malloc(newAllocatedCapacity);
if (UNLIKELY(p == nullptr)) {
throw std::bad_alloc();
}
newBuffer = static_cast<uint8_t*>(p);
- memcpy(newBuffer + minHeadroom, data_, length_);
+ if (length_ > 0) {
+ assert(data_ != nullptr);
+ memcpy(newBuffer + minHeadroom, data_, length_);
+ }
if (sharedInfo()) {
freeExtBuffer();
}
uint8_t* infoStart = (buf + mallocSize) - sizeof(SharedInfo);
SharedInfo* sharedInfo = new(infoStart) SharedInfo;
- *capacityReturn = infoStart - buf;
+ *capacityReturn = uint64_t(infoStart - buf);
*infoReturn = sharedInfo;
}
} while (p != this);
}
+size_t IOBuf::fillIov(struct iovec* iov, size_t len) const {
+ IOBuf const* p = this;
+ size_t i = 0;
+ while (i < len) {
+ // some code can get confused by empty iovs, so skip them
+ if (p->length() > 0) {
+ iov[i].iov_base = const_cast<uint8_t*>(p->data());
+ iov[i].iov_len = p->length();
+ i++;
+ }
+ p = p->next();
+ if (p == this) {
+ return i;
+ }
+ }
+ return 0;
+}
+
size_t IOBufHash::operator()(const IOBuf& buf) const {
folly::hash::SpookyHashV2 hasher;
hasher.Init(0, 0);
io::Cursor cursor(&buf);
for (;;) {
- auto p = cursor.peek();
- if (p.second == 0) {
+ auto b = cursor.peekBytes();
+ if (b.empty()) {
break;
}
- hasher.Update(p.first, p.second);
- cursor.skip(p.second);
+ hasher.Update(b.data(), b.size());
+ cursor.skip(b.size());
}
uint64_t h1;
uint64_t h2;
io::Cursor ca(&a);
io::Cursor cb(&b);
for (;;) {
- auto pa = ca.peek();
- auto pb = cb.peek();
- if (pa.second == 0 && pb.second == 0) {
+ auto ba = ca.peekBytes();
+ auto bb = cb.peekBytes();
+ if (ba.empty() && bb.empty()) {
return true;
- } else if (pa.second == 0 || pb.second == 0) {
+ } else if (ba.empty() || bb.empty()) {
return false;
}
- size_t n = std::min(pa.second, pb.second);
- DCHECK_GT(n, 0);
- if (memcmp(pa.first, pb.first, n)) {
+ size_t n = std::min(ba.size(), bb.size());
+ DCHECK_GT(n, 0u);
+ if (memcmp(ba.data(), bb.data(), n)) {
return false;
}
ca.skip(n);
}
}
-} // folly
+} // namespace folly
projects / folly.git / blobdiff