2 * Copyright 2014 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #define __STDC_LIMIT_MACROS
19 #include "folly/io/IOBuf.h"
21 #include "folly/Conv.h"
22 #include "folly/Likely.h"
23 #include "folly/Malloc.h"
24 #include "folly/Memory.h"
25 #include "folly/ScopeGuard.h"
32 using std::unique_ptr;
38 // This memory segment contains an IOBuf that is still in use
40 // This memory segment contains buffer data that is still in use
45 // When create() is called for buffers less than kDefaultCombinedBufSize,
46 // we allocate a single combined memory segment for the IOBuf and the data
47 // together. See the comments for createCombined()/createSeparate() for more
50 // (The size of 1k is largely just a guess here. We could could probably do
51 // benchmarks of real applications to see if adjusting this number makes a
52 // difference. Callers that know their exact use case can also explicitly
53 // call createCombined() or createSeparate().)
54 kDefaultCombinedBufSize = 1024
57 // Helper function for IOBuf::takeOwnership()
58 void takeOwnershipError(bool freeOnError, void* buf,
59 folly::IOBuf::FreeFunction freeFn,
69 freeFn(buf, userData);
71 // The user's free function is not allowed to throw.
72 // (We are already in the middle of throwing an exception, so
73 // we cannot let this exception go unhandled.)
78 } // unnamed namespace
82 struct IOBuf::HeapPrefix {
83 HeapPrefix(uint16_t flg)
87 // Reset magic to 0 on destruction. This is solely for debugging purposes
88 // to help catch bugs where someone tries to use HeapStorage after it has
94 std::atomic<uint16_t> flags;
97 struct IOBuf::HeapStorage {
99 // The IOBuf is last in the HeapStorage object.
100 // This way operator new will work even if allocating a subclass of IOBuf
101 // that requires more space.
105 struct IOBuf::HeapFullStorage {
111 IOBuf::SharedInfo::SharedInfo()
114 // Use relaxed memory ordering here. Since we are creating a new SharedInfo,
115 // no other threads should be referring to it yet.
116 refcount.store(1, std::memory_order_relaxed);
119 IOBuf::SharedInfo::SharedInfo(FreeFunction fn, void* arg)
122 // Use relaxed memory ordering here. Since we are creating a new SharedInfo,
123 // no other threads should be referring to it yet.
124 refcount.store(1, std::memory_order_relaxed);
127 void* IOBuf::operator new(size_t size) {
128 size_t fullSize = offsetof(HeapStorage, buf) + size;
129 auto* storage = static_cast<HeapStorage*>(malloc(fullSize));
130 // operator new is not allowed to return NULL
131 if (UNLIKELY(storage == nullptr)) {
132 throw std::bad_alloc();
135 new (&storage->prefix) HeapPrefix(kIOBufInUse);
136 return &(storage->buf);
139 void* IOBuf::operator new(size_t size, void* ptr) {
143 void IOBuf::operator delete(void* ptr) {
144 auto* storageAddr = static_cast<uint8_t*>(ptr) - offsetof(HeapStorage, buf);
145 auto* storage = reinterpret_cast<HeapStorage*>(storageAddr);
146 releaseStorage(storage, kIOBufInUse);
149 void IOBuf::releaseStorage(HeapStorage* storage, uint16_t freeFlags) {
150 CHECK_EQ(storage->prefix.magic, static_cast<uint16_t>(kHeapMagic));
152 // Use relaxed memory order here. If we are unlucky and happen to get
153 // out-of-date data the compare_exchange_weak() call below will catch
154 // it and load new data with memory_order_acq_rel.
155 auto flags = storage->prefix.flags.load(std::memory_order_acquire);
156 DCHECK_EQ((flags & freeFlags), freeFlags);
159 uint16_t newFlags = (flags & ~freeFlags);
161 // The storage space is now unused. Free it.
162 storage->prefix.HeapPrefix::~HeapPrefix();
167 // This storage segment still contains portions that are in use.
168 // Just clear the flags specified in freeFlags for now.
169 auto ret = storage->prefix.flags.compare_exchange_weak(
170 flags, newFlags, std::memory_order_acq_rel);
172 // We successfully updated the flags.
176 // We failed to update the flags. Some other thread probably updated them
177 // and cleared some of the other bits. Continue around the loop to see if
178 // we are the last user now, or if we need to try updating the flags again.
182 void IOBuf::freeInternalBuf(void* buf, void* userData) {
183 auto* storage = static_cast<HeapStorage*>(userData);
184 releaseStorage(storage, kDataInUse);
187 IOBuf::IOBuf(CreateOp, uint32_t capacity)
193 type_(kExtAllocated) {
194 allocExtBuffer(capacity, &buf_, &sharedInfo_, &capacity_);
198 IOBuf::IOBuf(CopyBufferOp op, const void* buf, uint32_t size,
199 uint32_t headroom, uint32_t minTailroom)
200 : IOBuf(CREATE, headroom + size + minTailroom) {
202 memcpy(writableData(), buf, size);
206 IOBuf::IOBuf(CopyBufferOp op, ByteRange br,
207 uint32_t headroom, uint32_t minTailroom)
208 : IOBuf(op, br.data(), br.size(), headroom, minTailroom) {
211 unique_ptr<IOBuf> IOBuf::create(uint32_t capacity) {
212 // For smaller-sized buffers, allocate the IOBuf, SharedInfo, and the buffer
213 // all with a single allocation.
215 // We don't do this for larger buffers since it can be wasteful if the user
216 // needs to reallocate the buffer but keeps using the same IOBuf object.
217 // In this case we can't free the data space until the IOBuf is also
218 // destroyed. Callers can explicitly call createCombined() or
219 // createSeparate() if they know their use case better, and know if they are
220 // likely to reallocate the buffer later.
221 if (capacity <= kDefaultCombinedBufSize) {
222 return createCombined(capacity);
224 return createSeparate(capacity);
227 unique_ptr<IOBuf> IOBuf::createCombined(uint32_t capacity) {
228 // To save a memory allocation, allocate space for the IOBuf object, the
229 // SharedInfo struct, and the data itself all with a single call to malloc().
230 size_t requiredStorage = offsetof(HeapFullStorage, align) + capacity;
231 size_t mallocSize = goodMallocSize(requiredStorage);
232 auto* storage = static_cast<HeapFullStorage*>(malloc(mallocSize));
234 new (&storage->hs.prefix) HeapPrefix(kIOBufInUse | kDataInUse);
235 new (&storage->shared) SharedInfo(freeInternalBuf, storage);
237 uint8_t* bufAddr = reinterpret_cast<uint8_t*>(&storage->align);
238 uint8_t* storageEnd = reinterpret_cast<uint8_t*>(storage) + mallocSize;
239 size_t actualCapacity = storageEnd - bufAddr;
240 unique_ptr<IOBuf> ret(new (&storage->hs.buf) IOBuf(
241 kCombinedAlloc, 0, bufAddr, actualCapacity,
242 bufAddr, 0, &storage->shared));
246 unique_ptr<IOBuf> IOBuf::createSeparate(uint32_t capacity) {
247 return make_unique<IOBuf>(CREATE, capacity);
250 unique_ptr<IOBuf> IOBuf::createChain(
251 size_t totalCapacity, uint32_t maxBufCapacity) {
252 unique_ptr<IOBuf> out = create(
253 std::min(totalCapacity, size_t(maxBufCapacity)));
254 size_t allocatedCapacity = out->capacity();
256 while (allocatedCapacity < totalCapacity) {
257 unique_ptr<IOBuf> newBuf = create(
258 std::min(totalCapacity - allocatedCapacity, size_t(maxBufCapacity)));
259 allocatedCapacity += newBuf->capacity();
260 out->prependChain(std::move(newBuf));
266 IOBuf::IOBuf(TakeOwnershipOp, void* buf, uint32_t capacity, uint32_t length,
267 FreeFunction freeFn, void* userData,
271 data_(static_cast<uint8_t*>(buf)),
272 buf_(static_cast<uint8_t*>(buf)),
275 flags_(kFlagFreeSharedInfo),
276 type_(kExtUserSupplied) {
278 sharedInfo_ = new SharedInfo(freeFn, userData);
280 takeOwnershipError(freeOnError, buf, freeFn, userData);
285 unique_ptr<IOBuf> IOBuf::takeOwnership(void* buf, uint32_t capacity,
291 // TODO: We could allocate the IOBuf object and SharedInfo all in a single
292 // memory allocation. We could use the existing HeapStorage class, and
293 // define a new kSharedInfoInUse flag. We could change our code to call
294 // releaseStorage(kFlagFreeSharedInfo) when this kFlagFreeSharedInfo,
295 // rather than directly calling delete.
297 // Note that we always pass freeOnError as false to the constructor.
298 // If the constructor throws we'll handle it below. (We have to handle
299 // allocation failures from make_unique too.)
300 return make_unique<IOBuf>(TAKE_OWNERSHIP, buf, capacity, length,
301 freeFn, userData, false);
303 takeOwnershipError(freeOnError, buf, freeFn, userData);
308 IOBuf::IOBuf(WrapBufferOp, const void* buf, uint32_t capacity)
309 : IOBuf(kExtUserOwned, kFlagUserOwned,
310 // We cast away the const-ness of the buffer here.
311 // This is okay since IOBuf users must use unshare() to create a copy
312 // of this buffer before writing to the buffer.
313 static_cast<uint8_t*>(const_cast<void*>(buf)), capacity,
314 static_cast<uint8_t*>(const_cast<void*>(buf)), capacity,
318 IOBuf::IOBuf(WrapBufferOp op, ByteRange br)
319 : IOBuf(op, br.data(), folly::to<uint32_t>(br.size())) {
322 unique_ptr<IOBuf> IOBuf::wrapBuffer(const void* buf, uint32_t capacity) {
323 return make_unique<IOBuf>(WRAP_BUFFER, buf, capacity);
326 IOBuf::IOBuf() noexcept {
329 IOBuf::IOBuf(IOBuf&& other) noexcept {
330 *this = std::move(other);
333 IOBuf::IOBuf(ExtBufTypeEnum type,
339 SharedInfo* sharedInfo)
348 sharedInfo_(sharedInfo) {
350 assert(data + length <= buf + capacity);
351 assert(static_cast<bool>(flags & kFlagUserOwned) ==
352 (sharedInfo == NULL));
356 // Destroying an IOBuf destroys the entire chain.
357 // Users of IOBuf should only explicitly delete the head of any chain.
358 // The other elements in the chain will be automatically destroyed.
359 while (next_ != this) {
360 // Since unlink() returns unique_ptr() and we don't store it,
361 // it will automatically delete the unlinked element.
362 (void)next_->unlink();
368 IOBuf& IOBuf::operator=(IOBuf&& other) noexcept {
369 // If we are part of a chain, delete the rest of the chain.
370 while (next_ != this) {
371 // Since unlink() returns unique_ptr() and we don't store it,
372 // it will automatically delete the unlinked element.
373 (void)next_->unlink();
376 // Decrement our refcount on the current buffer
379 // Take ownership of the other buffer's data
382 length_ = other.length_;
383 capacity_ = other.capacity_;
384 flags_ = other.flags_;
386 sharedInfo_ = other.sharedInfo_;
387 // Reset other so it is a clean state to be destroyed.
388 other.data_ = nullptr;
389 other.buf_ = nullptr;
392 other.flags_ = kFlagUserOwned;
393 other.type_ = kExtUserOwned;
394 other.sharedInfo_ = nullptr;
396 // If other was part of the chain, assume ownership of the rest of its chain.
397 // (It's only valid to perform move assignment on the head of a chain.)
398 if (other.next_ != &other) {
401 other.next_ = &other;
405 other.prev_ = &other;
408 // Sanity check to make sure that other is in a valid state to be destroyed.
409 DCHECK_EQ(other.prev_, &other);
410 DCHECK_EQ(other.next_, &other);
411 DCHECK(other.flags_ & kFlagUserOwned);
416 bool IOBuf::empty() const {
417 const IOBuf* current = this;
419 if (current->length() != 0) {
422 current = current->next_;
423 } while (current != this);
427 uint32_t IOBuf::countChainElements() const {
428 uint32_t numElements = 1;
429 for (IOBuf* current = next_; current != this; current = current->next_) {
435 uint64_t IOBuf::computeChainDataLength() const {
436 uint64_t fullLength = length_;
437 for (IOBuf* current = next_; current != this; current = current->next_) {
438 fullLength += current->length_;
443 void IOBuf::prependChain(unique_ptr<IOBuf>&& iobuf) {
444 // Take ownership of the specified IOBuf
445 IOBuf* other = iobuf.release();
447 // Remember the pointer to the tail of the other chain
448 IOBuf* otherTail = other->prev_;
450 // Hook up prev_->next_ to point at the start of the other chain,
451 // and other->prev_ to point at prev_
452 prev_->next_ = other;
453 other->prev_ = prev_;
455 // Hook up otherTail->next_ to point at us,
456 // and prev_ to point back at otherTail,
457 otherTail->next_ = this;
461 unique_ptr<IOBuf> IOBuf::clone() const {
462 unique_ptr<IOBuf> newHead(cloneOne());
464 for (IOBuf* current = next_; current != this; current = current->next_) {
465 newHead->prependChain(current->cloneOne());
471 unique_ptr<IOBuf> IOBuf::cloneOne() const {
473 flags_ |= kFlagMaybeShared;
475 unique_ptr<IOBuf> iobuf(new IOBuf(static_cast<ExtBufTypeEnum>(type_),
476 flags_, buf_, capacity_,
480 sharedInfo_->refcount.fetch_add(1, std::memory_order_acq_rel);
485 void IOBuf::unshareOneSlow() {
486 // Allocate a new buffer for the data
488 SharedInfo* sharedInfo;
489 uint32_t actualCapacity;
490 allocExtBuffer(capacity_, &buf, &sharedInfo, &actualCapacity);
493 // Maintain the same amount of headroom. Since we maintained the same
494 // minimum capacity we also maintain at least the same amount of tailroom.
495 uint32_t headlen = headroom();
496 memcpy(buf + headlen, data_, length_);
498 // Release our reference on the old buffer
500 // Make sure kFlagUserOwned, kFlagMaybeShared, and kFlagFreeSharedInfo
504 // Update the buffer pointers to point to the new buffer
505 data_ = buf + headlen;
507 sharedInfo_ = sharedInfo;
510 void IOBuf::unshareChained() {
511 // unshareChained() should only be called if we are part of a chain of
512 // multiple IOBufs. The caller should have already verified this.
515 IOBuf* current = this;
517 if (current->isSharedOne()) {
518 // we have to unshare
522 current = current->next_;
523 if (current == this) {
524 // None of the IOBufs in the chain are shared,
525 // so return without doing anything
530 // We have to unshare. Let coalesceSlow() do the work.
534 void IOBuf::coalesceSlow() {
535 // coalesceSlow() should only be called if we are part of a chain of multiple
536 // IOBufs. The caller should have already verified this.
539 // Compute the length of the entire chain
540 uint64_t newLength = 0;
543 newLength += end->length_;
545 } while (end != this);
547 coalesceAndReallocate(newLength, end);
548 // We should be only element left in the chain now
549 DCHECK(!isChained());
552 void IOBuf::coalesceSlow(size_t maxLength) {
553 // coalesceSlow() should only be called if we are part of a chain of multiple
554 // IOBufs. The caller should have already verified this.
556 DCHECK_LT(length_, maxLength);
558 // Compute the length of the entire chain
559 uint64_t newLength = 0;
562 newLength += end->length_;
564 if (newLength >= maxLength) {
568 throw std::overflow_error("attempted to coalesce more data than "
573 coalesceAndReallocate(newLength, end);
574 // We should have the requested length now
575 DCHECK_GE(length_, maxLength);
578 void IOBuf::coalesceAndReallocate(size_t newHeadroom,
581 size_t newTailroom) {
582 uint64_t newCapacity = newLength + newHeadroom + newTailroom;
583 if (newCapacity > UINT32_MAX) {
584 throw std::overflow_error("IOBuf chain too large to coalesce");
587 // Allocate space for the coalesced buffer.
588 // We always convert to an external buffer, even if we happened to be an
589 // internal buffer before.
592 uint32_t actualCapacity;
593 allocExtBuffer(newCapacity, &newBuf, &newInfo, &actualCapacity);
595 // Copy the data into the new buffer
596 uint8_t* newData = newBuf + newHeadroom;
597 uint8_t* p = newData;
598 IOBuf* current = this;
599 size_t remaining = newLength;
601 assert(current->length_ <= remaining);
602 remaining -= current->length_;
603 memcpy(p, current->data_, current->length_);
604 p += current->length_;
605 current = current->next_;
606 } while (current != end);
607 assert(remaining == 0);
609 // Point at the new buffer
612 // Make sure kFlagUserOwned, kFlagMaybeShared, and kFlagFreeSharedInfo
616 capacity_ = actualCapacity;
617 type_ = kExtAllocated;
619 sharedInfo_ = newInfo;
623 // Separate from the rest of our chain.
624 // Since we don't store the unique_ptr returned by separateChain(),
625 // this will immediately delete the returned subchain.
627 (void)separateChain(next_, current->prev_);
631 void IOBuf::decrementRefcount() {
632 // Externally owned buffers don't have a SharedInfo object and aren't managed
633 // by the reference count
634 if (flags_ & kFlagUserOwned) {
635 assert(sharedInfo_ == nullptr);
639 // Decrement the refcount
640 uint32_t newcnt = sharedInfo_->refcount.fetch_sub(
641 1, std::memory_order_acq_rel);
642 // Note that fetch_sub() returns the value before we decremented.
643 // If it is 1, we were the only remaining user; if it is greater there are
644 // still other users.
649 // We were the last user. Free the buffer
652 // Free the SharedInfo if it was allocated separately.
654 // This is only used by takeOwnership().
656 // To avoid this special case handling in decrementRefcount(), we could have
657 // takeOwnership() set a custom freeFn() that calls the user's free function
658 // then frees the SharedInfo object. (This would require that
659 // takeOwnership() store the user's free function with its allocated
660 // SharedInfo object.) However, handling this specially with a flag seems
661 // like it shouldn't be problematic.
662 if (flags_ & kFlagFreeSharedInfo) {
667 void IOBuf::reserveSlow(uint32_t minHeadroom, uint32_t minTailroom) {
668 size_t newCapacity = (size_t)length_ + minHeadroom + minTailroom;
669 DCHECK_LT(newCapacity, UINT32_MAX);
671 // reserveSlow() is dangerous if anyone else is sharing the buffer, as we may
672 // reallocate and free the original buffer. It should only ever be called if
673 // we are the only user of the buffer.
674 DCHECK(!isSharedOne());
676 // We'll need to reallocate the buffer.
677 // There are a few options.
678 // - If we have enough total room, move the data around in the buffer
679 // and adjust the data_ pointer.
680 // - If we're using an internal buffer, we'll switch to an external
681 // buffer with enough headroom and tailroom.
682 // - If we have enough headroom (headroom() >= minHeadroom) but not too much
683 // (so we don't waste memory), we can try one of two things, depending on
684 // whether we use jemalloc or not:
685 // - If using jemalloc, we can try to expand in place, avoiding a memcpy()
686 // - If not using jemalloc and we don't have too much to copy,
687 // we'll use realloc() (note that realloc might have to copy
688 // headroom + data + tailroom, see smartRealloc in folly/Malloc.h)
689 // - Otherwise, bite the bullet and reallocate.
690 if (headroom() + tailroom() >= minHeadroom + minTailroom) {
691 uint8_t* newData = writableBuffer() + minHeadroom;
692 memmove(newData, data_, length_);
697 size_t newAllocatedCapacity = goodExtBufferSize(newCapacity);
698 uint8_t* newBuffer = nullptr;
699 uint32_t newHeadroom = 0;
700 uint32_t oldHeadroom = headroom();
702 // If we have a buffer allocated with malloc and we just need more tailroom,
703 // try to use realloc()/rallocm() to grow the buffer in place.
704 if ((flags_ & kFlagUserOwned) == 0 && (sharedInfo_->freeFn == nullptr) &&
705 length_ != 0 && oldHeadroom >= minHeadroom) {
706 if (usingJEMalloc()) {
707 size_t headSlack = oldHeadroom - minHeadroom;
708 // We assume that tailroom is more useful and more important than
709 // headroom (not least because realloc / rallocm allow us to grow the
710 // buffer at the tail, but not at the head) So, if we have more headroom
711 // than we need, we consider that "wasted". We arbitrarily define "too
712 // much" headroom to be 25% of the capacity.
713 if (headSlack * 4 <= newCapacity) {
714 size_t allocatedCapacity = capacity() + sizeof(SharedInfo);
716 if (allocatedCapacity >= jemallocMinInPlaceExpandable) {
717 // rallocm can write to its 2nd arg even if it returns
718 // ALLOCM_ERR_NOT_MOVED. So, we pass a temporary to its 2nd arg and
719 // update newAllocatedCapacity only on success.
720 size_t allocatedSize;
721 int r = rallocm(&p, &allocatedSize, newAllocatedCapacity,
723 if (r == ALLOCM_SUCCESS) {
724 newBuffer = static_cast<uint8_t*>(p);
725 newHeadroom = oldHeadroom;
726 newAllocatedCapacity = allocatedSize;
727 } else if (r == ALLOCM_ERR_OOM) {
728 // shouldn't happen as we don't actually allocate new memory
729 // (due to ALLOCM_NO_MOVE)
730 throw std::bad_alloc();
732 // if ALLOCM_ERR_NOT_MOVED, do nothing, fall back to
733 // malloc/memcpy/free
736 } else { // Not using jemalloc
737 size_t copySlack = capacity() - length_;
738 if (copySlack * 2 <= length_) {
739 void* p = realloc(buf_, newAllocatedCapacity);
740 if (UNLIKELY(p == nullptr)) {
741 throw std::bad_alloc();
743 newBuffer = static_cast<uint8_t*>(p);
744 newHeadroom = oldHeadroom;
749 // None of the previous reallocation strategies worked (or we're using
750 // an internal buffer). malloc/copy/free.
751 if (newBuffer == nullptr) {
752 void* p = malloc(newAllocatedCapacity);
753 if (UNLIKELY(p == nullptr)) {
754 throw std::bad_alloc();
756 newBuffer = static_cast<uint8_t*>(p);
757 memcpy(newBuffer + minHeadroom, data_, length_);
758 if ((flags_ & kFlagUserOwned) == 0) {
761 newHeadroom = minHeadroom;
766 initExtBuffer(newBuffer, newAllocatedCapacity, &info, &cap);
768 if (flags_ & kFlagFreeSharedInfo) {
774 type_ = kExtAllocated;
777 data_ = newBuffer + newHeadroom;
778 // length_ is unchanged
781 void IOBuf::freeExtBuffer() {
782 DCHECK((flags_ & kFlagUserOwned) == 0);
784 if (sharedInfo_->freeFn) {
786 sharedInfo_->freeFn(buf_, sharedInfo_->userData);
788 // The user's free function should never throw. Otherwise we might
789 // throw from the IOBuf destructor. Other code paths like coalesce()
790 // also assume that decrementRefcount() cannot throw.
798 void IOBuf::allocExtBuffer(uint32_t minCapacity,
800 SharedInfo** infoReturn,
801 uint32_t* capacityReturn) {
802 size_t mallocSize = goodExtBufferSize(minCapacity);
803 uint8_t* buf = static_cast<uint8_t*>(malloc(mallocSize));
804 if (UNLIKELY(buf == NULL)) {
805 throw std::bad_alloc();
807 initExtBuffer(buf, mallocSize, infoReturn, capacityReturn);
811 size_t IOBuf::goodExtBufferSize(uint32_t minCapacity) {
812 // Determine how much space we should allocate. We'll store the SharedInfo
813 // for the external buffer just after the buffer itself. (We store it just
814 // after the buffer rather than just before so that the code can still just
815 // use free(buf_) to free the buffer.)
816 size_t minSize = static_cast<size_t>(minCapacity) + sizeof(SharedInfo);
817 // Add room for padding so that the SharedInfo will be aligned on an 8-byte
819 minSize = (minSize + 7) & ~7;
821 // Use goodMallocSize() to bump up the capacity to a decent size to request
822 // from malloc, so we can use all of the space that malloc will probably give
824 return goodMallocSize(minSize);
827 void IOBuf::initExtBuffer(uint8_t* buf, size_t mallocSize,
828 SharedInfo** infoReturn,
829 uint32_t* capacityReturn) {
830 // Find the SharedInfo storage at the end of the buffer
831 // and construct the SharedInfo.
832 uint8_t* infoStart = (buf + mallocSize) - sizeof(SharedInfo);
833 SharedInfo* sharedInfo = new(infoStart) SharedInfo;
835 size_t actualCapacity = infoStart - buf;
836 // On the unlikely possibility that the actual capacity is larger than can
837 // fit in a uint32_t after adding room for the refcount and calling
838 // goodMallocSize(), truncate downwards if necessary.
839 if (actualCapacity >= UINT32_MAX) {
840 *capacityReturn = UINT32_MAX;
842 *capacityReturn = actualCapacity;
845 *infoReturn = sharedInfo;
848 fbstring IOBuf::moveToFbString() {
849 // malloc-allocated buffers are just fine, everything else needs
850 // to be turned into one.
851 if ((flags_ & kFlagUserOwned) || // user owned, not ours to give up
852 sharedInfo_->freeFn != nullptr || // not malloc()-ed
853 headroom() != 0 || // malloc()-ed block doesn't start at beginning
854 tailroom() == 0 || // no room for NUL terminator
855 isShared() || // shared
856 isChained()) { // chained
857 // We might as well get rid of all head and tailroom if we're going
858 // to reallocate; we need 1 byte for NUL terminator.
859 coalesceAndReallocate(0, computeChainDataLength(), this, 1);
862 // Ensure NUL terminated
864 fbstring str(reinterpret_cast<char*>(writableData()),
865 length(), capacity(),
866 AcquireMallocatedString());
868 if (flags_ & kFlagFreeSharedInfo) {
872 // Reset to a state where we can be deleted cleanly
873 flags_ = kFlagUserOwned;
874 sharedInfo_ = nullptr;
880 IOBuf::Iterator IOBuf::cbegin() const {
881 return Iterator(this, this);
884 IOBuf::Iterator IOBuf::cend() const {
885 return Iterator(nullptr, nullptr);
888 folly::fbvector<struct iovec> IOBuf::getIov() const {
889 folly::fbvector<struct iovec> iov;
890 iov.reserve(countChainElements());
891 IOBuf const* p = this;
893 // some code can get confused by empty iovs, so skip them
894 if (p->length() > 0) {
895 iov.push_back({(void*)p->data(), p->length()});