X-Git-Url: http://plrg.eecs.uci.edu/git/?p=folly.git;a=blobdiff_plain;f=folly%2Fio%2Ftest%2FIOBufTest.cpp;h=ba92617752daa525c5b864aa6159ea034939db91;hp=b17be70887072539d06b5fadbe6e8f258a719a2f;hb=614ca62428e304c8d710f97aaf2881e8b6a297a4;hpb=51f6dea4137433b6b535f3d3348a625a5ca6c240 diff --git a/folly/io/test/IOBufTest.cpp b/folly/io/test/IOBufTest.cpp index b17be708..ba926177 100644 --- a/folly/io/test/IOBufTest.cpp +++ b/folly/io/test/IOBufTest.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2013 Facebook, Inc. + * Copyright 2015 Facebook, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,18 +14,17 @@ * limitations under the License. */ -#include "folly/io/IOBuf.h" -#include "folly/io/TypedIOBuf.h" - -// googletest requires std::tr1::tuple, not std::tuple -#include +#include +#include #include #include #include -#include "folly/Malloc.h" -#include "folly/Range.h" +#include +#include + +#include using folly::fbstring; using folly::fbvector; @@ -142,7 +141,26 @@ TEST(IOBuf, TakeOwnership) { iobuf3.reset(); EXPECT_EQ(1, deleteCount); - + deleteCount = 0; + { + uint32_t size4 = 1234; + uint8_t *buf4 = new uint8_t[size4]; + uint32_t length4 = 48; + IOBuf iobuf4(IOBuf::TAKE_OWNERSHIP, buf4, size4, length4, + deleteArrayBuffer, &deleteCount); + EXPECT_EQ(buf4, iobuf4.data()); + EXPECT_EQ(length4, iobuf4.length()); + EXPECT_EQ(buf4, iobuf4.buffer()); + EXPECT_EQ(size4, iobuf4.capacity()); + + IOBuf iobuf5 = std::move(iobuf4); + EXPECT_EQ(buf4, iobuf5.data()); + EXPECT_EQ(length4, iobuf5.length()); + EXPECT_EQ(buf4, iobuf5.buffer()); + EXPECT_EQ(size4, iobuf5.capacity()); + EXPECT_EQ(0, deleteCount); + } + EXPECT_EQ(1, deleteCount); } TEST(IOBuf, WrapBuffer) { @@ -161,6 +179,69 @@ TEST(IOBuf, WrapBuffer) { EXPECT_EQ(size2, iobuf2->length()); EXPECT_EQ(buf2.get(), iobuf2->buffer()); EXPECT_EQ(size2, iobuf2->capacity()); + + uint32_t size3 = 4321; + unique_ptr buf3(new uint8_t[size3]); + IOBuf iobuf3(IOBuf::WRAP_BUFFER, buf3.get(), size3); + EXPECT_EQ(buf3.get(), iobuf3.data()); + EXPECT_EQ(size3, iobuf3.length()); + EXPECT_EQ(buf3.get(), iobuf3.buffer()); + EXPECT_EQ(size3, iobuf3.capacity()); +} + +TEST(IOBuf, CreateCombined) { + // Create a combined IOBuf, then destroy it. + // The data buffer and IOBuf both become unused as part of the destruction + { + auto buf = IOBuf::createCombined(256); + EXPECT_FALSE(buf->isShared()); + } + + // Create a combined IOBuf, clone from it, and then destroy the original + // IOBuf. The data buffer cannot be deleted until the clone is also + // destroyed. + { + auto bufA = IOBuf::createCombined(256); + EXPECT_FALSE(bufA->isShared()); + auto bufB = bufA->clone(); + EXPECT_TRUE(bufA->isShared()); + EXPECT_TRUE(bufB->isShared()); + bufA.reset(); + EXPECT_FALSE(bufB->isShared()); + } + + // Create a combined IOBuf, then call reserve() to get a larger buffer. + // The IOBuf no longer points to the combined data buffer, but the + // overall memory segment cannot be deleted until the IOBuf is also + // destroyed. + { + auto buf = IOBuf::createCombined(256); + buf->reserve(0, buf->capacity() + 100); + } + + // Create a combined IOBuf, clone from it, then call unshare() on the original + // buffer. This creates a situation where bufB is pointing at the combined + // buffer associated with bufA, but bufA is now using a different buffer. + auto testSwap = [](bool resetAFirst) { + auto bufA = IOBuf::createCombined(256); + EXPECT_FALSE(bufA->isShared()); + auto bufB = bufA->clone(); + EXPECT_TRUE(bufA->isShared()); + EXPECT_TRUE(bufB->isShared()); + bufA->unshare(); + EXPECT_FALSE(bufA->isShared()); + EXPECT_FALSE(bufB->isShared()); + + if (resetAFirst) { + bufA.reset(); + bufB.reset(); + } else { + bufB.reset(); + bufA.reset(); + } + }; + testSwap(true); + testSwap(false); } void fillBuf(uint8_t* buf, uint32_t length, boost::mt19937& gen) { @@ -392,8 +473,7 @@ TEST(IOBuf, Chaining) { EXPECT_TRUE(iob1->isShared()); EXPECT_TRUE(iob1->isSharedOne()); - // since iob2 has a small internal buffer, it will never be shared - EXPECT_FALSE(iob2ptr->isSharedOne()); + EXPECT_TRUE(iob2ptr->isSharedOne()); EXPECT_TRUE(iob3ptr->isSharedOne()); EXPECT_TRUE(iob4ptr->isSharedOne()); EXPECT_TRUE(iob5ptr->isSharedOne()); @@ -432,6 +512,8 @@ TEST(IOBuf, Chaining) { EXPECT_EQ(0, arrayBufFreeCount); // Buffer lengths: 1500 20 1234 900 321 + // Attempting to gather more data than available should fail + EXPECT_THROW(chainClone->gather(4000), std::overflow_error); // Coalesce the first 3 buffers chainClone->gather(1521); EXPECT_EQ(3, chainClone->countChainElements()); @@ -494,6 +576,14 @@ TEST(IOBuf, Chaining) { EXPECT_EQ(3, iob2->computeChainDataLength()); } +void testFreeFn(void* buffer, void* ptr) { + uint32_t* freeCount = static_cast(ptr);; + delete[] static_cast(buffer); + if (freeCount) { + ++(*freeCount); + } +}; + TEST(IOBuf, Reserve) { uint32_t fillSeed = 0x23456789; boost::mt19937 gen(fillSeed); @@ -555,6 +645,26 @@ TEST(IOBuf, Reserve) { EXPECT_EQ(0, iob->headroom()); EXPECT_LE(2000, iob->tailroom()); } + + // Test reserving from a user-allocated buffer. + { + uint8_t* buf = static_cast(malloc(100)); + auto iob = IOBuf::takeOwnership(buf, 100); + iob->reserve(0, 2000); + EXPECT_EQ(0, iob->headroom()); + EXPECT_LE(2000, iob->tailroom()); + } + + // Test reserving from a user-allocated with a custom free function. + { + uint32_t freeCount{0}; + uint8_t* buf = new uint8_t[100]; + auto iob = IOBuf::takeOwnership(buf, 100, testFreeFn, &freeCount); + iob->reserve(0, 2000); + EXPECT_EQ(0, iob->headroom()); + EXPECT_LE(2000, iob->tailroom()); + EXPECT_EQ(1, freeCount); + } } TEST(IOBuf, copyBuffer) { @@ -576,6 +686,13 @@ TEST(IOBuf, copyBuffer) { EXPECT_EQ(3, buf->headroom()); EXPECT_EQ(0, buf->length()); EXPECT_LE(6, buf->tailroom()); + + // A stack-allocated version + IOBuf stackBuf(IOBuf::COPY_BUFFER, s, 1, 2); + EXPECT_EQ(1, stackBuf.headroom()); + EXPECT_EQ(s, std::string(reinterpret_cast(stackBuf.data()), + stackBuf.length())); + EXPECT_LE(2, stackBuf.tailroom()); } TEST(IOBuf, maybeCopyBuffer) { @@ -667,7 +784,7 @@ TEST(IOBuf, takeOwnershipUniquePtr) { destructorCount = 0; { std::unique_ptr - p(new OwnershipTestClass[2], customDeleteArray); + p(new OwnershipTestClass[2], CustomDeleter(customDeleteArray)); std::unique_ptr buf(IOBuf::takeOwnership(std::move(p), 2)); EXPECT_EQ(2 * sizeof(OwnershipTestClass), buf->length()); EXPECT_EQ(0, destructorCount); @@ -677,11 +794,7 @@ TEST(IOBuf, takeOwnershipUniquePtr) { } TEST(IOBuf, Alignment) { - // max_align_t doesn't exist in gcc 4.6.2 - struct MaxAlign { - char c; - } __attribute__((aligned)); - size_t alignment = alignof(MaxAlign); + size_t alignment = alignof(std::max_align_t); std::vector sizes {0, 1, 64, 256, 1024, 1 << 10}; for (size_t size : sizes) { @@ -706,13 +819,23 @@ TEST(TypedIOBuf, Simple) { EXPECT_EQ(i, typed.data()[i]); } } +enum BufType { + CREATE, + TAKE_OWNERSHIP_MALLOC, + TAKE_OWNERSHIP_CUSTOM, + USER_OWNED, +}; // chain element size, number of elements in chain, shared class MoveToFbStringTest - : public ::testing::TestWithParam> { + : public ::testing::TestWithParam> { protected: - void SetUp() { - std::tr1::tie(elementSize_, elementCount_, shared_) = GetParam(); + void SetUp() override { + elementSize_ = std::tr1::get<0>(GetParam()); + elementCount_ = std::tr1::get<1>(GetParam()); + shared_ = std::tr1::get<2>(GetParam()); + type_ = std::tr1::get<3>(GetParam()); + buf_ = makeBuf(); for (int i = 0; i < elementCount_ - 1; ++i) { buf_->prependChain(makeBuf()); @@ -727,9 +850,36 @@ class MoveToFbStringTest } std::unique_ptr makeBuf() { - auto buf = IOBuf::create(elementSize_); - memset(buf->writableTail(), 'x', elementSize_); - buf->append(elementSize_); + unique_ptr buf; + switch (type_) { + case CREATE: + buf = IOBuf::create(elementSize_); + buf->append(elementSize_); + break; + case TAKE_OWNERSHIP_MALLOC: { + void* data = malloc(elementSize_); + if (!data) { + throw std::bad_alloc(); + } + buf = IOBuf::takeOwnership(data, elementSize_); + break; + } + case TAKE_OWNERSHIP_CUSTOM: { + uint8_t* data = new uint8_t[elementSize_]; + buf = IOBuf::takeOwnership(data, elementSize_, testFreeFn); + break; + } + case USER_OWNED: { + unique_ptr data(new uint8_t[elementSize_]); + buf = IOBuf::wrapBuffer(data.get(), elementSize_); + ownedBuffers_.emplace_back(std::move(data)); + break; + } + default: + throw std::invalid_argument("unexpected buffer type parameter"); + break; + } + memset(buf->writableData(), 'x', elementSize_); return buf; } @@ -746,8 +896,10 @@ class MoveToFbStringTest int elementSize_; int elementCount_; bool shared_; + BufType type_; std::unique_ptr buf_; std::unique_ptr buf2_; + std::vector> ownedBuffers_; }; TEST_P(MoveToFbStringTest, Simple) { @@ -763,7 +915,9 @@ INSTANTIATE_TEST_CASE_P( ::testing::Combine( ::testing::Values(0, 1, 24, 256, 1 << 10, 1 << 20), // element size ::testing::Values(1, 2, 10), // element count - ::testing::Bool())); // shared + ::testing::Bool(), // shared + ::testing::Values(CREATE, TAKE_OWNERSHIP_MALLOC, + TAKE_OWNERSHIP_CUSTOM, USER_OWNED))); TEST(IOBuf, getIov) { uint32_t fillSeed = 0xdeadbeef; @@ -808,11 +962,282 @@ TEST(IOBuf, getIov) { buf->prev()->clear(); iov = buf->getIov(); EXPECT_EQ(count - 3, iov.size()); + + // test appending to an existing iovec array + iov.clear(); + const char localBuf[] = "hello"; + iov.push_back({(void*)localBuf, sizeof(localBuf)}); + iov.push_back({(void*)localBuf, sizeof(localBuf)}); + buf->appendToIov(&iov); + EXPECT_EQ(count - 1, iov.size()); + EXPECT_EQ(localBuf, iov[0].iov_base); + EXPECT_EQ(localBuf, iov[1].iov_base); + // The first two IOBufs were cleared, so the next iov entry + // should be the third IOBuf in the chain. + EXPECT_EQ(buf->next()->next()->data(), iov[2].iov_base); +} + +TEST(IOBuf, move) { + // Default allocate an IOBuf on the stack + IOBuf outerBuf; + char data[] = "foobar"; + uint32_t length = sizeof(data); + uint32_t actualCapacity{0}; + const void* ptr{nullptr}; + + { + // Create a small IOBuf on the stack. + // Note that IOBufs created on the stack always use an external buffer. + IOBuf b1(IOBuf::CREATE, 10); + actualCapacity = b1.capacity(); + EXPECT_GE(actualCapacity, 10); + EXPECT_EQ(0, b1.length()); + EXPECT_FALSE(b1.isShared()); + ptr = b1.data(); + ASSERT_TRUE(ptr != nullptr); + memcpy(b1.writableTail(), data, length); + b1.append(length); + EXPECT_EQ(length, b1.length()); + + // Use the move constructor + IOBuf b2(std::move(b1)); + EXPECT_EQ(ptr, b2.data()); + EXPECT_EQ(length, b2.length()); + EXPECT_EQ(actualCapacity, b2.capacity()); + EXPECT_FALSE(b2.isShared()); + + // Use the move assignment operator + outerBuf = std::move(b2); + // Close scope, destroying b1 and b2 + // (which are both be invalid now anyway after moving out of them) + } + + EXPECT_EQ(ptr, outerBuf.data()); + EXPECT_EQ(length, outerBuf.length()); + EXPECT_EQ(actualCapacity, outerBuf.capacity()); + EXPECT_FALSE(outerBuf.isShared()); +} + +namespace { +std::unique_ptr fromStr(StringPiece sp) { + return IOBuf::copyBuffer(ByteRange(sp)); +} +} // namespace + +TEST(IOBuf, HashAndEqual) { + folly::IOBufEqual eq; + folly::IOBufHash hash; + + EXPECT_TRUE(eq(nullptr, nullptr)); + EXPECT_EQ(0, hash(nullptr)); + + auto empty = IOBuf::create(0); + + EXPECT_TRUE(eq(*empty, *empty)); + EXPECT_TRUE(eq(empty, empty)); + + EXPECT_FALSE(eq(nullptr, empty)); + EXPECT_FALSE(eq(empty, nullptr)); + + EXPECT_EQ(hash(*empty), hash(empty)); + EXPECT_NE(0, hash(empty)); + + auto a = fromStr("hello"); + + EXPECT_TRUE(eq(*a, *a)); + EXPECT_TRUE(eq(a, a)); + + EXPECT_FALSE(eq(nullptr, a)); + EXPECT_FALSE(eq(a, nullptr)); + + EXPECT_EQ(hash(*a), hash(a)); + EXPECT_NE(0, hash(a)); + + auto b = fromStr("hello"); + + EXPECT_TRUE(eq(*a, *b)); + EXPECT_TRUE(eq(a, b)); + + EXPECT_EQ(hash(a), hash(b)); + + auto c = fromStr("hellow"); + + EXPECT_FALSE(eq(a, c)); + EXPECT_NE(hash(a), hash(c)); + + auto d = fromStr("world"); + + EXPECT_FALSE(eq(a, d)); + EXPECT_NE(hash(a), hash(d)); + + auto e = fromStr("helloworld"); + auto f = fromStr("hello"); + f->prependChain(fromStr("wo")); + f->prependChain(fromStr("rld")); + + EXPECT_TRUE(eq(e, f)); + EXPECT_EQ(hash(e), hash(f)); +} + +// reserveSlow() had a bug when reallocating the buffer in place. It would +// preserve old headroom if it's not too much (heuristically) but wouldn't +// adjust the requested amount of memory to account for that; the end result +// would be that reserve() would return with less tailroom than requested. +TEST(IOBuf, ReserveWithHeadroom) { + // This is assuming jemalloc, where we know that 4096 and 8192 bytes are + // valid (and consecutive) allocation sizes. We're hoping that our + // 4096-byte buffer can be expanded in place to 8192 (in practice, this + // usually happens). + const char data[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit"; + constexpr size_t reservedSize = 24; // sizeof(SharedInfo) + // chosen carefully so that the buffer is exactly 4096 bytes + IOBuf buf(IOBuf::CREATE, 4096 - reservedSize); + buf.advance(10); + memcpy(buf.writableData(), data, sizeof(data)); + buf.append(sizeof(data)); + EXPECT_EQ(sizeof(data), buf.length()); + + // Grow the buffer (hopefully in place); this would incorrectly reserve + // the 10 bytes of headroom, giving us 10 bytes less than requested. + size_t tailroom = 8192 - reservedSize - sizeof(data); + buf.reserve(0, tailroom); + EXPECT_LE(tailroom, buf.tailroom()); + EXPECT_EQ(sizeof(data), buf.length()); + EXPECT_EQ(0, memcmp(data, buf.data(), sizeof(data))); +} + +TEST(IOBuf, CopyConstructorAndAssignmentOperator) { + auto buf = IOBuf::create(4096); + append(buf, "hello world"); + auto buf2 = IOBuf::create(4096); + append(buf2, " goodbye"); + buf->prependChain(std::move(buf2)); + EXPECT_FALSE(buf->isShared()); + + { + auto copy = *buf; + EXPECT_TRUE(buf->isShared()); + EXPECT_TRUE(copy.isShared()); + EXPECT_EQ((void*)buf->data(), (void*)copy.data()); + EXPECT_NE(buf->next(), copy.next()); // actually different buffers + + auto copy2 = *buf; + copy2.coalesce(); + EXPECT_TRUE(buf->isShared()); + EXPECT_TRUE(copy.isShared()); + EXPECT_FALSE(copy2.isShared()); + + auto p = reinterpret_cast(copy2.data()); + EXPECT_EQ("hello world goodbye", std::string(p, copy2.length())); + } + + EXPECT_FALSE(buf->isShared()); + + { + folly::IOBuf newBuf(folly::IOBuf::CREATE, 4096); + EXPECT_FALSE(newBuf.isShared()); + + auto newBufCopy = newBuf; + EXPECT_TRUE(newBuf.isShared()); + EXPECT_TRUE(newBufCopy.isShared()); + + newBufCopy = *buf; + EXPECT_TRUE(buf->isShared()); + EXPECT_FALSE(newBuf.isShared()); + EXPECT_TRUE(newBufCopy.isShared()); + } + + EXPECT_FALSE(buf->isShared()); +} + +namespace { +// Use with string literals only +std::unique_ptr wrap(const char* str) { + return IOBuf::wrapBuffer(str, strlen(str)); +} + +std::unique_ptr copy(const char* str) { + // At least 1KiB of tailroom, to ensure an external buffer + return IOBuf::copyBuffer(str, strlen(str), 0, 1024); +} + +std::string toString(const folly::IOBuf& buf) { + std::string result; + result.reserve(buf.computeChainDataLength()); + for (auto& b : buf) { + result.append(reinterpret_cast(b.data()), b.size()); + } + return result; +} + +char* writableStr(folly::IOBuf& buf) { + return reinterpret_cast(buf.writableData()); +} + +} // namespace + +TEST(IOBuf, Managed) { + auto hello = "hello"; + auto buf1UP = wrap(hello); + auto buf1 = buf1UP.get(); + EXPECT_FALSE(buf1->isManagedOne()); + auto buf2UP = copy("world"); + auto buf2 = buf2UP.get(); + EXPECT_TRUE(buf2->isManagedOne()); + auto buf3UP = wrap(hello); + auto buf3 = buf3UP.get(); + auto buf4UP = buf2->clone(); + auto buf4 = buf4UP.get(); + + // buf1 and buf3 share the same memory (but are unmanaged) + EXPECT_FALSE(buf1->isManagedOne()); + EXPECT_FALSE(buf3->isManagedOne()); + EXPECT_TRUE(buf1->isSharedOne()); + EXPECT_TRUE(buf3->isSharedOne()); + EXPECT_EQ(buf1->data(), buf3->data()); + + // buf2 and buf4 share the same memory (but are managed) + EXPECT_TRUE(buf2->isManagedOne()); + EXPECT_TRUE(buf4->isManagedOne()); + EXPECT_TRUE(buf2->isSharedOne()); + EXPECT_TRUE(buf4->isSharedOne()); + EXPECT_EQ(buf2->data(), buf4->data()); + + buf1->prependChain(std::move(buf2UP)); + buf1->prependChain(std::move(buf3UP)); + buf1->prependChain(std::move(buf4UP)); + + EXPECT_EQ("helloworldhelloworld", toString(*buf1)); + EXPECT_FALSE(buf1->isManaged()); + + buf1->makeManaged(); + EXPECT_TRUE(buf1->isManaged()); + + // buf1 and buf3 are now unshared (because they were unmanaged) + EXPECT_TRUE(buf1->isManagedOne()); + EXPECT_TRUE(buf3->isManagedOne()); + EXPECT_FALSE(buf1->isSharedOne()); + EXPECT_FALSE(buf3->isSharedOne()); + EXPECT_NE(buf1->data(), buf3->data()); + + // buf2 and buf4 are still shared + EXPECT_TRUE(buf2->isManagedOne()); + EXPECT_TRUE(buf4->isManagedOne()); + EXPECT_TRUE(buf2->isSharedOne()); + EXPECT_TRUE(buf4->isSharedOne()); + EXPECT_EQ(buf2->data(), buf4->data()); + + // And verify that the truth is what we expect: modify a byte in buf1 and + // buf2, see that the change from buf1 is *not* reflected in buf3, but the + // change from buf2 is reflected in buf4. + writableStr(*buf1)[0] = 'j'; + writableStr(*buf2)[0] = 'x'; + EXPECT_EQ("jelloxorldhelloxorld", toString(*buf1)); } int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); - google::ParseCommandLineFlags(&argc, &argv, true); + gflags::ParseCommandLineFlags(&argc, &argv, true); return RUN_ALL_TESTS(); }