2 * Copyright 2013 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "folly/io/IOBuf.h"
19 #include <gflags/gflags.h>
20 #include <boost/random.hpp>
21 #include <gtest/gtest.h>
22 #include "folly/Benchmark.h"
23 #include "folly/Range.h"
24 #include "folly/io/Cursor.h"
26 DECLARE_bool(benchmark);
29 using std::unique_ptr;
30 using namespace folly::io;
32 TEST(IOBuf, RWCursor) {
33 unique_ptr<IOBuf> iobuf1(IOBuf::create(20));
35 unique_ptr<IOBuf> iobuf2(IOBuf::create(20));
38 IOBuf* iob2ptr = iobuf2.get();
39 iobuf1->prependChain(std::move(iobuf2));
41 EXPECT_TRUE(iobuf1->isChained());
43 RWPrivateCursor wcursor(iobuf1.get());
44 Cursor rcursor(iobuf1.get());
45 wcursor.writeLE((uint64_t)1);
46 wcursor.writeLE((uint64_t)1);
47 wcursor.writeLE((uint64_t)1);
48 wcursor.write((uint8_t)1);
50 EXPECT_EQ(1, rcursor.readLE<uint64_t>());
52 EXPECT_EQ(1, rcursor.readLE<uint32_t>());
54 EXPECT_EQ(0, rcursor.read<uint8_t>());
55 EXPECT_EQ(0, rcursor.read<uint8_t>());
56 EXPECT_EQ(0, rcursor.read<uint8_t>());
57 EXPECT_EQ(0, rcursor.read<uint8_t>());
58 EXPECT_EQ(1, rcursor.read<uint8_t>());
62 unique_ptr<IOBuf> iobuf1(IOBuf::create(20));
64 RWPrivateCursor wcursor(iobuf1.get());
65 wcursor.write((uint8_t)1);
66 wcursor.write((uint8_t)2);
67 Cursor cursor(iobuf1.get());
69 EXPECT_EQ(2, cursor.read<uint8_t>());
73 unique_ptr<IOBuf> iobuf1(IOBuf::create(20));
75 RWPrivateCursor wcursor(iobuf1.get());
76 wcursor.write((uint8_t)1);
77 wcursor.write((uint8_t)2);
78 wcursor.reset(iobuf1.get());
79 EXPECT_EQ(1, wcursor.read<uint8_t>());
82 TEST(IOBuf, copy_assign_convert) {
83 unique_ptr<IOBuf> iobuf1(IOBuf::create(20));
85 RWPrivateCursor wcursor(iobuf1.get());
86 RWPrivateCursor cursor2(wcursor);
87 RWPrivateCursor cursor3(iobuf1.get());
89 wcursor.write((uint8_t)1);
91 wcursor.write((uint8_t)2);
92 Cursor cursor4(wcursor);
93 RWPrivateCursor cursor5(wcursor);
94 wcursor.write((uint8_t)3);
96 EXPECT_EQ(1, cursor2.read<uint8_t>());
97 EXPECT_EQ(2, cursor3.read<uint8_t>());
98 EXPECT_EQ(3, cursor4.read<uint8_t>());
101 TEST(IOBuf, overloading) {
102 unique_ptr<IOBuf> iobuf1(IOBuf::create(20));
104 RWPrivateCursor wcursor(iobuf1.get());
106 wcursor.write((uint8_t)1);
107 Cursor cursor(iobuf1.get());
109 EXPECT_EQ(1, cursor.read<uint8_t>());
112 TEST(IOBuf, endian) {
113 unique_ptr<IOBuf> iobuf1(IOBuf::create(20));
115 RWPrivateCursor wcursor(iobuf1.get());
116 Cursor rcursor(iobuf1.get());
121 // Try a couple combinations to ensure they were generated correctly
126 EXPECT_EQ(v, rcursor.readBE<uint16_t>());
129 TEST(IOBuf, Cursor) {
130 unique_ptr<IOBuf> iobuf1(IOBuf::create(1));
132 RWPrivateCursor c(iobuf1.get());
133 c.write((uint8_t)40); // OK
135 c.write((uint8_t)10); // Bad write, checked should except.
136 EXPECT_EQ(true, false);
141 TEST(IOBuf, UnshareCursor) {
143 unique_ptr<IOBuf> iobuf1(IOBuf::wrapBuffer(&buf, 1));
144 unique_ptr<IOBuf> iobuf2(IOBuf::wrapBuffer(&buf, 1));
145 RWUnshareCursor c1(iobuf1.get());
146 RWUnshareCursor c2(iobuf2.get());
148 c1.write((uint8_t)10); // This should duplicate the two buffers.
149 uint8_t t = c2.read<uint8_t>();
152 iobuf1 = IOBuf::wrapBuffer(&buf, 1);
153 iobuf2 = IOBuf::wrapBuffer(&buf, 1);
154 RWPrivateCursor c3(iobuf1.get());
155 RWPrivateCursor c4(iobuf2.get());
157 c3.write((uint8_t)10); // This should _not_ duplicate the two buffers.
158 t = c4.read<uint8_t>();
163 void append(std::unique_ptr<IOBuf>& buf, folly::StringPiece data) {
164 EXPECT_LE(data.size(), buf->tailroom());
165 memcpy(buf->writableData(), data.data(), data.size());
166 buf->append(data.size());
169 void append(Appender& appender, folly::StringPiece data) {
170 appender.push(reinterpret_cast<const uint8_t*>(data.data()), data.size());
173 std::string toString(const IOBuf& buf) {
176 std::pair<const uint8_t*, size_t> p;
177 while ((p = cursor.peek()).second) {
178 str.append(reinterpret_cast<const char*>(p.first), p.second);
179 cursor.skip(p.second);
186 TEST(IOBuf, PullAndPeek) {
187 std::unique_ptr<IOBuf> iobuf1(IOBuf::create(10));
188 append(iobuf1, "he");
189 std::unique_ptr<IOBuf> iobuf2(IOBuf::create(10));
190 append(iobuf2, "llo ");
191 std::unique_ptr<IOBuf> iobuf3(IOBuf::create(10));
192 append(iobuf3, "world");
193 iobuf1->prependChain(std::move(iobuf2));
194 iobuf1->prependChain(std::move(iobuf3));
195 EXPECT_EQ(3, iobuf1->countChainElements());
196 EXPECT_EQ(11, iobuf1->computeChainDataLength());
199 memset(buf, 0, sizeof(buf));
200 Cursor(iobuf1.get()).pull(buf, 11);
201 EXPECT_EQ("hello world", std::string(buf));
203 memset(buf, 0, sizeof(buf));
204 EXPECT_EQ(11, Cursor(iobuf1.get()).pullAtMost(buf, 20));
205 EXPECT_EQ("hello world", std::string(buf));
207 EXPECT_THROW({Cursor(iobuf1.get()).pull(buf, 20);},
211 RWPrivateCursor cursor(iobuf1.get());
212 auto p = cursor.peek();
213 EXPECT_EQ("he", std::string(reinterpret_cast<const char*>(p.first),
215 cursor.skip(p.second);
217 EXPECT_EQ("llo ", std::string(reinterpret_cast<const char*>(p.first),
219 cursor.skip(p.second);
221 EXPECT_EQ("world", std::string(reinterpret_cast<const char*>(p.first),
223 cursor.skip(p.second);
224 EXPECT_EQ(3, iobuf1->countChainElements());
225 EXPECT_EQ(11, iobuf1->computeChainDataLength());
229 RWPrivateCursor cursor(iobuf1.get());
231 auto p = cursor.peek();
232 EXPECT_EQ("hello world", std::string(reinterpret_cast<const
233 char*>(p.first), p.second));
234 EXPECT_EQ(1, iobuf1->countChainElements());
235 EXPECT_EQ(11, iobuf1->computeChainDataLength());
239 TEST(IOBuf, cloneAndInsert) {
240 std::unique_ptr<IOBuf> iobuf1(IOBuf::create(10));
241 append(iobuf1, "he");
242 std::unique_ptr<IOBuf> iobuf2(IOBuf::create(10));
243 append(iobuf2, "llo ");
244 std::unique_ptr<IOBuf> iobuf3(IOBuf::create(10));
245 append(iobuf3, "world");
246 iobuf1->prependChain(std::move(iobuf2));
247 iobuf1->prependChain(std::move(iobuf3));
248 EXPECT_EQ(3, iobuf1->countChainElements());
249 EXPECT_EQ(11, iobuf1->computeChainDataLength());
251 std::unique_ptr<IOBuf> cloned;
253 Cursor(iobuf1.get()).clone(cloned, 3);
254 EXPECT_EQ(2, cloned->countChainElements());
255 EXPECT_EQ(3, cloned->computeChainDataLength());
258 EXPECT_EQ(11, Cursor(iobuf1.get()).cloneAtMost(cloned, 20));
259 EXPECT_EQ(3, cloned->countChainElements());
260 EXPECT_EQ(11, cloned->computeChainDataLength());
263 EXPECT_THROW({Cursor(iobuf1.get()).clone(cloned, 20);},
267 // Check that inserting in the middle of an iobuf splits
268 RWPrivateCursor cursor(iobuf1.get());
269 Cursor(iobuf1.get()).clone(cloned, 3);
270 EXPECT_EQ(2, cloned->countChainElements());
271 EXPECT_EQ(3, cloned->computeChainDataLength());
275 cursor.insert(std::move(cloned));
276 EXPECT_EQ(6, iobuf1->countChainElements());
277 EXPECT_EQ(14, iobuf1->computeChainDataLength());
278 // Check that nextBuf got set correctly
279 cursor.read<uint8_t>();
283 // Check that inserting at the end doesn't create empty buf
284 RWPrivateCursor cursor(iobuf1.get());
285 Cursor(iobuf1.get()).clone(cloned, 1);
286 EXPECT_EQ(1, cloned->countChainElements());
287 EXPECT_EQ(1, cloned->computeChainDataLength());
291 cursor.insert(std::move(cloned));
292 EXPECT_EQ(7, iobuf1->countChainElements());
293 EXPECT_EQ(15, iobuf1->computeChainDataLength());
294 // Check that nextBuf got set correctly
295 cursor.read<uint8_t>();
298 // Check that inserting at the beginning doesn't create empty buf
299 RWPrivateCursor cursor(iobuf1.get());
300 Cursor(iobuf1.get()).clone(cloned, 1);
301 EXPECT_EQ(1, cloned->countChainElements());
302 EXPECT_EQ(1, cloned->computeChainDataLength());
304 cursor.insert(std::move(cloned));
305 EXPECT_EQ(8, iobuf1->countChainElements());
306 EXPECT_EQ(16, iobuf1->computeChainDataLength());
307 // Check that nextBuf got set correctly
308 cursor.read<uint8_t>();
312 TEST(IOBuf, Appender) {
313 std::unique_ptr<IOBuf> head(IOBuf::create(10));
314 append(head, "hello");
316 Appender app(head.get(), 10);
317 uint32_t cap = head->capacity();
318 uint32_t len1 = app.length();
319 EXPECT_EQ(cap - 5, len1);
320 app.ensure(len1); // won't grow
321 EXPECT_EQ(len1, app.length());
322 app.ensure(len1 + 1); // will grow
323 EXPECT_LE(len1 + 1, app.length());
325 append(app, " world");
326 EXPECT_EQ("hello world", toString(*head));
329 TEST(IOBuf, QueueAppender) {
330 folly::IOBufQueue queue;
332 // Allocate 100 bytes at once, but don't grow past 1024
333 QueueAppender app(&queue, 100);
334 size_t n = 1024 / sizeof(uint32_t);
335 for (uint32_t i = 0; i < n; ++i) {
339 // There must be a goodMallocSize between 100 and 1024...
340 EXPECT_LT(1, queue.front()->countChainElements());
341 const IOBuf* buf = queue.front();
343 EXPECT_LE(100, buf->capacity());
345 } while (buf != queue.front());
347 Cursor cursor(queue.front());
348 for (uint32_t i = 0; i < n; ++i) {
349 EXPECT_EQ(i, cursor.readBE<uint32_t>());
352 EXPECT_THROW({cursor.readBE<uint32_t>();}, std::out_of_range);
355 TEST(IOBuf, CursorOperators) {
356 // Test operators on a single-item chain
358 std::unique_ptr<IOBuf> chain1(IOBuf::create(20));
361 Cursor curs1(chain1.get());
362 EXPECT_EQ(0, curs1 - chain1.get());
364 EXPECT_EQ(3, curs1 - chain1.get());
366 EXPECT_EQ(10, curs1 - chain1.get());
368 Cursor curs2(chain1.get());
369 EXPECT_EQ(0, curs2 - chain1.get());
370 EXPECT_EQ(10, curs1 - curs2);
371 EXPECT_THROW(curs2 - curs1, std::out_of_range);
374 // Test cross-chain operations
376 std::unique_ptr<IOBuf> chain1(IOBuf::create(20));
378 std::unique_ptr<IOBuf> chain2 = chain1->clone();
380 Cursor curs1(chain1.get());
381 Cursor curs2(chain2.get());
382 EXPECT_THROW(curs1 - curs2, std::out_of_range);
383 EXPECT_THROW(curs1 - chain2.get(), std::out_of_range);
386 // Test operations on multi-item chains
388 std::unique_ptr<IOBuf> chain(IOBuf::create(20));
390 chain->appendChain(chain->clone());
391 EXPECT_EQ(20, chain->computeChainDataLength());
393 Cursor curs1(chain.get());
395 Cursor curs2(chain.get());
397 EXPECT_EQ(2, curs1 - curs2);
398 EXPECT_EQ(5, curs1 - chain.get());
399 EXPECT_THROW(curs2 - curs1, std::out_of_range);
402 EXPECT_EQ(9, curs1 - curs2);
403 EXPECT_EQ(12, curs1 - chain.get());
404 EXPECT_THROW(curs2 - curs1, std::out_of_range);
407 EXPECT_EQ(2, curs1 - curs2);
408 EXPECT_THROW(curs2 - curs1, std::out_of_range);
412 TEST(IOBuf, StringOperations) {
413 // Test a single buffer with two null-terminated strings and an extra uint8_t
416 std::unique_ptr<IOBuf> chain(IOBuf::create(16));
417 Appender app(chain.get(), 0);
418 app.push(reinterpret_cast<const uint8_t*>("hello\0world\0\x01"), 13);
420 Cursor curs(chain.get());
421 EXPECT_STREQ("hello", curs.readTerminatedString().c_str());
422 EXPECT_STREQ("world", curs.readTerminatedString().c_str());
423 EXPECT_EQ(1, curs.read<uint8_t>());
426 // Test multiple buffers where the first is empty and the string starts in
427 // the second buffer.
429 std::unique_ptr<IOBuf> chain(IOBuf::create(8));
430 chain->prependChain(IOBuf::create(12));
431 Appender app(chain.get(), 0);
432 app.push(reinterpret_cast<const uint8_t*>("hello world\0"), 12);
434 Cursor curs(chain.get());
435 EXPECT_STREQ("hello world", curs.readTerminatedString().c_str());
438 // Test multiple buffers with a single null-terminated string spanning them
440 std::unique_ptr<IOBuf> chain(IOBuf::create(8));
441 chain->prependChain(IOBuf::create(8));
443 chain->next()->append(4);
444 RWPrivateCursor rwc(chain.get());
445 rwc.push(reinterpret_cast<const uint8_t*>("hello world\0"), 12);
447 Cursor curs(chain.get());
448 EXPECT_STREQ("hello world", curs.readTerminatedString().c_str());
451 // Test a reading a null-terminated string that's longer than the maximum
454 std::unique_ptr<IOBuf> chain(IOBuf::create(16));
455 Appender app(chain.get(), 0);
456 app.push(reinterpret_cast<const uint8_t*>("hello world\0"), 12);
458 Cursor curs(chain.get());
459 EXPECT_THROW(curs.readTerminatedString('\0', 5), std::length_error);
462 // Test reading a null-terminated string from a chain with an empty buffer at
465 std::unique_ptr<IOBuf> buf(IOBuf::create(8));
466 Appender app(buf.get(), 0);
467 app.push(reinterpret_cast<const uint8_t*>("hello\0"), 6);
468 std::unique_ptr<IOBuf> chain(IOBuf::create(8));
469 chain->prependChain(std::move(buf));
471 Cursor curs(chain.get());
472 EXPECT_STREQ("hello", curs.readTerminatedString().c_str());
475 // Test reading a two fixed-length strings from a single buffer with an extra
476 // uint8_t at the end
478 std::unique_ptr<IOBuf> chain(IOBuf::create(16));
479 Appender app(chain.get(), 0);
480 app.push(reinterpret_cast<const uint8_t*>("helloworld\x01"), 11);
482 Cursor curs(chain.get());
483 EXPECT_STREQ("hello", curs.readFixedString(5).c_str());
484 EXPECT_STREQ("world", curs.readFixedString(5).c_str());
485 EXPECT_EQ(1, curs.read<uint8_t>());
488 // Test multiple buffers where the first is empty and a fixed-length string
489 // starts in the second buffer.
491 std::unique_ptr<IOBuf> chain(IOBuf::create(8));
492 chain->prependChain(IOBuf::create(16));
493 Appender app(chain.get(), 0);
494 app.push(reinterpret_cast<const uint8_t*>("hello world"), 11);
496 Cursor curs(chain.get());
497 EXPECT_STREQ("hello world", curs.readFixedString(11).c_str());
500 // Test multiple buffers with a single fixed-length string spanning them
502 std::unique_ptr<IOBuf> chain(IOBuf::create(8));
503 chain->prependChain(IOBuf::create(8));
505 chain->next()->append(4);
506 RWPrivateCursor rwc(chain.get());
507 rwc.push(reinterpret_cast<const uint8_t*>("hello world"), 11);
509 Cursor curs(chain.get());
510 EXPECT_STREQ("hello world", curs.readFixedString(11).c_str());
513 // Test reading a fixed-length string from a chain with an empty buffer at
516 std::unique_ptr<IOBuf> buf(IOBuf::create(8));
517 Appender app(buf.get(), 0);
518 app.push(reinterpret_cast<const uint8_t*>("hello"), 5);
519 std::unique_ptr<IOBuf> chain(IOBuf::create(8));
520 chain->prependChain(std::move(buf));
522 Cursor curs(chain.get());
523 EXPECT_STREQ("hello", curs.readFixedString(5).c_str());
527 int benchmark_size = 1000;
528 unique_ptr<IOBuf> iobuf_benchmark;
530 unique_ptr<IOBuf> iobuf_read_benchmark;
532 template <class CursClass>
533 void runBenchmark() {
534 CursClass c(iobuf_benchmark.get());
536 for(int i = 0; i < benchmark_size; i++) {
541 BENCHMARK(rwPrivateCursorBenchmark, iters) {
543 runBenchmark<RWPrivateCursor>();
547 BENCHMARK(rwUnshareCursorBenchmark, iters) {
549 runBenchmark<RWUnshareCursor>();
554 BENCHMARK(cursorBenchmark, iters) {
556 Cursor c(iobuf_read_benchmark.get());
557 for(int i = 0; i < benchmark_size ; i++) {
563 BENCHMARK(skipBenchmark, iters) {
566 Cursor c(iobuf_read_benchmark.get());
567 for(int i = 0; i < benchmark_size ; i++) {
575 // _bin/folly/experimental/io/test/iobuf_cursor_test -benchmark
577 // Benchmark Iters Total t t/iter iter/sec
578 // ---------------------------------------------------------------------------
579 // rwPrivateCursorBenchmark 100000 142.9 ms 1.429 us 683.5 k
580 // rwUnshareCursorBenchmark 100000 309.3 ms 3.093 us 315.7 k
581 // cursorBenchmark 100000 741.4 ms 7.414 us 131.7 k
582 // skipBenchmark 100000 738.9 ms 7.389 us 132.2 k
586 // Linux dev2159.snc6.facebook.com 2.6.33-7_fbk15_104e4d0 #1 SMP
587 // Tue Oct 19 22:40:30 PDT 2010 x86_64 x86_64 x86_64 GNU/Linux
589 // 72GB RAM, 2 CPUs (Intel(R) Xeon(R) CPU L5630 @ 2.13GHz)
590 // hyperthreading disabled
592 int main(int argc, char** argv) {
593 testing::InitGoogleTest(&argc, argv);
594 google::ParseCommandLineFlags(&argc, &argv, true);
596 auto ret = RUN_ALL_TESTS();
598 if (ret == 0 && FLAGS_benchmark) {
599 iobuf_benchmark = IOBuf::create(benchmark_size);
600 iobuf_benchmark->append(benchmark_size);
602 iobuf_read_benchmark = IOBuf::create(1);
603 for (int i = 0; i < benchmark_size; i++) {
604 unique_ptr<IOBuf> iobuf2(IOBuf::create(1));
606 iobuf_read_benchmark->prependChain(std::move(iobuf2));
609 folly::runBenchmarks();