--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOLLY_ARENA_H_
+#error This file may only be included from Arena.h
+#endif
+
+// Implementation of Arena.h functions
+
+namespace folly {
+
+template <class Alloc>
+std::pair<typename Arena<Alloc>::Block*, size_t>
+Arena<Alloc>::Block::allocate(Alloc& alloc, size_t size, bool allowSlack) {
+ size_t allocSize = sizeof(Block) + size;
+ if (allowSlack) {
+ allocSize = ArenaAllocatorTraits<Alloc>::goodSize(alloc, allocSize);
+ }
+
+ void* mem = alloc.allocate(allocSize);
+ assert(isAligned(mem));
+ return std::make_pair(new (mem) Block(), allocSize - sizeof(Block));
+}
+
+template <class Alloc>
+void Arena<Alloc>::Block::deallocate(Alloc& alloc) {
+ this->~Block();
+ alloc.deallocate(this);
+}
+
+template <class Alloc>
+void* Arena<Alloc>::allocateSlow(size_t size) {
+ std::pair<Block*, size_t> p;
+ char* start;
+ if (size > minBlockSize()) {
+ // Allocate a large block for this chunk only, put it at the back of the
+ // list so it doesn't get used for small allocations; don't change ptr_
+ // and end_, let them point into a normal block (or none, if they're
+ // null)
+ p = Block::allocate(alloc(), size, false);
+ start = p.first->start();
+ blocks_.push_back(*p.first);
+ } else {
+ // Allocate a normal sized block and carve out size bytes from it
+ p = Block::allocate(alloc(), minBlockSize(), true);
+ start = p.first->start();
+ blocks_.push_front(*p.first);
+ ptr_ = start + size;
+ end_ = start + p.second;
+ }
+
+ assert(p.second >= size);
+ return start;
+}
+
+template <class Alloc>
+void Arena<Alloc>::merge(Arena<Alloc>&& other) {
+ blocks_.splice_after(blocks_.before_begin(), other.blocks_);
+ other.blocks_.clear();
+ other.ptr_ = other.end_ = nullptr;
+}
+
+template <class Alloc>
+Arena<Alloc>::~Arena() {
+ auto disposer = [this] (Block* b) { b->deallocate(this->alloc()); };
+ while (!blocks_.empty()) {
+ blocks_.pop_front_and_dispose(disposer);
+ }
+}
+
+} // namespace folly
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOLLY_ARENA_H_
+#define FOLLY_ARENA_H_
+
+#include <cassert>
+#include <utility>
+#include <limits>
+#include <boost/intrusive/slist.hpp>
+
+#include "folly/Likely.h"
+#include "folly/Malloc.h"
+
+namespace folly {
+
+/**
+ * Simple arena: allocate memory which gets freed when the arena gets
+ * destroyed.
+ *
+ * The arena itself allocates memory using a custom allocator which provides
+ * the following interface (same as required by StlAllocator in StlAllocator.h)
+ *
+ * void* allocate(size_t size);
+ * Allocate a block of size bytes, properly aligned to the maximum
+ * alignment required on your system; throw std::bad_alloc if the
+ * allocation can't be satisfied.
+ *
+ * void deallocate(void* ptr);
+ * Deallocate a previously allocated block.
+ *
+ * You may also specialize ArenaAllocatorTraits for your allocator type to
+ * provide:
+ *
+ * size_t goodSize(const Allocator& alloc, size_t size) const;
+ * Return a size (>= the provided size) that is considered "good" for your
+ * allocator (for example, if your allocator allocates memory in 4MB
+ * chunks, size should be rounded up to 4MB). The provided value is
+ * guaranteed to be rounded up to a multiple of the maximum alignment
+ * required on your system; the returned value must be also.
+ *
+ * An implementation that uses malloc() / free() is defined below, see
+ * SysAlloc / SysArena.
+ */
+template <class Alloc> struct ArenaAllocatorTraits;
+template <class Alloc>
+class Arena {
+ public:
+ explicit Arena(const Alloc& alloc,
+ size_t minBlockSize = kDefaultMinBlockSize)
+ : allocAndSize_(alloc, minBlockSize),
+ ptr_(nullptr),
+ end_(nullptr) {
+ }
+
+ ~Arena();
+
+ void* allocate(size_t size) {
+ size = roundUp(size);
+
+ if (LIKELY(end_ - ptr_ >= size)) {
+ // Fast path: there's enough room in the current block
+ char* r = ptr_;
+ ptr_ += size;
+ assert(isAligned(r));
+ return r;
+ }
+
+ // Not enough room in the current block
+ void* r = allocateSlow(size);
+ assert(isAligned(r));
+ return r;
+ }
+
+ void deallocate(void* p) {
+ // Deallocate? Never!
+ }
+
+ // Transfer ownership of all memory allocated from "other" to "this".
+ void merge(Arena&& other);
+
+ private:
+ // not copyable
+ Arena(const Arena&) = delete;
+ Arena& operator=(const Arena&) = delete;
+
+ // movable
+ Arena(Arena&&) = default;
+ Arena& operator=(Arena&&) = default;
+
+ struct Block;
+ typedef boost::intrusive::slist_member_hook<
+ boost::intrusive::tag<Arena>> BlockLink;
+
+ struct Block {
+ BlockLink link;
+
+ // Allocate a block with at least size bytes of storage.
+ // If allowSlack is true, allocate more than size bytes if convenient
+ // (via ArenaAllocatorTraits::goodSize()) as we'll try to pack small
+ // allocations in this block.
+ static std::pair<Block*, size_t> allocate(
+ Alloc& alloc, size_t size, bool allowSlack);
+ void deallocate(Alloc& alloc);
+
+ char* start() {
+ return reinterpret_cast<char*>(this + 1);
+ }
+
+ private:
+ Block() { }
+ ~Block() { }
+ } __attribute__((aligned));
+ // This should be alignas(std::max_align_t) but neither alignas nor
+ // max_align_t are supported by gcc 4.6.2.
+
+ public:
+ static constexpr size_t kDefaultMinBlockSize = 4096 - sizeof(Block);
+
+ private:
+ static constexpr size_t maxAlign = alignof(Block);
+ static constexpr bool isAligned(uintptr_t address) {
+ return (address & (maxAlign - 1)) == 0;
+ }
+ static bool isAligned(void* p) {
+ return isAligned(reinterpret_cast<uintptr_t>(p));
+ }
+
+ // Round up size so it's properly aligned
+ static constexpr size_t roundUp(size_t size) {
+ return (size + maxAlign - 1) & ~(maxAlign - 1);
+ }
+
+ // cache_last<true> makes the list keep a pointer to the last element, so we
+ // have push_back() and constant time splice_after()
+ typedef boost::intrusive::slist<
+ Block,
+ boost::intrusive::member_hook<Block, BlockLink, &Block::link>,
+ boost::intrusive::constant_time_size<false>,
+ boost::intrusive::cache_last<true>> BlockList;
+
+ void* allocateSlow(size_t size);
+
+ // Empty member optimization: package Alloc with a non-empty member
+ // in case Alloc is empty (as it is in the case of SysAlloc).
+ struct AllocAndSize : public Alloc {
+ explicit AllocAndSize(const Alloc& a, size_t s)
+ : Alloc(a), minBlockSize(s) {
+ }
+
+ size_t minBlockSize;
+ };
+
+ size_t minBlockSize() const {
+ return allocAndSize_.minBlockSize;
+ }
+ Alloc& alloc() { return allocAndSize_; }
+ const Alloc& alloc() const { return allocAndSize_; }
+
+ AllocAndSize allocAndSize_;
+ BlockList blocks_;
+ char* ptr_;
+ char* end_;
+};
+
+/**
+ * By default, don't pad the given size.
+ */
+template <class Alloc>
+struct ArenaAllocatorTraits {
+ static size_t goodSize(const Alloc& alloc, size_t size) {
+ return size;
+ }
+};
+
+/**
+ * Arena-compatible allocator that calls malloc() and free(); see
+ * goodMallocSize() in Malloc.h for goodSize().
+ */
+class SysAlloc {
+ public:
+ void* allocate(size_t size) {
+ void* mem = malloc(size);
+ if (!mem) throw std::bad_alloc();
+ return mem;
+ }
+
+ void deallocate(void* p) {
+ free(p);
+ }
+};
+
+template <>
+struct ArenaAllocatorTraits<SysAlloc> {
+ static size_t goodSize(const SysAlloc& alloc, size_t size) {
+ return goodMallocSize(size);
+ }
+};
+
+/**
+ * Arena that uses the system allocator (malloc / free)
+ */
+class SysArena : public Arena<SysAlloc> {
+ public:
+ explicit SysArena(size_t minBlockSize = kDefaultMinBlockSize)
+ : Arena<SysAlloc>(SysAlloc(), minBlockSize) {
+ }
+};
+
+} // namespace folly
+
+#include "folly/Arena-inl.h"
+
+#endif /* FOLLY_ARENA_H_ */
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOLLY_ATOMICHASHARRAY_H_
+#error "This should only be included by AtomicHashArray.h"
+#endif
+
+#include "folly/Bits.h"
+#include "folly/detail/AtomicHashUtils.h"
+
+namespace folly {
+
+// AtomicHashArray private constructor --
+template <class KeyT, class ValueT, class HashFcn>
+AtomicHashArray<KeyT, ValueT, HashFcn>::
+AtomicHashArray(size_t capacity, KeyT emptyKey, KeyT lockedKey,
+ KeyT erasedKey, double maxLoadFactor, size_t cacheSize)
+ : capacity_(capacity), maxEntries_(size_t(maxLoadFactor * capacity_ + 0.5)),
+ kEmptyKey_(emptyKey), kLockedKey_(lockedKey), kErasedKey_(erasedKey),
+ kAnchorMask_(nextPowTwo(capacity_) - 1), numEntries_(0, cacheSize),
+ numPendingEntries_(0, cacheSize), isFull_(0), numErases_(0) {
+}
+
+/*
+ * findInternal --
+ *
+ * Sets ret.second to value found and ret.index to index
+ * of key and returns true, or if key does not exist returns false and
+ * ret.index is set to capacity_.
+ */
+template <class KeyT, class ValueT, class HashFcn>
+typename AtomicHashArray<KeyT, ValueT, HashFcn>::SimpleRetT
+AtomicHashArray<KeyT, ValueT, HashFcn>::
+findInternal(const KeyT key_in) {
+ DCHECK_NE(key_in, kEmptyKey_);
+ DCHECK_NE(key_in, kLockedKey_);
+ DCHECK_NE(key_in, kErasedKey_);
+ for (size_t idx = keyToAnchorIdx(key_in), numProbes = 0;
+ ;
+ idx = probeNext(idx, numProbes)) {
+ const KeyT key = relaxedLoadKey(cells_[idx]);
+ if (LIKELY(key == key_in)) {
+ return SimpleRetT(idx, true);
+ }
+ if (UNLIKELY(key == kEmptyKey_)) {
+ // if we hit an empty element, this key does not exist
+ return SimpleRetT(capacity_, false);
+ }
+ ++numProbes;
+ if (UNLIKELY(numProbes >= capacity_)) {
+ // probed every cell...fail
+ return SimpleRetT(capacity_, false);
+ }
+ }
+}
+
+/*
+ * insertInternal --
+ *
+ * Returns false on failure due to key collision or full.
+ * Also sets ret.index to the index of the key. If the map is full, sets
+ * ret.index = capacity_. Also sets ret.second to cell value, thus if insert
+ * successful this will be what we just inserted, if there is a key collision
+ * this will be the previously inserted value, and if the map is full it is
+ * default.
+ */
+template <class KeyT, class ValueT, class HashFcn>
+typename AtomicHashArray<KeyT, ValueT, HashFcn>::SimpleRetT
+AtomicHashArray<KeyT, ValueT, HashFcn>::
+insertInternal(const value_type& record) {
+ const short NO_NEW_INSERTS = 1;
+ const short NO_PENDING_INSERTS = 2;
+ const KeyT key_in = record.first;
+ CHECK_NE(key_in, kEmptyKey_);
+ CHECK_NE(key_in, kLockedKey_);
+ CHECK_NE(key_in, kErasedKey_);
+ for (size_t idx = keyToAnchorIdx(key_in), numProbes = 0;
+ ;
+ idx = probeNext(idx, numProbes)) {
+ DCHECK_LT(idx, capacity_);
+ value_type* cell = &cells_[idx];
+ if (relaxedLoadKey(*cell) == kEmptyKey_) {
+ // NOTE: isFull_ is set based on numEntries_.readFast(), so it's
+ // possible to insert more than maxEntries_ entries. However, it's not
+ // possible to insert past capacity_.
+ ++numPendingEntries_;
+ if (isFull_.load(std::memory_order_acquire)) {
+ --numPendingEntries_;
+
+ // Before deciding whether this insert succeeded, this thread needs to
+ // wait until no other thread can add a new entry.
+
+ // Correctness assumes isFull_ is true at this point. If
+ // another thread now does ++numPendingEntries_, we expect it
+ // to pass the isFull_.load() test above. (It shouldn't insert
+ // a new entry.)
+ FOLLY_SPIN_WAIT(
+ isFull_.load(std::memory_order_acquire) != NO_PENDING_INSERTS
+ && numPendingEntries_.readFull() != 0
+ );
+ isFull_.store(NO_PENDING_INSERTS, std::memory_order_release);
+
+ if (relaxedLoadKey(*cell) == kEmptyKey_) {
+ // Don't insert past max load factor
+ return SimpleRetT(capacity_, false);
+ }
+ } else {
+ // An unallocated cell. Try once to lock it. If we succeed, insert here.
+ // If we fail, fall through to comparison below; maybe the insert that
+ // just beat us was for this very key....
+ if (tryLockCell(cell)) {
+ // Write the value - done before unlocking
+ try {
+ DCHECK(relaxedLoadKey(*cell) == kLockedKey_);
+ /*
+ * This happens using the copy constructor because we won't have
+ * constructed a lhs to use an assignment operator on when
+ * values are being set.
+ */
+ new (&cell->second) ValueT(record.second);
+ unlockCell(cell, key_in); // Sets the new key
+ } catch (...) {
+ unlockCell(cell, kEmptyKey_);
+ --numPendingEntries_;
+ throw;
+ }
+ DCHECK(relaxedLoadKey(*cell) == key_in);
+ --numPendingEntries_;
+ ++numEntries_; // This is a thread cached atomic increment :)
+ if (numEntries_.readFast() >= maxEntries_) {
+ isFull_.store(NO_NEW_INSERTS, std::memory_order_relaxed);
+ }
+ return SimpleRetT(idx, true);
+ }
+ --numPendingEntries_;
+ }
+ }
+ DCHECK(relaxedLoadKey(*cell) != kEmptyKey_);
+ if (kLockedKey_ == cellKeyPtr(*cell)->load(std::memory_order_acquire)) {
+ FOLLY_SPIN_WAIT(
+ kLockedKey_ == cellKeyPtr(*cell)->load(std::memory_order_acquire)
+ );
+ }
+ DCHECK(relaxedLoadKey(*cell) != kEmptyKey_);
+ DCHECK(relaxedLoadKey(*cell) != kLockedKey_);
+ if (key_in == relaxedLoadKey(*cell)) {
+ // Found an existing entry for our key, but we don't overwrite the
+ // previous value.
+ return SimpleRetT(idx, false);
+ }
+ ++numProbes;
+ if (UNLIKELY(numProbes >= capacity_)) {
+ // probed every cell...fail
+ return SimpleRetT(capacity_, false);
+ }
+ }
+}
+
+
+/*
+ * erase --
+ *
+ * This will attempt to erase the given key key_in if the key is found. It
+ * returns 1 iff the key was located and marked as erased, and 0 otherwise.
+ *
+ * Memory is not freed or reclaimed by erase, i.e. the cell containing the
+ * erased key will never be reused. If there's an associated value, we won't
+ * touch it either.
+ */
+template <class KeyT, class ValueT, class HashFcn>
+size_t AtomicHashArray<KeyT, ValueT, HashFcn>::
+erase(KeyT key_in) {
+ CHECK_NE(key_in, kEmptyKey_);
+ CHECK_NE(key_in, kLockedKey_);
+ CHECK_NE(key_in, kErasedKey_);
+ for (size_t idx = keyToAnchorIdx(key_in), numProbes = 0;
+ ;
+ idx = probeNext(idx, numProbes)) {
+ DCHECK_LT(idx, capacity_);
+ value_type* cell = &cells_[idx];
+ if (relaxedLoadKey(*cell) == kEmptyKey_ ||
+ relaxedLoadKey(*cell) == kLockedKey_) {
+ // If we hit an empty (or locked) element, this key does not exist. This
+ // is similar to how it's handled in find().
+ return 0;
+ }
+ if (key_in == relaxedLoadKey(*cell)) {
+ // Found an existing entry for our key, attempt to mark it erased.
+ // Some other thread may have erased our key, but this is ok.
+ KeyT expect = key_in;
+ if (cellKeyPtr(*cell)->compare_exchange_strong(expect, kErasedKey_)) {
+ numErases_.fetch_add(1, std::memory_order_relaxed);
+
+ // Even if there's a value in the cell, we won't delete (or even
+ // default construct) it because some other thread may be accessing it.
+ // Locking it meanwhile won't work either since another thread may be
+ // holding a pointer to it.
+
+ // We found the key and successfully erased it.
+ return 1;
+ }
+ // If another thread succeeds in erasing our key, we'll stop our search.
+ return 0;
+ }
+ ++numProbes;
+ if (UNLIKELY(numProbes >= capacity_)) {
+ // probed every cell...fail
+ return 0;
+ }
+ }
+}
+
+template <class KeyT, class ValueT, class HashFcn>
+const typename AtomicHashArray<KeyT, ValueT, HashFcn>::Config
+AtomicHashArray<KeyT, ValueT, HashFcn>::defaultConfig;
+
+template <class KeyT, class ValueT, class HashFcn>
+typename AtomicHashArray<KeyT, ValueT, HashFcn>::SmartPtr
+AtomicHashArray<KeyT, ValueT, HashFcn>::
+create(size_t maxSize, const Config& c) {
+ CHECK_LE(c.maxLoadFactor, 1.0);
+ CHECK_GT(c.maxLoadFactor, 0.0);
+ CHECK_NE(c.emptyKey, c.lockedKey);
+ size_t capacity = size_t(maxSize / c.maxLoadFactor);
+ size_t sz = sizeof(AtomicHashArray) + sizeof(value_type) * capacity;
+
+ std::unique_ptr<void, void(*)(void*)> mem(malloc(sz), free);
+ new(mem.get()) AtomicHashArray(capacity, c.emptyKey, c.lockedKey, c.erasedKey,
+ c.maxLoadFactor, c.entryCountThreadCacheSize);
+ SmartPtr map(static_cast<AtomicHashArray*>(mem.release()));
+
+ /*
+ * Mark all cells as empty.
+ *
+ * Note: we're bending the rules a little here accessing the key
+ * element in our cells even though the cell object has not been
+ * constructed, and casting them to atomic objects (see cellKeyPtr).
+ * (Also, in fact we never actually invoke the value_type
+ * constructor.) This is in order to avoid needing to default
+ * construct a bunch of value_type when we first start up: if you
+ * have an expensive default constructor for the value type this can
+ * noticably speed construction time for an AHA.
+ */
+ FOR_EACH_RANGE(i, 0, map->capacity_) {
+ cellKeyPtr(map->cells_[i])->store(map->kEmptyKey_,
+ std::memory_order_relaxed);
+ }
+ return map;
+}
+
+template <class KeyT, class ValueT, class HashFcn>
+void AtomicHashArray<KeyT, ValueT, HashFcn>::
+destroy(AtomicHashArray* p) {
+ assert(p);
+ FOR_EACH_RANGE(i, 0, p->capacity_) {
+ if (p->cells_[i].first != p->kEmptyKey_) {
+ p->cells_[i].~value_type();
+ }
+ }
+ p->~AtomicHashArray();
+ free(p);
+}
+
+// clear -- clears all keys and values in the map and resets all counters
+template <class KeyT, class ValueT, class HashFcn>
+void AtomicHashArray<KeyT, ValueT, HashFcn>::
+clear() {
+ FOR_EACH_RANGE(i, 0, capacity_) {
+ if (cells_[i].first != kEmptyKey_) {
+ cells_[i].~value_type();
+ *const_cast<KeyT*>(&cells_[i].first) = kEmptyKey_;
+ }
+ CHECK(cells_[i].first == kEmptyKey_);
+ }
+ numEntries_.set(0);
+ numPendingEntries_.set(0);
+ isFull_.store(0, std::memory_order_relaxed);
+ numErases_.store(0, std::memory_order_relaxed);
+}
+
+
+// Iterator implementation
+
+template <class KeyT, class ValueT, class HashFcn>
+template <class ContT, class IterVal>
+struct AtomicHashArray<KeyT, ValueT, HashFcn>::aha_iterator
+ : boost::iterator_facade<aha_iterator<ContT,IterVal>,
+ IterVal,
+ boost::forward_traversal_tag>
+{
+ explicit aha_iterator() : aha_(0) {}
+
+ // Conversion ctor for interoperability between const_iterator and
+ // iterator. The enable_if<> magic keeps us well-behaved for
+ // is_convertible<> (v. the iterator_facade documentation).
+ template<class OtherContT, class OtherVal>
+ aha_iterator(const aha_iterator<OtherContT,OtherVal>& o,
+ typename std::enable_if<
+ std::is_convertible<OtherVal*,IterVal*>::value >::type* = 0)
+ : aha_(o.aha_)
+ , offset_(o.offset_)
+ {}
+
+ explicit aha_iterator(ContT* array, size_t offset)
+ : aha_(array)
+ , offset_(offset)
+ {
+ advancePastEmpty();
+ }
+
+ // Returns unique index that can be used with findAt().
+ // WARNING: The following function will fail silently for hashtable
+ // with capacity > 2^32
+ uint32_t getIndex() const { return offset_; }
+
+ private:
+ friend class AtomicHashArray;
+ friend class boost::iterator_core_access;
+
+ void increment() {
+ ++offset_;
+ advancePastEmpty();
+ }
+
+ bool equal(const aha_iterator& o) const {
+ return aha_ == o.aha_ && offset_ == o.offset_;
+ }
+
+ IterVal& dereference() const {
+ return aha_->cells_[offset_];
+ }
+
+ void advancePastEmpty() {
+ while (offset_ < aha_->capacity_ && !isValid()) {
+ ++offset_;
+ }
+ }
+
+ bool isValid() const {
+ KeyT key = relaxedLoadKey(aha_->cells_[offset_]);
+ return key != aha_->kEmptyKey_ &&
+ key != aha_->kLockedKey_ &&
+ key != aha_->kErasedKey_;
+ }
+
+ private:
+ ContT* aha_;
+ size_t offset_;
+}; // aha_iterator
+
+} // namespace folly
+
+#undef FOLLY_SPIN_WAIT
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * AtomicHashArray is the building block for AtomicHashMap. It provides the
+ * core lock-free functionality, but is limitted by the fact that it cannot
+ * grow past it's initialization size and is a little more awkward (no public
+ * constructor, for example). If you're confident that you won't run out of
+ * space, don't mind the awkardness, and really need bare-metal performance,
+ * feel free to use AHA directly.
+ *
+ * Check out AtomicHashMap.h for more thorough documentation on perf and
+ * general pros and cons relative to other hash maps.
+ *
+ * @author Spencer Ahrens <sahrens@fb.com>
+ * @author Jordan DeLong <delong.j@fb.com>
+ */
+
+#ifndef FOLLY_ATOMICHASHARRAY_H_
+#define FOLLY_ATOMICHASHARRAY_H_
+
+#include <atomic>
+
+#include <boost/iterator/iterator_facade.hpp>
+#include <boost/noncopyable.hpp>
+
+#include "folly/Hash.h"
+#include "folly/ThreadCachedInt.h"
+
+namespace folly {
+
+template <class KeyT, class ValueT, class HashFcn = std::hash<KeyT>>
+class AtomicHashMap;
+
+template <class KeyT, class ValueT, class HashFcn = std::hash<KeyT>>
+class AtomicHashArray : boost::noncopyable {
+ static_assert((std::is_convertible<KeyT,int32_t>::value ||
+ std::is_convertible<KeyT,int64_t>::value),
+ "You are trying to use AtomicHashArray with disallowed key "
+ "types. You must use atomically compare-and-swappable integer "
+ "keys, or a different container class.");
+ public:
+ typedef KeyT key_type;
+ typedef ValueT mapped_type;
+ typedef std::pair<const KeyT, ValueT> value_type;
+ typedef std::size_t size_type;
+ typedef std::ptrdiff_t difference_type;
+ typedef value_type& reference;
+ typedef const value_type& const_reference;
+ typedef value_type* pointer;
+ typedef const value_type* const_pointer;
+
+ const size_t capacity_;
+ const size_t maxEntries_;
+ const KeyT kEmptyKey_;
+ const KeyT kLockedKey_;
+ const KeyT kErasedKey_;
+
+ template<class ContT, class IterVal>
+ struct aha_iterator;
+
+ typedef aha_iterator<const AtomicHashArray,const value_type> const_iterator;
+ typedef aha_iterator<AtomicHashArray,value_type> iterator;
+
+ // You really shouldn't need this if you use the SmartPtr provided by create,
+ // but if you really want to do something crazy like stick the released
+ // pointer into a DescriminatedPtr or something, you'll need this to clean up
+ // after yourself.
+ static void destroy(AtomicHashArray*);
+
+ private:
+ const size_t kAnchorMask_;
+
+ struct Deleter {
+ void operator()(AtomicHashArray* ptr) {
+ AtomicHashArray::destroy(ptr);
+ }
+ };
+
+ public:
+ typedef std::unique_ptr<AtomicHashArray, Deleter> SmartPtr;
+
+ /*
+ * create --
+ *
+ * Creates AtomicHashArray objects. Use instead of constructor/destructor.
+ *
+ * We do things this way in order to avoid the perf penalty of a second
+ * pointer indirection when composing these into AtomicHashMap, which needs
+ * to store an array of pointers so that it can perform atomic operations on
+ * them when growing.
+ *
+ * Instead of a mess of arguments, we take a max size and a Config struct to
+ * simulate named ctor parameters. The Config struct has sensible defaults
+ * for everything, but is overloaded - if you specify a positive capacity,
+ * that will be used directly instead of computing it based on
+ * maxLoadFactor.
+ *
+ * Create returns an AHA::SmartPtr which is a unique_ptr with a custom
+ * deleter to make sure everything is cleaned up properly.
+ */
+ struct Config {
+ KeyT emptyKey;
+ KeyT lockedKey;
+ KeyT erasedKey;
+ double maxLoadFactor;
+ int entryCountThreadCacheSize;
+ size_t capacity; // if positive, overrides maxLoadFactor
+
+ constexpr Config() : emptyKey(static_cast<KeyT>(-1ul)),
+ lockedKey(static_cast<KeyT>(-2ul)),
+ erasedKey(static_cast<KeyT>(-3ul)),
+ maxLoadFactor(0.8),
+ entryCountThreadCacheSize(1000),
+ capacity(0) {}
+ };
+
+ static const Config defaultConfig;
+ static SmartPtr create(size_t maxSize, const Config& = defaultConfig);
+
+ iterator find(KeyT k) {
+ return iterator(this, findInternal(k).idx);
+ }
+ const_iterator find(KeyT k) const {
+ return const_cast<AtomicHashArray*>(this)->find(k);
+ }
+
+ /*
+ * insert --
+ *
+ * Returns a pair with iterator to the element at r.first and bool success.
+ * Retrieve the index with ret.first.getIndex().
+ *
+ * Fails on key collision (does not overwrite) or if map becomes
+ * full, at which point no element is inserted, iterator is set to end(),
+ * and success is set false. On collisions, success is set false, but the
+ * iterator is set to the existing entry.
+ */
+ std::pair<iterator,bool> insert(const value_type& r) {
+ SimpleRetT ret = insertInternal(r);
+ return std::make_pair(iterator(this, ret.idx), ret.success);
+ }
+
+ // returns the number of elements erased - should never exceed 1
+ size_t erase(KeyT k);
+
+ // clears all keys and values in the map and resets all counters. Not thread
+ // safe.
+ void clear();
+
+ // Exact number of elements in the map - note that readFull() acquires a
+ // mutex. See folly/ThreadCachedInt.h for more details.
+ size_t size() const {
+ return numEntries_.readFull() -
+ numErases_.load(std::memory_order_relaxed);
+ }
+
+ bool empty() const { return size() == 0; }
+
+ iterator begin() { return iterator(this, 0); }
+ iterator end() { return iterator(this, capacity_); }
+ const_iterator begin() const { return const_iterator(this, 0); }
+ const_iterator end() const { return const_iterator(this, capacity_); }
+
+ // See AtomicHashMap::findAt - access elements directly
+ // WARNING: The following 2 functions will fail silently for hashtable
+ // with capacity > 2^32
+ iterator findAt(uint32_t idx) {
+ DCHECK_LT(idx, capacity_);
+ return iterator(this, idx);
+ }
+ const_iterator findAt(uint32_t idx) const {
+ return const_cast<AtomicHashArray*>(this)->findAt(idx);
+ }
+
+ iterator makeIter(size_t idx) { return iterator(this, idx); }
+ const_iterator makeIter(size_t idx) const {
+ return const_iterator(this, idx);
+ }
+
+ // The max load factor allowed for this map
+ double maxLoadFactor() const { return ((double) maxEntries_) / capacity_; }
+
+ void setEntryCountThreadCacheSize(uint32_t newSize) {
+ numEntries_.setCacheSize(newSize);
+ numPendingEntries_.setCacheSize(newSize);
+ }
+
+ int getEntryCountThreadCacheSize() const {
+ return numEntries_.getCacheSize();
+ }
+
+ /* Private data and helper functions... */
+
+ private:
+ friend class AtomicHashMap<KeyT,ValueT,HashFcn>;
+
+ struct SimpleRetT { size_t idx; bool success;
+ SimpleRetT(size_t i, bool s) : idx(i), success(s) {}
+ SimpleRetT() {}
+ };
+
+ SimpleRetT insertInternal(const value_type& record);
+
+ SimpleRetT findInternal(const KeyT key);
+
+ static std::atomic<KeyT>* cellKeyPtr(const value_type& r) {
+ // We need some illegal casting here in order to actually store
+ // our value_type as a std::pair<const,>. But a little bit of
+ // undefined behavior never hurt anyone ...
+ static_assert(sizeof(std::atomic<KeyT>) == sizeof(KeyT),
+ "std::atomic is implemented in an unexpected way for AHM");
+ return
+ const_cast<std::atomic<KeyT>*>(
+ reinterpret_cast<std::atomic<KeyT> const*>(&r.first));
+ }
+
+ static KeyT relaxedLoadKey(const value_type& r) {
+ return cellKeyPtr(r)->load(std::memory_order_relaxed);
+ }
+
+ // Fun with thread local storage - atomic increment is expensive
+ // (relatively), so we accumulate in the thread cache and periodically
+ // flush to the actual variable, and walk through the unflushed counts when
+ // reading the value, so be careful of calling size() too frequently. This
+ // increases insertion throughput several times over while keeping the count
+ // accurate.
+ ThreadCachedInt<int64_t> numEntries_; // Successful key inserts
+ ThreadCachedInt<int64_t> numPendingEntries_; // Used by insertInternal
+ std::atomic<int64_t> isFull_; // Used by insertInternal
+ std::atomic<int64_t> numErases_; // Successful key erases
+
+ value_type cells_[0]; // This must be the last field of this class
+
+ // Force constructor/destructor private since create/destroy should be
+ // used externally instead
+ AtomicHashArray(size_t capacity, KeyT emptyKey, KeyT lockedKey,
+ KeyT erasedKey, double maxLoadFactor, size_t cacheSize);
+
+ ~AtomicHashArray() {}
+
+ inline void unlockCell(value_type* const cell, KeyT newKey) {
+ cellKeyPtr(*cell)->store(newKey, std::memory_order_release);
+ }
+
+ inline bool tryLockCell(value_type* const cell) {
+ KeyT expect = kEmptyKey_;
+ return cellKeyPtr(*cell)->compare_exchange_strong(expect, kLockedKey_,
+ std::memory_order_acquire);
+ }
+
+ inline size_t keyToAnchorIdx(const KeyT k) const {
+ const size_t hashVal = HashFcn()(k);
+ const size_t probe = hashVal & kAnchorMask_;
+ return LIKELY(probe < capacity_) ? probe : hashVal % capacity_;
+ }
+
+ inline size_t probeNext(size_t idx, size_t numProbes) {
+ //idx += numProbes; // quadratic probing
+ idx += 1; // linear probing
+ // Avoid modulus because it's slow
+ return LIKELY(idx < capacity_) ? idx : (idx - capacity_);
+ }
+}; // AtomicHashArray
+
+} // namespace folly
+
+#include "AtomicHashArray-inl.h"
+
+#endif // FOLLY_ATOMICHASHARRAY_H_
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOLLY_ATOMICHASHMAP_H_
+#error "This should only be included by AtomicHashMap.h"
+#endif
+
+#include "folly/detail/AtomicHashUtils.h"
+
+namespace folly {
+
+template <class KeyT, class ValueT, class HashFcn>
+const typename AtomicHashMap<KeyT, ValueT, HashFcn>::Config
+AtomicHashMap<KeyT, ValueT, HashFcn>::defaultConfig;
+
+// AtomicHashMap constructor -- Atomic wrapper that allows growth
+// This class has a lot of overhead (184 Bytes) so only use for big maps
+template <typename KeyT, typename ValueT, typename HashFcn>
+AtomicHashMap<KeyT, ValueT, HashFcn>::
+AtomicHashMap(size_t size, const Config& config)
+ : kGrowthFrac_(1.0 - config.maxLoadFactor) {
+ CHECK(config.maxLoadFactor > 0.0 && config.maxLoadFactor < 1.0);
+ subMaps_[0].store(SubMap::create(size, config).release(),
+ std::memory_order_relaxed);
+ auto numSubMaps = kNumSubMaps_;
+ FOR_EACH_RANGE(i, 1, numSubMaps) {
+ subMaps_[i].store(nullptr, std::memory_order_relaxed);
+ }
+ numMapsAllocated_.store(1, std::memory_order_relaxed);
+}
+
+// insert --
+template <typename KeyT, typename ValueT, typename HashFcn>
+std::pair<typename AtomicHashMap<KeyT,ValueT,HashFcn>::iterator,bool>
+AtomicHashMap<KeyT, ValueT, HashFcn>::
+insert(const value_type& r) {
+ SimpleRetT ret = insertInternal(r);
+ SubMap* subMap = subMaps_[ret.i].load(std::memory_order_relaxed);
+ return std::make_pair(iterator(this, ret.i, subMap->makeIter(ret.j)),
+ ret.success);
+}
+
+// insertInternal -- Allocates new sub maps as existing ones fill up.
+template <typename KeyT, typename ValueT, typename HashFcn>
+typename AtomicHashMap<KeyT, ValueT, HashFcn>::SimpleRetT
+AtomicHashMap<KeyT, ValueT, HashFcn>::
+insertInternal(const value_type& r) {
+ beginInsertInternal:
+ int nextMapIdx = // this maintains our state
+ numMapsAllocated_.load(std::memory_order_acquire);
+ uint32_t idx = 0;
+ typename SubMap::SimpleRetT ret;
+ FOR_EACH_RANGE(i, 0, nextMapIdx) {
+ // insert in each map successively. If one succeeds, we're done!
+ SubMap* subMap = subMaps_[i].load(std::memory_order_relaxed);
+ ret = subMap->insertInternal(r);
+ if (ret.idx == subMap->capacity_) {
+ continue; //map is full, so try the next one
+ }
+ // Either collision or success - insert in either case
+ return SimpleRetT(i, ret.idx, ret.success);
+ }
+
+ // If we made it this far, all maps are full and we need to try to allocate
+ // the next one.
+
+ SubMap* primarySubMap = subMaps_[0].load(std::memory_order_relaxed);
+ if (nextMapIdx >= kNumSubMaps_ ||
+ primarySubMap->capacity_ * kGrowthFrac_ < 1.0) {
+ // Can't allocate any more sub maps.
+ throw AtomicHashMapFullError();
+ }
+
+ if (tryLockMap(nextMapIdx)) {
+ // Alloc a new map and shove it in. We can change whatever
+ // we want because other threads are waiting on us...
+ size_t numCellsAllocated = (size_t)
+ (primarySubMap->capacity_ *
+ std::pow(1.0 + kGrowthFrac_, nextMapIdx - 1));
+ size_t newSize = (int) (numCellsAllocated * kGrowthFrac_);
+ DCHECK(subMaps_[nextMapIdx].load(std::memory_order_relaxed) ==
+ (SubMap*)kLockedPtr_);
+ // create a new map using the settings stored in the first map
+
+ Config config;
+ config.emptyKey = primarySubMap->kEmptyKey_;
+ config.lockedKey = primarySubMap->kLockedKey_;
+ config.erasedKey = primarySubMap->kErasedKey_;
+ config.maxLoadFactor = primarySubMap->maxLoadFactor();
+ config.entryCountThreadCacheSize =
+ primarySubMap->getEntryCountThreadCacheSize();
+ subMaps_[nextMapIdx].store(SubMap::create(newSize, config).release(),
+ std::memory_order_relaxed);
+
+ // Publish the map to other threads.
+ numMapsAllocated_.fetch_add(1, std::memory_order_release);
+ DCHECK_EQ(nextMapIdx + 1,
+ numMapsAllocated_.load(std::memory_order_relaxed));
+ } else {
+ // If we lost the race, we'll have to wait for the next map to get
+ // allocated before doing any insertion here.
+ FOLLY_SPIN_WAIT(
+ nextMapIdx >= numMapsAllocated_.load(std::memory_order_acquire)
+ );
+ }
+
+ // Relaxed is ok here because either we just created this map, or we
+ // just did a spin wait with an acquire load on numMapsAllocated_.
+ SubMap* loadedMap = subMaps_[nextMapIdx].load(std::memory_order_relaxed);
+ DCHECK(loadedMap && loadedMap != (SubMap*)kLockedPtr_);
+ ret = loadedMap->insertInternal(r);
+ if (ret.idx != loadedMap->capacity_) {
+ return SimpleRetT(nextMapIdx, ret.idx, ret.success);
+ }
+ // We took way too long and the new map is already full...try again from
+ // the top (this should pretty much never happen).
+ goto beginInsertInternal;
+}
+
+// find --
+template <typename KeyT, typename ValueT, typename HashFcn>
+typename AtomicHashMap<KeyT, ValueT, HashFcn>::iterator
+AtomicHashMap<KeyT, ValueT, HashFcn>::
+find(KeyT k) {
+ SimpleRetT ret = findInternal(k);
+ if (ret.i >= numMapsAllocated_.load(std::memory_order_acquire)) {
+ return end();
+ }
+ SubMap* subMap = subMaps_[ret.i].load(std::memory_order_relaxed);
+ return iterator(this, ret.i, subMap->makeIter(ret.j));
+}
+
+template <typename KeyT, typename ValueT, typename HashFcn>
+typename AtomicHashMap<KeyT, ValueT, HashFcn>::const_iterator
+AtomicHashMap<KeyT, ValueT, HashFcn>::
+find(KeyT k) const {
+ return const_cast<AtomicHashMap*>(this)->find(k);
+}
+
+// findInternal --
+template <typename KeyT, typename ValueT, typename HashFcn>
+typename AtomicHashMap<KeyT, ValueT, HashFcn>::SimpleRetT
+AtomicHashMap<KeyT, ValueT, HashFcn>::
+findInternal(const KeyT k) const {
+ SubMap* const primaryMap = subMaps_[0].load(std::memory_order_relaxed);
+ typename SubMap::SimpleRetT ret = primaryMap->findInternal(k);
+ if (LIKELY(ret.idx != primaryMap->capacity_)) {
+ return SimpleRetT(0, ret.idx, ret.success);
+ }
+ int const numMaps = numMapsAllocated_.load(std::memory_order_acquire);
+ FOR_EACH_RANGE(i, 1, numMaps) {
+ // Check each map successively. If one succeeds, we're done!
+ SubMap* thisMap = subMaps_[i].load(std::memory_order_release);
+ ret = thisMap->findInternal(k);
+ if (LIKELY(ret.idx != thisMap->capacity_)) {
+ return SimpleRetT(i, ret.idx, ret.success);
+ }
+ }
+ // Didn't find our key...
+ return SimpleRetT(numMaps, 0, false);
+}
+
+// findAtInternal -- see encodeIndex() for details.
+template <typename KeyT, typename ValueT, typename HashFcn>
+typename AtomicHashMap<KeyT, ValueT, HashFcn>::SimpleRetT
+AtomicHashMap<KeyT, ValueT, HashFcn>::
+findAtInternal(uint32_t idx) const {
+ uint32_t subMapIdx, subMapOffset;
+ if (idx & kSecondaryMapBit_) {
+ // idx falls in a secondary map
+ idx &= ~kSecondaryMapBit_; // unset secondary bit
+ subMapIdx = idx >> kSubMapIndexShift_;
+ DCHECK_LT(subMapIdx, numMapsAllocated_.load(std::memory_order_relaxed));
+ subMapOffset = idx & kSubMapIndexMask_;
+ } else {
+ // idx falls in primary map
+ subMapIdx = 0;
+ subMapOffset = idx;
+ }
+ return SimpleRetT(subMapIdx, subMapOffset, true);
+}
+
+// erase --
+template <typename KeyT, typename ValueT, typename HashFcn>
+typename AtomicHashMap<KeyT, ValueT, HashFcn>::size_type
+AtomicHashMap<KeyT, ValueT, HashFcn>::
+erase(const KeyT k) {
+ int const numMaps = numMapsAllocated_.load(std::memory_order_acquire);
+ FOR_EACH_RANGE(i, 0, numMaps) {
+ // Check each map successively. If one succeeds, we're done!
+ if (subMaps_[i].load(std::memory_order_relaxed)->erase(k)) {
+ return 1;
+ }
+ }
+ // Didn't find our key...
+ return 0;
+}
+
+// capacity -- summation of capacities of all submaps
+template <typename KeyT, typename ValueT, typename HashFcn>
+size_t AtomicHashMap<KeyT, ValueT, HashFcn>::
+capacity() const {
+ size_t totalCap(0);
+ int const numMaps = numMapsAllocated_.load(std::memory_order_acquire);
+ FOR_EACH_RANGE(i, 0, numMaps) {
+ totalCap += subMaps_[i].load(std::memory_order_relaxed)->capacity_;
+ }
+ return totalCap;
+}
+
+// spaceRemaining --
+// number of new insertions until current submaps are all at max load
+template <typename KeyT, typename ValueT, typename HashFcn>
+size_t AtomicHashMap<KeyT, ValueT, HashFcn>::
+spaceRemaining() const {
+ size_t spaceRem(0);
+ int const numMaps = numMapsAllocated_.load(std::memory_order_acquire);
+ FOR_EACH_RANGE(i, 0, numMaps) {
+ SubMap* thisMap = subMaps_[i].load(std::memory_order_relaxed);
+ spaceRem += std::max(
+ 0,
+ thisMap->maxEntries_ - &thisMap->numEntries_.readFull()
+ );
+ }
+ return spaceRem;
+}
+
+// clear -- Wipes all keys and values from primary map and destroys
+// all secondary maps. Not thread safe.
+template <typename KeyT, typename ValueT, typename HashFcn>
+void AtomicHashMap<KeyT, ValueT, HashFcn>::
+clear() {
+ subMaps_[0].load(std::memory_order_relaxed)->clear();
+ int const numMaps = numMapsAllocated_
+ .load(std::memory_order_relaxed);
+ FOR_EACH_RANGE(i, 1, numMaps) {
+ SubMap* thisMap = subMaps_[i].load(std::memory_order_relaxed);
+ DCHECK(thisMap);
+ SubMap::destroy(thisMap);
+ subMaps_[i].store(nullptr, std::memory_order_relaxed);
+ }
+ numMapsAllocated_.store(1, std::memory_order_relaxed);
+}
+
+// size --
+template <typename KeyT, typename ValueT, typename HashFcn>
+size_t AtomicHashMap<KeyT, ValueT, HashFcn>::
+size() const {
+ size_t totalSize(0);
+ int const numMaps = numMapsAllocated_.load(std::memory_order_acquire);
+ FOR_EACH_RANGE(i, 0, numMaps) {
+ totalSize += subMaps_[i].load(std::memory_order_relaxed)->size();
+ }
+ return totalSize;
+}
+
+// encodeIndex -- Encode the submap index and offset into return.
+// index_ret must be pre-populated with the submap offset.
+//
+// We leave index_ret untouched when referring to the primary map
+// so it can be as large as possible (31 data bits). Max size of
+// secondary maps is limited by what can fit in the low 27 bits.
+//
+// Returns the following bit-encoded data in index_ret:
+// if subMap == 0 (primary map) =>
+// bit(s) value
+// 31 0
+// 0-30 submap offset (index_ret input)
+//
+// if subMap > 0 (secondary maps) =>
+// bit(s) value
+// 31 1
+// 27-30 which subMap
+// 0-26 subMap offset (index_ret input)
+template <typename KeyT, typename ValueT, typename HashFcn>
+inline uint32_t AtomicHashMap<KeyT, ValueT, HashFcn>::
+encodeIndex(uint32_t subMap, uint32_t offset) {
+ DCHECK_EQ(offset & kSecondaryMapBit_, 0); // offset can't be too big
+ if (subMap == 0) return offset;
+ // Make sure subMap isn't too big
+ DCHECK_EQ(subMap >> kNumSubMapBits_, 0);
+ // Make sure subMap bits of offset are clear
+ DCHECK_EQ(offset & (~kSubMapIndexMask_ | kSecondaryMapBit_), 0);
+
+ // Set high-order bits to encode which submap this index belongs to
+ return offset | (subMap << kSubMapIndexShift_) | kSecondaryMapBit_;
+}
+
+
+// Iterator implementation
+
+template <typename KeyT, typename ValueT, typename HashFcn>
+template<class ContT, class IterVal, class SubIt>
+struct AtomicHashMap<KeyT, ValueT, HashFcn>::ahm_iterator
+ : boost::iterator_facade<ahm_iterator<ContT,IterVal,SubIt>,
+ IterVal,
+ boost::forward_traversal_tag>
+{
+ explicit ahm_iterator() : ahm_(0) {}
+
+ // Conversion ctor for interoperability between const_iterator and
+ // iterator. The enable_if<> magic keeps us well-behaved for
+ // is_convertible<> (v. the iterator_facade documentation).
+ template<class OtherContT, class OtherVal, class OtherSubIt>
+ ahm_iterator(const ahm_iterator<OtherContT,OtherVal,OtherSubIt>& o,
+ typename std::enable_if<
+ std::is_convertible<OtherSubIt,SubIt>::value >::type* = 0)
+ : ahm_(o.ahm_)
+ , subMap_(o.subMap_)
+ , subIt_(o.subIt_)
+ {}
+
+ /*
+ * Returns the unique index that can be used for access directly
+ * into the data storage.
+ */
+ uint32_t getIndex() const {
+ CHECK(!isEnd());
+ return ahm_->encodeIndex(subMap_, subIt_.getIndex());
+ }
+
+ private:
+ friend class AtomicHashMap;
+ explicit ahm_iterator(ContT* ahm,
+ uint32_t subMap,
+ const SubIt& subIt)
+ : ahm_(ahm)
+ , subMap_(subMap)
+ , subIt_(subIt)
+ {
+ checkAdvanceToNextSubmap();
+ }
+
+ friend class boost::iterator_core_access;
+
+ void increment() {
+ CHECK(!isEnd());
+ ++subIt_;
+ checkAdvanceToNextSubmap();
+ }
+
+ bool equal(const ahm_iterator& other) const {
+ if (ahm_ != other.ahm_) {
+ return false;
+ }
+
+ if (isEnd() || other.isEnd()) {
+ return isEnd() == other.isEnd();
+ }
+
+ return subMap_ == other.subMap_ &&
+ subIt_ == other.subIt_;
+ }
+
+ IterVal& dereference() const {
+ return *subIt_;
+ }
+
+ bool isEnd() const { return ahm_ == nullptr; }
+
+ void checkAdvanceToNextSubmap() {
+ if (isEnd()) {
+ return;
+ }
+
+ SubMap* thisMap = ahm_->subMaps_[subMap_].
+ load(std::memory_order_relaxed);
+ if (subIt_ == thisMap->end()) {
+ // This sub iterator is done, advance to next one
+ if (subMap_ + 1 <
+ ahm_->numMapsAllocated_.load(std::memory_order_acquire)) {
+ ++subMap_;
+ thisMap = ahm_->subMaps_[subMap_].load(std::memory_order_relaxed);
+ subIt_ = thisMap->begin();
+ } else {
+ ahm_ = nullptr;
+ }
+ }
+ }
+
+ private:
+ ContT* ahm_;
+ uint32_t subMap_;
+ SubIt subIt_;
+}; // ahm_iterator
+
+} // namespace folly
+
+#undef FOLLY_SPIN_WAIT
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * AtomicHashMap --
+ *
+ * A high performance concurrent hash map with int32 or int64 keys. Supports
+ * insert, find(key), findAt(index), erase(key), size, and more. Memory cannot
+ * be freed or reclaimed by erase. Can grow to a maximum of about 18 times the
+ * initial capacity, but performance degrades linearly with growth. Can also be
+ * used as an object store with unique 32-bit references directly into the
+ * internal storage (retrieved with iterator::getIndex()).
+ *
+ * Advantages:
+ * - High performance (~2-4x tbb::concurrent_hash_map in heavily
+ * multi-threaded environments).
+ * - Efficient memory usage if initial capacity is not over estimated
+ * (especially for small keys and values).
+ * - Good fragmentation properties (only allocates in large slabs which can
+ * be reused with clear() and never move).
+ * - Can generate unique, long-lived 32-bit references for efficient lookup
+ * (see findAt()).
+ *
+ * Disadvantages:
+ * - Keys must be native int32 or int64, or explicitly converted.
+ * - Must be able to specify unique empty, locked, and erased keys
+ * - Performance degrades linearly as size grows beyond initialization
+ * capacity.
+ * - Max size limit of ~18x initial size (dependent on max load factor).
+ * - Memory is not freed or reclaimed by erase.
+ *
+ * Usage and Operation Details:
+ * Simple performance/memory tradeoff with maxLoadFactor. Higher load factors
+ * give better memory utilization but probe lengths increase, reducing
+ * performance.
+ *
+ * Implementation and Performance Details:
+ * AHArray is a fixed size contiguous block of value_type cells. When
+ * writing a cell, the key is locked while the rest of the record is
+ * written. Once done, the cell is unlocked by setting the key. find()
+ * is completely wait-free and doesn't require any non-relaxed atomic
+ * operations. AHA cannot grow beyond initialization capacity, but is
+ * faster because of reduced data indirection.
+ *
+ * AHMap is a wrapper around AHArray sub-maps that allows growth and provides
+ * an interface closer to the stl UnorderedAssociativeContainer concept. These
+ * sub-maps are allocated on the fly and are processed in series, so the more
+ * there are (from growing past initial capacity), the worse the performance.
+ *
+ * Insert returns false if there is a key collision and throws if the max size
+ * of the map is exceeded.
+ *
+ * Benchmark performance with 8 simultaneous threads processing 1 million
+ * unique <int64, int64> entries on a 4-core, 2.5 GHz machine:
+ *
+ * Load Factor Mem Efficiency usec/Insert usec/Find
+ * 50% 50% 0.19 0.05
+ * 85% 85% 0.20 0.06
+ * 90% 90% 0.23 0.08
+ * 95% 95% 0.27 0.10
+ *
+ * See folly/tests/AtomicHashMapTest.cpp for more benchmarks.
+ *
+ * @author Spencer Ahrens <sahrens@fb.com>
+ * @author Jordan DeLong <delong.j@fb.com>
+ *
+ */
+
+#ifndef FOLLY_ATOMICHASHMAP_H_
+#define FOLLY_ATOMICHASHMAP_H_
+
+#include <boost/iterator/iterator_facade.hpp>
+#include <boost/noncopyable.hpp>
+#include <boost/type_traits/is_convertible.hpp>
+#include <glog/logging.h>
+
+#include <stdexcept>
+#include <functional>
+#include <atomic>
+
+#include "folly/AtomicHashArray.h"
+#include "folly/Foreach.h"
+#include "folly/Hash.h"
+#include "folly/Likely.h"
+#include "folly/ThreadCachedInt.h"
+
+namespace folly {
+
+/*
+ * AtomicHashMap provides an interface somewhat similar to the
+ * UnorderedAssociativeContainer concept in C++. This does not
+ * exactly match this concept (or even the basic Container concept),
+ * because of some restrictions imposed by our datastructure.
+ *
+ * Specific differences (there are quite a few):
+ *
+ * - Efficiently thread safe for inserts (main point of this stuff),
+ * wait-free for lookups.
+ *
+ * - You can erase from this container, but the cell containing the key will
+ * not be free or reclaimed.
+ *
+ * - You can erase everything by calling clear() (and you must guarantee only
+ * one thread can be using the container to do that).
+ *
+ * - We aren't DefaultConstructible, CopyConstructible, Assignable, or
+ * EqualityComparable. (Most of these are probably not something
+ * you actually want to do with this anyway.)
+ *
+ * - We don't support the various bucket functions, rehash(),
+ * reserve(), or equal_range(). Also no constructors taking
+ * iterators, although this could change.
+ *
+ * - Several insertion functions, notably operator[], are not
+ * implemented. It is a little too easy to misuse these functions
+ * with this container, where part of the point is that when an
+ * insertion happens for a new key, it will atomically have the
+ * desired value.
+ *
+ * - The map has no templated insert() taking an iterator range, but
+ * we do provide an insert(key, value). The latter seems more
+ * frequently useful for this container (to avoid sprinkling
+ * make_pair everywhere), and providing both can lead to some gross
+ * template error messages.
+ *
+ * - Not Allocator-aware.
+ *
+ * - KeyT must be a 32 bit or 64 bit atomic integer type, and you must
+ * define special 'locked' and 'empty' key values in the ctor
+ *
+ * - We don't take the Hash function object as an instance in the
+ * constructor.
+ *
+ * - We don't take a Compare template parameter (since our keys must
+ * be integers, and the underlying hash array here uses atomic
+ * compare-and-swap instructions, we only allow normal integer
+ * comparisons).
+ */
+
+// Thrown when insertion fails due to running out of space for
+// submaps.
+struct AtomicHashMapFullError : std::runtime_error {
+ explicit AtomicHashMapFullError()
+ : std::runtime_error("AtomicHashMap is full")
+ {}
+};
+
+template<class KeyT, class ValueT, class HashFcn>
+class AtomicHashMap : boost::noncopyable {
+ typedef AtomicHashArray<KeyT, ValueT, HashFcn> SubMap;
+
+ public:
+ typedef KeyT key_type;
+ typedef ValueT mapped_type;
+ typedef std::pair<const KeyT, ValueT> value_type;
+ typedef HashFcn hasher;
+ typedef std::equal_to<KeyT> key_equal;
+ typedef value_type* pointer;
+ typedef value_type& reference;
+ typedef const value_type& const_reference;
+ typedef std::ptrdiff_t difference_type;
+ typedef std::size_t size_type;
+ typedef typename SubMap::Config Config;
+
+ template<class ContT, class IterVal, class SubIt>
+ struct ahm_iterator;
+
+ typedef ahm_iterator<const AtomicHashMap,
+ const value_type,
+ typename SubMap::const_iterator>
+ const_iterator;
+ typedef ahm_iterator<AtomicHashMap,
+ value_type,
+ typename SubMap::iterator>
+ iterator;
+
+ public:
+ const float kGrowthFrac_; // How much to grow when we run out of capacity.
+
+ // The constructor takes a finalSizeEst which is the optimal
+ // number of elements to maximize space utilization and performance,
+ // and a Config object to specify more advanced options.
+ static const Config defaultConfig;
+ explicit AtomicHashMap(size_t finalSizeEst, const Config& = defaultConfig);
+
+ ~AtomicHashMap() {
+ const int numMaps = numMapsAllocated_.load(std::memory_order_relaxed);
+ FOR_EACH_RANGE (i, 0, numMaps) {
+ SubMap* thisMap = subMaps_[i].load(std::memory_order_relaxed);
+ DCHECK(thisMap);
+ SubMap::destroy(thisMap);
+ }
+ }
+
+ key_equal key_eq() const { return key_eq(); }
+ hasher hash_function() const { return hasher(); }
+
+ // TODO: emplace() support would be nice.
+
+ /*
+ * insert --
+ *
+ * Returns a pair with iterator to the element at r.first and
+ * success. Retrieve the index with ret.first.getIndex().
+ *
+ * Does not overwrite on key collision, but returns an iterator to
+ * the existing element (since this could due to a race with
+ * another thread, it is often important to check this return
+ * value).
+ *
+ * Allocates new sub maps as the existing ones become full. If
+ * all sub maps are full, no element is inserted, and
+ * AtomicHashMapFullError is thrown.
+ */
+ std::pair<iterator,bool> insert(const value_type& r);
+ std::pair<iterator,bool> insert(key_type k, const mapped_type& v) {
+ return insert(value_type(k, v));
+ }
+
+ /*
+ * find --
+ *
+ * Returns an iterator into the map.
+ *
+ * If the key is not found, returns end().
+ */
+ iterator find(key_type k);
+ const_iterator find(key_type k) const;
+
+ /*
+ * erase --
+ *
+ * Erases key k from the map
+ *
+ * Returns 1 iff the key is found and erased, and 0 otherwise.
+ */
+ size_type erase(key_type k);
+
+ /*
+ * clear --
+ *
+ * Wipes all keys and values from primary map and destroys all secondary
+ * maps. Primary map remains allocated and thus the memory can be reused
+ * in place. Not thread safe.
+ *
+ */
+ void clear();
+
+ /*
+ * size --
+ *
+ * Returns the exact size of the map. Note this is not as cheap as typical
+ * size() implementations because, for each AtomicHashArray in this AHM, we
+ * need to grab a lock and accumulate the values from all the thread local
+ * counters. See folly/ThreadCachedInt.h for more details.
+ */
+ size_t size() const;
+
+ bool empty() const { return size() == 0; }
+
+ size_type count(key_type k) const {
+ return find(k) == end() ? 0 : 1;
+ }
+
+
+ /*
+ * findAt --
+ *
+ * Returns an iterator into the map.
+ *
+ * idx should only be an unmodified value returned by calling getIndex() on
+ * a valid iterator returned by find() or insert(). If idx is invalid you
+ * have a bug and the process aborts.
+ */
+ iterator findAt(uint32_t idx) {
+ SimpleRetT ret = findAtInternal(idx);
+ DCHECK_LT(ret.i, numSubMaps());
+ return iterator(this, ret.i,
+ subMaps_[ret.i].load(std::memory_order_relaxed)->makeIter(ret.j));
+ }
+ const_iterator findAt(uint32_t idx) const {
+ return const_cast<AtomicHashMap*>(this)->findAt(idx);
+ }
+
+ // Total capacity - summation of capacities of all submaps.
+ size_t capacity() const;
+
+ // Number of new insertions until current submaps are all at max load factor.
+ size_t spaceRemaining() const;
+
+ void setEntryCountThreadCacheSize(int32_t newSize) {
+ const int numMaps = numMapsAllocated_.load(std::memory_order_acquire);
+ for (int i = 0; i < numMaps; ++i) {
+ SubMap* map = subMaps_[i].load(std::memory_order_relaxed);
+ map->setEntryCountThreadCacheSize(newSize);
+ }
+ }
+
+ // Number of sub maps allocated so far to implement this map. The more there
+ // are, the worse the performance.
+ int numSubMaps() const {
+ return numMapsAllocated_.load(std::memory_order_acquire);
+ }
+
+ iterator begin() {
+ return iterator(this, 0,
+ subMaps_[0].load(std::memory_order_relaxed)->begin());
+ }
+
+ iterator end() {
+ return iterator();
+ }
+
+ const_iterator begin() const {
+ return const_iterator(this, 0,
+ subMaps_[0].load(std::memory_order_relaxed)->begin());
+ }
+
+ const_iterator end() const {
+ return const_iterator();
+ }
+
+ /* Advanced functions for direct access: */
+
+ inline uint32_t recToIdx(const value_type& r, bool mayInsert = true) {
+ SimpleRetT ret = mayInsert ? insertInternal(r) : findInternal(r.first);
+ return encodeIndex(ret.i, ret.j);
+ }
+
+ inline uint32_t keyToIdx(const KeyT k, bool mayInsert = false) {
+ return recToIdx(value_type(k), mayInsert);
+ }
+
+ inline const value_type& idxToRec(uint32_t idx) const {
+ SimpleRetT ret = findAtInternal(idx);
+ return subMaps_[ret.i].load(std::memory_order_relaxed)->idxToRec(ret.j);
+ }
+
+ /* Private data and helper functions... */
+
+ private:
+ // This limits primary submap size to 2^31 ~= 2 billion, secondary submap
+ // size to 2^(32 - kNumSubMapBits_ - 1) = 2^27 ~= 130 million, and num subMaps
+ // to 2^kNumSubMapBits_ = 16.
+ static const uint32_t kNumSubMapBits_ = 4;
+ static const uint32_t kSecondaryMapBit_ = 1u << 31; // Highest bit
+ static const uint32_t kSubMapIndexShift_ = 32 - kNumSubMapBits_ - 1;
+ static const uint32_t kSubMapIndexMask_ = (1 << kSubMapIndexShift_) - 1;
+ static const uint32_t kNumSubMaps_ = 1 << kNumSubMapBits_;
+ static const uintptr_t kLockedPtr_ = 0x88ul << 48; // invalid pointer
+
+ struct SimpleRetT { uint32_t i; size_t j; bool success;
+ SimpleRetT(uint32_t ii, size_t jj, bool s) : i(ii), j(jj), success(s) {}
+ SimpleRetT() {}
+ };
+
+ SimpleRetT insertInternal(const value_type& r);
+
+ SimpleRetT findInternal(const KeyT k) const;
+
+ SimpleRetT findAtInternal(const uint32_t idx) const;
+
+ std::atomic<SubMap*> subMaps_[kNumSubMaps_];
+ std::atomic<uint32_t> numMapsAllocated_;
+
+ inline bool tryLockMap(int idx) {
+ SubMap* val = nullptr;
+ return subMaps_[idx].compare_exchange_strong(val, (SubMap*)kLockedPtr_,
+ std::memory_order_acquire);
+ }
+
+ static inline uint32_t encodeIndex(uint32_t subMap, uint32_t subMapIdx);
+
+}; // AtomicHashMap
+
+} // namespace folly
+
+#include "AtomicHashMap-inl.h"
+
+#endif // FOLLY_ATOMICHASHMAP_H_
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @author Andrei Alexandrescu (andrei.alexandrescu@fb.com)
+
+#include "Benchmark.h"
+#include "Foreach.h"
+#include "String.h"
+#include <algorithm>
+#include <cmath>
+#include <iostream>
+#include <limits>
+#include <utility>
+#include <vector>
+
+using namespace std;
+
+DEFINE_bool(benchmark, false, "Run benchmarks.");
+
+namespace folly {
+
+BenchmarkSuspender::NanosecondsSpent BenchmarkSuspender::nsSpent;
+
+typedef function<uint64_t(unsigned int)> BenchmarkFun;
+static vector<tuple<const char*, const char*, BenchmarkFun>> benchmarks;
+
+// Add the global baseline
+BENCHMARK(globalBenchmarkBaseline) {
+ asm volatile("");
+}
+
+void detail::addBenchmarkImpl(const char* file, const char* name,
+ BenchmarkFun fun) {
+ benchmarks.emplace_back(file, name, std::move(fun));
+}
+
+/**
+ * Given a point, gives density at that point as a number 0.0 < x <=
+ * 1.0. The result is 1.0 if all samples are equal to where, and
+ * decreases near 0 if all points are far away from it. The density is
+ * computed with the help of a radial basis function.
+ */
+static double density(const double * begin, const double *const end,
+ const double where, const double bandwidth) {
+ assert(begin < end);
+ assert(bandwidth > 0.0);
+ double sum = 0.0;
+ FOR_EACH_RANGE (i, begin, end) {
+ auto d = (*i - where) / bandwidth;
+ sum += exp(- d * d);
+ }
+ return sum / (end - begin);
+}
+
+/**
+ * Computes mean and variance for a bunch of data points. Note that
+ * mean is currently not being used.
+ */
+static pair<double, double>
+meanVariance(const double * begin, const double *const end) {
+ assert(begin < end);
+ double sum = 0.0, sum2 = 0.0;
+ FOR_EACH_RANGE (i, begin, end) {
+ sum += *i;
+ sum2 += *i * *i;
+ }
+ auto const n = end - begin;
+ return make_pair(sum / n, sqrt((sum2 - sum * sum / n) / n));
+}
+
+/**
+ * Computes the mode of a sample set through brute force. Assumes
+ * input is sorted.
+ */
+static double mode(const double * begin, const double *const end) {
+ assert(begin < end);
+ // Lower bound and upper bound for result and their respective
+ // densities.
+ auto
+ result = 0.0,
+ bestDensity = 0.0;
+
+ // Get the variance so we pass it down to density()
+ auto const sigma = meanVariance(begin, end).second;
+ if (!sigma) {
+ // No variance means constant signal
+ return *begin;
+ }
+
+ FOR_EACH_RANGE (i, begin, end) {
+ assert(i == begin || *i >= i[-1]);
+ auto candidate = density(begin, end, *i, sigma * sqrt(2.0));
+ if (candidate > bestDensity) {
+ // Found a new best
+ bestDensity = candidate;
+ result = *i;
+ } else {
+ // Density is decreasing... we could break here if we definitely
+ // knew this is unimodal.
+ }
+ }
+
+ return result;
+}
+
+/**
+ * Given a bunch of benchmark samples, estimate the actual run time.
+ */
+static double estimateTime(double * begin, double * end) {
+ assert(begin < end);
+
+ // Current state of the art: get the minimum. After some
+ // experimentation, it seems taking the minimum is the best.
+
+ return *min_element(begin, end);
+
+ // What follows after estimates the time as the mode of the
+ // distribution.
+
+ // Select the awesomest (i.e. most frequent) result. We do this by
+ // sorting and then computing the longest run length.
+ sort(begin, end);
+
+ // Eliminate outliers. A time much larger than the minimum time is
+ // considered an outlier.
+ while (end[-1] > 2.0 * *begin) {
+ --end;
+ if (begin == end) {
+ LOG(INFO) << *begin;
+ }
+ assert(begin < end);
+ }
+
+ double result = 0;
+
+ /* Code used just for comparison purposes */ {
+ unsigned bestFrequency = 0;
+ unsigned candidateFrequency = 1;
+ double candidateValue = *begin;
+ for (auto current = begin + 1; ; ++current) {
+ if (current == end || *current != candidateValue) {
+ // Done with the current run, see if it was best
+ if (candidateFrequency > bestFrequency) {
+ bestFrequency = candidateFrequency;
+ result = candidateValue;
+ }
+ if (current == end) {
+ break;
+ }
+ // Start a new run
+ candidateValue = *current;
+ candidateFrequency = 1;
+ } else {
+ // Cool, inside a run, increase the frequency
+ ++candidateFrequency;
+ }
+ }
+ }
+
+ result = mode(begin, end);
+
+ return result;
+}
+
+static double runBenchmarkGetNSPerIteration(const BenchmarkFun& fun,
+ const double globalBaseline) {
+ // They key here is accuracy; too low numbers means the accuracy was
+ // coarse. We up the ante until we get to at least minNanoseconds
+ // timings.
+ static uint64_t resolutionInNs = 0, coarseResolutionInNs = 0;
+ if (!resolutionInNs) {
+ timespec ts;
+ CHECK_EQ(0, clock_getres(detail::DEFAULT_CLOCK_ID, &ts));
+ CHECK_EQ(0, ts.tv_sec) << "Clock sucks.";
+ CHECK_LT(0, ts.tv_nsec) << "Clock too fast for its own good.";
+ CHECK_EQ(1, ts.tv_nsec) << "Clock too coarse, upgrade your kernel.";
+ resolutionInNs = ts.tv_nsec;
+ }
+ // Whe choose a minimum minimum (sic) of 10,000 nanoseconds, but if
+ // the clock resolution is worse than that, it will be larger. In
+ // essence we're aiming at making the quantization noise 0.01%.
+ static const auto minNanoseconds = min(resolutionInNs * 100000, 1000000000UL);
+
+ // We do measurements in several epochs and take the minimum, to
+ // account for jitter.
+ static const unsigned int epochs = 1000;
+ // We establish a total time budget as we don't want a measurement
+ // to take too long. This will curtail the number of actual epochs.
+ static const uint64_t timeBudgetInNs = 1000000000;
+ timespec global;
+ CHECK_EQ(0, clock_gettime(CLOCK_REALTIME, &global));
+
+ double epochResults[epochs] = { 0 };
+ size_t actualEpochs = 0;
+
+ for (; actualEpochs < epochs; ++actualEpochs) {
+ for (unsigned int n = 1; n < (1U << 30); n *= 2) {
+ auto const nsecs = fun(n);
+ if (nsecs < minNanoseconds) {
+ continue;
+ }
+ // We got an accurate enough timing, done. But only save if
+ // smaller than the current result.
+ epochResults[actualEpochs] = max(0.0, double(nsecs) / n - globalBaseline);
+ // Done with the current epoch, we got a meaningful timing.
+ break;
+ }
+ timespec now;
+ CHECK_EQ(0, clock_gettime(CLOCK_REALTIME, &now));
+ if (detail::timespecDiff(now, global) >= timeBudgetInNs) {
+ // No more time budget available.
+ ++actualEpochs;
+ break;
+ }
+ }
+
+ // If the benchmark was basically drowned in baseline noise, it's
+ // possible it became negative.
+ return max(0.0, estimateTime(epochResults, epochResults + actualEpochs));
+}
+
+static string humanReadable(double n, unsigned int decimals) {
+ auto a = fabs(n);
+ char suffix = ' ';
+
+ if (a >= 1E21) {
+ // Too big to be comprehended by the puny human brain
+ suffix = '!';
+ n /= 1E21;
+ } else if (a >= 1E18) {
+ // "EXA" written with suffix 'X' so as to not create confusion
+ // with scientific notation.
+ suffix = 'X';
+ n /= 1E18;
+ } else if (a >= 1E15) {
+ // "PETA"
+ suffix = 'P';
+ n /= 1E15;
+ } else if (a >= 1E12) {
+ // "TERA"
+ suffix = 'T';
+ n /= 1E12;
+ } else if (a >= 1E9) {
+ // "GIGA"
+ suffix = 'G';
+ n /= 1E9;
+ } else if (a >= 1E6) {
+ // "MEGA"
+ suffix = 'M';
+ n /= 1E6;
+ } else if (a >= 1E3) {
+ // "KILO"
+ suffix = 'K';
+ n /= 1E3;
+ } else if (a == 0.0) {
+ suffix = ' ';
+ } else if (a < 1E-15) {
+ // too small
+ suffix = '?';
+ n *= 1E18;
+ } else if (a < 1E-12) {
+ // "femto"
+ suffix = 'f';
+ n *= 1E15;
+ } else if (a < 1E-9) {
+ // "pico"
+ suffix = 'p';
+ n *= 1E12;
+ } else if (a < 1E-6) {
+ // "nano"
+ suffix = 'n';
+ n *= 1E9;
+ } else if (a < 1E-3) {
+ // "micro"
+ suffix = 'u';
+ n *= 1E6;
+ } else if (a < 1) {
+ // "mili"
+ suffix = 'm';
+ n *= 1E3;
+ }
+
+ return stringPrintf("%*.*f%c", decimals + 3 + 1, decimals, n, suffix);
+}
+
+static void printBenchmarkResults(
+ const vector<tuple<const char*, const char*, double> >& data) {
+ // Width available
+ static const uint columns = 76;
+
+ // Compute the longest benchmark name
+ size_t longestName = 0;
+ FOR_EACH_RANGE (i, 1, benchmarks.size()) {
+ longestName = max(longestName, strlen(get<1>(benchmarks[i])));
+ }
+
+ // Print a horizontal rule
+ auto separator = [&](char pad) {
+ puts(string(columns, pad).c_str());
+ };
+
+ // Print header for a file
+ auto header = [&](const char* file) {
+ separator('=');
+ printf("%-*srelative ns/iter iters/s\n",
+ columns - 26, file);
+ separator('=');
+ };
+
+ double baselineNsPerIter = numeric_limits<double>::max();
+ const char* lastFile = "";
+
+ for (auto& datum : data) {
+ auto file = get<0>(datum);
+ if (strcmp(file, lastFile)) {
+ // New file starting
+ header(file);
+ lastFile = file;
+ }
+
+ string s = get<1>(datum);
+ if (s == "-") {
+ separator('-');
+ continue;
+ }
+ bool useBaseline /* = void */;
+ if (s[0] == '%') {
+ s.erase(0, 1);
+ useBaseline = true;
+ } else {
+ baselineNsPerIter = get<2>(datum);
+ useBaseline = false;
+ }
+ s.resize(columns - 27, ' ');
+ auto nsPerIter = get<2>(datum);
+ auto itersPerSec = 1E9 / nsPerIter;
+ if (!useBaseline) {
+ // Print without baseline
+ printf("%*s %s %s\n",
+ static_cast<int>(s.size()), s.c_str(),
+ humanReadable(nsPerIter, 2).c_str(),
+ humanReadable(itersPerSec, 2).c_str());
+ } else {
+ // Print with baseline
+ auto rel = baselineNsPerIter / nsPerIter * 100.0;
+ printf("%*s %7.2f%% %s %s\n",
+ static_cast<int>(s.size()), s.c_str(),
+ rel,
+ humanReadable(nsPerIter, 2).c_str(),
+ humanReadable(itersPerSec, 2).c_str());
+ }
+ }
+ separator('=');
+}
+
+void runBenchmarks() {
+ CHECK(!benchmarks.empty());
+
+ vector<tuple<const char*, const char*, double>> results;
+ results.reserve(benchmarks.size() - 1);
+
+ // PLEASE KEEP QUIET. MEASUREMENTS IN PROGRESS.
+
+ auto const globalBaseline = runBenchmarkGetNSPerIteration(
+ get<2>(benchmarks.front()), 0);
+ FOR_EACH_RANGE (i, 1, benchmarks.size()) {
+ auto elapsed = strcmp(get<1>(benchmarks[i]), "-") == 0
+ ? 0.0 // skip the separators
+ : runBenchmarkGetNSPerIteration(get<2>(benchmarks[i]),
+ globalBaseline);
+ results.emplace_back(get<0>(benchmarks[i]),
+ get<1>(benchmarks[i]), elapsed);
+ }
+
+ // PLEASE MAKE NOISE. MEASUREMENTS DONE.
+
+ printBenchmarkResults(results);
+}
+
+} // namespace folly
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOLLY_BENCHMARK_H_
+#define FOLLY_BENCHMARK_H_
+
+#include "folly/Preprocessor.h" // for FB_ANONYMOUS_VARIABLE
+#include <cassert>
+#include <ctime>
+#include <boost/function_types/function_arity.hpp>
+#include <functional>
+#include <glog/logging.h>
+#include <gflags/gflags.h>
+#include <limits>
+
+DECLARE_bool(benchmark);
+
+namespace folly {
+
+/**
+ * Runs all benchmarks defined. Usually put in main().
+ */
+void runBenchmarks();
+
+/**
+ * Runs all benchmarks defined if and only if the --benchmark flag has
+ * been passed to the program. Usually put in main().
+ */
+inline bool runBenchmarksOnFlag() {
+ if (FLAGS_benchmark) {
+ runBenchmarks();
+ }
+ return FLAGS_benchmark;
+}
+
+namespace detail {
+
+/**
+ * This is the clock ID used for measuring time. On older kernels, the
+ * resolution of this clock will be very coarse, which will cause the
+ * benchmarks to fail.
+ */
+enum Clock { DEFAULT_CLOCK_ID = CLOCK_REALTIME };
+
+/**
+ * Adds a benchmark wrapped in a std::function. Only used
+ * internally. Pass by value is intentional.
+ */
+void addBenchmarkImpl(const char* file,
+ const char* name,
+ std::function<uint64_t(unsigned int)>);
+
+/**
+ * Takes the difference between two timespec values. end is assumed to
+ * occur after start.
+ */
+inline uint64_t timespecDiff(timespec end, timespec start) {
+ if (end.tv_sec == start.tv_sec) {
+ assert(end.tv_nsec >= start.tv_nsec);
+ return end.tv_nsec - start.tv_nsec;
+ }
+ assert(end.tv_sec > start.tv_sec &&
+ end.tv_sec - start.tv_sec <
+ std::numeric_limits<uint64_t>::max() / 1000000000UL);
+ return (end.tv_sec - start.tv_sec) * 1000000000UL
+ + end.tv_nsec - start.tv_nsec;
+}
+
+/**
+ * Takes the difference between two sets of timespec values. The first
+ * two come from a high-resolution clock whereas the other two come
+ * from a low-resolution clock. The crux of the matter is that
+ * high-res values may be bogus as documented in
+ * http://linux.die.net/man/3/clock_gettime. The trouble is when the
+ * running process migrates from one CPU to another, which is more
+ * likely for long-running processes. Therefore we watch for high
+ * differences between the two timings.
+ *
+ * This function is subject to further improvements.
+ */
+inline uint64_t timespecDiff(timespec end, timespec start,
+ timespec endCoarse, timespec startCoarse) {
+ auto fine = timespecDiff(end, start);
+ auto coarse = timespecDiff(endCoarse, startCoarse);
+ if (coarse - fine >= 1000000) {
+ // The fine time is in all likelihood bogus
+ return coarse;
+ }
+ return fine;
+}
+
+} // namespace detail
+
+/**
+ * Supporting type for BENCHMARK_SUSPEND defined below.
+ */
+struct BenchmarkSuspender {
+ BenchmarkSuspender() {
+ CHECK_EQ(0, clock_gettime(detail::DEFAULT_CLOCK_ID, &start));
+ }
+
+ BenchmarkSuspender(const BenchmarkSuspender &) = delete;
+ BenchmarkSuspender(BenchmarkSuspender && rhs) {
+ start = rhs.start;
+ rhs.start.tv_nsec = rhs.start.tv_sec = 0;
+ }
+
+ BenchmarkSuspender& operator=(const BenchmarkSuspender &) = delete;
+ BenchmarkSuspender& operator=(BenchmarkSuspender && rhs) {
+ if (start.tv_nsec > 0 || start.tv_sec > 0) {
+ tally();
+ }
+ start = rhs.start;
+ rhs.start.tv_nsec = rhs.start.tv_sec = 0;
+ return *this;
+ }
+
+ ~BenchmarkSuspender() {
+ if (start.tv_nsec > 0 || start.tv_sec > 0) {
+ tally();
+ }
+ }
+
+ void dismiss() {
+ assert(start.tv_nsec > 0 || start.tv_sec > 0);
+ tally();
+ start.tv_nsec = start.tv_sec = 0;
+ }
+
+ void rehire() {
+ assert(start.tv_nsec == 0 || start.tv_sec == 0);
+ CHECK_EQ(0, clock_gettime(detail::DEFAULT_CLOCK_ID, &start));
+ }
+
+ /**
+ * This helps the macro definition. To get around the dangers of
+ * operator bool, returns a pointer to member (which allows no
+ * arithmetic).
+ */
+ operator int BenchmarkSuspender::*() const {
+ return nullptr;
+ }
+
+ /**
+ * Accumulates nanoseconds spent outside benchmark.
+ */
+ typedef uint64_t NanosecondsSpent;
+ static NanosecondsSpent nsSpent;
+
+private:
+ void tally() {
+ timespec end;
+ CHECK_EQ(0, clock_gettime(detail::DEFAULT_CLOCK_ID, &end));
+ nsSpent += detail::timespecDiff(end, start);
+ start = end;
+ }
+
+ timespec start;
+};
+
+/**
+ * Adds a benchmark. Usually not called directly but instead through
+ * the macro BENCHMARK defined below. The lambda function involved
+ * must take exactly one parameter of type unsigned, and the benchmark
+ * uses it with counter semantics (iteration occurs inside the
+ * function).
+ */
+template <typename Lambda>
+typename std::enable_if<
+ boost::function_types::function_arity<decltype(&Lambda::operator())>::value
+ == 2
+>::type
+addBenchmark(const char* file, const char* name, Lambda&& lambda) {
+ auto execute = [=](unsigned int times) {
+ BenchmarkSuspender::nsSpent = 0;
+ timespec start, end;
+
+ // CORE MEASUREMENT STARTS
+ CHECK_EQ(0, clock_gettime(detail::DEFAULT_CLOCK_ID, &start));
+ lambda(times);
+ CHECK_EQ(0, clock_gettime(detail::DEFAULT_CLOCK_ID, &end));
+ // CORE MEASUREMENT ENDS
+
+ return detail::timespecDiff(end, start) - BenchmarkSuspender::nsSpent;
+ };
+
+ detail::addBenchmarkImpl(file, name,
+ std::function<uint64_t(unsigned int)>(execute));
+}
+
+/**
+ * Adds a benchmark. Usually not called directly but instead through
+ * the macro BENCHMARK defined below. The lambda function involved
+ * must take zero parameters, and the benchmark calls it repeatedly
+ * (iteration occurs outside the function).
+ */
+template <typename Lambda>
+typename std::enable_if<
+ boost::function_types::function_arity<decltype(&Lambda::operator())>::value
+ == 1
+>::type
+addBenchmark(const char* file, const char* name, Lambda&& lambda) {
+ addBenchmark(file, name, [=](unsigned int times) {
+ while (times-- > 0) {
+ lambda();
+ }
+ });
+}
+
+/**
+ * Call doNotOptimizeAway(var) against variables that you use for
+ * benchmarking but otherwise are useless. The compiler tends to do a
+ * good job at eliminating unused variables, and this function fools
+ * it into thinking var is in fact needed.
+ */
+template <class T>
+void doNotOptimizeAway(T&& datum) {
+ asm volatile("" : "+r" (datum));
+}
+
+} // namespace folly
+
+/**
+ * Introduces a benchmark function. Used internally, see BENCHMARK and
+ * friends below.
+ */
+#define BENCHMARK_IMPL(funName, stringName, paramType, paramName) \
+ static void funName(paramType); \
+ static bool FB_ANONYMOUS_VARIABLE(follyBenchmarkUnused) = ( \
+ ::folly::addBenchmark(__FILE__, stringName, \
+ [](paramType paramName) { funName(paramName); }), \
+ true); \
+ static void funName(paramType paramName)
+
+/**
+ * Introduces a benchmark function. Use with either one one or two
+ * arguments. The first is the name of the benchmark. Use something
+ * descriptive, such as insertVectorBegin. The second argument may be
+ * missing, or could be a symbolic counter. The counter dictates how
+ * many internal iteration the benchmark does. Example:
+ *
+ * BENCHMARK(vectorPushBack) {
+ * vector<int> v;
+ * v.push_back(42);
+ * }
+ *
+ * BENCHMARK(insertVectorBegin, n) {
+ * vector<int> v;
+ * FOR_EACH_RANGE (i, 0, n) {
+ * v.insert(v.begin(), 42);
+ * }
+ * }
+ */
+#define BENCHMARK(name, ...) \
+ BENCHMARK_IMPL( \
+ name, \
+ FB_STRINGIZE(name), \
+ FB_ONE_OR_NONE(unsigned, ## __VA_ARGS__), \
+ __VA_ARGS__)
+
+/**
+ * Defines a benchmark that passes a parameter to another one. This is
+ * common for benchmarks that need a "problem size" in addition to
+ * "number of iterations". Consider:
+ *
+ * void pushBack(uint n, size_t initialSize) {
+ * vector<int> v;
+ * BENCHMARK_SUSPEND {
+ * v.resize(initialSize);
+ * }
+ * FOR_EACH_RANGE (i, 0, n) {
+ * v.push_back(i);
+ * }
+ * }
+ * BENCHMARK_PARAM(pushBack, 0)
+ * BENCHMARK_PARAM(pushBack, 1000)
+ * BENCHMARK_PARAM(pushBack, 1000000)
+ *
+ * The benchmark above estimates the speed of push_back at different
+ * initial sizes of the vector. The framework will pass 0, 1000, and
+ * 1000000 for initialSize, and the iteration count for n.
+ */
+#define BENCHMARK_PARAM(name, param) \
+ BENCHMARK_IMPL( \
+ FB_CONCATENATE(name, FB_CONCATENATE(_, param)), \
+ FB_STRINGIZE(name) "(" FB_STRINGIZE(param) ")", \
+ unsigned, \
+ iters) { \
+ name(iters, param); \
+ }
+
+/**
+ * Just like BENCHMARK, but prints the time relative to a
+ * baseline. The baseline is the most recent BENCHMARK() seen in
+ * lexical order. Example:
+ *
+ * // This is the baseline
+ * BENCHMARK(insertVectorBegin, n) {
+ * vector<int> v;
+ * FOR_EACH_RANGE (i, 0, n) {
+ * v.insert(v.begin(), 42);
+ * }
+ * }
+ *
+ * BENCHMARK_RELATIVE(insertListBegin, n) {
+ * list<int> s;
+ * FOR_EACH_RANGE (i, 0, n) {
+ * s.insert(s.begin(), 42);
+ * }
+ * }
+ *
+ * Any number of relative benchmark can be associated with a
+ * baseline. Another BENCHMARK() occurrence effectively establishes a
+ * new baseline.
+ */
+#define BENCHMARK_RELATIVE(name, ...) \
+ BENCHMARK_IMPL( \
+ name, \
+ "%" FB_STRINGIZE(name), \
+ FB_ONE_OR_NONE(unsigned, ## __VA_ARGS__), \
+ __VA_ARGS__)
+
+/**
+ * A combination of BENCHMARK_RELATIVE and BENCHMARK_PARAM.
+ */
+#define BENCHMARK_RELATIVE_PARAM(name, param) \
+ BENCHMARK_IMPL( \
+ FB_CONCATENATE(name, FB_CONCATENATE(_, param)), \
+ "%" FB_STRINGIZE(name) "(" FB_STRINGIZE(param) ")", \
+ unsigned, \
+ iters) { \
+ name(iters, param); \
+ }
+
+/**
+ * Draws a line of dashes.
+ */
+#define BENCHMARK_DRAW_LINE() \
+ static bool FB_ANONYMOUS_VARIABLE(follyBenchmarkUnused) = ( \
+ ::folly::addBenchmark(__FILE__, "-", []() { }), \
+ true);
+
+/**
+ * Allows execution of code that doesn't count torward the benchmark's
+ * time budget. Example:
+ *
+ * BENCHMARK_START_GROUP(insertVectorBegin, n) {
+ * vector<int> v;
+ * SUSPEND_BENCHMARK {
+ * v.reserve(n);
+ * }
+ * FOR_EACH_RANGE (i, 0, n) {
+ * v.insert(v.begin(), 42);
+ * }
+ * }
+ */
+#define BENCHMARK_SUSPEND \
+ if (auto FB_ANONYMOUS_VARIABLE(BENCHMARK_SUSPEND) = \
+ ::folly::BenchmarkSuspender()) {} \
+ else
+
+#endif // FOLLY_BENCHMARK_H_
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Various low-level, bit-manipulation routines.
+ *
+ * findFirstSet(x)
+ * find first (least significant) bit set in a value of an integral type,
+ * 1-based (like ffs()). 0 = no bits are set (x == 0)
+ *
+ * findLastSet(x)
+ * find last (most significant) bit set in a value of an integral type,
+ * 1-based. 0 = no bits are set (x == 0)
+ * for x != 0, findFirstSet(x) == 1 + floor(log2(x))
+ *
+ * nextPowTwo(x)
+ * Finds the next power of two >= x.
+ *
+ * Endian
+ * convert between native, big, and little endian representation
+ * Endian::big(x) big <-> native
+ * Endian::little(x) little <-> native
+ * Endian::swap(x) big <-> little
+ *
+ * BitIterator
+ * Wrapper around an iterator over an integral type that iterates
+ * over its underlying bits in MSb to LSb order
+ *
+ * findFirstSet(BitIterator begin, BitIterator end)
+ * return a BitIterator pointing to the first 1 bit in [begin, end), or
+ * end if all bits in [begin, end) are 0
+ *
+ * @author Tudor Bosman (tudorb@fb.com)
+ */
+
+#ifndef FOLLY_BITS_H_
+#define FOLLY_BITS_H_
+
+#include "folly/Portability.h"
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE 1
+#endif
+
+#include "folly/detail/BitIteratorDetail.h"
+#include "folly/Likely.h"
+
+#include <byteswap.h>
+#include <cassert>
+#include <cinttypes>
+#include <cstring> // for ffs, ffsl, ffsll
+#include <endian.h>
+#include <iterator>
+#include <limits>
+#include <type_traits>
+#include <boost/iterator/iterator_adaptor.hpp>
+#include <stdint.h>
+
+namespace folly {
+
+// Generate overloads for findFirstSet as wrappers around
+// appropriate ffs, ffsl, ffsll functions from glibc.
+// We first define these overloads for signed types (because ffs, ffsl, ffsll
+// take int, long, and long long as arguments, respectively) and then
+// define an overload for unsigned that forwards to the overload for the
+// corresponding signed type.
+template <class T>
+typename std::enable_if<
+ (std::is_integral<T>::value &&
+ std::is_signed<T>::value &&
+ (std::numeric_limits<T>::digits <= std::numeric_limits<int>::digits)),
+ unsigned int>::type
+ findFirstSet(T x) {
+ return ::ffs(static_cast<int>(x));
+}
+
+template <class T>
+typename std::enable_if<
+ (std::is_integral<T>::value &&
+ std::is_signed<T>::value &&
+ (std::numeric_limits<T>::digits > std::numeric_limits<int>::digits) &&
+ (std::numeric_limits<T>::digits <= std::numeric_limits<long>::digits)),
+ unsigned int>::type
+ findFirstSet(T x) {
+ return ::ffsl(static_cast<long>(x));
+}
+
+#ifdef FOLLY_HAVE_FFSLL
+
+template <class T>
+typename std::enable_if<
+ (std::is_integral<T>::value &&
+ std::is_signed<T>::value &&
+ (std::numeric_limits<T>::digits > std::numeric_limits<long>::digits) &&
+ (std::numeric_limits<T>::digits <= std::numeric_limits<long long>::digits)),
+ unsigned int>::type
+ findFirstSet(T x) {
+ return ::ffsll(static_cast<long long>(x));
+}
+
+#endif
+
+template <class T>
+typename std::enable_if<
+ (std::is_integral<T>::value &&
+ !std::is_signed<T>::value),
+ unsigned int>::type
+ findFirstSet(T x) {
+ // Note that conversion from an unsigned type to the corresponding signed
+ // type is technically implementation-defined, but will likely work
+ // on any impementation that uses two's complement.
+ return findFirstSet(static_cast<typename std::make_signed<T>::type>(x));
+}
+
+namespace detail {
+
+// Portable, but likely slow...
+inline unsigned int findLastSetPortable(uint64_t x) {
+ unsigned int r = (x != 0); // 1-based index, except for x==0
+ while (x >>= 1) {
+ ++r;
+ }
+ return r;
+}
+
+} // namespace detail
+
+#ifdef __GNUC__
+
+// findLastSet: return the 1-based index of the highest bit set
+// for x > 0, findLastSet(x) == 1 + floor(log2(x))
+template <class T>
+typename std::enable_if<
+ (std::is_integral<T>::value &&
+ std::is_unsigned<T>::value &&
+ (std::numeric_limits<T>::digits <=
+ std::numeric_limits<unsigned int>::digits)),
+ unsigned int>::type
+ findLastSet(T x) {
+ return x ? 8 * sizeof(unsigned int) - __builtin_clz(x) : 0;
+}
+
+template <class T>
+typename std::enable_if<
+ (std::is_integral<T>::value &&
+ std::is_unsigned<T>::value &&
+ (std::numeric_limits<T>::digits >
+ std::numeric_limits<unsigned int>::digits) &&
+ (std::numeric_limits<T>::digits <=
+ std::numeric_limits<unsigned long>::digits)),
+ unsigned int>::type
+ findLastSet(T x) {
+ return x ? 8 * sizeof(unsigned long) - __builtin_clzl(x) : 0;
+}
+
+template <class T>
+typename std::enable_if<
+ (std::is_integral<T>::value &&
+ std::is_unsigned<T>::value &&
+ (std::numeric_limits<T>::digits >
+ std::numeric_limits<unsigned long>::digits) &&
+ (std::numeric_limits<T>::digits <=
+ std::numeric_limits<unsigned long long>::digits)),
+ unsigned int>::type
+ findLastSet(T x) {
+ return x ? 8 * sizeof(unsigned long long) - __builtin_clzll(x) : 0;
+}
+
+#else /* !__GNUC__ */
+
+template <class T>
+typename std::enable_if<
+ (std::is_integral<T>::value &&
+ std::is_unsigned<T>::value),
+ unsigned int>::type
+ findLastSet(T x) {
+ return detail:findLastSetPortable(x);
+}
+
+#endif
+
+template <class T>
+typename std::enable_if<
+ (std::is_integral<T>::value &&
+ std::is_signed<T>::value),
+ unsigned int>::type
+ findLastSet(T x) {
+ return findLastSet(static_cast<typename std::make_unsigned<T>::type>(x));
+}
+
+namespace detail {
+
+template <class T>
+inline
+typename std::enable_if<
+ std::is_integral<T>::value && std::is_unsigned<T>::value,
+ T>::type
+nextPowTwoPortable(T v) {
+ if (UNLIKELY(v == 0)) {
+ return 1;
+ }
+
+ --v;
+ for (uint32_t i = 1; i < sizeof(T) * 8; i <<= 8) {
+ v |= (v >> i);
+ v |= (v >> (i << 1));
+ v |= (v >> (i << 2));
+ v |= (v >> (i << 3));
+ v |= (v >> (i << 4));
+ v |= (v >> (i << 5));
+ v |= (v >> (i << 6));
+ v |= (v >> (i << 7));
+ }
+ return v + 1;
+}
+
+} // namespace detail
+
+#ifdef __GNUC__
+
+template <class T>
+inline
+typename std::enable_if<
+ std::is_integral<T>::value && std::is_unsigned<T>::value,
+ T>::type
+nextPowTwo(T v) {
+ if (UNLIKELY(v == 0)) {
+ return 1;
+ }
+ return 1ul << findLastSet(v - 1);
+}
+
+#else /* __GNUC__ */
+
+template <class T>
+inline
+typename std::enable_if<
+ std::is_integral<T>::value && std::is_unsigned<T>::value,
+ T>::type
+nextPowTwo(T v) {
+ return detail::nextPowTwoPortable(v);
+}
+
+#endif /* __GNUC__ */
+
+
+
+/**
+ * Endianness detection and manipulation primitives.
+ */
+namespace detail {
+
+template <class T>
+struct EndianIntBase {
+ public:
+ static T swap(T x);
+};
+
+#define FB_GEN(t, fn) \
+template<> inline t EndianIntBase<t>::swap(t x) { return fn(x); }
+
+// fn(x) expands to (x) if the second argument is empty, which is exactly
+// what we want for [u]int8_t
+FB_GEN( int8_t,)
+FB_GEN(uint8_t,)
+FB_GEN( int64_t, bswap_64)
+FB_GEN(uint64_t, bswap_64)
+FB_GEN( int32_t, bswap_32)
+FB_GEN(uint32_t, bswap_32)
+FB_GEN( int16_t, bswap_16)
+FB_GEN(uint16_t, bswap_16)
+
+#undef FB_GEN
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+
+template <class T>
+struct EndianInt : public detail::EndianIntBase<T> {
+ public:
+ static T big(T x) { return EndianInt::swap(x); }
+ static T little(T x) { return x; }
+};
+
+#elif __BYTE_ORDER == __BIG_ENDIAN
+
+template <class T>
+struct EndianInt : public detail::EndianIntBase<T> {
+ public:
+ static T big(T x) { return x; }
+ static T little(T x) { return EndianInt::swap(x); }
+};
+
+#else
+# error Your machine uses a weird endianness!
+#endif /* __BYTE_ORDER */
+
+} // namespace detail
+
+// big* convert between native and big-endian representations
+// little* convert between native and little-endian representations
+// swap* convert between big-endian and little-endian representations
+//
+// ntohs, htons == big16
+// ntohl, htonl == big32
+#define FB_GEN1(fn, t, sz) \
+ static t fn##sz(t x) { return fn<t>(x); } \
+
+#define FB_GEN2(t, sz) \
+ FB_GEN1(swap, t, sz) \
+ FB_GEN1(big, t, sz) \
+ FB_GEN1(little, t, sz)
+
+#define FB_GEN(sz) \
+ FB_GEN2(uint##sz##_t, sz) \
+ FB_GEN2(int##sz##_t, sz)
+
+class Endian {
+ public:
+ template <class T> static T swap(T x) {
+ return detail::EndianInt<T>::swap(x);
+ }
+ template <class T> static T big(T x) {
+ return detail::EndianInt<T>::big(x);
+ }
+ template <class T> static T little(T x) {
+ return detail::EndianInt<T>::little(x);
+ }
+
+ FB_GEN(64)
+ FB_GEN(32)
+ FB_GEN(16)
+ FB_GEN(8)
+};
+
+#undef FB_GEN
+#undef FB_GEN2
+#undef FB_GEN1
+
+/**
+ * Fast bit iteration facility.
+ */
+
+
+template <class BaseIter> class BitIterator;
+template <class BaseIter>
+BitIterator<BaseIter> findFirstSet(BitIterator<BaseIter>,
+ BitIterator<BaseIter>);
+/**
+ * Wrapper around an iterator over an integer type that iterates
+ * over its underlying bits in LSb to MSb order.
+ *
+ * BitIterator models the same iterator concepts as the base iterator.
+ */
+template <class BaseIter>
+class BitIterator
+ : public bititerator_detail::BitIteratorBase<BaseIter>::type {
+ public:
+ /**
+ * Return the number of bits in an element of the underlying iterator.
+ */
+ static size_t bitsPerBlock() {
+ return std::numeric_limits<
+ typename std::make_unsigned<
+ typename std::iterator_traits<BaseIter>::value_type
+ >::type
+ >::digits;
+ }
+
+ /**
+ * Construct a BitIterator that points at a given bit offset (default 0)
+ * in iter.
+ */
+ explicit BitIterator(const BaseIter& iter, size_t bitOffset=0)
+ : bititerator_detail::BitIteratorBase<BaseIter>::type(iter),
+ bitOffset_(bitOffset) {
+ assert(bitOffset_ < bitsPerBlock());
+ }
+
+ size_t bitOffset() const {
+ return bitOffset_;
+ }
+
+ void advanceToNextBlock() {
+ bitOffset_ = 0;
+ ++this->base_reference();
+ }
+
+ BitIterator& operator=(const BaseIter& other) {
+ this->~BitIterator();
+ new (this) BitIterator(other);
+ return *this;
+ }
+
+ private:
+ friend class boost::iterator_core_access;
+ friend BitIterator findFirstSet<>(BitIterator, BitIterator);
+
+ typedef bititerator_detail::BitReference<
+ typename std::iterator_traits<BaseIter>::reference,
+ typename std::iterator_traits<BaseIter>::value_type
+ > BitRef;
+
+ void advanceInBlock(size_t n) {
+ bitOffset_ += n;
+ assert(bitOffset_ < bitsPerBlock());
+ }
+
+ BitRef dereference() const {
+ return BitRef(*this->base_reference(), bitOffset_);
+ }
+
+ void advance(ssize_t n) {
+ size_t bpb = bitsPerBlock();
+ ssize_t blocks = n / bpb;
+ bitOffset_ += n % bpb;
+ if (bitOffset_ >= bpb) {
+ bitOffset_ -= bpb;
+ ++blocks;
+ }
+ this->base_reference() += blocks;
+ }
+
+ void increment() {
+ if (++bitOffset_ == bitsPerBlock()) {
+ advanceToNextBlock();
+ }
+ }
+
+ void decrement() {
+ if (bitOffset_-- == 0) {
+ bitOffset_ = bitsPerBlock() - 1;
+ --this->base_reference();
+ }
+ }
+
+ bool equal(const BitIterator& other) const {
+ return (bitOffset_ == other.bitOffset_ &&
+ this->base_reference() == other.base_reference());
+ }
+
+ ssize_t distance_to(const BitIterator& other) const {
+ return
+ (other.base_reference() - this->base_reference()) * bitsPerBlock() +
+ (other.bitOffset_ - bitOffset_);
+ }
+
+ ssize_t bitOffset_;
+};
+
+/**
+ * Helper function, so you can write
+ * auto bi = makeBitIterator(container.begin());
+ */
+template <class BaseIter>
+BitIterator<BaseIter> makeBitIterator(const BaseIter& iter) {
+ return BitIterator<BaseIter>(iter);
+}
+
+
+/**
+ * Find first bit set in a range of bit iterators.
+ * 4.5x faster than the obvious std::find(begin, end, true);
+ */
+template <class BaseIter>
+BitIterator<BaseIter> findFirstSet(BitIterator<BaseIter> begin,
+ BitIterator<BaseIter> end) {
+ // shortcut to avoid ugly static_cast<>
+ static const typename BaseIter::value_type one = 1;
+
+ while (begin.base() != end.base()) {
+ typename BaseIter::value_type v = *begin.base();
+ // mask out the bits that don't matter (< begin.bitOffset)
+ v &= ~((one << begin.bitOffset()) - 1);
+ size_t firstSet = findFirstSet(v);
+ if (firstSet) {
+ --firstSet; // now it's 0-based
+ assert(firstSet >= begin.bitOffset());
+ begin.advanceInBlock(firstSet - begin.bitOffset());
+ return begin;
+ }
+ begin.advanceToNextBlock();
+ }
+
+ // now begin points to the same block as end
+ if (end.bitOffset() != 0) { // assume end is dereferenceable
+ typename BaseIter::value_type v = *begin.base();
+ // mask out the bits that don't matter (< begin.bitOffset)
+ v &= ~((one << begin.bitOffset()) - 1);
+ // mask out the bits that don't matter (>= end.bitOffset)
+ v &= (one << end.bitOffset()) - 1;
+ size_t firstSet = findFirstSet(v);
+ if (firstSet) {
+ --firstSet; // now it's 0-based
+ assert(firstSet >= begin.bitOffset());
+ begin.advanceInBlock(firstSet - begin.bitOffset());
+ return begin;
+ }
+ }
+
+ return end;
+}
+
+} // namespace folly
+
+#endif /* FOLLY_BITS_H_ */
+
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @author: Xin Liu <xliux@fb.com>
+
+#ifndef FOLLY_CONCURRENTSKIPLIST_INL_H_
+#define FOLLY_CONCURRENTSKIPLIST_INL_H_
+
+#include <algorithm>
+#include <climits>
+#include <cmath>
+#include <boost/random.hpp>
+
+#include <glog/logging.h>
+#include "folly/SmallLocks.h"
+#include "folly/ThreadLocal.h"
+
+namespace folly { namespace detail {
+
+template<typename ValT, typename NodeT> class csl_iterator;
+
+template<typename T>
+class SkipListNode : boost::noncopyable {
+ enum {
+ IS_HEAD_NODE = 1,
+ MARKED_FOR_REMOVAL = (1 << 1),
+ FULLY_LINKED = (1 << 2),
+ };
+ public:
+ typedef T value_type;
+
+ static SkipListNode* create(int height,
+ const value_type& data, bool isHead = false) {
+ DCHECK(height >= 1 && height < 64) << height;
+
+ size_t size = sizeof(SkipListNode) + height * sizeof(SkipListNode*);
+ auto* node = static_cast<SkipListNode*>(malloc(size));
+ new (node) SkipListNode(height);
+
+ node->spinLock_.init();
+ node->setFlags(0);
+
+ if (isHead) {
+ node->setIsHeadNode();
+ } else {
+ new (&(node->data_)) value_type(data);
+ }
+ return node;
+ }
+
+ static void destroy(SkipListNode* node) {
+ if (!node->isHeadNode()) {
+ node->data_.~value_type();
+ }
+ node->~SkipListNode();
+ free(node);
+ }
+
+ // assuming lock acquired
+ SkipListNode* promoteFrom(const SkipListNode* node) {
+ DCHECK(node != nullptr && height_ > node->height_);
+ setFlags(node->getFlags());
+ if (!isHeadNode()) {
+ new (&(data_)) value_type(node->data());
+ }
+ for (int i = 0; i < node->height_; ++i) {
+ setSkip(i, node->skip(i));
+ }
+ return this;
+ }
+
+ inline SkipListNode* skip(int layer) const {
+ DCHECK_LT(layer, height_);
+ return skip_[layer].load(std::memory_order_consume);
+ }
+
+ // next valid node as in the linked list
+ SkipListNode* next() {
+ SkipListNode* node;
+ for (node = skip(0);
+ (node != nullptr && node->markedForRemoval());
+ node = node->skip(0)) {}
+ return node;
+ }
+
+ void setSkip(uint8_t h, SkipListNode* next) {
+ DCHECK_LT(h, height_);
+ skip_[h].store(next, std::memory_order_release);
+ }
+
+ value_type& data() { return data_; }
+ const value_type& data() const { return data_; }
+ int maxLayer() const { return height_ - 1; }
+ int height() const { return height_; }
+
+ std::unique_lock<MicroSpinLock> acquireGuard() {
+ return std::unique_lock<MicroSpinLock>(spinLock_);
+ }
+
+ bool fullyLinked() const { return getFlags() & FULLY_LINKED; }
+ bool markedForRemoval() const { return getFlags() & MARKED_FOR_REMOVAL; }
+ bool isHeadNode() const { return getFlags() & IS_HEAD_NODE; }
+
+ void setIsHeadNode() {
+ setFlags(getFlags() | IS_HEAD_NODE);
+ }
+ void setFullyLinked() {
+ setFlags(getFlags() | FULLY_LINKED);
+ }
+ void setMarkedForRemoval() {
+ setFlags(getFlags() | MARKED_FOR_REMOVAL);
+ }
+
+ private:
+ ~SkipListNode() {
+ for (uint8_t i = 0; i < height_; ++i) {
+ skip_[i].~atomic();
+ }
+ }
+ explicit SkipListNode(uint8_t height) : height_(height) {
+ for (uint8_t i = 0; i < height_; ++i) {
+ new (&skip_[i]) std::atomic<SkipListNode*>(nullptr);
+ }
+ }
+
+ uint16_t getFlags() const {
+ return flags_.load(std::memory_order_consume);
+ }
+ void setFlags(uint16_t flags) {
+ flags_.store(flags, std::memory_order_release);
+ }
+
+ // TODO(xliu): on x86_64, it's possible to squeeze these into
+ // skip_[0] to maybe save 8 bytes depending on the data alignments.
+ // NOTE: currently this is x86_64 only anyway, due to the
+ // MicroSpinLock.
+ std::atomic<uint16_t> flags_;
+ const uint8_t height_;
+ MicroSpinLock spinLock_;
+
+ value_type data_;
+
+ std::atomic<SkipListNode*> skip_[0];
+};
+
+class SkipListRandomHeight {
+ enum { kMaxHeight = 64 };
+ public:
+ // make it a singleton.
+ static SkipListRandomHeight *instance() {
+ static SkipListRandomHeight instance_;
+ return &instance_;
+ }
+
+ int getHeight(int maxHeight) const {
+ DCHECK_LE(maxHeight, kMaxHeight) << "max height too big!";
+ double p = randomProb();
+ for (int i = 0; i < maxHeight; ++i) {
+ if (p < lookupTable_[i]) {
+ return i + 1;
+ }
+ }
+ return maxHeight;
+ }
+
+ size_t getSizeLimit(int height) const {
+ DCHECK_LT(height, kMaxHeight);
+ return sizeLimitTable_[height];
+ }
+
+ private:
+
+ SkipListRandomHeight() { initLookupTable(); }
+
+ void initLookupTable() {
+ // set skip prob = 1/E
+ static const double kProbInv = exp(1);
+ static const double kProb = 1.0 / kProbInv;
+ static const size_t kMaxSizeLimit = std::numeric_limits<size_t>::max();
+
+ double sizeLimit = 1;
+ double p = lookupTable_[0] = (1 - kProb);
+ sizeLimitTable_[0] = 1;
+ for (int i = 1; i < kMaxHeight - 1; ++i) {
+ p *= kProb;
+ sizeLimit *= kProbInv;
+ lookupTable_[i] = lookupTable_[i - 1] + p;
+ sizeLimitTable_[i] = sizeLimit > kMaxSizeLimit ?
+ kMaxSizeLimit :
+ static_cast<size_t>(sizeLimit);
+ }
+ lookupTable_[kMaxHeight - 1] = 1;
+ sizeLimitTable_[kMaxHeight - 1] = kMaxSizeLimit;
+ }
+
+ static double randomProb() {
+ static ThreadLocal<boost::lagged_fibonacci2281> rng_;
+ return (*rng_)();
+ }
+
+ double lookupTable_[kMaxHeight];
+ size_t sizeLimitTable_[kMaxHeight];
+};
+
+}}
+
+#endif // FOLLY_CONCURRENTSKIPLIST_INL_H_
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @author: Xin Liu <xliux@fb.com>
+//
+// A concurrent skip list (CSL) implementation.
+// Ref: http://www.cs.tau.ac.il/~shanir/nir-pubs-web/Papers/OPODIS2006-BA.pdf
+
+/*
+
+This implements a sorted associative container that supports only
+unique keys. (Similar to std::set.)
+
+Features:
+
+ 1. Small memory overhead: ~40% less memory overhead compared with
+ std::set (1.6 words per node versus 3). It has an minimum of 4
+ words (7 words if there nodes got deleted) per-list overhead
+ though.
+
+ 2. Read accesses (count, find iterator, skipper) are lock-free and
+ mostly wait-free (the only wait a reader may need to do is when
+ the node it is visiting is in a pending stage, i.e. deleting,
+ adding and not fully linked). Write accesses (remove, add) need
+ to acquire locks, but locks are local to the predecessor nodes
+ and/or successor nodes.
+
+ 3. Good high contention performance, comparable single-thread
+ performance. In the multithreaded case (12 workers), CSL tested
+ 10x faster than a RWSpinLocked std::set for an averaged sized
+ list (1K - 1M nodes).
+
+ Comparable read performance to std::set when single threaded,
+ especially when the list size is large, and scales better to
+ larger lists: when the size is small, CSL can be 20-50% slower on
+ find()/contains(). As the size gets large (> 1M elements),
+ find()/contains() can be 30% faster.
+
+ Iterating through a skiplist is similar to iterating through a
+ linked list, thus is much (2-6x) faster than on a std::set
+ (tree-based). This is especially true for short lists due to
+ better cache locality. Based on that, it's also faster to
+ intersect two skiplists.
+
+ 4. Lazy removal with GC support. The removed nodes get deleted when
+ the last Accessor to the skiplist is destroyed.
+
+Caveats:
+
+ 1. Write operations are usually 30% slower than std::set in a single
+ threaded environment.
+
+ 2. Need to have a head node for each list, which has a 4 word
+ overhead.
+
+ 3. When the list is quite small (< 1000 elements), single threaded
+ benchmarks show CSL can be 10x slower than std:set.
+
+ 4. The interface requires using an Accessor to access the skiplist.
+ (See below.)
+
+ 5. Currently x64 only, due to use of MicroSpinLock.
+
+ 6. Freed nodes will not be reclaimed as long as there are ongoing
+ uses of the list.
+
+Sample usage:
+
+ typedef ConcurrentSkipList<int> SkipListT;
+ shared_ptr<SkipListT> sl(SkipListT::createInstance(init_head_height);
+ {
+ // It's usually good practice to hold an accessor only during
+ // its necessary life cycle (but not in a tight loop as
+ // Accessor creation incurs ref-counting overhead).
+ //
+ // Holding it longer delays garbage-collecting the deleted
+ // nodes in the list.
+ SkipListT::Accessor accessor(sl);
+ accessor.insert(23);
+ accessor.erase(2);
+ for (auto &elem : accessor) {
+ // use elem to access data
+ }
+ ... ...
+ }
+
+ Another useful type is the Skipper accessor. This is useful if you
+ want to skip to locations in the way std::lower_bound() works,
+ i.e. it can be used for going through the list by skipping to the
+ node no less than a specified key. The Skipper keeps its location as
+ state, which makes it convenient for things like implementing
+ intersection of two sets efficiently, as it can start from the last
+ visited position.
+
+ {
+ SkipListT::Accessor accessor(sl);
+ SkipListT::Skipper skipper(accessor);
+ skipper.to(30);
+ if (skipper) {
+ CHECK_LE(30, *skipper);
+ }
+ ... ...
+ // GC may happen when the accessor gets destructed.
+ }
+*/
+
+#ifndef FOLLY_CONCURRENT_SKIP_LIST_H_
+#define FOLLY_CONCURRENT_SKIP_LIST_H_
+
+#include <algorithm>
+#include <climits>
+#include <type_traits>
+#include <utility>
+#include <vector>
+#include <atomic>
+#include <thread>
+#include <boost/iterator/iterator_facade.hpp>
+#include <boost/scoped_ptr.hpp>
+#include <boost/shared_ptr.hpp>
+
+#include <glog/logging.h>
+#include "folly/ConcurrentSkipList-inl.h"
+#include "folly/Likely.h"
+#include "folly/SmallLocks.h"
+
+namespace folly {
+
+template<typename T, typename Comp=std::less<T>, int MAX_HEIGHT=24>
+class ConcurrentSkipList {
+ // MAX_HEIGHT needs to be at least 2 to suppress compiler
+ // warnings/errors (Werror=uninitialized tiggered due to preds_[1]
+ // being treated as a scalar in the compiler).
+ static_assert(MAX_HEIGHT >= 2 && MAX_HEIGHT < 64,
+ "MAX_HEIGHT can only be in the range of [2, 64)");
+ typedef detail::SkipListNode<T> NodeType;
+ typedef std::unique_lock<folly::MicroSpinLock> ScopedLocker;
+ typedef ConcurrentSkipList<T, Comp, MAX_HEIGHT> SkipListType;
+
+ public:
+ typedef T value_type;
+ typedef T key_type;
+
+
+ typedef detail::csl_iterator<value_type, NodeType> iterator;
+ typedef detail::csl_iterator<const value_type, const NodeType> const_iterator;
+
+ class Accessor;
+ class Skipper;
+
+ // convenient function to get an Accessor to a new instance.
+ static Accessor create(int height=1) {
+ return Accessor(createInstance(height));
+ }
+
+ // create a shared_ptr skiplist object with initial head height.
+ static boost::shared_ptr<SkipListType> createInstance(int height=1) {
+ return boost::shared_ptr<SkipListType>(new SkipListType(height));
+ }
+
+ //===================================================================
+ // Below are implementation details.
+ // Please see ConcurrentSkipList::Accessor for stdlib-like APIs.
+ //===================================================================
+
+ ~ConcurrentSkipList() {
+ LOG_IF(FATAL, recycler_.refs() > 0)
+ << "number of accessors is not 0, " << recycler_.refs() << " instead!"
+ << " This shouldn't have happened!";
+ while (NodeType* current = head_.load(std::memory_order_relaxed)) {
+ NodeType* tmp = current->skip(0);
+ NodeType::destroy(current);
+ head_.store(tmp, std::memory_order_relaxed);
+ }
+ }
+
+ private:
+
+ static bool greater(const value_type &data, const NodeType *node) {
+ return node && Comp()(node->data(), data);
+ }
+
+ static bool less(const value_type &data, const NodeType *node) {
+ return (node == nullptr) || Comp()(data, node->data());
+ }
+
+ static int findInsertionPoint(NodeType *cur, int cur_layer,
+ const value_type &data,
+ NodeType *preds[], NodeType *succs[]) {
+ int foundLayer = -1;
+ NodeType *pred = cur;
+ NodeType *foundNode = nullptr;
+ for (int layer = cur_layer; layer >= 0; --layer) {
+ NodeType *node = pred->skip(layer);
+ while (greater(data, node)) {
+ pred = node;
+ node = node->skip(layer);
+ }
+ if (foundLayer == -1 && !less(data, node)) { // the two keys equal
+ foundLayer = layer;
+ foundNode = node;
+ }
+ preds[layer] = pred;
+
+ // if found, succs[0..foundLayer] need to point to the cached foundNode,
+ // as foundNode might be deleted at the same time thus pred->skip() can
+ // return NULL or another node.
+ succs[layer] = foundNode ? foundNode : node;
+ }
+ return foundLayer;
+ }
+
+ struct Recycler : private boost::noncopyable {
+ Recycler() : refs_(0), dirty_(false) { lock_.init(); }
+
+ ~Recycler() {
+ if (nodes_) {
+ for (auto& node : *nodes_) {
+ NodeType::destroy(node);
+ }
+ }
+ }
+
+ void add(NodeType* node) {
+ std::lock_guard<MicroSpinLock> g(lock_);
+ if (nodes_.get() == nullptr) {
+ nodes_.reset(new std::vector<NodeType*>(1, node));
+ } else {
+ nodes_->push_back(node);
+ }
+ DCHECK_GT(refs(), 0);
+ dirty_.store(true, std::memory_order_relaxed);
+ }
+
+ int refs() const {
+ return refs_.load(std::memory_order_relaxed);
+ }
+
+ int addRef() {
+ return refs_.fetch_add(1, std::memory_order_relaxed);
+ }
+
+ int release() {
+ // We don't expect to clean the recycler immediately everytime it is OK
+ // to do so. Here, it is possible that multiple accessors all release at
+ // the same time but nobody would clean the recycler here. If this
+ // happens, the recycler will usually still get cleaned when
+ // such a race doesn't happen. The worst case is the recycler will
+ // eventually get deleted along with the skiplist.
+ if (LIKELY(!dirty_.load(std::memory_order_relaxed) || refs() > 1)) {
+ return refs_.fetch_add(-1, std::memory_order_relaxed);
+ }
+
+ boost::scoped_ptr<std::vector<NodeType*> > newNodes;
+ {
+ std::lock_guard<MicroSpinLock> g(lock_);
+ if (nodes_.get() == nullptr || refs() > 1) {
+ return refs_.fetch_add(-1, std::memory_order_relaxed);
+ }
+ // once refs_ reaches 1 and there is no other accessor, it is safe to
+ // remove all the current nodes in the recycler, as we already acquired
+ // the lock here so no more new nodes can be added, even though new
+ // accessors may be added after that.
+ newNodes.swap(nodes_);
+ dirty_.store(false, std::memory_order_relaxed);
+ }
+
+ // TODO(xliu) should we spawn a thread to do this when there are large
+ // number of nodes in the recycler?
+ for (auto& node : *newNodes) {
+ NodeType::destroy(node);
+ }
+
+ // decrease the ref count at the very end, to minimize the
+ // chance of other threads acquiring lock_ to clear the deleted
+ // nodes again.
+ return refs_.fetch_add(-1, std::memory_order_relaxed);
+ }
+
+ private:
+ boost::scoped_ptr<std::vector<NodeType*>> nodes_;
+ std::atomic<int32_t> refs_; // current number of visitors to the list
+ std::atomic<bool> dirty_; // whether *nodes_ is non-empty
+ MicroSpinLock lock_; // protects access to *nodes_
+ }; // class ConcurrentSkipList::Recycler
+
+ explicit ConcurrentSkipList(int height) :
+ head_(NodeType::create(height, value_type(), true)), size_(0) {}
+
+ size_t size() const { return size_.load(std::memory_order_relaxed); }
+ int height() const {
+ return head_.load(std::memory_order_consume)->height();
+ }
+ int maxLayer() const { return height() - 1; }
+
+ size_t incrementSize(int delta) {
+ return size_.fetch_add(delta, std::memory_order_relaxed) + delta;
+ }
+
+ // Returns the node if found, nullptr otherwise.
+ NodeType* find(const value_type &data) {
+ auto ret = findNode(data);
+ if (ret.second && !ret.first->markedForRemoval()) return ret.first;
+ return nullptr;
+ }
+
+ // lock all the necessary nodes for changing (adding or removing) the list.
+ // returns true if all the lock acquried successfully and the related nodes
+ // are all validate (not in certain pending states), false otherwise.
+ bool lockNodesForChange(int nodeHeight,
+ ScopedLocker guards[MAX_HEIGHT],
+ NodeType *preds[MAX_HEIGHT],
+ NodeType *succs[MAX_HEIGHT],
+ bool adding=true) {
+ NodeType *pred, *succ, *prevPred = nullptr;
+ bool valid = true;
+ for (int layer = 0; valid && layer < nodeHeight; ++layer) {
+ pred = preds[layer];
+ DCHECK(pred != nullptr) << "layer=" << layer << " height=" << height()
+ << " nodeheight=" << nodeHeight;
+ succ = succs[layer];
+ if (pred != prevPred) {
+ guards[layer] = pred->acquireGuard();
+ prevPred = pred;
+ }
+ valid = !pred->markedForRemoval() &&
+ pred->skip(layer) == succ; // check again after locking
+
+ if (adding) { // when adding a node, the succ shouldn't be going away
+ valid = valid && (succ == nullptr || !succ->markedForRemoval());
+ }
+ }
+
+ return valid;
+ }
+
+ // Returns a paired value:
+ // pair.first always stores the pointer to the node with the same input key.
+ // It could be either the newly added data, or the existed data in the
+ // list with the same key.
+ // pair.second stores whether the data is added successfully:
+ // 0 means not added, otherwise reutrns the new size.
+ std::pair<NodeType*, size_t> addOrGetData(const value_type &data) {
+ NodeType *preds[MAX_HEIGHT], *succs[MAX_HEIGHT];
+ NodeType *newNode;
+ size_t newSize;
+ while (true) {
+ int max_layer = 0;
+ int layer = findInsertionPointGetMaxLayer(data, preds, succs, &max_layer);
+
+ if (layer >= 0) {
+ NodeType *nodeFound = succs[layer];
+ DCHECK(nodeFound != nullptr);
+ if (nodeFound->markedForRemoval()) {
+ continue; // if it's getting deleted retry finding node.
+ }
+ // wait until fully linked.
+ while (UNLIKELY(!nodeFound->fullyLinked())) {}
+ return std::make_pair(nodeFound, 0);
+ }
+
+ // need to capped at the original height -- the real height may have grown
+ int nodeHeight = detail::SkipListRandomHeight::instance()->
+ getHeight(max_layer + 1);
+
+ ScopedLocker guards[MAX_HEIGHT];
+ if (!lockNodesForChange(nodeHeight, guards, preds, succs)) {
+ continue; // give up the locks and retry until all valid
+ }
+
+ // locks acquired and all valid, need to modify the links under the locks.
+ newNode = NodeType::create(nodeHeight, data);
+ for (int layer = 0; layer < nodeHeight; ++layer) {
+ newNode->setSkip(layer, succs[layer]);
+ preds[layer]->setSkip(layer, newNode);
+ }
+
+ newNode->setFullyLinked();
+ newSize = incrementSize(1);
+ break;
+ }
+
+ int hgt = height();
+ size_t sizeLimit =
+ detail::SkipListRandomHeight::instance()->getSizeLimit(hgt);
+
+ if (hgt < MAX_HEIGHT && newSize > sizeLimit) {
+ growHeight(hgt + 1);
+ }
+ CHECK_GT(newSize, 0);
+ return std::make_pair(newNode, newSize);
+ }
+
+ bool remove(const value_type &data) {
+ NodeType *nodeToDelete = nullptr;
+ ScopedLocker nodeGuard;
+ bool isMarked = false;
+ int nodeHeight = 0;
+ NodeType* preds[MAX_HEIGHT], *succs[MAX_HEIGHT];
+
+ while (true) {
+ int max_layer = 0;
+ int layer = findInsertionPointGetMaxLayer(data, preds, succs, &max_layer);
+ if (!isMarked && (layer < 0 || !okToDelete(succs[layer], layer))) {
+ return false;
+ }
+
+ if (!isMarked) {
+ nodeToDelete = succs[layer];
+ nodeHeight = nodeToDelete->height();
+ nodeGuard = nodeToDelete->acquireGuard();
+ if (nodeToDelete->markedForRemoval()) return false;
+ nodeToDelete->setMarkedForRemoval();
+ isMarked = true;
+ }
+
+ // acquire pred locks from bottom layer up
+ ScopedLocker guards[MAX_HEIGHT];
+ if (!lockNodesForChange(nodeHeight, guards, preds, succs, false)) {
+ continue; // this will unlock all the locks
+ }
+
+ for (int layer = nodeHeight - 1; layer >= 0; --layer) {
+ preds[layer]->setSkip(layer, nodeToDelete->skip(layer));
+ }
+
+ incrementSize(-1);
+ break;
+ }
+ recycle(nodeToDelete);
+ return true;
+ }
+
+ const value_type *first() const {
+ auto node = head_.load(std::memory_order_consume)->skip(0);
+ return node ? &node->data() : nullptr;
+ }
+
+ const value_type *last() const {
+ NodeType *pred = head_.load(std::memory_order_consume);
+ NodeType *node = nullptr;
+ for (int layer = maxLayer(); layer >= 0; --layer) {
+ do {
+ node = pred->skip(layer);
+ if (node) pred = node;
+ } while (node != nullptr);
+ }
+ return pred == head_.load(std::memory_order_relaxed)
+ ? nullptr : &pred->data();
+ }
+
+ static bool okToDelete(NodeType *candidate, int layer) {
+ DCHECK(candidate != nullptr);
+ return candidate->fullyLinked() &&
+ candidate->maxLayer() == layer &&
+ !candidate->markedForRemoval();
+ }
+
+ // find node for insertion/deleting
+ int findInsertionPointGetMaxLayer(const value_type &data,
+ NodeType *preds[], NodeType *succs[], int *max_layer) const {
+ *max_layer = maxLayer();
+ return findInsertionPoint(head_.load(std::memory_order_consume),
+ *max_layer, data, preds, succs);
+ }
+
+ // Find node for access. Returns a paired values:
+ // pair.first = the first node that no-less than data value
+ // pair.second = 1 when the data value is founded, or 0 otherwise.
+ // This is like lower_bound, but not exact: we could have the node marked for
+ // removal so still need to check that.
+ std::pair<NodeType*, int> findNode(const value_type &data) const {
+ return findNodeDownRight(data);
+ }
+
+ // Find node by first stepping down then stepping right. Based on benchmark
+ // results, this is slightly faster than findNodeRightDown for better
+ // localality on the skipping pointers.
+ std::pair<NodeType*, int> findNodeDownRight(const value_type &data) const {
+ NodeType *pred = head_.load(std::memory_order_consume);
+ int ht = pred->height();
+ NodeType *node = nullptr;
+
+ bool found = false;
+ while (!found) {
+ // stepping down
+ for (; ht > 0 && less(data, pred->skip(ht - 1)); --ht) {}
+ if (ht == 0) return std::make_pair(pred->skip(0), 0); // not found
+
+ node = pred->skip(--ht); // node <= data now
+ // stepping right
+ while (greater(data, node)) {
+ pred = node;
+ node = node->skip(ht);
+ }
+ found = !less(data, node);
+ }
+ return std::make_pair(node, found);
+ }
+
+ // find node by first stepping right then stepping down.
+ // We still keep this for reference purposes.
+ std::pair<NodeType*, int> findNodeRightDown(const value_type &data) const {
+ NodeType *pred = head_.load(std::memory_order_consume);
+ NodeType *node = nullptr;
+ auto top = maxLayer();
+ int found = 0;
+ for (int layer = top; !found && layer >= 0; --layer) {
+ node = pred->skip(layer);
+ while (greater(data, node)) {
+ pred = node;
+ node = node->skip(layer);
+ }
+ found = !less(data, node);
+ }
+ return std::make_pair(node, found);
+ }
+
+ NodeType* lower_bound(const value_type &data) const {
+ auto node = findNode(data).first;
+ while (node != nullptr && node->markedForRemoval()) {
+ node = node->skip(0);
+ }
+ return node;
+ }
+
+ void growHeight(int height) {
+ NodeType* oldHead = head_.load(std::memory_order_consume);
+ if (oldHead->height() >= height) { // someone else already did this
+ return;
+ }
+
+ NodeType* newHead = NodeType::create(height, value_type(), true);
+
+ { // need to guard the head node in case others are adding/removing
+ // nodes linked to the head.
+ ScopedLocker g = oldHead->acquireGuard();
+ newHead->promoteFrom(oldHead);
+ NodeType* expected = oldHead;
+ if (!head_.compare_exchange_strong(expected, newHead,
+ std::memory_order_release)) {
+ // if someone has already done the swap, just return.
+ NodeType::destroy(newHead);
+ return;
+ }
+ oldHead->setMarkedForRemoval();
+ }
+ recycle(oldHead);
+ }
+
+ void recycle(NodeType *node) {
+ recycler_.add(node);
+ }
+
+ std::atomic<NodeType*> head_;
+ Recycler recycler_;
+ std::atomic<size_t> size_;
+};
+
+template<typename T, typename Comp, int MAX_HEIGHT>
+class ConcurrentSkipList<T, Comp, MAX_HEIGHT>::Accessor {
+ typedef detail::SkipListNode<T> NodeType;
+ typedef ConcurrentSkipList<T, Comp, MAX_HEIGHT> SkipListType;
+ public:
+ typedef T value_type;
+ typedef T key_type;
+ typedef T& reference;
+ typedef T* pointer;
+ typedef const T& const_reference;
+ typedef const T* const_pointer;
+ typedef size_t size_type;
+ typedef Comp key_compare;
+ typedef Comp value_compare;
+
+ typedef typename SkipListType::iterator iterator;
+ typedef typename SkipListType::const_iterator const_iterator;
+ typedef typename SkipListType::Skipper Skipper;
+
+ explicit Accessor(boost::shared_ptr<ConcurrentSkipList> skip_list)
+ : slHolder_(std::move(skip_list))
+ {
+ sl_ = slHolder_.get();
+ DCHECK(sl_ != nullptr);
+ sl_->recycler_.addRef();
+ }
+
+ // Unsafe initializer: the caller assumes the responsibility to keep
+ // skip_list valid during the whole life cycle of the Acessor.
+ explicit Accessor(ConcurrentSkipList *skip_list) : sl_(skip_list) {
+ DCHECK(sl_ != nullptr);
+ sl_->recycler_.addRef();
+ }
+
+ Accessor(const Accessor &accessor) :
+ sl_(accessor.sl_),
+ slHolder_(accessor.slHolder_) {
+ sl_->recycler_.addRef();
+ }
+
+ Accessor& operator=(const Accessor &accessor) {
+ if (this != &accessor) {
+ slHolder_ = accessor.slHolder_;
+ sl_->recycler_.release();
+ sl_ = accessor.sl_;
+ sl_->recycler_.addRef();
+ }
+ return *this;
+ }
+
+ ~Accessor() {
+ sl_->recycler_.release();
+ }
+
+ bool empty() const { return sl_->size() == 0; }
+ size_t size() const { return sl_->size(); }
+ size_type max_size() const { return std::numeric_limits<size_type>::max(); }
+
+ // returns end() if the value is not in the list, otherwise returns an
+ // iterator pointing to the data, and it's guaranteed that the data is valid
+ // as far as the Accessor is hold.
+ iterator find(const key_type &value) { return iterator(sl_->find(value)); }
+ const_iterator find(const key_type &value) const {
+ return iterator(sl_->find(value));
+ }
+ size_type count(const key_type &data) const { return contains(data); }
+
+ iterator begin() const {
+ NodeType* head = sl_->head_.load(std::memory_order_consume);
+ return iterator(head->next());
+ }
+ iterator end() const { return iterator(nullptr); }
+ const_iterator cbegin() const { return begin(); }
+ const_iterator cend() const { return end(); }
+
+ std::pair<iterator, bool> insert(const key_type &data) {
+ auto ret = sl_->addOrGetData(data);
+ return std::make_pair(iterator(ret.first), ret.second);
+ }
+ size_t erase(const key_type &data) { return remove(data); }
+
+ iterator lower_bound(const key_type &data) const {
+ return iterator(sl_->lower_bound(data));
+ }
+
+ size_t height() const { return sl_->height(); }
+
+ // first() returns pointer to the first element in the skiplist, or
+ // nullptr if empty.
+ //
+ // last() returns the pointer to the last element in the skiplist,
+ // nullptr if list is empty.
+ //
+ // Note: As concurrent writing can happen, first() is not
+ // guaranteed to be the min_element() in the list. Similarly
+ // last() is not guaranteed to be the max_element(), and both of them can
+ // be invalid (i.e. nullptr), so we name them differently from front() and
+ // tail() here.
+ const key_type *first() const { return sl_->first(); }
+ const key_type *last() const { return sl_->last(); }
+
+ // Try to remove the last element in the skip list.
+ //
+ // Returns true if we removed it, false if either the list is empty
+ // or a race condition happened (i.e. the used-to-be last element
+ // was already removed by another thread).
+ bool pop_back() {
+ auto last = sl_->last();
+ return last ? sl_->remove(*last) : false;
+ }
+
+ std::pair<key_type*, bool> addOrGetData(const key_type &data) {
+ auto ret = sl_->addOrGetData(data);
+ return std::make_pair(&ret.first->data(), ret.second);
+ }
+
+ SkipListType* skiplist() const { return sl_; }
+
+ // legacy interfaces
+ // TODO:(xliu) remove these.
+ // Returns true if the node is added successfully, false if not, i.e. the
+ // node with the same key already existed in the list.
+ bool contains(const key_type &data) const { return sl_->find(data); }
+ bool add(const key_type &data) { return sl_->addOrGetData(data).second; }
+ bool remove(const key_type &data) { return sl_->remove(data); }
+
+ private:
+ SkipListType *sl_;
+ boost::shared_ptr<SkipListType> slHolder_;
+};
+
+// implements forward iterator concept.
+template<typename ValT, typename NodeT>
+class detail::csl_iterator :
+ public boost::iterator_facade<csl_iterator<ValT, NodeT>,
+ ValT, boost::forward_traversal_tag> {
+ public:
+ typedef ValT value_type;
+ typedef value_type& reference;
+ typedef value_type* pointer;
+ typedef ptrdiff_t difference_type;
+
+ explicit csl_iterator(NodeT* node = nullptr) : node_(node) {}
+
+ template<typename OtherVal, typename OtherNode>
+ csl_iterator(const csl_iterator<OtherVal, OtherNode> &other,
+ typename std::enable_if<std::is_convertible<OtherVal, ValT>::value>::type*
+ = 0) : node_(other.node_) {}
+
+ size_t nodeSize() const {
+ return node_ == nullptr ? 0 :
+ node_->height() * sizeof(NodeT*) + sizeof(*this);
+ }
+
+ bool good() const { return node_ != nullptr; }
+
+ private:
+ friend class boost::iterator_core_access;
+ template<class,class> friend class csl_iterator;
+
+ void increment() { node_ = node_->next(); };
+ bool equal(const csl_iterator& other) const { return node_ == other.node_; }
+ value_type& dereference() const { return node_->data(); }
+
+ NodeT* node_;
+};
+
+// Skipper interface
+template<typename T, typename Comp, int MAX_HEIGHT>
+class ConcurrentSkipList<T, Comp, MAX_HEIGHT>::Skipper {
+ typedef detail::SkipListNode<T> NodeType;
+ typedef ConcurrentSkipList<T, Comp, MAX_HEIGHT> SkipListType;
+ typedef typename SkipListType::Accessor Accessor;
+
+ public:
+ typedef T value_type;
+ typedef T& reference;
+ typedef T* pointer;
+ typedef ptrdiff_t difference_type;
+
+ Skipper(const boost::shared_ptr<SkipListType>& skipList) :
+ accessor_(skipList) {
+ init();
+ }
+
+ Skipper(const Accessor& accessor) : accessor_(accessor) {
+ init();
+ }
+
+ void init() {
+ // need to cache the head node
+ NodeType* head_node = head();
+ headHeight_ = head_node->height();
+ for (int i = 0; i < headHeight_; ++i) {
+ preds_[i] = head_node;
+ succs_[i] = head_node->skip(i);
+ }
+ int max_layer = maxLayer();
+ for (int i = 0; i < max_layer; ++i) {
+ hints_[i] = i + 1;
+ }
+ hints_[max_layer] = max_layer;
+ }
+
+ // advance to the next node in the list.
+ Skipper& operator ++() {
+ preds_[0] = succs_[0];
+ succs_[0] = preds_[0]->skip(0);
+ int height = curHeight();
+ for (int i = 1; i < height && preds_[0] == succs_[i]; ++i) {
+ preds_[i] = succs_[i];
+ succs_[i] = preds_[i]->skip(i);
+ }
+ return *this;
+ }
+
+ bool good() const { return succs_[0] != nullptr; }
+
+ int maxLayer() const { return headHeight_ - 1; }
+
+ int curHeight() const {
+ // need to cap the height to the cached head height, as the current node
+ // might be some newly inserted node and also during the time period the
+ // head height may have grown.
+ return succs_[0] ? std::min(headHeight_, succs_[0]->height()) : 0;
+ }
+
+ const value_type &data() const {
+ DCHECK(succs_[0] != NULL);
+ return succs_[0]->data();
+ }
+
+ value_type &operator *() const {
+ DCHECK(succs_[0] != NULL);
+ return succs_[0]->data();
+ }
+
+ value_type *operator->() {
+ DCHECK(succs_[0] != NULL);
+ return &succs_[0]->data();
+ }
+
+ /*
+ * Skip to the position whose data is no less than the parameter.
+ * (I.e. the lower_bound).
+ *
+ * Returns true if the data is found, false otherwise.
+ */
+ bool to(const value_type &data) {
+ int layer = curHeight() - 1;
+ if (layer < 0) return false; // reaches the end of the list
+
+ int lyr = hints_[layer];
+ int max_layer = maxLayer();
+ while (SkipListType::greater(data, succs_[lyr]) && lyr < max_layer) {
+ ++lyr;
+ }
+ hints_[layer] = lyr; // update the hint
+
+ int foundLayer = SkipListType::
+ findInsertionPoint(preds_[lyr], lyr, data, preds_, succs_);
+ if (foundLayer < 0) return false;
+
+ DCHECK(succs_[0] != NULL) << "lyr=" << lyr << "; max_layer=" << max_layer;
+ return !succs_[0]->markedForRemoval();
+ }
+
+ private:
+ NodeType* head() const {
+ return accessor_.skiplist()->head_.load(std::memory_order_consume);
+ }
+
+ Accessor accessor_;
+ int headHeight_;
+ NodeType *succs_[MAX_HEIGHT], *preds_[MAX_HEIGHT];
+ uint8_t hints_[MAX_HEIGHT];
+};
+
+} // namespace folly
+
+#endif // FOLLY_CONCURRENT_SKIP_LIST_H_
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#define FOLLY_CONV_INTERNAL
+#include "folly/Conv.h"
+
+namespace folly {
+namespace detail {
+
+extern const char digit1[101] =
+ "00000000001111111111222222222233333333334444444444"
+ "55555555556666666666777777777788888888889999999999";
+extern const char digit2[101] =
+ "01234567890123456789012345678901234567890123456789"
+ "01234567890123456789012345678901234567890123456789";
+
+template <> const char *const MaxString<bool>::value = "true";
+template <> const char *const MaxString<uint8_t>::value = "255";
+template <> const char *const MaxString<uint16_t>::value = "65535";
+template <> const char *const MaxString<uint32_t>::value = "4294967295";
+#if __SIZEOF_LONG__ == 4
+template <> const char *const MaxString<unsigned long>::value =
+ "4294967295";
+#else
+template <> const char *const MaxString<unsigned long>::value =
+ "18446744073709551615";
+#endif
+static_assert(sizeof(unsigned long) >= 4,
+ "Wrong value for MaxString<unsigned long>::value,"
+ " please update.");
+template <> const char *const MaxString<unsigned long long>::value =
+ "18446744073709551615";
+static_assert(sizeof(unsigned long long) >= 8,
+ "Wrong value for MaxString<unsigned long long>::value"
+ ", please update.");
+
+inline bool bool_str_cmp(const char** b, size_t len, const char* value) {
+ // Can't use strncasecmp, since we want to ensure that the full value matches
+ const char* p = *b;
+ const char* e = *b + len;
+ const char* v = value;
+ while (*v != '\0') {
+ if (p == e || tolower(*p) != *v) { // value is already lowercase
+ return false;
+ }
+ ++p;
+ ++v;
+ }
+
+ *b = p;
+ return true;
+}
+
+bool str_to_bool(StringPiece* src) {
+ auto b = src->begin(), e = src->end();
+ for (;; ++b) {
+ FOLLY_RANGE_CHECK(b < e,
+ "No non-whitespace characters found in input string");
+ if (!isspace(*b)) break;
+ }
+
+ bool result;
+ size_t len = e - b;
+ switch (*b) {
+ case '0':
+ case '1': {
+ // Attempt to parse the value as an integer
+ StringPiece tmp(*src);
+ uint8_t value = to<uint8_t>(&tmp);
+ // Only accept 0 or 1
+ FOLLY_RANGE_CHECK(value <= 1,
+ "Integer overflow when parsing bool: must be 0 or 1");
+ b = tmp.begin();
+ result = (value == 1);
+ break;
+ }
+ case 'y':
+ case 'Y':
+ result = true;
+ if (!bool_str_cmp(&b, len, "yes")) {
+ ++b; // accept the single 'y' character
+ }
+ break;
+ case 'n':
+ case 'N':
+ result = false;
+ if (!bool_str_cmp(&b, len, "no")) {
+ ++b;
+ }
+ break;
+ case 't':
+ case 'T':
+ result = true;
+ if (!bool_str_cmp(&b, len, "true")) {
+ ++b;
+ }
+ break;
+ case 'f':
+ case 'F':
+ result = false;
+ if (!bool_str_cmp(&b, len, "false")) {
+ ++b;
+ }
+ break;
+ case 'o':
+ case 'O':
+ if (bool_str_cmp(&b, len, "on")) {
+ result = true;
+ } else if (bool_str_cmp(&b, len, "off")) {
+ result = false;
+ } else {
+ FOLLY_RANGE_CHECK(false, "Invalid value for bool");
+ }
+ break;
+ default:
+ FOLLY_RANGE_CHECK(false, "Invalid value for bool");
+ }
+
+ src->assign(b, e);
+ return result;
+}
+
+} // namespace detail
+} // namespace folly
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Converts anything to anything, with an emphasis on performance and
+ * safety.
+ *
+ * @author Andrei Alexandrescu (andrei.alexandrescu@fb.com)
+ */
+
+#ifndef FOLLY_BASE_CONV_H_
+#define FOLLY_BASE_CONV_H_
+
+#include "folly/FBString.h"
+#include "folly/Likely.h"
+#include "folly/Preprocessor.h"
+#include "folly/Range.h"
+
+#include <boost/implicit_cast.hpp>
+#include <type_traits>
+#include <limits>
+#include <string>
+#include <tuple>
+#include <stdexcept>
+#include <typeinfo>
+
+#include "double-conversion.h" // V8 JavaScript implementation
+
+#define FOLLY_RANGE_CHECK(condition, message) \
+ ((condition) ? (void)0 : throw std::range_error( \
+ (__FILE__ "(" + std::to_string((long long int) __LINE__) + "): " \
+ + (message)).c_str()))
+
+namespace folly {
+
+/*******************************************************************************
+ * Integral to integral
+ ******************************************************************************/
+
+/**
+ * Checked conversion from integral to integral. The checks are only
+ * performed when meaningful, e.g. conversion from int to long goes
+ * unchecked.
+ */
+template <class Tgt, class Src>
+typename std::enable_if<
+ std::is_integral<Src>::value && std::is_integral<Tgt>::value,
+ Tgt>::type
+to(const Src & value) {
+ /* static */ if (std::numeric_limits<Tgt>::max()
+ < std::numeric_limits<Src>::max()) {
+ FOLLY_RANGE_CHECK(value <= std::numeric_limits<Tgt>::max(),
+ "Overflow");
+ }
+ /* static */ if (std::is_signed<Src>::value &&
+ (!std::is_signed<Tgt>::value || sizeof(Src) > sizeof(Tgt))) {
+ FOLLY_RANGE_CHECK(value >= std::numeric_limits<Tgt>::min(),
+ "Negative overflow");
+ }
+ return static_cast<Tgt>(value);
+}
+
+/*******************************************************************************
+ * Floating point to floating point
+ ******************************************************************************/
+
+template <class Tgt, class Src>
+typename std::enable_if<
+ std::is_floating_point<Tgt>::value && std::is_floating_point<Src>::value,
+ Tgt>::type
+to(const Src & value) {
+ /* static */ if (std::numeric_limits<Tgt>::max() <
+ std::numeric_limits<Src>::max()) {
+ FOLLY_RANGE_CHECK(value <= std::numeric_limits<Tgt>::max(),
+ "Overflow");
+ FOLLY_RANGE_CHECK(value >= -std::numeric_limits<Tgt>::max(),
+ "Negative overflow");
+ }
+ return boost::implicit_cast<Tgt>(value);
+}
+
+/*******************************************************************************
+ * Anything to string
+ ******************************************************************************/
+
+namespace detail {
+
+template <class T> struct IsSomeString {
+ enum { value = std::is_same<T, std::string>::value
+ || std::is_same<T, fbstring>::value };
+};
+
+template <class T>
+const T& getLastElement(const T & v) {
+ return v;
+}
+
+template <class T, class... Ts>
+typename std::tuple_element<
+ sizeof...(Ts),
+ std::tuple<T, Ts...> >::type const&
+ getLastElement(const T& v, const Ts&... vs) {
+ return getLastElement(vs...);
+}
+
+/*******************************************************************************
+ * Conversions from integral types to string types.
+ ******************************************************************************/
+
+// Returns the offset of the formatted string from the start of
+// the supplied buffer. The new string will be at range
+// [buf+begin,buf+bufLen). Uint will be either uint32_t or uint64_t.
+template <class Uint>
+size_t uintToBuffer(char*const buffer, size_t bufLen, Uint v) {
+ extern const char digit1[101], digit2[101];
+ for (;;) {
+ if (v < 100) {
+ if (v < 10) {
+ buffer[--bufLen] = static_cast<char>(v + '0');
+ } else {
+ size_t r = static_cast<size_t>(v);
+ bufLen -= 2;
+ buffer[bufLen] = digit1[r];
+ buffer[bufLen + 1] = digit2[r];
+ }
+ break;
+ }
+ Uint t = v;
+ v /= 100;
+ size_t r = static_cast<size_t> (t - v * 100);
+ bufLen -= 2;
+ buffer[bufLen] = digit1[r];
+ buffer[bufLen + 1] = digit2[r];
+ }
+ return bufLen;
+}
+
+const size_t kMaxInt64BufLen = 21;// 19 + 1 for possible '-' sign + 1 for \0
+
+} // namespace detail
+
+/**
+ * A single char gets appended.
+ */
+template <class Tgt>
+void toAppend(char value, Tgt * result) {
+ *result += value;
+}
+
+/**
+ * Everything implicitly convertible to const char* gets appended.
+ */
+template <class Tgt, class Src>
+typename std::enable_if<
+ std::is_convertible<Src, const char*>::value
+ && detail::IsSomeString<Tgt>::value>::type
+toAppend(Src value, Tgt * result) {
+ // Treat null pointers like an empty string, as in:
+ // operator<<(std::ostream&, const char*).
+ const char* c = value;
+ if (c) {
+ result->append(value);
+ }
+}
+
+/**
+ * Strings get appended, too.
+ */
+template <class Tgt, class Src>
+typename std::enable_if<
+ detail::IsSomeString<Src>::value && detail::IsSomeString<Tgt>::value>::type
+toAppend(const Src& value, Tgt * result) {
+ result->append(value);
+}
+
+/**
+ * and StringPiece objects too
+ */
+template <class Tgt>
+typename std::enable_if<
+ detail::IsSomeString<Tgt>::value>::type
+toAppend(StringPiece value, Tgt * result) {
+ result->append(value.data(), value.size());
+}
+
+/**
+ * There's no implicit conversion from fbstring to other string types,
+ * so make a specialization.
+ */
+template <class Tgt>
+typename std::enable_if<
+ detail::IsSomeString<Tgt>::value>::type
+toAppend(const fbstring& value, Tgt * result) {
+ result->append(value.data(), value.size());
+}
+
+/**
+ * int32_t and int64_t to string (by appending) go through here. The
+ * result is APPENDED to a preexisting string passed as the second
+ * parameter. For convenience, the function also returns a reference
+ * to *result. This should be efficient with fbstring because fbstring
+ * incurs no dynamic allocation below 23 bytes and no number has more
+ * than 22 bytes in its textual representation (20 for digits, one for
+ * sign, one for the terminating 0).
+ */
+template <class Tgt, class Src>
+typename std::enable_if<
+ std::is_integral<Src>::value && std::is_signed<Src>::value
+ && detail::IsSomeString<Tgt>::value && sizeof(Src) >= 4>::type
+toAppend(Src value, Tgt * result) {
+ typedef typename std::make_unsigned<Src>::type Usrc;
+ char buffer[detail::kMaxInt64BufLen];
+ size_t begin;
+ if (value < 0) {
+ begin = detail::uintToBuffer(buffer, sizeof(buffer),
+ static_cast<Usrc>(-value));
+ DCHECK_GE(begin, 1);
+ buffer[--begin] = '-';
+ } else {
+ begin = detail::uintToBuffer(buffer, sizeof(buffer),
+ static_cast<Usrc>(value));
+ }
+ result->append(buffer + begin, buffer + sizeof(buffer));
+}
+
+/**
+ * As above, but for uint32_t and uint64_t.
+ */
+template <class Tgt, class Src>
+typename std::enable_if<
+ std::is_integral<Src>::value && !std::is_signed<Src>::value
+ && detail::IsSomeString<Tgt>::value && sizeof(Src) >= 4>::type
+toAppend(Src value, Tgt * result) {
+ char buffer[detail::kMaxInt64BufLen];
+ const size_t begin = detail::uintToBuffer(buffer, sizeof(buffer), value);
+ result->append(buffer + begin, buffer + sizeof(buffer));
+}
+
+/**
+ * All small signed and unsigned integers to string go through 32-bit
+ * types int32_t and uint32_t, respectively.
+ */
+template <class Tgt, class Src>
+typename std::enable_if<
+ std::is_integral<Src>::value
+ && detail::IsSomeString<Tgt>::value && sizeof(Src) < 4>::type
+toAppend(Src value, Tgt * result) {
+ typedef typename
+ std::conditional<std::is_signed<Src>::value, int64_t, uint64_t>::type
+ Intermediate;
+ toAppend<Tgt>(static_cast<Intermediate>(value), result);
+}
+
+/**
+ * Enumerated values get appended as integers.
+ */
+template <class Tgt, class Src>
+typename std::enable_if<
+ std::is_enum<Src>::value && detail::IsSomeString<Tgt>::value>::type
+toAppend(Src value, Tgt * result) {
+ /* static */ if (Src(-1) < 0) {
+ /* static */ if (sizeof(Src) <= sizeof(int)) {
+ toAppend(static_cast<int>(value), result);
+ } else {
+ toAppend(static_cast<long>(value), result);
+ }
+ } else {
+ /* static */ if (sizeof(Src) <= sizeof(int)) {
+ toAppend(static_cast<unsigned int>(value), result);
+ } else {
+ toAppend(static_cast<unsigned long>(value), result);
+ }
+ }
+}
+
+/*******************************************************************************
+ * Conversions from floating-point types to string types.
+ ******************************************************************************/
+
+/** Wrapper around DoubleToStringConverter **/
+template <class Tgt, class Src>
+typename std::enable_if<
+ std::is_floating_point<Src>::value
+ && detail::IsSomeString<Tgt>::value>::type
+toAppend(
+ Src value,
+ Tgt * result,
+ double_conversion::DoubleToStringConverter::DtoaMode mode,
+ unsigned int numDigits) {
+ using namespace double_conversion;
+ DoubleToStringConverter
+ conv(DoubleToStringConverter::NO_FLAGS,
+ "infinity", "NaN", 'E',
+ -6, // decimal in shortest low
+ 21, // decimal in shortest high
+ 6, // max leading padding zeros
+ 1); // max trailing padding zeros
+ char buffer[256];
+ StringBuilder builder(buffer, sizeof(buffer));
+ switch (mode) {
+ case DoubleToStringConverter::SHORTEST:
+ conv.ToShortest(value, &builder);
+ break;
+ case DoubleToStringConverter::FIXED:
+ conv.ToFixed(value, numDigits, &builder);
+ break;
+ default:
+ CHECK(mode == DoubleToStringConverter::PRECISION);
+ conv.ToPrecision(value, numDigits, &builder);
+ break;
+ }
+ const size_t length = builder.position();
+ builder.Finalize();
+ result->append(buffer, length);
+}
+
+/**
+ * As above, but for floating point
+ */
+template <class Tgt, class Src>
+typename std::enable_if<
+ std::is_floating_point<Src>::value
+ && detail::IsSomeString<Tgt>::value>::type
+toAppend(Src value, Tgt * result) {
+ toAppend(
+ value, result, double_conversion::DoubleToStringConverter::SHORTEST, 0);
+}
+
+/**
+ * Variadic conversion to string. Appends each element in turn.
+ */
+template <class T, class... Ts>
+typename std::enable_if<sizeof...(Ts) >= 2
+ && detail::IsSomeString<
+ typename std::remove_pointer<
+ typename std::tuple_element<
+ sizeof...(Ts) - 1, std::tuple<Ts...>
+ >::type>::type>::value>::type
+toAppend(const T& v, const Ts&... vs) {
+ toAppend(v, detail::getLastElement(vs...));
+ toAppend(vs...);
+}
+
+/**
+ * Variadic base case: do nothing.
+ */
+template <class Tgt>
+typename std::enable_if<detail::IsSomeString<Tgt>::value>::type
+toAppend(Tgt* result) {
+}
+
+/**
+ * to<SomeString>(v1, v2, ...) uses toAppend() (see below) as back-end
+ * for all types.
+ */
+template <class Tgt, class... Ts>
+typename std::enable_if<detail::IsSomeString<Tgt>::value, Tgt>::type
+to(const Ts&... vs) {
+ Tgt result;
+ toAppend(vs..., &result);
+ return result;
+}
+
+/*******************************************************************************
+ * Conversions from string types to integral types.
+ ******************************************************************************/
+
+namespace detail {
+
+/**
+ * Finds the first non-digit in a string. The number of digits
+ * searched depends on the precision of the Tgt integral. Assumes the
+ * string starts with NO whitespace and NO sign.
+ *
+ * The semantics of the routine is:
+ * for (;; ++b) {
+ * if (b >= e || !isdigit(*b)) return b;
+ * }
+ *
+ * Complete unrolling marks bottom-line (i.e. entire conversion)
+ * improvements of 20%.
+ */
+ template <class Tgt>
+ const char* findFirstNonDigit(const char* b, const char* e) {
+ for (; b < e; ++b) {
+ auto const c = static_cast<unsigned>(*b) - '0';
+ if (c >= 10) break;
+ }
+ return b;
+ }
+
+ // Maximum value of number when represented as a string
+ template <class T> struct MaxString {
+ static const char*const value;
+ };
+
+/**
+ * String represented as a pair of pointers to char to unsigned
+ * integrals. Assumes NO whitespace before or after, and also that the
+ * string is composed entirely of digits. Tgt must be unsigned, and no
+ * sign is allowed in the string (even it's '+'). String may be empty,
+ * in which case digits_to throws.
+ */
+ template <class Tgt>
+ Tgt digits_to(const char * b, const char * e) {
+
+ static_assert(!std::is_signed<Tgt>::value, "Unsigned type expected");
+ assert(b <= e);
+
+ const size_t size = e - b;
+
+ /* Although the string is entirely made of digits, we still need to
+ * check for overflow.
+ */
+ if (size >= std::numeric_limits<Tgt>::digits10 + 1) {
+ // Leading zeros? If so, recurse to keep things simple
+ if (b < e && *b == '0') {
+ for (++b;; ++b) {
+ if (b == e) return 0; // just zeros, e.g. "0000"
+ if (*b != '0') return digits_to<Tgt>(b, e);
+ }
+ }
+ FOLLY_RANGE_CHECK(size == std::numeric_limits<Tgt>::digits10 + 1 &&
+ strncmp(b, detail::MaxString<Tgt>::value, size) <= 0,
+ "Numeric overflow upon conversion");
+ }
+
+ // Here we know that the number won't overflow when
+ // converted. Proceed without checks.
+
+ static const Tgt power10[20] = {
+ static_cast<Tgt>(10000000000000000000UL),
+ static_cast<Tgt>(1000000000000000000UL),
+ static_cast<Tgt>(100000000000000000UL),
+ static_cast<Tgt>(10000000000000000UL),
+ static_cast<Tgt>(1000000000000000UL),
+ static_cast<Tgt>(100000000000000UL),
+ static_cast<Tgt>(10000000000000UL),
+ static_cast<Tgt>(1000000000000UL),
+ static_cast<Tgt>(100000000000UL),
+ static_cast<Tgt>(10000000000UL),
+ static_cast<Tgt>(1000000000UL),
+ static_cast<Tgt>(100000000UL),
+ static_cast<Tgt>(10000000UL),
+ static_cast<Tgt>(1000000UL),
+ static_cast<Tgt>(100000UL),
+ static_cast<Tgt>(10000UL),
+ static_cast<Tgt>(1000UL),
+ static_cast<Tgt>(100UL),
+ static_cast<Tgt>(10UL),
+ static_cast<Tgt>(1UL),
+ };
+
+ size_t powIdx = sizeof(power10) / sizeof(*power10) - size;
+ Tgt result = 0;
+
+ for (; e - b >= 4; b += 4, powIdx += 4) {
+ const auto c0 = static_cast<unsigned>(*b) - '0';
+ if (c0 >= 10) goto failure;
+ const auto r0 = power10[powIdx] * c0;
+ const auto c1 = static_cast<unsigned>(b[1]) - '0';
+ if (c1 >= 10) goto failure;
+ const auto r1 = power10[powIdx + 1] * c1;
+ const auto c2 = static_cast<unsigned>(b[2]) - '0';
+ if (c2 >= 10) goto failure;
+ const auto r2 = power10[powIdx + 2] * c2;
+ const auto c3 = static_cast<unsigned>(b[3]) - '0';
+ if (c3 >= 10) goto failure;
+ const auto r3 = power10[powIdx + 3] * c3;
+ result += r0 + r1 + r2 + r3;
+ }
+
+ switch (e - b) {
+ case 3: {
+ const auto c0 = static_cast<unsigned>(*b) - '0';
+ if (c0 >= 10) goto failure;
+ const auto c1 = static_cast<unsigned>(b[1]) - '0';
+ if (c1 >= 10) goto failure;
+ const auto c2 = static_cast<unsigned>(b[2]) - '0';
+ if (c2 >= 10) goto failure;
+ return result + 100 * c0 + 10 * c1 + c2;
+ }
+ case 2: {
+ const auto c0 = static_cast<unsigned>(*b) - '0';
+ if (c0 >= 10) goto failure;
+ const auto c1 = static_cast<unsigned>(b[1]) - '0';
+ if (c1 >= 10) goto failure;
+ return result + 10 * c0 + c1;
+ }
+ case 1: {
+ const auto c0 = static_cast<unsigned>(*b) - '0';
+ if (c0 >= 10) goto failure;
+ return result + c0;
+ }
+ }
+
+ assert(b == e);
+ FOLLY_RANGE_CHECK(size > 0, "Found no digits to convert in input");
+ return result;
+
+ failure:
+ throw std::range_error("Cannot convert string " +
+ std::string(e - size, e) + " to integral.");
+ }
+
+ bool str_to_bool(StringPiece * src);
+
+} // namespace detail
+
+/**
+ * String represented as a pair of pointers to char to unsigned
+ * integrals. Assumes NO whitespace before or after.
+ */
+template <class Tgt>
+typename std::enable_if<
+ std::is_integral<Tgt>::value && !std::is_signed<Tgt>::value
+ && !std::is_same<typename std::remove_cv<Tgt>::type, bool>::value,
+ Tgt>::type
+to(const char * b, const char * e) {
+ return detail::digits_to<Tgt>(b, e);
+}
+
+/**
+ * String represented as a pair of pointers to char to signed
+ * integrals. Assumes NO whitespace before or after. Allows an
+ * optional leading sign.
+ */
+template <class Tgt>
+typename std::enable_if<
+ std::is_integral<Tgt>::value && std::is_signed<Tgt>::value,
+ Tgt>::type
+to(const char * b, const char * e) {
+ FOLLY_RANGE_CHECK(b < e, "Empty input string in conversion to integral");
+ if (!isdigit(*b)) {
+ if (*b == '-') {
+ Tgt result = -to<typename std::make_unsigned<Tgt>::type>(b + 1, e);
+ FOLLY_RANGE_CHECK(result <= 0, "Negative overflow.");
+ return result;
+ }
+ FOLLY_RANGE_CHECK(*b == '+', "Invalid lead character");
+ ++b;
+ }
+ Tgt result = to<typename std::make_unsigned<Tgt>::type>(b, e);
+ FOLLY_RANGE_CHECK(result >= 0, "Overflow.");
+ return result;
+}
+
+/**
+ * Parsing strings to integrals. These routines differ from
+ * to<integral>(string) in that they take a POINTER TO a StringPiece
+ * and alter that StringPiece to reflect progress information.
+ */
+
+/**
+ * StringPiece to integrals, with progress information. Alters the
+ * StringPiece parameter to munch the already-parsed characters.
+ */
+template <class Tgt>
+typename std::enable_if<
+ std::is_integral<Tgt>::value
+ && !std::is_same<typename std::remove_cv<Tgt>::type, bool>::value,
+ Tgt>::type
+to(StringPiece * src) {
+
+ auto b = src->data(), past = src->data() + src->size();
+ for (;; ++b) {
+ FOLLY_RANGE_CHECK(b < past, "No digits found in input string");
+ if (!isspace(*b)) break;
+ }
+
+ auto m = b;
+
+ // First digit is customized because we test for sign
+ bool negative = false;
+ /* static */ if (std::is_signed<Tgt>::value) {
+ if (!isdigit(*m)) {
+ if (*m == '-') {
+ negative = true;
+ } else {
+ FOLLY_RANGE_CHECK(*m == '+', "Invalid leading character in conversion"
+ " to integral");
+ }
+ ++b;
+ ++m;
+ }
+ }
+ FOLLY_RANGE_CHECK(m < past, "No digits found in input string");
+ FOLLY_RANGE_CHECK(isdigit(*m), "Non-digit character found");
+ m = detail::findFirstNonDigit<Tgt>(m + 1, past);
+
+ Tgt result;
+ /* static */ if (!std::is_signed<Tgt>::value) {
+ result = detail::digits_to<typename std::make_unsigned<Tgt>::type>(b, m);
+ } else {
+ auto t = detail::digits_to<typename std::make_unsigned<Tgt>::type>(b, m);
+ if (negative) {
+ result = -t;
+ FOLLY_RANGE_CHECK(result <= 0, "Negative overflow");
+ } else {
+ result = t;
+ FOLLY_RANGE_CHECK(result >= 0, "Overflow");
+ }
+ }
+ src->advance(m - src->data());
+ return result;
+}
+
+/**
+ * StringPiece to bool, with progress information. Alters the
+ * StringPiece parameter to munch the already-parsed characters.
+ */
+template <class Tgt>
+typename std::enable_if<
+ std::is_same<typename std::remove_cv<Tgt>::type, bool>::value,
+ Tgt>::type
+to(StringPiece * src) {
+ return detail::str_to_bool(src);
+}
+
+namespace detail {
+
+/**
+ * Enforce that the suffix following a number is made up only of whitespace.
+ */
+inline void enforceWhitespace(const char* b, const char* e) {
+ for (; b != e; ++b) {
+ FOLLY_RANGE_CHECK(isspace(*b), to<std::string>("Non-whitespace: ", *b));
+ }
+}
+
+} // namespace detail
+
+/**
+ * String or StringPiece to integrals. Accepts leading and trailing
+ * whitespace, but no non-space trailing characters.
+ */
+template <class Tgt>
+typename std::enable_if<
+ std::is_integral<Tgt>::value,
+ Tgt>::type
+to(StringPiece src) {
+ Tgt result = to<Tgt>(&src);
+ detail::enforceWhitespace(src.data(), src.data() + src.size());
+ return result;
+}
+
+/*******************************************************************************
+ * Conversions from string types to floating-point types.
+ ******************************************************************************/
+
+/**
+ * StringPiece to double, with progress information. Alters the
+ * StringPiece parameter to munch the already-parsed characters.
+ */
+template <class Tgt>
+inline typename std::enable_if<
+ std::is_floating_point<Tgt>::value,
+ Tgt>::type
+to(StringPiece *const src) {
+ using namespace double_conversion;
+ static StringToDoubleConverter
+ conv(StringToDoubleConverter::ALLOW_TRAILING_JUNK
+ | StringToDoubleConverter::ALLOW_LEADING_SPACES,
+ 0.0,
+ // return this for junk input string
+ std::numeric_limits<double>::quiet_NaN(),
+ nullptr, nullptr);
+
+ FOLLY_RANGE_CHECK(!src->empty(), "No digits found in input string");
+
+ int length;
+ auto result = conv.StringToDouble(src->data(), src->size(),
+ &length); // processed char count
+
+ if (!std::isnan(result)) {
+ src->advance(length);
+ return result;
+ }
+
+ for (;; src->advance(1)) {
+ if (src->empty()) {
+ throw std::range_error("Unable to convert an empty string"
+ " to a floating point value.");
+ }
+ if (!isspace(src->front())) {
+ break;
+ }
+ }
+
+ // Was that "inf[inity]"?
+ if (src->size() >= 3 && toupper((*src)[0]) == 'I'
+ && toupper((*src)[1]) == 'N' && toupper((*src)[2]) == 'F') {
+ if (src->size() >= 8 &&
+ toupper((*src)[3]) == 'I' &&
+ toupper((*src)[4]) == 'N' &&
+ toupper((*src)[5]) == 'I' &&
+ toupper((*src)[6]) == 'T' &&
+ toupper((*src)[7]) == 'Y') {
+ src->advance(8);
+ } else {
+ src->advance(3);
+ }
+ return std::numeric_limits<Tgt>::infinity();
+ }
+
+ // Was that "-inf[inity]"?
+ if (src->size() >= 4 && toupper((*src)[0]) == '-'
+ && toupper((*src)[1]) == 'I' && toupper((*src)[2]) == 'N'
+ && toupper((*src)[3]) == 'F') {
+ if (src->size() >= 9 &&
+ toupper((*src)[4]) == 'I' &&
+ toupper((*src)[5]) == 'N' &&
+ toupper((*src)[6]) == 'I' &&
+ toupper((*src)[7]) == 'T' &&
+ toupper((*src)[8]) == 'Y') {
+ src->advance(9);
+ } else {
+ src->advance(4);
+ }
+ return -std::numeric_limits<Tgt>::infinity();
+ }
+
+ // "nan"?
+ if (src->size() >= 3 && toupper((*src)[0]) == 'N'
+ && toupper((*src)[1]) == 'A' && toupper((*src)[2]) == 'N') {
+ src->advance(3);
+ return std::numeric_limits<Tgt>::quiet_NaN();
+ }
+
+ // All bets are off
+ throw std::range_error("Unable to convert \"" + src->toString()
+ + "\" to a floating point value.");
+}
+
+/**
+ * Any string, const char*, or StringPiece to double.
+ */
+template <class Tgt>
+typename std::enable_if<
+ std::is_floating_point<Tgt>::value,
+ Tgt>::type
+to(StringPiece src) {
+ Tgt result = to<double>(&src);
+ detail::enforceWhitespace(src.data(), src.data() + src.size());
+ return result;
+}
+
+/*******************************************************************************
+ * Integral to floating point and back
+ ******************************************************************************/
+
+/**
+ * Checked conversion from integral to flating point and back. The
+ * result must be convertible back to the source type without loss of
+ * precision. This seems Draconian but sometimes is what's needed, and
+ * complements existing routines nicely. For various rounding
+ * routines, see <math>.
+ */
+template <class Tgt, class Src>
+typename std::enable_if<
+ (std::is_integral<Src>::value && std::is_floating_point<Tgt>::value)
+ ||
+ (std::is_floating_point<Src>::value && std::is_integral<Tgt>::value),
+ Tgt>::type
+to(const Src & value) {
+ Tgt result = value;
+ auto witness = static_cast<Src>(result);
+ if (value != witness) {
+ throw std::range_error(
+ to<std::string>("to<>: loss of precision when converting ", value,
+ " to type ", typeid(Tgt).name()).c_str());
+ }
+ return result;
+}
+
+/*******************************************************************************
+ * Enum to anything and back
+ ******************************************************************************/
+
+template <class Tgt, class Src>
+typename std::enable_if<std::is_enum<Src>::value, Tgt>::type
+to(const Src & value) {
+ // TODO: uncomment this when underlying_type is available
+ // return to<Tgt>(static_cast<typename std::underlying_type<Src>::type>(
+ // value));
+ /* static */ if (Src(-1) < 0) {
+ /* static */ if (sizeof(Src) <= sizeof(int)) {
+ return to<Tgt>(static_cast<int>(value));
+ } else {
+ return to<Tgt>(static_cast<long>(value));
+ }
+ } else {
+ /* static */ if (sizeof(Src) <= sizeof(int)) {
+ return to<Tgt>(static_cast<unsigned int>(value));
+ } else {
+ return to<Tgt>(static_cast<unsigned long>(value));
+ }
+ }
+}
+
+template <class Tgt, class Src>
+typename std::enable_if<std::is_enum<Tgt>::value, Tgt>::type
+to(const Src & value) {
+ // TODO: uncomment this when underlying_type is available
+ // return static_cast<Tgt>(
+ // to<typename std::underlying_type<Tgt>::type>(value));
+ /* static */ if (Tgt(-1) < 0) {
+ /* static */ if (sizeof(Tgt) <= sizeof(int)) {
+ return static_cast<Tgt>(to<int>(value));
+ } else {
+ return static_cast<Tgt>(to<long>(value));
+ }
+ } else {
+ /* static */ if (sizeof(Tgt) <= sizeof(int)) {
+ return static_cast<Tgt>(to<unsigned int>(value));
+ } else {
+ return static_cast<Tgt>(to<unsigned long>(value));
+ }
+ }
+}
+
+} // namespace folly
+
+// FOLLY_CONV_INTERNAL is defined by Conv.cpp. Keep the FOLLY_RANGE_CHECK
+// macro for use in Conv.cpp, but #undefine it everywhere else we are included,
+// to avoid defining this global macro name in other files that include Conv.h.
+#ifndef FOLLY_CONV_INTERNAL
+#undef FOLLY_RANGE_CHECK
+#endif
+
+#endif /* FOLLY_BASE_CONV_H_ */
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Discriminated pointer: Type-safe pointer to one of several types.
+ *
+ * Similar to boost::variant, but has no space overhead over a raw pointer, as
+ * it relies on the fact that (on x86_64) there are 16 unused bits in a
+ * pointer.
+ *
+ * @author Tudor Bosman (tudorb@fb.com)
+ */
+
+#ifndef FOLLY_DISCRIMINATEDPTR_H_
+#define FOLLY_DISCRIMINATEDPTR_H_
+
+#include <limits>
+#include <stdexcept>
+#include <glog/logging.h>
+#include "folly/Likely.h"
+#include "folly/detail/DiscriminatedPtrDetail.h"
+
+#ifndef __x86_64__
+# error "DiscriminatedPtr is x64-specific code."
+#endif
+
+namespace folly {
+
+/**
+ * Discriminated pointer.
+ *
+ * Given a list of types, a DiscriminatedPtr<Types...> may point to an object
+ * of one of the given types, or may be empty. DiscriminatedPtr is type-safe:
+ * you may only get a pointer to the type that you put in, otherwise get
+ * throws an exception (and get_nothrow returns nullptr)
+ *
+ * This pointer does not do any kind of lifetime management -- it's not a
+ * "smart" pointer. You are responsible for deallocating any memory used
+ * to hold pointees, if necessary.
+ */
+template <typename... Types>
+class DiscriminatedPtr {
+ // <, not <=, as our indexes are 1-based (0 means "empty")
+ static_assert(sizeof...(Types) < std::numeric_limits<uint16_t>::max(),
+ "too many types");
+
+ public:
+ /**
+ * Create an empty DiscriminatedPtr.
+ */
+ DiscriminatedPtr() : data_(0) {
+ }
+
+ /**
+ * Create a DiscriminatedPtr that points to an object of type T.
+ * Fails at compile time if T is not a valid type (listed in Types)
+ */
+ template <typename T>
+ explicit DiscriminatedPtr(T* ptr) {
+ set(ptr, typeIndex<T>());
+ }
+
+ /**
+ * Set this DiscriminatedPtr to point to an object of type T.
+ * Fails at compile time if T is not a valid type (listed in Types)
+ */
+ template <typename T>
+ void set(T* ptr) {
+ set(ptr, typeIndex<T>());
+ }
+
+ /**
+ * Get a pointer to the object that this DiscriminatedPtr points to, if it is
+ * of type T. Fails at compile time if T is not a valid type (listed in
+ * Types), and returns nullptr if this DiscriminatedPtr is empty or points to
+ * an object of a different type.
+ */
+ template <typename T>
+ T* get_nothrow() noexcept {
+ void* p = LIKELY(hasType<T>()) ? ptr() : nullptr;
+ return static_cast<T*>(p);
+ }
+
+ template <typename T>
+ const T* get_nothrow() const noexcept {
+ const void* p = LIKELY(hasType<T>()) ? ptr() : nullptr;
+ return static_cast<const T*>(p);
+ }
+
+ /**
+ * Get a pointer to the object that this DiscriminatedPtr points to, if it is
+ * of type T. Fails at compile time if T is not a valid type (listed in
+ * Types), and throws std::invalid_argument if this DiscriminatedPtr is empty
+ * or points to an object of a different type.
+ */
+ template <typename T>
+ T* get() {
+ if (UNLIKELY(!hasType<T>())) {
+ throw std::invalid_argument("Invalid type");
+ }
+ return static_cast<T*>(ptr());
+ }
+
+ template <typename T>
+ const T* get() const {
+ if (UNLIKELY(!hasType<T>())) {
+ throw std::invalid_argument("Invalid type");
+ }
+ return static_cast<const T*>(ptr());
+ }
+
+ /**
+ * Return true iff this DiscriminatedPtr is empty.
+ */
+ bool empty() const {
+ return index() == 0;
+ }
+
+ /**
+ * Return true iff the object pointed by this DiscriminatedPtr has type T,
+ * false otherwise. Fails at compile time if T is not a valid type (listed
+ * in Types...)
+ */
+ template <typename T>
+ bool hasType() const {
+ return index() == typeIndex<T>();
+ }
+
+ /**
+ * Clear this DiscriminatedPtr, making it empty.
+ */
+ void clear() {
+ data_ = 0;
+ }
+
+ /**
+ * Assignment operator from a pointer of type T.
+ */
+ template <typename T>
+ DiscriminatedPtr& operator=(T* ptr) {
+ set(ptr);
+ return *this;
+ }
+
+ /**
+ * Apply a visitor to this object, calling the appropriate overload for
+ * the type currently stored in DiscriminatedPtr. Throws invalid_argument
+ * if the DiscriminatedPtr is empty.
+ *
+ * The visitor must meet the following requirements:
+ *
+ * - The visitor must allow invocation as a function by overloading
+ * operator(), unambiguously accepting all values of type T* (or const T*)
+ * for all T in Types...
+ * - All operations of the function object on T* (or const T*) must
+ * return the same type (or a static_assert will fire).
+ */
+ template <typename V>
+ typename dptr_detail::VisitorResult<V, Types...>::type apply(V&& visitor) {
+ size_t n = index();
+ if (n == 0) throw std::invalid_argument("Empty DiscriminatedPtr");
+ return dptr_detail::ApplyVisitor<V, Types...>()(
+ n, std::forward<V>(visitor), ptr());
+ }
+
+ template <typename V>
+ typename dptr_detail::ConstVisitorResult<V, Types...>::type apply(V&& visitor)
+ const {
+ size_t n = index();
+ if (n == 0) throw std::invalid_argument("Empty DiscriminatedPtr");
+ return dptr_detail::ApplyConstVisitor<V, Types...>()(
+ n, std::forward<V>(visitor), ptr());
+ }
+
+ private:
+ /**
+ * Get the 1-based type index of T in Types.
+ */
+ template <typename T>
+ size_t typeIndex() const {
+ return dptr_detail::GetTypeIndex<T, Types...>::value;
+ }
+
+ uint16_t index() const { return data_ >> 48; }
+ void* ptr() const {
+ return reinterpret_cast<void*>(data_ & ((1ULL << 48) - 1));
+ }
+
+ void set(void* p, uint16_t v) {
+ uintptr_t ip = reinterpret_cast<uintptr_t>(p);
+ CHECK(!(ip >> 48));
+ ip |= static_cast<uintptr_t>(v) << 48;
+ data_ = ip;
+ }
+
+ /**
+ * We store a pointer in the least significant 48 bits of data_, and a type
+ * index (0 = empty, or 1-based index in Types) in the most significant 16
+ * bits. We rely on the fact that pointers have their most significant 16
+ * bits clear on x86_64.
+ */
+ uintptr_t data_;
+};
+
+} // namespace folly
+
+#endif /* FOLLY_DISCRIMINATEDPTR_H_ */
+
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @author: Andrei Alexandrescu (aalexandre)
+// String type.
+
+#ifndef FOLLY_BASE_FBSTRING_H_
+#define FOLLY_BASE_FBSTRING_H_
+
+/**
+ fbstring's behavior can be configured via two macro definitions, as
+ follows. Normally, fbstring does not write a '\0' at the end of
+ each string whenever it changes the underlying characters. Instead,
+ it lazily writes the '\0' whenever either c_str() or data()
+ called.
+
+ This is standard-compliant behavior and may save costs in some
+ circumstances. However, it may be surprising to some client code
+ because c_str() and data() are const member functions (fbstring
+ uses the "mutable" storage class for its own state).
+
+ In order to appease client code that expects fbstring to be
+ zero-terminated at all times, if the preprocessor symbol
+ FBSTRING_CONSERVATIVE is defined, fbstring does exactly that,
+ i.e. it goes the extra mile to guarantee a '\0' is always planted
+ at the end of its data.
+
+ On the contrary, if the desire is to debug faulty client code that
+ unduly assumes the '\0' is present, fbstring plants a '^' (i.e.,
+ emphatically NOT a zero) at the end of each string if
+ FBSTRING_PERVERSE is defined. (Calling c_str() or data() still
+ writes the '\0', of course.)
+
+ The preprocessor symbols FBSTRING_PERVERSE and
+ FBSTRING_CONSERVATIVE cannot be defined simultaneously. This is
+ enforced during preprocessing.
+*/
+
+//#define FBSTRING_PERVERSE
+//#define FBSTRING_CONSERVATIVE
+
+#ifdef FBSTRING_PERVERSE
+#ifdef FBSTRING_CONSERVATIVE
+#error Cannot define both FBSTRING_PERVERSE and FBSTRING_CONSERVATIVE.
+#endif
+#endif
+
+// This file appears in two locations: inside fbcode and in the
+// libstdc++ source code (when embedding fbstring as std::string).
+// To aid in this schizophrenic use, two macros are defined in
+// c++config.h:
+// _LIBSTDCXX_FBSTRING - Set inside libstdc++. This is useful to
+// gate use inside fbcode v. libstdc++
+#include <bits/c++config.h>
+
+#ifdef _LIBSTDCXX_FBSTRING
+
+#pragma GCC system_header
+
+// Handle the cases where the fbcode version (folly/Malloc.h) is included
+// either before or after this inclusion. */home/engshare/third-party/src/
+// libgcc/libgcc-4.6.2/gcc-4.6.2-20111027/libstdc++-v3/include/bits/
+// basic_string.h* has a more detailed explanation of why this is necessary.
+#ifdef FOLLY_MALLOC_H_
+#undef FOLLY_MALLOC_H_
+#include "basic_fbstring_malloc.h"
+#else
+#include "basic_fbstring_malloc.h"
+#undef FOLLY_MALLOC_H_
+#endif
+
+#else // !_LIBSTDCXX_FBSTRING
+
+#include <string>
+#include <cstring>
+#include <cassert>
+
+#include "folly/Traits.h"
+#include "folly/Malloc.h"
+#include "folly/Hash.h"
+
+#endif
+
+#include <atomic>
+#include <limits>
+#include <type_traits>
+
+#ifdef _LIBSTDCXX_FBSTRING
+namespace std _GLIBCXX_VISIBILITY(default) {
+_GLIBCXX_BEGIN_NAMESPACE_VERSION
+#else
+namespace folly {
+#endif
+
+namespace fbstring_detail {
+
+template <class InIt, class OutIt>
+inline
+OutIt copy_n(InIt b,
+ typename std::iterator_traits<InIt>::difference_type n,
+ OutIt d) {
+ for (; n != 0; --n, ++b, ++d) {
+ assert((const void*)&*d != &*b);
+ *d = *b;
+ }
+ return d;
+}
+
+template <class Pod, class T>
+inline void pod_fill(Pod* b, Pod* e, T c) {
+ assert(b && e && b <= e);
+ /*static*/ if (sizeof(T) == 1) {
+ memset(b, c, e - b);
+ } else {
+ auto const ee = b + ((e - b) & ~7u);
+ for (; b != ee; b += 8) {
+ b[0] = c;
+ b[1] = c;
+ b[2] = c;
+ b[3] = c;
+ b[4] = c;
+ b[5] = c;
+ b[6] = c;
+ b[7] = c;
+ }
+ // Leftovers
+ for (; b != e; ++b) {
+ *b = c;
+ }
+ }
+}
+
+/*
+ * Lightly structured memcpy, simplifies copying PODs and introduces
+ * some asserts
+ */
+template <class Pod>
+inline Pod* pod_copy(const Pod* b, const Pod* e, Pod* d) {
+ assert(e >= b);
+ assert(d >= e || d + (e - b) <= b);
+ const size_t s = e - b;
+ std::memcpy(d, b, s * sizeof(*b));
+ return d + s;
+}
+
+/*
+ * Lightly structured memmove, simplifies copying PODs and introduces
+ * some asserts
+ */
+template <class Pod>
+inline void pod_move(const Pod* b, const Pod* e, Pod* d) {
+ assert(e >= b);
+ memmove(d, b, (e - b) * sizeof(*b));
+}
+
+} // namespace fbstring_detail
+
+/**
+ * Defines a special acquisition method for constructing fbstring
+ * objects. AcquireMallocatedString means that the user passes a
+ * pointer to a malloc-allocated string that the fbstring object will
+ * take into custody.
+ */
+enum class AcquireMallocatedString {};
+
+/*
+ * fbstring_core_model is a mock-up type that defines all required
+ * signatures of a fbstring core. The fbstring class itself uses such
+ * a core object to implement all of the numerous member functions
+ * required by the standard.
+ *
+ * If you want to define a new core, copy the definition below and
+ * implement the primitives. Then plug the core into basic_fbstring as
+ * a template argument.
+
+template <class Char>
+class fbstring_core_model {
+public:
+ fbstring_core_model();
+ fbstring_core_model(const fbstring_core_model &);
+ ~fbstring_core_model();
+ // Returns a pointer to string's buffer (currently only contiguous
+ // strings are supported). The pointer is guaranteed to be valid
+ // until the next call to a non-const member function.
+ const Char * data() const;
+ // Much like data(), except the string is prepared to support
+ // character-level changes. This call is a signal for
+ // e.g. reference-counted implementation to fork the data. The
+ // pointer is guaranteed to be valid until the next call to a
+ // non-const member function.
+ Char * mutable_data();
+ // Returns a pointer to string's buffer and guarantees that a
+ // readable '\0' lies right after the buffer. The pointer is
+ // guaranteed to be valid until the next call to a non-const member
+ // function.
+ const Char * c_str() const;
+ // Shrinks the string by delta characters. Asserts that delta <=
+ // size().
+ void shrink(size_t delta);
+ // Expands the string by delta characters (i.e. after this call
+ // size() will report the old size() plus delta) but without
+ // initializing the expanded region. The caller is expected to fill
+ // the expanded area appropriately.
+ void expand_noinit(size_t delta);
+ // Expands the string by one character and sets the last character
+ // to c.
+ void push_back(Char c);
+ // Returns the string's size.
+ size_t size() const;
+ // Returns the string's capacity, i.e. maximum size that the string
+ // can grow to without reallocation. Note that for reference counted
+ // strings that's technically a lie - even assigning characters
+ // within the existing size would cause a reallocation.
+ size_t capacity() const;
+ // Returns true if the data underlying the string is actually shared
+ // across multiple strings (in a refcounted fashion).
+ bool isShared() const;
+ // Makes sure that at least minCapacity characters are available for
+ // the string without reallocation. For reference-counted strings,
+ // it should fork the data even if minCapacity < size().
+ void reserve(size_t minCapacity);
+private:
+ // Do not implement
+ fbstring_core_model& operator=(const fbstring_core_model &);
+};
+*/
+
+/**
+ * This is the core of the string. The code should work on 32- and
+ * 64-bit architectures and with any Char size. Porting to big endian
+ * architectures would require some changes.
+ *
+ * The storage is selected as follows (assuming we store one-byte
+ * characters on a 64-bit machine): (a) "small" strings between 0 and
+ * 23 chars are stored in-situ without allocation (the rightmost byte
+ * stores the size); (b) "medium" strings from 24 through 254 chars
+ * are stored in malloc-allocated memory that is copied eagerly; (c)
+ * "large" strings of 255 chars and above are stored in a similar
+ * structure as medium arrays, except that the string is
+ * reference-counted and copied lazily. the reference count is
+ * allocated right before the character array.
+ *
+ * The discriminator between these three strategies sits in the two
+ * most significant bits of the rightmost char of the storage. If
+ * neither is set, then the string is small (and its length sits in
+ * the lower-order bits of that rightmost character). If the MSb is
+ * set, the string is medium width. If the second MSb is set, then the
+ * string is large.
+ */
+template <class Char> class fbstring_core {
+public:
+ fbstring_core() {
+ // Only initialize the tag, will set the MSBs (i.e. the small
+ // string size) to zero too
+ ml_.capacity_ = maxSmallSize << (8 * (sizeof(size_t) - 1));
+ // or: setSmallSize(0);
+ writeTerminator();
+ assert(category() == isSmall && size() == 0);
+ }
+
+ fbstring_core(const fbstring_core & rhs) {
+ assert(&rhs != this);
+ // Simplest case first: small strings are bitblitted
+ if (rhs.category() == isSmall) {
+ assert(offsetof(MediumLarge, data_) == 0);
+ assert(offsetof(MediumLarge, size_) == sizeof(ml_.data_));
+ assert(offsetof(MediumLarge, capacity_) == 2 * sizeof(ml_.data_));
+ const size_t size = rhs.smallSize();
+ if (size == 0) {
+ ml_.capacity_ = rhs.ml_.capacity_;
+ writeTerminator();
+ } else {
+ // Just write the whole thing, don't look at details. In
+ // particular we need to copy capacity anyway because we want
+ // to set the size (don't forget that the last character,
+ // which stores a short string's length, is shared with the
+ // ml_.capacity field).
+ ml_ = rhs.ml_;
+ }
+ assert(category() == isSmall && this->size() == rhs.size());
+ } else if (rhs.category() == isLarge) {
+ // Large strings are just refcounted
+ ml_ = rhs.ml_;
+ RefCounted::incrementRefs(ml_.data_);
+ assert(category() == isLarge && size() == rhs.size());
+ } else {
+ // Medium strings are copied eagerly. Don't forget to allocate
+ // one extra Char for the null terminator.
+ auto const allocSize =
+ goodMallocSize((1 + rhs.ml_.size_) * sizeof(Char));
+ ml_.data_ = static_cast<Char*>(malloc(allocSize));
+ fbstring_detail::pod_copy(rhs.ml_.data_,
+ // 1 for terminator
+ rhs.ml_.data_ + rhs.ml_.size_ + 1,
+ ml_.data_);
+ // No need for writeTerminator() here, we copied one extra
+ // element just above.
+ ml_.size_ = rhs.ml_.size_;
+ ml_.capacity_ = (allocSize / sizeof(Char) - 1) | isMedium;
+ assert(category() == isMedium);
+ }
+ assert(size() == rhs.size());
+ assert(memcmp(data(), rhs.data(), size() * sizeof(Char)) == 0);
+ }
+
+ fbstring_core(fbstring_core&& goner) {
+ if (goner.category() == isSmall) {
+ // Just copy, leave the goner in peace
+ new(this) fbstring_core(goner.small_, goner.smallSize());
+ } else {
+ // Take goner's guts
+ ml_ = goner.ml_;
+ // Clean goner's carcass
+ goner.setSmallSize(0);
+ }
+ }
+
+ fbstring_core(const Char *const data, const size_t size) {
+ // Simplest case first: small strings are bitblitted
+ if (size <= maxSmallSize) {
+ // Layout is: Char* data_, size_t size_, size_t capacity_
+ /*static_*/assert(sizeof(*this) == sizeof(Char*) + 2 * sizeof(size_t));
+ /*static_*/assert(sizeof(Char*) == sizeof(size_t));
+ // sizeof(size_t) must be a power of 2
+ /*static_*/assert((sizeof(size_t) & (sizeof(size_t) - 1)) == 0);
+
+ // If data is aligned, use fast word-wise copying. Otherwise,
+ // use conservative memcpy.
+ if (reinterpret_cast<size_t>(data) & (sizeof(size_t) - 1)) {
+ fbstring_detail::pod_copy(data, data + size, small_);
+ } else {
+ // Copy one word (64 bits) at a time
+ const size_t byteSize = size * sizeof(Char);
+ if (byteSize > 2 * sizeof(size_t)) {
+ // Copy three words
+ ml_.capacity_ = reinterpret_cast<const size_t*>(data)[2];
+ copyTwo:
+ ml_.size_ = reinterpret_cast<const size_t*>(data)[1];
+ copyOne:
+ ml_.data_ = *reinterpret_cast<Char**>(const_cast<Char*>(data));
+ } else if (byteSize > sizeof(size_t)) {
+ // Copy two words
+ goto copyTwo;
+ } else if (size > 0) {
+ // Copy one word
+ goto copyOne;
+ }
+ }
+ setSmallSize(size);
+ } else if (size <= maxMediumSize) {
+ // Medium strings are allocated normally. Don't forget to
+ // allocate one extra Char for the terminating null.
+ auto const allocSize = goodMallocSize((1 + size) * sizeof(Char));
+ ml_.data_ = static_cast<Char*>(malloc(allocSize));
+ fbstring_detail::pod_copy(data, data + size, ml_.data_);
+ ml_.size_ = size;
+ ml_.capacity_ = (allocSize / sizeof(Char) - 1) | isMedium;
+ } else {
+ // Large strings are allocated differently
+ size_t effectiveCapacity = size;
+ auto const newRC = RefCounted::create(data, & effectiveCapacity);
+ ml_.data_ = newRC->data_;
+ ml_.size_ = size;
+ ml_.capacity_ = effectiveCapacity | isLarge;
+ }
+ writeTerminator();
+ assert(this->size() == size);
+ assert(memcmp(this->data(), data, size * sizeof(Char)) == 0);
+ }
+
+ ~fbstring_core() {
+ auto const c = category();
+ if (c == isSmall) {
+ return;
+ }
+ if (c == isMedium) {
+ free(ml_.data_);
+ return;
+ }
+ RefCounted::decrementRefs(ml_.data_);
+ }
+
+ // Snatches a previously mallocated string. The parameter "size"
+ // is the size of the string, and the parameter "capacity" is the size
+ // of the mallocated block. The string must be \0-terminated, so
+ // data[size] == '\0' and capacity >= size + 1.
+ //
+ // So if you want a 2-character string, pass malloc(3) as "data", pass 2 as
+ // "size", and pass 3 as "capacity".
+ fbstring_core(Char *const data, const size_t size,
+ const size_t capacity,
+ AcquireMallocatedString) {
+ if (size > 0) {
+ assert(capacity > size);
+ assert(data[size] == '\0');
+ // Use the medium string storage
+ ml_.data_ = data;
+ ml_.size_ = size;
+ ml_.capacity_ = capacity | isMedium;
+ } else {
+ // No need for the memory
+ free(data);
+ setSmallSize(0);
+ }
+ }
+
+ // swap below doesn't test whether &rhs == this (and instead
+ // potentially does extra work) on the premise that the rarity of
+ // that situation actually makes the check more expensive than is
+ // worth.
+ void swap(fbstring_core & rhs) {
+ auto const t = ml_;
+ ml_ = rhs.ml_;
+ rhs.ml_ = t;
+ }
+
+ // In C++11 data() and c_str() are 100% equivalent.
+ const Char * data() const {
+ return c_str();
+ }
+
+ Char * mutable_data() {
+ auto const c = category();
+ if (c == isSmall) {
+ return small_;
+ }
+ assert(c == isMedium || c == isLarge);
+ if (c == isLarge && RefCounted::refs(ml_.data_) > 1) {
+ // Ensure unique.
+ size_t effectiveCapacity = ml_.capacity();
+ auto const newRC = RefCounted::create(& effectiveCapacity);
+ // If this fails, someone placed the wrong capacity in an
+ // fbstring.
+ assert(effectiveCapacity >= ml_.capacity());
+ fbstring_detail::pod_copy(ml_.data_, ml_.data_ + ml_.size_ + 1,
+ newRC->data_);
+ RefCounted::decrementRefs(ml_.data_);
+ ml_.data_ = newRC->data_;
+ // No need to call writeTerminator(), we have + 1 above.
+ }
+ return ml_.data_;
+ }
+
+ const Char * c_str() const {
+ auto const c = category();
+#ifdef FBSTRING_PERVERSE
+ if (c == isSmall) {
+ assert(small_[smallSize()] == TERMINATOR || smallSize() == maxSmallSize
+ || small_[smallSize()] == '\0');
+ small_[smallSize()] = '\0';
+ return small_;
+ }
+ assert(c == isMedium || c == isLarge);
+ assert(ml_.data_[ml_.size_] == TERMINATOR || ml_.data_[ml_.size_] == '\0');
+ ml_.data_[ml_.size_] = '\0';
+#elif defined(FBSTRING_CONSERVATIVE)
+ if (c == isSmall) {
+ assert(small_[smallSize()] == '\0');
+ return small_;
+ }
+ assert(c == isMedium || c == isLarge);
+ assert(ml_.data_[ml_.size_] == '\0');
+#else
+ if (c == isSmall) {
+ small_[smallSize()] = '\0';
+ return small_;
+ }
+ assert(c == isMedium || c == isLarge);
+ ml_.data_[ml_.size_] = '\0';
+#endif
+ return ml_.data_;
+ }
+
+ void shrink(const size_t delta) {
+ if (category() == isSmall) {
+ // Check for underflow
+ assert(delta <= smallSize());
+ setSmallSize(smallSize() - delta);
+ } else if (category() == isMedium || RefCounted::refs(ml_.data_) == 1) {
+ // Medium strings and unique large strings need no special
+ // handling.
+ assert(ml_.size_ >= delta);
+ ml_.size_ -= delta;
+ } else {
+ assert(ml_.size_ >= delta);
+ // Shared large string, must make unique. This is because of the
+ // durn terminator must be written, which may trample the shared
+ // data.
+ if (delta) {
+ fbstring_core(ml_.data_, ml_.size_ - delta).swap(*this);
+ }
+ // No need to write the terminator.
+ return;
+ }
+ writeTerminator();
+ }
+
+ void reserve(size_t minCapacity) {
+ if (category() == isLarge) {
+ // Ensure unique
+ if (RefCounted::refs(ml_.data_) > 1) {
+ // We must make it unique regardless; in-place reallocation is
+ // useless if the string is shared. In order to not surprise
+ // people, reserve the new block at current capacity or
+ // more. That way, a string's capacity never shrinks after a
+ // call to reserve.
+ minCapacity = std::max(minCapacity, ml_.capacity());
+ auto const newRC = RefCounted::create(& minCapacity);
+ fbstring_detail::pod_copy(ml_.data_, ml_.data_ + ml_.size_ + 1,
+ newRC->data_);
+ // Done with the old data. No need to call writeTerminator(),
+ // we have + 1 above.
+ RefCounted::decrementRefs(ml_.data_);
+ ml_.data_ = newRC->data_;
+ ml_.capacity_ = minCapacity | isLarge;
+ // size remains unchanged
+ } else {
+ // String is not shared, so let's try to realloc (if needed)
+ if (minCapacity > ml_.capacity()) {
+ // Asking for more memory
+ auto const newRC =
+ RefCounted::reallocate(ml_.data_, ml_.size_,
+ ml_.capacity(), minCapacity);
+ ml_.data_ = newRC->data_;
+ ml_.capacity_ = minCapacity | isLarge;
+ writeTerminator();
+ }
+ assert(capacity() >= minCapacity);
+ }
+ } else if (category() == isMedium) {
+ // String is not shared
+ if (minCapacity <= ml_.capacity()) {
+ return; // nothing to do, there's enough room
+ }
+ if (minCapacity <= maxMediumSize) {
+ // Keep the string at medium size. Don't forget to allocate
+ // one extra Char for the terminating null.
+ size_t capacityBytes = goodMallocSize((1 + minCapacity) * sizeof(Char));
+ ml_.data_ = static_cast<Char *>(
+ smartRealloc(
+ ml_.data_,
+ ml_.size_ * sizeof(Char),
+ ml_.capacity() * sizeof(Char),
+ capacityBytes));
+ writeTerminator();
+ ml_.capacity_ = (capacityBytes / sizeof(Char) - 1) | isMedium;
+ } else {
+ // Conversion from medium to large string
+ fbstring_core nascent;
+ // Will recurse to another branch of this function
+ nascent.reserve(minCapacity);
+ nascent.ml_.size_ = ml_.size_;
+ fbstring_detail::pod_copy(ml_.data_, ml_.data_ + ml_.size_,
+ nascent.ml_.data_);
+ nascent.swap(*this);
+ writeTerminator();
+ assert(capacity() >= minCapacity);
+ }
+ } else {
+ assert(category() == isSmall);
+ if (minCapacity > maxMediumSize) {
+ // large
+ auto const newRC = RefCounted::create(& minCapacity);
+ auto const size = smallSize();
+ fbstring_detail::pod_copy(small_, small_ + size + 1, newRC->data_);
+ // No need for writeTerminator(), we wrote it above with + 1.
+ ml_.data_ = newRC->data_;
+ ml_.size_ = size;
+ ml_.capacity_ = minCapacity | isLarge;
+ assert(capacity() >= minCapacity);
+ } else if (minCapacity > maxSmallSize) {
+ // medium
+ // Don't forget to allocate one extra Char for the terminating null
+ auto const allocSizeBytes =
+ goodMallocSize((1 + minCapacity) * sizeof(Char));
+ auto const data = static_cast<Char*>(malloc(allocSizeBytes));
+ auto const size = smallSize();
+ fbstring_detail::pod_copy(small_, small_ + size + 1, data);
+ // No need for writeTerminator(), we wrote it above with + 1.
+ ml_.data_ = data;
+ ml_.size_ = size;
+ ml_.capacity_ = (allocSizeBytes / sizeof(Char) - 1) | isMedium;
+ } else {
+ // small
+ // Nothing to do, everything stays put
+ }
+ }
+ assert(capacity() >= minCapacity);
+ }
+
+ void expand_noinit(const size_t delta) {
+ // Strategy is simple: make room, then change size
+ assert(capacity() >= size());
+ size_t sz, newSz, cp;
+ if (category() == isSmall) {
+ sz = smallSize();
+ newSz = sz + delta;
+ if (newSz <= maxSmallSize) {
+ setSmallSize(newSz);
+ writeTerminator();
+ return;
+ }
+ cp = maxSmallSize;
+ } else {
+ sz = ml_.size_;
+ newSz = sz + delta;
+ cp = capacity();
+ }
+ if (newSz > cp) reserve(newSz);
+ assert(capacity() >= newSz);
+ // Category can't be small - we took care of that above
+ assert(category() == isMedium || category() == isLarge);
+ ml_.size_ = newSz;
+ writeTerminator();
+ assert(size() == newSz);
+ }
+
+ void push_back(Char c) {
+ assert(capacity() >= size());
+ size_t sz, cp;
+ if (category() == isSmall) {
+ sz = smallSize();
+ if (sz < maxSmallSize) {
+ setSmallSize(sz + 1);
+ small_[sz] = c;
+ writeTerminator();
+ return;
+ }
+ reserve(maxSmallSize * 3 / 2);
+ } else {
+ sz = ml_.size_;
+ cp = ml_.capacity();
+ if (sz == cp) reserve(cp * 3 / 2);
+ }
+ assert(capacity() >= sz + 1);
+ // Category can't be small - we took care of that above
+ assert(category() == isMedium || category() == isLarge);
+ ml_.size_ = sz + 1;
+ mutable_data()[sz] = c;
+ writeTerminator();
+ }
+
+ size_t size() const {
+ return category() == isSmall ? smallSize() : ml_.size_;
+ }
+
+ size_t capacity() const {
+ switch (category()) {
+ case isSmall:
+ return maxSmallSize;
+ case isLarge:
+ // For large-sized strings, a multi-referenced chunk has no
+ // available capacity. This is because any attempt to append
+ // data would trigger a new allocation.
+ if (RefCounted::refs(ml_.data_) > 1) return ml_.size_;
+ default: {}
+ }
+ return ml_.capacity();
+ }
+
+ bool isShared() const {
+ return category() == isLarge && RefCounted::refs(ml_.data_) > 1;
+ }
+
+#ifdef FBSTRING_PERVERSE
+ enum { TERMINATOR = '^' };
+#else
+ enum { TERMINATOR = '\0' };
+#endif
+
+ void writeTerminator() {
+#if defined(FBSTRING_PERVERSE) || defined(FBSTRING_CONSERVATIVE)
+ if (category() == isSmall) {
+ const auto s = smallSize();
+ if (s != maxSmallSize) {
+ small_[s] = TERMINATOR;
+ }
+ } else {
+ ml_.data_[ml_.size_] = TERMINATOR;
+ }
+#endif
+ }
+
+private:
+ // Disabled
+ fbstring_core & operator=(const fbstring_core & rhs);
+
+ struct MediumLarge {
+ Char * data_;
+ size_t size_;
+ size_t capacity_;
+
+ size_t capacity() const {
+ return capacity_ & capacityExtractMask;
+ }
+ };
+
+ struct RefCounted {
+ std::atomic<size_t> refCount_;
+ Char data_[1];
+
+ static RefCounted * fromData(Char * p) {
+ return static_cast<RefCounted*>(
+ static_cast<void*>(
+ static_cast<unsigned char*>(static_cast<void*>(p))
+ - offsetof(RefCounted, data_)));
+ }
+
+ static size_t refs(Char * p) {
+ return fromData(p)->refCount_.load(std::memory_order_acquire);
+ }
+
+ static void incrementRefs(Char * p) {
+ fromData(p)->refCount_.fetch_add(1, std::memory_order_acq_rel);
+ }
+
+ static void decrementRefs(Char * p) {
+ auto const dis = fromData(p);
+ size_t oldcnt = dis->refCount_.fetch_sub(1, std::memory_order_acq_rel);
+ assert(oldcnt > 0);
+ if (oldcnt == 1) {
+ free(dis);
+ }
+ }
+
+ static RefCounted * create(size_t * size) {
+ // Don't forget to allocate one extra Char for the terminating
+ // null. In this case, however, one Char is already part of the
+ // struct.
+ const size_t allocSize = goodMallocSize(
+ sizeof(RefCounted) + *size * sizeof(Char));
+ auto result = static_cast<RefCounted*>(malloc(allocSize));
+ result->refCount_.store(1, std::memory_order_release);
+ *size = (allocSize - sizeof(RefCounted)) / sizeof(Char);
+ return result;
+ }
+
+ static RefCounted * create(const Char * data, size_t * size) {
+ const size_t effectiveSize = *size;
+ auto result = create(size);
+ fbstring_detail::pod_copy(data, data + effectiveSize, result->data_);
+ return result;
+ }
+
+ static RefCounted * reallocate(Char *const data,
+ const size_t currentSize,
+ const size_t currentCapacity,
+ const size_t newCapacity) {
+ assert(newCapacity > 0 && newCapacity > currentSize);
+ auto const dis = fromData(data);
+ assert(dis->refCount_.load(std::memory_order_acquire) == 1);
+ // Don't forget to allocate one extra Char for the terminating
+ // null. In this case, however, one Char is already part of the
+ // struct.
+ auto result = static_cast<RefCounted*>(
+ smartRealloc(dis,
+ sizeof(RefCounted) + currentSize * sizeof(Char),
+ sizeof(RefCounted) + currentCapacity * sizeof(Char),
+ sizeof(RefCounted) + newCapacity * sizeof(Char)));
+ assert(result->refCount_.load(std::memory_order_acquire) == 1);
+ return result;
+ }
+ };
+
+ union {
+ mutable Char small_[sizeof(MediumLarge) / sizeof(Char)];
+ mutable MediumLarge ml_;
+ };
+
+ enum {
+ lastChar = sizeof(MediumLarge) - 1,
+ maxSmallSize = lastChar / sizeof(Char),
+ maxMediumSize = 254 / sizeof(Char), // coincides with the small
+ // bin size in dlmalloc
+ categoryExtractMask = sizeof(size_t) == 4 ? 0xC0000000 : 0xC000000000000000,
+ capacityExtractMask = ~categoryExtractMask,
+ };
+ static_assert(!(sizeof(MediumLarge) % sizeof(Char)),
+ "Corrupt memory layout for fbstring.");
+
+ enum Category {
+ isSmall = 0,
+ isMedium = sizeof(size_t) == 4 ? 0x80000000 : 0x8000000000000000,
+ isLarge = sizeof(size_t) == 4 ? 0x40000000 : 0x4000000000000000,
+ };
+
+ Category category() const {
+ // Assumes little endian
+ return static_cast<Category>(ml_.capacity_ & categoryExtractMask);
+ }
+
+ size_t smallSize() const {
+ assert(category() == isSmall && small_[maxSmallSize] <= maxSmallSize);
+ return static_cast<size_t>(maxSmallSize)
+ - static_cast<size_t>(small_[maxSmallSize]);
+ }
+
+ void setSmallSize(size_t s) {
+ // Warning: this should work with uninitialized strings too,
+ // so don't assume anything about the previous value of
+ // small_[maxSmallSize].
+ assert(s <= maxSmallSize);
+ small_[maxSmallSize] = maxSmallSize - s;
+ }
+};
+
+#ifndef _LIBSTDCXX_FBSTRING
+/**
+ * Dummy fbstring core that uses an actual std::string. This doesn't
+ * make any sense - it's just for testing purposes.
+ */
+template <class Char>
+class dummy_fbstring_core {
+public:
+ dummy_fbstring_core() {
+ }
+ dummy_fbstring_core(const dummy_fbstring_core& another)
+ : backend_(another.backend_) {
+ }
+ dummy_fbstring_core(const Char * s, size_t n)
+ : backend_(s, n) {
+ }
+ void swap(dummy_fbstring_core & rhs) {
+ backend_.swap(rhs.backend_);
+ }
+ const Char * data() const {
+ return backend_.data();
+ }
+ Char * mutable_data() {
+ //assert(!backend_.empty());
+ return &*backend_.begin();
+ }
+ void shrink(size_t delta) {
+ assert(delta <= size());
+ backend_.resize(size() - delta);
+ }
+ void expand_noinit(size_t delta) {
+ backend_.resize(size() + delta);
+ }
+ void push_back(Char c) {
+ backend_.push_back(c);
+ }
+ size_t size() const {
+ return backend_.size();
+ }
+ size_t capacity() const {
+ return backend_.capacity();
+ }
+ bool isShared() const {
+ return false;
+ }
+ void reserve(size_t minCapacity) {
+ backend_.reserve(minCapacity);
+ }
+
+private:
+ std::basic_string<Char> backend_;
+};
+#endif // !_LIBSTDCXX_FBSTRING
+
+/**
+ * This is the basic_string replacement. For conformity,
+ * basic_fbstring takes the same template parameters, plus the last
+ * one which is the core.
+ */
+#ifdef _LIBSTDCXX_FBSTRING
+template <typename E, class T, class A, class Storage>
+#else
+template <typename E,
+ class T = std::char_traits<E>,
+ class A = std::allocator<E>,
+ class Storage = fbstring_core<E> >
+#endif
+class basic_fbstring {
+
+ static void enforce(
+ bool condition,
+ void (*throw_exc)(const char*),
+ const char* msg) {
+ if (!condition) throw_exc(msg);
+ }
+
+ bool isSane() const {
+ return
+ begin() <= end() &&
+ empty() == (size() == 0) &&
+ empty() == (begin() == end()) &&
+ size() <= max_size() &&
+ capacity() <= max_size() &&
+ size() <= capacity() &&
+ (begin()[size()] == Storage::TERMINATOR || begin()[size()] == '\0');
+ }
+
+ struct Invariant;
+ friend struct Invariant;
+ struct Invariant {
+#ifndef NDEBUG
+ explicit Invariant(const basic_fbstring& s) : s_(s) {
+ assert(s_.isSane());
+ }
+ ~Invariant() {
+ assert(s_.isSane());
+ }
+ private:
+ const basic_fbstring& s_;
+#else
+ explicit Invariant(const basic_fbstring&) {}
+#endif
+ Invariant& operator=(const Invariant&);
+ };
+
+public:
+ // types
+ typedef T traits_type;
+ typedef typename traits_type::char_type value_type;
+ typedef A allocator_type;
+ typedef typename A::size_type size_type;
+ typedef typename A::difference_type difference_type;
+
+ typedef typename A::reference reference;
+ typedef typename A::const_reference const_reference;
+ typedef typename A::pointer pointer;
+ typedef typename A::const_pointer const_pointer;
+
+ typedef E* iterator;
+ typedef const E* const_iterator;
+ typedef std::reverse_iterator<iterator
+#ifdef NO_ITERATOR_TRAITS
+ , value_type
+#endif
+ > reverse_iterator;
+ typedef std::reverse_iterator<const_iterator
+#ifdef NO_ITERATOR_TRAITS
+ , const value_type
+#endif
+ > const_reverse_iterator;
+
+ static const size_type npos; // = size_type(-1)
+
+private:
+ static void procrustes(size_type& n, size_type nmax) {
+ if (n > nmax) n = nmax;
+ }
+
+public:
+ // 21.3.1 construct/copy/destroy
+ explicit basic_fbstring(const A& a = A()) {
+ }
+
+ basic_fbstring(const basic_fbstring& str)
+ : store_(str.store_) {
+ }
+
+ // Move constructor
+ basic_fbstring(basic_fbstring&& goner) : store_(std::move(goner.store_)) {
+ }
+
+#ifndef _LIBSTDCXX_FBSTRING
+ // This is defined for compatibility with std::string
+ /* implicit */ basic_fbstring(const std::string& str)
+ : store_(str.data(), str.size()) {
+ }
+#endif
+
+ basic_fbstring(const basic_fbstring& str, size_type pos,
+ size_type n = npos, const A& a = A()) {
+ assign(str, pos, n);
+ }
+
+ /* implicit */ basic_fbstring(const value_type* s, const A& a = A())
+ : store_(s, s ? traits_type::length(s) : ({
+ basic_fbstring<char> err = __PRETTY_FUNCTION__;
+ err += ": null pointer initializer not valid";
+ std::__throw_logic_error(err.c_str());
+ 0;
+ })) {
+ }
+
+ basic_fbstring(const value_type* s, size_type n, const A& a = A())
+ : store_(s, n) {
+ }
+
+ basic_fbstring(size_type n, value_type c, const A& a = A()) {
+ store_.expand_noinit(n);
+ auto const data = store_.mutable_data();
+ fbstring_detail::pod_fill(data, data + n, c);
+ store_.writeTerminator();
+ }
+
+ template <class InIt>
+ basic_fbstring(InIt begin, InIt end,
+ typename std::enable_if<
+ !std::is_same<typename std::remove_const<InIt>::type,
+ value_type*>::value, const A>::type & a = A()) {
+ assign(begin, end);
+ }
+
+ // Specialization for const char*, const char*
+ basic_fbstring(const value_type* b, const value_type* e)
+ : store_(b, e - b) {
+ }
+
+ // Nonstandard constructor
+ basic_fbstring(value_type *s, size_type n, size_type c,
+ AcquireMallocatedString a)
+ : store_(s, n, c, a) {
+ }
+
+ ~basic_fbstring() {
+ }
+
+ basic_fbstring& operator=(const basic_fbstring & lhs) {
+ if (&lhs == this) {
+ return *this;
+ }
+ auto const oldSize = size();
+ auto const srcSize = lhs.size();
+ if (capacity() >= srcSize && !store_.isShared()) {
+ // great, just copy the contents
+ if (oldSize < srcSize)
+ store_.expand_noinit(srcSize - oldSize);
+ else
+ store_.shrink(oldSize - srcSize);
+ assert(size() == srcSize);
+ fbstring_detail::pod_copy(lhs.begin(), lhs.end(), begin());
+ store_.writeTerminator();
+ } else {
+ // need to reallocate, so we may as well create a brand new string
+ basic_fbstring(lhs).swap(*this);
+ }
+ return *this;
+ }
+
+ // Move assignment
+ basic_fbstring& operator=(basic_fbstring&& goner) {
+ // No need of this anymore
+ this->~basic_fbstring();
+ // Move the goner into this
+ new(&store_) fbstring_core<E>(std::move(goner.store_));
+ return *this;
+ }
+
+#ifndef _LIBSTDCXX_FBSTRING
+ // Compatibility with std::string
+ basic_fbstring & operator=(const std::string & rhs) {
+ return assign(rhs.data(), rhs.size());
+ }
+
+ // Compatibility with std::string
+ std::string toStdString() const {
+ return std::string(data(), size());
+ }
+#else
+ // A lot of code in fbcode still uses this method, so keep it here for now.
+ const basic_fbstring& toStdString() const {
+ return *this;
+ }
+#endif
+
+ basic_fbstring& operator=(const value_type* s) {
+ return assign(s);
+ }
+
+ basic_fbstring& operator=(value_type c) {
+ if (empty()) {
+ store_.expand_noinit(1);
+ } else if (store_.isShared()) {
+ basic_fbstring(1, c).swap(*this);
+ return *this;
+ } else {
+ store_.shrink(size() - 1);
+ }
+ *store_.mutable_data() = c;
+ store_.writeTerminator();
+ return *this;
+ }
+
+ // 21.3.2 iterators:
+ iterator begin() { return store_.mutable_data(); }
+
+ const_iterator begin() const { return store_.data(); }
+
+ iterator end() {
+ return store_.mutable_data() + store_.size();
+ }
+
+ const_iterator end() const {
+ return store_.data() + store_.size();
+ }
+
+ reverse_iterator rbegin() {
+ return reverse_iterator(end());
+ }
+
+ const_reverse_iterator rbegin() const {
+ return const_reverse_iterator(end());
+ }
+
+ reverse_iterator rend() {
+ return reverse_iterator(begin());
+ }
+
+ const_reverse_iterator rend() const {
+ return const_reverse_iterator(begin());
+ }
+
+ // Non-standard functions. They intentionally return by value to
+ // reduce pressure on the reference counting mechanism.
+ value_type front() const { return *begin(); }
+ value_type back() const {
+ assert(!empty());
+ return begin()[size() - 1];
+ }
+ void pop_back() { assert(!empty()); store_.shrink(1); }
+
+ // 21.3.3 capacity:
+ size_type size() const { return store_.size(); }
+
+ size_type length() const { return size(); }
+
+ size_type max_size() const {
+ return std::numeric_limits<size_type>::max();
+ }
+
+ void resize(const size_type n, const value_type c = value_type()) {
+ auto size = this->size();
+ if (n <= size) {
+ store_.shrink(size - n);
+ } else {
+ // Do this in two steps to minimize slack memory copied (see
+ // smartRealloc).
+ auto const capacity = this->capacity();
+ assert(capacity >= size);
+ if (size < capacity) {
+ auto delta = std::min(n, capacity) - size;
+ store_.expand_noinit(delta);
+ fbstring_detail::pod_fill(begin() + size, end(), c);
+ size += delta;
+ if (size == n) {
+ store_.writeTerminator();
+ return;
+ }
+ assert(size < n);
+ }
+ auto const delta = n - size;
+ store_.expand_noinit(delta);
+ fbstring_detail::pod_fill(end() - delta, end(), c);
+ store_.writeTerminator();
+ }
+ assert(this->size() == n);
+ }
+
+ size_type capacity() const { return store_.capacity(); }
+
+ void reserve(size_type res_arg = 0) {
+ enforce(res_arg <= max_size(), std::__throw_length_error, "");
+ store_.reserve(res_arg);
+ }
+
+ void clear() { resize(0); }
+
+ bool empty() const { return size() == 0; }
+
+ // 21.3.4 element access:
+ const_reference operator[](size_type pos) const {
+ return *(c_str() + pos);
+ }
+
+ reference operator[](size_type pos) {
+ if (pos == size()) {
+ // Just call c_str() to make sure '\0' is present
+ c_str();
+ }
+ return *(begin() + pos);
+ }
+
+ const_reference at(size_type n) const {
+ enforce(n <= size(), std::__throw_out_of_range, "");
+ return (*this)[n];
+ }
+
+ reference at(size_type n) {
+ enforce(n < size(), std::__throw_out_of_range, "");
+ return (*this)[n];
+ }
+
+ // 21.3.5 modifiers:
+ basic_fbstring& operator+=(const basic_fbstring& str) {
+ return append(str);
+ }
+
+ basic_fbstring& operator+=(const value_type* s) {
+ return append(s);
+ }
+
+ basic_fbstring& operator+=(const value_type c) {
+ push_back(c);
+ return *this;
+ }
+
+ basic_fbstring& append(const basic_fbstring& str) {
+#ifndef NDEBUG
+ auto desiredSize = size() + str.size();
+#endif
+ append(str.data(), str.size());
+ assert(size() == desiredSize);
+ return *this;
+ }
+
+ basic_fbstring& append(const basic_fbstring& str, const size_type pos,
+ size_type n) {
+ const size_type sz = str.size();
+ enforce(pos <= sz, std::__throw_out_of_range, "");
+ procrustes(n, sz - pos);
+ return append(str.data() + pos, n);
+ }
+
+ basic_fbstring& append(const value_type* s, const size_type n) {
+#ifndef NDEBUG
+ auto oldSize = size();
+#endif
+ Invariant checker(*this);
+ (void) checker;
+ static std::less_equal<const value_type*> le;
+ if (le(data(), s) && !le(data() + size(), s)) {// aliasing
+ assert(le(s + n, data() + size()));
+ const size_type offset = s - data();
+ store_.reserve(size() + n);
+ // Restore the source
+ s = data() + offset;
+ }
+ store_.expand_noinit(n);
+ fbstring_detail::pod_copy(s, s + n, end() - n);
+ store_.writeTerminator();
+ assert(size() == oldSize + n);
+ return *this;
+ }
+
+ basic_fbstring& append(const value_type* s) {
+ return append(s, traits_type::length(s));
+ }
+
+ basic_fbstring& append(size_type n, value_type c) {
+ resize(size() + n, c);
+ return *this;
+ }
+
+ template<class InputIterator>
+ basic_fbstring& append(InputIterator first, InputIterator last) {
+ insert(end(), first, last);
+ return *this;
+ }
+
+ void push_back(const value_type c) { // primitive
+ store_.push_back(c);
+ }
+
+ basic_fbstring& assign(const basic_fbstring& str) {
+ if (&str == this) return *this;
+ return assign(str.data(), str.size());
+ }
+
+ basic_fbstring& assign(const basic_fbstring& str, const size_type pos,
+ size_type n) {
+ const size_type sz = str.size();
+ enforce(pos <= sz, std::__throw_out_of_range, "");
+ procrustes(n, sz - pos);
+ return assign(str.data() + pos, n);
+ }
+
+ basic_fbstring& assign(const value_type* s, const size_type n) {
+ Invariant checker(*this);
+ (void) checker;
+ if (size() >= n) {
+ std::copy(s, s + n, begin());
+ resize(n);
+ assert(size() == n);
+ } else {
+ const value_type *const s2 = s + size();
+ std::copy(s, s2, begin());
+ append(s2, n - size());
+ assert(size() == n);
+ }
+ store_.writeTerminator();
+ assert(size() == n);
+ return *this;
+ }
+
+ basic_fbstring& assign(const value_type* s) {
+ return assign(s, traits_type::length(s));
+ }
+
+ template <class ItOrLength, class ItOrChar>
+ basic_fbstring& assign(ItOrLength first_or_n, ItOrChar last_or_c) {
+ return replace(begin(), end(), first_or_n, last_or_c);
+ }
+
+ basic_fbstring& insert(size_type pos1, const basic_fbstring& str) {
+ return insert(pos1, str.data(), str.size());
+ }
+
+ basic_fbstring& insert(size_type pos1, const basic_fbstring& str,
+ size_type pos2, size_type n) {
+ enforce(pos2 <= str.length(), std::__throw_out_of_range, "");
+ procrustes(n, str.length() - pos2);
+ return insert(pos1, str.data() + pos2, n);
+ }
+
+ basic_fbstring& insert(size_type pos, const value_type* s, size_type n) {
+ enforce(pos <= length(), std::__throw_out_of_range, "");
+ insert(begin() + pos, s, s + n);
+ return *this;
+ }
+
+ basic_fbstring& insert(size_type pos, const value_type* s) {
+ return insert(pos, s, traits_type::length(s));
+ }
+
+ basic_fbstring& insert(size_type pos, size_type n, value_type c) {
+ enforce(pos <= length(), std::__throw_out_of_range, "");
+ insert(begin() + pos, n, c);
+ return *this;
+ }
+
+ iterator insert(const iterator p, const value_type c) {
+ const size_type pos = p - begin();
+ insert(p, 1, c);
+ return begin() + pos;
+ }
+
+private:
+ template <int i> class Selector {};
+
+ basic_fbstring& insertImplDiscr(iterator p,
+ size_type n, value_type c, Selector<1>) {
+ Invariant checker(*this);
+ (void) checker;
+ assert(p >= begin() && p <= end());
+ if (capacity() - size() < n) {
+ const size_type sz = p - begin();
+ reserve(size() + n);
+ p = begin() + sz;
+ }
+ const iterator oldEnd = end();
+ if( n < size_type(oldEnd - p)) {
+ append(oldEnd - n, oldEnd);
+ //std::copy(
+ // reverse_iterator(oldEnd - n),
+ // reverse_iterator(p),
+ // reverse_iterator(oldEnd));
+ fbstring_detail::pod_move(&*p, &*oldEnd - n, &*p + n);
+ std::fill(p, p + n, c);
+ } else {
+ append(n - (end() - p), c);
+ append(p, oldEnd);
+ std::fill(p, oldEnd, c);
+ }
+ store_.writeTerminator();
+ return *this;
+ }
+
+ template<class InputIter>
+ basic_fbstring& insertImplDiscr(iterator i,
+ InputIter b, InputIter e, Selector<0>) {
+ insertImpl(i, b, e,
+ typename std::iterator_traits<InputIter>::iterator_category());
+ return *this;
+ }
+
+ template <class FwdIterator>
+ void insertImpl(iterator i,
+ FwdIterator s1, FwdIterator s2, std::forward_iterator_tag) {
+ Invariant checker(*this);
+ (void) checker;
+ const size_type pos = i - begin();
+ const typename std::iterator_traits<FwdIterator>::difference_type n2 =
+ std::distance(s1, s2);
+ assert(n2 >= 0);
+ using namespace fbstring_detail;
+ assert(pos <= size());
+
+ const typename std::iterator_traits<FwdIterator>::difference_type maxn2 =
+ capacity() - size();
+ if (maxn2 < n2) {
+ // realloc the string
+ reserve(size() + n2);
+ i = begin() + pos;
+ }
+ if (pos + n2 <= size()) {
+ const iterator tailBegin = end() - n2;
+ store_.expand_noinit(n2);
+ fbstring_detail::pod_copy(tailBegin, tailBegin + n2, end() - n2);
+ std::copy(reverse_iterator(tailBegin), reverse_iterator(i),
+ reverse_iterator(tailBegin + n2));
+ std::copy(s1, s2, i);
+ } else {
+ FwdIterator t = s1;
+ const size_type old_size = size();
+ std::advance(t, old_size - pos);
+ const size_t newElems = std::distance(t, s2);
+ store_.expand_noinit(n2);
+ std::copy(t, s2, begin() + old_size);
+ fbstring_detail::pod_copy(data() + pos, data() + old_size,
+ begin() + old_size + newElems);
+ std::copy(s1, t, i);
+ }
+ store_.writeTerminator();
+ }
+
+ template <class InputIterator>
+ void insertImpl(iterator i,
+ InputIterator b, InputIterator e, std::input_iterator_tag) {
+ basic_fbstring temp(begin(), i);
+ for (; b != e; ++b) {
+ temp.push_back(*b);
+ }
+ temp.append(i, end());
+ swap(temp);
+ }
+
+public:
+ template <class ItOrLength, class ItOrChar>
+ void insert(iterator p, ItOrLength first_or_n, ItOrChar last_or_c) {
+ Selector<std::numeric_limits<ItOrLength>::is_specialized> sel;
+ insertImplDiscr(p, first_or_n, last_or_c, sel);
+ }
+
+ basic_fbstring& erase(size_type pos = 0, size_type n = npos) {
+ Invariant checker(*this);
+ (void) checker;
+ enforce(pos <= length(), std::__throw_out_of_range, "");
+ procrustes(n, length() - pos);
+ std::copy(begin() + pos + n, end(), begin() + pos);
+ resize(length() - n);
+ return *this;
+ }
+
+ iterator erase(iterator position) {
+ const size_type pos(position - begin());
+ enforce(pos <= size(), std::__throw_out_of_range, "");
+ erase(pos, 1);
+ return begin() + pos;
+ }
+
+ iterator erase(iterator first, iterator last) {
+ const size_type pos(first - begin());
+ erase(pos, last - first);
+ return begin() + pos;
+ }
+
+ // Replaces at most n1 chars of *this, starting with pos1 with the
+ // content of str
+ basic_fbstring& replace(size_type pos1, size_type n1,
+ const basic_fbstring& str) {
+ return replace(pos1, n1, str.data(), str.size());
+ }
+
+ // Replaces at most n1 chars of *this, starting with pos1,
+ // with at most n2 chars of str starting with pos2
+ basic_fbstring& replace(size_type pos1, size_type n1,
+ const basic_fbstring& str,
+ size_type pos2, size_type n2) {
+ enforce(pos2 <= str.length(), std::__throw_out_of_range, "");
+ return replace(pos1, n1, str.data() + pos2,
+ std::min(n2, str.size() - pos2));
+ }
+
+ // Replaces at most n1 chars of *this, starting with pos, with chars from s
+ basic_fbstring& replace(size_type pos, size_type n1, const value_type* s) {
+ return replace(pos, n1, s, traits_type::length(s));
+ }
+
+ // Replaces at most n1 chars of *this, starting with pos, with n2
+ // occurences of c
+ //
+ // consolidated with
+ //
+ // Replaces at most n1 chars of *this, starting with pos, with at
+ // most n2 chars of str. str must have at least n2 chars.
+ template <class StrOrLength, class NumOrChar>
+ basic_fbstring& replace(size_type pos, size_type n1,
+ StrOrLength s_or_n2, NumOrChar n_or_c) {
+ Invariant checker(*this);
+ (void) checker;
+ enforce(pos <= size(), std::__throw_out_of_range, "");
+ procrustes(n1, length() - pos);
+ const iterator b = begin() + pos;
+ return replace(b, b + n1, s_or_n2, n_or_c);
+ }
+
+ basic_fbstring& replace(iterator i1, iterator i2, const basic_fbstring& str) {
+ return replace(i1, i2, str.data(), str.length());
+ }
+
+ basic_fbstring& replace(iterator i1, iterator i2, const value_type* s) {
+ return replace(i1, i2, s, traits_type::length(s));
+ }
+
+private:
+ basic_fbstring& replaceImplDiscr(iterator i1, iterator i2,
+ const value_type* s, size_type n,
+ Selector<2>) {
+ assert(i1 <= i2);
+ assert(begin() <= i1 && i1 <= end());
+ assert(begin() <= i2 && i2 <= end());
+ return replace(i1, i2, s, s + n);
+ }
+
+ basic_fbstring& replaceImplDiscr(iterator i1, iterator i2,
+ size_type n2, value_type c, Selector<1>) {
+ const size_type n1 = i2 - i1;
+ if (n1 > n2) {
+ std::fill(i1, i1 + n2, c);
+ erase(i1 + n2, i2);
+ } else {
+ std::fill(i1, i2, c);
+ insert(i2, n2 - n1, c);
+ }
+ assert(isSane());
+ return *this;
+ }
+
+ template <class InputIter>
+ basic_fbstring& replaceImplDiscr(iterator i1, iterator i2,
+ InputIter b, InputIter e,
+ Selector<0>) {
+ replaceImpl(i1, i2, b, e,
+ typename std::iterator_traits<InputIter>::iterator_category());
+ return *this;
+ }
+
+private:
+ template <class FwdIterator, class P>
+ bool replaceAliased(iterator i1, iterator i2,
+ FwdIterator s1, FwdIterator s2, P*) {
+ return false;
+ }
+
+ template <class FwdIterator>
+ bool replaceAliased(iterator i1, iterator i2,
+ FwdIterator s1, FwdIterator s2, value_type*) {
+ static const std::less_equal<const value_type*> le =
+ std::less_equal<const value_type*>();
+ const bool aliased = le(&*begin(), &*s1) && le(&*s1, &*end());
+ if (!aliased) {
+ return false;
+ }
+ // Aliased replace, copy to new string
+ basic_fbstring temp;
+ temp.reserve(size() - (i2 - i1) + std::distance(s1, s2));
+ temp.append(begin(), i1).append(s1, s2).append(i2, end());
+ swap(temp);
+ return true;
+ }
+
+public:
+ template <class FwdIterator>
+ void replaceImpl(iterator i1, iterator i2,
+ FwdIterator s1, FwdIterator s2, std::forward_iterator_tag) {
+ Invariant checker(*this);
+ (void) checker;
+
+ // Handle aliased replace
+ if (replaceAliased(i1, i2, s1, s2, &*s1)) {
+ return;
+ }
+
+ auto const n1 = i2 - i1;
+ assert(n1 >= 0);
+ auto const n2 = std::distance(s1, s2);
+ assert(n2 >= 0);
+
+ if (n1 > n2) {
+ // shrinks
+ std::copy(s1, s2, i1);
+ erase(i1 + n2, i2);
+ } else {
+ // grows
+ fbstring_detail::copy_n(s1, n1, i1);
+ std::advance(s1, n1);
+ insert(i2, s1, s2);
+ }
+ assert(isSane());
+ }
+
+ template <class InputIterator>
+ void replaceImpl(iterator i1, iterator i2,
+ InputIterator b, InputIterator e, std::input_iterator_tag) {
+ basic_fbstring temp(begin(), i1);
+ temp.append(b, e).append(i2, end());
+ swap(temp);
+ }
+
+public:
+ template <class T1, class T2>
+ basic_fbstring& replace(iterator i1, iterator i2,
+ T1 first_or_n_or_s, T2 last_or_c_or_n) {
+ const bool
+ num1 = std::numeric_limits<T1>::is_specialized,
+ num2 = std::numeric_limits<T2>::is_specialized;
+ return replaceImplDiscr(
+ i1, i2, first_or_n_or_s, last_or_c_or_n,
+ Selector<num1 ? (num2 ? 1 : -1) : (num2 ? 2 : 0)>());
+ }
+
+ size_type copy(value_type* s, size_type n, size_type pos = 0) const {
+ enforce(pos <= size(), std::__throw_out_of_range, "");
+ procrustes(n, size() - pos);
+
+ fbstring_detail::pod_copy(
+ data() + pos,
+ data() + pos + n,
+ s);
+ return n;
+ }
+
+ void swap(basic_fbstring& rhs) {
+ store_.swap(rhs.store_);
+ }
+
+ // 21.3.6 string operations:
+ const value_type* c_str() const {
+ return store_.c_str();
+ }
+
+ const value_type* data() const { return c_str(); }
+
+ allocator_type get_allocator() const {
+ return allocator_type();
+ }
+
+ size_type find(const basic_fbstring& str, size_type pos = 0) const {
+ return find(str.data(), pos, str.length());
+ }
+
+ size_type find(const value_type* needle, const size_type pos,
+ const size_type nsize) const {
+ if (!nsize) return pos;
+ auto const size = this->size();
+ if (nsize + pos > size) return npos;
+ // Don't use std::search, use a Boyer-Moore-like trick by comparing
+ // the last characters first
+ auto const haystack = data();
+ auto const nsize_1 = nsize - 1;
+ auto const lastNeedle = needle[nsize_1];
+
+ // Boyer-Moore skip value for the last char in the needle. Zero is
+ // not a valid value; skip will be computed the first time it's
+ // needed.
+ size_type skip = 0;
+
+ const E * i = haystack + pos;
+ auto iEnd = haystack + size - nsize_1;
+
+ while (i < iEnd) {
+ // Boyer-Moore: match the last element in the needle
+ while (i[nsize_1] != lastNeedle) {
+ if (++i == iEnd) {
+ // not found
+ return npos;
+ }
+ }
+ // Here we know that the last char matches
+ // Continue in pedestrian mode
+ for (size_t j = 0; ; ) {
+ assert(j < nsize);
+ if (i[j] != needle[j]) {
+ // Not found, we can skip
+ // Compute the skip value lazily
+ if (skip == 0) {
+ skip = 1;
+ while (skip <= nsize_1 && needle[nsize_1 - skip] != lastNeedle) {
+ ++skip;
+ }
+ }
+ i += skip;
+ break;
+ }
+ // Check if done searching
+ if (++j == nsize) {
+ // Yay
+ return i - haystack;
+ }
+ }
+ }
+ return npos;
+ }
+
+ size_type find(const value_type* s, size_type pos = 0) const {
+ return find(s, pos, traits_type::length(s));
+ }
+
+ size_type find (value_type c, size_type pos = 0) const {
+ return find(&c, pos, 1);
+ }
+
+ size_type rfind(const basic_fbstring& str, size_type pos = npos) const {
+ return rfind(str.data(), pos, str.length());
+ }
+
+ size_type rfind(const value_type* s, size_type pos, size_type n) const {
+ if (n > length()) return npos;
+ pos = std::min(pos, length() - n);
+ if (n == 0) return pos;
+
+ const_iterator i(begin() + pos);
+ for (; ; --i) {
+ if (traits_type::eq(*i, *s)
+ && traits_type::compare(&*i, s, n) == 0) {
+ return i - begin();
+ }
+ if (i == begin()) break;
+ }
+ return npos;
+ }
+
+ size_type rfind(const value_type* s, size_type pos = npos) const {
+ return rfind(s, pos, traits_type::length(s));
+ }
+
+ size_type rfind(value_type c, size_type pos = npos) const {
+ return rfind(&c, pos, 1);
+ }
+
+ size_type find_first_of(const basic_fbstring& str, size_type pos = 0) const {
+ return find_first_of(str.data(), pos, str.length());
+ }
+
+ size_type find_first_of(const value_type* s,
+ size_type pos, size_type n) const {
+ if (pos > length() || n == 0) return npos;
+ const_iterator i(begin() + pos),
+ finish(end());
+ for (; i != finish; ++i) {
+ if (traits_type::find(s, n, *i) != 0) {
+ return i - begin();
+ }
+ }
+ return npos;
+ }
+
+ size_type find_first_of(const value_type* s, size_type pos = 0) const {
+ return find_first_of(s, pos, traits_type::length(s));
+ }
+
+ size_type find_first_of(value_type c, size_type pos = 0) const {
+ return find_first_of(&c, pos, 1);
+ }
+
+ size_type find_last_of (const basic_fbstring& str,
+ size_type pos = npos) const {
+ return find_last_of(str.data(), pos, str.length());
+ }
+
+ size_type find_last_of (const value_type* s, size_type pos,
+ size_type n) const {
+ if (!empty() && n > 0) {
+ pos = std::min(pos, length() - 1);
+ const_iterator i(begin() + pos);
+ for (;; --i) {
+ if (traits_type::find(s, n, *i) != 0) {
+ return i - begin();
+ }
+ if (i == begin()) break;
+ }
+ }
+ return npos;
+ }
+
+ size_type find_last_of (const value_type* s,
+ size_type pos = npos) const {
+ return find_last_of(s, pos, traits_type::length(s));
+ }
+
+ size_type find_last_of (value_type c, size_type pos = npos) const {
+ return find_last_of(&c, pos, 1);
+ }
+
+ size_type find_first_not_of(const basic_fbstring& str,
+ size_type pos = 0) const {
+ return find_first_not_of(str.data(), pos, str.size());
+ }
+
+ size_type find_first_not_of(const value_type* s, size_type pos,
+ size_type n) const {
+ if (pos < length()) {
+ const_iterator
+ i(begin() + pos),
+ finish(end());
+ for (; i != finish; ++i) {
+ if (traits_type::find(s, n, *i) == 0) {
+ return i - begin();
+ }
+ }
+ }
+ return npos;
+ }
+
+ size_type find_first_not_of(const value_type* s,
+ size_type pos = 0) const {
+ return find_first_not_of(s, pos, traits_type::length(s));
+ }
+
+ size_type find_first_not_of(value_type c, size_type pos = 0) const {
+ return find_first_not_of(&c, pos, 1);
+ }
+
+ size_type find_last_not_of(const basic_fbstring& str,
+ size_type pos = npos) const {
+ return find_last_not_of(str.data(), pos, str.length());
+ }
+
+ size_type find_last_not_of(const value_type* s, size_type pos,
+ size_type n) const {
+ if (!this->empty()) {
+ pos = std::min(pos, size() - 1);
+ const_iterator i(begin() + pos);
+ for (;; --i) {
+ if (traits_type::find(s, n, *i) == 0) {
+ return i - begin();
+ }
+ if (i == begin()) break;
+ }
+ }
+ return npos;
+ }
+
+ size_type find_last_not_of(const value_type* s,
+ size_type pos = npos) const {
+ return find_last_not_of(s, pos, traits_type::length(s));
+ }
+
+ size_type find_last_not_of (value_type c, size_type pos = npos) const {
+ return find_last_not_of(&c, pos, 1);
+ }
+
+ basic_fbstring substr(size_type pos = 0, size_type n = npos) const {
+ enforce(pos <= size(), std::__throw_out_of_range, "");
+ return basic_fbstring(data() + pos, std::min(n, size() - pos));
+ }
+
+ int compare(const basic_fbstring& str) const {
+ // FIX due to Goncalo N M de Carvalho July 18, 2005
+ return compare(0, size(), str);
+ }
+
+ int compare(size_type pos1, size_type n1,
+ const basic_fbstring& str) const {
+ return compare(pos1, n1, str.data(), str.size());
+ }
+
+ int compare(size_type pos1, size_type n1,
+ const value_type* s) const {
+ return compare(pos1, n1, s, traits_type::length(s));
+ }
+
+ int compare(size_type pos1, size_type n1,
+ const value_type* s, size_type n2) const {
+ enforce(pos1 <= size(), std::__throw_out_of_range, "");
+ procrustes(n1, size() - pos1);
+ // The line below fixed by Jean-Francois Bastien, 04-23-2007. Thanks!
+ const int r = traits_type::compare(pos1 + data(), s, std::min(n1, n2));
+ return r != 0 ? r : n1 > n2 ? 1 : n1 < n2 ? -1 : 0;
+ }
+
+ int compare(size_type pos1, size_type n1,
+ const basic_fbstring& str,
+ size_type pos2, size_type n2) const {
+ enforce(pos2 <= str.size(), std::__throw_out_of_range, "");
+ return compare(pos1, n1, str.data() + pos2,
+ std::min(n2, str.size() - pos2));
+ }
+
+ // Code from Jean-Francois Bastien (03/26/2007)
+ int compare(const value_type* s) const {
+ // Could forward to compare(0, size(), s, traits_type::length(s))
+ // but that does two extra checks
+ const size_type n1(size()), n2(traits_type::length(s));
+ const int r = traits_type::compare(data(), s, std::min(n1, n2));
+ return r != 0 ? r : n1 > n2 ? 1 : n1 < n2 ? -1 : 0;
+ }
+
+private:
+ // Data
+ Storage store_;
+};
+
+// non-member functions
+// C++11 21.4.8.1/2
+template <typename E, class T, class A, class S>
+inline
+basic_fbstring<E, T, A, S> operator+(const basic_fbstring<E, T, A, S>& lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+
+ basic_fbstring<E, T, A, S> result;
+ result.reserve(lhs.size() + rhs.size());
+ result.append(lhs).append(rhs);
+ return std::move(result);
+}
+
+// C++11 21.4.8.1/2
+template <typename E, class T, class A, class S>
+inline
+basic_fbstring<E, T, A, S> operator+(basic_fbstring<E, T, A, S>&& lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+ return std::move(lhs.append(rhs));
+}
+
+// C++11 21.4.8.1/3
+template <typename E, class T, class A, class S>
+inline
+basic_fbstring<E, T, A, S> operator+(const basic_fbstring<E, T, A, S>& lhs,
+ basic_fbstring<E, T, A, S>&& rhs) {
+ if (rhs.capacity() >= lhs.size() + rhs.size()) {
+ // Good, at least we don't need to reallocate
+ return std::move(rhs.insert(0, lhs));
+ }
+ // Meh, no go. Forward to operator+(const&, const&).
+ auto const& rhsC = rhs;
+ return lhs + rhsC;
+}
+
+// C++11 21.4.8.1/4
+template <typename E, class T, class A, class S>
+inline
+basic_fbstring<E, T, A, S> operator+(basic_fbstring<E, T, A, S>&& lhs,
+ basic_fbstring<E, T, A, S>&& rhs) {
+ return std::move(lhs.append(rhs));
+}
+
+template <typename E, class T, class A, class S>
+inline
+basic_fbstring<E, T, A, S> operator+(
+ const typename basic_fbstring<E, T, A, S>::value_type* lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+ //
+ basic_fbstring<E, T, A, S> result;
+ const typename basic_fbstring<E, T, A, S>::size_type len =
+ basic_fbstring<E, T, A, S>::traits_type::length(lhs);
+ result.reserve(len + rhs.size());
+ result.append(lhs, len).append(rhs);
+ return result;
+}
+
+template <typename E, class T, class A, class S>
+inline
+basic_fbstring<E, T, A, S> operator+(
+ typename basic_fbstring<E, T, A, S>::value_type lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+
+ basic_fbstring<E, T, A, S> result;
+ result.reserve(1 + rhs.size());
+ result.push_back(lhs);
+ result.append(rhs);
+ return result;
+}
+
+template <typename E, class T, class A, class S>
+inline
+basic_fbstring<E, T, A, S> operator+(
+ const basic_fbstring<E, T, A, S>& lhs,
+ const typename basic_fbstring<E, T, A, S>::value_type* rhs) {
+
+ typedef typename basic_fbstring<E, T, A, S>::size_type size_type;
+ typedef typename basic_fbstring<E, T, A, S>::traits_type traits_type;
+
+ basic_fbstring<E, T, A, S> result;
+ const size_type len = traits_type::length(rhs);
+ result.reserve(lhs.size() + len);
+ result.append(lhs).append(rhs, len);
+ return result;
+}
+
+template <typename E, class T, class A, class S>
+inline
+basic_fbstring<E, T, A, S> operator+(
+ const basic_fbstring<E, T, A, S>& lhs,
+ typename basic_fbstring<E, T, A, S>::value_type rhs) {
+
+ basic_fbstring<E, T, A, S> result;
+ result.reserve(lhs.size() + 1);
+ result.append(lhs);
+ result.push_back(rhs);
+ return result;
+}
+
+template <typename E, class T, class A, class S>
+inline
+bool operator==(const basic_fbstring<E, T, A, S>& lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+ return lhs.compare(rhs) == 0; }
+
+template <typename E, class T, class A, class S>
+inline
+bool operator==(const typename basic_fbstring<E, T, A, S>::value_type* lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+ return rhs == lhs; }
+
+template <typename E, class T, class A, class S>
+inline
+bool operator==(const basic_fbstring<E, T, A, S>& lhs,
+ const typename basic_fbstring<E, T, A, S>::value_type* rhs) {
+ return lhs.compare(rhs) == 0; }
+
+template <typename E, class T, class A, class S>
+inline
+bool operator!=(const basic_fbstring<E, T, A, S>& lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+ return !(lhs == rhs); }
+
+template <typename E, class T, class A, class S>
+inline
+bool operator!=(const typename basic_fbstring<E, T, A, S>::value_type* lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+ return !(lhs == rhs); }
+
+template <typename E, class T, class A, class S>
+inline
+bool operator!=(const basic_fbstring<E, T, A, S>& lhs,
+ const typename basic_fbstring<E, T, A, S>::value_type* rhs) {
+ return !(lhs == rhs); }
+
+template <typename E, class T, class A, class S>
+inline
+bool operator<(const basic_fbstring<E, T, A, S>& lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+ return lhs.compare(rhs) < 0; }
+
+template <typename E, class T, class A, class S>
+inline
+bool operator<(const basic_fbstring<E, T, A, S>& lhs,
+ const typename basic_fbstring<E, T, A, S>::value_type* rhs) {
+ return lhs.compare(rhs) < 0; }
+
+template <typename E, class T, class A, class S>
+inline
+bool operator<(const typename basic_fbstring<E, T, A, S>::value_type* lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+ return rhs.compare(lhs) > 0; }
+
+template <typename E, class T, class A, class S>
+inline
+bool operator>(const basic_fbstring<E, T, A, S>& lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+ return rhs < lhs; }
+
+template <typename E, class T, class A, class S>
+inline
+bool operator>(const basic_fbstring<E, T, A, S>& lhs,
+ const typename basic_fbstring<E, T, A, S>::value_type* rhs) {
+ return rhs < lhs; }
+
+template <typename E, class T, class A, class S>
+inline
+bool operator>(const typename basic_fbstring<E, T, A, S>::value_type* lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+ return rhs < lhs; }
+
+template <typename E, class T, class A, class S>
+inline
+bool operator<=(const basic_fbstring<E, T, A, S>& lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+ return !(rhs < lhs); }
+
+template <typename E, class T, class A, class S>
+inline
+bool operator<=(const basic_fbstring<E, T, A, S>& lhs,
+ const typename basic_fbstring<E, T, A, S>::value_type* rhs) {
+ return !(rhs < lhs); }
+
+template <typename E, class T, class A, class S>
+inline
+bool operator<=(const typename basic_fbstring<E, T, A, S>::value_type* lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+ return !(rhs < lhs); }
+
+template <typename E, class T, class A, class S>
+inline
+bool operator>=(const basic_fbstring<E, T, A, S>& lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+ return !(lhs < rhs); }
+
+template <typename E, class T, class A, class S>
+inline
+bool operator>=(const basic_fbstring<E, T, A, S>& lhs,
+ const typename basic_fbstring<E, T, A, S>::value_type* rhs) {
+ return !(lhs < rhs); }
+
+template <typename E, class T, class A, class S>
+inline
+bool operator>=(const typename basic_fbstring<E, T, A, S>::value_type* lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+ return !(lhs < rhs);
+}
+
+// subclause 21.3.7.8:
+template <typename E, class T, class A, class S>
+void swap(basic_fbstring<E, T, A, S>& lhs, basic_fbstring<E, T, A, S>& rhs) {
+ lhs.swap(rhs);
+}
+
+// TODO: make this faster.
+template <typename E, class T, class A, class S>
+inline
+std::basic_istream<
+ typename basic_fbstring<E, T, A, S>::value_type,
+ typename basic_fbstring<E, T, A, S>::traits_type>&
+ operator>>(
+ std::basic_istream<typename basic_fbstring<E, T, A, S>::value_type,
+ typename basic_fbstring<E, T, A, S>::traits_type>& is,
+ basic_fbstring<E, T, A, S>& str) {
+ typename std::basic_istream<E, T>::sentry sentry(is);
+ typedef std::basic_istream<typename basic_fbstring<E, T, A, S>::value_type,
+ typename basic_fbstring<E, T, A, S>::traits_type>
+ __istream_type;
+ typedef typename __istream_type::ios_base __ios_base;
+ size_t extracted = 0;
+ auto err = __ios_base::goodbit;
+ if (sentry) {
+ auto n = is.width();
+ if (n == 0) {
+ n = str.max_size();
+ }
+ str.erase();
+ auto got = is.rdbuf()->sgetc();
+ for (; extracted != n && got != T::eof() && !isspace(got); ++extracted) {
+ // Whew. We get to store this guy
+ str.push_back(got);
+ got = is.rdbuf()->snextc();
+ }
+ if (got == T::eof()) {
+ err |= __ios_base::eofbit;
+ is.width(0);
+ }
+ }
+ if (!extracted) {
+ err |= __ios_base::failbit;
+ }
+ if (err) {
+ is.setstate(err);
+ }
+ return is;
+}
+
+template <typename E, class T, class A, class S>
+inline
+std::basic_ostream<typename basic_fbstring<E, T, A, S>::value_type,
+ typename basic_fbstring<E, T, A, S>::traits_type>&
+operator<<(
+ std::basic_ostream<typename basic_fbstring<E, T, A, S>::value_type,
+ typename basic_fbstring<E, T, A, S>::traits_type>& os,
+ const basic_fbstring<E, T, A, S>& str) {
+ os.write(str.data(), str.size());
+ return os;
+}
+
+#ifndef _LIBSTDCXX_FBSTRING
+
+template <typename E, class T, class A, class S>
+inline
+std::basic_istream<typename basic_fbstring<E, T, A, S>::value_type,
+ typename basic_fbstring<E, T, A, S>::traits_type>&
+getline(
+ std::basic_istream<typename basic_fbstring<E, T, A, S>::value_type,
+ typename basic_fbstring<E, T, A, S>::traits_type>& is,
+ basic_fbstring<E, T, A, S>& str,
+ typename basic_fbstring<E, T, A, S>::value_type delim) {
+ // Use the nonstandard getdelim()
+ char * buf = NULL;
+ size_t size = 0;
+ for (;;) {
+ // This looks quadratic but it really depends on realloc
+ auto const newSize = size + 128;
+ buf = static_cast<char*>(realloc(buf, newSize));
+ is.getline(buf + size, newSize - size, delim);
+ if (is.bad() || is.eof() || !is.fail()) {
+ // done by either failure, end of file, or normal read
+ size += std::strlen(buf + size);
+ break;
+ }
+ // Here we have failed due to too short a buffer
+ // Minus one to discount the terminating '\0'
+ size = newSize - 1;
+ assert(buf[size] == 0);
+ // Clear the error so we can continue reading
+ is.clear();
+ }
+ basic_fbstring<E, T, A, S> result(buf, size, size + 1,
+ AcquireMallocatedString());
+ result.swap(str);
+ return is;
+}
+
+template <typename E, class T, class A, class S>
+inline
+std::basic_istream<typename basic_fbstring<E, T, A, S>::value_type,
+ typename basic_fbstring<E, T, A, S>::traits_type>&
+getline(
+ std::basic_istream<typename basic_fbstring<E, T, A, S>::value_type,
+ typename basic_fbstring<E, T, A, S>::traits_type>& is,
+ basic_fbstring<E, T, A, S>& str) {
+ // Just forward to the version with a delimiter
+ return getline(is, str, '\n');
+}
+
+#endif
+
+template <typename E1, class T, class A, class S>
+const typename basic_fbstring<E1, T, A, S>::size_type
+basic_fbstring<E1, T, A, S>::npos =
+ static_cast<typename basic_fbstring<E1, T, A, S>::size_type>(-1);
+
+#ifndef _LIBSTDCXX_FBSTRING
+// basic_string compatiblity routines
+
+template <typename E, class T, class A, class S>
+inline
+bool operator==(const basic_fbstring<E, T, A, S>& lhs,
+ const std::string& rhs) {
+ return lhs.compare(0, lhs.size(), rhs.data(), rhs.size()) == 0;
+}
+
+template <typename E, class T, class A, class S>
+inline
+bool operator==(const std::string& lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+ return rhs == lhs;
+}
+
+template <typename E, class T, class A, class S>
+inline
+bool operator!=(const basic_fbstring<E, T, A, S>& lhs,
+ const std::string& rhs) {
+ return !(lhs == rhs);
+}
+
+template <typename E, class T, class A, class S>
+inline
+bool operator!=(const std::string& lhs,
+ const basic_fbstring<E, T, A, S>& rhs) {
+ return !(lhs == rhs);
+}
+
+#if !defined(_LIBSTDCXX_FBSTRING)
+typedef basic_fbstring<char> fbstring;
+#endif
+
+// fbstring is relocatable
+template <class T, class R, class A, class S>
+FOLLY_ASSUME_RELOCATABLE(basic_fbstring<T, R, A, S>);
+
+#else
+_GLIBCXX_END_NAMESPACE_VERSION
+#endif
+
+} // namespace folly
+
+#ifndef _LIBSTDCXX_FBSTRING
+
+namespace std {
+template <>
+struct hash< ::folly::fbstring> {
+ size_t operator()(const ::folly::fbstring& s) const {
+ return ::folly::hash::fnv32(s.c_str());
+ }
+};
+}
+
+#endif // _LIBSTDCXX_FBSTRING
+
+#endif // FOLLY_BASE_FBSTRING_H_
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Andrei Alexandrescu (aalexandre)
+
+/**
+ * Vector type. Drop-in replacement for std::vector featuring
+ * significantly faster primitives, see e.g. benchmark results at
+ * https:*phabricator.fb.com/D235852.
+ *
+ * In order for a type to be used with fbvector, it must be
+ * relocatable, see Traits.h.
+ *
+ * For user-defined types you must specialize templates
+ * appropriately. Consult Traits.h for ways to do so and for a handy
+ * family of macros FOLLY_ASSUME_FBVECTOR_COMPATIBLE*.
+ *
+ * For more information and documentation see folly/docs/FBVector.md
+ */
+
+#ifndef FOLLY_FBVECTOR_H_
+#define FOLLY_FBVECTOR_H_
+
+#include "folly/Foreach.h"
+#include "folly/Malloc.h"
+#include "folly/Traits.h"
+#include <iterator>
+#include <algorithm>
+#include <stdexcept>
+#include <limits>
+#include <cassert>
+#include <boost/type_traits.hpp>
+#include <boost/operators.hpp>
+#include <boost/utility/enable_if.hpp>
+#include <type_traits>
+
+namespace folly {
+/**
+ * Forward declaration for use by FOLLY_ASSUME_FBVECTOR_COMPATIBLE_2,
+ * see folly/Traits.h.
+ */
+template <typename T, class Allocator = std::allocator<T> >
+class fbvector;
+}
+
+// You can define an fbvector of fbvectors.
+FOLLY_ASSUME_FBVECTOR_COMPATIBLE_2(folly::fbvector);
+
+namespace folly {
+namespace fbvector_detail {
+
+/**
+ * isForwardIterator<T>::value yields true if T is a forward iterator
+ * or better, and false otherwise.
+ */
+template <class It> struct isForwardIterator {
+ enum { value = boost::is_convertible<
+ typename std::iterator_traits<It>::iterator_category,
+ std::forward_iterator_tag>::value
+ };
+};
+
+/**
+ * Destroys all elements in the range [b, e). If the type referred to
+ * by the iterators has a trivial destructor, does nothing.
+ */
+template <class It>
+void destroyRange(It b, It e) {
+ typedef typename boost::remove_reference<decltype(*b)>::type T;
+ if (boost::has_trivial_destructor<T>::value) return;
+ for (; b != e; ++b) {
+ (*b).~T();
+ }
+}
+
+/**
+ * Moves the "interesting" part of value to the uninitialized memory
+ * at address addr, and leaves value in a destroyable state.
+ */
+
+template <class T>
+typename boost::enable_if_c<
+ boost::has_trivial_assign<T>::value
+>::type
+uninitialized_destructive_move(T& value, T* addr) {
+ // Just assign the thing; this is most efficient
+ *addr = value;
+}
+
+template <class T>
+typename boost::enable_if_c<
+ !boost::has_trivial_assign<T>::value &&
+ boost::has_nothrow_constructor<T>::value
+>::type
+uninitialized_destructive_move(T& value, T* addr) {
+ // Cheap default constructor - move and reinitialize
+ memcpy(addr, &value, sizeof(T));
+ new(&value) T;
+}
+
+template <class T>
+typename std::enable_if<
+ !boost::has_trivial_assign<T>::value &&
+ !boost::has_nothrow_constructor<T>::value
+>::type
+uninitialized_destructive_move(T& value, T* addr) {
+ // User defined move construction.
+
+ // TODO: we should probably prefer this over the above memcpy()
+ // version when the type has a user-defined move constructor. We
+ // don't right now because 4.6 doesn't implement
+ // std::is_move_constructible<> yet.
+ new (addr) T(std::move(value));
+}
+
+/**
+ * Fills n objects of type T starting at address b with T's default
+ * value. If the operation throws, destroys all objects constructed so
+ * far and calls free(b).
+ */
+template <class T>
+void uninitializedFillDefaultOrFree(T * b, size_t n) {
+ if (boost::is_arithmetic<T>::value || boost::is_pointer<T>::value) {
+ if (n <= 16384 / sizeof(T)) {
+ memset(b, 0, n * sizeof(T));
+ } else {
+ goto duff_fill;
+ }
+ } else if (boost::has_nothrow_constructor<T>::value) {
+ duff_fill:
+ auto i = b;
+ auto const e1 = b + (n & ~size_t(7));
+ for (; i != e1; i += 8) {
+ new(i) T();
+ new(i + 1) T();
+ new(i + 2) T();
+ new(i + 3) T();
+ new(i + 4) T();
+ new(i + 5) T();
+ new(i + 6) T();
+ new(i + 7) T();
+ }
+ for (auto const e = b + n; i != e; ++i) {
+ new(i) T();
+ }
+ } else {
+ // Conservative approach
+ auto i = b;
+ try {
+ for (auto const e = b + n; i != e; ++i) {
+ new(i) T;
+ }
+ } catch (...) {
+ destroyRange(b, i);
+ free(b);
+ throw;
+ }
+ }
+}
+
+/**
+ * Fills n objects of type T starting at address b with value. If the
+ * operation throws, destroys all objects constructed so far and calls
+ * free(b).
+ */
+template <class T>
+void uninitializedFillOrFree(T * b, size_t n, const T& value) {
+ auto const e = b + n;
+ if (boost::has_trivial_copy<T>::value) {
+ auto i = b;
+ auto const e1 = b + (n & ~size_t(7));
+ for (; i != e1; i += 8) {
+ new(i) T(value);
+ new(i + 1) T(value);
+ new(i + 2) T(value);
+ new(i + 3) T(value);
+ new(i + 4) T(value);
+ new(i + 5) T(value);
+ new(i + 6) T(value);
+ new(i + 7) T(value);
+ }
+ for (; i != e; ++i) {
+ new(i) T(value);
+ }
+ } else {
+ // Conservative approach
+ auto i = b;
+ try {
+ for (; i != e; ++i) {
+ new(i) T(value);
+ }
+ } catch (...) {
+ destroyRange(b, i);
+ free(b);
+ throw;
+ }
+ }
+}
+} // namespace fbvector_detail
+
+/**
+ * This is the std::vector replacement. For conformity, fbvector takes
+ * the same template parameters, but it doesn't use the
+ * allocator. Instead, it uses malloc, and when present, jemalloc's
+ * extensions.
+ */
+template <class T, class Allocator>
+class fbvector : private boost::totally_ordered<fbvector<T,Allocator> > {
+ bool isSane() const {
+ return
+ begin() <= end() &&
+ empty() == (size() == 0) &&
+ empty() == (begin() == end()) &&
+ size() <= max_size() &&
+ capacity() <= max_size() &&
+ size() <= capacity() &&
+
+ // Either we have no capacity or our pointers should make sense:
+ ((!b_ && !e_ && !z_) || (b_ != z_ && e_ <= z_));
+ }
+
+ struct Invariant {
+#ifndef NDEBUG
+ explicit Invariant(const fbvector& s) : s_(s) {
+ assert(s_.isSane());
+ }
+ ~Invariant() {
+ assert(s_.isSane());
+ }
+ private:
+ const fbvector& s_;
+#else
+ explicit Invariant(const fbvector&) {}
+#endif
+ Invariant& operator=(const Invariant&);
+ };
+
+public:
+
+// types:
+ typedef T value_type;
+ typedef value_type& reference;
+ typedef const value_type& const_reference;
+ typedef T* iterator;
+ typedef const T* const_iterator;
+ typedef size_t size_type;
+ typedef ssize_t difference_type;
+ // typedef typename allocator_traits<Allocator>::pointer pointer;
+ // typedef typename allocator_traits<Allocator>::const_pointer const_pointer;
+ typedef Allocator allocator_type;
+ typedef typename Allocator::pointer pointer;
+ typedef typename Allocator::const_pointer const_pointer;
+ typedef std::reverse_iterator<iterator> reverse_iterator;
+ typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+
+// 23.3.6.1 construct/copy/destroy:
+ fbvector() : b_(NULL), e_(NULL), z_(NULL) {}
+
+ explicit fbvector(const Allocator&) {
+ new(this) fbvector;
+ }
+
+ explicit fbvector(const size_type n) {
+ if (n == 0) {
+ b_ = e_ = z_ = 0;
+ return;
+ }
+
+ auto const nBytes = goodMallocSize(n * sizeof(T));
+ b_ = static_cast<T*>(malloc(nBytes));
+ fbvector_detail::uninitializedFillDefaultOrFree(b_, n);
+ e_ = b_ + n;
+ z_ = b_ + nBytes / sizeof(T);
+ }
+
+ fbvector(const size_type n, const T& value) {
+ if (!n) {
+ b_ = e_ = z_ = 0;
+ return;
+ }
+
+ auto const nBytes = goodMallocSize(n * sizeof(T));
+ b_ = static_cast<T*>(malloc(nBytes));
+ fbvector_detail::uninitializedFillOrFree(b_, n, value);
+ e_ = b_ + n;
+ z_ = b_ + nBytes / sizeof(T);
+ }
+
+ fbvector(const size_type n, const T& value, const Allocator&) {
+ new(this) fbvector(n, value);
+ }
+
+ template <class InputIteratorOrNum>
+ fbvector(InputIteratorOrNum first, InputIteratorOrNum last) {
+ new(this) fbvector;
+ assign(first, last);
+ }
+
+ template <class InputIterator>
+ fbvector(InputIterator first, InputIterator last,
+ const Allocator&) {
+ new(this) fbvector(first, last);
+ }
+
+ fbvector(const fbvector& rhs) {
+ new(this) fbvector(rhs.begin(), rhs.end());
+ }
+ fbvector(const fbvector& rhs, const Allocator&) {
+ new(this) fbvector(rhs);
+ }
+
+ fbvector(fbvector&& o, const Allocator& = Allocator())
+ : b_(o.b_)
+ , e_(o.e_)
+ , z_(o.z_)
+ {
+ o.b_ = o.e_ = o.z_ = 0;
+ }
+
+ fbvector(std::initializer_list<T> il, const Allocator& = Allocator()) {
+ new(this) fbvector(il.begin(), il.end());
+ }
+
+ ~fbvector() {
+ // fbvector only works with relocatable objects. We insert this
+ // static check inside the destructor because pretty much any
+ // instantiation of fbvector<T> will generate the destructor (and
+ // therefore refuse compilation if the assertion fails). To see
+ // how you can enable IsRelocatable for your type, refer to the
+ // definition of IsRelocatable in Traits.h.
+ BOOST_STATIC_ASSERT(IsRelocatable<T>::value);
+ if (!b_) return;
+ fbvector_detail::destroyRange(b_, e_);
+ free(b_);
+ }
+ fbvector& operator=(const fbvector& rhs) {
+ assign(rhs.begin(), rhs.end());
+ return *this;
+ }
+
+ fbvector& operator=(fbvector&& v) {
+ clear();
+ swap(v);
+ return *this;
+ }
+
+ fbvector& operator=(std::initializer_list<T> il) {
+ assign(il.begin(), il.end());
+ return *this;
+ }
+
+ bool operator==(const fbvector& rhs) const {
+ return size() == rhs.size() && std::equal(begin(), end(), rhs.begin());
+ }
+
+ bool operator<(const fbvector& rhs) const {
+ return std::lexicographical_compare(begin(), end(),
+ rhs.begin(), rhs.end());
+ }
+
+private:
+ template <class InputIterator>
+ void assignImpl(InputIterator first, InputIterator last, boost::false_type) {
+ // Pair of iterators
+ if (fbvector_detail::isForwardIterator<InputIterator>::value) {
+ if (b_ <= &*first && &*first < e_) {
+ // Aliased assign, work on the side
+ fbvector result(first, last);
+ result.swap(*this);
+ return;
+ }
+
+ auto const oldSize = size();
+ auto const newSize = std::distance(first, last);
+
+ if (static_cast<difference_type>(oldSize) >= newSize) {
+ // No reallocation, nice
+ auto const newEnd = std::copy(first, last, b_);
+ fbvector_detail::destroyRange(newEnd, e_);
+ e_ = newEnd;
+ return;
+ }
+
+ // Must reallocate - just do it on the side
+ auto const nBytes = goodMallocSize(newSize * sizeof(T));
+ auto const b = static_cast<T*>(malloc(nBytes));
+ std::uninitialized_copy(first, last, b);
+ this->fbvector::~fbvector();
+ b_ = b;
+ e_ = b + newSize;
+ z_ = b_ + nBytes / sizeof(T);
+ } else {
+ // Input iterator sucks
+ FOR_EACH (i, *this) {
+ if (first == last) {
+ fbvector_detail::destroyRange(i, e_);
+ e_ = i;
+ return;
+ }
+ *i = *first;
+ ++first;
+ }
+ FOR_EACH_RANGE (i, first, last) {
+ push_back(*i);
+ }
+ }
+ }
+
+ void assignImpl(const size_type newSize, const T value, boost::true_type) {
+ // Arithmetic type, forward back to unambiguous definition
+ assign(newSize, value);
+ }
+
+public:
+ // Classic ambiguity (and a lot of unnecessary complexity) in
+ // std::vector: assign(10, 20) for vector<int> means "assign 10
+ // elements all having the value 20" but is intercepted by the
+ // two-iterators overload assign(first, last). So we need to
+ // disambiguate here. There is no pretty solution. We use here
+ // overloading based on is_arithmetic. Method insert has the same
+ // issue (and the same solution in this implementation).
+ template <class InputIteratorOrNum>
+ void assign(InputIteratorOrNum first, InputIteratorOrNum last) {
+ assignImpl(first, last, boost::is_arithmetic<InputIteratorOrNum>());
+ }
+
+ void assign(const size_type newSize, const T& value) {
+ if (b_ <= &value && &value < e_) {
+ // Need to check for aliased assign, sigh
+ return assign(newSize, T(value));
+ }
+
+ auto const oldSize = size();
+ if (oldSize >= newSize) {
+ // No reallocation, nice
+ auto const newEnd = b_ + newSize;
+ fbvector_detail::destroyRange(newEnd, e_);
+ e_ = newEnd;
+ return;
+ }
+
+ // Need to reallocate
+ if (reserve_in_place(newSize)) {
+ // Careful here, fill and uninitialized_fill may throw. The
+ // latter is transactional, so no need to worry about a
+ // buffer partially filled in case of exception.
+ std::fill(b_, e_, value);
+ auto const newEnd = b_ + newSize;
+ std::uninitialized_fill(e_, newEnd, value);
+ e_ = newEnd;
+ return;
+ }
+
+ // Cannot expand or jemalloc not present at all; must just
+ // allocate a new chunk and discard the old one. This is
+ // tantamount with creating a new fbvector altogether. This won't
+ // recurse infinitely; the constructor implements its own.
+ fbvector temp(newSize, value);
+ temp.swap(*this);
+ }
+
+ void assign(std::initializer_list<T> il) {
+ assign(il.begin(), il.end());
+ }
+
+ allocator_type get_allocator() const {
+ // whatevs
+ return allocator_type();
+ }
+
+// iterators:
+ iterator begin() {
+ return b_;
+ }
+ const_iterator begin() const {
+ return b_;
+ }
+ iterator end() {
+ return e_;
+ }
+ const_iterator end() const {
+ return e_;
+ }
+ reverse_iterator rbegin() {
+ return reverse_iterator(end());
+ }
+ const_reverse_iterator rbegin() const {
+ return const_reverse_iterator(end());
+ }
+ reverse_iterator rend() {
+ return reverse_iterator(begin());
+ }
+ const_reverse_iterator rend() const {
+ return const_reverse_iterator(begin());
+ }
+ const_iterator cbegin() const {
+ return b_;
+ }
+ const_iterator cend() const {
+ return e_;
+ }
+
+// 23.3.6.2 capacity:
+ size_type size() const {
+ return e_ - b_;
+ }
+
+ size_type max_size() {
+ // good luck gettin' there
+ return ~size_type(0);
+ }
+
+ void resize(const size_type sz) {
+ auto const oldSize = size();
+ if (sz <= oldSize) {
+ auto const newEnd = b_ + sz;
+ fbvector_detail::destroyRange(newEnd, e_);
+ e_ = newEnd;
+ } else {
+ // Must expand
+ reserve(sz);
+ auto newEnd = b_ + sz;
+ std::uninitialized_fill(e_, newEnd, T());
+ e_ = newEnd;
+ }
+ }
+
+ void resize(const size_type sz, const T& c) {
+ auto const oldSize = size();
+ if (sz <= oldSize) {
+ auto const newEnd = b_ + sz;
+ fbvector_detail::destroyRange(newEnd, e_);
+ e_ = newEnd;
+ } else {
+ // Must expand
+ reserve(sz);
+ auto newEnd = b_ + sz;
+ std::uninitialized_fill(e_, newEnd, c);
+ e_ = newEnd;
+ }
+ }
+
+ size_type capacity() const {
+ return z_ - b_;
+ }
+ bool empty() const {
+ return b_ == e_;
+ }
+
+private:
+ bool reserve_in_place(const size_type n) {
+ auto const crtCapacity = capacity();
+ if (n <= crtCapacity) return true;
+ if (!rallocm) return false;
+
+ // using jemalloc's API. Don't forget that jemalloc can never grow
+ // in place blocks smaller than 4096 bytes.
+ auto const crtCapacityBytes = crtCapacity * sizeof(T);
+ if (crtCapacityBytes < jemallocMinInPlaceExpandable) return false;
+
+ auto const newCapacityBytes = goodMallocSize(n * sizeof(T));
+ void* p = b_;
+ if (rallocm(&p, NULL, newCapacityBytes, 0, ALLOCM_NO_MOVE)
+ != ALLOCM_SUCCESS) {
+ return false;
+ }
+
+ // Managed to expand in place, reflect that in z_
+ assert(b_ == p);
+ z_ = b_ + newCapacityBytes / sizeof(T);
+ return true;
+ }
+
+ void reserve_with_move(const size_type n) {
+ // Here we can be sure we'll need to do a full reallocation
+ auto const crtCapacity = capacity();
+ assert(crtCapacity < n); // reserve_in_place should have taken
+ // care of this
+ auto const newCapacityBytes = goodMallocSize(n * sizeof(T));
+ auto b = static_cast<T*>(malloc(newCapacityBytes));
+ auto const oldSize = size();
+ memcpy(b, b_, oldSize * sizeof(T));
+ // Done with the old chunk. Free but don't call destructors!
+ free(b_);
+ b_ = b;
+ e_ = b_ + oldSize;
+ z_ = b_ + newCapacityBytes / sizeof(T);
+ // done with the old chunk
+ }
+
+public:
+ void reserve(const size_type n) {
+ if (reserve_in_place(n)) return;
+ reserve_with_move(n);
+ }
+
+ void shrink_to_fit() {
+ if (!rallocm) return;
+
+ // using jemalloc's API. Don't forget that jemalloc can never
+ // shrink in place blocks smaller than 4096 bytes.
+ void* p = b_;
+ auto const crtCapacityBytes = capacity() * sizeof(T);
+ auto const newCapacityBytes = goodMallocSize(size() * sizeof(T));
+ if (crtCapacityBytes >= jemallocMinInPlaceExpandable &&
+ rallocm(&p, NULL, newCapacityBytes, 0, ALLOCM_NO_MOVE)
+ == ALLOCM_SUCCESS) {
+ // Celebrate
+ z_ = b_ + newCapacityBytes / sizeof(T);
+ }
+ }
+
+// element access
+ reference operator[](size_type n) {
+ assert(n < size());
+ return b_[n];
+ }
+ const_reference operator[](size_type n) const {
+ assert(n < size());
+ return b_[n];
+ }
+ const_reference at(size_type n) const {
+ if (n > size()) {
+ throw std::out_of_range("fbvector: index is greater than size.");
+ }
+ return (*this)[n];
+ }
+ reference at(size_type n) {
+ auto const& cThis = *this;
+ return const_cast<reference>(cThis.at(n));
+ }
+ reference front() {
+ assert(!empty());
+ return *b_;
+ }
+ const_reference front() const {
+ assert(!empty());
+ return *b_;
+ }
+ reference back() {
+ assert(!empty());
+ return e_[-1];
+ }
+ const_reference back() const {
+ assert(!empty());
+ return e_[-1];
+ }
+
+// 23.3.6.3 data access
+ T* data() {
+ return b_;
+ }
+ const T* data() const {
+ return b_;
+ }
+
+private:
+ size_t computePushBackCapacity() const {
+ return empty() ? std::max(64 / sizeof(T), size_t(1))
+ : capacity() < jemallocMinInPlaceExpandable ? capacity() * 2
+ : (capacity() * 3) / 2;
+ }
+
+public:
+// 23.3.6.4 modifiers:
+ template <class... Args>
+ void emplace_back(Args&&... args) {
+ if (e_ == z_) {
+ if (!reserve_in_place(size() + 1)) {
+ reserve_with_move(computePushBackCapacity());
+ }
+ }
+ new (e_) T(std::forward<Args>(args)...);
+ ++e_;
+ }
+
+ void push_back(T x) {
+ if (e_ == z_) {
+ if (!reserve_in_place(size() + 1)) {
+ reserve_with_move(computePushBackCapacity());
+ }
+ }
+ fbvector_detail::uninitialized_destructive_move(x, e_);
+ ++e_;
+ }
+
+private:
+ bool expand() {
+ if (!rallocm) return false;
+ auto const capBytes = capacity() * sizeof(T);
+ if (capBytes < jemallocMinInPlaceExpandable) return false;
+ auto const newCapBytes = goodMallocSize(capBytes + sizeof(T));
+ void * bv = b_;
+ if (rallocm(&bv, NULL, newCapBytes, 0, ALLOCM_NO_MOVE) != ALLOCM_SUCCESS) {
+ return false;
+ }
+ // Managed to expand in place
+ assert(bv == b_); // nothing moved
+ z_ = b_ + newCapBytes / sizeof(T);
+ assert(capacity() > capBytes / sizeof(T));
+ return true;
+ }
+
+public:
+ void pop_back() {
+ assert(!empty());
+ --e_;
+ if (!boost::has_trivial_destructor<T>::value) {
+ e_->T::~T();
+ }
+ }
+ // template <class... Args>
+ // iterator emplace(const_iterator position, Args&&... args);
+
+ iterator insert(const_iterator position, T x) {
+ size_t newSize; // intentionally uninitialized
+ if (e_ == z_ && !reserve_in_place(newSize = size() + 1)) {
+ // Can't reserve in place, make a copy
+ auto const offset = position - cbegin();
+ fbvector tmp;
+ tmp.reserve(newSize);
+ memcpy(tmp.b_, b_, offset * sizeof(T));
+ fbvector_detail::uninitialized_destructive_move(
+ x,
+ tmp.b_ + offset);
+ memcpy(tmp.b_ + offset + 1, b_ + offset, (size() - offset) * sizeof(T));
+ // Brutally reassign this to refer to tmp's guts
+ free(b_);
+ b_ = tmp.b_;
+ e_ = b_ + newSize;
+ z_ = tmp.z_;
+ // get rid of tmp's guts
+ new(&tmp) fbvector;
+ return begin() + offset;
+ }
+ // Here we have enough room
+ memmove(const_cast<T*>(&*position) + 1,
+ const_cast<T*>(&*position),
+ sizeof(T) * (e_ - position));
+ fbvector_detail::uninitialized_destructive_move(
+ x,
+ const_cast<T*>(&*position));
+ ++e_;
+ return const_cast<iterator>(position);
+ }
+
+ iterator insert(const_iterator position, const size_type n, const T& x) {
+ if (e_ + n >= z_) {
+ if (b_ <= &x && &x < e_) {
+ // Ew, aliased insert
+ auto copy = x;
+ return insert(position, n, copy);
+ }
+ auto const m = position - b_;
+ reserve(size() + n);
+ position = b_ + m;
+ }
+ memmove(const_cast<T*>(position) + n,
+ position,
+ sizeof(T) * (e_ - position));
+ if (boost::has_trivial_copy<T>::value) {
+ std::uninitialized_fill(const_cast<T*>(position),
+ const_cast<T*>(position) + n,
+ x);
+ } else {
+ try {
+ std::uninitialized_fill(const_cast<T*>(position),
+ const_cast<T*>(position) + n,
+ x);
+ } catch (...) {
+ // Oops, put things back where they were
+ memmove(const_cast<T*>(position),
+ position + n,
+ sizeof(T) * (e_ - position));
+ throw;
+ }
+ }
+ e_ += n;
+ return const_cast<iterator>(position);
+ }
+
+private:
+ template <class InputIterator>
+ iterator insertImpl(const_iterator position,
+ InputIterator first, InputIterator last,
+ boost::false_type) {
+ // Pair of iterators
+ if (fbvector_detail::isForwardIterator<InputIterator>::value) {
+ // Can compute distance
+ auto const n = std::distance(first, last);
+ if (e_ + n >= z_) {
+ if (b_ <= &*first && &*first < e_) {
+ // Ew, aliased insert
+ goto conservative;
+ }
+ auto const m = position - b_;
+ reserve(size() + n);
+ position = b_ + m;
+ }
+ memmove(const_cast<T*>(position) + n,
+ position,
+ sizeof(T) * (e_ - position));
+ try {
+ std::uninitialized_copy(first, last,
+ const_cast<T*>(position));
+ } catch (...) {
+ // Oops, put things back where they were
+ memmove(const_cast<T*>(position),
+ position + n,
+ sizeof(T) * (e_ - position));
+ throw;
+ }
+ e_ += n;
+ return const_cast<iterator>(position);
+ } else {
+ // Cannot compute distance, crappy approach
+ // TODO: OPTIMIZE
+ conservative:
+ fbvector result(cbegin(), position);
+ auto const offset = result.size();
+ FOR_EACH_RANGE (i, first, last) {
+ result.push_back(*i);
+ }
+ result.insert(result.end(), position, cend());
+ result.swap(*this);
+ return begin() + offset;
+ }
+ }
+
+ iterator insertImpl(const_iterator position,
+ const size_type count, const T value, boost::true_type) {
+ // Forward back to unambiguous function
+ return insert(position, count, value);
+ }
+
+public:
+ template <class InputIteratorOrNum>
+ iterator insert(const_iterator position, InputIteratorOrNum first,
+ InputIteratorOrNum last) {
+ return insertImpl(position, first, last,
+ boost::is_arithmetic<InputIteratorOrNum>());
+ }
+
+ iterator insert(const_iterator position, std::initializer_list<T> il) {
+ return insert(position, il.begin(), il.end());
+ }
+
+ iterator erase(const_iterator position) {
+ if (position == e_) return e_;
+ auto p = const_cast<T*>(position);
+ (*p).T::~T();
+ memmove(p, p + 1, sizeof(T) * (e_ - p - 1));
+ --e_;
+ return p;
+ }
+
+ iterator erase(const_iterator first, const_iterator last) {
+ assert(first <= last);
+ auto p1 = const_cast<T*>(first);
+ auto p2 = const_cast<T*>(last);
+ fbvector_detail::destroyRange(p1, p2);
+ memmove(p1, last, sizeof(T) * (e_ - last));
+ e_ -= last - first;
+ return p1;
+ }
+
+ void swap(fbvector& rhs) {
+ std::swap(b_, rhs.b_);
+ std::swap(e_, rhs.e_);
+ std::swap(z_, rhs.z_);
+ }
+
+ void clear() {
+ fbvector_detail::destroyRange(b_, e_);
+ e_ = b_;
+ }
+
+private:
+ // Data
+ T *b_, *e_, *z_;
+};
+
+template <class T, class A>
+bool operator!=(const fbvector<T, A>& lhs,
+ const fbvector<T, A>& rhs) {
+ return !(lhs == rhs);
+}
+
+template <class T, class A>
+void swap(fbvector<T, A>& lhs, fbvector<T, A>& rhs) {
+ lhs.swap(rhs);
+}
+
+/**
+ * Resizes *v to exactly n elements. May reallocate the vector to a
+ * smaller buffer if too much space will be left unused.
+ */
+template <class T>
+static void compactResize(folly::fbvector<T> * v, size_t size) {
+ auto const oldCap = v->capacity();
+ if (oldCap > size + 1024 && size < oldCap * 0.3) {
+ // Too much slack memory, reallocate a smaller buffer
+ auto const oldSize = v->size();
+ if (size <= oldSize) {
+ // Shrink
+ folly::fbvector<T>(v->begin(), v->begin() + size).swap(*v);
+ } else {
+ // Expand
+ folly::fbvector<T> temp;
+ temp.reserve(size);
+ copy(v->begin(), v->end(), back_inserter(temp));
+ temp.resize(size);
+ temp.swap(*v);
+ }
+ } else {
+ // Nolo contendere
+ v->resize(size);
+ }
+}
+
+} // namespace folly
+
+#endif // FOLLY_FBVECTOR_H_
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOLLY_BASE_FOREACH_H_
+#define FOLLY_BASE_FOREACH_H_
+
+/*
+ * Iterim macros (until we have C++0x range-based for) that simplify
+ * writing loops of the form
+ *
+ * for (Container<data>::iterator i = c.begin(); i != c.end(); ++i) statement
+ *
+ * Just replace the above with:
+ *
+ * FOR_EACH (i, c) statement
+ *
+ * and everything is taken care of.
+ *
+ * The implementation is a bit convoluted to make sure the container is
+ * only evaluated once (however, keep in mind that c.end() is evaluated
+ * at every pass through the loop). To ensure the container is not
+ * evaluated multiple times, the macro defines one do-nothing if
+ * statement to inject the Boolean variable FOR_EACH_state1, and then a
+ * for statement that is executed only once, which defines the variable
+ * FOR_EACH_state2 holding a reference to the container being
+ * iterated. The workhorse is the last loop, which uses the just defined
+ * reference FOR_EACH_state2.
+ *
+ * The state variables are nested so they don't interfere; you can use
+ * FOR_EACH multiple times in the same scope, either at the same level or
+ * nested.
+ *
+ * In optimized builds g++ eliminates the extra gymnastics entirely and
+ * generates code 100% identical to the handwritten loop.
+ *
+ * This will not work with temporary containers. Consider BOOST_FOREACH
+ * if you need that.
+ */
+
+#include <boost/type_traits/remove_cv.hpp>
+
+namespace folly { namespace detail {
+
+/*
+ * Simple template for obtaining the unqualified type given a generic
+ * type T. For example, if T is const int,
+ * typeof(remove_cv_from_expression(T())) yields int. Due to a bug in
+ * g++, you need to actually use
+ * typeof(remove_cv_from_expression(T())) instead of typename
+ * boost::remove_cv<T>::type. Note that the function
+ * remove_cv_from_expression is never defined - use it only inside
+ * typeof.
+ */
+template <class T> typename boost::remove_cv<T>::type
+remove_cv_from_expression(T value);
+
+}}
+
+/*
+ * Use a "reference reference" (auto&&) to take advantage of reference
+ * collapsing rules, if available. In this case, FOR_EACH* will work with
+ * temporary containers.
+ */
+#define FB_AUTO_RR(x, y) auto&& x = y
+
+/*
+ * The first AUTO should be replaced by decltype((c)) &
+ * FOR_EACH_state2, but bugs in gcc prevent that from functioning
+ * properly. The second pair of parens in decltype is actually
+ * required, see
+ * cpp-next.com/archive/2011/04/appearing-and-disappearing-consts-in-c/
+ */
+#define FOR_EACH(i, c) \
+ if (bool FOR_EACH_state1 = false) {} else \
+ for (auto & FOR_EACH_state2 = (c); \
+ !FOR_EACH_state1; FOR_EACH_state1 = true) \
+ for (auto i = FOR_EACH_state2.begin(); \
+ i != FOR_EACH_state2.end(); ++i)
+
+/*
+ * Similar to FOR_EACH, but iterates the container backwards by
+ * using rbegin() and rend().
+ */
+#define FOR_EACH_R(i, c) \
+ if (bool FOR_EACH_R_state1 = false) {} else \
+ for (auto & FOR_EACH_R_state2 = (c); \
+ !FOR_EACH_R_state1; FOR_EACH_R_state1 = true) \
+ for (auto i = FOR_EACH_R_state2.rbegin(); \
+ i != FOR_EACH_R_state2.rend(); ++i)
+
+/*
+ * Similar to FOR_EACH but also allows client to specify a 'count' variable
+ * to track the current iteration in the loop (starting at zero).
+ * Similar to python's enumerate() function. For example:
+ * string commaSeparatedValues = "VALUES: ";
+ * FOR_EACH_ENUMERATE(ii, value, columns) { // don't want comma at the end!
+ * commaSeparatedValues += (ii == 0) ? *value : string(",") + *value;
+ * }
+ */
+#define FOR_EACH_ENUMERATE(count, i, c) \
+ if (bool FOR_EACH_state1 = false) {} else \
+ for (auto & FOR_EACH_state2 = (c); \
+ !FOR_EACH_state1; FOR_EACH_state1 = true) \
+ if (size_t FOR_EACH_privateCount = 0) {} else \
+ if (const size_t& count = FOR_EACH_privateCount) {} else \
+ for (auto i = FOR_EACH_state2.begin(); \
+ i != FOR_EACH_state2.end(); ++FOR_EACH_privateCount, ++i)
+
+/**
+ * Similar to FOR_EACH, but gives the user the key and value for each entry in
+ * the container, instead of just the iterator to the entry. For example:
+ * map<string, string> testMap;
+ * FOR_EACH_KV(key, value, testMap) {
+ * cout << key << " " << value;
+ * }
+ */
+#define FOR_EACH_KV(k, v, c) \
+ if (unsigned int FOR_EACH_state1 = 0) {} else \
+ for (FB_AUTO_RR(FOR_EACH_state2, (c)); \
+ !FOR_EACH_state1; FOR_EACH_state1 = 1) \
+ for (auto FOR_EACH_state3 = FOR_EACH_state2.begin(); \
+ FOR_EACH_state3 != FOR_EACH_state2.end(); \
+ FOR_EACH_state1 == 2 \
+ ? ((FOR_EACH_state1 = 0), ++FOR_EACH_state3) \
+ : (FOR_EACH_state3 = FOR_EACH_state2.end())) \
+ for (auto &k = FOR_EACH_state3->first; \
+ !FOR_EACH_state1; ++FOR_EACH_state1) \
+ for (auto &v = FOR_EACH_state3->second; \
+ !FOR_EACH_state1; ++FOR_EACH_state1)
+
+namespace folly { namespace detail {
+
+// Boost 1.48 lacks has_less, we emulate a subset of it here.
+template <typename T, typename U>
+class HasLess {
+ struct BiggerThanChar { char unused[2]; };
+ template <typename C, typename D> static char test(decltype(C() < D())*);
+ template <typename, typename> static BiggerThanChar test(...);
+public:
+ enum { value = sizeof(test<T, U>(0)) == 1 };
+};
+
+/**
+ * notThereYet helps the FOR_EACH_RANGE macro by opportunistically
+ * using "<" instead of "!=" whenever available when checking for loop
+ * termination. This makes e.g. examples such as FOR_EACH_RANGE (i,
+ * 10, 5) execute zero iterations instead of looping virtually
+ * forever. At the same time, some iterator types define "!=" but not
+ * "<". The notThereYet function will dispatch differently for those.
+ *
+ * Below is the correct implementation of notThereYet. It is disabled
+ * because of a bug in Boost 1.46: The filesystem::path::iterator
+ * defines operator< (via boost::iterator_facade), but that in turn
+ * uses distance_to which is undefined for that particular
+ * iterator. So HasLess (defined above) identifies
+ * boost::filesystem::path as properly comparable with <, but in fact
+ * attempting to do so will yield a compile-time error.
+ *
+ * The else branch (active) contains a conservative
+ * implementation.
+ */
+
+#if 0
+
+template <class T, class U>
+typename std::enable_if<HasLess<T, U>::value, bool>::type
+notThereYet(T& iter, const U& end) {
+ return iter < end;
+}
+
+template <class T, class U>
+typename std::enable_if<!HasLess<T, U>::value, bool>::type
+notThereYet(T& iter, const U& end) {
+ return iter != end;
+}
+
+#else
+
+template <class T, class U>
+typename std::enable_if<
+ (std::is_arithmetic<T>::value && std::is_arithmetic<U>::value) ||
+ (std::is_pointer<T>::value && std::is_pointer<U>::value),
+ bool>::type
+notThereYet(T& iter, const U& end) {
+ return iter < end;
+}
+
+template <class T, class U>
+typename std::enable_if<
+ !(
+ (std::is_arithmetic<T>::value && std::is_arithmetic<U>::value) ||
+ (std::is_pointer<T>::value && std::is_pointer<U>::value)
+ ),
+ bool>::type
+notThereYet(T& iter, const U& end) {
+ return iter != end;
+}
+
+#endif
+
+
+/**
+ * downTo is similar to notThereYet, but in reverse - it helps the
+ * FOR_EACH_RANGE_R macro.
+ */
+template <class T, class U>
+typename std::enable_if<HasLess<U, T>::value, bool>::type
+downTo(T& iter, const U& begin) {
+ return begin < iter--;
+}
+
+template <class T, class U>
+typename std::enable_if<!HasLess<U, T>::value, bool>::type
+downTo(T& iter, const U& begin) {
+ if (iter == begin) return false;
+ --iter;
+ return true;
+}
+
+} }
+
+/*
+ * Iteration with given limits. end is assumed to be reachable from
+ * begin. end is evaluated every pass through the loop.
+ *
+ * NOTE: The type of the loop variable should be the common type of "begin"
+ * and "end". e.g. If "begin" is "int" but "end" is "long", we want "i"
+ * to be "long". This is done by getting the type of (true ? begin : end)
+ */
+#define FOR_EACH_RANGE(i, begin, end) \
+ for (auto i = (true ? (begin) : (end)); \
+ ::folly::detail::notThereYet(i, (end)); \
+ ++i)
+
+/*
+ * Iteration with given limits. begin is assumed to be reachable from
+ * end by successive decrements. begin is evaluated every pass through
+ * the loop.
+ *
+ * NOTE: The type of the loop variable should be the common type of "begin"
+ * and "end". e.g. If "begin" is "int" but "end" is "long", we want "i"
+ * to be "long". This is done by getting the type of (false ? begin : end)
+ */
+#define FOR_EACH_RANGE_R(i, begin, end) \
+ for (auto i = (false ? (begin) : (end)); ::folly::detail::downTo(i, (begin));)
+
+#endif
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOLLY_FORMAT_H_
+#error This file may only be included from Format.h.
+#endif
+
+namespace folly {
+
+namespace detail {
+
+extern const char formatHexUpper[256][2];
+extern const char formatHexLower[256][2];
+extern const char formatOctal[512][3];
+extern const char formatBinary[256][8];
+
+const size_t kMaxHexLength = 2 * sizeof(uintmax_t);
+const size_t kMaxOctalLength = 3 * sizeof(uintmax_t);
+const size_t kMaxBinaryLength = 8 * sizeof(uintmax_t);
+
+/**
+ * Convert an unsigned to hex, using repr (which maps from each possible
+ * 2-hex-bytes value to the 2-character representation).
+ *
+ * Just like folly::detail::uintToBuffer in Conv.h, writes at the *end* of
+ * the supplied buffer and returns the offset of the beginning of the string
+ * from the start of the buffer. The formatted string will be in range
+ * [buf+begin, buf+bufLen).
+ */
+template <class Uint>
+size_t uintToHex(char* buffer, size_t bufLen, Uint v,
+ const char (&repr)[256][2]) {
+ for (; v >= 256; v >>= 8) {
+ auto b = v & 0xff;
+ bufLen -= 2;
+ buffer[bufLen] = repr[b][0];
+ buffer[bufLen + 1] = repr[b][1];
+ }
+ buffer[--bufLen] = repr[v][1];
+ if (v >= 16) {
+ buffer[--bufLen] = repr[v][0];
+ }
+ return bufLen;
+}
+
+/**
+ * Convert an unsigned to hex, using lower-case letters for the digits
+ * above 9. See the comments for uintToHex.
+ */
+template <class Uint>
+inline size_t uintToHexLower(char* buffer, size_t bufLen, Uint v) {
+ return uintToHex(buffer, bufLen, v, formatHexLower);
+}
+
+/**
+ * Convert an unsigned to hex, using upper-case letters for the digits
+ * above 9. See the comments for uintToHex.
+ */
+template <class Uint>
+inline size_t uintToHexUpper(char* buffer, size_t bufLen, Uint v) {
+ return uintToHex(buffer, bufLen, v, formatHexUpper);
+}
+
+/**
+ * Convert an unsigned to octal.
+ *
+ * Just like folly::detail::uintToBuffer in Conv.h, writes at the *end* of
+ * the supplied buffer and returns the offset of the beginning of the string
+ * from the start of the buffer. The formatted string will be in range
+ * [buf+begin, buf+bufLen).
+ */
+template <class Uint>
+size_t uintToOctal(char* buffer, size_t bufLen, Uint v) {
+ auto& repr = formatOctal;
+ for (; v >= 512; v >>= 9) {
+ auto b = v & 0x1ff;
+ bufLen -= 3;
+ buffer[bufLen] = repr[b][0];
+ buffer[bufLen + 1] = repr[b][1];
+ buffer[bufLen + 2] = repr[b][2];
+ }
+ buffer[--bufLen] = repr[v][2];
+ if (v >= 8) {
+ buffer[--bufLen] = repr[v][1];
+ }
+ if (v >= 64) {
+ buffer[--bufLen] = repr[v][0];
+ }
+ return bufLen;
+}
+
+/**
+ * Convert an unsigned to binary.
+ *
+ * Just like folly::detail::uintToBuffer in Conv.h, writes at the *end* of
+ * the supplied buffer and returns the offset of the beginning of the string
+ * from the start of the buffer. The formatted string will be in range
+ * [buf+begin, buf+bufLen).
+ */
+template <class Uint>
+size_t uintToBinary(char* buffer, size_t bufLen, Uint v) {
+ auto& repr = formatBinary;
+ if (v == 0) {
+ buffer[--bufLen] = '0';
+ return bufLen;
+ }
+ for (; v; v >>= 8) {
+ auto b = v & 0xff;
+ bufLen -= 8;
+ memcpy(buffer + bufLen, &(repr[b][0]), 8);
+ }
+ while (buffer[bufLen] == '0') {
+ ++bufLen;
+ }
+ return bufLen;
+}
+
+} // namespace detail
+
+
+template <bool containerMode, class... Args>
+Formatter<containerMode, Args...>::Formatter(StringPiece str, Args&&... args)
+ : str_(str),
+ values_(FormatValue<typename std::decay<Args>::type>(
+ std::forward<Args>(args))...) {
+ static_assert(!containerMode || sizeof...(Args) == 1,
+ "Exactly one argument required in container mode");
+}
+
+template <bool containerMode, class... Args>
+template <class Output>
+void Formatter<containerMode, Args...>::operator()(Output& out) const {
+ auto p = str_.begin();
+ auto end = str_.end();
+
+ // Copy raw string (without format specifiers) to output;
+ // not as simple as we'd like, as we still need to translate "}}" to "}"
+ // and throw if we see any lone "}"
+ auto outputString = [&out] (StringPiece s) {
+ auto p = s.begin();
+ auto end = s.end();
+ while (p != end) {
+ auto q = static_cast<const char*>(memchr(p, '}', end - p));
+ if (!q) {
+ out(StringPiece(p, end));
+ break;
+ }
+ ++q;
+ out(StringPiece(p, q));
+ p = q;
+
+ if (p == end || *p != '}') {
+ throw std::invalid_argument(
+ "folly::format: single '}' in format string");
+ }
+ ++p;
+ }
+ };
+
+ int nextArg = 0;
+ bool hasDefaultArgIndex = false;
+ bool hasExplicitArgIndex = false;
+ while (p != end) {
+ auto q = static_cast<const char*>(memchr(p, '{', end - p));
+ if (!q) {
+ outputString(StringPiece(p, end));
+ break;
+ }
+ outputString(StringPiece(p, q));
+ p = q + 1;
+
+ if (p == end) {
+ throw std::invalid_argument(
+ "folly::format: '}' at end of format string");
+ }
+
+ // "{{" -> "{"
+ if (*p == '{') {
+ out(StringPiece(p, 1));
+ ++p;
+ continue;
+ }
+
+ // Format string
+ q = static_cast<const char*>(memchr(p, '}', end - p));
+ if (q == end) {
+ throw std::invalid_argument("folly::format: missing ending '}'");
+ }
+ FormatArg arg(StringPiece(p, q));
+ p = q + 1;
+
+ int argIndex = 0;
+ auto piece = arg.splitKey<true>(); // empty key component is okay
+ if (containerMode) { // static
+ if (piece.empty()) {
+ arg.setNextIntKey(nextArg++);
+ hasDefaultArgIndex = true;
+ } else {
+ arg.setNextKey(piece);
+ hasExplicitArgIndex = true;
+ }
+ } else {
+ if (piece.empty()) {
+ argIndex = nextArg++;
+ hasDefaultArgIndex = true;
+ } else {
+ try {
+ argIndex = to<int>(piece);
+ } catch (const std::out_of_range& e) {
+ arg.error("argument index must be integer");
+ }
+ arg.enforce(argIndex >= 0, "argument index must be non-negative");
+ hasExplicitArgIndex = true;
+ }
+ }
+
+ if (hasDefaultArgIndex && hasExplicitArgIndex) {
+ throw std::invalid_argument(
+ "folly::format: may not have both default and explicit arg indexes");
+ }
+
+ doFormat(argIndex, arg, out);
+ }
+}
+
+namespace format_value {
+
+template <class FormatCallback>
+void formatString(StringPiece val, FormatArg& arg, FormatCallback& cb) {
+ if (arg.precision != FormatArg::kDefaultPrecision &&
+ val.size() > arg.precision) {
+ val.reset(val.data(), arg.precision);
+ }
+
+ constexpr int padBufSize = 128;
+ char padBuf[padBufSize];
+
+ // Output padding, no more than padBufSize at once
+ auto pad = [&padBuf, &cb, padBufSize] (int chars) {
+ while (chars) {
+ int n = std::min(chars, padBufSize);
+ cb(StringPiece(padBuf, n));
+ chars -= n;
+ }
+ };
+
+ int padRemaining = 0;
+ if (arg.width != FormatArg::kDefaultWidth && val.size() < arg.width) {
+ char fill = arg.fill == FormatArg::kDefaultFill ? ' ' : arg.fill;
+ int padChars = arg.width - val.size();
+ memset(padBuf, fill, std::min(padBufSize, padChars));
+
+ switch (arg.align) {
+ case FormatArg::Align::DEFAULT:
+ case FormatArg::Align::LEFT:
+ padRemaining = padChars;
+ break;
+ case FormatArg::Align::CENTER:
+ pad(padChars / 2);
+ padRemaining = padChars - padChars / 2;
+ break;
+ case FormatArg::Align::RIGHT:
+ case FormatArg::Align::PAD_AFTER_SIGN:
+ pad(padChars);
+ break;
+ default:
+ abort();
+ break;
+ }
+ }
+
+ cb(val);
+
+ if (padRemaining) {
+ pad(padRemaining);
+ }
+}
+
+template <class FormatCallback>
+void formatNumber(StringPiece val, int prefixLen, FormatArg& arg,
+ FormatCallback& cb) {
+ // precision means something different for numbers
+ arg.precision = FormatArg::kDefaultPrecision;
+ if (arg.align == FormatArg::Align::DEFAULT) {
+ arg.align = FormatArg::Align::RIGHT;
+ } else if (prefixLen && arg.align == FormatArg::Align::PAD_AFTER_SIGN) {
+ // Split off the prefix, then do any padding if necessary
+ cb(val.subpiece(0, prefixLen));
+ val.advance(prefixLen);
+ arg.width = std::max(arg.width - prefixLen, 0);
+ }
+ format_value::formatString(val, arg, cb);
+}
+
+template <class FormatCallback, bool containerMode, class... Args>
+void formatFormatter(const Formatter<containerMode, Args...>& formatter,
+ FormatArg& arg,
+ FormatCallback& cb) {
+ if (arg.width == FormatArg::kDefaultWidth &&
+ arg.precision == FormatArg::kDefaultPrecision) {
+ // nothing to do
+ formatter(cb);
+ } else if (arg.align != FormatArg::Align::LEFT &&
+ arg.align != FormatArg::Align::DEFAULT) {
+ // We can only avoid creating a temporary string if we align left,
+ // as we'd need to know the size beforehand otherwise
+ format_value::formatString(formatter.fbstr(), arg, cb);
+ } else {
+ auto fn = [&arg, &cb] (StringPiece sp) mutable {
+ int sz = static_cast<int>(sp.size());
+ if (arg.precision != FormatArg::kDefaultPrecision) {
+ sz = std::min(arg.precision, sz);
+ sp.reset(sp.data(), sz);
+ arg.precision -= sz;
+ }
+ if (!sp.empty()) {
+ cb(sp);
+ if (arg.width != FormatArg::kDefaultWidth) {
+ arg.width = std::max(arg.width - sz, 0);
+ }
+ }
+ };
+ formatter(fn);
+ if (arg.width != FormatArg::kDefaultWidth && arg.width != 0) {
+ // Rely on formatString to do appropriate padding
+ format_value::formatString(StringPiece(), arg, cb);
+ }
+ }
+}
+
+} // namespace format_value
+
+// Definitions for default FormatValue classes
+
+// Integral types (except bool)
+template <class T>
+class FormatValue<
+ T, typename std::enable_if<
+ std::is_integral<T>::value &&
+ !std::is_same<T, bool>::value>::type>
+ {
+ public:
+ explicit FormatValue(T val) : val_(val) { }
+ template <class FormatCallback>
+ void format(FormatArg& arg, FormatCallback& cb) const {
+ arg.validate(FormatArg::Type::INTEGER);
+ doFormat(arg, cb);
+ }
+
+ template <class FormatCallback>
+ void doFormat(FormatArg& arg, FormatCallback& cb) const {
+ char presentation = arg.presentation;
+ if (presentation == FormatArg::kDefaultPresentation) {
+ presentation = std::is_same<T, char>::value ? 'c' : 'd';
+ }
+
+ // Do all work as unsigned, we'll add the prefix ('0' or '0x' if necessary)
+ // and sign ourselves.
+ typedef typename std::make_unsigned<T>::type UT;
+ UT uval;
+ char sign;
+ if (std::is_signed<T>::value) {
+ if (val_ < 0) {
+ uval = static_cast<UT>(-val_);
+ sign = '-';
+ } else {
+ uval = static_cast<UT>(val_);
+ switch (arg.sign) {
+ case FormatArg::Sign::PLUS_OR_MINUS:
+ sign = '+';
+ break;
+ case FormatArg::Sign::SPACE_OR_MINUS:
+ sign = ' ';
+ break;
+ default:
+ sign = '\0';
+ break;
+ }
+ }
+ } else {
+ uval = val_;
+ sign = '\0';
+
+ arg.enforce(arg.sign == FormatArg::Sign::DEFAULT,
+ "sign specifications not allowed for unsigned values");
+ }
+
+ // max of:
+ // #x: 0x prefix + 16 bytes = 18 bytes
+ // #o: 0 prefix + 22 bytes = 23 bytes
+ // #b: 0b prefix + 64 bytes = 65 bytes
+ // ,d: 26 bytes (including thousands separators!)
+ // + nul terminator
+ // + 3 for sign and prefix shenanigans (see below)
+ constexpr size_t valBufSize = 69;
+ char valBuf[valBufSize];
+ char* valBufBegin = nullptr;
+ char* valBufEnd = nullptr;
+
+ // Defer to sprintf
+ auto useSprintf = [&] (const char* format) mutable {
+ valBufBegin = valBuf + 3; // room for sign and base prefix
+ valBufEnd = valBufBegin + sprintf(valBufBegin, format,
+ static_cast<uintmax_t>(uval));
+ };
+
+ int prefixLen = 0;
+
+ switch (presentation) {
+ case 'n': // TODO(tudorb): locale awareness?
+ case 'd':
+ arg.enforce(!arg.basePrefix,
+ "base prefix not allowed with '", presentation,
+ "' specifier");
+ if (arg.thousandsSeparator) {
+ useSprintf("%'ju");
+ } else {
+ // Use uintToBuffer, faster than sprintf
+ valBufEnd = valBuf + valBufSize - 1;
+ valBufBegin = valBuf + detail::uintToBuffer(valBuf, valBufSize - 1,
+ uval);
+ }
+ break;
+ case 'c':
+ arg.enforce(!arg.basePrefix,
+ "base prefix not allowed with '", presentation,
+ "' specifier");
+ arg.enforce(!arg.thousandsSeparator,
+ "thousands separator (',') not allowed with '",
+ presentation, "' specifier");
+ valBufBegin = valBuf + 3;
+ *valBufBegin = static_cast<char>(uval);
+ valBufEnd = valBufBegin + 1;
+ break;
+ case 'o':
+ case 'O':
+ arg.enforce(!arg.thousandsSeparator,
+ "thousands separator (',') not allowed with '",
+ presentation, "' specifier");
+ valBufEnd = valBuf + valBufSize - 1;
+ valBufBegin = valBuf + detail::uintToOctal(valBuf, valBufSize - 1, uval);
+ if (arg.basePrefix) {
+ *--valBufBegin = '0';
+ prefixLen = 1;
+ }
+ break;
+ case 'x':
+ arg.enforce(!arg.thousandsSeparator,
+ "thousands separator (',') not allowed with '",
+ presentation, "' specifier");
+ valBufEnd = valBuf + valBufSize - 1;
+ valBufBegin = valBuf + detail::uintToHexLower(valBuf, valBufSize - 1,
+ uval);
+ if (arg.basePrefix) {
+ *--valBufBegin = 'x';
+ *--valBufBegin = '0';
+ prefixLen = 2;
+ }
+ break;
+ case 'X':
+ arg.enforce(!arg.thousandsSeparator,
+ "thousands separator (',') not allowed with '",
+ presentation, "' specifier");
+ valBufEnd = valBuf + valBufSize - 1;
+ valBufBegin = valBuf + detail::uintToHexUpper(valBuf, valBufSize - 1,
+ uval);
+ if (arg.basePrefix) {
+ *--valBufBegin = 'X';
+ *--valBufBegin = '0';
+ prefixLen = 2;
+ }
+ break;
+ case 'b':
+ case 'B':
+ arg.enforce(!arg.thousandsSeparator,
+ "thousands separator (',') not allowed with '",
+ presentation, "' specifier");
+ valBufEnd = valBuf + valBufSize - 1;
+ valBufBegin = valBuf + detail::uintToBinary(valBuf, valBufSize - 1,
+ uval);
+ if (arg.basePrefix) {
+ *--valBufBegin = presentation; // 0b or 0B
+ *--valBufBegin = '0';
+ prefixLen = 2;
+ }
+ break;
+ default:
+ arg.error("invalid specifier '", presentation, "'");
+ }
+
+ if (sign) {
+ *--valBufBegin = sign;
+ ++prefixLen;
+ }
+
+ format_value::formatNumber(StringPiece(valBufBegin, valBufEnd), prefixLen,
+ arg, cb);
+ }
+
+ private:
+ T val_;
+};
+
+// Bool
+template <>
+class FormatValue<bool> {
+ public:
+ explicit FormatValue(bool val) : val_(val) { }
+
+ template <class FormatCallback>
+ void format(FormatArg& arg, FormatCallback& cb) const {
+ if (arg.presentation == FormatArg::kDefaultPresentation) {
+ arg.validate(FormatArg::Type::OTHER);
+ format_value::formatString(val_ ? "true" : "false", arg, cb);
+ } else { // number
+ FormatValue<int>(val_).format(arg, cb);
+ }
+ }
+
+ private:
+ bool val_;
+};
+
+// double
+template <>
+class FormatValue<double> {
+ public:
+ explicit FormatValue(double val) : val_(val) { }
+
+ template <class FormatCallback>
+ void format(FormatArg& arg, FormatCallback& cb) const {
+ using ::double_conversion::DoubleToStringConverter;
+ using ::double_conversion::StringBuilder;
+
+ arg.validate(FormatArg::Type::FLOAT);
+
+ if (arg.presentation == FormatArg::kDefaultPresentation) {
+ arg.presentation = 'g';
+ }
+
+ const char* infinitySymbol = isupper(arg.presentation) ? "INF" : "inf";
+ const char* nanSymbol = isupper(arg.presentation) ? "NAN" : "nan";
+ char exponentSymbol = isupper(arg.presentation) ? 'E' : 'e';
+
+ if (arg.precision == FormatArg::kDefaultPrecision) {
+ arg.precision = 6;
+ }
+
+ bool done = false;
+
+ // 2+: for null terminator and optional sign shenanigans.
+ char buf[2 + std::max({
+ (2 + DoubleToStringConverter::kMaxFixedDigitsBeforePoint +
+ DoubleToStringConverter::kMaxFixedDigitsAfterPoint),
+ (8 + DoubleToStringConverter::kMaxExponentialDigits),
+ (7 + DoubleToStringConverter::kMaxPrecisionDigits)})];
+ StringBuilder builder(buf + 1, sizeof(buf) - 1);
+
+ char plusSign;
+ switch (arg.sign) {
+ case FormatArg::Sign::PLUS_OR_MINUS:
+ plusSign = '+';
+ break;
+ case FormatArg::Sign::SPACE_OR_MINUS:
+ plusSign = ' ';
+ break;
+ default:
+ plusSign = '\0';
+ break;
+ };
+
+ double val = val_;
+ switch (arg.presentation) {
+ case '%':
+ val *= 100;
+ case 'f':
+ case 'F':
+ {
+ if (arg.precision >
+ DoubleToStringConverter::kMaxFixedDigitsAfterPoint) {
+ arg.precision = DoubleToStringConverter::kMaxFixedDigitsAfterPoint;
+ }
+ DoubleToStringConverter conv(
+ DoubleToStringConverter::EMIT_POSITIVE_EXPONENT_SIGN,
+ infinitySymbol,
+ nanSymbol,
+ exponentSymbol,
+ -4, arg.precision,
+ 0, 0);
+ arg.enforce(conv.ToFixed(val, arg.precision, &builder),
+ "fixed double conversion failed");
+ }
+ break;
+ case 'e':
+ case 'E':
+ {
+ if (arg.precision > DoubleToStringConverter::kMaxExponentialDigits) {
+ arg.precision = DoubleToStringConverter::kMaxExponentialDigits;
+ }
+
+ DoubleToStringConverter conv(
+ DoubleToStringConverter::EMIT_POSITIVE_EXPONENT_SIGN,
+ infinitySymbol,
+ nanSymbol,
+ exponentSymbol,
+ -4, arg.precision,
+ 0, 0);
+ CHECK(conv.ToExponential(val, arg.precision, &builder));
+ }
+ break;
+ case 'n': // should be locale-aware, but isn't
+ case 'g':
+ case 'G':
+ {
+ if (arg.precision < DoubleToStringConverter::kMinPrecisionDigits) {
+ arg.precision = DoubleToStringConverter::kMinPrecisionDigits;
+ } else if (arg.precision >
+ DoubleToStringConverter::kMaxPrecisionDigits) {
+ arg.precision = DoubleToStringConverter::kMaxPrecisionDigits;
+ }
+ DoubleToStringConverter conv(
+ DoubleToStringConverter::EMIT_POSITIVE_EXPONENT_SIGN,
+ infinitySymbol,
+ nanSymbol,
+ exponentSymbol,
+ -4, arg.precision,
+ 0, 0);
+ CHECK(conv.ToShortest(val, &builder));
+ }
+ break;
+ default:
+ arg.error("invalid specifier '", arg.presentation, "'");
+ }
+
+ int len = builder.position();
+ builder.Finalize();
+ DCHECK_GT(len, 0);
+
+ // Add '+' or ' ' sign if needed
+ char* p = buf + 1;
+ // anything that's neither negative nor nan
+ int prefixLen = 0;
+ if (plusSign && (*p != '-' && *p != 'n' && *p != 'N')) {
+ *--p = plusSign;
+ ++len;
+ prefixLen = 1;
+ } else if (*p == '-') {
+ prefixLen = 1;
+ }
+
+ format_value::formatNumber(StringPiece(p, len), prefixLen, arg, cb);
+ }
+
+ private:
+ double val_;
+};
+
+// float (defer to double)
+template <>
+class FormatValue<float> {
+ public:
+ explicit FormatValue(float val) : val_(val) { }
+
+ template <class FormatCallback>
+ void format(FormatArg& arg, FormatCallback& cb) const {
+ FormatValue<double>(val_).format(arg, cb);
+ }
+
+ private:
+ float val_;
+};
+
+// Sring-y types (implicitly convertible to StringPiece, except char*)
+template <class T>
+class FormatValue<
+ T, typename std::enable_if<
+ (!std::is_pointer<T>::value ||
+ !std::is_same<char, typename std::decay<
+ typename std::remove_pointer<T>::type>::type>::value) &&
+ std::is_convertible<T, StringPiece>::value>::type>
+ {
+ public:
+ explicit FormatValue(StringPiece val) : val_(val) { }
+
+ template <class FormatCallback>
+ void format(FormatArg& arg, FormatCallback& cb) const {
+ if (arg.keyEmpty()) {
+ arg.validate(FormatArg::Type::OTHER);
+ arg.enforce(arg.presentation == FormatArg::kDefaultPresentation ||
+ arg.presentation == 's',
+ "invalid specifier '", arg.presentation, "'");
+ format_value::formatString(val_, arg, cb);
+ } else {
+ FormatValue<char>(val_.at(arg.splitIntKey())).format(arg, cb);
+ }
+ }
+
+ private:
+ StringPiece val_;
+};
+
+// Null
+template <>
+class FormatValue<std::nullptr_t> {
+ public:
+ explicit FormatValue(std::nullptr_t) { }
+
+ template <class FormatCallback>
+ void format(FormatArg& arg, FormatCallback& cb) const {
+ arg.validate(FormatArg::Type::OTHER);
+ arg.enforce(arg.presentation == FormatArg::kDefaultPresentation,
+ "invalid specifier '", arg.presentation, "'");
+ format_value::formatString("(null)", arg, cb);
+ }
+};
+
+// Partial specialization of FormatValue for char*
+template <class T>
+class FormatValue<
+ T*,
+ typename std::enable_if<
+ std::is_same<char, typename std::decay<T>::type>::value>::type>
+ {
+ public:
+ explicit FormatValue(T* val) : val_(val) { }
+
+ template <class FormatCallback>
+ void format(FormatArg& arg, FormatCallback& cb) const {
+ if (arg.keyEmpty()) {
+ if (!val_) {
+ FormatValue<std::nullptr_t>(nullptr).format(arg, cb);
+ } else {
+ FormatValue<StringPiece>(val_).format(arg, cb);
+ }
+ } else {
+ FormatValue<typename std::decay<T>::type>(
+ val_[arg.splitIntKey()]).format(arg, cb);
+ }
+ }
+
+ private:
+ T* val_;
+};
+
+// Partial specialization of FormatValue for void*
+template <class T>
+class FormatValue<
+ T*,
+ typename std::enable_if<
+ std::is_same<void, typename std::decay<T>::type>::value>::type>
+ {
+ public:
+ explicit FormatValue(T* val) : val_(val) { }
+
+ template <class FormatCallback>
+ void format(FormatArg& arg, FormatCallback& cb) const {
+ if (!val_) {
+ FormatValue<std::nullptr_t>(nullptr).format(arg, cb);
+ } else {
+ // Print as a pointer, in hex.
+ arg.validate(FormatArg::Type::OTHER);
+ arg.enforce(arg.presentation == FormatArg::kDefaultPresentation,
+ "invalid specifier '", arg.presentation, "'");
+ arg.basePrefix = true;
+ arg.presentation = 'x';
+ if (arg.align == FormatArg::Align::DEFAULT) {
+ arg.align = FormatArg::Align::LEFT;
+ }
+ FormatValue<uintptr_t>(
+ reinterpret_cast<uintptr_t>(val_)).doFormat(arg, cb);
+ }
+ }
+
+ private:
+ T* val_;
+};
+
+// Partial specialization of FormatValue for other pointers
+template <class T>
+class FormatValue<
+ T*,
+ typename std::enable_if<
+ !std::is_same<char, typename std::decay<T>::type>::value &&
+ !std::is_same<void, typename std::decay<T>::type>::value>::type>
+ {
+ public:
+ explicit FormatValue(T* val) : val_(val) { }
+
+ template <class FormatCallback>
+ void format(FormatArg& arg, FormatCallback& cb) const {
+ if (arg.keyEmpty()) {
+ FormatValue<void*>((void*)val_).format(arg, cb);
+ } else {
+ FormatValue<typename std::decay<T>::type>(
+ val_[arg.splitIntKey()]).format(arg, cb);
+ }
+ }
+ private:
+ T* val_;
+};
+
+namespace detail {
+
+// Shortcut, so we don't have to use enable_if everywhere
+struct FormatTraitsBase {
+ typedef void enabled;
+};
+
+// Traits that define enabled, value_type, and at() for anything
+// indexable with integral keys: pointers, arrays, vectors, and maps
+// with integral keys
+template <class T, class Enable=void> struct IndexableTraits;
+
+// Base class for sequences (vectors, deques)
+template <class C>
+struct IndexableTraitsSeq : public FormatTraitsBase {
+ typedef C container_type;
+ typedef typename C::value_type value_type;
+ static const value_type& at(const C& c, int idx) {
+ return c.at(idx);
+ }
+};
+
+// Base class for associative types (maps)
+template <class C>
+struct IndexableTraitsAssoc : public FormatTraitsBase {
+ typedef typename C::value_type::second_type value_type;
+ static const value_type& at(const C& c, int idx) {
+ return c.at(static_cast<typename C::key_type>(idx));
+ }
+};
+
+// std::array
+template <class T, size_t N>
+struct IndexableTraits<std::array<T, N>>
+ : public IndexableTraitsSeq<std::array<T, N>> {
+};
+
+// std::vector
+template <class T, class A>
+struct IndexableTraits<std::vector<T, A>>
+ : public IndexableTraitsSeq<std::vector<T, A>> {
+};
+
+// std::deque
+template <class T, class A>
+struct IndexableTraits<std::deque<T, A>>
+ : public IndexableTraitsSeq<std::deque<T, A>> {
+};
+
+// fbvector
+template <class T, class A>
+struct IndexableTraits<fbvector<T, A>>
+ : public IndexableTraitsSeq<fbvector<T, A>> {
+};
+
+// small_vector
+template <class T, size_t M, class A, class B, class C>
+struct IndexableTraits<small_vector<T, M, A, B, C>>
+ : public IndexableTraitsSeq<small_vector<T, M, A, B, C>> {
+};
+
+// std::map with integral keys
+template <class K, class T, class C, class A>
+struct IndexableTraits<
+ std::map<K, T, C, A>,
+ typename std::enable_if<std::is_integral<K>::value>::type>
+ : public IndexableTraitsAssoc<std::map<K, T, C, A>> {
+};
+
+// std::unordered_map with integral keys
+template <class K, class T, class H, class E, class A>
+struct IndexableTraits<
+ std::unordered_map<K, T, H, E, A>,
+ typename std::enable_if<std::is_integral<K>::value>::type>
+ : public IndexableTraitsAssoc<std::unordered_map<K, T, H, E, A>> {
+};
+
+} // namespace detail
+
+// Partial specialization of FormatValue for integer-indexable containers
+template <class T>
+class FormatValue<
+ T,
+ typename detail::IndexableTraits<T>::enabled> {
+ public:
+ explicit FormatValue(const T& val) : val_(val) { }
+
+ template <class FormatCallback>
+ void format(FormatArg& arg, FormatCallback& cb) const {
+ FormatValue<typename std::decay<
+ typename detail::IndexableTraits<T>::value_type>::type>(
+ detail::IndexableTraits<T>::at(
+ val_, arg.splitIntKey())).format(arg, cb);
+ }
+
+ private:
+ const T& val_;
+};
+
+namespace detail {
+
+// Define enabled, key_type, convert from StringPiece to the key types
+// that we support
+template <class T> struct KeyFromStringPiece;
+
+// std::string
+template <>
+struct KeyFromStringPiece<std::string> : public FormatTraitsBase {
+ typedef std::string key_type;
+ static std::string convert(StringPiece s) {
+ return s.toString();
+ }
+ typedef void enabled;
+};
+
+// fbstring
+template <>
+struct KeyFromStringPiece<fbstring> : public FormatTraitsBase {
+ typedef fbstring key_type;
+ static fbstring convert(StringPiece s) {
+ return s.toFbstring();
+ }
+};
+
+// StringPiece
+template <>
+struct KeyFromStringPiece<StringPiece> : public FormatTraitsBase {
+ typedef StringPiece key_type;
+ static StringPiece convert(StringPiece s) {
+ return s;
+ }
+};
+
+// Base class for associative types keyed by strings
+template <class T> struct KeyableTraitsAssoc : public FormatTraitsBase {
+ typedef typename T::key_type key_type;
+ typedef typename T::value_type::second_type value_type;
+ static const value_type& at(const T& map, StringPiece key) {
+ return map.at(KeyFromStringPiece<key_type>::convert(key));
+ }
+};
+
+// Define enabled, key_type, value_type, at() for supported string-keyed
+// types
+template <class T, class Enabled=void> struct KeyableTraits;
+
+// std::map with string key
+template <class K, class T, class C, class A>
+struct KeyableTraits<
+ std::map<K, T, C, A>,
+ typename KeyFromStringPiece<K>::enabled>
+ : public KeyableTraitsAssoc<std::map<K, T, C, A>> {
+};
+
+// std::unordered_map with string key
+template <class K, class T, class H, class E, class A>
+struct KeyableTraits<
+ std::unordered_map<K, T, H, E, A>,
+ typename KeyFromStringPiece<K>::enabled>
+ : public KeyableTraitsAssoc<std::unordered_map<K, T, H, E, A>> {
+};
+
+} // namespace detail
+
+// Partial specialization of FormatValue for string-keyed containers
+template <class T>
+class FormatValue<
+ T,
+ typename detail::KeyableTraits<T>::enabled> {
+ public:
+ explicit FormatValue(const T& val) : val_(val) { }
+
+ template <class FormatCallback>
+ void format(FormatArg& arg, FormatCallback& cb) const {
+ FormatValue<typename std::decay<
+ typename detail::KeyableTraits<T>::value_type>::type>(
+ detail::KeyableTraits<T>::at(
+ val_, arg.splitKey())).format(arg, cb);
+ }
+
+ private:
+ const T& val_;
+};
+
+// Partial specialization of FormatValue for pairs
+template <class A, class B>
+class FormatValue<std::pair<A, B>> {
+ public:
+ explicit FormatValue(const std::pair<A, B>& val) : val_(val) { }
+
+ template <class FormatCallback>
+ void format(FormatArg& arg, FormatCallback& cb) const {
+ int key = arg.splitIntKey();
+ switch (key) {
+ case 0:
+ FormatValue<typename std::decay<A>::type>(val_.first).format(arg, cb);
+ break;
+ case 1:
+ FormatValue<typename std::decay<B>::type>(val_.second).format(arg, cb);
+ break;
+ default:
+ arg.error("invalid index for pair");
+ }
+ }
+
+ private:
+ const std::pair<A, B>& val_;
+};
+
+// Partial specialization of FormatValue for tuples
+template <class... Args>
+class FormatValue<std::tuple<Args...>> {
+ typedef std::tuple<Args...> Tuple;
+ public:
+ explicit FormatValue(const Tuple& val) : val_(val) { }
+
+ template <class FormatCallback>
+ void format(FormatArg& arg, FormatCallback& cb) const {
+ int key = arg.splitIntKey();
+ arg.enforce(key >= 0, "tuple index must be non-negative");
+ doFormat(key, arg, cb);
+ }
+
+ private:
+ static constexpr size_t valueCount = std::tuple_size<Tuple>::value;
+
+ template <size_t K, class Callback>
+ typename std::enable_if<K == valueCount>::type
+ doFormatFrom(size_t i, FormatArg& arg, Callback& cb) const {
+ arg.enforce("tuple index out of range, max=", i);
+ }
+
+ template <size_t K, class Callback>
+ typename std::enable_if<(K < valueCount)>::type
+ doFormatFrom(size_t i, FormatArg& arg, Callback& cb) const {
+ if (i == K) {
+ FormatValue<typename std::decay<
+ typename std::tuple_element<K, Tuple>::type>::type>(
+ std::get<K>(val_)).format(arg, cb);
+ } else {
+ doFormatFrom<K+1>(i, arg, cb);
+ }
+ }
+
+ template <class Callback>
+ void doFormat(size_t i, FormatArg& arg, Callback& cb) const {
+ return doFormatFrom<0>(i, arg, cb);
+ }
+
+ const Tuple& val_;
+};
+
+/**
+ * Formatter objects can be appended to strings, and therefore they're
+ * compatible with folly::toAppend and folly::to.
+ */
+template <class Tgt, bool containerMode, class... Args>
+typename std::enable_if<
+ detail::IsSomeString<Tgt>::value>::type
+toAppend(const Formatter<containerMode, Args...>& value, Tgt * result) {
+ value.appendTo(*result);
+}
+
+} // namespace folly
--- /dev/null
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "folly/Format.h"
+
+namespace folly {
+namespace detail {
+
+extern const FormatArg::Align formatAlignTable[];
+extern const FormatArg::Sign formatSignTable[];
+
+} // namespace detail
+
+using namespace folly::detail;
+
+void FormatArg::initSlow() {
+ auto b = fullArgString.begin();
+ auto end = fullArgString.end();
+
+ // Parse key
+ auto p = static_cast<const char*>(memchr(b, ':', end - b));
+ if (!p) {
+ key_ = StringPiece(b, end);
+ return;
+ }
+ key_ = StringPiece(b, p);
+
+ if (*p == ':') {
+ // parse format spec
+ if (++p == end) return;
+
+ // fill/align, or just align
+ Align a;
+ if (p + 1 != end &&
+ (a = formatAlignTable[static_cast<unsigned char>(p[1])]) !=
+ Align::INVALID) {
+ fill = *p;
+ align = a;
+ p += 2;
+ if (p == end) return;
+ } else if ((a = formatAlignTable[static_cast<unsigned char>(*p)]) !=
+ Align::INVALID) {
+ align = a;
+ if (++p == end) return;
+ }
+
+ Sign s;
+ unsigned char uSign = static_cast<unsigned char>(*p);
+ if ((s = formatSignTable[uSign]) != Sign::INVALID) {
+ sign = s;
+ if (++p == end) return;
+ }
+
+ if (*p == '#') {
+ basePrefix = true;
+ if (++p == end) return;
+ }
+
+ if (*p == '0') {
+ enforce(align == Align::DEFAULT, "alignment specified twice");
+ fill = '0';
+ align = Align::PAD_AFTER_SIGN;
+ if (++p == end) return;
+ }
+
+ if (*p >= '0' && *p <= '9') {
+ auto b = p;
+ do {
+ ++p;
+ } while (p != end && *p >= '0' && *p <= '9');
+ width = to<int>(StringPiece(b, p));
+
+ if (p == end) return;
+ }
+
+ if (*p == ',') {
+ thousandsSeparator = true;
+ if (++p == end) return;
+ }
+
+ if (*p == '.') {
+ auto b = ++p;
+ while (p != end && *p >= '0' && *p <= '9') {
+ ++p;
+ }
+ precision = to<int>(StringPiece(b, p));
+
+ if (p == end) return;
+ }
+
+ presentation = *p;
+ if (++p == end) return;
+ }
+
+ error("extra characters in format string");
+}
+
+void FormatArg::validate(Type type) const {
+ enforce(keyEmpty(), "index not allowed");
+ switch (type) {
+ case Type::INTEGER:
+ enforce(precision == kDefaultPrecision,
+ "precision not allowed on integers");
+ break;
+ case Type::FLOAT:
+ enforce(!basePrefix,
+ "base prefix ('#') specifier only allowed on integers");
+ enforce(!thousandsSeparator,
+ "thousands separator (',') only allowed on integers");
+ break;
+ case Type::OTHER:
+ enforce(align != Align::PAD_AFTER_SIGN,
+ "'='alignment only allowed on numbers");
+ enforce(sign == Sign::DEFAULT,
+ "sign
projects / folly.git / commitdiff
