From: Tom Jackson Date: Wed, 29 Jan 2014 21:50:24 +0000 (-0800) Subject: Promoting out of experimental X-Git-Tag: v0.22.0~704 X-Git-Url: http://plrg.eecs.uci.edu/git/?a=commitdiff_plain;h=faf7b5c67ce08ec1fbdd847349bf6f95aa46dd86;p=folly.git Promoting out of experimental Summary: At long last, promoting this out of experimental. Also, while I'm at it, I've separated the tests and benchmarks into their corresponding parts. Redirect headers provided. Test Plan: Unit tests, contbuild. Reviewed By: marcelo.juchem@fb.com FB internal diff: D1151911 --- diff --git a/folly/experimental/CombineGen-inl.h b/folly/experimental/CombineGen-inl.h deleted file mode 100644 index 3b859df8..00000000 --- a/folly/experimental/CombineGen-inl.h +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Copyright 2014 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FOLLY_COMBINEGEN_H_ -#error This file may only be included from folly/experimental/CombineGen.h -#endif - -#include -#include -#include -#include - -namespace folly { -namespace gen { -namespace detail { - -/** - * Interleave - * - * Alternate values from a sequence with values from a sequence container. - * Stops once we run out of values from either source. - */ -template -class Interleave : public Operator> { - // see comment about copies in CopiedSource - const std::shared_ptr container_; - public: - explicit Interleave(Container container) - : container_(new Container(std::move(container))) {} - - template - class Generator : public GenImpl> { - Source source_; - const std::shared_ptr container_; - typedef const typename Container::value_type& ConstRefType; - - static_assert(std::is_same::value, - "Only matching types may be interleaved"); - public: - explicit Generator(Source source, - const std::shared_ptr container) - : source_(std::move(source)), - container_(container) { } - - template - bool apply(Handler&& handler) const { - auto iter = container_->begin(); - return source_.apply([&](const Value& value) -> bool { - if (iter == container_->end()) { - return false; - } - if (!handler(value)) { - return false; - } - if (!handler(*iter)) { - return false; - } - iter++; - return true; - }); - } - }; - - template> - Gen compose(GenImpl&& source) const { - return Gen(std::move(source.self()), container_); - } - - template> - Gen compose(const GenImpl& source) const { - return Gen(source.self(), container_); - } -}; - -/** - * Zip - * - * Combine inputs from Source with values from a sequence container by merging - * them into a tuple. - * - */ -template -class Zip : public Operator> { - // see comment about copies in CopiedSource - const std::shared_ptr container_; - public: - explicit Zip(Container container) - : container_(new Container(std::move(container))) {} - - template::type, - typename std::decay::type>> - class Generator : public GenImpl> { - Source source_; - const std::shared_ptr container_; - public: - explicit Generator(Source source, - const std::shared_ptr container) - : source_(std::move(source)), - container_(container) { } - - template - bool apply(Handler&& handler) const { - auto iter = container_->begin(); - return (source_.apply([&](Value1 value) -> bool { - if (iter == container_->end()) { - return false; - } - if (!handler(std::make_tuple(std::forward(value), *iter))) { - return false; - } - ++iter; - return true; - })); - } - }; - - template> - Gen compose(GenImpl&& source) const { - return Gen(std::move(source.self()), container_); - } - - template> - Gen compose(const GenImpl& source) const { - return Gen(source.self(), container_); - } -}; - -template -auto add_to_tuple(std::tuple t1, std::tuple t2) -> -std::tuple { - return std::tuple_cat(std::move(t1), std::move(t2)); -} - -template -auto add_to_tuple(std::tuple t1, Type2&& t2) -> -decltype(std::tuple_cat(std::move(t1), - std::make_tuple(std::forward(t2)))) { - return std::tuple_cat(std::move(t1), - std::make_tuple(std::forward(t2))); -} - -template -auto add_to_tuple(Type1&& t1, std::tuple t2) -> -decltype(std::tuple_cat(std::make_tuple(std::forward(t1)), - std::move(t2))) { - return std::tuple_cat(std::make_tuple(std::forward(t1)), - std::move(t2)); -} - -template -auto add_to_tuple(Type1&& t1, Type2&& t2) -> -decltype(std::make_tuple(std::forward(t1), - std::forward(t2))) { - return std::make_tuple(std::forward(t1), - std::forward(t2)); -} - -// Merges a 2-tuple into a single tuple (get<0> could already be a tuple) -class MergeTuples { - public: - template - auto operator()(Tuple&& value) const -> - decltype(add_to_tuple(std::get<0>(std::forward(value)), - std::get<1>(std::forward(value)))) { - static_assert(std::tuple_size< - typename std::remove_reference::type - >::value == 2, - "Can only merge tuples of size 2"); - return add_to_tuple(std::get<0>(std::forward(value)), - std::get<1>(std::forward(value))); - } -}; - -} // namespace detail - -static const detail::Map tuple_flatten; - -// TODO(mcurtiss): support zip() for N>1 operands. Because of variadic problems, -// this might not be easily possible until gcc4.8 is available. -template::type>> -Zip zip(Source&& source) { - return Zip(std::forward(source)); -} - -} // namespace gen -} // namespace folly diff --git a/folly/experimental/CombineGen.h b/folly/experimental/CombineGen.h index b3bc3bb0..82e5b0d9 100644 --- a/folly/experimental/CombineGen.h +++ b/folly/experimental/CombineGen.h @@ -13,35 +13,5 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#ifndef FOLLY_COMBINEGEN_H_ -#define FOLLY_COMBINEGEN_H_ - -#include "folly/experimental/Gen.h" - -namespace folly { -namespace gen { -namespace detail { - -template -class Interleave; - -template -class Zip; - -} // namespace detail - -template::type, - class Interleave = detail::Interleave> -Interleave interleave(Source2&& source2) { - return Interleave(std::forward(source2)); -} - -} // namespace gen -} // namespace folly - -#include "folly/experimental/CombineGen-inl.h" - -#endif /* FOLLY_COMBINEGEN_H_ */ - +#pragma GCC message "folly::gen has moved to folly/gen/*.h" +#include "folly/gen/Combine.h" diff --git a/folly/experimental/FileGen-inl.h b/folly/experimental/FileGen-inl.h deleted file mode 100644 index c3a7f94b..00000000 --- a/folly/experimental/FileGen-inl.h +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright 2014 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FOLLY_FILEGEN_H_ -#error This file may only be included from folly/experimental/FileGen.h -#endif - -#include - -#include "folly/experimental/StringGen.h" - -namespace folly { -namespace gen { -namespace detail { - -class FileReader : public GenImpl { - public: - FileReader(File file, std::unique_ptr buffer) - : file_(std::move(file)), - buffer_(std::move(buffer)) { - buffer_->clear(); - } - - template - bool apply(Body&& body) const { - for (;;) { - ssize_t n; - do { - n = ::read(file_.fd(), buffer_->writableTail(), buffer_->capacity()); - } while (n == -1 && errno == EINTR); - if (n == -1) { - throw std::system_error(errno, std::system_category(), "read failed"); - } - if (n == 0) { - return true; - } - if (!body(ByteRange(buffer_->tail(), n))) { - return false; - } - } - } - - // Technically, there could be infinite files (e.g. /dev/random), but people - // who open those can do so at their own risk. - static constexpr bool infinite = false; - - private: - File file_; - std::unique_ptr buffer_; -}; - -class FileWriter : public Operator { - public: - FileWriter(File file, std::unique_ptr buffer) - : file_(std::move(file)), - buffer_(std::move(buffer)) { - if (buffer_) { - buffer_->clear(); - } - } - - template - void compose(const GenImpl& source) const { - auto fn = [&](ByteRange v) { - if (!this->buffer_ || v.size() >= this->buffer_->capacity()) { - this->flushBuffer(); - this->write(v); - } else { - if (v.size() > this->buffer_->tailroom()) { - this->flushBuffer(); - } - memcpy(this->buffer_->writableTail(), v.data(), v.size()); - this->buffer_->append(v.size()); - } - }; - - // Iterate - source.foreach(std::move(fn)); - - flushBuffer(); - file_.close(); - } - - private: - void write(ByteRange v) const { - ssize_t n; - while (!v.empty()) { - do { - n = ::write(file_.fd(), v.data(), v.size()); - } while (n == -1 && errno == EINTR); - if (n == -1) { - throw std::system_error(errno, std::system_category(), - "write() failed"); - } - v.advance(n); - } - } - - void flushBuffer() const { - if (buffer_ && buffer_->length() != 0) { - write(ByteRange(buffer_->data(), buffer_->length())); - buffer_->clear(); - } - } - - mutable File file_; - std::unique_ptr buffer_; -}; - -} // !detail - -/** - * Generator which reads lines from a file. - * Note: This produces StringPieces which reference temporary strings which are - * only valid during iteration. - */ -inline auto byLine(File file, char delim = '\n') - -> decltype(fromFile(std::move(file)) - | eachAs() - | resplit(delim)) { - return fromFile(std::move(file)) - | eachAs() - | resplit(delim); -} - -inline auto byLine(int fd, char delim = '\n') - -> decltype(byLine(File(fd), delim)) { return byLine(File(fd), delim); } - -inline auto byLine(const char* f, char delim = '\n') - -> decltype(byLine(File(f), delim)) { return byLine(File(f), delim); } - -}} // !folly::gen diff --git a/folly/experimental/FileGen.h b/folly/experimental/FileGen.h index e9ecfa76..7b8ac658 100644 --- a/folly/experimental/FileGen.h +++ b/folly/experimental/FileGen.h @@ -13,61 +13,5 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#ifndef FOLLY_FILEGEN_H_ -#define FOLLY_FILEGEN_H_ - -#include "folly/File.h" -#include "folly/experimental/Gen.h" -#include "folly/io/IOBuf.h" - -namespace folly { -namespace gen { - -namespace detail { -class FileReader; -class FileWriter; -} // namespace detail - -/** - * Generator that reads from a file with a buffer of the given size. - * Reads must be buffered (the generator interface expects the generator - * to hold each value). - */ -template -S fromFile(File file, size_t bufferSize=4096) { - return S(std::move(file), IOBuf::create(bufferSize)); -} - -/** - * Generator that reads from a file using a given buffer. - */ -template -S fromFile(File file, std::unique_ptr buffer) { - return S(std::move(file), std::move(buffer)); -} - -/** - * Sink that writes to a file with a buffer of the given size. - * If bufferSize is 0, writes will be unbuffered. - */ -template -S toFile(File file, size_t bufferSize=4096) { - return S(std::move(file), bufferSize ? nullptr : IOBuf::create(bufferSize)); -} - -/** - * Sink that writes to a file using a given buffer. - * If the buffer is nullptr, writes will be unbuffered. - */ -template -S toFile(File file, std::unique_ptr buffer) { - return S(std::move(file), std::move(buffer)); -} - -}} // !folly::gen - -#include "folly/experimental/FileGen-inl.h" - -#endif /* FOLLY_FILEGEN_H_ */ - +#pragma message "folly::gen has moved to folly/gen/*.h" +#include "folly/gen/File.h" diff --git a/folly/experimental/Gen-inl.h b/folly/experimental/Gen-inl.h deleted file mode 100644 index c21529d8..00000000 --- a/folly/experimental/Gen-inl.h +++ /dev/null @@ -1,2150 +0,0 @@ -/* - * Copyright 2014 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Ignore shadowing warnings within this file, so includers can use -Wshadow. -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wshadow" - -namespace folly { namespace gen { - -/** - * IsCompatibleSignature - Trait type for testing whether a given Functor - * matches an expected signature. - * - * Usage: - * IsCompatibleSignature::value - */ -template -class IsCompatibleSignature { - static constexpr bool value = false; -}; - -template -class IsCompatibleSignature { - template()(std::declval()...)), - bool good = std::is_same::value> - static constexpr bool testArgs(int* p) { - return good; - } - - template - static constexpr bool testArgs(...) { - return false; - } -public: - static constexpr bool value = testArgs(nullptr); -}; - -/** - * ArgumentReference - For determining ideal argument type to receive a value. - */ -template -struct ArgumentReference : - public std::conditional::value, - T, // T& -> T&, T&& -> T&&, const T& -> const T& - typename std::conditional< - std::is_const::value, - T&, // const int -> const int& - T&& // int -> int&& - >::type> {}; - -/** - * FBounded - Helper type for the curiously recurring template pattern, used - * heavily here to enable inlining and obviate virtual functions - */ -template -struct FBounded { - const Self& self() const { - return *static_cast(this); - } - - Self& self() { - return *static_cast(this); - } -}; - -/** - * Operator - Core abstraction of an operation which may be applied to a - * generator. All operators implement a method compose(), which takes a - * generator and produces an output generator. - */ -template -class Operator : public FBounded { - public: - /** - * compose() - Must be implemented by child class to compose a new Generator - * out of a given generator. This function left intentionally unimplemented. - */ - template - ResultGen compose(const GenImpl& source) const; - - protected: - Operator() = default; - Operator(const Operator&) = default; - Operator(Operator&&) = default; -}; - -/** - * operator|() - For composing two operators without binding it to a - * particular generator. - */ -template> -Composed operator|(const Operator& left, - const Operator& right) { - return Composed(left.self(), right.self()); -} - -template> -Composed operator|(const Operator& left, - Operator&& right) { - return Composed(left.self(), std::move(right.self())); -} - -template> -Composed operator|(Operator&& left, - const Operator& right) { - return Composed(std::move(left.self()), right.self()); -} - -template> -Composed operator|(Operator&& left, - Operator&& right) { - return Composed(std::move(left.self()), std::move(right.self())); -} - -/** - * GenImpl - Core abstraction of a generator, an object which produces values by - * passing them to a given handler lambda. All generator implementations must - * implement apply(). foreach() may also be implemented to special case the - * condition where the entire sequence is consumed. - */ -template -class GenImpl : public FBounded { - protected: - // To prevent slicing - GenImpl() = default; - GenImpl(const GenImpl&) = default; - GenImpl(GenImpl&&) = default; - - public: - typedef Value ValueType; - typedef typename std::decay::type StorageType; - - /** - * apply() - Send all values produced by this generator to given handler until - * the handler returns false. Returns false if and only if the handler passed - * in returns false. Note: It should return true even if it completes (without - * the handler returning false), as 'Chain' uses the return value of apply to - * determine if it should process the second object in its chain. - */ - template - bool apply(Handler&& handler) const; - - /** - * foreach() - Send all values produced by this generator to given lambda. - */ - template - void foreach(Body&& body) const { - this->self().apply([&](Value value) -> bool { - static_assert(!infinite, "Cannot call foreach on infinite GenImpl"); - body(std::forward(value)); - return true; - }); - } - - // Child classes should override if the sequence generated is *definitely* - // infinite. 'infinite' may be false_type for some infinite sequences - // (due the the Halting Problem). - static constexpr bool infinite = false; -}; - -template> -Chain operator+(const GenImpl& left, - const GenImpl& right) { - static_assert( - std::is_same::value, - "Generators may ony be combined if Values are the exact same type."); - return Chain(left.self(), right.self()); -} - -template> -Chain operator+(const GenImpl& left, - GenImpl&& right) { - static_assert( - std::is_same::value, - "Generators may ony be combined if Values are the exact same type."); - return Chain(left.self(), std::move(right.self())); -} - -template> -Chain operator+(GenImpl&& left, - const GenImpl& right) { - static_assert( - std::is_same::value, - "Generators may ony be combined if Values are the exact same type."); - return Chain(std::move(left.self()), right.self()); -} - -template> -Chain operator+(GenImpl&& left, - GenImpl&& right) { - static_assert( - std::is_same::value, - "Generators may ony be combined if Values are the exact same type."); - return Chain(std::move(left.self()), std::move(right.self())); -} - -/** - * operator|() which enables foreach-like usage: - * gen | [](Value v) -> void {...}; - */ -template -typename std::enable_if< - IsCompatibleSignature::value>::type -operator|(const GenImpl& gen, Handler&& handler) { - static_assert(!Gen::infinite, - "Cannot pull all values from an infinite sequence."); - gen.self().foreach(std::forward(handler)); -} - -/** - * operator|() which enables foreach-like usage with 'break' support: - * gen | [](Value v) -> bool { return shouldContinue(); }; - */ -template -typename std::enable_if< - IsCompatibleSignature::value, bool>::type -operator|(const GenImpl& gen, Handler&& handler) { - return gen.self().apply(std::forward(handler)); -} - -/** - * operator|() for composing generators with operators, similar to boosts' range - * adaptors: - * gen | map(square) | sum - */ -template -auto operator|(const GenImpl& gen, const Operator& op) -> -decltype(op.self().compose(gen.self())) { - return op.self().compose(gen.self()); -} - -template -auto operator|(GenImpl&& gen, const Operator& op) -> -decltype(op.self().compose(std::move(gen.self()))) { - return op.self().compose(std::move(gen.self())); -} - -namespace detail { - -/* - * ReferencedSource - Generate values from an STL-like container using - * iterators from .begin() until .end(). Value type defaults to the type of - * *container->begin(). For std::vector, this would be int&. Note that the - * value here is a reference, so the values in the vector will be passed by - * reference to downstream operators. - * - * This type is primarily used through the 'from' helper method, like: - * - * string& longestName = from(names) - * | maxBy([](string& s) { return s.size() }); - */ -template -class ReferencedSource : - public GenImpl> { - Container* container_; -public: - explicit ReferencedSource(Container* container) - : container_(container) {} - - template - void foreach(Body&& body) const { - for (auto& value : *container_) { - body(std::forward(value)); - } - } - - template - bool apply(Handler&& handler) const { - for (auto& value : *container_) { - if (!handler(std::forward(value))) { - return false; - } - } - return true; - } -}; - -/** - * CopiedSource - For producing values from eagerly from a sequence of values - * whose storage is owned by this class. Useful for preparing a generator for - * use after a source collection will no longer be available, or for when the - * values are specified literally with an initializer list. - * - * This type is primarily used through the 'fromCopy' function, like: - * - * auto sourceCopy = fromCopy(makeAVector()); - * auto sum = sourceCopy | sum; - * auto max = sourceCopy | max; - * - * Though it is also used for the initializer_list specialization of from(). - */ -template -class CopiedSource : - public GenImpl> { - static_assert( - !std::is_reference::value, "StorageType must be decayed"); - public: - // Generator objects are often copied during normal construction as they are - // encapsulated by downstream generators. It would be bad if this caused - // a copy of the entire container each time, and since we're only exposing a - // const reference to the value, it's safe to share it between multiple - // generators. - static_assert( - !std::is_reference::value, - "Can't copy into a reference"); - std::shared_ptr copy_; -public: - typedef Container ContainerType; - - template - explicit CopiedSource(const SourceContainer& container) - : copy_(new Container(begin(container), end(container))) {} - - explicit CopiedSource(Container&& container) : - copy_(new Container(std::move(container))) {} - - // To enable re-use of cached results. - CopiedSource(const CopiedSource& source) - : copy_(source.copy_) {} - - template - void foreach(Body&& body) const { - for (const auto& value : *copy_) { - body(value); - } - } - - template - bool apply(Handler&& handler) const { - // The collection may be reused by others, we can't allow it to be changed. - for (const auto& value : *copy_) { - if (!handler(value)) { - return false; - } - } - return true; - } -}; - -/** - * Sequence - For generating values from beginning value, incremented along the - * way with the ++ and += operators. Iteration may continue indefinitely by - * setting the 'endless' template parameter to true. If set to false, iteration - * will stop when value reaches 'end', either inclusively or exclusively, - * depending on the template parameter 'endInclusive'. Value type specified - * explicitly. - * - * This type is primarily used through the 'seq' and 'range' function, like: - * - * int total = seq(1, 10) | sum; - * auto indexes = range(0, 10); - */ -template -class Sequence : public GenImpl> { - static_assert(!std::is_reference::value && - !std::is_const::value, "Value mustn't be const or ref."); - Value bounds_[endless ? 1 : 2]; -public: - explicit Sequence(Value begin) - : bounds_{std::move(begin)} { - static_assert(endless, "Must supply 'end'"); - } - - Sequence(Value begin, - Value end) - : bounds_{std::move(begin), std::move(end)} {} - - template - bool apply(Handler&& handler) const { - Value value = bounds_[0]; - for (;endless || value < bounds_[1]; ++value) { - const Value& arg = value; - if (!handler(arg)) { - return false; - } - } - if (endInclusive && value == bounds_[1]) { - const Value& arg = value; - if (!handler(arg)) { - return false; - } - } - return true; - } - - template - void foreach(Body&& body) const { - Value value = bounds_[0]; - for (;endless || value < bounds_[1]; ++value) { - const Value& arg = value; - body(arg); - } - if (endInclusive && value == bounds_[1]) { - const Value& arg = value; - body(arg); - } - } - - static constexpr bool infinite = endless; -}; - -/** - * Chain - For concatenating the values produced by two Generators. - * - * This type is primarily used through using '+' to combine generators, like: - * - * auto nums = seq(1, 10) + seq(20, 30); - * int total = nums | sum; - */ -template -class Chain : public GenImpl> { - First first_; - Second second_; -public: - explicit Chain(First first, Second second) - : first_(std::move(first)) - , second_(std::move(second)) {} - - template - bool apply(Handler&& handler) const { - return first_.apply(std::forward(handler)) - && second_.apply(std::forward(handler)); - } - - template - void foreach(Body&& body) const { - first_.foreach(std::forward(body)); - second_.foreach(std::forward(body)); - } - - static constexpr bool infinite = First::infinite || Second::infinite; -}; - -/** - * GenratorBuilder - Helper for GENERTATOR macro. - **/ -template -struct GeneratorBuilder { - template> - Yield operator+(Source&& source) { - return Yield(std::forward(source)); - } -}; - -/** - * Yield - For producing values from a user-defined generator by way of a - * 'yield' function. - **/ -template -class Yield : public GenImpl> { - Source source_; - public: - explicit Yield(Source source) - : source_(std::move(source)) { - } - - template - bool apply(Handler&& handler) const { - struct Break {}; - auto body = [&](Value value) { - if (!handler(std::forward(value))) { - throw Break(); - } - }; - try { - source_(body); - return true; - } catch (Break&) { - return false; - } - } - - template - void foreach(Body&& body) const { - source_(std::forward(body)); - } -}; - -template -class Empty : public GenImpl> { - public: - template - bool apply(Handler&&) const { return true; } -}; - -/* - * Operators - */ - -/** - * Map - For producing a sequence of values by passing each value from a source - * collection through a predicate. - * - * This type is usually used through the 'map' or 'mapped' helper function: - * - * auto squares = seq(1, 10) | map(square) | asVector; - */ -template -class Map : public Operator> { - Predicate pred_; - public: - Map() {} - - explicit Map(Predicate pred) - : pred_(std::move(pred)) - { } - - template::type - >::type> - class Generator : - public GenImpl> { - Source source_; - Predicate pred_; - public: - explicit Generator(Source source, const Predicate& pred) - : source_(std::move(source)), pred_(pred) {} - - template - void foreach(Body&& body) const { - source_.foreach([&](Value value) { - body(pred_(std::forward(value))); - }); - } - - template - bool apply(Handler&& handler) const { - return source_.apply([&](Value value) { - return handler(pred_(std::forward(value))); - }); - } - - static constexpr bool infinite = Source::infinite; - }; - - template> - Gen compose(GenImpl&& source) const { - return Gen(std::move(source.self()), pred_); - } - - template> - Gen compose(const GenImpl& source) const { - return Gen(source.self(), pred_); - } -}; - - -/** - * Filter - For filtering values from a source sequence by a predicate. - * - * This type is usually used through the 'filter' helper function, like: - * - * auto nonEmpty = from(strings) - * | filter([](const string& str) -> bool { - * return !str.empty(); - * }); - */ -template -class Filter : public Operator> { - Predicate pred_; - public: - Filter() {} - explicit Filter(Predicate pred) - : pred_(std::move(pred)) - { } - - template - class Generator : public GenImpl> { - Source source_; - Predicate pred_; - public: - explicit Generator(Source source, const Predicate& pred) - : source_(std::move(source)), pred_(pred) {} - - template - void foreach(Body&& body) const { - source_.foreach([&](Value value) { - if (pred_(std::forward(value))) { - body(std::forward(value)); - } - }); - } - - template - bool apply(Handler&& handler) const { - return source_.apply([&](Value value) -> bool { - if (pred_(std::forward(value))) { - return handler(std::forward(value)); - } - return true; - }); - } - - static constexpr bool infinite = Source::infinite; - }; - - template> - Gen compose(GenImpl&& source) const { - return Gen(std::move(source.self()), pred_); - } - - template> - Gen compose(const GenImpl& source) const { - return Gen(source.self(), pred_); - } -}; - -/** - * Until - For producing values from a source until a predicate is satisfied. - * - * This type is usually used through the 'until' helper function, like: - * - * auto best = from(sortedItems) - * | until([](Item& item) { return item.score > 100; }) - * | asVector; - */ -template -class Until : public Operator> { - Predicate pred_; - public: - Until() {} - explicit Until(Predicate pred) - : pred_(std::move(pred)) - {} - - template - class Generator : public GenImpl> { - Source source_; - Predicate pred_; - public: - explicit Generator(Source source, const Predicate& pred) - : source_(std::move(source)), pred_(pred) {} - - template - bool apply(Handler&& handler) const { - bool cancelled = false; - source_.apply([&](Value value) -> bool { - if (pred_(value)) { // un-forwarded to disable move - return false; - } - if (!handler(std::forward(value))) { - cancelled = true; - return false; - } - return true; - }); - return !cancelled; - } - }; - - template> - Gen compose(GenImpl&& source) const { - return Gen(std::move(source.self()), pred_); - } - - template> - Gen compose(const GenImpl& source) const { - return Gen(source.self(), pred_); - } - - // Theoretically an 'until' might stop an infinite - static constexpr bool infinite = false; -}; - -/** - * Take - For producing up to N values from a source. - * - * This type is usually used through the 'take' helper function, like: - * - * auto best = from(docs) - * | orderByDescending(scoreDoc) - * | take(10); - */ -class Take : public Operator { - size_t count_; - public: - explicit Take(size_t count) - : count_(count) {} - - template - class Generator : - public GenImpl> { - Source source_; - size_t count_; - public: - explicit Generator(Source source, size_t count) - : source_(std::move(source)) , count_(count) {} - - template - bool apply(Handler&& handler) const { - if (count_ == 0) { return false; } - size_t n = count_; - bool cancelled = false; - source_.apply([&](Value value) -> bool { - if (!handler(std::forward(value))) { - cancelled = true; - return false; - } - return --n; - }); - return !cancelled; - } - }; - - template> - Gen compose(GenImpl&& source) const { - return Gen(std::move(source.self()), count_); - } - - template> - Gen compose(const GenImpl& source) const { - return Gen(source.self(), count_); - } -}; - -/** - * Sample - For taking a random sample of N elements from a sequence - * (without replacement). - */ -template -class Sample : public Operator> { - size_t count_; - Random rng_; - public: - explicit Sample(size_t count, Random rng) - : count_(count), rng_(std::move(rng)) {} - - template::type> - class Generator : - public GenImpl> { - static_assert(!Source::infinite, "Cannot sample infinite source!"); - // It's too easy to bite ourselves if random generator is only 16-bit - static_assert(Random::max() >= std::numeric_limits::max() - 1, - "Random number generator must support big values"); - Source source_; - size_t count_; - mutable Rand rng_; - public: - explicit Generator(Source source, size_t count, Random rng) - : source_(std::move(source)) , count_(count), rng_(std::move(rng)) {} - - template - bool apply(Handler&& handler) const { - if (count_ == 0) { return false; } - std::vector v; - v.reserve(count_); - // use reservoir sampling to give each source value an equal chance - // of appearing in our output. - size_t n = 1; - source_.foreach([&](Value value) -> void { - if (v.size() < count_) { - v.push_back(std::forward(value)); - } else { - // alternatively, we could create a std::uniform_int_distribution - // instead of using modulus, but benchmarks show this has - // substantial overhead. - size_t index = rng_() % n; - if (index < v.size()) { - v[index] = std::forward(value); - } - } - ++n; - }); - - // output is unsorted! - for (auto& val: v) { - if (!handler(std::move(val))) { - return false; - } - } - return true; - } - }; - - template> - Gen compose(GenImpl&& source) const { - return Gen(std::move(source.self()), count_, rng_); - } - - template> - Gen compose(const GenImpl& source) const { - return Gen(source.self(), count_, rng_); - } -}; - -/** - * Skip - For skipping N items from the beginning of a source generator. - * - * This type is usually used through the 'skip' helper function, like: - * - * auto page = from(results) - * | skip(pageSize * startPage) - * | take(10); - */ -class Skip : public Operator { - size_t count_; - public: - explicit Skip(size_t count) - : count_(count) {} - - template - class Generator : - public GenImpl> { - Source source_; - size_t count_; - public: - explicit Generator(Source source, size_t count) - : source_(std::move(source)) , count_(count) {} - - template - void foreach(Body&& body) const { - if (count_ == 0) { - source_.foreach(body); - return; - } - size_t n = 0; - source_.foreach([&](Value value) { - if (n < count_) { - ++n; - } else { - body(std::forward(value)); - } - }); - } - - template - bool apply(Handler&& handler) const { - if (count_ == 0) { - return source_.apply(std::forward(handler)); - } - size_t n = 0; - return source_.apply([&](Value value) -> bool { - if (n < count_) { - ++n; - return true; - } - return handler(std::forward(value)); - }); - } - - static constexpr bool infinite = Source::infinite; - }; - - template> - Gen compose(GenImpl&& source) const { - return Gen(std::move(source.self()), count_); - } - - template> - Gen compose(const GenImpl& source) const { - return Gen(source.self(), count_); - } -}; - -/** - * Order - For ordering a sequence of values from a source by key. - * The key is extracted by the given selector functor, and this key is then - * compared using the specified comparator. - * - * This type is usually used through the 'order' helper function, like: - * - * auto closest = from(places) - * | orderBy([](Place& p) { - * return -distance(p.location, here); - * }) - * | take(10); - */ -template -class Order : public Operator> { - Selector selector_; - Comparer comparer_; - public: - Order() {} - - explicit Order(Selector selector) - : selector_(std::move(selector)) - {} - - Order(Selector selector, - Comparer comparer) - : selector_(std::move(selector)) - , comparer_(std::move(comparer)) - {} - - template::type, - class Result = typename std::result_of::type> - class Generator : - public GenImpl> { - static_assert(!Source::infinite, "Cannot sort infinite source!"); - Source source_; - Selector selector_; - Comparer comparer_; - - typedef std::vector VectorType; - - VectorType asVector() const { - auto comparer = [&](const StorageType& a, const StorageType& b) { - return comparer_(selector_(a), selector_(b)); - }; - auto vals = source_ | as(); - std::sort(vals.begin(), vals.end(), comparer); - return std::move(vals); - } - public: - Generator(Source source, - Selector selector, - Comparer comparer) - : source_(std::move(source)), - selector_(std::move(selector)), - comparer_(std::move(comparer)) {} - - VectorType operator|(const Collect&) const { - return asVector(); - } - - VectorType operator|(const CollectTemplate&) const { - return asVector(); - } - - template - void foreach(Body&& body) const { - for (auto& value : asVector()) { - body(std::move(value)); - } - } - - template - bool apply(Handler&& handler) const { - auto comparer = [&](const StorageType& a, const StorageType& b) { - // swapped for minHeap - return comparer_(selector_(b), selector_(a)); - }; - auto heap = source_ | as(); - std::make_heap(heap.begin(), heap.end(), comparer); - while (!heap.empty()) { - std::pop_heap(heap.begin(), heap.end(), comparer); - if (!handler(std::move(heap.back()))) { - return false; - } - heap.pop_back(); - } - return true; - } - }; - - template> - Gen compose(GenImpl&& source) const { - return Gen(std::move(source.self()), selector_, comparer_); - } - - template> - Gen compose(const GenImpl& source) const { - return Gen(source.self(), selector_, comparer_); - } -}; - -/* - * TypeAssertion - For verifying the exact type of the value produced by a - * generator. Useful for testing and debugging, and acts as a no-op at runtime. - * Pass-through at runtime. Used through the 'assert_type<>()' factory method - * like so: - * - * auto c = from(vector) | assert_type() | sum; - * - */ -template -class TypeAssertion : public Operator> { - public: - template - const Source& compose(const GenImpl& source) const { - static_assert(std::is_same::value, - "assert_type() check failed"); - return source.self(); - } - - template - Source&& compose(GenImpl&& source) const { - static_assert(std::is_same::value, - "assert_type() check failed"); - return std::move(source.self()); - } -}; - -/** - * Distinct - For filtering duplicates out of a sequence. A selector may be - * provided to generate a key to uniquify for each value. - * - * This type is usually used through the 'distinct' helper function, like: - * - * auto closest = from(results) - * | distinctBy([](Item& i) { - * return i.target; - * }) - * | take(10); - */ -template -class Distinct : public Operator> { - Selector selector_; - public: - Distinct() {} - - explicit Distinct(Selector selector) - : selector_(std::move(selector)) - {} - - template - class Generator : public GenImpl> { - Source source_; - Selector selector_; - - typedef typename std::decay::type StorageType; - - // selector_ cannot be passed an rvalue or it would end up passing the husk - // of a value to the downstream operators. - typedef const StorageType& ParamType; - - typedef typename std::result_of::type KeyType; - typedef typename std::decay::type KeyStorageType; - - public: - Generator(Source source, - Selector selector) - : source_(std::move(source)), - selector_(std::move(selector)) {} - - template - void foreach(Body&& body) const { - std::unordered_set keysSeen; - source_.foreach([&](Value value) { - if (keysSeen.insert(selector_(ParamType(value))).second) { - body(std::forward(value)); - } - }); - } - - template - bool apply(Handler&& handler) const { - std::unordered_set keysSeen; - return source_.apply([&](Value value) -> bool { - if (keysSeen.insert(selector_(ParamType(value))).second) { - return handler(std::forward(value)); - } - return true; - }); - } - }; - - template> - Gen compose(GenImpl&& source) const { - return Gen(std::move(source.self()), selector_); - } - - template> - Gen compose(const GenImpl& source) const { - return Gen(source.self(), selector_); - } -}; - -/** - * Batch - For producing fixed-size batches of each value from a source. - * - * This type is usually used through the 'batch' helper function: - * - * auto batchSums - * = seq(1, 10) - * | batch(3) - * | map([](const std::vector& batch) { - * return from(batch) | sum; - * }) - * | as(); - */ -class Batch : public Operator { - size_t batchSize_; - public: - explicit Batch(size_t batchSize) - : batchSize_(batchSize) { - if (batchSize_ == 0) { - throw std::invalid_argument("Batch size must be non-zero!"); - } - } - - template::type, - class VectorType = std::vector> - class Generator : - public GenImpl> { - Source source_; - size_t batchSize_; - public: - explicit Generator(Source source, size_t batchSize) - : source_(std::move(source)) - , batchSize_(batchSize) {} - - template - bool apply(Handler&& handler) const { - VectorType batch_; - batch_.reserve(batchSize_); - bool shouldContinue = source_.apply([&](Value value) -> bool { - batch_.push_back(std::forward(value)); - if (batch_.size() == batchSize_) { - bool needMore = handler(batch_); - batch_.clear(); - return needMore; - } - // Always need more if the handler is not called. - return true; - }); - // Flush everything, if and only if `handler` hasn't returned false. - if (shouldContinue && !batch_.empty()) { - shouldContinue = handler(batch_); - batch_.clear(); - } - return shouldContinue; - } - - static constexpr bool infinite = Source::infinite; - }; - - template> - Gen compose(GenImpl&& source) const { - return Gen(std::move(source.self()), batchSize_); - } - - template> - Gen compose(const GenImpl& source) const { - return Gen(source.self(), batchSize_); - } -}; - -/** - * Composed - For building up a pipeline of operations to perform, absent any - * particular source generator. Useful for building up custom pipelines. - * - * This type is usually used by just piping two operators together: - * - * auto valuesOf = filter([](Optional& o) { return o.hasValue(); }) - * | map([](Optional& o) -> int& { return o.value(); }); - * - * auto valuesIncluded = from(optionals) | valuesOf | as(); - */ -template -class Composed : public Operator> { - First first_; - Second second_; - public: - Composed() {} - - Composed(First first, Second second) - : first_(std::move(first)) - , second_(std::move(second)) {} - - template() - .compose(std::declval())), - class SecondRet = decltype(std::declval() - .compose(std::declval()))> - SecondRet compose(const GenImpl& source) const { - return second_.compose(first_.compose(source.self())); - } - - template() - .compose(std::declval())), - class SecondRet = decltype(std::declval() - .compose(std::declval()))> - SecondRet compose(GenImpl&& source) const { - return second_.compose(first_.compose(std::move(source.self()))); - } -}; - -/* - * Sinks - */ - -/** - * FoldLeft - Left-associative functional fold. For producing an aggregate value - * from a seed and a folder function. Useful for custom aggregators on a - * sequence. - * - * This type is primarily used through the 'foldl' helper method, like: - * - * double movingAverage = from(values) - * | foldl(0.0, [](double avg, double sample) { - * return sample * 0.1 + avg * 0.9; - * }); - */ -template -class FoldLeft : public Operator> { - Seed seed_; - Fold fold_; - public: - FoldLeft() {} - FoldLeft(Seed seed, - Fold fold) - : seed_(std::move(seed)) - , fold_(std::move(fold)) - {} - - template - Seed compose(const GenImpl& source) const { - static_assert(!Source::infinite, "Cannot foldl infinite source"); - Seed accum = seed_; - source | [&](Value v) { - accum = fold_(std::move(accum), std::forward(v)); - }; - return accum; - } -}; - -/** - * First - For finding the first value in a sequence. - * - * This type is primarily used through the 'first' static value, like: - * - * int firstThreeDigitPrime = seq(100) | filter(isPrime) | first; - */ -class First : public Operator { - public: - First() { } - - template::type> - StorageType compose(const GenImpl& source) const { - Optional accum; - source | [&](Value v) -> bool { - accum = std::forward(v); - return false; - }; - if (!accum.hasValue()) { - throw EmptySequence(); - } - return std::move(accum.value()); - } -}; - - -/** - * Any - For determining whether any values in a sequence satisfy a predicate. - * - * This type is primarily used through the 'any' static value, like: - * - * bool any20xPrimes = seq(200, 210) | filter(isPrime) | any; - * - * Note that it may also be used like so: - * - * bool any20xPrimes = seq(200, 210) | any(isPrime); - * - */ -class Any : public Operator { - public: - Any() { } - - template - bool compose(const GenImpl& source) const { - bool any = false; - source | [&](Value v) -> bool { - any = true; - return false; - }; - return any; - } - - /** - * Convenience function for use like: - * - * bool found = gen | any([](int i) { return i * i > 100; }); - */ - template, - class Composed = Composed> - Composed operator()(Predicate pred) const { - return Composed(Filter(std::move(pred)), Any()); - } -}; - -/** - * All - For determining whether all values in a sequence satisfy a predicate. - * - * This type is primarily used through the 'any' static value, like: - * - * bool valid = from(input) | all(validate); - * - * Note: Passing an empty sequence through 'all()' will always return true. - */ -template -class All : public Operator> { - Predicate pred_; - public: - All() {} - explicit All(Predicate pred) - : pred_(std::move(pred)) - { } - - template - bool compose(const GenImpl& source) const { - static_assert(!Source::infinite, "Cannot call 'all' on infinite source"); - bool all = true; - source | [&](Value v) -> bool { - if (!pred_(std::forward(v))) { - all = false; - return false; - } - return true; - }; - return all; - } -}; - -/** - * Reduce - Functional reduce, for recursively combining values from a source - * using a reducer function until there is only one item left. Useful for - * combining values when an empty sequence doesn't make sense. - * - * This type is primarily used through the 'reduce' helper method, like: - * - * sring longest = from(names) - * | reduce([](string&& best, string& current) { - * return best.size() >= current.size() ? best : current; - * }); - */ -template -class Reduce : public Operator> { - Reducer reducer_; - public: - Reduce() {} - explicit Reduce(Reducer reducer) - : reducer_(std::move(reducer)) - {} - - template::type> - StorageType compose(const GenImpl& source) const { - Optional accum; - source | [&](Value v) { - if (accum.hasValue()) { - accum = reducer_(std::move(accum.value()), std::forward(v)); - } else { - accum = std::forward(v); - } - }; - if (!accum.hasValue()) { - throw EmptySequence(); - } - return accum.value(); - } -}; - -/** - * Count - for simply counting the items in a collection. - * - * This type is usually used through its singleton, 'count': - * - * auto shortPrimes = seq(1, 100) | filter(isPrime) | count; - */ -class Count : public Operator { - public: - Count() { } - - template - size_t compose(const GenImpl& source) const { - static_assert(!Source::infinite, "Cannot count infinite source"); - return foldl(size_t(0), - [](size_t accum, Value v) { - return accum + 1; - }).compose(source); - } -}; - -/** - * Sum - For simply summing up all the values from a source. - * - * This type is usually used through its singleton, 'sum': - * - * auto gaussSum = seq(1, 100) | sum; - */ -class Sum : public Operator { - public: - Sum() : Operator() {} - - template::type> - StorageType compose(const GenImpl& source) const { - static_assert(!Source::infinite, "Cannot sum infinite source"); - return foldl(StorageType(0), - [](StorageType&& accum, Value v) { - return std::move(accum) + std::forward(v); - }).compose(source); - } -}; - -/** - * Contains - For testing whether a value matching the given value is contained - * in a sequence. - * - * This type should be used through the 'contains' helper method, like: - * - * bool contained = seq(1, 10) | map(square) | contains(49); - */ -template -class Contains : public Operator> { - Needle needle_; - public: - explicit Contains(Needle needle) - : needle_(std::move(needle)) - {} - - template::type> - bool compose(const GenImpl& source) const { - static_assert(!Source::infinite, - "Calling contains on an infinite source might cause " - "an infinite loop."); - return !(source | [this](Value value) { - return !(needle_ == std::forward(value)); - }); - } -}; - -/** - * Min - For a value which minimizes a key, where the key is determined by a - * given selector, and compared by given comparer. - * - * This type is usually used through the singletone 'min' or through the helper - * functions 'minBy' and 'maxBy'. - * - * auto oldest = from(people) - * | minBy([](Person& p) { - * return p.dateOfBirth; - * }); - */ -template -class Min : public Operator> { - Selector selector_; - Comparer comparer_; - public: - Min() {} - - explicit Min(Selector selector) - : selector_(std::move(selector)) - {} - - Min(Selector selector, - Comparer comparer) - : selector_(std::move(selector)) - , comparer_(std::move(comparer)) - {} - - template::type, - class Key = typename std::decay< - typename std::result_of::type - >::type> - StorageType compose(const GenImpl& source) const { - Optional min; - Optional minKey; - source | [&](Value v) { - Key key = selector_(std::forward(v)); - if (!minKey.hasValue() || comparer_(key, minKey.value())) { - minKey = key; - min = std::forward(v); - } - }; - if (!min.hasValue()) { - throw EmptySequence(); - } - return min.value(); - } -}; - -/** - * Append - For collecting values from a source into a given output container - * by appending. - * - * This type is usually used through the helper function 'appendTo', like: - * - * vector ids; - * from(results) | map([](Person& p) { return p.id }) - * | appendTo(ids); - */ -template -class Append : public Operator> { - Collection* collection_; - public: - explicit Append(Collection* collection) - : collection_(collection) - {} - - template - Collection& compose(const GenImpl& source) const { - source | [&](Value v) { - collection_->insert(collection_->end(), std::forward(v)); - }; - return *collection_; - } -}; - -/** - * Collect - For collecting values from a source in a collection of the desired - * type. - * - * This type is usually used through the helper function 'as', like: - * - * std::string upper = from(stringPiece) - * | map(&toupper) - * | as(); - */ -template -class Collect : public Operator> { - public: - Collect() { } - - template::type> - Collection compose(const GenImpl& source) const { - Collection collection; - source | [&](Value v) { - collection.insert(collection.end(), std::forward(v)); - }; - return collection; - } -}; - - -/** - * CollectTemplate - For collecting values from a source in a collection - * constructed using the specified template type. Given the type of values - * produced by the given generator, the collection type will be: - * Container> - * - * The allocator defaults to std::allocator, so this may be used for the STL - * containers by simply using operators like 'as', 'as', - * 'as'. 'as', here is the helper method which is the usual means of - * consturcting this operator. - * - * Example: - * - * set uniqueNames = from(names) | as(); - */ -template class Container, - template class Allocator> -class CollectTemplate : public Operator> { - public: - CollectTemplate() { } - - template::type, - class Collection = Container>> - Collection compose(const GenImpl& source) const { - Collection collection; - source | [&](Value v) { - collection.insert(collection.end(), std::forward(v)); - }; - return collection; - } -}; - -/** - * Concat - For flattening generators of generators. - * - * This type is usually used through the 'concat' static value, like: - * - * auto edges = - * from(nodes) - * | map([](Node& x) { - * return from(x.neighbors) - * | map([&](Node& y) { - * return Edge(x, y); - * }); - * }) - * | concat - * | as(); - */ -class Concat : public Operator { - public: - Concat() { } - - template::type::ValueType> - class Generator : - public GenImpl> { - Source source_; - public: - explicit Generator(Source source) - : source_(std::move(source)) {} - - template - bool apply(Handler&& handler) const { - return source_.apply([&](Inner inner) -> bool { - return inner.apply(std::forward(handler)); - }); - } - - template - void foreach(Body&& body) const { - source_.foreach([&](Inner inner) { - inner.foreach(std::forward(body)); - }); - } - - static constexpr bool infinite = Source::infinite; - }; - - template> - Gen compose(GenImpl&& source) const { - return Gen(std::move(source.self())); - } - - template> - Gen compose(const GenImpl& source) const { - return Gen(source.self()); - } -}; - -/** - * RangeConcat - For flattening generators of iterables. - * - * This type is usually used through the 'rconcat' static value, like: - * - * map> adjacency; - * auto sinks = - * from(adjacency) - * | get<1>() - * | rconcat() - * | as(); - */ -class RangeConcat : public Operator { - public: - RangeConcat() { } - - template::RefType> - class Generator - : public GenImpl> { - Source source_; - public: - explicit Generator(Source source) - : source_(std::move(source)) {} - - template - void foreach(Body&& body) const { - source_.foreach([&](Range range) { - for (auto& value : range) { - body(value); - } - }); - } - - template - bool apply(Handler&& handler) const { - return source_.apply([&](Range range) -> bool { - for (auto& value : range) { - if (!handler(value)) { - return false; - } - } - return true; - }); - } - }; - - template> - Gen compose(GenImpl&& source) const { - return Gen(std::move(source.self())); - } - - template> - Gen compose(const GenImpl& source) const { - return Gen(source.self()); - } -}; - - -/** - * GuardImpl - For handling exceptions from downstream computation. Requires the - * type of exception to catch, and handler function to invoke in the event of - * the exception. Note that the handler may: - * 1) return true to continue processing the sequence - * 2) return false to end the sequence immediately - * 3) throw, to pass the exception to the next catch - * The handler must match the signature 'bool(Exception&, Value)'. - * - * This type is used through the `guard` helper, like so: - * - * auto indexes - * = byLine(STDIN_FILENO) - * | guard([](std::runtime_error& e, - * StringPiece sp) { - * LOG(ERROR) << sp << ": " << e.str(); - * return true; // continue processing subsequent lines - * }) - * | eachTo() - * | as(); - * - * TODO(tjackson): Rename this back to Guard. - **/ -template -class GuardImpl : public Operator> { - ErrorHandler handler_; - public: - GuardImpl(ErrorHandler handler) - : handler_(std::move(handler)) {} - - template - class Generator : public GenImpl> { - Source source_; - ErrorHandler handler_; - public: - explicit Generator(Source source, - ErrorHandler handler) - : source_(std::move(source)), - handler_(std::move(handler)) {} - - template - bool apply(Handler&& handler) const { - return source_.apply([&](Value value) -> bool { - try { - handler(std::forward(value)); - return true; - } catch (Exception& e) { - return handler_(e, std::forward(value)); - } - }); - } - - static constexpr bool infinite = Source::infinite; - }; - - template> - Gen compose(GenImpl&& source) const { - return Gen(std::move(source.self()), handler_); - } - - template> - Gen compose(const GenImpl& source) const { - return Gen(source.self(), handler_); - } -}; - -/** - * Cycle - For repeating a sequence forever. - * - * This type is usually used through the 'cycle' static value, like: - * - * auto tests - * = from(samples) - * | cycle - * | take(100); - */ -class Cycle : public Operator { - off_t limit_; // -1 for infinite - public: - Cycle() - : limit_(-1) { } - - explicit Cycle(off_t limit) - : limit_(limit) { } - - template - class Generator : public GenImpl> { - Source source_; - off_t limit_; // -1 for infinite - public: - explicit Generator(Source source, off_t limit) - : source_(std::move(source)) - , limit_(limit) {} - - template - bool apply(Handler&& handler) const { - bool cont; - auto handler2 = [&](Value value) { - cont = handler(std::forward(value)); - return cont; - }; - for (off_t count = 0; count != limit_; ++count) { - cont = false; - source_.apply(handler2); - if (!cont) { - return false; - } - } - return true; - } - - // not actually infinite, since an empty generator will end the cycles. - static constexpr bool infinite = Source::infinite; - }; - - template> - Gen compose(GenImpl&& source) const { - return Gen(std::move(source.self()), limit_); - } - - template> - Gen compose(const GenImpl& source) const { - return Gen(source.self(), limit_); - } - - /** - * Convenience function for use like: - * - * auto tripled = gen | cycle(3); - */ - Cycle operator()(off_t limit) const { - return Cycle(limit); - } -}; - -/** - * Dereference - For dereferencing a sequence of pointers while filtering out - * null pointers. - * - * This type is usually used through the 'dereference' static value, like: - * - * auto refs = from(ptrs) | dereference; - */ -class Dereference : public Operator { - public: - Dereference() {} - - template())> - class Generator : public GenImpl> { - Source source_; - public: - explicit Generator(Source source) - : source_(std::move(source)) {} - - template - void foreach(Body&& body) const { - source_.foreach([&](Value value) { - if (value) { - return body(*value); - } - }); - } - - template - bool apply(Handler&& handler) const { - return source_.apply([&](Value value) -> bool { - if (value) { - return handler(*value); - } - return true; - }); - } - - // not actually infinite, since an empty generator will end the cycles. - static constexpr bool infinite = Source::infinite; - }; - - template> - Gen compose(GenImpl&& source) const { - return Gen(std::move(source.self())); - } - - template> - Gen compose(const GenImpl& source) const { - return Gen(source.self()); - } -}; - -} //::detail - -/** - * VirtualGen - For wrapping template types in simple polymorphic wrapper. - **/ -template -class VirtualGen : public GenImpl> { - class WrapperBase { - public: - virtual ~WrapperBase() {} - virtual bool apply(const std::function& handler) const = 0; - virtual void foreach(const std::function& body) const = 0; - virtual std::unique_ptr clone() const = 0; - }; - - template - class WrapperImpl : public WrapperBase { - Wrapped wrapped_; - public: - explicit WrapperImpl(Wrapped wrapped) - : wrapped_(std::move(wrapped)) { - } - - virtual bool apply(const std::function& handler) const { - return wrapped_.apply(handler); - } - - virtual void foreach(const std::function& body) const { - wrapped_.foreach(body); - } - - virtual std::unique_ptr clone() const { - return std::unique_ptr(new WrapperImpl(wrapped_)); - } - }; - - std::unique_ptr wrapper_; - - public: - template - /* implicit */ VirtualGen(Self source) - : wrapper_(new WrapperImpl(std::move(source))) - { } - - VirtualGen(VirtualGen&& source) - : wrapper_(std::move(source.wrapper_)) - { } - - VirtualGen(const VirtualGen& source) - : wrapper_(source.wrapper_->clone()) - { } - - VirtualGen& operator=(const VirtualGen& source) { - wrapper_.reset(source.wrapper_->clone()); - return *this; - } - - VirtualGen& operator=(VirtualGen&& source) { - wrapper_= std::move(source.wrapper_); - return *this; - } - - bool apply(const std::function& handler) const { - return wrapper_->apply(handler); - } - - void foreach(const std::function& body) const { - wrapper_->foreach(body); - } -}; - -/** - * non-template operators, statically defined to avoid the need for anything but - * the header. - */ -static const detail::Sum sum; - -static const detail::Count count; - -static const detail::First first; - -/** - * Use directly for detecting any values, or as a function to detect values - * which pass a predicate: - * - * auto nonempty = g | any; - * auto evens = g | any(even); - */ -static const detail::Any any; - -static const detail::Min min; - -static const detail::Min max; - -static const detail::Order order; - -static const detail::Distinct distinct; - -static const detail::Map move; - -static const detail::Concat concat; - -static const detail::RangeConcat rconcat; - -/** - * Use directly for infinite sequences, or as a function to limit cycle count. - * - * auto forever = g | cycle; - * auto thrice = g | cycle(3); - */ -static const detail::Cycle cycle; - -static const detail::Dereference dereference; - -inline detail::Take take(size_t count) { - return detail::Take(count); -} - -template -inline detail::Sample sample(size_t count, Random rng = Random()) { - return detail::Sample(count, std::move(rng)); -} - -inline detail::Skip skip(size_t count) { - return detail::Skip(count); -} - -inline detail::Batch batch(size_t batchSize) { - return detail::Batch(batchSize); -} - -}} //folly::gen - -#pragma GCC diagnostic pop diff --git a/folly/experimental/Gen.h b/folly/experimental/Gen.h index fdbb712c..3142b541 100644 --- a/folly/experimental/Gen.h +++ b/folly/experimental/Gen.h @@ -13,639 +13,5 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "folly/Range.h" -#include "folly/Optional.h" -#include "folly/Conv.h" - -/** - * Generator-based Sequence Comprehensions in C++, akin to C#'s LINQ - * @author Tom Jackson - * - * This library makes it possible to write declarative comprehensions for - * processing sequences of values efficiently in C++. The operators should be - * familiar to those with experience in functional programming, and the - * performance will be virtually identical to the equivalent, boilerplate C++ - * implementations. - * - * Generator objects may be created from either an stl-like container (anything - * supporting begin() and end()), from sequences of values, or from another - * generator (see below). To create a generator that pulls values from a vector, - * for example, one could write: - * - * vector names { "Jack", "Jill", "Sara", "Tom" }; - * auto gen = from(names); - * - * Generators are composed by building new generators out of old ones through - * the use of operators. These are reminicent of shell pipelines, and afford - * similar composition. Lambda functions are used liberally to describe how to - * handle individual values: - * - * auto lengths = gen - * | mapped([](const fbstring& name) { return name.size(); }); - * - * Generators are lazy; they don't actually perform any work until they need to. - * As an example, the 'lengths' generator (above) won't actually invoke the - * provided lambda until values are needed: - * - * auto lengthVector = lengths | as(); - * auto totalLength = lengths | sum; - * - * 'auto' is useful in here because the actual types of the generators objects - * are usually complicated and implementation-sensitive. - * - * If a simpler type is desired (for returning, as an example), VirtualGen - * may be used to wrap the generator in a polymorphic wrapper: - * - * VirtualGen powersOfE() { - * return seq(1) | mapped(&expf); - * } - * - * To learn more about this library, including the use of infinite generators, - * see the examples in the comments, or the docs (coming soon). -*/ - -namespace folly { namespace gen { - -template -class GenImpl; - -template -class Operator; - -class EmptySequence : public std::exception { -public: - virtual const char* what() const noexcept { - return "This operation cannot be called on an empty sequence"; - } -}; - -class Less { -public: - template - auto operator()(const First& first, const Second& second) const -> - decltype(first < second) { - return first < second; - } -}; - -class Greater { -public: - template - auto operator()(const First& first, const Second& second) const -> - decltype(first > second) { - return first > second; - } -}; - -template -class Get { -public: - template - auto operator()(Value&& value) const -> - decltype(std::get(std::forward(value))) { - return std::get(std::forward(value)); - } -}; - -template -class MemberFunction { - public: - typedef Result (Class::*MemberPtr)(); - private: - MemberPtr member_; - public: - explicit MemberFunction(MemberPtr member) - : member_(member) - {} - - Result operator()(Class&& x) const { - return (x.*member_)(); - } - - Result operator()(Class& x) const { - return (x.*member_)(); - } -}; - -template -class ConstMemberFunction{ - public: - typedef Result (Class::*MemberPtr)() const; - private: - MemberPtr member_; - public: - explicit ConstMemberFunction(MemberPtr member) - : member_(member) - {} - - Result operator()(const Class& x) const { - return (x.*member_)(); - } -}; - -template -class Field { - public: - typedef FieldType (Class::*FieldPtr); - private: - FieldPtr field_; - public: - explicit Field(FieldPtr field) - : field_(field) - {} - - const FieldType& operator()(const Class& x) const { - return x.*field_; - } - - FieldType& operator()(Class& x) const { - return x.*field_; - } - - FieldType&& operator()(Class&& x) const { - return std::move(x.*field_); - } -}; - -class Move { -public: - template - auto operator()(Value&& value) const -> - decltype(std::move(std::forward(value))) { - return std::move(std::forward(value)); - } -}; - -class Identity { -public: - template - auto operator()(Value&& value) const -> - decltype(std::forward(value)) { - return std::forward(value); - } -}; - -template -class Cast { - public: - template - Dest operator()(Value&& value) const { - return Dest(std::forward(value)); - } -}; - -template -class To { - public: - template - Dest operator()(Value&& value) const { - return ::folly::to(std::forward(value)); - } -}; - -// Specialization to allow String->StringPiece conversion -template <> -class To { - public: - StringPiece operator()(StringPiece src) const { - return src; - } -}; - -namespace detail { - -template -struct FBounded; - -/* - * Type Traits - */ -template -struct ValueTypeOfRange { - private: - static Container container_; - public: - typedef decltype(*std::begin(container_)) - RefType; - typedef typename std::decay::type - StorageType; -}; - - -/* - * Sources - */ -template::RefType> -class ReferencedSource; - -template::type>> -class CopiedSource; - -template -class Sequence; - -template -class Chain; - -template -class Yield; - -template -class Empty; - - -/* - * Operators - */ -template -class Map; - -template -class Filter; - -template -class Until; - -class Take; - -template -class Sample; - -class Skip; - -template -class Order; - -template -class Distinct; - -template -class Composed; - -template -class TypeAssertion; - -class Concat; - -class RangeConcat; - -class Cycle; - -class Batch; - -class Dereference; - -/* - * Sinks - */ -template -class FoldLeft; - -class First; - -class Any; - -template -class All; - -template -class Reduce; - -class Sum; - -template -class Min; - -template -class Collect; - -template class Collection = std::vector, - template class Allocator = std::allocator> -class CollectTemplate; - -template -class Append; - -template -struct GeneratorBuilder; - -template -class Contains; - -template -class GuardImpl; - -} - -/** - * Polymorphic wrapper - **/ -template -class VirtualGen; - -/* - * Source Factories - */ -template> -From fromConst(const Container& source) { - return From(&source); -} - -template> -From from(Container& source) { - return From(&source); -} - -template::StorageType, - class CopyOf = detail::CopiedSource> -CopyOf fromCopy(Container&& source) { - return CopyOf(std::forward(source)); -} - -template> -From from(std::initializer_list source) { - return From(source); -} - -template> -From from(Container&& source) { - return From(std::move(source)); -} - -template> -Gen range(Value begin, Value end) { - return Gen(begin, end); -} - -template> -Gen seq(Value first, Value last) { - return Gen(first, last); -} - -template> -Gen seq(Value begin) { - return Gen(begin); -} - -template> -Yield generator(Source&& source) { - return Yield(std::forward(source)); -} - -/* - * Create inline generator, used like: - * - * auto gen = GENERATOR(int) { yield(1); yield(2); }; - */ -#define GENERATOR(TYPE) \ - ::folly::gen::detail::GeneratorBuilder() + \ - [=](const std::function& yield) - -/* - * empty() - for producing empty sequences. - */ -template -detail::Empty empty() { - return {}; -} - -/* - * Operator Factories - */ -template> -Map mapped(Predicate pred = Predicate()) { - return Map(std::move(pred)); -} - -template> -Map map(Predicate pred = Predicate()) { - return Map(std::move(pred)); -} - -/* - * member(...) - For extracting a member from each value. - * - * vector strings = ...; - * auto sizes = from(strings) | member(&string::size); - * - * If a member is const overridden (like 'front()'), pass template parameter - * 'Const' to select the const version, or 'Mutable' to select the non-const - * version: - * - * auto heads = from(strings) | member(&string::front); - */ -enum MemberType { - Const, - Mutable -}; - -template, - class Map = detail::Map> -typename std::enable_if::type -member(Return (Class::*member)() const) { - return Map(Mem(member)); -} - -template, - class Map = detail::Map> -typename std::enable_if::type -member(Return (Class::*member)()) { - return Map(Mem(member)); -} - -/* - * field(...) - For extracting a field from each value. - * - * vector items = ...; - * auto names = from(items) | field(&Item::name); - * - * Note that if the values of the generator are rvalues, any non-reference - * fields will be rvalues as well. As an example, the code below does not copy - * any strings, only moves them: - * - * auto namesVector = from(items) - * | move - * | field(&Item::name) - * | as(); - */ -template, - class Map = detail::Map> -Map field(FieldType Class::*field) { - return Map(Field(field)); -} - -template> -Filter filter(Predicate pred = Predicate()) { - return Filter(std::move(pred)); -} - -template> -All all(Predicate pred = Predicate()) { - return All(std::move(pred)); -} - -template> -Until until(Predicate pred = Predicate()) { - return Until(std::move(pred)); -} - -template> -Order orderBy(Selector selector = Identity(), - Comparer comparer = Comparer()) { - return Order(std::move(selector), - std::move(comparer)); -} - -template> -Order orderByDescending(Selector selector = Identity()) { - return Order(std::move(selector)); -} - -template> -Distinct distinctBy(Selector selector = Identity()) { - return Distinct(std::move(selector)); -} - -template>> -Get get() { - return Get(); -} - -// construct Dest from each value -template >> -Cast eachAs() { - return Cast(); -} - -// call folly::to on each value -template >> -To eachTo() { - return To(); -} - -template -detail::TypeAssertion assert_type() { - return {}; -} - -/* - * Sink Factories - */ -template> -FoldLeft foldl(Seed seed = Seed(), - Fold fold = Fold()) { - return FoldLeft(std::move(seed), - std::move(fold)); -} - -template> -Reduce reduce(Reducer reducer = Reducer()) { - return Reduce(std::move(reducer)); -} - -template> -Min minBy(Selector selector = Selector()) { - return Min(std::move(selector)); -} - -template> -MaxBy maxBy(Selector selector = Selector()) { - return MaxBy(std::move(selector)); -} - -template> -Collect as() { - return Collect(); -} - -template class Container = std::vector, - template class Allocator = std::allocator, - class Collect = detail::CollectTemplate> -Collect as() { - return Collect(); -} - -template> -Append appendTo(Collection& collection) { - return Append(&collection); -} - -template::type>> -Contains contains(Needle&& needle) { - return Contains(std::forward(needle)); -} - -template::type>> -GuardImpl guard(ErrorHandler&& handler) { - return GuardImpl(std::forward(handler)); -} - -}} // folly::gen - -#include "folly/experimental/Gen-inl.h" +#pragma message "folly::gen has moved to folly/gen/*.h" +#include "folly/gen/Base.h" diff --git a/folly/experimental/StringGen-inl.h b/folly/experimental/StringGen-inl.h deleted file mode 100644 index 98f9edc7..00000000 --- a/folly/experimental/StringGen-inl.h +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright 2014 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FOLLY_STRINGGEN_H_ -#error This file may only be included from folly/experimental/StringGen.h -#endif - -#include "folly/Conv.h" -#include "folly/String.h" -#include "folly/io/IOBuf.h" - -namespace folly { -namespace gen { -namespace detail { - -inline bool splitPrefix(StringPiece& in, StringPiece& prefix, char delimiter) { - auto p = static_cast(memchr(in.data(), delimiter, in.size())); - if (p) { - prefix.assign(in.data(), p); - in.assign(p + 1, in.end()); - return true; - } - prefix.clear(); - return false; -} - -inline const char* ch(const unsigned char* p) { - return reinterpret_cast(p); -} - -class StringResplitter : public Operator { - char delimiter_; - public: - explicit StringResplitter(char delimiter) : delimiter_(delimiter) { } - - template - class Generator : public GenImpl> { - Source source_; - char delimiter_; - public: - Generator(Source source, char delimiter) - : source_(std::move(source)), delimiter_(delimiter) { } - - template - bool apply(Body&& body) const { - std::unique_ptr buffer; - - auto fn = [&](StringPiece in) -> bool { - StringPiece prefix; - bool found = splitPrefix(in, prefix, this->delimiter_); - if (found && buffer && buffer->length() != 0) { - // Append to end of buffer, return line - if (!prefix.empty()) { - buffer->reserve(0, prefix.size()); - memcpy(buffer->writableTail(), prefix.data(), prefix.size()); - buffer->append(prefix.size()); - } - if (!body(StringPiece(ch(buffer->data()), buffer->length()))) { - return false; - } - buffer->clear(); - found = splitPrefix(in, prefix, this->delimiter_); - } - // Buffer is empty, return lines directly from input (no buffer) - while (found) { - if (!body(prefix)) { - return false; - } - found = splitPrefix(in, prefix, this->delimiter_); - } - if (!in.empty()) { - // Incomplete line left, append to buffer - if (!buffer) { - // Arbitrarily assume that we have half a line and get enough - // room for twice that. - constexpr size_t kDefaultLineSize = 256; - buffer = IOBuf::create(std::max(kDefaultLineSize, 2 * in.size())); - } - buffer->reserve(0, in.size()); - memcpy(buffer->writableTail(), in.data(), in.size()); - buffer->append(in.size()); - } - return true; - }; - - // Iterate - if (!source_.apply(std::move(fn))) { - return false; - } - - // Incomplete last line - if (buffer && buffer->length() != 0) { - if (!body(StringPiece(ch(buffer->data()), buffer->length()))) { - return false; - } - } - return true; - } - - static constexpr bool infinite = Source::infinite; - }; - - template> - Gen compose(GenImpl&& source) const { - return Gen(std::move(source.self()), delimiter_); - } - - template> - Gen compose(const GenImpl& source) const { - return Gen(source.self(), delimiter_); - } -}; - -class SplitStringSource : public GenImpl { - StringPiece source_; - char delimiter_; - public: - SplitStringSource(const StringPiece& source, - char delimiter) - : source_(source) - , delimiter_(delimiter) { } - - template - bool apply(Body&& body) const { - StringPiece rest(source_); - StringPiece prefix; - while (splitPrefix(rest, prefix, this->delimiter_)) { - if (!body(prefix)) { - return false; - } - } - if (!rest.empty()) { - if (!body(rest)) { - return false; - } - } - return true; - } -}; - -/** - * Unsplit - For joining tokens from a generator into a string. This is - * the inverse of `split` above. - * - * This type is primarily used through the 'unsplit' function. - */ -template -class Unsplit : public Operator> { - Delimiter delimiter_; - public: - Unsplit(const Delimiter& delimiter) - : delimiter_(delimiter) { - } - - template - Output compose(const GenImpl& source) const { - Output outputBuffer; - UnsplitBuffer unsplitter(delimiter_, &outputBuffer); - unsplitter.compose(source); - return outputBuffer; - } -}; - -/** - * UnsplitBuffer - For joining tokens from a generator into a string, - * and inserting them into a custom buffer. - * - * This type is primarily used through the 'unsplit' function. - */ -template -class UnsplitBuffer : public Operator> { - Delimiter delimiter_; - OutputBuffer* outputBuffer_; - public: - UnsplitBuffer(const Delimiter& delimiter, OutputBuffer* outputBuffer) - : delimiter_(delimiter) - , outputBuffer_(outputBuffer) { - CHECK(outputBuffer); - } - - template - void compose(const GenImpl& source) const { - // If the output buffer is empty, we skip inserting the delimiter for the - // first element. - bool skipDelim = outputBuffer_->empty(); - source | [&](Value v) { - if (skipDelim) { - skipDelim = false; - toAppend(std::forward(v), outputBuffer_); - } else { - toAppend(delimiter_, std::forward(v), outputBuffer_); - } - }; - } -}; - - -/** - * Hack for static for-like constructs - */ -template -inline Target passthrough(Target target) { return target; } - -#pragma GCC diagnostic push -#ifdef __clang__ -// Clang isn't happy with eatField() hack below. -#pragma GCC diagnostic ignored "-Wreturn-stack-address" -#endif // __clang__ - -/** - * ParseToTuple - For splitting a record and immediatlely converting it to a - * target tuple type. Primary used through the 'eachToTuple' helper, like so: - * - * auto config - * = split("1:a 2:b", ' ') - * | eachToTuple() - * | as>>(); - * - */ -template -class SplitTo { - Delimiter delimiter_; - public: - explicit SplitTo(Delimiter delimiter) - : delimiter_(delimiter) {} - - TargetContainer operator()(StringPiece line) const { - int i = 0; - StringPiece fields[sizeof...(Targets)]; - // HACK(tjackson): Used for referencing fields[] corresponding to variadic - // template parameters. - auto eatField = [&]() -> StringPiece& { return fields[i++]; }; - if (!split(delimiter_, - line, - detail::passthrough(eatField())...)) { - throw std::runtime_error("field count mismatch"); - } - i = 0; - return TargetContainer(To()(eatField())...); - } -}; - -#pragma GCC diagnostic pop - -} // namespace detail - -} // namespace gen -} // namespace folly diff --git a/folly/experimental/StringGen.h b/folly/experimental/StringGen.h index 55b945d2..c8a6571c 100644 --- a/folly/experimental/StringGen.h +++ b/folly/experimental/StringGen.h @@ -13,144 +13,5 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#ifndef FOLLY_STRINGGEN_H_ -#define FOLLY_STRINGGEN_H_ - -#include "folly/Range.h" -#include "folly/experimental/Gen.h" - -namespace folly { -namespace gen { - -namespace detail { -class StringResplitter; -class SplitStringSource; - -template -class Unsplit; - -template -class UnsplitBuffer; - -template -class SplitTo; - -} // namespace detail - -/** - * Split the output from a generator into StringPiece "lines" delimited by - * the given delimiter. Delimters are NOT included in the output. - * - * resplit() behaves as if the input strings were concatenated into one long - * string and then split. - */ -// make this a template so we don't require StringResplitter to be complete -// until use -template -S resplit(char delimiter) { - return S(delimiter); -} - -template -S split(const StringPiece& source, char delimiter) { - return S(source, delimiter); -} - -/* - * Joins a sequence of tokens into a string, with the chosen delimiter. - * - * E.G. - * fbstring result = split("a,b,c", ",") | unsplit(","); - * assert(result == "a,b,c"); - * - * std::string result = split("a,b,c", ",") | unsplit(" "); - * assert(result == "a b c"); - */ - - -// NOTE: The template arguments are reversed to allow the user to cleanly -// specify the output type while still inferring the type of the delimiter. -template> -Unsplit unsplit(const Delimiter& delimiter) { - return Unsplit(delimiter); -} - -template> -Unsplit unsplit(const char* delimiter) { - return Unsplit(delimiter); -} - -/* - * Joins a sequence of tokens into a string, appending them to the output - * buffer. If the output buffer is empty, an initial delimiter will not be - * inserted at the start. - * - * E.G. - * std::string buffer; - * split("a,b,c", ",") | unsplit(",", &buffer); - * assert(buffer == "a,b,c"); - * - * std::string anotherBuffer("initial"); - * split("a,b,c", ",") | unsplit(",", &anotherbuffer); - * assert(anotherBuffer == "initial,a,b,c"); - */ -template> -UnsplitBuffer unsplit(Delimiter delimiter, OutputBuffer* outputBuffer) { - return UnsplitBuffer(delimiter, outputBuffer); -} - -template> -UnsplitBuffer unsplit(const char* delimiter, OutputBuffer* outputBuffer) { - return UnsplitBuffer(delimiter, outputBuffer); -} - - -template -detail::Map, char, Targets...>> -eachToTuple(char delim) { - return detail::Map< - detail::SplitTo, char, Targets...>>( - detail::SplitTo, char, Targets...>(delim)); -} - -template -detail::Map, fbstring, Targets...>> -eachToTuple(StringPiece delim) { - return detail::Map< - detail::SplitTo, fbstring, Targets...>>( - detail::SplitTo, fbstring, Targets...>(delim)); -} - -template -detail::Map, char, First, Second>> -eachToPair(char delim) { - return detail::Map< - detail::SplitTo, char, First, Second>>( - detail::SplitTo, char, First, Second>(delim)); -} - -template -detail::Map, fbstring, First, Second>> -eachToPair(StringPiece delim) { - return detail::Map< - detail::SplitTo, fbstring, First, Second>>( - detail::SplitTo, fbstring, First, Second>( - to(delim))); -} - -} // namespace gen -} // namespace folly - -#include "folly/experimental/StringGen-inl.h" - -#endif /* FOLLY_STRINGGEN_H_ */ - +#pragma message "folly::gen has moved to folly/gen/*.h" +#include "folly/gen/String.h" diff --git a/folly/experimental/test/GenBenchmark.cpp b/folly/experimental/test/GenBenchmark.cpp deleted file mode 100644 index 52f3e0a2..00000000 --- a/folly/experimental/test/GenBenchmark.cpp +++ /dev/null @@ -1,678 +0,0 @@ -/* - * Copyright 2014 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "folly/experimental/Gen.h" -#include "folly/experimental/StringGen.h" -#include "folly/experimental/FileGen.h" -#include "folly/String.h" - -#include -#include - -#include - -#include "folly/Benchmark.h" - -using namespace folly; -using namespace folly::gen; -using std::ostream; -using std::pair; -using std::set; -using std::vector; -using std::tuple; - -static std::atomic testSize(1000); -static vector testVector = - seq(1, testSize.load()) - | mapped([](int) { return rand(); }) - | as(); - -static vector testStrVector = - seq(1, testSize.load()) - | eachTo() - | as(); - -static vector> testVectorVector = - seq(1, 100) - | map([](int i) { - return seq(1, i) | as(); - }) - | as(); -static vector strings = - from(testVector) - | eachTo() - | as(); - -auto square = [](int x) { return x * x; }; -auto add = [](int a, int b) { return a + b; }; -auto multiply = [](int a, int b) { return a * b; }; - -BENCHMARK(Sum_Basic_NoGen, iters) { - int limit = testSize.load(); - int s = 0; - while (iters--) { - for (int i = 0; i < limit; ++i) { - s += i; - } - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(Sum_Basic_Gen, iters) { - int limit = testSize.load(); - int s = 0; - while (iters--) { - s += range(0, limit) | sum; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_DRAW_LINE() - -BENCHMARK(Sum_Vector_NoGen, iters) { - int s = 0; - while (iters--) { - for (auto& i : testVector) { - s += i; - } - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(Sum_Vector_Gen, iters) { - int s = 0; - while (iters--) { - s += from(testVector) | sum; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_DRAW_LINE() - -BENCHMARK(Member, iters) { - int s = 0; - while(iters--) { - s += from(strings) - | member(&fbstring::size) - | sum; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(MapMember, iters) { - int s = 0; - while(iters--) { - s += from(strings) - | map([](const fbstring& x) { return x.size(); }) - | sum; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_DRAW_LINE() - -BENCHMARK(Count_Vector_NoGen, iters) { - int s = 0; - while (iters--) { - for (auto& i : testVector) { - if (i * 2 < rand()) { - ++s; - } - } - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(Count_Vector_Gen, iters) { - int s = 0; - while (iters--) { - s += from(testVector) - | filter([](int i) { - return i * 2 < rand(); - }) - | count; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_DRAW_LINE() - -BENCHMARK(Fib_Sum_NoGen, iters) { - int s = 0; - while (iters--) { - auto fib = [](int limit) -> vector { - vector ret; - int a = 0; - int b = 1; - for (int i = 0; i * 2 < limit; ++i) { - ret.push_back(a += b); - ret.push_back(b += a); - } - return ret; - }; - for (auto& v : fib(testSize.load())) { - s += v; - } - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(Fib_Sum_Gen, iters) { - int s = 0; - while (iters--) { - auto fib = GENERATOR(int) { - int a = 0; - int b = 1; - for (;;) { - yield(a += b); - yield(b += a); - } - }; - s += fib | take(testSize.load()) | sum; - } - folly::doNotOptimizeAway(s); -} - -struct FibYielder { - template - void operator()(Yield&& yield) const { - int a = 0; - int b = 1; - for (;;) { - yield(a += b); - yield(b += a); - } - } -}; - -BENCHMARK_RELATIVE(Fib_Sum_Gen_Static, iters) { - int s = 0; - while (iters--) { - auto fib = generator(FibYielder()); - s += fib | take(testSize.load()) | sum; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_DRAW_LINE() - -BENCHMARK(VirtualGen_0Virtual, iters) { - int s = 0; - while (iters--) { - auto numbers = seq(1, 10000); - auto squares = numbers | map(square); - auto quads = squares | map(square); - s += quads | sum; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(VirtualGen_1Virtual, iters) { - int s = 0; - while (iters--) { - VirtualGen numbers = seq(1, 10000); - auto squares = numbers | map(square); - auto quads = squares | map(square); - s += quads | sum; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(VirtualGen_2Virtual, iters) { - int s = 0; - while (iters--) { - VirtualGen numbers = seq(1, 10000); - VirtualGen squares = numbers | map(square); - auto quads = squares | map(square); - s += quads | sum; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(VirtualGen_3Virtual, iters) { - int s = 0; - while (iters--) { - VirtualGen numbers = seq(1, 10000); - VirtualGen squares = numbers | map(square); - VirtualGen quads = squares | map(square); - s += quads | sum; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_DRAW_LINE() - -BENCHMARK(Concat_NoGen, iters) { - int s = 0; - while (iters--) { - for (auto& v : testVectorVector) { - for (auto& i : v) { - s += i; - } - } - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(Concat_Gen, iters) { - int s = 0; - while (iters--) { - s += from(testVectorVector) | rconcat | sum; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_DRAW_LINE() - -BENCHMARK(Composed_NoGen, iters) { - int s = 0; - while (iters--) { - for (auto& i : testVector) { - s += i * i; - } - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(Composed_Gen, iters) { - int s = 0; - auto sumSq = map(square) | sum; - while (iters--) { - s += from(testVector) | sumSq; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(Composed_GenRegular, iters) { - int s = 0; - while (iters--) { - s += from(testVector) | map(square) | sum; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_DRAW_LINE() - -BENCHMARK(Sample, iters) { - size_t s = 0; - while (iters--) { - auto sampler = seq(1, 10 * 1000 * 1000) | sample(1000); - s += (sampler | sum); - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_DRAW_LINE() - -namespace { - -const char* const kLine = "The quick brown fox jumped over the lazy dog.\n"; -const size_t kLineCount = 10000; -std::string bigLines; -const size_t kSmallLineSize = 17; -std::vector smallLines; - -void initStringResplitterBenchmark() { - bigLines.reserve(kLineCount * strlen(kLine)); - for (size_t i = 0; i < kLineCount; ++i) { - bigLines += kLine; - } - size_t remaining = bigLines.size(); - size_t pos = 0; - while (remaining) { - size_t n = std::min(kSmallLineSize, remaining); - smallLines.push_back(bigLines.substr(pos, n)); - pos += n; - remaining -= n; - } -} - -size_t len(folly::StringPiece s) { return s.size(); } - -} // namespace - -BENCHMARK(StringResplitter_Big, iters) { - size_t s = 0; - while (iters--) { - s += from({bigLines}) | resplit('\n') | map(&len) | sum; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(StringResplitter_Small, iters) { - size_t s = 0; - while (iters--) { - s += from(smallLines) | resplit('\n') | map(&len) | sum; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_DRAW_LINE() - -BENCHMARK(StringSplit_Old, iters) { - size_t s = 0; - std::string line(kLine); - while (iters--) { - std::vector parts; - split(' ', line, parts); - s += parts.size(); - } - folly::doNotOptimizeAway(s); -} - - -BENCHMARK_RELATIVE(StringSplit_Gen_Vector, iters) { - size_t s = 0; - StringPiece line(kLine); - while (iters--) { - s += (split(line, ' ') | as()).size(); - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_DRAW_LINE() - -BENCHMARK(StringSplit_Old_ReuseVector, iters) { - size_t s = 0; - std::string line(kLine); - std::vector parts; - while (iters--) { - parts.clear(); - split(' ', line, parts); - s += parts.size(); - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(StringSplit_Gen_ReuseVector, iters) { - size_t s = 0; - StringPiece line(kLine); - std::vector parts; - while (iters--) { - parts.clear(); - split(line, ' ') | appendTo(parts); - s += parts.size(); - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(StringSplit_Gen, iters) { - size_t s = 0; - StringPiece line(kLine); - while (iters--) { - s += split(line, ' ') | count; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(StringSplit_Gen_Take, iters) { - size_t s = 0; - StringPiece line(kLine); - while (iters--) { - s += split(line, ' ') | take(10) | count; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_DRAW_LINE() - -BENCHMARK(StringUnsplit_Old, iters) { - size_t s = 0; - while (iters--) { - fbstring joined; - join(',', testStrVector, joined); - s += joined.size(); - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(StringUnsplit_Old_ReusedBuffer, iters) { - size_t s = 0; - fbstring joined; - while (iters--) { - joined.clear(); - join(',', testStrVector, joined); - s += joined.size(); - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(StringUnsplit_Gen, iters) { - size_t s = 0; - StringPiece line(kLine); - while (iters--) { - fbstring joined = from(testStrVector) | unsplit(','); - s += joined.size(); - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(StringUnsplit_Gen_ReusedBuffer, iters) { - size_t s = 0; - fbstring buffer; - while (iters--) { - buffer.clear(); - from(testStrVector) | unsplit(',', &buffer); - s += buffer.size(); - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_DRAW_LINE() - -void StringUnsplit_Gen(size_t iters, size_t joinSize) { - std::vector v; - BENCHMARK_SUSPEND { - FOR_EACH_RANGE(i, 0, joinSize) { - v.push_back(to(rand())); - } - } - size_t s = 0; - fbstring buffer; - while (iters--) { - buffer.clear(); - from(v) | unsplit(',', &buffer); - s += buffer.size(); - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_PARAM(StringUnsplit_Gen, 1000) -BENCHMARK_RELATIVE_PARAM(StringUnsplit_Gen, 2000) -BENCHMARK_RELATIVE_PARAM(StringUnsplit_Gen, 4000) -BENCHMARK_RELATIVE_PARAM(StringUnsplit_Gen, 8000) - -BENCHMARK_DRAW_LINE() - -fbstring records -= seq(1, 1000) - | mapped([](size_t i) { - return folly::to(i, ' ', i * i, ' ', i * i * i); - }) - | unsplit('\n'); - -BENCHMARK(Records_EachToTuple, iters) { - size_t s = 0; - for (size_t i = 0; i < iters; i += 1000) { - s += split(records, '\n') - | eachToTuple(' ') - | get<1>() - | sum; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(Records_VectorStringPieceReused, iters) { - size_t s = 0; - std::vector fields; - for (size_t i = 0; i < iters; i += 1000) { - s += split(records, '\n') - | mapped([&](StringPiece line) { - fields.clear(); - folly::split(' ', line, fields); - CHECK(fields.size() == 3); - return std::make_tuple( - folly::to(fields[0]), - folly::to(fields[1]), - StringPiece(fields[2])); - }) - | get<1>() - | sum; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(Records_VectorStringPiece, iters) { - size_t s = 0; - for (size_t i = 0; i < iters; i += 1000) { - s += split(records, '\n') - | mapped([](StringPiece line) { - std::vector fields; - folly::split(' ', line, fields); - CHECK(fields.size() == 3); - return std::make_tuple( - folly::to(fields[0]), - folly::to(fields[1]), - StringPiece(fields[2])); - }) - | get<1>() - | sum; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_RELATIVE(Records_VectorString, iters) { - size_t s = 0; - for (size_t i = 0; i < iters; i += 1000) { - s += split(records, '\n') - | mapped([](StringPiece line) { - std::vector fields; - folly::split(' ', line, fields); - CHECK(fields.size() == 3); - return std::make_tuple( - folly::to(fields[0]), - folly::to(fields[1]), - StringPiece(fields[2])); - }) - | get<1>() - | sum; - } - folly::doNotOptimizeAway(s); -} - -BENCHMARK_DRAW_LINE() - -BENCHMARK(ByLine_Pipes, iters) { - std::thread thread; - int rfd; - int wfd; - BENCHMARK_SUSPEND { - int p[2]; - CHECK_ERR(::pipe(p)); - rfd = p[0]; - wfd = p[1]; - thread = std::thread([wfd, iters] { - char x = 'x'; - PCHECK(::write(wfd, &x, 1) == 1); // signal startup - FILE* f = fdopen(wfd, "w"); - PCHECK(f); - for (int i = 1; i <= iters; ++i) { - fprintf(f, "%d\n", i); - } - fclose(f); - }); - char buf; - PCHECK(::read(rfd, &buf, 1) == 1); // wait for startup - } - - auto s = byLine(File(rfd)) | eachTo() | sum; - folly::doNotOptimizeAway(s); - - BENCHMARK_SUSPEND { - ::close(rfd); - CHECK_EQ(s, int64_t(iters) * (iters + 1) / 2); - thread.join(); - } -} - -// ============================================================================ -// folly/experimental/test/GenBenchmark.cpp relative time/iter iters/s -// ============================================================================ -// Sum_Basic_NoGen 374.39ns 2.67M -// Sum_Basic_Gen 101.05% 370.48ns 2.70M -// ---------------------------------------------------------------------------- -// Sum_Vector_NoGen 198.84ns 5.03M -// Sum_Vector_Gen 98.14% 202.60ns 4.94M -// ---------------------------------------------------------------------------- -// Member 4.56us 219.11K -// MapMember 400.21% 1.14us 876.89K -// ---------------------------------------------------------------------------- -// Count_Vector_NoGen 13.99us 71.47K -// Count_Vector_Gen 106.73% 13.11us 76.28K -// ---------------------------------------------------------------------------- -// Fib_Sum_NoGen 4.27us 234.07K -// Fib_Sum_Gen 43.18% 9.90us 101.06K -// Fib_Sum_Gen_Static 92.08% 4.64us 215.53K -// ---------------------------------------------------------------------------- -// VirtualGen_0Virtual 12.07us 82.83K -// VirtualGen_1Virtual 32.46% 37.19us 26.89K -// VirtualGen_2Virtual 24.36% 49.55us 20.18K -// VirtualGen_3Virtual 18.16% 66.49us 15.04K -// ---------------------------------------------------------------------------- -// Concat_NoGen 1.90us 527.40K -// Concat_Gen 86.73% 2.19us 457.39K -// ---------------------------------------------------------------------------- -// Composed_NoGen 546.18ns 1.83M -// Composed_Gen 100.41% 543.93ns 1.84M -// Composed_GenRegular 100.42% 543.92ns 1.84M -// ---------------------------------------------------------------------------- -// Sample 146.68ms 6.82 -// ---------------------------------------------------------------------------- -// StringResplitter_Big 124.80us 8.01K -// StringResplitter_Small 15.11% 825.74us 1.21K -// ---------------------------------------------------------------------------- -// StringSplit_Old 393.49ns 2.54M -// StringSplit_Gen_Vector 121.47% 323.93ns 3.09M -// ---------------------------------------------------------------------------- -// StringSplit_Old_ReuseVector 80.77ns 12.38M -// StringSplit_Gen_ReuseVector 102.02% 79.17ns 12.63M -// StringSplit_Gen 123.78% 65.25ns 15.32M -// StringSplit_Gen_Take 123.44% 65.43ns 15.28M -// ---------------------------------------------------------------------------- -// StringUnsplit_Old 29.36us 34.06K -// StringUnsplit_Old_ReusedBuffer 100.25% 29.29us 34.14K -// StringUnsplit_Gen 103.38% 28.40us 35.21K -// StringUnsplit_Gen_ReusedBuffer 109.85% 26.73us 37.41K -// ---------------------------------------------------------------------------- -// StringUnsplit_Gen(1000) 32.30us 30.96K -// StringUnsplit_Gen(2000) 49.75% 64.93us 15.40K -// StringUnsplit_Gen(4000) 24.74% 130.60us 7.66K -// StringUnsplit_Gen(8000) 12.31% 262.35us 3.81K -// ---------------------------------------------------------------------------- -// Records_EachToTuple 75.03ns 13.33M -// Records_VectorStringPieceReused 81.79% 91.74ns 10.90M -// Records_VectorStringPiece 36.47% 205.77ns 4.86M -// Records_VectorString 12.90% 581.70ns 1.72M -// ---------------------------------------------------------------------------- -// ByLine_Pipes 121.68ns 8.22M -// ============================================================================ - -int main(int argc, char *argv[]) { - google::ParseCommandLineFlags(&argc, &argv, true); - initStringResplitterBenchmark(); - runBenchmarks(); - return 0; -} diff --git a/folly/experimental/test/GenTest.cpp b/folly/experimental/test/GenTest.cpp deleted file mode 100644 index 35c1414b..00000000 --- a/folly/experimental/test/GenTest.cpp +++ /dev/null @@ -1,1425 +0,0 @@ -/* - * Copyright 2014 Facebook, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include - -#include "folly/FBString.h" -#include "folly/FBVector.h" -#include "folly/Format.h" -#include "folly/MapUtil.h" -#include "folly/Memory.h" -#include "folly/dynamic.h" -#include "folly/experimental/CombineGen.h" -#include "folly/experimental/FileGen.h" -#include "folly/experimental/Gen.h" -#include "folly/experimental/StringGen.h" -#include "folly/experimental/TestUtil.h" - -using namespace folly::gen; -using namespace folly; -using std::make_tuple; -using std::ostream; -using std::pair; -using std::set; -using std::string; -using std::tuple; -using std::unique_ptr; -using std::vector; - -#define EXPECT_SAME(A, B) \ - static_assert(std::is_same::value, "Mismatched: " #A ", " #B) -EXPECT_SAME(int&&, typename ArgumentReference::type); -EXPECT_SAME(int&, typename ArgumentReference::type); -EXPECT_SAME(const int&, typename ArgumentReference::type); -EXPECT_SAME(const int&, typename ArgumentReference::type); - -template -ostream& operator<<(ostream& os, const set& values) { - return os << from(values); -} - -template -ostream& operator<<(ostream& os, const vector& values) { - os << "["; - for (auto& value : values) { - if (&value != &values.front()) { - os << " "; - } - os << value; - } - return os << "]"; -} - -auto square = [](int x) { return x * x; }; -auto add = [](int a, int b) { return a + b; }; -auto multiply = [](int a, int b) { return a * b; }; - -auto product = foldl(1, multiply); - -template -ostream& operator<<(ostream& os, const pair& pair) { - return os << "(" << pair.first << ", " << pair.second << ")"; -} - -TEST(Gen, Count) { - auto gen = seq(1, 10); - EXPECT_EQ(10, gen | count); - EXPECT_EQ(5, gen | take(5) | count); -} - -TEST(Gen, Sum) { - auto gen = seq(1, 10); - EXPECT_EQ((1 + 10) * 10 / 2, gen | sum); - EXPECT_EQ((1 + 5) * 5 / 2, gen | take(5) | sum); -} - -TEST(Gen, Foreach) { - auto gen = seq(1, 4); - int accum = 0; - gen | [&](int x) { accum += x; }; - EXPECT_EQ(10, accum); - int accum2 = 0; - gen | take(3) | [&](int x) { accum2 += x; }; - EXPECT_EQ(6, accum2); -} - -TEST(Gen, Map) { - auto expected = vector{4, 9, 16}; - auto gen = from({2, 3, 4}) | map(square); - EXPECT_EQ((vector{4, 9, 16}), gen | as()); - EXPECT_EQ((vector{4, 9}), gen | take(2) | as()); -} - -TEST(Gen, Member) { - struct Counter { - Counter(int start = 0) - : c(start) - {} - - int count() const { return c; } - int incr() { return ++c; } - - int& ref() { return c; } - const int& ref() const { return c; } - private: - int c; - }; - auto counters = seq(1, 10) | eachAs() | as(); - EXPECT_EQ(10 * (1 + 10) / 2, - from(counters) - | member(&Counter::count) - | sum); - EXPECT_EQ(10 * (2 + 11) / 2, - from(counters) - | member(&Counter::incr) - | sum); - EXPECT_EQ(10 * (2 + 11) / 2, - from(counters) - | member(&Counter::count) - | sum); - - // type-verifications - auto m = empty(); - auto c = empty(); - m | member(&Counter::incr) | assert_type(); - m | member(&Counter::count) | assert_type(); - m | member(&Counter::count) | assert_type(); - m | member(&Counter::ref) | assert_type(); - m | member(&Counter::ref) | assert_type(); - c | member(&Counter::ref) | assert_type(); -} - -TEST(Gen, Field) { - struct X { - X() : a(2), b(3), c(4), d(b) {} - - const int a; - int b; - mutable int c; - int& d; // can't access this with a field pointer. - }; - - std::vector xs(1); - EXPECT_EQ(2, from(xs) - | field(&X::a) - | first); - EXPECT_EQ(3, from(xs) - | field(&X::b) - | first); - EXPECT_EQ(4, from(xs) - | field(&X::c) - | first); - // type-verification - empty() | field(&X::a) | assert_type(); - empty() | field(&X::b) | assert_type(); - empty() | field(&X::c) | assert_type(); - empty() | field(&X::a) | assert_type(); - empty() | field(&X::b) | assert_type(); - empty() | field(&X::c) | assert_type(); - // references don't imply ownership so they're not moved - empty() | field(&X::a) | assert_type(); - empty() | field(&X::b) | assert_type(); - // 'mutable' has no effect on field pointers, by C++ spec - empty() | field(&X::c) | assert_type(); - - // can't form pointer-to-reference field: empty() | field(&X::d) -} - -TEST(Gen, Seq) { - // cover the fenceposts of the loop unrolling - for (int n = 1; n < 100; ++n) { - EXPECT_EQ(n, seq(1, n) | count); - EXPECT_EQ(n + 1, seq(1) | take(n + 1) | count); - } -} - -TEST(Gen, Range) { - // cover the fenceposts of the loop unrolling - for (int n = 1; n < 100; ++n) { - EXPECT_EQ(range(0, n) | count, n); - } -} - -TEST(Gen, FromIterators) { - vector source {2, 3, 5, 7, 11}; - auto gen = from(makeRange(source.begin() + 1, source.end() - 1)); - EXPECT_EQ(3 * 5 * 7, gen | product); -} - -TEST(Gen, FromMap) { - auto source = seq(0, 10) - | map([](int i) { return std::make_pair(i, i * i); }) - | as>(); - auto gen = fromConst(source) - | map([&](const std::pair& p) { - return p.second - p.first; - }); - EXPECT_EQ(330, gen | sum); -} - -TEST(Gen, Filter) { - const auto expected = vector{1, 2, 4, 5, 7, 8}; - auto actual = - seq(1, 9) - | filter([](int x) { return x % 3; }) - | as>(); - EXPECT_EQ(expected, actual); -} - -TEST(Gen, Contains) { - { - auto gen = - seq(1, 9) - | map(square); - EXPECT_TRUE(gen | contains(49)); - EXPECT_FALSE(gen | contains(50)); - } - { - auto gen = - seq(1) // infinite, to prove laziness - | map(square) - | eachTo(); - - // std::string gen, const char* needle - EXPECT_TRUE(gen | take(9999) | contains("49")); - } -} - -TEST(Gen, Take) { - { - auto expected = vector{1, 4, 9, 16}; - auto actual = - seq(1, 1000) - | mapped([](int x) { return x * x; }) - | take(4) - | as>(); - EXPECT_EQ(expected, actual); - } - { - auto expected = vector{ 0, 1, 4, 5, 8 }; - auto actual - = ((seq(0) | take(2)) + - (seq(4) | take(2)) + - (seq(8) | take(2))) - | take(5) - | as(); - EXPECT_EQ(expected, actual); - } - { - auto expected = vector{ 0, 1, 4, 5, 8 }; - auto actual - = seq(0) - | mapped([](int i) { - return seq(i * 4) | take(2); - }) - | concat - | take(5) - | as(); - EXPECT_EQ(expected, actual); - } -} - -TEST(Gen, Sample) { - std::mt19937 rnd(42); - - auto sampler = - seq(1, 100) - | sample(50, rnd); - std::unordered_map hits; - const int kNumIters = 80; - for (int i = 0; i < kNumIters; i++) { - auto vec = sampler | as>(); - EXPECT_EQ(vec.size(), 50); - auto uniq = fromConst(vec) | as>(); - EXPECT_EQ(uniq.size(), vec.size()); // sampling without replacement - for (auto v: vec) { - ++hits[v]; - } - } - - // In 80 separate samples of our range, we should have seen every value - // at least once and no value all 80 times. (The odds of either of those - // events is 1/2^80). - EXPECT_EQ(hits.size(), 100); - for (auto hit: hits) { - EXPECT_GT(hit.second, 0); - EXPECT_LT(hit.second, kNumIters); - } - - auto small = - seq(1, 5) - | sample(10); - EXPECT_EQ((small | sum), 15); - EXPECT_EQ((small | take(3) | count), 3); -} - -TEST(Gen, Skip) { - auto gen = - seq(1, 1000) - | mapped([](int x) { return x * x; }) - | skip(4) - | take(4); - EXPECT_EQ((vector{25, 36, 49, 64}), gen | as()); -} - -TEST(Gen, Until) { - { - auto expected = vector{1, 4, 9, 16}; - auto actual - = seq(1, 1000) - | mapped([](int x) { return x * x; }) - | until([](int x) { return x > 20; }) - | as>(); - EXPECT_EQ(expected, actual); - } - { - auto expected = vector{ 0, 1, 4, 5, 8 }; - auto actual - = ((seq(0) | until([](int i) { return i > 1; })) + - (seq(4) | until([](int i) { return i > 5; })) + - (seq(8) | until([](int i) { return i > 9; }))) - | until([](int i) { return i > 8; }) - | as>(); - EXPECT_EQ(expected, actual); - } - /* - { - auto expected = vector{ 0, 1, 5, 6, 10 }; - auto actual - = seq(0) - | mapped([](int i) { - return seq(i * 5) | until([=](int j) { return j > i * 5 + 1; }); - }) - | concat - | until([](int i) { return i > 10; }) - | as>(); - EXPECT_EQ(expected, actual); - } - */ -} - -auto even = [](int i) -> bool { return i % 2 == 0; }; -auto odd = [](int i) -> bool { return i % 2 == 1; }; - -TEST(CombineGen, Interleave) { - { // large (infinite) base, small container - auto base = seq(1) | filter(odd); - auto toInterleave = seq(1, 6) | filter(even); - auto interleaved = base | interleave(toInterleave | as()); - EXPECT_EQ(interleaved | as(), vector({1, 2, 3, 4, 5, 6})); - } - { // small base, large container - auto base = seq(1) | filter(odd) | take(3); - auto toInterleave = seq(1) | filter(even) | take(50); - auto interleaved = base | interleave(toInterleave | as()); - EXPECT_EQ(interleaved | as(), - vector({1, 2, 3, 4, 5, 6})); - } -} - -TEST(CombineGen, Zip) { - auto base0 = seq(1); - // We rely on std::move(fbvector) emptying the source vector - auto zippee = fbvector{"one", "two", "three"}; - { - auto combined = base0 - | zip(zippee) - | as(); - ASSERT_EQ(combined.size(), 3); - EXPECT_EQ(std::get<0>(combined[0]), 1); - EXPECT_EQ(std::get<1>(combined[0]), "one"); - EXPECT_EQ(std::get<0>(combined[1]), 2); - EXPECT_EQ(std::get<1>(combined[1]), "two"); - EXPECT_EQ(std::get<0>(combined[2]), 3); - EXPECT_EQ(std::get<1>(combined[2]), "three"); - ASSERT_FALSE(zippee.empty()); - EXPECT_FALSE(zippee.front().empty()); // shouldn't have been move'd - } - - { // same as top, but using std::move. - auto combined = base0 - | zip(std::move(zippee)) - | as(); - ASSERT_EQ(combined.size(), 3); - EXPECT_EQ(std::get<0>(combined[0]), 1); - EXPECT_TRUE(zippee.empty()); - } - - { // same as top, but base is truncated - auto baseFinite = seq(1) | take(1); - auto combined = baseFinite - | zip(vector{"one", "two", "three"}) - | as(); - ASSERT_EQ(combined.size(), 1); - EXPECT_EQ(std::get<0>(combined[0]), 1); - EXPECT_EQ(std::get<1>(combined[0]), "one"); - } -} - -TEST(CombineGen, TupleFlatten) { - vector> intStringTupleVec{ - tuple{1, "1"}, - tuple{2, "2"}, - tuple{3, "3"}, - }; - - vector> charTupleVec{ - tuple{'A'}, - tuple{'B'}, - tuple{'C'}, - tuple{'D'}, - }; - - vector doubleVec{ - 1.0, - 4.0, - 9.0, - 16.0, - 25.0, - }; - - auto zipped1 = from(intStringTupleVec) - | zip(charTupleVec) - | assert_type, tuple>>() - | as(); - EXPECT_EQ(std::get<0>(zipped1[0]), std::make_tuple(1, "1")); - EXPECT_EQ(std::get<1>(zipped1[0]), std::make_tuple('A')); - - auto zipped2 = from(zipped1) - | tuple_flatten - | assert_type&&>() - | as(); - ASSERT_EQ(zipped2.size(), 3); - EXPECT_EQ(zipped2[0], std::make_tuple(1, "1", 'A')); - - auto zipped3 = from(charTupleVec) - | zip(intStringTupleVec) - | tuple_flatten - | assert_type&&>() - | as(); - ASSERT_EQ(zipped3.size(), 3); - EXPECT_EQ(zipped3[0], std::make_tuple('A', 1, "1")); - - auto zipped4 = from(intStringTupleVec) - | zip(doubleVec) - | tuple_flatten - | assert_type&&>() - | as(); - ASSERT_EQ(zipped4.size(), 3); - EXPECT_EQ(zipped4[0], std::make_tuple(1, "1", 1.0)); - - auto zipped5 = from(doubleVec) - | zip(doubleVec) - | assert_type>() - | tuple_flatten // essentially a no-op - | assert_type&&>() - | as(); - ASSERT_EQ(zipped5.size(), 5); - EXPECT_EQ(zipped5[0], std::make_tuple(1.0, 1.0)); - - auto zipped6 = from(intStringTupleVec) - | zip(charTupleVec) - | tuple_flatten - | zip(doubleVec) - | tuple_flatten - | assert_type&&>() - | as(); - ASSERT_EQ(zipped6.size(), 3); - EXPECT_EQ(zipped6[0], std::make_tuple(1, "1", 'A', 1.0)); -} - -TEST(Gen, Composed) { - // Operator, Operator - auto valuesOf = - filter([](Optional& o) { return o.hasValue(); }) - | map([](Optional& o) -> int& { return o.value(); }); - std::vector> opts { - none, 4, none, 6, none - }; - EXPECT_EQ(4 * 4 + 6 * 6, from(opts) | valuesOf | map(square) | sum); - // Operator, Sink - auto sumOpt = valuesOf | sum; - EXPECT_EQ(10, from(opts) | sumOpt); -} - -TEST(Gen, Chain) { - std::vector nums {2, 3, 5, 7}; - std::map mappings { { 3, 9}, {5, 25} }; - auto gen = from(nums) + (from(mappings) | get<1>()); - EXPECT_EQ(51, gen | sum); - EXPECT_EQ(5, gen | take(2) | sum); - EXPECT_EQ(26, gen | take(5) | sum); -} - -TEST(Gen, Concat) { - std::vector> nums {{2, 3}, {5, 7}}; - auto gen = from(nums) | rconcat; - EXPECT_EQ(17, gen | sum); - EXPECT_EQ(10, gen | take(3) | sum); -} - -TEST(Gen, ConcatGen) { - auto gen = seq(1, 10) - | map([](int i) { return seq(1, i); }) - | concat; - EXPECT_EQ(220, gen | sum); - EXPECT_EQ(10, gen | take(6) | sum); -} - -TEST(Gen, ConcatAlt) { - std::vector> nums {{2, 3}, {5, 7}}; - auto actual = from(nums) - | map([](std::vector& v) { return from(v); }) - | concat - | sum; - auto expected = 17; - EXPECT_EQ(expected, actual); -} - -TEST(Gen, Order) { - auto expected = vector{0, 3, 5, 6, 7, 8, 9}; - auto actual = - from({8, 6, 7, 5, 3, 0, 9}) - | order - | as(); - EXPECT_EQ(expected, actual); -} - -TEST(Gen, OrderMoved) { - auto expected = vector{0, 9, 25, 36, 49, 64, 81}; - auto actual = - from({8, 6, 7, 5, 3, 0, 9}) - | move - | order - | map(square) - | as(); - EXPECT_EQ(expected, actual); -} - -TEST(Gen, OrderTake) { - auto expected = vector{9, 8, 7}; - auto actual = - from({8, 6, 7, 5, 3, 0, 9}) - | orderByDescending(square) - | take(3) - | as(); - EXPECT_EQ(expected, actual); -} - -TEST(Gen, Distinct) { - auto expected = vector{3, 1, 2}; - auto actual = - from({3, 1, 3, 2, 1, 2, 3}) - | distinct - | as(); - EXPECT_EQ(expected, actual); -} - -TEST(Gen, DistinctBy) { // 0 1 4 9 6 5 6 9 4 1 0 - auto expected = vector{0, 1, 2, 3, 4, 5}; - auto actual = - seq(0, 100) - | distinctBy([](int i) { return i * i % 10; }) - | as(); - EXPECT_EQ(expected, actual); -} - -TEST(Gen, DistinctMove) { // 0 1 4 9 6 5 6 9 4 1 0 - auto expected = vector{0, 1, 2, 3, 4, 5}; - auto actual = - seq(0, 100) - | mapped([](int i) { return std::unique_ptr(new int(i)); }) - // see comment below about selector parameters for Distinct - | distinctBy([](const std::unique_ptr& pi) { return *pi * *pi % 10; }) - | mapped([](std::unique_ptr pi) { return *pi; }) - | as(); - - // NOTE(tjackson): the following line intentionally doesn't work: - // | distinctBy([](std::unique_ptr pi) { return *pi * *pi % 10; }) - // This is because distinctBy because the selector intentionally requires a - // const reference. If it required a move-reference, the value might get - // gutted by the selector before said value could be passed to downstream - // operators. - EXPECT_EQ(expected, actual); -} - -TEST(Gen, MinBy) { - EXPECT_EQ(7, seq(1, 10) - | minBy([](int i) -> double { - double d = i - 6.8; - return d * d; - })); -} - -TEST(Gen, MaxBy) { - auto gen = from({"three", "eleven", "four"}); - - EXPECT_EQ("eleven", gen | maxBy(&strlen)); -} - -TEST(Gen, Append) { - fbstring expected = "facebook"; - fbstring actual = "face"; - from(StringPiece("book")) | appendTo(actual); - EXPECT_EQ(expected, actual); -} - -TEST(Gen, FromRValue) { - { - // AFAICT The C++ Standard does not specify what happens to the rvalue - // reference of a std::vector when it is used as the 'other' for an rvalue - // constructor. Use fbvector because we're sure its size will be zero in - // this case. - folly::fbvector v({1,2,3,4}); - auto q1 = from(v); - EXPECT_EQ(v.size(), 4); // ensure that the lvalue version was called! - auto expected = 1 * 2 * 3 * 4; - EXPECT_EQ(expected, q1 | product); - - auto q2 = from(std::move(v)); - EXPECT_EQ(v.size(), 0); // ensure that rvalue version was called - EXPECT_EQ(expected, q2 | product); - } - { - auto expected = 7; - auto q = from([] {return vector({3,7,5}); }()); - EXPECT_EQ(expected, q | max); - } - { - for (auto size: {5, 1024, 16384, 1<<20}) { - auto q1 = from(vector(size, 2)); - auto q2 = from(vector(size, 3)); - // If the rvalue specialization is broken/gone, then the compiler will - // (disgustingly!) just store a *reference* to the temporary object, - // which is bad. Try to catch this by allocating two temporary vectors - // of the same size, so that they'll probably use the same underlying - // buffer if q1's vector is destructed before q2's vector is constructed. - EXPECT_EQ(size * 2 + size * 3, (q1 | sum) + (q2 | sum)); - } - } - { - auto q = from(set{1,2,3,2,1}); - EXPECT_EQ(q | sum, 6); - } -} - -TEST(Gen, OrderBy) { - auto expected = vector{5, 6, 4, 7, 3, 8, 2, 9, 1, 10}; - auto actual = - seq(1, 10) - | orderBy([](int x) { return (5.1 - x) * (5.1 - x); }) - | as(); - EXPECT_EQ(expected, actual); -} - -TEST(Gen, Foldl) { - int expected = 2 * 3 * 4 * 5; - auto actual = - seq(2, 5) - | foldl(1, multiply); - EXPECT_EQ(expected, actual); -} - -TEST(Gen, Reduce) { - int expected = 2 + 3 + 4 + 5; - auto actual = seq(2, 5) | reduce(add); - EXPECT_EQ(expected, actual); -} - -TEST(Gen, ReduceBad) { - auto gen = seq(1) | take(0); - try { - EXPECT_TRUE(true); - gen | reduce(add); - EXPECT_TRUE(false); - } catch (...) { - } -} - -TEST(Gen, Moves) { - std::vector> ptrs; - ptrs.emplace_back(new int(1)); - EXPECT_NE(ptrs.front().get(), nullptr); - auto ptrs2 = from(ptrs) | move | as(); - EXPECT_EQ(ptrs.front().get(), nullptr); - EXPECT_EQ(**ptrs2.data(), 1); -} - -TEST(Gen, First) { - auto gen = - seq(0) - | filter([](int x) { return x > 3; }); - EXPECT_EQ(4, gen | first); -} - -TEST(Gen, FromCopy) { - vector v {3, 5}; - auto src = from(v); - auto copy = fromCopy(v); - EXPECT_EQ(8, src | sum); - EXPECT_EQ(8, copy | sum); - v[1] = 7; - EXPECT_EQ(10, src | sum); - EXPECT_EQ(8, copy | sum); -} - -TEST(Gen, Get) { - std::map pairs { - {1, 1}, - {2, 4}, - {3, 9}, - {4, 16}, - }; - auto pairSrc = from(pairs); - auto keys = pairSrc | get<0>(); - auto values = pairSrc | get<1>(); - EXPECT_EQ(10, keys | sum); - EXPECT_EQ(30, values | sum); - EXPECT_EQ(30, keys | map(square) | sum); - pairs[5] = 25; - EXPECT_EQ(15, keys | sum); - EXPECT_EQ(55, values | sum); - - vector> tuples { - make_tuple(1, 1, 1), - make_tuple(2, 4, 8), - make_tuple(3, 9, 27), - }; - EXPECT_EQ(36, from(tuples) | get<2>() | sum); -} - -TEST(Gen, Any) { - EXPECT_TRUE(seq(0) | any); - EXPECT_TRUE(seq(0, 1) | any); - EXPECT_TRUE(seq(0, 10) | any([](int i) { return i == 7; })); - EXPECT_FALSE(seq(0, 10) | any([](int i) { return i == 11; })); - - EXPECT_TRUE(from({1}) | any); - EXPECT_FALSE(range(0, 0) | any); - EXPECT_FALSE(from({1}) | take(0) | any); -} - -TEST(Gen, All) { - EXPECT_TRUE(seq(0, 10) | all([](int i) { return i < 11; })); - EXPECT_FALSE(seq(0, 10) | all([](int i) { return i < 5; })); - EXPECT_FALSE(seq(0) | take(9999) | all([](int i) { return i < 10; })); - - // empty lists satisfies all - EXPECT_TRUE(seq(0) | take(0) | all([](int i) { return i < 50; })); - EXPECT_TRUE(seq(0) | take(0) | all([](int i) { return i > 50; })); -} - -TEST(Gen, Yielders) { - auto gen = GENERATOR(int) { - for (int i = 1; i <= 5; ++i) { - yield(i); - } - yield(7); - for (int i = 3; ; ++i) { - yield(i * i); - } - }; - vector expected { - 1, 2, 3, 4, 5, 7, 9, 16, 25 - }; - EXPECT_EQ(expected, gen | take(9) | as()); -} - -TEST(Gen, NestedYield) { - auto nums = GENERATOR(int) { - for (int i = 1; ; ++i) { - yield(i); - } - }; - auto gen = GENERATOR(int) { - nums | take(10) | yield; - seq(1, 5) | [&](int i) { - yield(i); - }; - }; - EXPECT_EQ(70, gen | sum); -} - -TEST(Gen, MapYielders) { - auto gen = seq(1, 5) - | map([](int n) { - return GENERATOR(int) { - int i; - for (i = 1; i < n; ++i) - yield(i); - for (; i >= 1; --i) - yield(i); - }; - }) - | concat; - vector expected { - 1, - 1, 2, 1, - 1, 2, 3, 2, 1, - 1, 2, 3, 4, 3, 2, 1, - 1, 2, 3, 4, 5, 4, 3, 2, 1, - }; - EXPECT_EQ(expected, gen | as()); -} - -TEST(Gen, VirtualGen) { - VirtualGen v(seq(1, 10)); - EXPECT_EQ(55, v | sum); - v = v | map(square); - EXPECT_EQ(385, v | sum); - v = v | take(5); - EXPECT_EQ(55, v | sum); - EXPECT_EQ(30, v | take(4) | sum); -} - - -TEST(Gen, CustomType) { - struct Foo{ - int y; - }; - auto gen = from({Foo{2}, Foo{3}}) - | map([](const Foo& f) { return f.y; }); - EXPECT_EQ(5, gen | sum); -} - -TEST(Gen, NoNeedlessCopies) { - auto gen = seq(1, 5) - | map([](int x) { return unique_ptr(new int(x)); }) - | map([](unique_ptr p) { return p; }) - | map([](unique_ptr&& p) { return std::move(p); }) - | map([](const unique_ptr& p) { return *p; }); - EXPECT_EQ(15, gen | sum); - EXPECT_EQ(6, gen | take(3) | sum); -} - -namespace { - -class TestIntSeq : public GenImpl { - public: - TestIntSeq() { } - - template - bool apply(Body&& body) const { - for (int i = 1; i < 6; ++i) { - if (!body(i)) { - return false; - } - } - return true; - } - - TestIntSeq(TestIntSeq&&) = default; - TestIntSeq& operator=(TestIntSeq&&) = default; - TestIntSeq(const TestIntSeq&) = delete; - TestIntSeq& operator=(const TestIntSeq&) = delete; -}; - -} // namespace - -TEST(Gen, NoGeneratorCopies) { - EXPECT_EQ(15, TestIntSeq() | sum); - auto x = TestIntSeq() | take(3); - EXPECT_EQ(6, std::move(x) | sum); -} - -TEST(Gen, FromArray) { - int source[] = {2, 3, 5, 7}; - auto gen = from(source); - EXPECT_EQ(2 * 3 * 5 * 7, gen | product); -} - -TEST(Gen, FromStdArray) { - std::array source {{2, 3, 5, 7}}; - auto gen = from(source); - EXPECT_EQ(2 * 3 * 5 * 7, gen | product); -} - -TEST(Gen, StringConcat) { - auto gen = seq(1, 10) - | map([](int n) { return folly::to(n); }) - | rconcat; - EXPECT_EQ("12345678910", gen | as()); -} - -struct CopyCounter { - static int alive; - int copies; - int moves; - - CopyCounter() : copies(0), moves(0) { - ++alive; - } - - CopyCounter(CopyCounter&& source) { - *this = std::move(source); - ++alive; - } - - CopyCounter(const CopyCounter& source) { - *this = source; - ++alive; - } - - ~CopyCounter() { - --alive; - } - - CopyCounter& operator=(const CopyCounter& source) { - this->copies = source.copies + 1; - this->moves = source.moves; - return *this; - } - - CopyCounter& operator=(CopyCounter&& source) { - this->copies = source.copies; - this->moves = source.moves + 1; - return *this; - } -}; - -int CopyCounter::alive = 0; - -TEST(Gen, CopyCount) { - vector originals; - originals.emplace_back(); - EXPECT_EQ(1, originals.size()); - EXPECT_EQ(0, originals.back().copies); - - vector copies = from(originals) | as(); - EXPECT_EQ(1, copies.back().copies); - EXPECT_EQ(0, copies.back().moves); - - vector moves = from(originals) | move | as(); - EXPECT_EQ(0, moves.back().copies); - EXPECT_EQ(1, moves.back().moves); -} - -// test dynamics with various layers of nested arrays. -TEST(Gen, Dynamic) { - dynamic array1 = {1, 2}; - EXPECT_EQ(dynamic(3), from(array1) | sum); - dynamic array2 = {{1}, {1, 2}}; - EXPECT_EQ(dynamic(4), from(array2) | rconcat | sum); - dynamic array3 = {{{1}}, {{1}, {1, 2}}}; - EXPECT_EQ(dynamic(5), from(array3) | rconcat | rconcat | sum); -} - -TEST(Gen, DynamicObject) { - const dynamic obj = dynamic::object(1, 2)(3, 4); - EXPECT_EQ(dynamic(4), from(obj.keys()) | sum); - EXPECT_EQ(dynamic(6), from(obj.values()) | sum); - EXPECT_EQ(dynamic(4), from(obj.items()) | get<0>() | sum); - EXPECT_EQ(dynamic(6), from(obj.items()) | get<1>() | sum); -} - -TEST(Gen, Collect) { - auto s = from({7, 6, 5, 4, 3}) | as>(); - EXPECT_EQ(s.size(), 5); -} - -TEST(StringGen, EmptySplit) { - auto collect = eachTo() | as(); - { - auto pieces = split("", ',') | collect; - EXPECT_EQ(0, pieces.size()); - } - - // The last delimiter is eaten, just like std::getline - { - auto pieces = split(",", ',') | collect; - EXPECT_EQ(1, pieces.size()); - EXPECT_EQ("", pieces[0]); - } - - { - auto pieces = split(",,", ',') | collect; - EXPECT_EQ(2, pieces.size()); - EXPECT_EQ("", pieces[0]); - EXPECT_EQ("", pieces[1]); - } - - { - auto pieces = split(",,", ',') | take(1) | collect; - EXPECT_EQ(1, pieces.size()); - EXPECT_EQ("", pieces[0]); - } -} - -TEST(Gen, Cycle) { - { - auto s = from({1, 2}); - EXPECT_EQ((vector { 1, 2, 1, 2, 1 }), - s | cycle | take(5) | as()); - } - { - auto s = from({1, 2}); - EXPECT_EQ((vector { 1, 2, 1, 2 }), - s | cycle(2) | as()); - } - { - auto s = from({1, 2, 3}); - EXPECT_EQ((vector { 1, 2, 1, 2, 1 }), - s | take(2) | cycle | take(5) | as()); - } - { - auto s = empty(); - EXPECT_EQ((vector { }), - s | cycle | take(4) | as()); - } - { - int count = 3; - int* pcount = &count; - auto countdown = GENERATOR(int) { - ASSERT_GE(*pcount, 0) - << "Cycle should have stopped when it didnt' get values!"; - for (int i = 1; i <= *pcount; ++i) { - yield(i); - } - --*pcount; - }; - auto s = countdown; - EXPECT_EQ((vector { 1, 2, 3, 1, 2, 1}), - s | cycle | as()); - } -} - -TEST(Gen, Dereference) { - { - const int x = 4, y = 2; - auto s = from({&x, nullptr, &y}); - EXPECT_EQ(6, s | dereference | sum); - } - { - vector a { 1, 2 }; - vector b { 3, 4 }; - vector*> pv { &a, nullptr, &b }; - from(pv) - | dereference - | [&](vector& v) { - v.push_back(5); - }; - EXPECT_EQ(3, a.size()); - EXPECT_EQ(3, b.size()); - EXPECT_EQ(5, a.back()); - EXPECT_EQ(5, b.back()); - } - { - vector> maps { - { - { 2, 31 }, - { 3, 41 }, - }, - { - { 3, 52 }, - { 4, 62 }, - }, - { - { 4, 73 }, - { 5, 83 }, - }, - }; - EXPECT_EQ( - 93, - from(maps) - | map([](std::map& m) { - return get_ptr(m, 3); - }) - | dereference - | sum); - } - { - vector> ups; - ups.emplace_back(new int(3)); - ups.emplace_back(); - ups.emplace_back(new int(7)); - EXPECT_EQ(10, from(ups) | dereference | sum); - EXPECT_EQ(10, from(ups) | move | dereference | sum); - } -} - -TEST(StringGen, Split) { - auto collect = eachTo() | as(); - { - auto pieces = split("hello,, world, goodbye, meow", ',') | collect; - EXPECT_EQ(5, pieces.size()); - EXPECT_EQ("hello", pieces[0]); - EXPECT_EQ("", pieces[1]); - EXPECT_EQ(" world", pieces[2]); - EXPECT_EQ(" goodbye", pieces[3]); - EXPECT_EQ(" meow", pieces[4]); - } - - { - auto pieces = split("hello,, world, goodbye, meow", ',') - | take(3) | collect; - EXPECT_EQ(3, pieces.size()); - EXPECT_EQ("hello", pieces[0]); - EXPECT_EQ("", pieces[1]); - EXPECT_EQ(" world", pieces[2]); - } - - { - auto pieces = split("hello,, world, goodbye, meow", ',') - | take(5) | collect; - EXPECT_EQ(5, pieces.size()); - EXPECT_EQ("hello", pieces[0]); - EXPECT_EQ("", pieces[1]); - EXPECT_EQ(" world", pieces[2]); - } -} - -TEST(StringGen, EmptyResplit) { - auto collect = eachTo() | as(); - { - auto pieces = from({""}) | resplit(',') | collect; - EXPECT_EQ(0, pieces.size()); - } - - // The last delimiter is eaten, just like std::getline - { - auto pieces = from({","}) | resplit(',') | collect; - EXPECT_EQ(1, pieces.size()); - EXPECT_EQ("", pieces[0]); - } - - { - auto pieces = from({",,"}) | resplit(',') | collect; - EXPECT_EQ(2, pieces.size()); - EXPECT_EQ("", pieces[0]); - EXPECT_EQ("", pieces[1]); - } -} - -TEST(StringGen, EachToTuple) { - { - auto lines = "2:1.414:yo 3:1.732:hi"; - auto actual - = split(lines, ' ') - | eachToTuple(':') - | as(); - vector> expected { - make_tuple(2, 1.414, "yo"), - make_tuple(3, 1.732, "hi"), - }; - EXPECT_EQ(expected, actual); - } - { - auto lines = "2 3"; - auto actual - = split(lines, ' ') - | eachToTuple(',') - | as(); - vector> expected { - make_tuple(2), - make_tuple(3), - }; - EXPECT_EQ(expected, actual); - } - { - // StringPiece target - auto lines = "1:cat 2:dog"; - auto actual - = split(lines, ' ') - | eachToTuple(':') - | as(); - vector> expected { - make_tuple(1, "cat"), - make_tuple(2, "dog"), - }; - EXPECT_EQ(expected, actual); - } - { - // Empty field - auto lines = "2:tjackson:4 3::5"; - auto actual - = split(lines, ' ') - | eachToTuple(':') - | as(); - vector> expected { - make_tuple(2, "tjackson", 4), - make_tuple(3, "", 5), - }; - EXPECT_EQ(expected, actual); - } - { - // Excess fields - auto lines = "1:2 3:4:5"; - EXPECT_THROW((split(lines, ' ') - | eachToTuple(':') - | as()), - std::runtime_error); - } - { - // Missing fields - auto lines = "1:2:3 4:5"; - EXPECT_THROW((split(lines, ' ') - | eachToTuple(':') - | as()), - std::runtime_error); - } -} - -TEST(StringGen, EachToPair) { - { - // char delimiters - auto lines = "2:1.414 3:1.732"; - auto actual - = split(lines, ' ') - | eachToPair(':') - | as>(); - std::map expected { - { 3, 1.732 }, - { 2, 1.414 }, - }; - EXPECT_EQ(expected, actual); - } - { - // string delimiters - auto lines = "ab=>cd ef=>gh"; - auto actual - = split(lines, ' ') - | eachToPair("=>") - | as>(); - std::map expected { - { "ab", "cd" }, - { "ef", "gh" }, - }; - EXPECT_EQ(expected, actual); - } -} - -TEST(StringGen, Resplit) { - auto collect = eachTo() | as(); - { - auto pieces = from({"hello,, world, goodbye, meow"}) | - resplit(',') | collect; - EXPECT_EQ(5, pieces.size()); - EXPECT_EQ("hello", pieces[0]); - EXPECT_EQ("", pieces[1]); - EXPECT_EQ(" world", pieces[2]); - EXPECT_EQ(" goodbye", pieces[3]); - EXPECT_EQ(" meow", pieces[4]); - } - { - auto pieces = from({"hel", "lo,", ", world", ", goodbye, m", "eow"}) | - resplit(',') | collect; - EXPECT_EQ(5, pieces.size()); - EXPECT_EQ("hello", pieces[0]); - EXPECT_EQ("", pieces[1]); - EXPECT_EQ(" world", pieces[2]); - EXPECT_EQ(" goodbye", pieces[3]); - EXPECT_EQ(" meow", pieces[4]); - } -} - -template -void runUnsplitSuite(F fn) { - fn("hello, world"); - fn("hello,world,goodbye"); - fn(" "); - fn(""); - fn(", "); - fn(", a, b,c"); -} - -TEST(StringGen, Unsplit) { - - auto basicFn = [](const StringPiece& s) { - EXPECT_EQ(split(s, ',') | unsplit(','), s); - }; - - auto existingBuffer = [](const StringPiece& s) { - folly::fbstring buffer("asdf"); - split(s, ',') | unsplit(',', &buffer); - auto expected = folly::to( - "asdf", s.empty() ? "" : ",", s); - EXPECT_EQ(expected, buffer); - }; - - auto emptyBuffer = [](const StringPiece& s) { - std::string buffer; - split(s, ',') | unsplit(',', &buffer); - EXPECT_EQ(s, buffer); - }; - - auto stringDelim = [](const StringPiece& s) { - EXPECT_EQ(s, split(s, ',') | unsplit(",")); - std::string buffer; - split(s, ',') | unsplit(",", &buffer); - EXPECT_EQ(buffer, s); - }; - - runUnsplitSuite(basicFn); - runUnsplitSuite(existingBuffer); - runUnsplitSuite(emptyBuffer); - runUnsplitSuite(stringDelim); - EXPECT_EQ("1, 2, 3", seq(1, 3) | unsplit(", ")); -} - -TEST(FileGen, ByLine) { - auto collect = eachTo() | as(); - test::TemporaryFile file("ByLine"); - static const std::string lines( - "Hello world\n" - "This is the second line\n" - "\n" - "\n" - "a few empty lines above\n" - "incomplete last line"); - EXPECT_EQ(lines.size(), write(file.fd(), lines.data(), lines.size())); - - auto expected = from({lines}) | resplit('\n') | collect; - auto found = byLine(file.path().c_str()) | collect; - - EXPECT_TRUE(expected == found); -} - -class FileGenBufferedTest : public ::testing::TestWithParam { }; - -TEST_P(FileGenBufferedTest, FileWriter) { - size_t bufferSize = GetParam(); - test::TemporaryFile file("FileWriter"); - - static const std::string lines( - "Hello world\n" - "This is the second line\n" - "\n" - "\n" - "a few empty lines above\n"); - - auto src = from({lines, lines, lines, lines, lines, lines, lines, lines}); - auto collect = eachTo() | as(); - auto expected = src | resplit('\n') | collect; - - src | eachAs() | toFile(File(file.fd()), bufferSize); - auto found = byLine(file.path().c_str()) | collect; - - EXPECT_TRUE(expected == found); -} - -INSTANTIATE_TEST_CASE_P( - DifferentBufferSizes, - FileGenBufferedTest, - ::testing::Values(0, 1, 2, 4, 8, 64, 4096)); - -TEST(Gen, Guard) { - using std::runtime_error; - EXPECT_THROW(from({"1", "a", "3"}) - | eachTo() - | sum, - runtime_error); - EXPECT_EQ(4, - from({"1", "a", "3"}) - | guard([](runtime_error&, const char*) { - return true; // continue - }) - | eachTo() - | sum); - EXPECT_EQ(1, - from({"1", "a", "3"}) - | guard([](runtime_error&, const char*) { - return false; // break - }) - | eachTo() - | sum); - EXPECT_THROW(from({"1", "a", "3"}) - | guard([](runtime_error&, const char* v) { - if (v[0] == 'a') { - throw; - } - return true; - }) - | eachTo() - | sum, - runtime_error); -} - -TEST(Gen, Batch) { - EXPECT_EQ((vector> { {1} }), - seq(1, 1) | batch(5) | as()); - EXPECT_EQ((vector> { {1, 2, 3}, {4, 5, 6}, {7, 8, 9}, {10, 11} }), - seq(1, 11) | batch(3) | as()); - EXPECT_THROW(seq(1, 1) | batch(0) | as(), - std::invalid_argument); -} - -TEST(Gen, BatchMove) { - auto expected = vector>{ {0, 1}, {2, 3}, {4} }; - auto actual = - seq(0, 4) - | mapped([](int i) { return std::unique_ptr(new int(i)); }) - | batch(2) - | mapped([](std::vector>& pVector) { - std::vector iVector; - for (const auto& p : pVector) { - iVector.push_back(*p); - }; - return iVector; - }) - | as(); - EXPECT_EQ(expected, actual); -} - -int main(int argc, char *argv[]) { - testing::InitGoogleTest(&argc, argv); - google::ParseCommandLineFlags(&argc, &argv, true); - return RUN_ALL_TESTS(); -} diff --git a/folly/gen/Base-inl.h b/folly/gen/Base-inl.h new file mode 100644 index 00000000..d627ed55 --- /dev/null +++ b/folly/gen/Base-inl.h @@ -0,0 +1,1822 @@ +/* + * Copyright 2014 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FOLLY_GEN_BASE_H +#error This file may only be included from folly/gen/Base.h +#endif + +// Ignore shadowing warnings within this file, so includers can use -Wshadow. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wshadow" + +namespace folly { namespace gen { + +/** + * ArgumentReference - For determining ideal argument type to receive a value. + */ +template +struct ArgumentReference + : public std::conditional< + std::is_reference::value, + T, // T& -> T&, T&& -> T&&, const T& -> const T& + typename std::conditional::value, + T&, // const int -> const int& + T&& // int -> int&& + >::type> {}; + +namespace detail { + +/* + * ReferencedSource - Generate values from an STL-like container using + * iterators from .begin() until .end(). Value type defaults to the type of + * *container->begin(). For std::vector, this would be int&. Note that the + * value here is a reference, so the values in the vector will be passed by + * reference to downstream operators. + * + * This type is primarily used through the 'from' helper method, like: + * + * string& longestName = from(names) + * | maxBy([](string& s) { return s.size() }); + */ +template +class ReferencedSource : + public GenImpl> { + Container* container_; +public: + explicit ReferencedSource(Container* container) + : container_(container) {} + + template + void foreach(Body&& body) const { + for (auto& value : *container_) { + body(std::forward(value)); + } + } + + template + bool apply(Handler&& handler) const { + for (auto& value : *container_) { + if (!handler(std::forward(value))) { + return false; + } + } + return true; + } +}; + +/** + * CopiedSource - For producing values from eagerly from a sequence of values + * whose storage is owned by this class. Useful for preparing a generator for + * use after a source collection will no longer be available, or for when the + * values are specified literally with an initializer list. + * + * This type is primarily used through the 'fromCopy' function, like: + * + * auto sourceCopy = fromCopy(makeAVector()); + * auto sum = sourceCopy | sum; + * auto max = sourceCopy | max; + * + * Though it is also used for the initializer_list specialization of from(). + */ +template +class CopiedSource : + public GenImpl> { + static_assert( + !std::is_reference::value, "StorageType must be decayed"); + public: + // Generator objects are often copied during normal construction as they are + // encapsulated by downstream generators. It would be bad if this caused + // a copy of the entire container each time, and since we're only exposing a + // const reference to the value, it's safe to share it between multiple + // generators. + static_assert( + !std::is_reference::value, + "Can't copy into a reference"); + std::shared_ptr copy_; +public: + typedef Container ContainerType; + + template + explicit CopiedSource(const SourceContainer& container) + : copy_(new Container(begin(container), end(container))) {} + + explicit CopiedSource(Container&& container) : + copy_(new Container(std::move(container))) {} + + // To enable re-use of cached results. + CopiedSource(const CopiedSource& source) + : copy_(source.copy_) {} + + template + void foreach(Body&& body) const { + for (const auto& value : *copy_) { + body(value); + } + } + + template + bool apply(Handler&& handler) const { + // The collection may be reused by others, we can't allow it to be changed. + for (const auto& value : *copy_) { + if (!handler(value)) { + return false; + } + } + return true; + } +}; + +/** + * Sequence - For generating values from beginning value, incremented along the + * way with the ++ and += operators. Iteration may continue indefinitely by + * setting the 'endless' template parameter to true. If set to false, iteration + * will stop when value reaches 'end', either inclusively or exclusively, + * depending on the template parameter 'endInclusive'. Value type specified + * explicitly. + * + * This type is primarily used through the 'seq' and 'range' function, like: + * + * int total = seq(1, 10) | sum; + * auto indexes = range(0, 10); + */ +template +class Sequence : public GenImpl> { + static_assert(!std::is_reference::value && + !std::is_const::value, "Value mustn't be const or ref."); + Value bounds_[endless ? 1 : 2]; +public: + explicit Sequence(Value begin) + : bounds_{std::move(begin)} { + static_assert(endless, "Must supply 'end'"); + } + + Sequence(Value begin, + Value end) + : bounds_{std::move(begin), std::move(end)} {} + + template + bool apply(Handler&& handler) const { + Value value = bounds_[0]; + for (;endless || value < bounds_[1]; ++value) { + const Value& arg = value; + if (!handler(arg)) { + return false; + } + } + if (endInclusive && value == bounds_[1]) { + const Value& arg = value; + if (!handler(arg)) { + return false; + } + } + return true; + } + + template + void foreach(Body&& body) const { + Value value = bounds_[0]; + for (;endless || value < bounds_[1]; ++value) { + const Value& arg = value; + body(arg); + } + if (endInclusive && value == bounds_[1]) { + const Value& arg = value; + body(arg); + } + } + + static constexpr bool infinite = endless; +}; + +/** + * GenratorBuilder - Helper for GENERTATOR macro. + **/ +template +struct GeneratorBuilder { + template> + Yield operator+(Source&& source) { + return Yield(std::forward(source)); + } +}; + +/** + * Yield - For producing values from a user-defined generator by way of a + * 'yield' function. + **/ +template +class Yield : public GenImpl> { + Source source_; + public: + explicit Yield(Source source) + : source_(std::move(source)) { + } + + template + bool apply(Handler&& handler) const { + struct Break {}; + auto body = [&](Value value) { + if (!handler(std::forward(value))) { + throw Break(); + } + }; + try { + source_(body); + return true; + } catch (Break&) { + return false; + } + } + + template + void foreach(Body&& body) const { + source_(std::forward(body)); + } +}; + +template +class Empty : public GenImpl> { + public: + template + bool apply(Handler&&) const { return true; } +}; + +/* + * Operators + */ + +/** + * Map - For producing a sequence of values by passing each value from a source + * collection through a predicate. + * + * This type is usually used through the 'map' or 'mapped' helper function: + * + * auto squares = seq(1, 10) | map(square) | asVector; + */ +template +class Map : public Operator> { + Predicate pred_; + public: + Map() {} + + explicit Map(Predicate pred) + : pred_(std::move(pred)) + { } + + template::type + >::type> + class Generator : + public GenImpl> { + Source source_; + Predicate pred_; + public: + explicit Generator(Source source, const Predicate& pred) + : source_(std::move(source)), pred_(pred) {} + + template + void foreach(Body&& body) const { + source_.foreach([&](Value value) { + body(pred_(std::forward(value))); + }); + } + + template + bool apply(Handler&& handler) const { + return source_.apply([&](Value value) { + return handler(pred_(std::forward(value))); + }); + } + + static constexpr bool infinite = Source::infinite; + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self()), pred_); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self(), pred_); + } +}; + + +/** + * Filter - For filtering values from a source sequence by a predicate. + * + * This type is usually used through the 'filter' helper function, like: + * + * auto nonEmpty = from(strings) + * | filter([](const string& str) -> bool { + * return !str.empty(); + * }); + */ +template +class Filter : public Operator> { + Predicate pred_; + public: + Filter() {} + explicit Filter(Predicate pred) + : pred_(std::move(pred)) + { } + + template + class Generator : public GenImpl> { + Source source_; + Predicate pred_; + public: + explicit Generator(Source source, const Predicate& pred) + : source_(std::move(source)), pred_(pred) {} + + template + void foreach(Body&& body) const { + source_.foreach([&](Value value) { + if (pred_(std::forward(value))) { + body(std::forward(value)); + } + }); + } + + template + bool apply(Handler&& handler) const { + return source_.apply([&](Value value) -> bool { + if (pred_(std::forward(value))) { + return handler(std::forward(value)); + } + return true; + }); + } + + static constexpr bool infinite = Source::infinite; + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self()), pred_); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self(), pred_); + } +}; + +/** + * Until - For producing values from a source until a predicate is satisfied. + * + * This type is usually used through the 'until' helper function, like: + * + * auto best = from(sortedItems) + * | until([](Item& item) { return item.score > 100; }) + * | asVector; + */ +template +class Until : public Operator> { + Predicate pred_; + public: + Until() {} + explicit Until(Predicate pred) + : pred_(std::move(pred)) + {} + + template + class Generator : public GenImpl> { + Source source_; + Predicate pred_; + public: + explicit Generator(Source source, const Predicate& pred) + : source_(std::move(source)), pred_(pred) {} + + template + bool apply(Handler&& handler) const { + bool cancelled = false; + source_.apply([&](Value value) -> bool { + if (pred_(value)) { // un-forwarded to disable move + return false; + } + if (!handler(std::forward(value))) { + cancelled = true; + return false; + } + return true; + }); + return !cancelled; + } + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self()), pred_); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self(), pred_); + } + + // Theoretically an 'until' might stop an infinite + static constexpr bool infinite = false; +}; + +/** + * Take - For producing up to N values from a source. + * + * This type is usually used through the 'take' helper function, like: + * + * auto best = from(docs) + * | orderByDescending(scoreDoc) + * | take(10); + */ +class Take : public Operator { + size_t count_; + public: + explicit Take(size_t count) + : count_(count) {} + + template + class Generator : + public GenImpl> { + Source source_; + size_t count_; + public: + explicit Generator(Source source, size_t count) + : source_(std::move(source)) , count_(count) {} + + template + bool apply(Handler&& handler) const { + if (count_ == 0) { return false; } + size_t n = count_; + bool cancelled = false; + source_.apply([&](Value value) -> bool { + if (!handler(std::forward(value))) { + cancelled = true; + return false; + } + return --n; + }); + return !cancelled; + } + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self()), count_); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self(), count_); + } +}; + +/** + * Sample - For taking a random sample of N elements from a sequence + * (without replacement). + */ +template +class Sample : public Operator> { + size_t count_; + Random rng_; + public: + explicit Sample(size_t count, Random rng) + : count_(count), rng_(std::move(rng)) {} + + template::type> + class Generator : + public GenImpl> { + static_assert(!Source::infinite, "Cannot sample infinite source!"); + // It's too easy to bite ourselves if random generator is only 16-bit + static_assert(Random::max() >= std::numeric_limits::max() - 1, + "Random number generator must support big values"); + Source source_; + size_t count_; + mutable Rand rng_; + public: + explicit Generator(Source source, size_t count, Random rng) + : source_(std::move(source)) , count_(count), rng_(std::move(rng)) {} + + template + bool apply(Handler&& handler) const { + if (count_ == 0) { return false; } + std::vector v; + v.reserve(count_); + // use reservoir sampling to give each source value an equal chance + // of appearing in our output. + size_t n = 1; + source_.foreach([&](Value value) -> void { + if (v.size() < count_) { + v.push_back(std::forward(value)); + } else { + // alternatively, we could create a std::uniform_int_distribution + // instead of using modulus, but benchmarks show this has + // substantial overhead. + size_t index = rng_() % n; + if (index < v.size()) { + v[index] = std::forward(value); + } + } + ++n; + }); + + // output is unsorted! + for (auto& val: v) { + if (!handler(std::move(val))) { + return false; + } + } + return true; + } + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self()), count_, rng_); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self(), count_, rng_); + } +}; + +/** + * Skip - For skipping N items from the beginning of a source generator. + * + * This type is usually used through the 'skip' helper function, like: + * + * auto page = from(results) + * | skip(pageSize * startPage) + * | take(10); + */ +class Skip : public Operator { + size_t count_; + public: + explicit Skip(size_t count) + : count_(count) {} + + template + class Generator : + public GenImpl> { + Source source_; + size_t count_; + public: + explicit Generator(Source source, size_t count) + : source_(std::move(source)) , count_(count) {} + + template + void foreach(Body&& body) const { + if (count_ == 0) { + source_.foreach(body); + return; + } + size_t n = 0; + source_.foreach([&](Value value) { + if (n < count_) { + ++n; + } else { + body(std::forward(value)); + } + }); + } + + template + bool apply(Handler&& handler) const { + if (count_ == 0) { + return source_.apply(std::forward(handler)); + } + size_t n = 0; + return source_.apply([&](Value value) -> bool { + if (n < count_) { + ++n; + return true; + } + return handler(std::forward(value)); + }); + } + + static constexpr bool infinite = Source::infinite; + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self()), count_); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self(), count_); + } +}; + +/** + * Order - For ordering a sequence of values from a source by key. + * The key is extracted by the given selector functor, and this key is then + * compared using the specified comparator. + * + * This type is usually used through the 'order' helper function, like: + * + * auto closest = from(places) + * | orderBy([](Place& p) { + * return -distance(p.location, here); + * }) + * | take(10); + */ +template +class Order : public Operator> { + Selector selector_; + Comparer comparer_; + public: + Order() {} + + explicit Order(Selector selector) + : selector_(std::move(selector)) + {} + + Order(Selector selector, + Comparer comparer) + : selector_(std::move(selector)) + , comparer_(std::move(comparer)) + {} + + template::type, + class Result = typename std::result_of::type> + class Generator : + public GenImpl> { + static_assert(!Source::infinite, "Cannot sort infinite source!"); + Source source_; + Selector selector_; + Comparer comparer_; + + typedef std::vector VectorType; + + VectorType asVector() const { + auto comparer = [&](const StorageType& a, const StorageType& b) { + return comparer_(selector_(a), selector_(b)); + }; + auto vals = source_ | as(); + std::sort(vals.begin(), vals.end(), comparer); + return std::move(vals); + } + public: + Generator(Source source, + Selector selector, + Comparer comparer) + : source_(std::move(source)), + selector_(std::move(selector)), + comparer_(std::move(comparer)) {} + + VectorType operator|(const Collect&) const { + return asVector(); + } + + VectorType operator|(const CollectTemplate&) const { + return asVector(); + } + + template + void foreach(Body&& body) const { + for (auto& value : asVector()) { + body(std::move(value)); + } + } + + template + bool apply(Handler&& handler) const { + auto comparer = [&](const StorageType& a, const StorageType& b) { + // swapped for minHeap + return comparer_(selector_(b), selector_(a)); + }; + auto heap = source_ | as(); + std::make_heap(heap.begin(), heap.end(), comparer); + while (!heap.empty()) { + std::pop_heap(heap.begin(), heap.end(), comparer); + if (!handler(std::move(heap.back()))) { + return false; + } + heap.pop_back(); + } + return true; + } + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self()), selector_, comparer_); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self(), selector_, comparer_); + } +}; + +/* + * TypeAssertion - For verifying the exact type of the value produced by a + * generator. Useful for testing and debugging, and acts as a no-op at runtime. + * Pass-through at runtime. Used through the 'assert_type<>()' factory method + * like so: + * + * auto c = from(vector) | assert_type() | sum; + * + */ +template +class TypeAssertion : public Operator> { + public: + template + const Source& compose(const GenImpl& source) const { + static_assert(std::is_same::value, + "assert_type() check failed"); + return source.self(); + } + + template + Source&& compose(GenImpl&& source) const { + static_assert(std::is_same::value, + "assert_type() check failed"); + return std::move(source.self()); + } +}; + +/** + * Distinct - For filtering duplicates out of a sequence. A selector may be + * provided to generate a key to uniquify for each value. + * + * This type is usually used through the 'distinct' helper function, like: + * + * auto closest = from(results) + * | distinctBy([](Item& i) { + * return i.target; + * }) + * | take(10); + */ +template +class Distinct : public Operator> { + Selector selector_; + public: + Distinct() {} + + explicit Distinct(Selector selector) + : selector_(std::move(selector)) + {} + + template + class Generator : public GenImpl> { + Source source_; + Selector selector_; + + typedef typename std::decay::type StorageType; + + // selector_ cannot be passed an rvalue or it would end up passing the husk + // of a value to the downstream operators. + typedef const StorageType& ParamType; + + typedef typename std::result_of::type KeyType; + typedef typename std::decay::type KeyStorageType; + + public: + Generator(Source source, + Selector selector) + : source_(std::move(source)), + selector_(std::move(selector)) {} + + template + void foreach(Body&& body) const { + std::unordered_set keysSeen; + source_.foreach([&](Value value) { + if (keysSeen.insert(selector_(ParamType(value))).second) { + body(std::forward(value)); + } + }); + } + + template + bool apply(Handler&& handler) const { + std::unordered_set keysSeen; + return source_.apply([&](Value value) -> bool { + if (keysSeen.insert(selector_(ParamType(value))).second) { + return handler(std::forward(value)); + } + return true; + }); + } + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self()), selector_); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self(), selector_); + } +}; + +/** + * Batch - For producing fixed-size batches of each value from a source. + * + * This type is usually used through the 'batch' helper function: + * + * auto batchSums + * = seq(1, 10) + * | batch(3) + * | map([](const std::vector& batch) { + * return from(batch) | sum; + * }) + * | as(); + */ +class Batch : public Operator { + size_t batchSize_; + public: + explicit Batch(size_t batchSize) + : batchSize_(batchSize) { + if (batchSize_ == 0) { + throw std::invalid_argument("Batch size must be non-zero!"); + } + } + + template::type, + class VectorType = std::vector> + class Generator : + public GenImpl> { + Source source_; + size_t batchSize_; + public: + explicit Generator(Source source, size_t batchSize) + : source_(std::move(source)) + , batchSize_(batchSize) {} + + template + bool apply(Handler&& handler) const { + VectorType batch_; + batch_.reserve(batchSize_); + bool shouldContinue = source_.apply([&](Value value) -> bool { + batch_.push_back(std::forward(value)); + if (batch_.size() == batchSize_) { + bool needMore = handler(batch_); + batch_.clear(); + return needMore; + } + // Always need more if the handler is not called. + return true; + }); + // Flush everything, if and only if `handler` hasn't returned false. + if (shouldContinue && !batch_.empty()) { + shouldContinue = handler(batch_); + batch_.clear(); + } + return shouldContinue; + } + + static constexpr bool infinite = Source::infinite; + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self()), batchSize_); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self(), batchSize_); + } +}; +/* + * Sinks + */ + +/** + * FoldLeft - Left-associative functional fold. For producing an aggregate value + * from a seed and a folder function. Useful for custom aggregators on a + * sequence. + * + * This type is primarily used through the 'foldl' helper method, like: + * + * double movingAverage = from(values) + * | foldl(0.0, [](double avg, double sample) { + * return sample * 0.1 + avg * 0.9; + * }); + */ +template +class FoldLeft : public Operator> { + Seed seed_; + Fold fold_; + public: + FoldLeft() {} + FoldLeft(Seed seed, + Fold fold) + : seed_(std::move(seed)) + , fold_(std::move(fold)) + {} + + template + Seed compose(const GenImpl& source) const { + static_assert(!Source::infinite, "Cannot foldl infinite source"); + Seed accum = seed_; + source | [&](Value v) { + accum = fold_(std::move(accum), std::forward(v)); + }; + return accum; + } +}; + +/** + * First - For finding the first value in a sequence. + * + * This type is primarily used through the 'first' static value, like: + * + * int firstThreeDigitPrime = seq(100) | filter(isPrime) | first; + */ +class First : public Operator { + public: + First() { } + + template::type> + StorageType compose(const GenImpl& source) const { + Optional accum; + source | [&](Value v) -> bool { + accum = std::forward(v); + return false; + }; + if (!accum.hasValue()) { + throw EmptySequence(); + } + return std::move(accum.value()); + } +}; + + +/** + * Any - For determining whether any values in a sequence satisfy a predicate. + * + * This type is primarily used through the 'any' static value, like: + * + * bool any20xPrimes = seq(200, 210) | filter(isPrime) | any; + * + * Note that it may also be used like so: + * + * bool any20xPrimes = seq(200, 210) | any(isPrime); + * + */ +class Any : public Operator { + public: + Any() { } + + template + bool compose(const GenImpl& source) const { + bool any = false; + source | [&](Value v) -> bool { + any = true; + return false; + }; + return any; + } + + /** + * Convenience function for use like: + * + * bool found = gen | any([](int i) { return i * i > 100; }); + */ + template, + class Composed = Composed> + Composed operator()(Predicate pred) const { + return Composed(Filter(std::move(pred)), Any()); + } +}; + +/** + * All - For determining whether all values in a sequence satisfy a predicate. + * + * This type is primarily used through the 'any' static value, like: + * + * bool valid = from(input) | all(validate); + * + * Note: Passing an empty sequence through 'all()' will always return true. + */ +template +class All : public Operator> { + Predicate pred_; + public: + All() {} + explicit All(Predicate pred) + : pred_(std::move(pred)) + { } + + template + bool compose(const GenImpl& source) const { + static_assert(!Source::infinite, "Cannot call 'all' on infinite source"); + bool all = true; + source | [&](Value v) -> bool { + if (!pred_(std::forward(v))) { + all = false; + return false; + } + return true; + }; + return all; + } +}; + +/** + * Reduce - Functional reduce, for recursively combining values from a source + * using a reducer function until there is only one item left. Useful for + * combining values when an empty sequence doesn't make sense. + * + * This type is primarily used through the 'reduce' helper method, like: + * + * sring longest = from(names) + * | reduce([](string&& best, string& current) { + * return best.size() >= current.size() ? best : current; + * }); + */ +template +class Reduce : public Operator> { + Reducer reducer_; + public: + Reduce() {} + explicit Reduce(Reducer reducer) + : reducer_(std::move(reducer)) + {} + + template::type> + StorageType compose(const GenImpl& source) const { + Optional accum; + source | [&](Value v) { + if (accum.hasValue()) { + accum = reducer_(std::move(accum.value()), std::forward(v)); + } else { + accum = std::forward(v); + } + }; + if (!accum.hasValue()) { + throw EmptySequence(); + } + return accum.value(); + } +}; + +/** + * Count - for simply counting the items in a collection. + * + * This type is usually used through its singleton, 'count': + * + * auto shortPrimes = seq(1, 100) | filter(isPrime) | count; + */ +class Count : public Operator { + public: + Count() { } + + template + size_t compose(const GenImpl& source) const { + static_assert(!Source::infinite, "Cannot count infinite source"); + return foldl(size_t(0), + [](size_t accum, Value v) { + return accum + 1; + }).compose(source); + } +}; + +/** + * Sum - For simply summing up all the values from a source. + * + * This type is usually used through its singleton, 'sum': + * + * auto gaussSum = seq(1, 100) | sum; + */ +class Sum : public Operator { + public: + Sum() : Operator() {} + + template::type> + StorageType compose(const GenImpl& source) const { + static_assert(!Source::infinite, "Cannot sum infinite source"); + return foldl(StorageType(0), + [](StorageType&& accum, Value v) { + return std::move(accum) + std::forward(v); + }).compose(source); + } +}; + +/** + * Contains - For testing whether a value matching the given value is contained + * in a sequence. + * + * This type should be used through the 'contains' helper method, like: + * + * bool contained = seq(1, 10) | map(square) | contains(49); + */ +template +class Contains : public Operator> { + Needle needle_; + public: + explicit Contains(Needle needle) + : needle_(std::move(needle)) + {} + + template::type> + bool compose(const GenImpl& source) const { + static_assert(!Source::infinite, + "Calling contains on an infinite source might cause " + "an infinite loop."); + return !(source | [this](Value value) { + return !(needle_ == std::forward(value)); + }); + } +}; + +/** + * Min - For a value which minimizes a key, where the key is determined by a + * given selector, and compared by given comparer. + * + * This type is usually used through the singletone 'min' or through the helper + * functions 'minBy' and 'maxBy'. + * + * auto oldest = from(people) + * | minBy([](Person& p) { + * return p.dateOfBirth; + * }); + */ +template +class Min : public Operator> { + Selector selector_; + Comparer comparer_; + public: + Min() {} + + explicit Min(Selector selector) + : selector_(std::move(selector)) + {} + + Min(Selector selector, + Comparer comparer) + : selector_(std::move(selector)) + , comparer_(std::move(comparer)) + {} + + template::type, + class Key = typename std::decay< + typename std::result_of::type + >::type> + StorageType compose(const GenImpl& source) const { + Optional min; + Optional minKey; + source | [&](Value v) { + Key key = selector_(std::forward(v)); + if (!minKey.hasValue() || comparer_(key, minKey.value())) { + minKey = key; + min = std::forward(v); + } + }; + if (!min.hasValue()) { + throw EmptySequence(); + } + return min.value(); + } +}; + +/** + * Append - For collecting values from a source into a given output container + * by appending. + * + * This type is usually used through the helper function 'appendTo', like: + * + * vector ids; + * from(results) | map([](Person& p) { return p.id }) + * | appendTo(ids); + */ +template +class Append : public Operator> { + Collection* collection_; + public: + explicit Append(Collection* collection) + : collection_(collection) + {} + + template + Collection& compose(const GenImpl& source) const { + source | [&](Value v) { + collection_->insert(collection_->end(), std::forward(v)); + }; + return *collection_; + } +}; + +/** + * Collect - For collecting values from a source in a collection of the desired + * type. + * + * This type is usually used through the helper function 'as', like: + * + * std::string upper = from(stringPiece) + * | map(&toupper) + * | as(); + */ +template +class Collect : public Operator> { + public: + Collect() { } + + template::type> + Collection compose(const GenImpl& source) const { + Collection collection; + source | [&](Value v) { + collection.insert(collection.end(), std::forward(v)); + }; + return collection; + } +}; + + +/** + * CollectTemplate - For collecting values from a source in a collection + * constructed using the specified template type. Given the type of values + * produced by the given generator, the collection type will be: + * Container> + * + * The allocator defaults to std::allocator, so this may be used for the STL + * containers by simply using operators like 'as', 'as', + * 'as'. 'as', here is the helper method which is the usual means of + * consturcting this operator. + * + * Example: + * + * set uniqueNames = from(names) | as(); + */ +template class Container, + template class Allocator> +class CollectTemplate : public Operator> { + public: + CollectTemplate() { } + + template::type, + class Collection = Container>> + Collection compose(const GenImpl& source) const { + Collection collection; + source | [&](Value v) { + collection.insert(collection.end(), std::forward(v)); + }; + return collection; + } +}; + +/** + * Concat - For flattening generators of generators. + * + * This type is usually used through the 'concat' static value, like: + * + * auto edges = + * from(nodes) + * | map([](Node& x) { + * return from(x.neighbors) + * | map([&](Node& y) { + * return Edge(x, y); + * }); + * }) + * | concat + * | as(); + */ +class Concat : public Operator { + public: + Concat() { } + + template::type::ValueType> + class Generator : + public GenImpl> { + Source source_; + public: + explicit Generator(Source source) + : source_(std::move(source)) {} + + template + bool apply(Handler&& handler) const { + return source_.apply([&](Inner inner) -> bool { + return inner.apply(std::forward(handler)); + }); + } + + template + void foreach(Body&& body) const { + source_.foreach([&](Inner inner) { + inner.foreach(std::forward(body)); + }); + } + + static constexpr bool infinite = Source::infinite; + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self())); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self()); + } +}; + +/** + * RangeConcat - For flattening generators of iterables. + * + * This type is usually used through the 'rconcat' static value, like: + * + * map> adjacency; + * auto sinks = + * from(adjacency) + * | get<1>() + * | rconcat() + * | as(); + */ +class RangeConcat : public Operator { + public: + RangeConcat() { } + + template::RefType> + class Generator + : public GenImpl> { + Source source_; + public: + explicit Generator(Source source) + : source_(std::move(source)) {} + + template + void foreach(Body&& body) const { + source_.foreach([&](Range range) { + for (auto& value : range) { + body(value); + } + }); + } + + template + bool apply(Handler&& handler) const { + return source_.apply([&](Range range) -> bool { + for (auto& value : range) { + if (!handler(value)) { + return false; + } + } + return true; + }); + } + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self())); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self()); + } +}; + + +/** + * GuardImpl - For handling exceptions from downstream computation. Requires the + * type of exception to catch, and handler function to invoke in the event of + * the exception. Note that the handler may: + * 1) return true to continue processing the sequence + * 2) return false to end the sequence immediately + * 3) throw, to pass the exception to the next catch + * The handler must match the signature 'bool(Exception&, Value)'. + * + * This type is used through the `guard` helper, like so: + * + * auto indexes + * = byLine(STDIN_FILENO) + * | guard([](std::runtime_error& e, + * StringPiece sp) { + * LOG(ERROR) << sp << ": " << e.str(); + * return true; // continue processing subsequent lines + * }) + * | eachTo() + * | as(); + * + * TODO(tjackson): Rename this back to Guard. + **/ +template +class GuardImpl : public Operator> { + ErrorHandler handler_; + public: + GuardImpl(ErrorHandler handler) + : handler_(std::move(handler)) {} + + template + class Generator : public GenImpl> { + Source source_; + ErrorHandler handler_; + public: + explicit Generator(Source source, + ErrorHandler handler) + : source_(std::move(source)), + handler_(std::move(handler)) {} + + template + bool apply(Handler&& handler) const { + return source_.apply([&](Value value) -> bool { + try { + handler(std::forward(value)); + return true; + } catch (Exception& e) { + return handler_(e, std::forward(value)); + } + }); + } + + static constexpr bool infinite = Source::infinite; + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self()), handler_); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self(), handler_); + } +}; + +/** + * Cycle - For repeating a sequence forever. + * + * This type is usually used through the 'cycle' static value, like: + * + * auto tests + * = from(samples) + * | cycle + * | take(100); + */ +class Cycle : public Operator { + off_t limit_; // -1 for infinite + public: + Cycle() + : limit_(-1) { } + + explicit Cycle(off_t limit) + : limit_(limit) { } + + template + class Generator : public GenImpl> { + Source source_; + off_t limit_; // -1 for infinite + public: + explicit Generator(Source source, off_t limit) + : source_(std::move(source)) + , limit_(limit) {} + + template + bool apply(Handler&& handler) const { + bool cont; + auto handler2 = [&](Value value) { + cont = handler(std::forward(value)); + return cont; + }; + for (off_t count = 0; count != limit_; ++count) { + cont = false; + source_.apply(handler2); + if (!cont) { + return false; + } + } + return true; + } + + // not actually infinite, since an empty generator will end the cycles. + static constexpr bool infinite = Source::infinite; + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self()), limit_); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self(), limit_); + } + + /** + * Convenience function for use like: + * + * auto tripled = gen | cycle(3); + */ + Cycle operator()(off_t limit) const { + return Cycle(limit); + } +}; + +/** + * Dereference - For dereferencing a sequence of pointers while filtering out + * null pointers. + * + * This type is usually used through the 'dereference' static value, like: + * + * auto refs = from(ptrs) | dereference; + */ +class Dereference : public Operator { + public: + Dereference() {} + + template())> + class Generator : public GenImpl> { + Source source_; + public: + explicit Generator(Source source) + : source_(std::move(source)) {} + + template + void foreach(Body&& body) const { + source_.foreach([&](Value value) { + if (value) { + return body(*value); + } + }); + } + + template + bool apply(Handler&& handler) const { + return source_.apply([&](Value value) -> bool { + if (value) { + return handler(*value); + } + return true; + }); + } + + // not actually infinite, since an empty generator will end the cycles. + static constexpr bool infinite = Source::infinite; + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self())); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self()); + } +}; + +} //::detail + +/** + * VirtualGen - For wrapping template types in simple polymorphic wrapper. + **/ +template +class VirtualGen : public GenImpl> { + class WrapperBase { + public: + virtual ~WrapperBase() {} + virtual bool apply(const std::function& handler) const = 0; + virtual void foreach(const std::function& body) const = 0; + virtual std::unique_ptr clone() const = 0; + }; + + template + class WrapperImpl : public WrapperBase { + Wrapped wrapped_; + public: + explicit WrapperImpl(Wrapped wrapped) + : wrapped_(std::move(wrapped)) { + } + + virtual bool apply(const std::function& handler) const { + return wrapped_.apply(handler); + } + + virtual void foreach(const std::function& body) const { + wrapped_.foreach(body); + } + + virtual std::unique_ptr clone() const { + return std::unique_ptr(new WrapperImpl(wrapped_)); + } + }; + + std::unique_ptr wrapper_; + + public: + template + /* implicit */ VirtualGen(Self source) + : wrapper_(new WrapperImpl(std::move(source))) + { } + + VirtualGen(VirtualGen&& source) + : wrapper_(std::move(source.wrapper_)) + { } + + VirtualGen(const VirtualGen& source) + : wrapper_(source.wrapper_->clone()) + { } + + VirtualGen& operator=(const VirtualGen& source) { + wrapper_.reset(source.wrapper_->clone()); + return *this; + } + + VirtualGen& operator=(VirtualGen&& source) { + wrapper_= std::move(source.wrapper_); + return *this; + } + + bool apply(const std::function& handler) const { + return wrapper_->apply(handler); + } + + void foreach(const std::function& body) const { + wrapper_->foreach(body); + } +}; + +/** + * non-template operators, statically defined to avoid the need for anything but + * the header. + */ +static const detail::Sum sum; + +static const detail::Count count; + +static const detail::First first; + +/** + * Use directly for detecting any values, or as a function to detect values + * which pass a predicate: + * + * auto nonempty = g | any; + * auto evens = g | any(even); + */ +static const detail::Any any; + +static const detail::Min min; + +static const detail::Min max; + +static const detail::Order order; + +static const detail::Distinct distinct; + +static const detail::Map move; + +static const detail::Concat concat; + +static const detail::RangeConcat rconcat; + +/** + * Use directly for infinite sequences, or as a function to limit cycle count. + * + * auto forever = g | cycle; + * auto thrice = g | cycle(3); + */ +static const detail::Cycle cycle; + +static const detail::Dereference dereference; + +inline detail::Take take(size_t count) { + return detail::Take(count); +} + +template +inline detail::Sample sample(size_t count, Random rng = Random()) { + return detail::Sample(count, std::move(rng)); +} + +inline detail::Skip skip(size_t count) { + return detail::Skip(count); +} + +inline detail::Batch batch(size_t batchSize) { + return detail::Batch(batchSize); +} + +}} //folly::gen + +#pragma GCC diagnostic pop diff --git a/folly/gen/Base.h b/folly/gen/Base.h new file mode 100644 index 00000000..3b6cb7ca --- /dev/null +++ b/folly/gen/Base.h @@ -0,0 +1,642 @@ +/* + * Copyright 2013 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef FOLLY_GEN_BASE_H +#define FOLLY_GEN_BASE_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "folly/Range.h" +#include "folly/Optional.h" +#include "folly/Conv.h" +#include "folly/gen/Core.h" + +/** + * Generator-based Sequence Comprehensions in C++, akin to C#'s LINQ + * @author Tom Jackson + * + * This library makes it possible to write declarative comprehensions for + * processing sequences of values efficiently in C++. The operators should be + * familiar to those with experience in functional programming, and the + * performance will be virtually identical to the equivalent, boilerplate C++ + * implementations. + * + * Generator objects may be created from either an stl-like container (anything + * supporting begin() and end()), from sequences of values, or from another + * generator (see below). To create a generator that pulls values from a vector, + * for example, one could write: + * + * vector names { "Jack", "Jill", "Sara", "Tom" }; + * auto gen = from(names); + * + * Generators are composed by building new generators out of old ones through + * the use of operators. These are reminicent of shell pipelines, and afford + * similar composition. Lambda functions are used liberally to describe how to + * handle individual values: + * + * auto lengths = gen + * | mapped([](const fbstring& name) { return name.size(); }); + * + * Generators are lazy; they don't actually perform any work until they need to. + * As an example, the 'lengths' generator (above) won't actually invoke the + * provided lambda until values are needed: + * + * auto lengthVector = lengths | as(); + * auto totalLength = lengths | sum; + * + * 'auto' is useful in here because the actual types of the generators objects + * are usually complicated and implementation-sensitive. + * + * If a simpler type is desired (for returning, as an example), VirtualGen + * may be used to wrap the generator in a polymorphic wrapper: + * + * VirtualGen powersOfE() { + * return seq(1) | mapped(&expf); + * } + * + * To learn more about this library, including the use of infinite generators, + * see the examples in the comments, or the docs (coming soon). +*/ + +namespace folly { namespace gen { + +class EmptySequence : public std::exception { +public: + virtual const char* what() const noexcept { + return "This operation cannot be called on an empty sequence"; + } +}; + +class Less { +public: + template + auto operator()(const First& first, const Second& second) const -> + decltype(first < second) { + return first < second; + } +}; + +class Greater { +public: + template + auto operator()(const First& first, const Second& second) const -> + decltype(first > second) { + return first > second; + } +}; + +template +class Get { +public: + template + auto operator()(Value&& value) const -> + decltype(std::get(std::forward(value))) { + return std::get(std::forward(value)); + } +}; + +template +class MemberFunction { + public: + typedef Result (Class::*MemberPtr)(); + private: + MemberPtr member_; + public: + explicit MemberFunction(MemberPtr member) + : member_(member) + {} + + Result operator()(Class&& x) const { + return (x.*member_)(); + } + + Result operator()(Class& x) const { + return (x.*member_)(); + } +}; + +template +class ConstMemberFunction{ + public: + typedef Result (Class::*MemberPtr)() const; + private: + MemberPtr member_; + public: + explicit ConstMemberFunction(MemberPtr member) + : member_(member) + {} + + Result operator()(const Class& x) const { + return (x.*member_)(); + } +}; + +template +class Field { + public: + typedef FieldType (Class::*FieldPtr); + private: + FieldPtr field_; + public: + explicit Field(FieldPtr field) + : field_(field) + {} + + const FieldType& operator()(const Class& x) const { + return x.*field_; + } + + FieldType& operator()(Class& x) const { + return x.*field_; + } + + FieldType&& operator()(Class&& x) const { + return std::move(x.*field_); + } +}; + +class Move { +public: + template + auto operator()(Value&& value) const -> + decltype(std::move(std::forward(value))) { + return std::move(std::forward(value)); + } +}; + +class Identity { +public: + template + auto operator()(Value&& value) const -> + decltype(std::forward(value)) { + return std::forward(value); + } +}; + +template +class Cast { + public: + template + Dest operator()(Value&& value) const { + return Dest(std::forward(value)); + } +}; + +template +class To { + public: + template + Dest operator()(Value&& value) const { + return ::folly::to(std::forward(value)); + } +}; + +// Specialization to allow String->StringPiece conversion +template <> +class To { + public: + StringPiece operator()(StringPiece src) const { + return src; + } +}; + +namespace detail { + +template +struct FBounded; + +/* + * Type Traits + */ +template +struct ValueTypeOfRange { + private: + static Container container_; + public: + typedef decltype(*std::begin(container_)) + RefType; + typedef typename std::decay::type + StorageType; +}; + + +/* + * Sources + */ +template::RefType> +class ReferencedSource; + +template::type>> +class CopiedSource; + +template +class Sequence; + +template +class Yield; + +template +class Empty; + + +/* + * Operators + */ +template +class Map; + +template +class Filter; + +template +class Until; + +class Take; + +template +class Sample; + +class Skip; + +template +class Order; + +template +class Distinct; + +template +class TypeAssertion; + +class Concat; + +class RangeConcat; + +class Cycle; + +class Batch; + +class Dereference; + +/* + * Sinks + */ +template +class FoldLeft; + +class First; + +class Any; + +template +class All; + +template +class Reduce; + +class Sum; + +template +class Min; + +template +class Collect; + +template class Collection = std::vector, + template class Allocator = std::allocator> +class CollectTemplate; + +template +class Append; + +template +struct GeneratorBuilder; + +template +class Contains; + +template +class GuardImpl; + +} + +/** + * Polymorphic wrapper + **/ +template +class VirtualGen; + +/* + * Source Factories + */ +template> +From fromConst(const Container& source) { + return From(&source); +} + +template> +From from(Container& source) { + return From(&source); +} + +template::StorageType, + class CopyOf = detail::CopiedSource> +CopyOf fromCopy(Container&& source) { + return CopyOf(std::forward(source)); +} + +template> +From from(std::initializer_list source) { + return From(source); +} + +template> +From from(Container&& source) { + return From(std::move(source)); +} + +template> +Gen range(Value begin, Value end) { + return Gen(begin, end); +} + +template> +Gen seq(Value first, Value last) { + return Gen(first, last); +} + +template> +Gen seq(Value begin) { + return Gen(begin); +} + +template> +Yield generator(Source&& source) { + return Yield(std::forward(source)); +} + +/* + * Create inline generator, used like: + * + * auto gen = GENERATOR(int) { yield(1); yield(2); }; + */ +#define GENERATOR(TYPE) \ + ::folly::gen::detail::GeneratorBuilder() + \ + [=](const std::function& yield) + +/* + * empty() - for producing empty sequences. + */ +template +detail::Empty empty() { + return {}; +} + +/* + * Operator Factories + */ +template> +Map mapped(Predicate pred = Predicate()) { + return Map(std::move(pred)); +} + +template> +Map map(Predicate pred = Predicate()) { + return Map(std::move(pred)); +} + +/* + * member(...) - For extracting a member from each value. + * + * vector strings = ...; + * auto sizes = from(strings) | member(&string::size); + * + * If a member is const overridden (like 'front()'), pass template parameter + * 'Const' to select the const version, or 'Mutable' to select the non-const + * version: + * + * auto heads = from(strings) | member(&string::front); + */ +enum MemberType { + Const, + Mutable +}; + +template, + class Map = detail::Map> +typename std::enable_if::type +member(Return (Class::*member)() const) { + return Map(Mem(member)); +} + +template, + class Map = detail::Map> +typename std::enable_if::type +member(Return (Class::*member)()) { + return Map(Mem(member)); +} + +/* + * field(...) - For extracting a field from each value. + * + * vector items = ...; + * auto names = from(items) | field(&Item::name); + * + * Note that if the values of the generator are rvalues, any non-reference + * fields will be rvalues as well. As an example, the code below does not copy + * any strings, only moves them: + * + * auto namesVector = from(items) + * | move + * | field(&Item::name) + * | as(); + */ +template, + class Map = detail::Map> +Map field(FieldType Class::*field) { + return Map(Field(field)); +} + +template> +Filter filter(Predicate pred = Predicate()) { + return Filter(std::move(pred)); +} + +template> +All all(Predicate pred = Predicate()) { + return All(std::move(pred)); +} + +template> +Until until(Predicate pred = Predicate()) { + return Until(std::move(pred)); +} + +template> +Order orderBy(Selector selector = Identity(), + Comparer comparer = Comparer()) { + return Order(std::move(selector), + std::move(comparer)); +} + +template> +Order orderByDescending(Selector selector = Identity()) { + return Order(std::move(selector)); +} + +template> +Distinct distinctBy(Selector selector = Identity()) { + return Distinct(std::move(selector)); +} + +template>> +Get get() { + return Get(); +} + +// construct Dest from each value +template >> +Cast eachAs() { + return Cast(); +} + +// call folly::to on each value +template >> +To eachTo() { + return To(); +} + +template +detail::TypeAssertion assert_type() { + return {}; +} + +/* + * Sink Factories + */ +template> +FoldLeft foldl(Seed seed = Seed(), + Fold fold = Fold()) { + return FoldLeft(std::move(seed), + std::move(fold)); +} + +template> +Reduce reduce(Reducer reducer = Reducer()) { + return Reduce(std::move(reducer)); +} + +template> +Min minBy(Selector selector = Selector()) { + return Min(std::move(selector)); +} + +template> +MaxBy maxBy(Selector selector = Selector()) { + return MaxBy(std::move(selector)); +} + +template> +Collect as() { + return Collect(); +} + +template class Container = std::vector, + template class Allocator = std::allocator, + class Collect = detail::CollectTemplate> +Collect as() { + return Collect(); +} + +template> +Append appendTo(Collection& collection) { + return Append(&collection); +} + +template::type>> +Contains contains(Needle&& needle) { + return Contains(std::forward(needle)); +} + +template::type>> +GuardImpl guard(ErrorHandler&& handler) { + return GuardImpl(std::forward(handler)); +} + +}} // folly::gen + +#include "folly/gen/Base-inl.h" + +#endif // FOLLY_GEN_BASE_H diff --git a/folly/gen/Combine-inl.h b/folly/gen/Combine-inl.h new file mode 100644 index 00000000..667de7f2 --- /dev/null +++ b/folly/gen/Combine-inl.h @@ -0,0 +1,217 @@ +/* + * Copyright 2014 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FOLLY_GEN_COMBINE_H +#error This file may only be included from folly/gen/Combine.h +#endif + +#include +#include +#include +#include + +namespace folly { +namespace gen { +namespace detail { + +/** + * Interleave + * + * Alternate values from a sequence with values from a sequence container. + * Stops once we run out of values from either source. + */ +template +class Interleave : public Operator> { + // see comment about copies in CopiedSource + const std::shared_ptr container_; + public: + explicit Interleave(Container container) + : container_(new Container(std::move(container))) {} + + template + class Generator : public GenImpl> { + Source source_; + const std::shared_ptr container_; + typedef const typename Container::value_type& ConstRefType; + + static_assert(std::is_same::value, + "Only matching types may be interleaved"); + public: + explicit Generator(Source source, + const std::shared_ptr container) + : source_(std::move(source)), + container_(container) { } + + template + bool apply(Handler&& handler) const { + auto iter = container_->begin(); + return source_.apply([&](const Value& value) -> bool { + if (iter == container_->end()) { + return false; + } + if (!handler(value)) { + return false; + } + if (!handler(*iter)) { + return false; + } + iter++; + return true; + }); + } + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self()), container_); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self(), container_); + } +}; + +/** + * Zip + * + * Combine inputs from Source with values from a sequence container by merging + * them into a tuple. + * + */ +template +class Zip : public Operator> { + // see comment about copies in CopiedSource + const std::shared_ptr container_; + public: + explicit Zip(Container container) + : container_(new Container(std::move(container))) {} + + template::type, + typename std::decay::type>> + class Generator : public GenImpl> { + Source source_; + const std::shared_ptr container_; + public: + explicit Generator(Source source, + const std::shared_ptr container) + : source_(std::move(source)), + container_(container) { } + + template + bool apply(Handler&& handler) const { + auto iter = container_->begin(); + return (source_.apply([&](Value1 value) -> bool { + if (iter == container_->end()) { + return false; + } + if (!handler(std::make_tuple(std::forward(value), *iter))) { + return false; + } + ++iter; + return true; + })); + } + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self()), container_); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self(), container_); + } +}; + +template +auto add_to_tuple(std::tuple t1, std::tuple t2) -> +std::tuple { + return std::tuple_cat(std::move(t1), std::move(t2)); +} + +template +auto add_to_tuple(std::tuple t1, Type2&& t2) -> +decltype(std::tuple_cat(std::move(t1), + std::make_tuple(std::forward(t2)))) { + return std::tuple_cat(std::move(t1), + std::make_tuple(std::forward(t2))); +} + +template +auto add_to_tuple(Type1&& t1, std::tuple t2) -> +decltype(std::tuple_cat(std::make_tuple(std::forward(t1)), + std::move(t2))) { + return std::tuple_cat(std::make_tuple(std::forward(t1)), + std::move(t2)); +} + +template +auto add_to_tuple(Type1&& t1, Type2&& t2) -> +decltype(std::make_tuple(std::forward(t1), + std::forward(t2))) { + return std::make_tuple(std::forward(t1), + std::forward(t2)); +} + +// Merges a 2-tuple into a single tuple (get<0> could already be a tuple) +class MergeTuples { + public: + template + auto operator()(Tuple&& value) const -> + decltype(add_to_tuple(std::get<0>(std::forward(value)), + std::get<1>(std::forward(value)))) { + static_assert(std::tuple_size< + typename std::remove_reference::type + >::value == 2, + "Can only merge tuples of size 2"); + return add_to_tuple(std::get<0>(std::forward(value)), + std::get<1>(std::forward(value))); + } +}; + +} // namespace detail + +static const detail::Map tuple_flatten; + +// TODO(mcurtiss): support zip() for N>1 operands. Because of variadic problems, +// this might not be easily possible until gcc4.8 is available. +template::type>> +Zip zip(Source&& source) { + return Zip(std::forward(source)); +} + +} // namespace gen +} // namespace folly diff --git a/folly/gen/Combine.h b/folly/gen/Combine.h new file mode 100644 index 00000000..ff519c4d --- /dev/null +++ b/folly/gen/Combine.h @@ -0,0 +1,45 @@ +/* + * Copyright 2013 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef FOLLY_GEN_COMBINE_H +#define FOLLY_GEN_COMBINE_H + +#include "folly/gen/Base.h" + +namespace folly { +namespace gen { +namespace detail { + +template +class Interleave; + +template +class Zip; + +} // namespace detail + +template::type, + class Interleave = detail::Interleave> +Interleave interleave(Source2&& source2) { + return Interleave(std::forward(source2)); +} + +} // namespace gen +} // namespace folly + +#include "folly/gen/Combine-inl.h" + +#endif // FOLLY_GEN_COMBINE_H diff --git a/folly/gen/Core-inl.h b/folly/gen/Core-inl.h new file mode 100644 index 00000000..9808487f --- /dev/null +++ b/folly/gen/Core-inl.h @@ -0,0 +1,364 @@ +/* + * Copyright 2014 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FOLLY_GEN_CORE_H +#error This file may only be included from folly/gen/Core.h +#endif + +// Ignore shadowing warnings within this file, so includers can use -Wshadow. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wshadow" + +namespace folly { namespace gen { + +/** + * IsCompatibleSignature - Trait type for testing whether a given Functor + * matches an expected signature. + * + * Usage: + * IsCompatibleSignature::value + */ +template +class IsCompatibleSignature { + static constexpr bool value = false; +}; + +template +class IsCompatibleSignature { + template()(std::declval()...)), + bool good = std::is_same::value> + static constexpr bool testArgs(int* p) { + return good; + } + + template + static constexpr bool testArgs(...) { + return false; + } +public: + static constexpr bool value = testArgs(nullptr); +}; + +/** + * FBounded - Helper type for the curiously recurring template pattern, used + * heavily here to enable inlining and obviate virtual functions + */ +template +struct FBounded { + const Self& self() const { + return *static_cast(this); + } + + Self& self() { + return *static_cast(this); + } +}; + +/** + * Operator - Core abstraction of an operation which may be applied to a + * generator. All operators implement a method compose(), which takes a + * generator and produces an output generator. + */ +template +class Operator : public FBounded { + public: + /** + * compose() - Must be implemented by child class to compose a new Generator + * out of a given generator. This function left intentionally unimplemented. + */ + template + ResultGen compose(const GenImpl& source) const; + + protected: + Operator() = default; + Operator(const Operator&) = default; + Operator(Operator&&) = default; +}; + +/** + * operator|() - For composing two operators without binding it to a + * particular generator. + */ +template> +Composed operator|(const Operator& left, + const Operator& right) { + return Composed(left.self(), right.self()); +} + +template> +Composed operator|(const Operator& left, + Operator&& right) { + return Composed(left.self(), std::move(right.self())); +} + +template> +Composed operator|(Operator&& left, + const Operator& right) { + return Composed(std::move(left.self()), right.self()); +} + +template> +Composed operator|(Operator&& left, + Operator&& right) { + return Composed(std::move(left.self()), std::move(right.self())); +} + +/** + * GenImpl - Core abstraction of a generator, an object which produces values by + * passing them to a given handler lambda. All generator implementations must + * implement apply(). foreach() may also be implemented to special case the + * condition where the entire sequence is consumed. + */ +template +class GenImpl : public FBounded { + protected: + // To prevent slicing + GenImpl() = default; + GenImpl(const GenImpl&) = default; + GenImpl(GenImpl&&) = default; + + public: + typedef Value ValueType; + typedef typename std::decay::type StorageType; + + /** + * apply() - Send all values produced by this generator to given handler until + * the handler returns false. Returns false if and only if the handler passed + * in returns false. Note: It should return true even if it completes (without + * the handler returning false), as 'Chain' uses the return value of apply to + * determine if it should process the second object in its chain. + */ + template + bool apply(Handler&& handler) const; + + /** + * foreach() - Send all values produced by this generator to given lambda. + */ + template + void foreach(Body&& body) const { + this->self().apply([&](Value value) -> bool { + static_assert(!infinite, "Cannot call foreach on infinite GenImpl"); + body(std::forward(value)); + return true; + }); + } + + // Child classes should override if the sequence generated is *definitely* + // infinite. 'infinite' may be false_type for some infinite sequences + // (due the the Halting Problem). + static constexpr bool infinite = false; +}; + +template> +Chain operator+(const GenImpl& left, + const GenImpl& right) { + static_assert( + std::is_same::value, + "Generators may ony be combined if Values are the exact same type."); + return Chain(left.self(), right.self()); +} + +template> +Chain operator+(const GenImpl& left, + GenImpl&& right) { + static_assert( + std::is_same::value, + "Generators may ony be combined if Values are the exact same type."); + return Chain(left.self(), std::move(right.self())); +} + +template> +Chain operator+(GenImpl&& left, + const GenImpl& right) { + static_assert( + std::is_same::value, + "Generators may ony be combined if Values are the exact same type."); + return Chain(std::move(left.self()), right.self()); +} + +template> +Chain operator+(GenImpl&& left, + GenImpl&& right) { + static_assert( + std::is_same::value, + "Generators may ony be combined if Values are the exact same type."); + return Chain(std::move(left.self()), std::move(right.self())); +} + +/** + * operator|() which enables foreach-like usage: + * gen | [](Value v) -> void {...}; + */ +template +typename std::enable_if< + IsCompatibleSignature::value>::type +operator|(const GenImpl& gen, Handler&& handler) { + static_assert(!Gen::infinite, + "Cannot pull all values from an infinite sequence."); + gen.self().foreach(std::forward(handler)); +} + +/** + * operator|() which enables foreach-like usage with 'break' support: + * gen | [](Value v) -> bool { return shouldContinue(); }; + */ +template +typename std::enable_if< + IsCompatibleSignature::value, bool>::type +operator|(const GenImpl& gen, Handler&& handler) { + return gen.self().apply(std::forward(handler)); +} + +/** + * operator|() for composing generators with operators, similar to boosts' range + * adaptors: + * gen | map(square) | sum + */ +template +auto operator|(const GenImpl& gen, const Operator& op) -> +decltype(op.self().compose(gen.self())) { + return op.self().compose(gen.self()); +} + +template +auto operator|(GenImpl&& gen, const Operator& op) -> +decltype(op.self().compose(std::move(gen.self()))) { + return op.self().compose(std::move(gen.self())); +} + +namespace detail { + +/** + * Composed - For building up a pipeline of operations to perform, absent any + * particular source generator. Useful for building up custom pipelines. + * + * This type is usually used by just piping two operators together: + * + * auto valuesOf = filter([](Optional& o) { return o.hasValue(); }) + * | map([](Optional& o) -> int& { return o.value(); }); + * + * auto valuesIncluded = from(optionals) | valuesOf | as(); + */ +template +class Composed : public Operator> { + First first_; + Second second_; + public: + Composed() {} + + Composed(First first, Second second) + : first_(std::move(first)) + , second_(std::move(second)) {} + + template() + .compose(std::declval())), + class SecondRet = decltype(std::declval() + .compose(std::declval()))> + SecondRet compose(const GenImpl& source) const { + return second_.compose(first_.compose(source.self())); + } + + template() + .compose(std::declval())), + class SecondRet = decltype(std::declval() + .compose(std::declval()))> + SecondRet compose(GenImpl&& source) const { + return second_.compose(first_.compose(std::move(source.self()))); + } +}; + +/** + * Chain - For concatenating the values produced by two Generators. + * + * This type is primarily used through using '+' to combine generators, like: + * + * auto nums = seq(1, 10) + seq(20, 30); + * int total = nums | sum; + */ +template +class Chain : public GenImpl> { + First first_; + Second second_; +public: + explicit Chain(First first, Second second) + : first_(std::move(first)) + , second_(std::move(second)) {} + + template + bool apply(Handler&& handler) const { + return first_.apply(std::forward(handler)) + && second_.apply(std::forward(handler)); + } + + template + void foreach(Body&& body) const { + first_.foreach(std::forward(body)); + second_.foreach(std::forward(body)); + } + + static constexpr bool infinite = First::infinite || Second::infinite; +}; + +} // detail + +}} // folly::gen + +#pragma GCC diagnostic pop diff --git a/folly/gen/Core.h b/folly/gen/Core.h new file mode 100644 index 00000000..d470199c --- /dev/null +++ b/folly/gen/Core.h @@ -0,0 +1,45 @@ +/* + * Copyright 2014 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FOLLY_GEN_CORE_H +#define FOLLY_GEN_CORE_H + +namespace folly { namespace gen { + +template +class GenImpl; + +template +class Operator; + +namespace detail { + +template +struct FBounded; + +template +class Composed; + +template +class Chain; + +} // detail + +}} // folly::gen + +#include "folly/gen/Core-inl.h" + +#endif // FOLLY_GEN_CORE_H diff --git a/folly/gen/File-inl.h b/folly/gen/File-inl.h new file mode 100644 index 00000000..268bd816 --- /dev/null +++ b/folly/gen/File-inl.h @@ -0,0 +1,145 @@ +/* + * Copyright 2014 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FOLLY_GEN_FILE_H +#error This file may only be included from folly/gen/File.h +#endif + +#include + +#include "folly/gen/String.h" + +namespace folly { +namespace gen { +namespace detail { + +class FileReader : public GenImpl { + public: + FileReader(File file, std::unique_ptr buffer) + : file_(std::move(file)), + buffer_(std::move(buffer)) { + buffer_->clear(); + } + + template + bool apply(Body&& body) const { + for (;;) { + ssize_t n; + do { + n = ::read(file_.fd(), buffer_->writableTail(), buffer_->capacity()); + } while (n == -1 && errno == EINTR); + if (n == -1) { + throw std::system_error(errno, std::system_category(), "read failed"); + } + if (n == 0) { + return true; + } + if (!body(ByteRange(buffer_->tail(), n))) { + return false; + } + } + } + + // Technically, there could be infinite files (e.g. /dev/random), but people + // who open those can do so at their own risk. + static constexpr bool infinite = false; + + private: + File file_; + std::unique_ptr buffer_; +}; + +class FileWriter : public Operator { + public: + FileWriter(File file, std::unique_ptr buffer) + : file_(std::move(file)), + buffer_(std::move(buffer)) { + if (buffer_) { + buffer_->clear(); + } + } + + template + void compose(const GenImpl& source) const { + auto fn = [&](ByteRange v) { + if (!this->buffer_ || v.size() >= this->buffer_->capacity()) { + this->flushBuffer(); + this->write(v); + } else { + if (v.size() > this->buffer_->tailroom()) { + this->flushBuffer(); + } + memcpy(this->buffer_->writableTail(), v.data(), v.size()); + this->buffer_->append(v.size()); + } + }; + + // Iterate + source.foreach(std::move(fn)); + + flushBuffer(); + file_.close(); + } + + private: + void write(ByteRange v) const { + ssize_t n; + while (!v.empty()) { + do { + n = ::write(file_.fd(), v.data(), v.size()); + } while (n == -1 && errno == EINTR); + if (n == -1) { + throw std::system_error(errno, std::system_category(), + "write() failed"); + } + v.advance(n); + } + } + + void flushBuffer() const { + if (buffer_ && buffer_->length() != 0) { + write(ByteRange(buffer_->data(), buffer_->length())); + buffer_->clear(); + } + } + + mutable File file_; + std::unique_ptr buffer_; +}; + +} // !detail + +/** + * Generator which reads lines from a file. + * Note: This produces StringPieces which reference temporary strings which are + * only valid during iteration. + */ +inline auto byLine(File file, char delim = '\n') + -> decltype(fromFile(std::move(file)) + | eachAs() + | resplit(delim)) { + return fromFile(std::move(file)) + | eachAs() + | resplit(delim); +} + +inline auto byLine(int fd, char delim = '\n') + -> decltype(byLine(File(fd), delim)) { return byLine(File(fd), delim); } + +inline auto byLine(const char* f, char delim = '\n') + -> decltype(byLine(File(f), delim)) { return byLine(File(f), delim); } + +}} // !folly::gen diff --git a/folly/gen/File.h b/folly/gen/File.h new file mode 100644 index 00000000..ded64c20 --- /dev/null +++ b/folly/gen/File.h @@ -0,0 +1,72 @@ +/* + * Copyright 2013 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FOLLY_GEN_FILE_H +#define FOLLY_GEN_FILE_H + +#include "folly/File.h" +#include "folly/gen/Base.h" +#include "folly/io/IOBuf.h" + +namespace folly { +namespace gen { + +namespace detail { +class FileReader; +class FileWriter; +} // namespace detail + +/** + * Generator that reads from a file with a buffer of the given size. + * Reads must be buffered (the generator interface expects the generator + * to hold each value). + */ +template +S fromFile(File file, size_t bufferSize=4096) { + return S(std::move(file), IOBuf::create(bufferSize)); +} + +/** + * Generator that reads from a file using a given buffer. + */ +template +S fromFile(File file, std::unique_ptr buffer) { + return S(std::move(file), std::move(buffer)); +} + +/** + * Sink that writes to a file with a buffer of the given size. + * If bufferSize is 0, writes will be unbuffered. + */ +template +S toFile(File file, size_t bufferSize=4096) { + return S(std::move(file), bufferSize ? nullptr : IOBuf::create(bufferSize)); +} + +/** + * Sink that writes to a file using a given buffer. + * If the buffer is nullptr, writes will be unbuffered. + */ +template +S toFile(File file, std::unique_ptr buffer) { + return S(std::move(file), std::move(buffer)); +} + +}} // !folly::gen + +#include "folly/gen/File-inl.h" + +#endif // FOLLY_GEN_FILE_H diff --git a/folly/gen/String-inl.h b/folly/gen/String-inl.h new file mode 100644 index 00000000..e29f2a57 --- /dev/null +++ b/folly/gen/String-inl.h @@ -0,0 +1,271 @@ +/* + * Copyright 2014 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FOLLY_GEN_STRING_H +#error This file may only be included from folly/gen/String.h +#endif + +#include "folly/Conv.h" +#include "folly/String.h" +#include "folly/io/IOBuf.h" + +namespace folly { +namespace gen { +namespace detail { + +inline bool splitPrefix(StringPiece& in, StringPiece& prefix, char delimiter) { + auto p = static_cast(memchr(in.data(), delimiter, in.size())); + if (p) { + prefix.assign(in.data(), p); + in.assign(p + 1, in.end()); + return true; + } + prefix.clear(); + return false; +} + +inline const char* ch(const unsigned char* p) { + return reinterpret_cast(p); +} + +class StringResplitter : public Operator { + char delimiter_; + public: + explicit StringResplitter(char delimiter) : delimiter_(delimiter) { } + + template + class Generator : public GenImpl> { + Source source_; + char delimiter_; + public: + Generator(Source source, char delimiter) + : source_(std::move(source)), delimiter_(delimiter) { } + + template + bool apply(Body&& body) const { + std::unique_ptr buffer; + + auto fn = [&](StringPiece in) -> bool { + StringPiece prefix; + bool found = splitPrefix(in, prefix, this->delimiter_); + if (found && buffer && buffer->length() != 0) { + // Append to end of buffer, return line + if (!prefix.empty()) { + buffer->reserve(0, prefix.size()); + memcpy(buffer->writableTail(), prefix.data(), prefix.size()); + buffer->append(prefix.size()); + } + if (!body(StringPiece(ch(buffer->data()), buffer->length()))) { + return false; + } + buffer->clear(); + found = splitPrefix(in, prefix, this->delimiter_); + } + // Buffer is empty, return lines directly from input (no buffer) + while (found) { + if (!body(prefix)) { + return false; + } + found = splitPrefix(in, prefix, this->delimiter_); + } + if (!in.empty()) { + // Incomplete line left, append to buffer + if (!buffer) { + // Arbitrarily assume that we have half a line and get enough + // room for twice that. + constexpr size_t kDefaultLineSize = 256; + buffer = IOBuf::create(std::max(kDefaultLineSize, 2 * in.size())); + } + buffer->reserve(0, in.size()); + memcpy(buffer->writableTail(), in.data(), in.size()); + buffer->append(in.size()); + } + return true; + }; + + // Iterate + if (!source_.apply(std::move(fn))) { + return false; + } + + // Incomplete last line + if (buffer && buffer->length() != 0) { + if (!body(StringPiece(ch(buffer->data()), buffer->length()))) { + return false; + } + } + return true; + } + + static constexpr bool infinite = Source::infinite; + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self()), delimiter_); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self(), delimiter_); + } +}; + +class SplitStringSource : public GenImpl { + StringPiece source_; + char delimiter_; + public: + SplitStringSource(const StringPiece& source, + char delimiter) + : source_(source) + , delimiter_(delimiter) { } + + template + bool apply(Body&& body) const { + StringPiece rest(source_); + StringPiece prefix; + while (splitPrefix(rest, prefix, this->delimiter_)) { + if (!body(prefix)) { + return false; + } + } + if (!rest.empty()) { + if (!body(rest)) { + return false; + } + } + return true; + } +}; + +/** + * Unsplit - For joining tokens from a generator into a string. This is + * the inverse of `split` above. + * + * This type is primarily used through the 'unsplit' function. + */ +template +class Unsplit : public Operator> { + Delimiter delimiter_; + public: + Unsplit(const Delimiter& delimiter) + : delimiter_(delimiter) { + } + + template + Output compose(const GenImpl& source) const { + Output outputBuffer; + UnsplitBuffer unsplitter(delimiter_, &outputBuffer); + unsplitter.compose(source); + return outputBuffer; + } +}; + +/** + * UnsplitBuffer - For joining tokens from a generator into a string, + * and inserting them into a custom buffer. + * + * This type is primarily used through the 'unsplit' function. + */ +template +class UnsplitBuffer : public Operator> { + Delimiter delimiter_; + OutputBuffer* outputBuffer_; + public: + UnsplitBuffer(const Delimiter& delimiter, OutputBuffer* outputBuffer) + : delimiter_(delimiter) + , outputBuffer_(outputBuffer) { + CHECK(outputBuffer); + } + + template + void compose(const GenImpl& source) const { + // If the output buffer is empty, we skip inserting the delimiter for the + // first element. + bool skipDelim = outputBuffer_->empty(); + source | [&](Value v) { + if (skipDelim) { + skipDelim = false; + toAppend(std::forward(v), outputBuffer_); + } else { + toAppend(delimiter_, std::forward(v), outputBuffer_); + } + }; + } +}; + + +/** + * Hack for static for-like constructs + */ +template +inline Target passthrough(Target target) { return target; } + +#pragma GCC diagnostic push +#ifdef __clang__ +// Clang isn't happy with eatField() hack below. +#pragma GCC diagnostic ignored "-Wreturn-stack-address" +#endif // __clang__ + +/** + * ParseToTuple - For splitting a record and immediatlely converting it to a + * target tuple type. Primary used through the 'eachToTuple' helper, like so: + * + * auto config + * = split("1:a 2:b", ' ') + * | eachToTuple() + * | as>>(); + * + */ +template +class SplitTo { + Delimiter delimiter_; + public: + explicit SplitTo(Delimiter delimiter) + : delimiter_(delimiter) {} + + TargetContainer operator()(StringPiece line) const { + int i = 0; + StringPiece fields[sizeof...(Targets)]; + // HACK(tjackson): Used for referencing fields[] corresponding to variadic + // template parameters. + auto eatField = [&]() -> StringPiece& { return fields[i++]; }; + if (!split(delimiter_, + line, + detail::passthrough(eatField())...)) { + throw std::runtime_error("field count mismatch"); + } + i = 0; + return TargetContainer(To()(eatField())...); + } +}; + +#pragma GCC diagnostic pop + +} // namespace detail + +} // namespace gen +} // namespace folly diff --git a/folly/gen/String.h b/folly/gen/String.h new file mode 100644 index 00000000..3f4497b1 --- /dev/null +++ b/folly/gen/String.h @@ -0,0 +1,155 @@ +/* + * Copyright 2013 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FOLLY_GEN_STRING_H +#define FOLLY_GEN_STRING_H + +#include "folly/Range.h" +#include "folly/gen/Base.h" + +namespace folly { +namespace gen { + +namespace detail { +class StringResplitter; +class SplitStringSource; + +template +class Unsplit; + +template +class UnsplitBuffer; + +template +class SplitTo; + +} // namespace detail + +/** + * Split the output from a generator into StringPiece "lines" delimited by + * the given delimiter. Delimters are NOT included in the output. + * + * resplit() behaves as if the input strings were concatenated into one long + * string and then split. + */ +// make this a template so we don't require StringResplitter to be complete +// until use +template +S resplit(char delimiter) { + return S(delimiter); +} + +template +S split(const StringPiece& source, char delimiter) { + return S(source, delimiter); +} + +/* + * Joins a sequence of tokens into a string, with the chosen delimiter. + * + * E.G. + * fbstring result = split("a,b,c", ",") | unsplit(","); + * assert(result == "a,b,c"); + * + * std::string result = split("a,b,c", ",") | unsplit(" "); + * assert(result == "a b c"); + */ + + +// NOTE: The template arguments are reversed to allow the user to cleanly +// specify the output type while still inferring the type of the delimiter. +template> +Unsplit unsplit(const Delimiter& delimiter) { + return Unsplit(delimiter); +} + +template> +Unsplit unsplit(const char* delimiter) { + return Unsplit(delimiter); +} + +/* + * Joins a sequence of tokens into a string, appending them to the output + * buffer. If the output buffer is empty, an initial delimiter will not be + * inserted at the start. + * + * E.G. + * std::string buffer; + * split("a,b,c", ",") | unsplit(",", &buffer); + * assert(buffer == "a,b,c"); + * + * std::string anotherBuffer("initial"); + * split("a,b,c", ",") | unsplit(",", &anotherbuffer); + * assert(anotherBuffer == "initial,a,b,c"); + */ +template> +UnsplitBuffer unsplit(Delimiter delimiter, OutputBuffer* outputBuffer) { + return UnsplitBuffer(delimiter, outputBuffer); +} + +template> +UnsplitBuffer unsplit(const char* delimiter, OutputBuffer* outputBuffer) { + return UnsplitBuffer(delimiter, outputBuffer); +} + + +template +detail::Map, char, Targets...>> +eachToTuple(char delim) { + return detail::Map< + detail::SplitTo, char, Targets...>>( + detail::SplitTo, char, Targets...>(delim)); +} + +template +detail::Map, fbstring, Targets...>> +eachToTuple(StringPiece delim) { + return detail::Map< + detail::SplitTo, fbstring, Targets...>>( + detail::SplitTo, fbstring, Targets...>(delim)); +} + +template +detail::Map, char, First, Second>> +eachToPair(char delim) { + return detail::Map< + detail::SplitTo, char, First, Second>>( + detail::SplitTo, char, First, Second>(delim)); +} + +template +detail::Map, fbstring, First, Second>> +eachToPair(StringPiece delim) { + return detail::Map< + detail::SplitTo, fbstring, First, Second>>( + detail::SplitTo, fbstring, First, Second>( + to(delim))); +} + +} // namespace gen +} // namespace folly + +#include "folly/gen/String-inl.h" + +#endif // FOLLY_GEN_STRING_H diff --git a/folly/gen/test/BaseBenchmark.cpp b/folly/gen/test/BaseBenchmark.cpp new file mode 100644 index 00000000..26009743 --- /dev/null +++ b/folly/gen/test/BaseBenchmark.cpp @@ -0,0 +1,344 @@ +/* + * Copyright 2014 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "folly/Benchmark.h" +#include "folly/gen/Base.h" + +using namespace folly; +using namespace folly::gen; +using std::pair; +using std::set; +using std::vector; +using std::tuple; + +static std::atomic testSize(1000); +static vector testVector = + seq(1, testSize.load()) + | mapped([](int) { return rand(); }) + | as(); + +static vector> testVectorVector = + seq(1, 100) + | map([](int i) { + return seq(1, i) | as(); + }) + | as(); +static vector strings = + from(testVector) + | eachTo() + | as(); + +auto square = [](int x) { return x * x; }; +auto add = [](int a, int b) { return a + b; }; +auto multiply = [](int a, int b) { return a * b; }; + +BENCHMARK(Sum_Basic_NoGen, iters) { + int limit = testSize.load(); + int s = 0; + while (iters--) { + for (int i = 0; i < limit; ++i) { + s += i; + } + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(Sum_Basic_Gen, iters) { + int limit = testSize.load(); + int s = 0; + while (iters--) { + s += range(0, limit) | sum; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_DRAW_LINE() + +BENCHMARK(Sum_Vector_NoGen, iters) { + int s = 0; + while (iters--) { + for (auto& i : testVector) { + s += i; + } + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(Sum_Vector_Gen, iters) { + int s = 0; + while (iters--) { + s += from(testVector) | sum; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_DRAW_LINE() + +BENCHMARK(Member, iters) { + int s = 0; + while(iters--) { + s += from(strings) + | member(&fbstring::size) + | sum; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(MapMember, iters) { + int s = 0; + while(iters--) { + s += from(strings) + | map([](const fbstring& x) { return x.size(); }) + | sum; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_DRAW_LINE() + +BENCHMARK(Count_Vector_NoGen, iters) { + int s = 0; + while (iters--) { + for (auto& i : testVector) { + if (i * 2 < rand()) { + ++s; + } + } + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(Count_Vector_Gen, iters) { + int s = 0; + while (iters--) { + s += from(testVector) + | filter([](int i) { + return i * 2 < rand(); + }) + | count; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_DRAW_LINE() + +BENCHMARK(Fib_Sum_NoGen, iters) { + int s = 0; + while (iters--) { + auto fib = [](int limit) -> vector { + vector ret; + int a = 0; + int b = 1; + for (int i = 0; i * 2 < limit; ++i) { + ret.push_back(a += b); + ret.push_back(b += a); + } + return ret; + }; + for (auto& v : fib(testSize.load())) { + s += v; + } + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(Fib_Sum_Gen, iters) { + int s = 0; + while (iters--) { + auto fib = GENERATOR(int) { + int a = 0; + int b = 1; + for (;;) { + yield(a += b); + yield(b += a); + } + }; + s += fib | take(testSize.load()) | sum; + } + folly::doNotOptimizeAway(s); +} + +struct FibYielder { + template + void operator()(Yield&& yield) const { + int a = 0; + int b = 1; + for (;;) { + yield(a += b); + yield(b += a); + } + } +}; + +BENCHMARK_RELATIVE(Fib_Sum_Gen_Static, iters) { + int s = 0; + while (iters--) { + auto fib = generator(FibYielder()); + s += fib | take(testSize.load()) | sum; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_DRAW_LINE() + +BENCHMARK(VirtualGen_0Virtual, iters) { + int s = 0; + while (iters--) { + auto numbers = seq(1, 10000); + auto squares = numbers | map(square); + auto quads = squares | map(square); + s += quads | sum; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(VirtualGen_1Virtual, iters) { + int s = 0; + while (iters--) { + VirtualGen numbers = seq(1, 10000); + auto squares = numbers | map(square); + auto quads = squares | map(square); + s += quads | sum; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(VirtualGen_2Virtual, iters) { + int s = 0; + while (iters--) { + VirtualGen numbers = seq(1, 10000); + VirtualGen squares = numbers | map(square); + auto quads = squares | map(square); + s += quads | sum; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(VirtualGen_3Virtual, iters) { + int s = 0; + while (iters--) { + VirtualGen numbers = seq(1, 10000); + VirtualGen squares = numbers | map(square); + VirtualGen quads = squares | map(square); + s += quads | sum; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_DRAW_LINE() + +BENCHMARK(Concat_NoGen, iters) { + int s = 0; + while (iters--) { + for (auto& v : testVectorVector) { + for (auto& i : v) { + s += i; + } + } + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(Concat_Gen, iters) { + int s = 0; + while (iters--) { + s += from(testVectorVector) | rconcat | sum; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_DRAW_LINE() + +BENCHMARK(Composed_NoGen, iters) { + int s = 0; + while (iters--) { + for (auto& i : testVector) { + s += i * i; + } + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(Composed_Gen, iters) { + int s = 0; + auto sumSq = map(square) | sum; + while (iters--) { + s += from(testVector) | sumSq; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(Composed_GenRegular, iters) { + int s = 0; + while (iters--) { + s += from(testVector) | map(square) | sum; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_DRAW_LINE() + +BENCHMARK(Sample, iters) { + size_t s = 0; + while (iters--) { + auto sampler = seq(1, 10 * 1000 * 1000) | sample(1000); + s += (sampler | sum); + } + folly::doNotOptimizeAway(s); +} + +// Results from an Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz +// ============================================================================ +// folly/gen/test/BaseBenchmark.cpp relative time/iter iters/s +// ============================================================================ +// Sum_Basic_NoGen 372.39ns 2.69M +// Sum_Basic_Gen 195.96% 190.03ns 5.26M +// ---------------------------------------------------------------------------- +// Sum_Vector_NoGen 200.41ns 4.99M +// Sum_Vector_Gen 77.14% 259.81ns 3.85M +// ---------------------------------------------------------------------------- +// Member 4.56us 219.42K +// MapMember 400.47% 1.14us 878.73K +// ---------------------------------------------------------------------------- +// Count_Vector_NoGen 13.96us 71.64K +// Count_Vector_Gen 86.05% 16.22us 61.65K +// ---------------------------------------------------------------------------- +// Fib_Sum_NoGen 2.21us 452.63K +// Fib_Sum_Gen 23.94% 9.23us 108.36K +// Fib_Sum_Gen_Static 48.77% 4.53us 220.73K +// ---------------------------------------------------------------------------- +// VirtualGen_0Virtual 9.60us 104.13K +// VirtualGen_1Virtual 28.00% 34.30us 29.15K +// VirtualGen_2Virtual 22.62% 42.46us 23.55K +// VirtualGen_3Virtual 16.96% 56.64us 17.66K +// ---------------------------------------------------------------------------- +// Concat_NoGen 2.20us 453.66K +// Concat_Gen 109.49% 2.01us 496.70K +// ---------------------------------------------------------------------------- +// Composed_NoGen 545.32ns 1.83M +// Composed_Gen 87.94% 620.07ns 1.61M +// Composed_GenRegular 88.13% 618.74ns 1.62M +// ---------------------------------------------------------------------------- +// Sample 176.48ms 5.67 +// ============================================================================ + +int main(int argc, char *argv[]) { + google::ParseCommandLineFlags(&argc, &argv, true); + runBenchmarks(); + return 0; +} diff --git a/folly/gen/test/BaseTest.cpp b/folly/gen/test/BaseTest.cpp new file mode 100644 index 00000000..3a0b500c --- /dev/null +++ b/folly/gen/test/BaseTest.cpp @@ -0,0 +1,998 @@ +/* + * Copyright 2014 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "folly/FBVector.h" +#include "folly/MapUtil.h" +#include "folly/Memory.h" +#include "folly/dynamic.h" +#include "folly/gen/Base.h" +#include "folly/experimental/TestUtil.h" + +using namespace folly::gen; +using namespace folly; +using std::make_tuple; +using std::ostream; +using std::pair; +using std::set; +using std::string; +using std::tuple; +using std::unique_ptr; +using std::vector; + +#define EXPECT_SAME(A, B) \ + static_assert(std::is_same::value, "Mismatched: " #A ", " #B) +EXPECT_SAME(int&&, typename ArgumentReference::type); +EXPECT_SAME(int&, typename ArgumentReference::type); +EXPECT_SAME(const int&, typename ArgumentReference::type); +EXPECT_SAME(const int&, typename ArgumentReference::type); + +template +ostream& operator<<(ostream& os, const set& values) { + return os << from(values); +} + +template +ostream& operator<<(ostream& os, const vector& values) { + os << "["; + for (auto& value : values) { + if (&value != &values.front()) { + os << " "; + } + os << value; + } + return os << "]"; +} + +auto square = [](int x) { return x * x; }; +auto add = [](int a, int b) { return a + b; }; +auto multiply = [](int a, int b) { return a * b; }; + +auto product = foldl(1, multiply); + +template +ostream& operator<<(ostream& os, const pair& pair) { + return os << "(" << pair.first << ", " << pair.second << ")"; +} + +TEST(Gen, Count) { + auto gen = seq(1, 10); + EXPECT_EQ(10, gen | count); + EXPECT_EQ(5, gen | take(5) | count); +} + +TEST(Gen, Sum) { + auto gen = seq(1, 10); + EXPECT_EQ((1 + 10) * 10 / 2, gen | sum); + EXPECT_EQ((1 + 5) * 5 / 2, gen | take(5) | sum); +} + +TEST(Gen, Foreach) { + auto gen = seq(1, 4); + int accum = 0; + gen | [&](int x) { accum += x; }; + EXPECT_EQ(10, accum); + int accum2 = 0; + gen | take(3) | [&](int x) { accum2 += x; }; + EXPECT_EQ(6, accum2); +} + +TEST(Gen, Map) { + auto expected = vector{4, 9, 16}; + auto gen = from({2, 3, 4}) | map(square); + EXPECT_EQ((vector{4, 9, 16}), gen | as()); + EXPECT_EQ((vector{4, 9}), gen | take(2) | as()); +} + +TEST(Gen, Member) { + struct Counter { + Counter(int start = 0) + : c(start) + {} + + int count() const { return c; } + int incr() { return ++c; } + + int& ref() { return c; } + const int& ref() const { return c; } + private: + int c; + }; + auto counters = seq(1, 10) | eachAs() | as(); + EXPECT_EQ(10 * (1 + 10) / 2, + from(counters) + | member(&Counter::count) + | sum); + EXPECT_EQ(10 * (2 + 11) / 2, + from(counters) + | member(&Counter::incr) + | sum); + EXPECT_EQ(10 * (2 + 11) / 2, + from(counters) + | member(&Counter::count) + | sum); + + // type-verifications + auto m = empty(); + auto c = empty(); + m | member(&Counter::incr) | assert_type(); + m | member(&Counter::count) | assert_type(); + m | member(&Counter::count) | assert_type(); + m | member(&Counter::ref) | assert_type(); + m | member(&Counter::ref) | assert_type(); + c | member(&Counter::ref) | assert_type(); +} + +TEST(Gen, Field) { + struct X { + X() : a(2), b(3), c(4), d(b) {} + + const int a; + int b; + mutable int c; + int& d; // can't access this with a field pointer. + }; + + std::vector xs(1); + EXPECT_EQ(2, from(xs) + | field(&X::a) + | first); + EXPECT_EQ(3, from(xs) + | field(&X::b) + | first); + EXPECT_EQ(4, from(xs) + | field(&X::c) + | first); + // type-verification + empty() | field(&X::a) | assert_type(); + empty() | field(&X::b) | assert_type(); + empty() | field(&X::c) | assert_type(); + empty() | field(&X::a) | assert_type(); + empty() | field(&X::b) | assert_type(); + empty() | field(&X::c) | assert_type(); + // references don't imply ownership so they're not moved + empty() | field(&X::a) | assert_type(); + empty() | field(&X::b) | assert_type(); + // 'mutable' has no effect on field pointers, by C++ spec + empty() | field(&X::c) | assert_type(); + + // can't form pointer-to-reference field: empty() | field(&X::d) +} + +TEST(Gen, Seq) { + // cover the fenceposts of the loop unrolling + for (int n = 1; n < 100; ++n) { + EXPECT_EQ(n, seq(1, n) | count); + EXPECT_EQ(n + 1, seq(1) | take(n + 1) | count); + } +} + +TEST(Gen, Range) { + // cover the fenceposts of the loop unrolling + for (int n = 1; n < 100; ++n) { + EXPECT_EQ(range(0, n) | count, n); + } +} + +TEST(Gen, FromIterators) { + vector source {2, 3, 5, 7, 11}; + auto gen = from(makeRange(source.begin() + 1, source.end() - 1)); + EXPECT_EQ(3 * 5 * 7, gen | product); +} + +TEST(Gen, FromMap) { + auto source = seq(0, 10) + | map([](int i) { return std::make_pair(i, i * i); }) + | as>(); + auto gen = fromConst(source) + | map([&](const std::pair& p) { + return p.second - p.first; + }); + EXPECT_EQ(330, gen | sum); +} + +TEST(Gen, Filter) { + const auto expected = vector{1, 2, 4, 5, 7, 8}; + auto actual = + seq(1, 9) + | filter([](int x) { return x % 3; }) + | as>(); + EXPECT_EQ(expected, actual); +} + +TEST(Gen, Contains) { + { + auto gen = + seq(1, 9) + | map(square); + EXPECT_TRUE(gen | contains(49)); + EXPECT_FALSE(gen | contains(50)); + } + { + auto gen = + seq(1) // infinite, to prove laziness + | map(square) + | eachTo(); + + // std::string gen, const char* needle + EXPECT_TRUE(gen | take(9999) | contains("49")); + } +} + +TEST(Gen, Take) { + { + auto expected = vector{1, 4, 9, 16}; + auto actual = + seq(1, 1000) + | mapped([](int x) { return x * x; }) + | take(4) + | as>(); + EXPECT_EQ(expected, actual); + } + { + auto expected = vector{ 0, 1, 4, 5, 8 }; + auto actual + = ((seq(0) | take(2)) + + (seq(4) | take(2)) + + (seq(8) | take(2))) + | take(5) + | as(); + EXPECT_EQ(expected, actual); + } + { + auto expected = vector{ 0, 1, 4, 5, 8 }; + auto actual + = seq(0) + | mapped([](int i) { + return seq(i * 4) | take(2); + }) + | concat + | take(5) + | as(); + EXPECT_EQ(expected, actual); + } +} + +TEST(Gen, Sample) { + std::mt19937 rnd(42); + + auto sampler = + seq(1, 100) + | sample(50, rnd); + std::unordered_map hits; + const int kNumIters = 80; + for (int i = 0; i < kNumIters; i++) { + auto vec = sampler | as>(); + EXPECT_EQ(vec.size(), 50); + auto uniq = fromConst(vec) | as>(); + EXPECT_EQ(uniq.size(), vec.size()); // sampling without replacement + for (auto v: vec) { + ++hits[v]; + } + } + + // In 80 separate samples of our range, we should have seen every value + // at least once and no value all 80 times. (The odds of either of those + // events is 1/2^80). + EXPECT_EQ(hits.size(), 100); + for (auto hit: hits) { + EXPECT_GT(hit.second, 0); + EXPECT_LT(hit.second, kNumIters); + } + + auto small = + seq(1, 5) + | sample(10); + EXPECT_EQ((small | sum), 15); + EXPECT_EQ((small | take(3) | count), 3); +} + +TEST(Gen, Skip) { + auto gen = + seq(1, 1000) + | mapped([](int x) { return x * x; }) + | skip(4) + | take(4); + EXPECT_EQ((vector{25, 36, 49, 64}), gen | as()); +} + +TEST(Gen, Until) { + { + auto expected = vector{1, 4, 9, 16}; + auto actual + = seq(1, 1000) + | mapped([](int x) { return x * x; }) + | until([](int x) { return x > 20; }) + | as>(); + EXPECT_EQ(expected, actual); + } + { + auto expected = vector{ 0, 1, 4, 5, 8 }; + auto actual + = ((seq(0) | until([](int i) { return i > 1; })) + + (seq(4) | until([](int i) { return i > 5; })) + + (seq(8) | until([](int i) { return i > 9; }))) + | until([](int i) { return i > 8; }) + | as>(); + EXPECT_EQ(expected, actual); + } + /* + { + auto expected = vector{ 0, 1, 5, 6, 10 }; + auto actual + = seq(0) + | mapped([](int i) { + return seq(i * 5) | until([=](int j) { return j > i * 5 + 1; }); + }) + | concat + | until([](int i) { return i > 10; }) + | as>(); + EXPECT_EQ(expected, actual); + } + */ +} + +TEST(Gen, Composed) { + // Operator, Operator + auto valuesOf = + filter([](Optional& o) { return o.hasValue(); }) + | map([](Optional& o) -> int& { return o.value(); }); + std::vector> opts { + none, 4, none, 6, none + }; + EXPECT_EQ(4 * 4 + 6 * 6, from(opts) | valuesOf | map(square) | sum); + // Operator, Sink + auto sumOpt = valuesOf | sum; + EXPECT_EQ(10, from(opts) | sumOpt); +} + +TEST(Gen, Chain) { + std::vector nums {2, 3, 5, 7}; + std::map mappings { { 3, 9}, {5, 25} }; + auto gen = from(nums) + (from(mappings) | get<1>()); + EXPECT_EQ(51, gen | sum); + EXPECT_EQ(5, gen | take(2) | sum); + EXPECT_EQ(26, gen | take(5) | sum); +} + +TEST(Gen, Concat) { + std::vector> nums {{2, 3}, {5, 7}}; + auto gen = from(nums) | rconcat; + EXPECT_EQ(17, gen | sum); + EXPECT_EQ(10, gen | take(3) | sum); +} + +TEST(Gen, ConcatGen) { + auto gen = seq(1, 10) + | map([](int i) { return seq(1, i); }) + | concat; + EXPECT_EQ(220, gen | sum); + EXPECT_EQ(10, gen | take(6) | sum); +} + +TEST(Gen, ConcatAlt) { + std::vector> nums {{2, 3}, {5, 7}}; + auto actual = from(nums) + | map([](std::vector& v) { return from(v); }) + | concat + | sum; + auto expected = 17; + EXPECT_EQ(expected, actual); +} + +TEST(Gen, Order) { + auto expected = vector{0, 3, 5, 6, 7, 8, 9}; + auto actual = + from({8, 6, 7, 5, 3, 0, 9}) + | order + | as(); + EXPECT_EQ(expected, actual); +} + +TEST(Gen, OrderMoved) { + auto expected = vector{0, 9, 25, 36, 49, 64, 81}; + auto actual = + from({8, 6, 7, 5, 3, 0, 9}) + | move + | order + | map(square) + | as(); + EXPECT_EQ(expected, actual); +} + +TEST(Gen, OrderTake) { + auto expected = vector{9, 8, 7}; + auto actual = + from({8, 6, 7, 5, 3, 0, 9}) + | orderByDescending(square) + | take(3) + | as(); + EXPECT_EQ(expected, actual); +} + +TEST(Gen, Distinct) { + auto expected = vector{3, 1, 2}; + auto actual = + from({3, 1, 3, 2, 1, 2, 3}) + | distinct + | as(); + EXPECT_EQ(expected, actual); +} + +TEST(Gen, DistinctBy) { // 0 1 4 9 6 5 6 9 4 1 0 + auto expected = vector{0, 1, 2, 3, 4, 5}; + auto actual = + seq(0, 100) + | distinctBy([](int i) { return i * i % 10; }) + | as(); + EXPECT_EQ(expected, actual); +} + +TEST(Gen, DistinctMove) { // 0 1 4 9 6 5 6 9 4 1 0 + auto expected = vector{0, 1, 2, 3, 4, 5}; + auto actual = + seq(0, 100) + | mapped([](int i) { return std::unique_ptr(new int(i)); }) + // see comment below about selector parameters for Distinct + | distinctBy([](const std::unique_ptr& pi) { return *pi * *pi % 10; }) + | mapped([](std::unique_ptr pi) { return *pi; }) + | as(); + + // NOTE(tjackson): the following line intentionally doesn't work: + // | distinctBy([](std::unique_ptr pi) { return *pi * *pi % 10; }) + // This is because distinctBy because the selector intentionally requires a + // const reference. If it required a move-reference, the value might get + // gutted by the selector before said value could be passed to downstream + // operators. + EXPECT_EQ(expected, actual); +} + +TEST(Gen, MinBy) { + EXPECT_EQ(7, seq(1, 10) + | minBy([](int i) -> double { + double d = i - 6.8; + return d * d; + })); +} + +TEST(Gen, MaxBy) { + auto gen = from({"three", "eleven", "four"}); + + EXPECT_EQ("eleven", gen | maxBy(&strlen)); +} + +TEST(Gen, Append) { + string expected = "facebook"; + string actual = "face"; + from(StringPiece("book")) | appendTo(actual); + EXPECT_EQ(expected, actual); +} + +TEST(Gen, FromRValue) { + { + // AFAICT The C++ Standard does not specify what happens to the rvalue + // reference of a std::vector when it is used as the 'other' for an rvalue + // constructor. Use fbvector because we're sure its size will be zero in + // this case. + fbvector v({1,2,3,4}); + auto q1 = from(v); + EXPECT_EQ(v.size(), 4); // ensure that the lvalue version was called! + auto expected = 1 * 2 * 3 * 4; + EXPECT_EQ(expected, q1 | product); + + auto q2 = from(std::move(v)); + EXPECT_EQ(v.size(), 0); // ensure that rvalue version was called + EXPECT_EQ(expected, q2 | product); + } + { + auto expected = 7; + auto q = from([] {return vector({3,7,5}); }()); + EXPECT_EQ(expected, q | max); + } + { + for (auto size: {5, 1024, 16384, 1<<20}) { + auto q1 = from(vector(size, 2)); + auto q2 = from(vector(size, 3)); + // If the rvalue specialization is broken/gone, then the compiler will + // (disgustingly!) just store a *reference* to the temporary object, + // which is bad. Try to catch this by allocating two temporary vectors + // of the same size, so that they'll probably use the same underlying + // buffer if q1's vector is destructed before q2's vector is constructed. + EXPECT_EQ(size * 2 + size * 3, (q1 | sum) + (q2 | sum)); + } + } + { + auto q = from(set{1,2,3,2,1}); + EXPECT_EQ(q | sum, 6); + } +} + +TEST(Gen, OrderBy) { + auto expected = vector{5, 6, 4, 7, 3, 8, 2, 9, 1, 10}; + auto actual = + seq(1, 10) + | orderBy([](int x) { return (5.1 - x) * (5.1 - x); }) + | as(); + EXPECT_EQ(expected, actual); +} + +TEST(Gen, Foldl) { + int expected = 2 * 3 * 4 * 5; + auto actual = + seq(2, 5) + | foldl(1, multiply); + EXPECT_EQ(expected, actual); +} + +TEST(Gen, Reduce) { + int expected = 2 + 3 + 4 + 5; + auto actual = seq(2, 5) | reduce(add); + EXPECT_EQ(expected, actual); +} + +TEST(Gen, ReduceBad) { + auto gen = seq(1) | take(0); + try { + EXPECT_TRUE(true); + gen | reduce(add); + EXPECT_TRUE(false); + } catch (...) { + } +} + +TEST(Gen, Moves) { + std::vector> ptrs; + ptrs.emplace_back(new int(1)); + EXPECT_NE(ptrs.front().get(), nullptr); + auto ptrs2 = from(ptrs) | move | as(); + EXPECT_EQ(ptrs.front().get(), nullptr); + EXPECT_EQ(**ptrs2.data(), 1); +} + +TEST(Gen, First) { + auto gen = + seq(0) + | filter([](int x) { return x > 3; }); + EXPECT_EQ(4, gen | first); +} + +TEST(Gen, FromCopy) { + vector v {3, 5}; + auto src = from(v); + auto copy = fromCopy(v); + EXPECT_EQ(8, src | sum); + EXPECT_EQ(8, copy | sum); + v[1] = 7; + EXPECT_EQ(10, src | sum); + EXPECT_EQ(8, copy | sum); +} + +TEST(Gen, Get) { + std::map pairs { + {1, 1}, + {2, 4}, + {3, 9}, + {4, 16}, + }; + auto pairSrc = from(pairs); + auto keys = pairSrc | get<0>(); + auto values = pairSrc | get<1>(); + EXPECT_EQ(10, keys | sum); + EXPECT_EQ(30, values | sum); + EXPECT_EQ(30, keys | map(square) | sum); + pairs[5] = 25; + EXPECT_EQ(15, keys | sum); + EXPECT_EQ(55, values | sum); + + vector> tuples { + make_tuple(1, 1, 1), + make_tuple(2, 4, 8), + make_tuple(3, 9, 27), + }; + EXPECT_EQ(36, from(tuples) | get<2>() | sum); +} + +TEST(Gen, Any) { + EXPECT_TRUE(seq(0) | any); + EXPECT_TRUE(seq(0, 1) | any); + EXPECT_TRUE(seq(0, 10) | any([](int i) { return i == 7; })); + EXPECT_FALSE(seq(0, 10) | any([](int i) { return i == 11; })); + + EXPECT_TRUE(from({1}) | any); + EXPECT_FALSE(range(0, 0) | any); + EXPECT_FALSE(from({1}) | take(0) | any); +} + +TEST(Gen, All) { + EXPECT_TRUE(seq(0, 10) | all([](int i) { return i < 11; })); + EXPECT_FALSE(seq(0, 10) | all([](int i) { return i < 5; })); + EXPECT_FALSE(seq(0) | take(9999) | all([](int i) { return i < 10; })); + + // empty lists satisfies all + EXPECT_TRUE(seq(0) | take(0) | all([](int i) { return i < 50; })); + EXPECT_TRUE(seq(0) | take(0) | all([](int i) { return i > 50; })); +} + +TEST(Gen, Yielders) { + auto gen = GENERATOR(int) { + for (int i = 1; i <= 5; ++i) { + yield(i); + } + yield(7); + for (int i = 3; ; ++i) { + yield(i * i); + } + }; + vector expected { + 1, 2, 3, 4, 5, 7, 9, 16, 25 + }; + EXPECT_EQ(expected, gen | take(9) | as()); +} + +TEST(Gen, NestedYield) { + auto nums = GENERATOR(int) { + for (int i = 1; ; ++i) { + yield(i); + } + }; + auto gen = GENERATOR(int) { + nums | take(10) | yield; + seq(1, 5) | [&](int i) { + yield(i); + }; + }; + EXPECT_EQ(70, gen | sum); +} + +TEST(Gen, MapYielders) { + auto gen = seq(1, 5) + | map([](int n) { + return GENERATOR(int) { + int i; + for (i = 1; i < n; ++i) + yield(i); + for (; i >= 1; --i) + yield(i); + }; + }) + | concat; + vector expected { + 1, + 1, 2, 1, + 1, 2, 3, 2, 1, + 1, 2, 3, 4, 3, 2, 1, + 1, 2, 3, 4, 5, 4, 3, 2, 1, + }; + EXPECT_EQ(expected, gen | as()); +} + +TEST(Gen, VirtualGen) { + VirtualGen v(seq(1, 10)); + EXPECT_EQ(55, v | sum); + v = v | map(square); + EXPECT_EQ(385, v | sum); + v = v | take(5); + EXPECT_EQ(55, v | sum); + EXPECT_EQ(30, v | take(4) | sum); +} + + +TEST(Gen, CustomType) { + struct Foo{ + int y; + }; + auto gen = from({Foo{2}, Foo{3}}) + | map([](const Foo& f) { return f.y; }); + EXPECT_EQ(5, gen | sum); +} + +TEST(Gen, NoNeedlessCopies) { + auto gen = seq(1, 5) + | map([](int x) { return unique_ptr(new int(x)); }) + | map([](unique_ptr p) { return p; }) + | map([](unique_ptr&& p) { return std::move(p); }) + | map([](const unique_ptr& p) { return *p; }); + EXPECT_EQ(15, gen | sum); + EXPECT_EQ(6, gen | take(3) | sum); +} + +namespace { + +class TestIntSeq : public GenImpl { + public: + TestIntSeq() { } + + template + bool apply(Body&& body) const { + for (int i = 1; i < 6; ++i) { + if (!body(i)) { + return false; + } + } + return true; + } + + TestIntSeq(TestIntSeq&&) = default; + TestIntSeq& operator=(TestIntSeq&&) = default; + TestIntSeq(const TestIntSeq&) = delete; + TestIntSeq& operator=(const TestIntSeq&) = delete; +}; + +} // namespace + +TEST(Gen, NoGeneratorCopies) { + EXPECT_EQ(15, TestIntSeq() | sum); + auto x = TestIntSeq() | take(3); + EXPECT_EQ(6, std::move(x) | sum); +} + +TEST(Gen, FromArray) { + int source[] = {2, 3, 5, 7}; + auto gen = from(source); + EXPECT_EQ(2 * 3 * 5 * 7, gen | product); +} + +TEST(Gen, FromStdArray) { + std::array source {{2, 3, 5, 7}}; + auto gen = from(source); + EXPECT_EQ(2 * 3 * 5 * 7, gen | product); +} + +TEST(Gen, StringConcat) { + auto gen = seq(1, 10) + | eachTo() + | rconcat; + EXPECT_EQ("12345678910", gen | as()); +} + +struct CopyCounter { + static int alive; + int copies; + int moves; + + CopyCounter() : copies(0), moves(0) { + ++alive; + } + + CopyCounter(CopyCounter&& source) { + *this = std::move(source); + ++alive; + } + + CopyCounter(const CopyCounter& source) { + *this = source; + ++alive; + } + + ~CopyCounter() { + --alive; + } + + CopyCounter& operator=(const CopyCounter& source) { + this->copies = source.copies + 1; + this->moves = source.moves; + return *this; + } + + CopyCounter& operator=(CopyCounter&& source) { + this->copies = source.copies; + this->moves = source.moves + 1; + return *this; + } +}; + +int CopyCounter::alive = 0; + +TEST(Gen, CopyCount) { + vector originals; + originals.emplace_back(); + EXPECT_EQ(1, originals.size()); + EXPECT_EQ(0, originals.back().copies); + + vector copies = from(originals) | as(); + EXPECT_EQ(1, copies.back().copies); + EXPECT_EQ(0, copies.back().moves); + + vector moves = from(originals) | move | as(); + EXPECT_EQ(0, moves.back().copies); + EXPECT_EQ(1, moves.back().moves); +} + +// test dynamics with various layers of nested arrays. +TEST(Gen, Dynamic) { + dynamic array1 = {1, 2}; + EXPECT_EQ(dynamic(3), from(array1) | sum); + dynamic array2 = {{1}, {1, 2}}; + EXPECT_EQ(dynamic(4), from(array2) | rconcat | sum); + dynamic array3 = {{{1}}, {{1}, {1, 2}}}; + EXPECT_EQ(dynamic(5), from(array3) | rconcat | rconcat | sum); +} + +TEST(Gen, DynamicObject) { + const dynamic obj = dynamic::object(1, 2)(3, 4); + EXPECT_EQ(dynamic(4), from(obj.keys()) | sum); + EXPECT_EQ(dynamic(6), from(obj.values()) | sum); + EXPECT_EQ(dynamic(4), from(obj.items()) | get<0>() | sum); + EXPECT_EQ(dynamic(6), from(obj.items()) | get<1>() | sum); +} + +TEST(Gen, Collect) { + auto s = from({7, 6, 5, 4, 3}) | as>(); + EXPECT_EQ(s.size(), 5); +} + + +TEST(Gen, Cycle) { + { + auto s = from({1, 2}); + EXPECT_EQ((vector { 1, 2, 1, 2, 1 }), + s | cycle | take(5) | as()); + } + { + auto s = from({1, 2}); + EXPECT_EQ((vector { 1, 2, 1, 2 }), + s | cycle(2) | as()); + } + { + auto s = from({1, 2, 3}); + EXPECT_EQ((vector { 1, 2, 1, 2, 1 }), + s | take(2) | cycle | take(5) | as()); + } + { + auto s = empty(); + EXPECT_EQ((vector { }), + s | cycle | take(4) | as()); + } + { + int count = 3; + int* pcount = &count; + auto countdown = GENERATOR(int) { + ASSERT_GE(*pcount, 0) + << "Cycle should have stopped when it didnt' get values!"; + for (int i = 1; i <= *pcount; ++i) { + yield(i); + } + --*pcount; + }; + auto s = countdown; + EXPECT_EQ((vector { 1, 2, 3, 1, 2, 1}), + s | cycle | as()); + } +} + +TEST(Gen, Dereference) { + { + const int x = 4, y = 2; + auto s = from({&x, nullptr, &y}); + EXPECT_EQ(6, s | dereference | sum); + } + { + vector a { 1, 2 }; + vector b { 3, 4 }; + vector*> pv { &a, nullptr, &b }; + from(pv) + | dereference + | [&](vector& v) { + v.push_back(5); + }; + EXPECT_EQ(3, a.size()); + EXPECT_EQ(3, b.size()); + EXPECT_EQ(5, a.back()); + EXPECT_EQ(5, b.back()); + } + { + vector> maps { + { + { 2, 31 }, + { 3, 41 }, + }, + { + { 3, 52 }, + { 4, 62 }, + }, + { + { 4, 73 }, + { 5, 83 }, + }, + }; + EXPECT_EQ( + 93, + from(maps) + | map([](std::map& m) { + return get_ptr(m, 3); + }) + | dereference + | sum); + } + { + vector> ups; + ups.emplace_back(new int(3)); + ups.emplace_back(); + ups.emplace_back(new int(7)); + EXPECT_EQ(10, from(ups) | dereference | sum); + EXPECT_EQ(10, from(ups) | move | dereference | sum); + } +} + +TEST(Gen, Guard) { + using std::runtime_error; + EXPECT_THROW(from({"1", "a", "3"}) + | eachTo() + | sum, + runtime_error); + EXPECT_EQ(4, + from({"1", "a", "3"}) + | guard([](runtime_error&, const char*) { + return true; // continue + }) + | eachTo() + | sum); + EXPECT_EQ(1, + from({"1", "a", "3"}) + | guard([](runtime_error&, const char*) { + return false; // break + }) + | eachTo() + | sum); + EXPECT_THROW(from({"1", "a", "3"}) + | guard([](runtime_error&, const char* v) { + if (v[0] == 'a') { + throw; + } + return true; + }) + | eachTo() + | sum, + runtime_error); +} + +TEST(Gen, Batch) { + EXPECT_EQ((vector> { {1} }), + seq(1, 1) | batch(5) | as()); + EXPECT_EQ((vector> { {1, 2, 3}, {4, 5, 6}, {7, 8, 9}, {10, 11} }), + seq(1, 11) | batch(3) | as()); + EXPECT_THROW(seq(1, 1) | batch(0) | as(), + std::invalid_argument); +} + +TEST(Gen, BatchMove) { + auto expected = vector>{ {0, 1}, {2, 3}, {4} }; + auto actual = + seq(0, 4) + | mapped([](int i) { return std::unique_ptr(new int(i)); }) + | batch(2) + | mapped([](std::vector>& pVector) { + std::vector iVector; + for (const auto& p : pVector) { + iVector.push_back(*p); + }; + return iVector; + }) + | as(); + EXPECT_EQ(expected, actual); +} + +int main(int argc, char *argv[]) { + testing::InitGoogleTest(&argc, argv); + google::ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +} diff --git a/folly/gen/test/CombineTest.cpp b/folly/gen/test/CombineTest.cpp new file mode 100644 index 00000000..8f879888 --- /dev/null +++ b/folly/gen/test/CombineTest.cpp @@ -0,0 +1,167 @@ +/* + * Copyright 2014 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include + +#include "folly/Range.h" +#include "folly/FBVector.h" +#include "folly/experimental/TestUtil.h" +#include "folly/gen/Base.h" +#include "folly/gen/Combine.h" + +using namespace folly::gen; +using namespace folly; +using std::string; +using std::vector; +using std::tuple; + +auto even = [](int i) -> bool { return i % 2 == 0; }; +auto odd = [](int i) -> bool { return i % 2 == 1; }; + +TEST(CombineGen, Interleave) { + { // large (infinite) base, small container + auto base = seq(1) | filter(odd); + auto toInterleave = seq(1, 6) | filter(even); + auto interleaved = base | interleave(toInterleave | as()); + EXPECT_EQ(interleaved | as(), vector({1, 2, 3, 4, 5, 6})); + } + { // small base, large container + auto base = seq(1) | filter(odd) | take(3); + auto toInterleave = seq(1) | filter(even) | take(50); + auto interleaved = base | interleave(toInterleave | as()); + EXPECT_EQ(interleaved | as(), + vector({1, 2, 3, 4, 5, 6})); + } +} + +TEST(CombineGen, Zip) { + auto base0 = seq(1); + // We rely on std::move(fbvector) emptying the source vector + auto zippee = fbvector{"one", "two", "three"}; + { + auto combined = base0 + | zip(zippee) + | as(); + ASSERT_EQ(combined.size(), 3); + EXPECT_EQ(std::get<0>(combined[0]), 1); + EXPECT_EQ(std::get<1>(combined[0]), "one"); + EXPECT_EQ(std::get<0>(combined[1]), 2); + EXPECT_EQ(std::get<1>(combined[1]), "two"); + EXPECT_EQ(std::get<0>(combined[2]), 3); + EXPECT_EQ(std::get<1>(combined[2]), "three"); + ASSERT_FALSE(zippee.empty()); + EXPECT_FALSE(zippee.front().empty()); // shouldn't have been move'd + } + + { // same as top, but using std::move. + auto combined = base0 + | zip(std::move(zippee)) + | as(); + ASSERT_EQ(combined.size(), 3); + EXPECT_EQ(std::get<0>(combined[0]), 1); + EXPECT_TRUE(zippee.empty()); + } + + { // same as top, but base is truncated + auto baseFinite = seq(1) | take(1); + auto combined = baseFinite + | zip(vector{"one", "two", "three"}) + | as(); + ASSERT_EQ(combined.size(), 1); + EXPECT_EQ(std::get<0>(combined[0]), 1); + EXPECT_EQ(std::get<1>(combined[0]), "one"); + } +} + +TEST(CombineGen, TupleFlatten) { + vector> intStringTupleVec{ + tuple{1, "1"}, + tuple{2, "2"}, + tuple{3, "3"}, + }; + + vector> charTupleVec{ + tuple{'A'}, + tuple{'B'}, + tuple{'C'}, + tuple{'D'}, + }; + + vector doubleVec{ + 1.0, + 4.0, + 9.0, + 16.0, + 25.0, + }; + + auto zipped1 = from(intStringTupleVec) + | zip(charTupleVec) + | assert_type, tuple>>() + | as(); + EXPECT_EQ(std::get<0>(zipped1[0]), std::make_tuple(1, "1")); + EXPECT_EQ(std::get<1>(zipped1[0]), std::make_tuple('A')); + + auto zipped2 = from(zipped1) + | tuple_flatten + | assert_type&&>() + | as(); + ASSERT_EQ(zipped2.size(), 3); + EXPECT_EQ(zipped2[0], std::make_tuple(1, "1", 'A')); + + auto zipped3 = from(charTupleVec) + | zip(intStringTupleVec) + | tuple_flatten + | assert_type&&>() + | as(); + ASSERT_EQ(zipped3.size(), 3); + EXPECT_EQ(zipped3[0], std::make_tuple('A', 1, "1")); + + auto zipped4 = from(intStringTupleVec) + | zip(doubleVec) + | tuple_flatten + | assert_type&&>() + | as(); + ASSERT_EQ(zipped4.size(), 3); + EXPECT_EQ(zipped4[0], std::make_tuple(1, "1", 1.0)); + + auto zipped5 = from(doubleVec) + | zip(doubleVec) + | assert_type>() + | tuple_flatten // essentially a no-op + | assert_type&&>() + | as(); + ASSERT_EQ(zipped5.size(), 5); + EXPECT_EQ(zipped5[0], std::make_tuple(1.0, 1.0)); + + auto zipped6 = from(intStringTupleVec) + | zip(charTupleVec) + | tuple_flatten + | zip(doubleVec) + | tuple_flatten + | assert_type&&>() + | as(); + ASSERT_EQ(zipped6.size(), 3); + EXPECT_EQ(zipped6[0], std::make_tuple(1, "1", 'A', 1.0)); +} + +int main(int argc, char *argv[]) { + testing::InitGoogleTest(&argc, argv); + google::ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +} diff --git a/folly/gen/test/FileBenchmark.cpp b/folly/gen/test/FileBenchmark.cpp new file mode 100644 index 00000000..339fd293 --- /dev/null +++ b/folly/gen/test/FileBenchmark.cpp @@ -0,0 +1,70 @@ +/* + * Copyright 2014 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "folly/Benchmark.h" +#include "folly/File.h" +#include "folly/gen/Base.h" +#include "folly/gen/File.h" + +using namespace folly::gen; + +BENCHMARK(ByLine_Pipes, iters) { + std::thread thread; + int rfd; + int wfd; + BENCHMARK_SUSPEND { + int p[2]; + CHECK_ERR(::pipe(p)); + rfd = p[0]; + wfd = p[1]; + thread = std::thread([wfd, iters] { + char x = 'x'; + PCHECK(::write(wfd, &x, 1) == 1); // signal startup + FILE* f = fdopen(wfd, "w"); + PCHECK(f); + for (int i = 1; i <= iters; ++i) { + fprintf(f, "%d\n", i); + } + fclose(f); + }); + char buf; + PCHECK(::read(rfd, &buf, 1) == 1); // wait for startup + } + + auto s = byLine(folly::File(rfd)) | eachTo() | sum; + folly::doNotOptimizeAway(s); + + BENCHMARK_SUSPEND { + ::close(rfd); + CHECK_EQ(s, int64_t(iters) * (iters + 1) / 2); + thread.join(); + } +} + +// Results from an Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz +// ============================================================================ +// folly/gen/test/FileBenchmark.cpp relative time/iter iters/s +// ============================================================================ +// ByLine_Pipes 148.63ns 6.73M +// ============================================================================ + +int main(int argc, char *argv[]) { + google::ParseCommandLineFlags(&argc, &argv, true); + folly::runBenchmarks(); + return 0; +} diff --git a/folly/gen/test/FileTest.cpp b/folly/gen/test/FileTest.cpp new file mode 100644 index 00000000..fe9b3540 --- /dev/null +++ b/folly/gen/test/FileTest.cpp @@ -0,0 +1,80 @@ +/* + * Copyright 2014 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#include "folly/File.h" +#include "folly/Range.h" +#include "folly/experimental/TestUtil.h" +#include "folly/gen/Base.h" +#include "folly/gen/File.h" + +using namespace folly::gen; +using namespace folly; +using std::string; +using std::vector; + +TEST(FileGen, ByLine) { + auto collect = eachTo() | as(); + test::TemporaryFile file("ByLine"); + static const std::string lines( + "Hello world\n" + "This is the second line\n" + "\n" + "\n" + "a few empty lines above\n" + "incomplete last line"); + EXPECT_EQ(lines.size(), write(file.fd(), lines.data(), lines.size())); + + auto expected = from({lines}) | resplit('\n') | collect; + auto found = byLine(file.path().c_str()) | collect; + + EXPECT_TRUE(expected == found); +} + +class FileGenBufferedTest : public ::testing::TestWithParam { }; + +TEST_P(FileGenBufferedTest, FileWriter) { + size_t bufferSize = GetParam(); + test::TemporaryFile file("FileWriter"); + + static const std::string lines( + "Hello world\n" + "This is the second line\n" + "\n" + "\n" + "a few empty lines above\n"); + + auto src = from({lines, lines, lines, lines, lines, lines, lines, lines}); + auto collect = eachTo() | as(); + auto expected = src | resplit('\n') | collect; + + src | eachAs() | toFile(File(file.fd()), bufferSize); + auto found = byLine(file.path().c_str()) | collect; + + EXPECT_TRUE(expected == found); +} + +INSTANTIATE_TEST_CASE_P( + DifferentBufferSizes, + FileGenBufferedTest, + ::testing::Values(0, 1, 2, 4, 8, 64, 4096)); +int main(int argc, char *argv[]) { + testing::InitGoogleTest(&argc, argv); + google::ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +} diff --git a/folly/gen/test/StringBenchmark.cpp b/folly/gen/test/StringBenchmark.cpp new file mode 100644 index 00000000..5c143ec4 --- /dev/null +++ b/folly/gen/test/StringBenchmark.cpp @@ -0,0 +1,329 @@ +/* + * Copyright 2014 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "folly/Benchmark.h" +#include "folly/String.h" +#include "folly/gen/Base.h" +#include "folly/gen/String.h" + +using namespace folly; +using namespace folly::gen; +using std::pair; +using std::set; +using std::vector; +using std::tuple; + +namespace { + +static std::atomic testSize(1000); +static vector testStrVector + = seq(1, testSize.load()) + | eachTo() + | as(); + +const char* const kLine = "The quick brown fox jumped over the lazy dog.\n"; +const size_t kLineCount = 10000; +std::string bigLines; +const size_t kSmallLineSize = 17; +std::vector smallLines; + +void initStringResplitterBenchmark() { + bigLines.reserve(kLineCount * strlen(kLine)); + for (size_t i = 0; i < kLineCount; ++i) { + bigLines += kLine; + } + size_t remaining = bigLines.size(); + size_t pos = 0; + while (remaining) { + size_t n = std::min(kSmallLineSize, remaining); + smallLines.push_back(bigLines.substr(pos, n)); + pos += n; + remaining -= n; + } +} + +size_t len(folly::StringPiece s) { return s.size(); } + +} // namespace + +BENCHMARK(StringResplitter_Big, iters) { + size_t s = 0; + while (iters--) { + s += from({bigLines}) | resplit('\n') | map(&len) | sum; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(StringResplitter_Small, iters) { + size_t s = 0; + while (iters--) { + s += from(smallLines) | resplit('\n') | map(&len) | sum; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_DRAW_LINE() + +BENCHMARK(StringSplit_Old, iters) { + size_t s = 0; + std::string line(kLine); + while (iters--) { + std::vector parts; + split(' ', line, parts); + s += parts.size(); + } + folly::doNotOptimizeAway(s); +} + + +BENCHMARK_RELATIVE(StringSplit_Gen_Vector, iters) { + size_t s = 0; + StringPiece line(kLine); + while (iters--) { + s += (split(line, ' ') | as()).size(); + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_DRAW_LINE() + +BENCHMARK(StringSplit_Old_ReuseVector, iters) { + size_t s = 0; + std::string line(kLine); + std::vector parts; + while (iters--) { + parts.clear(); + split(' ', line, parts); + s += parts.size(); + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(StringSplit_Gen_ReuseVector, iters) { + size_t s = 0; + StringPiece line(kLine); + std::vector parts; + while (iters--) { + parts.clear(); + split(line, ' ') | appendTo(parts); + s += parts.size(); + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(StringSplit_Gen, iters) { + size_t s = 0; + StringPiece line(kLine); + while (iters--) { + s += split(line, ' ') | count; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(StringSplit_Gen_Take, iters) { + size_t s = 0; + StringPiece line(kLine); + while (iters--) { + s += split(line, ' ') | take(10) | count; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_DRAW_LINE() + +BENCHMARK(StringUnsplit_Old, iters) { + size_t s = 0; + while (iters--) { + fbstring joined; + join(',', testStrVector, joined); + s += joined.size(); + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(StringUnsplit_Old_ReusedBuffer, iters) { + size_t s = 0; + fbstring joined; + while (iters--) { + joined.clear(); + join(',', testStrVector, joined); + s += joined.size(); + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(StringUnsplit_Gen, iters) { + size_t s = 0; + StringPiece line(kLine); + while (iters--) { + fbstring joined = from(testStrVector) | unsplit(','); + s += joined.size(); + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(StringUnsplit_Gen_ReusedBuffer, iters) { + size_t s = 0; + fbstring buffer; + while (iters--) { + buffer.clear(); + from(testStrVector) | unsplit(',', &buffer); + s += buffer.size(); + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_DRAW_LINE() + +void StringUnsplit_Gen(size_t iters, size_t joinSize) { + std::vector v; + BENCHMARK_SUSPEND { + FOR_EACH_RANGE(i, 0, joinSize) { + v.push_back(to(rand())); + } + } + size_t s = 0; + fbstring buffer; + while (iters--) { + buffer.clear(); + from(v) | unsplit(',', &buffer); + s += buffer.size(); + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_PARAM(StringUnsplit_Gen, 1000) +BENCHMARK_RELATIVE_PARAM(StringUnsplit_Gen, 2000) +BENCHMARK_RELATIVE_PARAM(StringUnsplit_Gen, 4000) +BENCHMARK_RELATIVE_PARAM(StringUnsplit_Gen, 8000) + +BENCHMARK_DRAW_LINE() + +fbstring records += seq(1, 1000) + | mapped([](size_t i) { + return folly::to(i, ' ', i * i, ' ', i * i * i); + }) + | unsplit('\n'); + +BENCHMARK(Records_EachToTuple, iters) { + size_t s = 0; + for (size_t i = 0; i < iters; i += 1000) { + s += split(records, '\n') + | eachToTuple(' ') + | get<1>() + | sum; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(Records_VectorStringPieceReused, iters) { + size_t s = 0; + std::vector fields; + for (size_t i = 0; i < iters; i += 1000) { + s += split(records, '\n') + | mapped([&](StringPiece line) { + fields.clear(); + folly::split(' ', line, fields); + CHECK(fields.size() == 3); + return std::make_tuple( + folly::to(fields[0]), + folly::to(fields[1]), + StringPiece(fields[2])); + }) + | get<1>() + | sum; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(Records_VectorStringPiece, iters) { + size_t s = 0; + for (size_t i = 0; i < iters; i += 1000) { + s += split(records, '\n') + | mapped([](StringPiece line) { + std::vector fields; + folly::split(' ', line, fields); + CHECK(fields.size() == 3); + return std::make_tuple( + folly::to(fields[0]), + folly::to(fields[1]), + StringPiece(fields[2])); + }) + | get<1>() + | sum; + } + folly::doNotOptimizeAway(s); +} + +BENCHMARK_RELATIVE(Records_VectorString, iters) { + size_t s = 0; + for (size_t i = 0; i < iters; i += 1000) { + s += split(records, '\n') + | mapped([](StringPiece line) { + std::vector fields; + folly::split(' ', line, fields); + CHECK(fields.size() == 3); + return std::make_tuple( + folly::to(fields[0]), + folly::to(fields[1]), + StringPiece(fields[2])); + }) + | get<1>() + | sum; + } + folly::doNotOptimizeAway(s); +} + +// Results from an Intel(R) Xeon(R) CPU E5-2660 0 @ 2.20GHz +// ============================================================================ +// folly/gen/test/StringBenchmark.cpp relative time/iter iters/s +// ============================================================================ +// StringResplitter_Big 108.58us 9.21K +// StringResplitter_Small 10.60% 1.02ms 976.48 +// ---------------------------------------------------------------------------- +// StringSplit_Old 357.82ns 2.79M +// StringSplit_Gen_Vector 105.10% 340.46ns 2.94M +// ---------------------------------------------------------------------------- +// StringSplit_Old_ReuseVector 96.45ns 10.37M +// StringSplit_Gen_ReuseVector 124.01% 77.78ns 12.86M +// StringSplit_Gen 140.10% 68.85ns 14.52M +// StringSplit_Gen_Take 122.97% 78.44ns 12.75M +// ---------------------------------------------------------------------------- +// StringUnsplit_Old 42.99us 23.26K +// StringUnsplit_Old_ReusedBuffer 100.48% 42.79us 23.37K +// StringUnsplit_Gen 96.37% 44.61us 22.42K +// StringUnsplit_Gen_ReusedBuffer 116.96% 36.76us 27.20K +// ---------------------------------------------------------------------------- +// StringUnsplit_Gen(1000) 44.71us 22.37K +// StringUnsplit_Gen(2000) 49.28% 90.72us 11.02K +// StringUnsplit_Gen(4000) 24.05% 185.91us 5.38K +// StringUnsplit_Gen(8000) 12.23% 365.42us 2.74K +// ---------------------------------------------------------------------------- +// Records_EachToTuple 101.43us 9.86K +// Records_VectorStringPieceReused 93.72% 108.22us 9.24K +// Records_VectorStringPiece 37.14% 273.11us 3.66K +// Records_VectorString 16.70% 607.47us 1.65K +// ============================================================================ + +int main(int argc, char *argv[]) { + google::ParseCommandLineFlags(&argc, &argv, true); + initStringResplitterBenchmark(); + runBenchmarks(); + return 0; +} diff --git a/folly/gen/test/StringTest.cpp b/folly/gen/test/StringTest.cpp new file mode 100644 index 00000000..17fb9c09 --- /dev/null +++ b/folly/gen/test/StringTest.cpp @@ -0,0 +1,286 @@ +/* + * Copyright 2014 Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "folly/gen/String.h" + +using namespace folly::gen; +using namespace folly; +using std::make_tuple; +using std::ostream; +using std::pair; +using std::string; +using std::tuple; +using std::unique_ptr; +using std::vector; + +TEST(StringGen, EmptySplit) { + auto collect = eachTo() | as(); + { + auto pieces = split("", ',') | collect; + EXPECT_EQ(0, pieces.size()); + } + + // The last delimiter is eaten, just like std::getline + { + auto pieces = split(",", ',') | collect; + EXPECT_EQ(1, pieces.size()); + EXPECT_EQ("", pieces[0]); + } + + { + auto pieces = split(",,", ',') | collect; + EXPECT_EQ(2, pieces.size()); + EXPECT_EQ("", pieces[0]); + EXPECT_EQ("", pieces[1]); + } + + { + auto pieces = split(",,", ',') | take(1) | collect; + EXPECT_EQ(1, pieces.size()); + EXPECT_EQ("", pieces[0]); + } +} + +TEST(StringGen, Split) { + auto collect = eachTo() | as(); + { + auto pieces = split("hello,, world, goodbye, meow", ',') | collect; + EXPECT_EQ(5, pieces.size()); + EXPECT_EQ("hello", pieces[0]); + EXPECT_EQ("", pieces[1]); + EXPECT_EQ(" world", pieces[2]); + EXPECT_EQ(" goodbye", pieces[3]); + EXPECT_EQ(" meow", pieces[4]); + } + + { + auto pieces = split("hello,, world, goodbye, meow", ',') + | take(3) | collect; + EXPECT_EQ(3, pieces.size()); + EXPECT_EQ("hello", pieces[0]); + EXPECT_EQ("", pieces[1]); + EXPECT_EQ(" world", pieces[2]); + } + + { + auto pieces = split("hello,, world, goodbye, meow", ',') + | take(5) | collect; + EXPECT_EQ(5, pieces.size()); + EXPECT_EQ("hello", pieces[0]); + EXPECT_EQ("", pieces[1]); + EXPECT_EQ(" world", pieces[2]); + } +} + +TEST(StringGen, EmptyResplit) { + auto collect = eachTo() | as(); + { + auto pieces = from({""}) | resplit(',') | collect; + EXPECT_EQ(0, pieces.size()); + } + + // The last delimiter is eaten, just like std::getline + { + auto pieces = from({","}) | resplit(',') | collect; + EXPECT_EQ(1, pieces.size()); + EXPECT_EQ("", pieces[0]); + } + + { + auto pieces = from({",,"}) | resplit(',') | collect; + EXPECT_EQ(2, pieces.size()); + EXPECT_EQ("", pieces[0]); + EXPECT_EQ("", pieces[1]); + } +} + +TEST(StringGen, EachToTuple) { + { + auto lines = "2:1.414:yo 3:1.732:hi"; + auto actual + = split(lines, ' ') + | eachToTuple(':') + | as(); + vector> expected { + make_tuple(2, 1.414, "yo"), + make_tuple(3, 1.732, "hi"), + }; + EXPECT_EQ(expected, actual); + } + { + auto lines = "2 3"; + auto actual + = split(lines, ' ') + | eachToTuple(',') + | as(); + vector> expected { + make_tuple(2), + make_tuple(3), + }; + EXPECT_EQ(expected, actual); + } + { + // StringPiece target + auto lines = "1:cat 2:dog"; + auto actual + = split(lines, ' ') + | eachToTuple(':') + | as(); + vector> expected { + make_tuple(1, "cat"), + make_tuple(2, "dog"), + }; + EXPECT_EQ(expected, actual); + } + { + // Empty field + auto lines = "2:tjackson:4 3::5"; + auto actual + = split(lines, ' ') + | eachToTuple(':') + | as(); + vector> expected { + make_tuple(2, "tjackson", 4), + make_tuple(3, "", 5), + }; + EXPECT_EQ(expected, actual); + } + { + // Excess fields + auto lines = "1:2 3:4:5"; + EXPECT_THROW((split(lines, ' ') + | eachToTuple(':') + | as()), + std::runtime_error); + } + { + // Missing fields + auto lines = "1:2:3 4:5"; + EXPECT_THROW((split(lines, ' ') + | eachToTuple(':') + | as()), + std::runtime_error); + } +} + +TEST(StringGen, EachToPair) { + { + // char delimiters + auto lines = "2:1.414 3:1.732"; + auto actual + = split(lines, ' ') + | eachToPair(':') + | as>(); + std::map expected { + { 3, 1.732 }, + { 2, 1.414 }, + }; + EXPECT_EQ(expected, actual); + } + { + // string delimiters + auto lines = "ab=>cd ef=>gh"; + auto actual + = split(lines, ' ') + | eachToPair("=>") + | as>(); + std::map expected { + { "ab", "cd" }, + { "ef", "gh" }, + }; + EXPECT_EQ(expected, actual); + } +} + +TEST(StringGen, Resplit) { + auto collect = eachTo() | as(); + { + auto pieces = from({"hello,, world, goodbye, meow"}) | + resplit(',') | collect; + EXPECT_EQ(5, pieces.size()); + EXPECT_EQ("hello", pieces[0]); + EXPECT_EQ("", pieces[1]); + EXPECT_EQ(" world", pieces[2]); + EXPECT_EQ(" goodbye", pieces[3]); + EXPECT_EQ(" meow", pieces[4]); + } + { + auto pieces = from({"hel", "lo,", ", world", ", goodbye, m", "eow"}) | + resplit(',') | collect; + EXPECT_EQ(5, pieces.size()); + EXPECT_EQ("hello", pieces[0]); + EXPECT_EQ("", pieces[1]); + EXPECT_EQ(" world", pieces[2]); + EXPECT_EQ(" goodbye", pieces[3]); + EXPECT_EQ(" meow", pieces[4]); + } +} + +template +void runUnsplitSuite(F fn) { + fn("hello, world"); + fn("hello,world,goodbye"); + fn(" "); + fn(""); + fn(", "); + fn(", a, b,c"); +} + +TEST(StringGen, Unsplit) { + + auto basicFn = [](StringPiece s) { + EXPECT_EQ(split(s, ',') | unsplit(','), s); + }; + + auto existingBuffer = [](StringPiece s) { + folly::fbstring buffer("asdf"); + split(s, ',') | unsplit(',', &buffer); + auto expected = folly::to( + "asdf", s.empty() ? "" : ",", s); + EXPECT_EQ(expected, buffer); + }; + + auto emptyBuffer = [](StringPiece s) { + std::string buffer; + split(s, ',') | unsplit(',', &buffer); + EXPECT_EQ(s, buffer); + }; + + auto stringDelim = [](StringPiece s) { + EXPECT_EQ(s, split(s, ',') | unsplit(",")); + std::string buffer; + split(s, ',') | unsplit(",", &buffer); + EXPECT_EQ(buffer, s); + }; + + runUnsplitSuite(basicFn); + runUnsplitSuite(existingBuffer); + runUnsplitSuite(emptyBuffer); + runUnsplitSuite(stringDelim); + EXPECT_EQ("1, 2, 3", seq(1, 3) | unsplit(", ")); +} + +int main(int argc, char *argv[]) { + testing::InitGoogleTest(&argc, argv); + google::ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +}