HACK: New Gen operators: zip, interleave
authorMike Curtiss <mcurtiss@fb.com>
Sat, 2 Mar 2013 06:27:56 +0000 (22:27 -0800)
committerOwen Yamauchi <oyamauchi@fb.com>
Mon, 3 Jun 2013 19:23:28 +0000 (12:23 -0700)
Summary:
Zip: inspired by python's zip()
o Combine a generator with the contents of a container to form
a tuple.  Note that we combine with a container (and not
another generator) because of a fundamental constraint
in how control-flow in Generators works.  Containers give us 90%
of the utility without all the hassle.  We could theoretically
also add a version of zip where the extra source is generated
concurrently in another thread.

Interleave: similar to zip, but inspired by Clojure's interleave()
o Instead of creating a tuple like zip, just flatten the values.

Added some tuple creation/concatenation functions.  These are mostly
meant as a way to enable zip'ing multiple containers together into an
N-tuple. (My variadic-fu was not strong enough to get this working
within a single Zip function).

Test Plan: Added unit-tests

Reviewed By: tjackson@fb.com

FB internal diff: D740518

folly/experimental/CombineGen-inl.h [new file with mode: 0644]
folly/experimental/CombineGen.h [new file with mode: 0644]
folly/experimental/Gen-inl.h
folly/experimental/test/GenTest.cpp

diff --git a/folly/experimental/CombineGen-inl.h b/folly/experimental/CombineGen-inl.h
new file mode 100644 (file)
index 0000000..bb3ba6b
--- /dev/null
@@ -0,0 +1,217 @@
+/*
+ * Copyright 2013 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOLLY_COMBINEGEN_H_
+#error This file may only be included from folly/experimental/CombineGen.h
+#endif
+
+#include <iterator>
+#include <system_error>
+#include <tuple>
+#include <type_traits>
+
+namespace folly {
+namespace gen {
+namespace detail {
+
+/**
+ * Interleave
+ *
+ * Alternate values from a sequence with values from a sequence container.
+ * Stops once we run out of values from either source.
+ */
+template<class Container>
+class Interleave : public Operator<Interleave<Container>> {
+  // see comment about copies in CopiedSource
+  const std::shared_ptr<const Container> container_;
+ public:
+  explicit Interleave(Container container)
+    : container_(new Container(std::move(container))) {}
+
+  template<class Value,
+           class Source>
+  class Generator : public GenImpl<Value, Generator<Value, Source>> {
+    Source source_;
+    const std::shared_ptr<const Container> container_;
+    typedef const typename Container::value_type& ConstRefType;
+
+    static_assert(std::is_same<const Value&, ConstRefType>::value,
+                  "Only matching types may be interleaved");
+  public:
+    explicit Generator(Source source,
+                       const std::shared_ptr<const Container> container)
+      : source_(std::move(source)),
+        container_(container) { }
+
+    template<class Handler>
+    bool apply(Handler&& handler) const {
+      auto iter = container_->begin();
+      return source_.apply([&](const Value& value) -> bool {
+            if (iter == container_->end()) {
+              return false;
+            }
+            if (!handler(value)) {
+              return false;
+            }
+            if (!handler(*iter)) {
+              return false;
+            }
+            iter++;
+            return true;
+        });
+    }
+  };
+
+  template<class Value2,
+           class Source,
+           class Gen = Generator<Value2,Source>>
+  Gen compose(GenImpl<Value2, Source>&& source) const {
+    return Gen(std::move(source.self()), container_);
+  }
+
+  template<class Value2,
+           class Source,
+           class Gen = Generator<Value2,Source>>
+  Gen compose(const GenImpl<Value2, Source>& source) const {
+    return Gen(source.self(), container_);
+  }
+};
+
+/**
+ * Zip
+ *
+ * Combine inputs from Source with values from a sequence container by merging
+ * them into a tuple.
+ *
+ */
+template<class Container>
+class Zip : public Operator<Zip<Container>> {
+  // see comment about copies in CopiedSource
+  const std::shared_ptr<const Container> container_;
+ public:
+  explicit Zip(Container container)
+    : container_(new Container(std::move(container))) {}
+
+  template<class Value1,
+           class Source,
+           class Value2 = decltype(*std::begin(*container_)),
+           class Result = std::tuple<typename std::decay<Value1>::type,
+                                     typename std::decay<Value2>::type>>
+  class Generator : public GenImpl<Result,
+                                   Generator<Value1,Source,Value2,Result>> {
+    Source source_;
+    const std::shared_ptr<const Container> container_;
+  public:
+    explicit Generator(Source source,
+                       const std::shared_ptr<const Container> container)
+      : source_(std::move(source)),
+        container_(container) { }
+
+    template<class Handler>
+    bool apply(Handler&& handler) const {
+      auto iter = container_->begin();
+      return (source_.apply([&](Value1 value) -> bool {
+            if (iter == container_->end()) {
+              return false;
+            }
+            if (!handler(std::make_tuple(std::forward<Value1>(value), *iter))) {
+              return false;
+            }
+            ++iter;
+            return true;
+          }));
+    }
+  };
+
+  template<class Source,
+           class Value,
+           class Gen = Generator<Value, Source>>
+  Gen compose(GenImpl<Value, Source>&& source) const {
+    return Gen(std::move(source.self()), container_);
+  }
+
+  template<class Source,
+           class Value,
+           class Gen = Generator<Value, Source>>
+  Gen compose(const GenImpl<Value, Source>& source) const {
+    return Gen(source.self(), container_);
+  }
+};
+
+template<class... Types1,
+         class... Types2>
+auto add_to_tuple(std::tuple<Types1...> t1, std::tuple<Types2...> t2) ->
+std::tuple<Types1..., Types2...> {
+  return std::tuple_cat(std::move(t1), std::move(t2));
+}
+
+template<class... Types1,
+         class Type2>
+auto add_to_tuple(std::tuple<Types1...> t1, Type2&& t2) ->
+decltype(std::tuple_cat(std::move(t1),
+                        std::make_tuple(std::forward<Type2>(t2)))) {
+  return std::tuple_cat(std::move(t1),
+                        std::make_tuple(std::forward<Type2>(t2)));
+}
+
+template<class Type1,
+         class... Types2>
+auto add_to_tuple(Type1&& t1, std::tuple<Types2...> t2) ->
+decltype(std::tuple_cat(std::make_tuple(std::forward<Type1>(t1)),
+                        std::move(t2))) {
+  return std::tuple_cat(std::make_tuple(std::forward<Type1>(t1)),
+                        std::move(t2));
+}
+
+template<class Type1,
+         class Type2>
+auto add_to_tuple(Type1&& t1, Type2&& t2) ->
+decltype(std::make_tuple(std::forward<Type1>(t1),
+                         std::forward<Type2>(t2))) {
+  return std::make_tuple(std::forward<Type1>(t1),
+                         std::forward<Type2>(t2));
+}
+
+// Merges a 2-tuple into a single tuple (get<0> could already be a tuple)
+class MergeTuples {
+ public:
+  template<class Tuple>
+  auto operator()(Tuple&& value) const ->
+  decltype(add_to_tuple(std::get<0>(std::forward<Tuple>(value)),
+                        std::get<1>(std::forward<Tuple>(value)))) {
+    static_assert(std::tuple_size<
+                    typename std::remove_reference<Tuple>::type
+                    >::value == 2,
+                  "Can only merge tuples of size 2");
+    return add_to_tuple(std::get<0>(std::forward<Tuple>(value)),
+                        std::get<1>(std::forward<Tuple>(value)));
+  }
+};
+
+}  // namespace detail
+
+static const detail::Map<detail::MergeTuples> tuple_flatten;
+
+// TODO(mcurtiss): support zip() for N>1 operands. Because of variadic problems,
+// this might not be easily possible until gcc4.8 is available.
+template<class Source,
+         class Zip = detail::Zip<typename std::decay<Source>::type>>
+Zip zip(Source&& source) {
+  return Zip(std::forward<Source>(source));
+}
+
+}  // namespace gen
+}  // namespace folly
diff --git a/folly/experimental/CombineGen.h b/folly/experimental/CombineGen.h
new file mode 100644 (file)
index 0000000..89585be
--- /dev/null
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2013 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FOLLY_COMBINEGEN_H_
+#define FOLLY_COMBINEGEN_H_
+
+#include "folly/experimental/Gen.h"
+
+namespace folly {
+namespace gen {
+namespace detail {
+
+template<class Container>
+class Interleave;
+
+template<class Container>
+class Zip;
+
+}  // namespace detail
+
+template<class Source2,
+         class Source2Decayed = typename std::decay<Source2>::type,
+         class Interleave = detail::Interleave<Source2Decayed>>
+Interleave interleave(Source2&& source2) {
+  return Interleave(std::forward<Source2>(source2));
+}
+
+}  // namespace gen
+}  // namespace folly
+
+#include "folly/experimental/CombineGen-inl.h"
+
+#endif /* FOLLY_COMBINEGEN_H_ */
+
index f916878c48b8861a6c59f7623a0b4c2f28ddf2df..39d63fced4e1da1745de5073a133bb1128ae549c 100644 (file)
@@ -1828,4 +1828,4 @@ inline detail::Skip skip(size_t count) {
   return detail::Skip(count);
 }
 
-}} //folly::gen::detail
+}} //folly::gen
index 8c7b826f7fe755e6c5b7ce152bfebb710752fa86..d732e97e46f6f7d884d4821c1830b9fdc60fce83 100644 (file)
 #include <vector>
 #include "folly/experimental/Gen.h"
 #include "folly/experimental/StringGen.h"
+#include "folly/experimental/CombineGen.h"
 #include "folly/experimental/FileGen.h"
 #include "folly/experimental/TestUtil.h"
+#include "folly/FBString.h"
 #include "folly/FBVector.h"
 #include "folly/Format.h"
 #include "folly/dynamic.h"
@@ -38,7 +40,6 @@ using std::vector;
 using std::string;
 using std::tuple;
 using std::make_tuple;
-//using std::unordered_map;
 
 #define EXPECT_SAME(A, B) \
   static_assert(std::is_same<A, B>::value, "Mismatched: " #A ", " #B)
@@ -300,6 +301,136 @@ TEST(Gen, Until) {
   EXPECT_EQ(31, gen | count);
 }
 
+auto even = [](int i) -> bool { return i % 2 == 0; };
+auto odd = [](int i) -> bool { return i % 2 == 1; };
+
+TEST(CombineGen, Interleave) {
+  { // large (infinite) base, small container
+    auto base = seq(1) | filter(odd);
+    auto toInterleave = seq(1, 6) | filter(even);
+    auto interleaved = base | interleave(toInterleave | as<vector>());
+    EXPECT_EQ(interleaved | as<vector>(), vector<int>({1, 2, 3, 4, 5, 6}));
+  }
+  { // small base, large container
+    auto base = seq(1) | filter(odd) | take(3);
+    auto toInterleave = seq(1) | filter(even) | take(50);
+    auto interleaved = base | interleave(toInterleave | as<vector>());
+    EXPECT_EQ(interleaved | as<vector>(),
+              vector<int>({1, 2, 3, 4, 5, 6}));
+  }
+}
+
+TEST(CombineGen, Zip) {
+  auto base0 = seq(1);
+  // We rely on std::move(fbvector) emptying the source vector
+  auto zippee = fbvector<string>{"one", "two", "three"};
+  {
+    auto combined = base0
+      | zip(zippee)
+      | as<vector>();
+    ASSERT_EQ(combined.size(), 3);
+    EXPECT_EQ(std::get<0>(combined[0]), 1);
+    EXPECT_EQ(std::get<1>(combined[0]), "one");
+    EXPECT_EQ(std::get<0>(combined[1]), 2);
+    EXPECT_EQ(std::get<1>(combined[1]), "two");
+    EXPECT_EQ(std::get<0>(combined[2]), 3);
+    EXPECT_EQ(std::get<1>(combined[2]), "three");
+    ASSERT_FALSE(zippee.empty());
+    EXPECT_FALSE(zippee.front().empty());  // shouldn't have been move'd
+  }
+
+  { // same as top, but using std::move.
+    auto combined = base0
+      | zip(std::move(zippee))
+      | as<vector>();
+    ASSERT_EQ(combined.size(), 3);
+    EXPECT_EQ(std::get<0>(combined[0]), 1);
+    EXPECT_TRUE(zippee.empty());
+  }
+
+  { // same as top, but base is truncated
+    auto baseFinite = seq(1) | take(1);
+    auto combined = baseFinite
+      | zip(vector<string>{"one", "two", "three"})
+      | as<vector>();
+    ASSERT_EQ(combined.size(), 1);
+    EXPECT_EQ(std::get<0>(combined[0]), 1);
+    EXPECT_EQ(std::get<1>(combined[0]), "one");
+  }
+}
+
+TEST(CombineGen, TupleFlatten) {
+  vector<tuple<int,string>> intStringTupleVec{
+    tuple<int,string>{1, "1"},
+    tuple<int,string>{2, "2"},
+    tuple<int,string>{3, "3"},
+  };
+
+  vector<tuple<char>> charTupleVec{
+    tuple<char>{'A'},
+    tuple<char>{'B'},
+    tuple<char>{'C'},
+    tuple<char>{'D'},
+  };
+
+  vector<double> doubleVec{
+    1.0,
+    4.0,
+    9.0,
+    16.0,
+    25.0,
+  };
+
+  auto zipped1 = from(intStringTupleVec)
+    | zip(charTupleVec)
+    | assert_type<tuple<tuple<int, string>, tuple<char>>>()
+    | as<vector>();
+  EXPECT_EQ(std::get<0>(zipped1[0]), std::make_tuple(1, "1"));
+  EXPECT_EQ(std::get<1>(zipped1[0]), std::make_tuple('A'));
+
+  auto zipped2 = from(zipped1)
+    | tuple_flatten
+    | assert_type<tuple<int, string, char>&&>()
+    | as<vector>();
+  ASSERT_EQ(zipped2.size(), 3);
+  EXPECT_EQ(zipped2[0], std::make_tuple(1, "1", 'A'));
+
+  auto zipped3 = from(charTupleVec)
+    | zip(intStringTupleVec)
+    | tuple_flatten
+    | assert_type<tuple<char, int, string>&&>()
+    | as<vector>();
+  ASSERT_EQ(zipped3.size(), 3);
+  EXPECT_EQ(zipped3[0], std::make_tuple('A', 1, "1"));
+
+  auto zipped4 = from(intStringTupleVec)
+    | zip(doubleVec)
+    | tuple_flatten
+    | assert_type<tuple<int, string, double>&&>()
+    | as<vector>();
+  ASSERT_EQ(zipped4.size(), 3);
+  EXPECT_EQ(zipped4[0], std::make_tuple(1, "1", 1.0));
+
+  auto zipped5 = from(doubleVec)
+    | zip(doubleVec)
+    | assert_type<tuple<double, double>>()
+    | tuple_flatten  // essentially a no-op
+    | assert_type<tuple<double, double>&&>()
+    | as<vector>();
+  ASSERT_EQ(zipped5.size(), 5);
+  EXPECT_EQ(zipped5[0], std::make_tuple(1.0, 1.0));
+
+  auto zipped6 = from(intStringTupleVec)
+    | zip(charTupleVec)
+    | tuple_flatten
+    | zip(doubleVec)
+    | tuple_flatten
+    | assert_type<tuple<int, string, char, double>&&>()
+    | as<vector>();
+  ASSERT_EQ(zipped6.size(), 3);
+  EXPECT_EQ(zipped6[0], std::make_tuple(1, "1", 'A', 1.0));
+}
+
 TEST(Gen, Composed) {
   // Operator, Operator
   auto valuesOf =