2 * Copyright 2015 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include <glog/logging.h>
18 #include <gtest/gtest.h>
23 #include <folly/gen/String.h>
25 using namespace folly::gen;
26 using namespace folly;
27 using std::make_tuple;
32 using std::unique_ptr;
35 TEST(StringGen, EmptySplit) {
36 auto collect = eachTo<std::string>() | as<vector>();
38 auto pieces = split("", ',') | collect;
39 EXPECT_EQ(0, pieces.size());
42 // The last delimiter is eaten, just like std::getline
44 auto pieces = split(",", ',') | collect;
45 EXPECT_EQ(1, pieces.size());
46 EXPECT_EQ("", pieces[0]);
50 auto pieces = split(",,", ',') | collect;
51 EXPECT_EQ(2, pieces.size());
52 EXPECT_EQ("", pieces[0]);
53 EXPECT_EQ("", pieces[1]);
57 auto pieces = split(",,", ',') | take(1) | collect;
58 EXPECT_EQ(1, pieces.size());
59 EXPECT_EQ("", pieces[0]);
63 TEST(StringGen, Split) {
64 auto collect = eachTo<std::string>() | as<vector>();
66 auto pieces = split("hello,, world, goodbye, meow", ',') | collect;
67 EXPECT_EQ(5, pieces.size());
68 EXPECT_EQ("hello", pieces[0]);
69 EXPECT_EQ("", pieces[1]);
70 EXPECT_EQ(" world", pieces[2]);
71 EXPECT_EQ(" goodbye", pieces[3]);
72 EXPECT_EQ(" meow", pieces[4]);
76 auto pieces = split("hello,, world, goodbye, meow", ',')
78 EXPECT_EQ(3, pieces.size());
79 EXPECT_EQ("hello", pieces[0]);
80 EXPECT_EQ("", pieces[1]);
81 EXPECT_EQ(" world", pieces[2]);
85 auto pieces = split("hello,, world, goodbye, meow", ",")
87 EXPECT_EQ(5, pieces.size());
88 EXPECT_EQ("hello", pieces[0]);
89 EXPECT_EQ("", pieces[1]);
90 EXPECT_EQ(" world", pieces[2]);
94 auto pieces = split("hello,, world, goodbye, meow", ", ")
96 EXPECT_EQ(4, pieces.size());
97 EXPECT_EQ("hello,", pieces[0]);
98 EXPECT_EQ("world", pieces[1]);
99 EXPECT_EQ("goodbye", pieces[2]);
100 EXPECT_EQ("meow", pieces[3]);
104 TEST(StringGen, SplitByNewLine) {
105 auto collect = eachTo<std::string>() | as<vector>();
107 auto pieces = lines("hello\n\n world\r\n goodbye\r me\n\row") | collect;
108 EXPECT_EQ(7, pieces.size());
109 EXPECT_EQ("hello", pieces[0]);
110 EXPECT_EQ("", pieces[1]);
111 EXPECT_EQ(" world", pieces[2]);
112 EXPECT_EQ(" goodbye", pieces[3]);
113 EXPECT_EQ(" me", pieces[4]);
114 EXPECT_EQ("", pieces[5]);
115 EXPECT_EQ("ow", pieces[6]);
119 TEST(StringGen, EmptyResplit) {
120 auto collect = eachTo<std::string>() | as<vector>();
122 auto pieces = from({""}) | resplit(',') | collect;
123 EXPECT_EQ(0, pieces.size());
126 // The last delimiter is eaten, just like std::getline
128 auto pieces = from({","}) | resplit(',') | collect;
129 EXPECT_EQ(1, pieces.size());
130 EXPECT_EQ("", pieces[0]);
134 auto pieces = from({",,"}) | resplit(',') | collect;
135 EXPECT_EQ(2, pieces.size());
136 EXPECT_EQ("", pieces[0]);
137 EXPECT_EQ("", pieces[1]);
141 TEST(StringGen, EachToTuple) {
143 auto lines = "2:1.414:yo 3:1.732:hi";
146 | eachToTuple<int, double, std::string>(':')
148 vector<tuple<int, double, std::string>> expected {
149 make_tuple(2, 1.414, "yo"),
150 make_tuple(3, 1.732, "hi"),
152 EXPECT_EQ(expected, actual);
158 | eachToTuple<int>(',')
160 vector<tuple<int>> expected {
164 EXPECT_EQ(expected, actual);
167 // StringPiece target
168 auto lines = "1:cat 2:dog";
171 | eachToTuple<int, StringPiece>(':')
173 vector<tuple<int, StringPiece>> expected {
174 make_tuple(1, "cat"),
175 make_tuple(2, "dog"),
177 EXPECT_EQ(expected, actual);
181 auto lines = "2:tjackson:4 3::5";
184 | eachToTuple<int, fbstring, int>(':')
186 vector<tuple<int, fbstring, int>> expected {
187 make_tuple(2, "tjackson", 4),
188 make_tuple(3, "", 5),
190 EXPECT_EQ(expected, actual);
194 auto lines = "1:2 3:4:5";
195 EXPECT_THROW((split(lines, ' ')
196 | eachToTuple<int, int>(':')
202 auto lines = "1:2:3 4:5";
203 EXPECT_THROW((split(lines, ' ')
204 | eachToTuple<int, int, int>(':')
210 TEST(StringGen, EachToPair) {
213 auto lines = "2:1.414 3:1.732";
216 | eachToPair<int, double>(':')
217 | as<std::map<int, double>>();
218 std::map<int, double> expected {
222 EXPECT_EQ(expected, actual);
226 auto lines = "ab=>cd ef=>gh";
229 | eachToPair<string, string>("=>")
230 | as<std::map<string, string>>();
231 std::map<string, string> expected {
235 EXPECT_EQ(expected, actual);
239 TEST(StringGen, Resplit) {
240 auto collect = eachTo<std::string>() | as<vector>();
242 auto pieces = from({"hello,, world, goodbye, meow"}) |
243 resplit(',') | collect;
244 EXPECT_EQ(5, pieces.size());
245 EXPECT_EQ("hello", pieces[0]);
246 EXPECT_EQ("", pieces[1]);
247 EXPECT_EQ(" world", pieces[2]);
248 EXPECT_EQ(" goodbye", pieces[3]);
249 EXPECT_EQ(" meow", pieces[4]);
252 auto pieces = from({"hel", "lo,", ", world", ", goodbye, m", "eow"}) |
253 resplit(',') | collect;
254 EXPECT_EQ(5, pieces.size());
255 EXPECT_EQ("hello", pieces[0]);
256 EXPECT_EQ("", pieces[1]);
257 EXPECT_EQ(" world", pieces[2]);
258 EXPECT_EQ(" goodbye", pieces[3]);
259 EXPECT_EQ(" meow", pieces[4]);
263 void checkResplitMaxLength(vector<string> ins,
266 vector<string> outs) {
267 vector<std::string> pieces;
268 auto splitter = streamSplitter(delim, [&pieces](StringPiece s) {
269 pieces.push_back(string(s.begin(), s.end()));
272 for (const auto& in : ins) {
277 EXPECT_EQ(outs.size(), pieces.size());
278 for (size_t i = 0; i < outs.size(); ++i) {
279 EXPECT_EQ(outs[i], pieces[i]);
282 // Also check the concatenated input against the same output
283 if (ins.size() > 1) {
284 checkResplitMaxLength({folly::join("", ins)}, delim, maxLength, outs);
288 TEST(StringGen, ResplitMaxLength) {
289 checkResplitMaxLength(
290 {"hel", "lo,", ", world", ", goodbye, m", "ew"}, ',', 5,
291 {"hello", ",", ",", " worl", "d,", " good", "bye,", " mew"}
293 // " meow" cannot be "end of stream", since it's maxLength long
294 checkResplitMaxLength(
295 {"hel", "lo,", ", world", ", goodbye, m", "eow"}, ',', 5,
296 {"hello", ",", ",", " worl", "d,", " good", "bye,", " meow", ""}
298 checkResplitMaxLength(
299 {"||", "", "", "", "|a|b", "cdefghijklmn", "|opqrst",
300 "uvwx|y|||", "z", "0123456789", "|", ""}, '|', 2,
301 {"|", "|", "|", "a|", "bc", "de", "fg", "hi", "jk", "lm", "n|", "op", "qr",
302 "st", "uv", "wx", "|", "y|", "|", "|", "z0", "12", "34", "56", "78", "9|",
308 void runUnsplitSuite(F fn) {
310 fn("hello,world,goodbye");
317 TEST(StringGen, Unsplit) {
319 auto basicFn = [](StringPiece s) {
320 EXPECT_EQ(split(s, ',') | unsplit(','), s);
323 auto existingBuffer = [](StringPiece s) {
324 folly::fbstring buffer("asdf");
325 split(s, ',') | unsplit(',', &buffer);
326 auto expected = folly::to<folly::fbstring>(
327 "asdf", s.empty() ? "" : ",", s);
328 EXPECT_EQ(expected, buffer);
331 auto emptyBuffer = [](StringPiece s) {
333 split(s, ',') | unsplit(',', &buffer);
334 EXPECT_EQ(s, buffer);
337 auto stringDelim = [](StringPiece s) {
338 EXPECT_EQ(s, split(s, ',') | unsplit(","));
340 split(s, ',') | unsplit(",", &buffer);
341 EXPECT_EQ(buffer, s);
344 runUnsplitSuite(basicFn);
345 runUnsplitSuite(existingBuffer);
346 runUnsplitSuite(emptyBuffer);
347 runUnsplitSuite(stringDelim);
348 EXPECT_EQ("1, 2, 3", seq(1, 3) | unsplit(", "));
351 TEST(StringGen, Batch) {
352 std::vector<std::string> chunks{
353 "on", "e\nt", "w", "o", "\nthr", "ee\nfo", "ur\n",
355 std::vector<std::string> lines{
356 "one", "two", "three", "four",
358 EXPECT_EQ(4, from(chunks) | resplit('\n') | count);
359 EXPECT_EQ(4, from(chunks) | resplit('\n') | batch(2) | rconcat | count);
360 EXPECT_EQ(4, from(chunks) | resplit('\n') | batch(3) | rconcat | count);
361 EXPECT_EQ(lines, from(chunks) | resplit('\n') | eachTo<std::string>() |
362 batch(3) | rconcat | as<vector>());
365 int main(int argc, char *argv[]) {
366 testing::InitGoogleTest(&argc, argv);
367 gflags::ParseCommandLineFlags(&argc, &argv, true);
368 return RUN_ALL_TESTS();