2 * Copyright 2012 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "folly/json.h"
18 #include <gtest/gtest.h>
19 #include <gflags/gflags.h>
23 #include <boost/next_prior.hpp>
24 #include "folly/Benchmark.h"
27 using folly::parseJson;
31 auto val = parseJson("\"I \u2665 UTF-8\"");
32 EXPECT_EQ("I \u2665 UTF-8", val.asString());
33 val = parseJson("\"I \\u2665 UTF-8\"");
34 EXPECT_EQ("I \u2665 UTF-8", val.asString());
35 val = parseJson("\"I \U0001D11E playing in G-clef\"");
36 EXPECT_EQ("I \U0001D11E playing in G-clef", val.asString());
38 val = parseJson("\"I \\uD834\\uDD1E playing in G-clef\"");
39 EXPECT_EQ("I \U0001D11E playing in G-clef", val.asString());
43 auto num = parseJson("12");
44 EXPECT_TRUE(num.isInt());
46 num = parseJson("12e5");
47 EXPECT_TRUE(num.isDouble());
49 auto numAs1 = num.asDouble();
50 EXPECT_EQ(numAs1, 12e5);
52 EXPECT_EQ(num, 1200000);
54 auto largeNumber = parseJson("4611686018427387904");
55 EXPECT_TRUE(largeNumber.isInt());
56 EXPECT_EQ(largeNumber, 4611686018427387904L);
58 auto negative = parseJson("-123");
59 EXPECT_EQ(negative, -123);
61 auto bfalse = parseJson("false");
62 auto btrue = parseJson("true");
63 EXPECT_EQ(bfalse, false);
64 EXPECT_EQ(btrue, true);
66 auto null = parseJson("null");
67 EXPECT_TRUE(null == nullptr);
69 auto doub1 = parseJson("12.0");
70 auto doub2 = parseJson("12e2");
71 EXPECT_EQ(doub1, 12.0);
72 EXPECT_EQ(doub2, 12e2);
73 EXPECT_EQ(std::numeric_limits<double>::infinity(),
74 parseJson("Infinity").asDouble());
75 EXPECT_EQ(-std::numeric_limits<double>::infinity(),
76 parseJson("-Infinity").asDouble());
77 EXPECT_TRUE(std::isnan(parseJson("NaN").asDouble()));
78 EXPECT_THROW(parseJson("infinity"), std::runtime_error);
79 EXPECT_THROW(parseJson("inf"), std::runtime_error);
80 EXPECT_THROW(parseJson("nan"), std::runtime_error);
82 auto array = parseJson(
83 "[12,false, false , null , [12e4,32, [], 12]]");
84 EXPECT_EQ(array.size(), 5);
85 if (array.size() == 5) {
86 EXPECT_EQ(boost::prior(array.end())->size(), 4);
91 parseJson("\n[12,\n\nnotvalidjson");
92 } catch (const std::exception& e) {
100 } catch (const std::exception& e) {
107 parseJson("{\"foo\":12,\"bar\":42} \"something\"");
108 } catch (const std::exception& e) {
114 dynamic anotherVal = dynamic::object
120 dynamic::object("a", "b")
130 // Print then parse and get the same thing, hopefully.
131 auto value = parseJson(toJson(anotherVal));
132 EXPECT_EQ(value, anotherVal);
134 // Test an object with non-string values.
135 dynamic something = folly::parseJson(
136 "{\"old_value\":40,\"changed\":true,\"opened\":false}");
137 dynamic expected = dynamic::object
141 EXPECT_EQ(something, expected);
144 TEST(Json, JavascriptSafe) {
145 auto badDouble = (1ll << 63ll) + 1;
146 dynamic badDyn = badDouble;
147 EXPECT_EQ(folly::toJson(badDouble), folly::to<folly::fbstring>(badDouble));
148 folly::json::serialization_opts opts;
149 opts.javascript_safe = true;
150 EXPECT_ANY_THROW(folly::json::serialize(badDouble, opts));
152 auto okDouble = 1ll << 63ll;
153 dynamic okDyn = okDouble;
154 EXPECT_EQ(folly::toJson(okDouble), folly::to<folly::fbstring>(okDouble));
157 TEST(Json, Produce) {
158 auto value = parseJson(R"( "f\"oo" )");
159 EXPECT_EQ(toJson(value), R"("f\"oo")");
160 value = parseJson("\"Control code: \001 \002 \x1f\"");
161 EXPECT_EQ(toJson(value), R"("Control code: \u0001 \u0002 \u001f")");
165 dynamic d = dynamic::object;
168 auto str = toJson(d);
169 } catch (std::exception const& e) {
170 // We're not allowed to have non-string keys in json.
176 TEST(Json, JsonEscape) {
177 folly::json::serialization_opts opts;
179 folly::json::serialize("\b\f\n\r\x01\t\\\"/\v\a", opts),
180 R"("\b\f\n\r\u0001\t\\\"/\u000b\u0007")");
183 TEST(Json, JsonNonAsciiEncoding) {
184 folly::json::serialization_opts opts;
185 opts.encode_non_ascii = true;
188 EXPECT_EQ(folly::json::serialize("\x1f", opts), R"("\u001f")");
189 EXPECT_EQ(folly::json::serialize("\xc2\xa2", opts), R"("\u00a2")");
190 EXPECT_EQ(folly::json::serialize("\xe2\x82\xac", opts), R"("\u20ac")");
192 // multiple unicode encodings
194 folly::json::serialize("\x1f\xe2\x82\xac", opts),
195 R"("\u001f\u20ac")");
197 folly::json::serialize("\x1f\xc2\xa2\xe2\x82\xac", opts),
198 R"("\u001f\u00a2\u20ac")");
200 folly::json::serialize("\xc2\x80\xef\xbf\xbf", opts),
201 R"("\u0080\uffff")");
203 folly::json::serialize("\xe0\xa0\x80\xdf\xbf", opts),
204 R"("\u0800\u07ff")");
206 // first possible sequence of a certain length
207 EXPECT_EQ(folly::json::serialize("\xc2\x80", opts), R"("\u0080")");
208 EXPECT_EQ(folly::json::serialize("\xe0\xa0\x80", opts), R"("\u0800")");
210 // last possible sequence of a certain length
211 EXPECT_EQ(folly::json::serialize("\xdf\xbf", opts), R"("\u07ff")");
212 EXPECT_EQ(folly::json::serialize("\xef\xbf\xbf", opts), R"("\uffff")");
214 // other boundary conditions
215 EXPECT_EQ(folly::json::serialize("\xed\x9f\xbf", opts), R"("\ud7ff")");
216 EXPECT_EQ(folly::json::serialize("\xee\x80\x80", opts), R"("\ue000")");
217 EXPECT_EQ(folly::json::serialize("\xef\xbf\xbd", opts), R"("\ufffd")");
219 // incomplete sequences
220 EXPECT_ANY_THROW(folly::json::serialize("a\xed\x9f", opts));
221 EXPECT_ANY_THROW(folly::json::serialize("b\xee\x80", opts));
222 EXPECT_ANY_THROW(folly::json::serialize("c\xef\xbf", opts));
225 EXPECT_ANY_THROW(folly::json::serialize("\xfe", opts));
226 EXPECT_ANY_THROW(folly::json::serialize("\xff", opts));
228 // Sample overlong sequences
229 EXPECT_ANY_THROW(folly::json::serialize("\xc0\xaf", opts));
230 EXPECT_ANY_THROW(folly::json::serialize("\xe0\x80\xaf", opts));
232 // Maximum overlong sequences
233 EXPECT_ANY_THROW(folly::json::serialize("\xc1\xbf", opts));
234 EXPECT_ANY_THROW(folly::json::serialize("\x30\x9f\xbf", opts));
236 // illegal code positions
237 EXPECT_ANY_THROW(folly::json::serialize("\xed\xa0\x80", opts));
238 EXPECT_ANY_THROW(folly::json::serialize("\xed\xbf\xbf", opts));
240 // Overlong representation of NUL character
241 EXPECT_ANY_THROW(folly::json::serialize("\xc0\x80", opts));
242 EXPECT_ANY_THROW(folly::json::serialize("\xe0\x80\x80", opts));
244 // Longer than 3 byte encodings
245 EXPECT_ANY_THROW(folly::json::serialize("\xf4\x8f\xbf\xbf", opts));
246 EXPECT_ANY_THROW(folly::json::serialize("\xed\xaf\xbf\xed\xbf\xbf", opts));
249 TEST(Json, UTF8Retention) {
251 // test retention with valid utf8 strings
252 folly::fbstring input = "\u2665";
253 folly::fbstring jsonInput = folly::toJson(input);
254 folly::fbstring output = folly::parseJson(jsonInput).asString();
255 folly::fbstring jsonOutput = folly::toJson(output);
257 LOG(INFO) << "input: " << input
258 <<" => json: " << jsonInput;
259 LOG(INFO) << "output: " << output
260 <<" => json: " << jsonOutput;
262 EXPECT_EQ(input, output);
263 EXPECT_EQ(jsonInput, jsonOutput);
265 // test retention with invalid utf8 - note that non-ascii chars are retained
266 // as is, and no unicode encoding is attempted so no exception is thrown.
268 folly::toJson("a\xe0\xa0\x80z\xc0\x80"),
269 "\"a\xe0\xa0\x80z\xc0\x80\""
273 TEST(Json, UTF8EncodeNonAsciiRetention) {
275 folly::json::serialization_opts opts;
276 opts.encode_non_ascii = true;
278 // test encode_non_ascii valid utf8 strings
279 folly::fbstring input = "\u2665";
280 folly::fbstring jsonInput = folly::json::serialize(input, opts);
281 folly::fbstring output = folly::parseJson(jsonInput).asString();
282 folly::fbstring jsonOutput = folly::json::serialize(output, opts);
284 LOG(INFO) << "input: " << input
285 <<" => json: " << jsonInput;
286 LOG(INFO) << "output: " << output
287 <<" => json: " << jsonOutput;
289 EXPECT_EQ(input, output);
290 EXPECT_EQ(jsonInput, jsonOutput);
292 // test encode_non_ascii with invalid utf8 - note that an attempt to encode
293 // non-ascii to unicode will result is a utf8 validation and throw exceptions.
294 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts));
295 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xe0\x80\x80", opts));
298 TEST(Json, UTF8Validation) {
299 folly::json::serialization_opts opts;
300 opts.validate_utf8 = true;
302 // test validate_utf8 valid utf8 strings - note that we only validate the
303 // for utf8 but don't encode non-ascii to unicode so they are retained as is.
304 EXPECT_EQ(folly::json::serialize("a\xc2\x80z", opts), "\"a\xc2\x80z\"");
306 folly::json::serialize("a\xe0\xa0\x80z", opts),
307 "\"a\xe0\xa0\x80z\"");
309 folly::json::serialize("a\xe0\xa0\x80m\xc2\x80z", opts),
310 "\"a\xe0\xa0\x80m\xc2\x80z\"");
312 // test validate_utf8 with invalid utf8
313 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts));
314 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xe0\x80\x80", opts));
317 BENCHMARK(jsonSerialize, iters) {
318 folly::json::serialization_opts opts;
319 for (int i = 0; i < iters; ++i) {
320 folly::json::serialize(
321 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
322 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
323 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
324 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
325 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
326 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
327 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
328 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
329 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
330 "qwerty \xc2\x80 \xef\xbf\xbf poiuy",
335 BENCHMARK(jsonSerializeWithNonAsciiEncoding, iters) {
336 folly::json::serialization_opts opts;
337 opts.encode_non_ascii = true;
339 for (int i = 0; i < iters; ++i) {
340 folly::json::serialize(
341 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
342 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
343 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
344 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
345 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
346 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
347 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
348 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
349 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
350 "qwerty \xc2\x80 \xef\xbf\xbf poiuy",
355 BENCHMARK(jsonSerializeWithUtf8Validation, iters) {
356 folly::json::serialization_opts opts;
357 opts.validate_utf8 = true;
359 for (int i = 0; i < iters; ++i) {
360 folly::json::serialize(
361 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
362 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
363 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
364 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
365 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
366 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
367 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
368 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
369 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
370 "qwerty \xc2\x80 \xef\xbf\xbf poiuy",
375 BENCHMARK(parseSmallStringWithUtf, iters) {
376 for (int i = 0; i < iters << 4; ++i) {
377 parseJson("\"I \\u2665 UTF-8 thjasdhkjh blah blah blah\"");
381 BENCHMARK(parseNormalString, iters) {
382 for (int i = 0; i < iters << 4; ++i) {
383 parseJson("\"akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk\"");
387 BENCHMARK(parseBigString, iters) {
388 for (int i = 0; i < iters; ++i) {
390 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
391 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
392 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
393 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
394 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
395 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
396 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
397 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
398 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
399 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
400 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
405 int main(int argc, char** argv) {
406 testing::InitGoogleTest(&argc, argv);
407 google::ParseCommandLineFlags(&argc, &argv, true);
408 if (FLAGS_benchmark) {
409 folly::runBenchmarks();
411 return RUN_ALL_TESTS();