2 * Copyright 2011-present Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
18 #include <boost/next_prior.hpp>
20 #include <folly/json.h>
21 #include <folly/portability/GTest.h>
24 using folly::parseJson;
28 auto val = parseJson(u8"\"I \u2665 UTF-8\"");
29 EXPECT_EQ(u8"I \u2665 UTF-8", val.asString());
30 val = parseJson("\"I \\u2665 UTF-8\"");
31 EXPECT_EQ(u8"I \u2665 UTF-8", val.asString());
32 val = parseJson(u8"\"I \U0001D11E playing in G-clef\"");
33 EXPECT_EQ(u8"I \U0001D11E playing in G-clef", val.asString());
35 val = parseJson("\"I \\uD834\\uDD1E playing in G-clef\"");
36 EXPECT_EQ(u8"I \U0001D11E playing in G-clef", val.asString());
40 auto num = parseJson("12");
41 EXPECT_TRUE(num.isInt());
43 num = parseJson("12e5");
44 EXPECT_TRUE(num.isDouble());
46 auto numAs1 = num.asDouble();
47 EXPECT_EQ(numAs1, 12e5);
49 EXPECT_EQ(num, 1200000);
51 auto largeNumber = parseJson("4611686018427387904");
52 EXPECT_TRUE(largeNumber.isInt());
53 EXPECT_EQ(largeNumber, 4611686018427387904L);
55 auto negative = parseJson("-123");
56 EXPECT_EQ(negative, -123);
58 auto bfalse = parseJson("false");
59 auto btrue = parseJson("true");
60 EXPECT_EQ(bfalse, false);
61 EXPECT_EQ(btrue, true);
63 auto null = parseJson("null");
64 EXPECT_TRUE(null == nullptr);
66 auto doub1 = parseJson("12.0");
67 auto doub2 = parseJson("12e2");
68 EXPECT_EQ(doub1, 12.0);
69 EXPECT_EQ(doub2, 12e2);
70 EXPECT_EQ(std::numeric_limits<double>::infinity(),
71 parseJson("Infinity").asDouble());
72 EXPECT_EQ(-std::numeric_limits<double>::infinity(),
73 parseJson("-Infinity").asDouble());
74 EXPECT_TRUE(std::isnan(parseJson("NaN").asDouble()));
77 EXPECT_THROW(parseJson("infinity"), std::runtime_error);
78 EXPECT_THROW(parseJson("inf"), std::runtime_error);
79 EXPECT_THROW(parseJson("Inf"), std::runtime_error);
80 EXPECT_THROW(parseJson("INF"), std::runtime_error);
81 EXPECT_THROW(parseJson("nan"), std::runtime_error);
82 EXPECT_THROW(parseJson("NAN"), std::runtime_error);
84 auto array = parseJson(
85 "[12,false, false , null , [12e4,32, [], 12]]");
86 EXPECT_EQ(array.size(), 5);
87 if (array.size() == 5) {
88 EXPECT_EQ(boost::prior(array.end())->size(), 4);
91 EXPECT_THROW(parseJson("\n[12,\n\nnotvalidjson"),
94 EXPECT_THROW(parseJson("12e2e2"),
97 EXPECT_THROW(parseJson("{\"foo\":12,\"bar\":42} \"something\""),
100 dynamic value = dynamic::object
106 dynamic::object("a", "b")
110 dynamic::array("heh"),
116 // Print then parse and get the same thing, hopefully.
117 EXPECT_EQ(value, parseJson(toJson(value)));
120 // Test an object with non-string values.
121 dynamic something = parseJson(
122 "{\"old_value\":40,\"changed\":true,\"opened\":false}");
123 dynamic expected = dynamic::object
127 EXPECT_EQ(something, expected);
130 TEST(Json, ParseTrailingComma) {
131 folly::json::serialization_opts on, off;
132 on.allow_trailing_comma = true;
133 off.allow_trailing_comma = false;
135 dynamic arr = dynamic::array(1, 2);
136 EXPECT_EQ(arr, parseJson("[1, 2]", on));
137 EXPECT_EQ(arr, parseJson("[1, 2,]", on));
138 EXPECT_EQ(arr, parseJson("[1, 2, ]", on));
139 EXPECT_EQ(arr, parseJson("[1, 2 , ]", on));
140 EXPECT_EQ(arr, parseJson("[1, 2 ,]", on));
141 EXPECT_THROW(parseJson("[1, 2,]", off), std::runtime_error);
143 dynamic obj = dynamic::object("a", 1);
144 EXPECT_EQ(obj, parseJson("{\"a\": 1}", on));
145 EXPECT_EQ(obj, parseJson("{\"a\": 1,}", on));
146 EXPECT_EQ(obj, parseJson("{\"a\": 1, }", on));
147 EXPECT_EQ(obj, parseJson("{\"a\": 1 , }", on));
148 EXPECT_EQ(obj, parseJson("{\"a\": 1 ,}", on));
149 EXPECT_THROW(parseJson("{\"a\":1,}", off), std::runtime_error);
152 TEST(Json, BoolConversion) {
153 EXPECT_TRUE(parseJson("42").asBool());
156 TEST(Json, JavascriptSafe) {
157 auto badDouble = int64_t((1ULL << 63ULL) + 1);
158 dynamic badDyn = badDouble;
159 EXPECT_EQ(folly::toJson(badDouble), folly::to<std::string>(badDouble));
160 folly::json::serialization_opts opts;
161 opts.javascript_safe = true;
162 EXPECT_ANY_THROW(folly::json::serialize(badDouble, opts));
164 auto okDouble = int64_t(1ULL << 63ULL);
165 dynamic okDyn = okDouble;
166 EXPECT_EQ(folly::toJson(okDouble), folly::to<std::string>(okDouble));
169 TEST(Json, Produce) {
170 auto value = parseJson(R"( "f\"oo" )");
171 EXPECT_EQ(toJson(value), R"("f\"oo")");
172 value = parseJson("\"Control code: \001 \002 \x1f\"");
173 EXPECT_EQ(toJson(value), R"("Control code: \u0001 \u0002 \u001f")");
175 // We're not allowed to have non-string keys in json.
176 EXPECT_THROW(toJson(dynamic::object("abc", "xyz")(42.33, "asd")),
179 // Check Infinity/Nan
180 folly::json::serialization_opts opts;
181 opts.allow_nan_inf = true;
182 EXPECT_EQ("Infinity", folly::json::serialize(parseJson("Infinity"), opts));
183 EXPECT_EQ("NaN", folly::json::serialize(parseJson("NaN"), opts));
186 TEST(Json, JsonEscape) {
187 folly::json::serialization_opts opts;
189 folly::json::serialize("\b\f\n\r\x01\t\\\"/\v\a", opts),
190 R"("\b\f\n\r\u0001\t\\\"/\u000b\u0007")");
193 TEST(Json, EscapeCornerCases) {
194 // The escaping logic uses some bitwise operations to determine
195 // which bytes need escaping 8 bytes at a time. Test that this logic
196 // is correct regardless of positions by planting 2 characters that
197 // may need escaping at each possible position and checking the
198 // result, for varying string lengths.
200 folly::json::serialization_opts opts;
201 opts.validate_utf8 = true;
204 std::string expected;
205 for (bool ascii : {true, false}) {
206 opts.encode_non_ascii = ascii;
208 for (size_t len = 2; len < 32; ++len) {
209 for (size_t i = 0; i < len; ++i) {
210 for (size_t j = 0; j < len; ++j) {
218 expected.push_back('"');
219 for (size_t pos = 0; pos < len; ++pos) {
222 expected.append("\\\\");
223 } else if (pos == j) {
224 s.append("\xe2\x82\xac");
225 expected.append(ascii ? "\\u20ac" : "\xe2\x82\xac");
228 expected.push_back('x');
231 expected.push_back('"');
233 EXPECT_EQ(folly::json::serialize(s, opts), expected) << ascii;
240 TEST(Json, JsonNonAsciiEncoding) {
241 folly::json::serialization_opts opts;
242 opts.encode_non_ascii = true;
245 EXPECT_EQ(folly::json::serialize("\x1f", opts), R"("\u001f")");
246 EXPECT_EQ(folly::json::serialize("\xc2\xa2", opts), R"("\u00a2")");
247 EXPECT_EQ(folly::json::serialize("\xe2\x82\xac", opts), R"("\u20ac")");
249 // multiple unicode encodings
251 folly::json::serialize("\x1f\xe2\x82\xac", opts),
252 R"("\u001f\u20ac")");
254 folly::json::serialize("\x1f\xc2\xa2\xe2\x82\xac", opts),
255 R"("\u001f\u00a2\u20ac")");
257 folly::json::serialize("\xc2\x80\xef\xbf\xbf", opts),
258 R"("\u0080\uffff")");
260 folly::json::serialize("\xe0\xa0\x80\xdf\xbf", opts),
261 R"("\u0800\u07ff")");
263 // first possible sequence of a certain length
264 EXPECT_EQ(folly::json::serialize("\xc2\x80", opts), R"("\u0080")");
265 EXPECT_EQ(folly::json::serialize("\xe0\xa0\x80", opts), R"("\u0800")");
267 // last possible sequence of a certain length
268 EXPECT_EQ(folly::json::serialize("\xdf\xbf", opts), R"("\u07ff")");
269 EXPECT_EQ(folly::json::serialize("\xef\xbf\xbf", opts), R"("\uffff")");
271 // other boundary conditions
272 EXPECT_EQ(folly::json::serialize("\xed\x9f\xbf", opts), R"("\ud7ff")");
273 EXPECT_EQ(folly::json::serialize("\xee\x80\x80", opts), R"("\ue000")");
274 EXPECT_EQ(folly::json::serialize("\xef\xbf\xbd", opts), R"("\ufffd")");
276 // incomplete sequences
277 EXPECT_ANY_THROW(folly::json::serialize("a\xed\x9f", opts));
278 EXPECT_ANY_THROW(folly::json::serialize("b\xee\x80", opts));
279 EXPECT_ANY_THROW(folly::json::serialize("c\xef\xbf", opts));
282 EXPECT_ANY_THROW(folly::json::serialize("\xfe", opts));
283 EXPECT_ANY_THROW(folly::json::serialize("\xff", opts));
285 // Sample overlong sequences
286 EXPECT_ANY_THROW(folly::json::serialize("\xc0\xaf", opts));
287 EXPECT_ANY_THROW(folly::json::serialize("\xe0\x80\xaf", opts));
289 // Maximum overlong sequences
290 EXPECT_ANY_THROW(folly::json::serialize("\xc1\xbf", opts));
291 EXPECT_ANY_THROW(folly::json::serialize("\x30\x9f\xbf", opts));
293 // illegal code positions
294 EXPECT_ANY_THROW(folly::json::serialize("\xed\xa0\x80", opts));
295 EXPECT_ANY_THROW(folly::json::serialize("\xed\xbf\xbf", opts));
297 // Overlong representation of NUL character
298 EXPECT_ANY_THROW(folly::json::serialize("\xc0\x80", opts));
299 EXPECT_ANY_THROW(folly::json::serialize("\xe0\x80\x80", opts));
301 // Longer than 3 byte encodings
302 EXPECT_ANY_THROW(folly::json::serialize("\xf4\x8f\xbf\xbf", opts));
303 EXPECT_ANY_THROW(folly::json::serialize("\xed\xaf\xbf\xed\xbf\xbf", opts));
306 TEST(Json, UTF8Retention) {
308 // test retention with valid utf8 strings
309 std::string input = u8"\u2665";
310 std::string jsonInput = folly::toJson(input);
311 std::string output = folly::parseJson(jsonInput).asString();
312 std::string jsonOutput = folly::toJson(output);
314 EXPECT_EQ(input, output);
315 EXPECT_EQ(jsonInput, jsonOutput);
317 // test retention with invalid utf8 - note that non-ascii chars are retained
318 // as is, and no unicode encoding is attempted so no exception is thrown.
320 folly::toJson("a\xe0\xa0\x80z\xc0\x80"),
321 "\"a\xe0\xa0\x80z\xc0\x80\""
325 TEST(Json, UTF8EncodeNonAsciiRetention) {
327 folly::json::serialization_opts opts;
328 opts.encode_non_ascii = true;
330 // test encode_non_ascii valid utf8 strings
331 std::string input = u8"\u2665";
332 std::string jsonInput = folly::json::serialize(input, opts);
333 std::string output = folly::parseJson(jsonInput).asString();
334 std::string jsonOutput = folly::json::serialize(output, opts);
336 EXPECT_EQ(input, output);
337 EXPECT_EQ(jsonInput, jsonOutput);
339 // test encode_non_ascii with invalid utf8 - note that an attempt to encode
340 // non-ascii to unicode will result is a utf8 validation and throw exceptions.
341 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts));
342 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xe0\x80\x80", opts));
345 TEST(Json, UTF8Validation) {
346 folly::json::serialization_opts opts;
347 opts.validate_utf8 = true;
349 // test validate_utf8 valid utf8 strings - note that we only validate the
350 // for utf8 but don't encode non-ascii to unicode so they are retained as is.
351 EXPECT_EQ(folly::json::serialize("a\xc2\x80z", opts), "\"a\xc2\x80z\"");
353 folly::json::serialize("a\xe0\xa0\x80z", opts),
354 "\"a\xe0\xa0\x80z\"");
356 folly::json::serialize("a\xe0\xa0\x80m\xc2\x80z", opts),
357 "\"a\xe0\xa0\x80m\xc2\x80z\"");
359 // test validate_utf8 with invalid utf8
360 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts));
361 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xe0\x80\x80", opts));
363 opts.skip_invalid_utf8 = true;
365 folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts),
366 u8"\"a\xe0\xa0\x80z\ufffd\ufffd\"");
368 folly::json::serialize("a\xe0\xa0\x80z\xc0\x80\x80", opts),
369 u8"\"a\xe0\xa0\x80z\ufffd\ufffd\ufffd\"");
371 folly::json::serialize("z\xc0\x80z\xe0\xa0\x80", opts),
372 u8"\"z\ufffd\ufffdz\xe0\xa0\x80\"");
374 opts.encode_non_ascii = true;
375 EXPECT_EQ(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts),
376 "\"a\\u0800z\\ufffd\\ufffd\"");
377 EXPECT_EQ(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80\x80", opts),
378 "\"a\\u0800z\\ufffd\\ufffd\\ufffd\"");
379 EXPECT_EQ(folly::json::serialize("z\xc0\x80z\xe0\xa0\x80", opts),
380 "\"z\\ufffd\\ufffdz\\u0800\"");
385 TEST(Json, ParseNonStringKeys) {
387 EXPECT_EQ("a", parseJson("{\"a\":[]}").items().begin()->first.asString());
389 // check that we don't allow non-string keys as this violates the
390 // strict JSON spec (though it is emitted by the output of
391 // folly::dynamic with operator <<).
392 EXPECT_THROW(parseJson("{1:[]}"), std::runtime_error);
394 // check that we can parse colloquial JSON if the option is set
395 folly::json::serialization_opts opts;
396 opts.allow_non_string_keys = true;
398 auto val = parseJson("{1:[]}", opts);
399 EXPECT_EQ(1, val.items().begin()->first.asInt());
402 // test we can still read in strings
403 auto sval = parseJson("{\"a\":[]}", opts);
404 EXPECT_EQ("a", sval.items().begin()->first.asString());
406 // test we can read in doubles
407 auto dval = parseJson("{1.5:[]}", opts);
408 EXPECT_EQ(1.5, dval.items().begin()->first.asDouble());
411 TEST(Json, ParseDoubleFallback) {
413 EXPECT_THROW(parseJson("{\"a\":847605071342477600000000000000}"),
415 EXPECT_THROW(parseJson("{\"a\":-9223372036854775809}"),
417 EXPECT_THROW(parseJson("{\"a\":9223372036854775808}"),
419 EXPECT_EQ(std::numeric_limits<int64_t>::min(),
420 parseJson("{\"a\":-9223372036854775808}").items().begin()
422 EXPECT_EQ(std::numeric_limits<int64_t>::max(),
423 parseJson("{\"a\":9223372036854775807}").items().begin()->second.asInt());
424 // with double_fallback
425 folly::json::serialization_opts opts;
426 opts.double_fallback = true;
427 EXPECT_EQ(847605071342477600000000000000.0,
428 parseJson("{\"a\":847605071342477600000000000000}",
429 opts).items().begin()->second.asDouble());
430 EXPECT_EQ(847605071342477600000000000000.0,
431 parseJson("{\"a\": 847605071342477600000000000000}",
432 opts).items().begin()->second.asDouble());
433 EXPECT_EQ(847605071342477600000000000000.0,
434 parseJson("{\"a\":847605071342477600000000000000 }",
435 opts).items().begin()->second.asDouble());
436 EXPECT_EQ(847605071342477600000000000000.0,
437 parseJson("{\"a\": 847605071342477600000000000000 }",
438 opts).items().begin()->second.asDouble());
439 EXPECT_EQ(std::numeric_limits<int64_t>::min(),
440 parseJson("{\"a\":-9223372036854775808}",
441 opts).items().begin()->second.asInt());
442 EXPECT_EQ(std::numeric_limits<int64_t>::max(),
443 parseJson("{\"a\":9223372036854775807}",
444 opts).items().begin()->second.asInt());
445 // show that some precision gets lost
446 EXPECT_EQ(847605071342477612345678900000.0,
447 parseJson("{\"a\":847605071342477612345678912345}",
448 opts).items().begin()->second.asDouble());
450 toJson(parseJson(R"({"a":-9223372036854775808})", opts)),
451 R"({"a":-9223372036854775808})");
454 TEST(Json, ParseNumbersAsStrings) {
455 folly::json::serialization_opts opts;
456 opts.parse_numbers_as_strings = true;
457 auto parse = [&](std::string number) {
458 return parseJson(number, opts).asString();
461 EXPECT_EQ("0", parse("0"));
462 EXPECT_EQ("1234", parse("1234"));
463 EXPECT_EQ("3.00", parse("3.00"));
464 EXPECT_EQ("3.14", parse("3.14"));
465 EXPECT_EQ("0.1234", parse("0.1234"));
466 EXPECT_EQ("0.0", parse("0.0"));
467 EXPECT_EQ("46845131213548676854213265486468451312135486768542132",
468 parse("46845131213548676854213265486468451312135486768542132"));
469 EXPECT_EQ("-468451312135486768542132654864684513121354867685.5e4",
470 parse("-468451312135486768542132654864684513121354867685.5e4"));
471 EXPECT_EQ("6.62607004e-34", parse("6.62607004e-34"));
472 EXPECT_EQ("6.62607004E+34", parse("6.62607004E+34"));
473 EXPECT_EQ("Infinity", parse("Infinity"));
474 EXPECT_EQ("-Infinity", parse("-Infinity"));
475 EXPECT_EQ("NaN", parse("NaN"));
477 EXPECT_THROW(parse("ThisIsWrong"), std::runtime_error);
478 EXPECT_THROW(parse("34-2"), std::runtime_error);
479 EXPECT_THROW(parse(""), std::runtime_error);
480 EXPECT_THROW(parse("-"), std::runtime_error);
481 EXPECT_THROW(parse("34-e2"), std::runtime_error);
482 EXPECT_THROW(parse("34e2.4"), std::runtime_error);
483 EXPECT_THROW(parse("infinity"), std::runtime_error);
484 EXPECT_THROW(parse("nan"), std::runtime_error);
487 TEST(Json, SortKeys) {
488 folly::json::serialization_opts opts_on, opts_off, opts_custom_sort;
489 opts_on.sort_keys = true;
490 opts_off.sort_keys = false;
492 opts_custom_sort.sort_keys = false; // should not be required
493 opts_custom_sort.sort_keys_by = [](
494 folly::dynamic const& a, folly::dynamic const& b) {
495 // just an inverse sort
499 dynamic value = dynamic::object
505 dynamic::object("a", "b")
509 dynamic::array("heh"),
515 std::string sorted_keys =
516 R"({"a":[{"a":"b","c":"d"},12.5,"Yo Dawg",["heh"],null],)"
517 R"("another":32.2,"foo":"bar","junk":12})";
519 std::string inverse_sorted_keys =
520 R"({"junk":12,"foo":"bar","another":32.2,)"
521 R"("a":[{"c":"d","a":"b"},12.5,"Yo Dawg",["heh"],null]})";
523 EXPECT_EQ(value, parseJson(folly::json::serialize(value, opts_on)));
524 EXPECT_EQ(value, parseJson(folly::json::serialize(value, opts_off)));
525 EXPECT_EQ(value, parseJson(folly::json::serialize(value, opts_custom_sort)));
527 EXPECT_EQ(sorted_keys, folly::json::serialize(value, opts_on));
528 EXPECT_NE(sorted_keys, folly::json::serialize(value, opts_off));
530 inverse_sorted_keys, folly::json::serialize(value, opts_custom_sort));
533 TEST(Json, PrintTo) {
534 std::ostringstream oss;
536 dynamic value = dynamic::object
540 (true, false) // include non-string keys
551 dynamic::object("a", "b")
555 dynamic::array("heh"),
561 std::string expected =
586 PrintTo(value, &oss);
587 EXPECT_EQ(expected, oss.str());
590 TEST(Json, RecursionLimit) {
592 for (int i = 0; i < 1000; i++) {
593 in.append("{\"x\":");
596 for (int i = 0; i < 1000; i++) {
599 EXPECT_ANY_THROW(parseJson(in));
601 folly::json::serialization_opts opts_high_recursion_limit;
602 opts_high_recursion_limit.recursion_limit = 10000;
603 parseJson(in, opts_high_recursion_limit);