2 * Copyright 2015 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include <folly/FileUtil.h>
18 #include <folly/json.h>
19 #include <gtest/gtest.h>
20 #include <gflags/gflags.h>
24 #include <boost/next_prior.hpp>
25 #include <folly/Benchmark.h>
28 using folly::parseJson;
32 auto val = parseJson("\"I \u2665 UTF-8\"");
33 EXPECT_EQ("I \u2665 UTF-8", val.asString());
34 val = parseJson("\"I \\u2665 UTF-8\"");
35 EXPECT_EQ("I \u2665 UTF-8", val.asString());
36 val = parseJson("\"I \U0001D11E playing in G-clef\"");
37 EXPECT_EQ("I \U0001D11E playing in G-clef", val.asString());
39 val = parseJson("\"I \\uD834\\uDD1E playing in G-clef\"");
40 EXPECT_EQ("I \U0001D11E playing in G-clef", val.asString());
44 auto num = parseJson("12");
45 EXPECT_TRUE(num.isInt());
47 num = parseJson("12e5");
48 EXPECT_TRUE(num.isDouble());
50 auto numAs1 = num.asDouble();
51 EXPECT_EQ(numAs1, 12e5);
53 EXPECT_EQ(num, 1200000);
55 auto largeNumber = parseJson("4611686018427387904");
56 EXPECT_TRUE(largeNumber.isInt());
57 EXPECT_EQ(largeNumber, 4611686018427387904L);
59 auto negative = parseJson("-123");
60 EXPECT_EQ(negative, -123);
62 auto bfalse = parseJson("false");
63 auto btrue = parseJson("true");
64 EXPECT_EQ(bfalse, false);
65 EXPECT_EQ(btrue, true);
67 auto null = parseJson("null");
68 EXPECT_TRUE(null == nullptr);
70 auto doub1 = parseJson("12.0");
71 auto doub2 = parseJson("12e2");
72 EXPECT_EQ(doub1, 12.0);
73 EXPECT_EQ(doub2, 12e2);
74 EXPECT_EQ(std::numeric_limits<double>::infinity(),
75 parseJson("Infinity").asDouble());
76 EXPECT_EQ(-std::numeric_limits<double>::infinity(),
77 parseJson("-Infinity").asDouble());
78 EXPECT_TRUE(std::isnan(parseJson("NaN").asDouble()));
81 EXPECT_THROW(parseJson("infinity"), std::runtime_error);
82 EXPECT_THROW(parseJson("inf"), std::runtime_error);
83 EXPECT_THROW(parseJson("nan"), std::runtime_error);
85 auto array = parseJson(
86 "[12,false, false , null , [12e4,32, [], 12]]");
87 EXPECT_EQ(array.size(), 5);
88 if (array.size() == 5) {
89 EXPECT_EQ(boost::prior(array.end())->size(), 4);
92 EXPECT_THROW(parseJson("\n[12,\n\nnotvalidjson"),
95 EXPECT_THROW(parseJson("12e2e2"),
98 EXPECT_THROW(parseJson("{\"foo\":12,\"bar\":42} \"something\""),
101 dynamic value = dynamic::object
107 dynamic::object("a", "b")
117 // Print then parse and get the same thing, hopefully.
118 EXPECT_EQ(value, parseJson(toJson(value)));
121 // Test an object with non-string values.
122 dynamic something = parseJson(
123 "{\"old_value\":40,\"changed\":true,\"opened\":false}");
124 dynamic expected = dynamic::object
128 EXPECT_EQ(something, expected);
131 TEST(Json, ParseTrailingComma) {
132 folly::json::serialization_opts on, off;
133 on.allow_trailing_comma = true;
134 off.allow_trailing_comma = false;
136 dynamic arr { 1, 2 };
137 EXPECT_EQ(arr, parseJson("[1, 2]", on));
138 EXPECT_EQ(arr, parseJson("[1, 2,]", on));
139 EXPECT_EQ(arr, parseJson("[1, 2, ]", on));
140 EXPECT_EQ(arr, parseJson("[1, 2 , ]", on));
141 EXPECT_EQ(arr, parseJson("[1, 2 ,]", on));
142 EXPECT_THROW(parseJson("[1, 2,]", off), std::runtime_error);
144 dynamic obj = dynamic::object("a", 1);
145 EXPECT_EQ(obj, parseJson("{\"a\": 1}", on));
146 EXPECT_EQ(obj, parseJson("{\"a\": 1,}", on));
147 EXPECT_EQ(obj, parseJson("{\"a\": 1, }", on));
148 EXPECT_EQ(obj, parseJson("{\"a\": 1 , }", on));
149 EXPECT_EQ(obj, parseJson("{\"a\": 1 ,}", on));
150 EXPECT_THROW(parseJson("{\"a\":1,}", off), std::runtime_error);
153 TEST(Json, JavascriptSafe) {
154 auto badDouble = (1ll << 63ll) + 1;
155 dynamic badDyn = badDouble;
156 EXPECT_EQ(folly::toJson(badDouble), folly::to<folly::fbstring>(badDouble));
157 folly::json::serialization_opts opts;
158 opts.javascript_safe = true;
159 EXPECT_ANY_THROW(folly::json::serialize(badDouble, opts));
161 auto okDouble = 1ll << 63ll;
162 dynamic okDyn = okDouble;
163 EXPECT_EQ(folly::toJson(okDouble), folly::to<folly::fbstring>(okDouble));
166 TEST(Json, Produce) {
167 auto value = parseJson(R"( "f\"oo" )");
168 EXPECT_EQ(toJson(value), R"("f\"oo")");
169 value = parseJson("\"Control code: \001 \002 \x1f\"");
170 EXPECT_EQ(toJson(value), R"("Control code: \u0001 \u0002 \u001f")");
172 // We're not allowed to have non-string keys in json.
173 EXPECT_THROW(toJson(dynamic::object("abc", "xyz")(42.33, "asd")),
177 TEST(Json, JsonEscape) {
178 folly::json::serialization_opts opts;
180 folly::json::serialize("\b\f\n\r\x01\t\\\"/\v\a", opts),
181 R"("\b\f\n\r\u0001\t\\\"/\u000b\u0007")");
184 TEST(Json, JsonNonAsciiEncoding) {
185 folly::json::serialization_opts opts;
186 opts.encode_non_ascii = true;
189 EXPECT_EQ(folly::json::serialize("\x1f", opts), R"("\u001f")");
190 EXPECT_EQ(folly::json::serialize("\xc2\xa2", opts), R"("\u00a2")");
191 EXPECT_EQ(folly::json::serialize("\xe2\x82\xac", opts), R"("\u20ac")");
193 // multiple unicode encodings
195 folly::json::serialize("\x1f\xe2\x82\xac", opts),
196 R"("\u001f\u20ac")");
198 folly::json::serialize("\x1f\xc2\xa2\xe2\x82\xac", opts),
199 R"("\u001f\u00a2\u20ac")");
201 folly::json::serialize("\xc2\x80\xef\xbf\xbf", opts),
202 R"("\u0080\uffff")");
204 folly::json::serialize("\xe0\xa0\x80\xdf\xbf", opts),
205 R"("\u0800\u07ff")");
207 // first possible sequence of a certain length
208 EXPECT_EQ(folly::json::serialize("\xc2\x80", opts), R"("\u0080")");
209 EXPECT_EQ(folly::json::serialize("\xe0\xa0\x80", opts), R"("\u0800")");
211 // last possible sequence of a certain length
212 EXPECT_EQ(folly::json::serialize("\xdf\xbf", opts), R"("\u07ff")");
213 EXPECT_EQ(folly::json::serialize("\xef\xbf\xbf", opts), R"("\uffff")");
215 // other boundary conditions
216 EXPECT_EQ(folly::json::serialize("\xed\x9f\xbf", opts), R"("\ud7ff")");
217 EXPECT_EQ(folly::json::serialize("\xee\x80\x80", opts), R"("\ue000")");
218 EXPECT_EQ(folly::json::serialize("\xef\xbf\xbd", opts), R"("\ufffd")");
220 // incomplete sequences
221 EXPECT_ANY_THROW(folly::json::serialize("a\xed\x9f", opts));
222 EXPECT_ANY_THROW(folly::json::serialize("b\xee\x80", opts));
223 EXPECT_ANY_THROW(folly::json::serialize("c\xef\xbf", opts));
226 EXPECT_ANY_THROW(folly::json::serialize("\xfe", opts));
227 EXPECT_ANY_THROW(folly::json::serialize("\xff", opts));
229 // Sample overlong sequences
230 EXPECT_ANY_THROW(folly::json::serialize("\xc0\xaf", opts));
231 EXPECT_ANY_THROW(folly::json::serialize("\xe0\x80\xaf", opts));
233 // Maximum overlong sequences
234 EXPECT_ANY_THROW(folly::json::serialize("\xc1\xbf", opts));
235 EXPECT_ANY_THROW(folly::json::serialize("\x30\x9f\xbf", opts));
237 // illegal code positions
238 EXPECT_ANY_THROW(folly::json::serialize("\xed\xa0\x80", opts));
239 EXPECT_ANY_THROW(folly::json::serialize("\xed\xbf\xbf", opts));
241 // Overlong representation of NUL character
242 EXPECT_ANY_THROW(folly::json::serialize("\xc0\x80", opts));
243 EXPECT_ANY_THROW(folly::json::serialize("\xe0\x80\x80", opts));
245 // Longer than 3 byte encodings
246 EXPECT_ANY_THROW(folly::json::serialize("\xf4\x8f\xbf\xbf", opts));
247 EXPECT_ANY_THROW(folly::json::serialize("\xed\xaf\xbf\xed\xbf\xbf", opts));
250 TEST(Json, UTF8Retention) {
252 // test retention with valid utf8 strings
253 folly::fbstring input = "\u2665";
254 folly::fbstring jsonInput = folly::toJson(input);
255 folly::fbstring output = folly::parseJson(jsonInput).asString();
256 folly::fbstring jsonOutput = folly::toJson(output);
258 EXPECT_EQ(input, output);
259 EXPECT_EQ(jsonInput, jsonOutput);
261 // test retention with invalid utf8 - note that non-ascii chars are retained
262 // as is, and no unicode encoding is attempted so no exception is thrown.
264 folly::toJson("a\xe0\xa0\x80z\xc0\x80"),
265 "\"a\xe0\xa0\x80z\xc0\x80\""
269 TEST(Json, UTF8EncodeNonAsciiRetention) {
271 folly::json::serialization_opts opts;
272 opts.encode_non_ascii = true;
274 // test encode_non_ascii valid utf8 strings
275 folly::fbstring input = "\u2665";
276 folly::fbstring jsonInput = folly::json::serialize(input, opts);
277 folly::fbstring output = folly::parseJson(jsonInput).asString();
278 folly::fbstring jsonOutput = folly::json::serialize(output, opts);
280 EXPECT_EQ(input, output);
281 EXPECT_EQ(jsonInput, jsonOutput);
283 // test encode_non_ascii with invalid utf8 - note that an attempt to encode
284 // non-ascii to unicode will result is a utf8 validation and throw exceptions.
285 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts));
286 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xe0\x80\x80", opts));
289 TEST(Json, UTF8Validation) {
290 folly::json::serialization_opts opts;
291 opts.validate_utf8 = true;
293 // test validate_utf8 valid utf8 strings - note that we only validate the
294 // for utf8 but don't encode non-ascii to unicode so they are retained as is.
295 EXPECT_EQ(folly::json::serialize("a\xc2\x80z", opts), "\"a\xc2\x80z\"");
297 folly::json::serialize("a\xe0\xa0\x80z", opts),
298 "\"a\xe0\xa0\x80z\"");
300 folly::json::serialize("a\xe0\xa0\x80m\xc2\x80z", opts),
301 "\"a\xe0\xa0\x80m\xc2\x80z\"");
303 // test validate_utf8 with invalid utf8
304 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts));
305 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xe0\x80\x80", opts));
307 opts.skip_invalid_utf8 = true;
308 EXPECT_EQ(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts),
309 "\"a\xe0\xa0\x80z\ufffd\ufffd\"");
310 EXPECT_EQ(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80\x80", opts),
311 "\"a\xe0\xa0\x80z\ufffd\ufffd\ufffd\"");
312 EXPECT_EQ(folly::json::serialize("z\xc0\x80z\xe0\xa0\x80", opts),
313 "\"z\ufffd\ufffdz\xe0\xa0\x80\"");
315 opts.encode_non_ascii = true;
316 EXPECT_EQ(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts),
317 "\"a\\u0800z\\ufffd\\ufffd\"");
318 EXPECT_EQ(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80\x80", opts),
319 "\"a\\u0800z\\ufffd\\ufffd\\ufffd\"");
320 EXPECT_EQ(folly::json::serialize("z\xc0\x80z\xe0\xa0\x80", opts),
321 "\"z\\ufffd\\ufffdz\\u0800\"");
326 TEST(Json, ParseNonStringKeys) {
328 EXPECT_EQ("a", parseJson("{\"a\":[]}").items().begin()->first.asString());
330 // check that we don't allow non-string keys as this violates the
331 // strict JSON spec (though it is emitted by the output of
332 // folly::dynamic with operator <<).
333 EXPECT_THROW(parseJson("{1:[]}"), std::runtime_error);
335 // check that we can parse colloquial JSON if the option is set
336 folly::json::serialization_opts opts;
337 opts.allow_non_string_keys = true;
339 auto val = parseJson("{1:[]}", opts);
340 EXPECT_EQ(1, val.items().begin()->first.asInt());
343 // test we can still read in strings
344 auto sval = parseJson("{\"a\":[]}", opts);
345 EXPECT_EQ("a", sval.items().begin()->first.asString());
347 // test we can read in doubles
348 auto dval = parseJson("{1.5:[]}", opts);
349 EXPECT_EQ(1.5, dval.items().begin()->first.asDouble());
352 TEST(Json, SortKeys) {
353 folly::json::serialization_opts opts_on, opts_off;
354 opts_on.sort_keys = true;
355 opts_off.sort_keys = false;
357 dynamic value = dynamic::object
363 dynamic::object("a", "b")
373 std::string sorted_keys =
374 R"({"a":[{"a":"b","c":"d"},12.5,"Yo Dawg",["heh"],null],)"
375 R"("another":32.2,"foo":"bar","junk":12})";
377 EXPECT_EQ(value, parseJson(folly::json::serialize(value, opts_on)));
378 EXPECT_EQ(value, parseJson(folly::json::serialize(value, opts_off)));
380 EXPECT_EQ(sorted_keys, folly::json::serialize(value, opts_on));
383 TEST(Json, StripComments) {
384 const std::string kTestDir = "folly/test/";
385 const std::string kTestFile = "json_test_data/commented.json";
386 const std::string kTestExpected = "json_test_data/commented.json.exp";
389 std::string expectedStr;
390 if (!folly::readFile(kTestFile.data(), testStr) &&
391 !folly::readFile((kTestDir + kTestFile).data(), testStr)) {
392 FAIL() << "can not read test file " << kTestFile;
394 if (!folly::readFile(kTestExpected.data(), expectedStr) &&
395 !folly::readFile((kTestDir + kTestExpected).data(), expectedStr)) {
396 FAIL() << "can not read test file " << kTestExpected;
398 EXPECT_EQ(expectedStr, folly::json::stripComments(testStr));
401 BENCHMARK(jsonSerialize, iters) {
402 folly::json::serialization_opts opts;
403 for (size_t i = 0; i < iters; ++i) {
404 folly::json::serialize(
405 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
406 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
407 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
408 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
409 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
410 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
411 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
412 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
413 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
414 "qwerty \xc2\x80 \xef\xbf\xbf poiuy",
419 BENCHMARK(jsonSerializeWithNonAsciiEncoding, iters) {
420 folly::json::serialization_opts opts;
421 opts.encode_non_ascii = true;
423 for (size_t i = 0; i < iters; ++i) {
424 folly::json::serialize(
425 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
426 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
427 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
428 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
429 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
430 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
431 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
432 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
433 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
434 "qwerty \xc2\x80 \xef\xbf\xbf poiuy",
439 BENCHMARK(jsonSerializeWithUtf8Validation, iters) {
440 folly::json::serialization_opts opts;
441 opts.validate_utf8 = true;
443 for (size_t i = 0; i < iters; ++i) {
444 folly::json::serialize(
445 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
446 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
447 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
448 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
449 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
450 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
451 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
452 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
453 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
454 "qwerty \xc2\x80 \xef\xbf\xbf poiuy",
459 BENCHMARK(parseSmallStringWithUtf, iters) {
460 for (size_t i = 0; i < iters << 4; ++i) {
461 parseJson("\"I \\u2665 UTF-8 thjasdhkjh blah blah blah\"");
465 BENCHMARK(parseNormalString, iters) {
466 for (size_t i = 0; i < iters << 4; ++i) {
467 parseJson("\"akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk\"");
471 BENCHMARK(parseBigString, iters) {
472 for (size_t i = 0; i < iters; ++i) {
474 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
475 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
476 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
477 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
478 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
479 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
480 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
481 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
482 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
483 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
484 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
489 BENCHMARK(toJson, iters) {
490 dynamic something = parseJson(
491 "{\"old_value\":40,\"changed\":true,\"opened\":false,\"foo\":[1,2,3,4,5,6]}"
494 for (size_t i = 0; i < iters; i++) {
499 int main(int argc, char** argv) {
500 testing::InitGoogleTest(&argc, argv);
501 gflags::ParseCommandLineFlags(&argc, &argv, true);
502 if (FLAGS_benchmark) {
503 folly::runBenchmarks();
505 return RUN_ALL_TESTS();