2 * Copyright 2013 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include "folly/json.h"
18 #include <gtest/gtest.h>
19 #include <gflags/gflags.h>
23 #include <boost/next_prior.hpp>
24 #include "folly/Benchmark.h"
27 using folly::parseJson;
31 auto val = parseJson("\"I \u2665 UTF-8\"");
32 EXPECT_EQ("I \u2665 UTF-8", val.asString());
33 val = parseJson("\"I \\u2665 UTF-8\"");
34 EXPECT_EQ("I \u2665 UTF-8", val.asString());
35 val = parseJson("\"I \U0001D11E playing in G-clef\"");
36 EXPECT_EQ("I \U0001D11E playing in G-clef", val.asString());
38 val = parseJson("\"I \\uD834\\uDD1E playing in G-clef\"");
39 EXPECT_EQ("I \U0001D11E playing in G-clef", val.asString());
43 auto num = parseJson("12");
44 EXPECT_TRUE(num.isInt());
46 num = parseJson("12e5");
47 EXPECT_TRUE(num.isDouble());
49 auto numAs1 = num.asDouble();
50 EXPECT_EQ(numAs1, 12e5);
52 EXPECT_EQ(num, 1200000);
54 auto largeNumber = parseJson("4611686018427387904");
55 EXPECT_TRUE(largeNumber.isInt());
56 EXPECT_EQ(largeNumber, 4611686018427387904L);
58 auto negative = parseJson("-123");
59 EXPECT_EQ(negative, -123);
61 auto bfalse = parseJson("false");
62 auto btrue = parseJson("true");
63 EXPECT_EQ(bfalse, false);
64 EXPECT_EQ(btrue, true);
66 auto null = parseJson("null");
67 EXPECT_TRUE(null == nullptr);
69 auto doub1 = parseJson("12.0");
70 auto doub2 = parseJson("12e2");
71 EXPECT_EQ(doub1, 12.0);
72 EXPECT_EQ(doub2, 12e2);
73 EXPECT_EQ(std::numeric_limits<double>::infinity(),
74 parseJson("Infinity").asDouble());
75 EXPECT_EQ(-std::numeric_limits<double>::infinity(),
76 parseJson("-Infinity").asDouble());
77 EXPECT_TRUE(std::isnan(parseJson("NaN").asDouble()));
78 EXPECT_THROW(parseJson("infinity"), std::runtime_error);
79 EXPECT_THROW(parseJson("inf"), std::runtime_error);
80 EXPECT_THROW(parseJson("nan"), std::runtime_error);
82 auto array = parseJson(
83 "[12,false, false , null , [12e4,32, [], 12]]");
84 EXPECT_EQ(array.size(), 5);
85 if (array.size() == 5) {
86 EXPECT_EQ(boost::prior(array.end())->size(), 4);
91 parseJson("\n[12,\n\nnotvalidjson");
92 } catch (const std::exception& e) {
100 } catch (const std::exception& e) {
107 parseJson("{\"foo\":12,\"bar\":42} \"something\"");
108 } catch (const std::exception& e) {
114 dynamic anotherVal = dynamic::object
120 dynamic::object("a", "b")
130 // Print then parse and get the same thing, hopefully.
131 auto value = parseJson(toJson(anotherVal));
132 EXPECT_EQ(value, anotherVal);
134 // Test an object with non-string values.
135 dynamic something = folly::parseJson(
136 "{\"old_value\":40,\"changed\":true,\"opened\":false}");
137 dynamic expected = dynamic::object
141 EXPECT_EQ(something, expected);
144 TEST(Json, JavascriptSafe) {
145 auto badDouble = (1ll << 63ll) + 1;
146 dynamic badDyn = badDouble;
147 EXPECT_EQ(folly::toJson(badDouble), folly::to<folly::fbstring>(badDouble));
148 folly::json::serialization_opts opts;
149 opts.javascript_safe = true;
150 EXPECT_ANY_THROW(folly::json::serialize(badDouble, opts));
152 auto okDouble = 1ll << 63ll;
153 dynamic okDyn = okDouble;
154 EXPECT_EQ(folly::toJson(okDouble), folly::to<folly::fbstring>(okDouble));
157 TEST(Json, Produce) {
158 auto value = parseJson(R"( "f\"oo" )");
159 EXPECT_EQ(toJson(value), R"("f\"oo")");
160 value = parseJson("\"Control code: \001 \002 \x1f\"");
161 EXPECT_EQ(toJson(value), R"("Control code: \u0001 \u0002 \u001f")");
165 dynamic d = dynamic::object;
168 auto str = toJson(d);
169 } catch (std::exception const& e) {
170 // We're not allowed to have non-string keys in json.
176 TEST(Json, JsonEscape) {
177 folly::json::serialization_opts opts;
179 folly::json::serialize("\b\f\n\r\x01\t\\\"/\v\a", opts),
180 R"("\b\f\n\r\u0001\t\\\"/\u000b\u0007")");
183 TEST(Json, JsonNonAsciiEncoding) {
184 folly::json::serialization_opts opts;
185 opts.encode_non_ascii = true;
188 EXPECT_EQ(folly::json::serialize("\x1f", opts), R"("\u001f")");
189 EXPECT_EQ(folly::json::serialize("\xc2\xa2", opts), R"("\u00a2")");
190 EXPECT_EQ(folly::json::serialize("\xe2\x82\xac", opts), R"("\u20ac")");
192 // multiple unicode encodings
194 folly::json::serialize("\x1f\xe2\x82\xac", opts),
195 R"("\u001f\u20ac")");
197 folly::json::serialize("\x1f\xc2\xa2\xe2\x82\xac", opts),
198 R"("\u001f\u00a2\u20ac")");
200 folly::json::serialize("\xc2\x80\xef\xbf\xbf", opts),
201 R"("\u0080\uffff")");
203 folly::json::serialize("\xe0\xa0\x80\xdf\xbf", opts),
204 R"("\u0800\u07ff")");
206 // first possible sequence of a certain length
207 EXPECT_EQ(folly::json::serialize("\xc2\x80", opts), R"("\u0080")");
208 EXPECT_EQ(folly::json::serialize("\xe0\xa0\x80", opts), R"("\u0800")");
210 // last possible sequence of a certain length
211 EXPECT_EQ(folly::json::serialize("\xdf\xbf", opts), R"("\u07ff")");
212 EXPECT_EQ(folly::json::serialize("\xef\xbf\xbf", opts), R"("\uffff")");
214 // other boundary conditions
215 EXPECT_EQ(folly::json::serialize("\xed\x9f\xbf", opts), R"("\ud7ff")");
216 EXPECT_EQ(folly::json::serialize("\xee\x80\x80", opts), R"("\ue000")");
217 EXPECT_EQ(folly::json::serialize("\xef\xbf\xbd", opts), R"("\ufffd")");
219 // incomplete sequences
220 EXPECT_ANY_THROW(folly::json::serialize("a\xed\x9f", opts));
221 EXPECT_ANY_THROW(folly::json::serialize("b\xee\x80", opts));
222 EXPECT_ANY_THROW(folly::json::serialize("c\xef\xbf", opts));
225 EXPECT_ANY_THROW(folly::json::serialize("\xfe", opts));
226 EXPECT_ANY_THROW(folly::json::serialize("\xff", opts));
228 // Sample overlong sequences
229 EXPECT_ANY_THROW(folly::json::serialize("\xc0\xaf", opts));
230 EXPECT_ANY_THROW(folly::json::serialize("\xe0\x80\xaf", opts));
232 // Maximum overlong sequences
233 EXPECT_ANY_THROW(folly::json::serialize("\xc1\xbf", opts));
234 EXPECT_ANY_THROW(folly::json::serialize("\x30\x9f\xbf", opts));
236 // illegal code positions
237 EXPECT_ANY_THROW(folly::json::serialize("\xed\xa0\x80", opts));
238 EXPECT_ANY_THROW(folly::json::serialize("\xed\xbf\xbf", opts));
240 // Overlong representation of NUL character
241 EXPECT_ANY_THROW(folly::json::serialize("\xc0\x80", opts));
242 EXPECT_ANY_THROW(folly::json::serialize("\xe0\x80\x80", opts));
244 // Longer than 3 byte encodings
245 EXPECT_ANY_THROW(folly::json::serialize("\xf4\x8f\xbf\xbf", opts));
246 EXPECT_ANY_THROW(folly::json::serialize("\xed\xaf\xbf\xed\xbf\xbf", opts));
249 TEST(Json, UTF8Retention) {
251 // test retention with valid utf8 strings
252 folly::fbstring input = "\u2665";
253 folly::fbstring jsonInput = folly::toJson(input);
254 folly::fbstring output = folly::parseJson(jsonInput).asString();
255 folly::fbstring jsonOutput = folly::toJson(output);
257 LOG(INFO) << "input: " << input
258 <<" => json: " << jsonInput;
259 LOG(INFO) << "output: " << output
260 <<" => json: " << jsonOutput;
262 EXPECT_EQ(input, output);
263 EXPECT_EQ(jsonInput, jsonOutput);
265 // test retention with invalid utf8 - note that non-ascii chars are retained
266 // as is, and no unicode encoding is attempted so no exception is thrown.
268 folly::toJson("a\xe0\xa0\x80z\xc0\x80"),
269 "\"a\xe0\xa0\x80z\xc0\x80\""
273 TEST(Json, UTF8EncodeNonAsciiRetention) {
275 folly::json::serialization_opts opts;
276 opts.encode_non_ascii = true;
278 // test encode_non_ascii valid utf8 strings
279 folly::fbstring input = "\u2665";
280 folly::fbstring jsonInput = folly::json::serialize(input, opts);
281 folly::fbstring output = folly::parseJson(jsonInput).asString();
282 folly::fbstring jsonOutput = folly::json::serialize(output, opts);
284 LOG(INFO) << "input: " << input
285 <<" => json: " << jsonInput;
286 LOG(INFO) << "output: " << output
287 <<" => json: " << jsonOutput;
289 EXPECT_EQ(input, output);
290 EXPECT_EQ(jsonInput, jsonOutput);
292 // test encode_non_ascii with invalid utf8 - note that an attempt to encode
293 // non-ascii to unicode will result is a utf8 validation and throw exceptions.
294 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts));
295 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xe0\x80\x80", opts));
298 TEST(Json, UTF8Validation) {
299 folly::json::serialization_opts opts;
300 opts.validate_utf8 = true;
302 // test validate_utf8 valid utf8 strings - note that we only validate the
303 // for utf8 but don't encode non-ascii to unicode so they are retained as is.
304 EXPECT_EQ(folly::json::serialize("a\xc2\x80z", opts), "\"a\xc2\x80z\"");
306 folly::json::serialize("a\xe0\xa0\x80z", opts),
307 "\"a\xe0\xa0\x80z\"");
309 folly::json::serialize("a\xe0\xa0\x80m\xc2\x80z", opts),
310 "\"a\xe0\xa0\x80m\xc2\x80z\"");
312 // test validate_utf8 with invalid utf8
313 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts));
314 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xe0\x80\x80", opts));
318 TEST(Json, ParseNonStringKeys) {
320 EXPECT_EQ("a", parseJson("{\"a\":[]}").items().begin()->first.asString());
322 // check that we don't allow non-string keys as this violates the
323 // strict JSON spec (though it is emitted by the output of
324 // folly::dynamic with operator <<).
325 EXPECT_THROW(parseJson("{1:[]}"), std::runtime_error);
327 // check that we can parse colloquial JSON if the option is set
328 folly::json::serialization_opts opts;
329 opts.allow_non_string_keys = true;
331 auto val = parseJson("{1:[]}", opts);
332 EXPECT_EQ(1, val.items().begin()->first.asInt());
335 // test we can still read in strings
336 auto sval = parseJson("{\"a\":[]}", opts);
337 EXPECT_EQ("a", sval.items().begin()->first.asString());
339 // test we can read in doubles
340 auto dval = parseJson("{1.5:[]}", opts);
341 EXPECT_EQ(1.5, dval.items().begin()->first.asDouble());
344 BENCHMARK(jsonSerialize, iters) {
345 folly::json::serialization_opts opts;
346 for (int i = 0; i < iters; ++i) {
347 folly::json::serialize(
348 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
349 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
350 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
351 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
352 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
353 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
354 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
355 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
356 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
357 "qwerty \xc2\x80 \xef\xbf\xbf poiuy",
362 BENCHMARK(jsonSerializeWithNonAsciiEncoding, iters) {
363 folly::json::serialization_opts opts;
364 opts.encode_non_ascii = true;
366 for (int i = 0; i < iters; ++i) {
367 folly::json::serialize(
368 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
369 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
370 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
371 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
372 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
373 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
374 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
375 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
376 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
377 "qwerty \xc2\x80 \xef\xbf\xbf poiuy",
382 BENCHMARK(jsonSerializeWithUtf8Validation, iters) {
383 folly::json::serialization_opts opts;
384 opts.validate_utf8 = true;
386 for (int i = 0; i < iters; ++i) {
387 folly::json::serialize(
388 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
389 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
390 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
391 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
392 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
393 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
394 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
395 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
396 "qwerty \xc2\x80 \xef\xbf\xbf poiuy"
397 "qwerty \xc2\x80 \xef\xbf\xbf poiuy",
402 BENCHMARK(parseSmallStringWithUtf, iters) {
403 for (int i = 0; i < iters << 4; ++i) {
404 parseJson("\"I \\u2665 UTF-8 thjasdhkjh blah blah blah\"");
408 BENCHMARK(parseNormalString, iters) {
409 for (int i = 0; i < iters << 4; ++i) {
410 parseJson("\"akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk\"");
414 BENCHMARK(parseBigString, iters) {
415 for (int i = 0; i < iters; ++i) {
417 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
418 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
419 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
420 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
421 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
422 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
423 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
424 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
425 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
426 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
427 "akjhfk jhkjlakjhfk jhkjlakjhfk jhkjl akjhfk"
432 int main(int argc, char** argv) {
433 testing::InitGoogleTest(&argc, argv);
434 google::ParseCommandLineFlags(&argc, &argv, true);
435 if (FLAGS_benchmark) {
436 folly::runBenchmarks();
438 return RUN_ALL_TESTS();