2 * Copyright 2015 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include <folly/json.h>
19 #include <gtest/gtest.h>
20 #include <gflags/gflags.h>
22 #include <boost/next_prior.hpp>
25 using folly::parseJson;
29 auto val = parseJson("\"I \u2665 UTF-8\"");
30 EXPECT_EQ("I \u2665 UTF-8", val.asString());
31 val = parseJson("\"I \\u2665 UTF-8\"");
32 EXPECT_EQ("I \u2665 UTF-8", val.asString());
33 val = parseJson("\"I \U0001D11E playing in G-clef\"");
34 EXPECT_EQ("I \U0001D11E playing in G-clef", val.asString());
36 val = parseJson("\"I \\uD834\\uDD1E playing in G-clef\"");
37 EXPECT_EQ("I \U0001D11E playing in G-clef", val.asString());
41 auto num = parseJson("12");
42 EXPECT_TRUE(num.isInt());
44 num = parseJson("12e5");
45 EXPECT_TRUE(num.isDouble());
47 auto numAs1 = num.asDouble();
48 EXPECT_EQ(numAs1, 12e5);
50 EXPECT_EQ(num, 1200000);
52 auto largeNumber = parseJson("4611686018427387904");
53 EXPECT_TRUE(largeNumber.isInt());
54 EXPECT_EQ(largeNumber, 4611686018427387904L);
56 auto negative = parseJson("-123");
57 EXPECT_EQ(negative, -123);
59 auto bfalse = parseJson("false");
60 auto btrue = parseJson("true");
61 EXPECT_EQ(bfalse, false);
62 EXPECT_EQ(btrue, true);
64 auto null = parseJson("null");
65 EXPECT_TRUE(null == nullptr);
67 auto doub1 = parseJson("12.0");
68 auto doub2 = parseJson("12e2");
69 EXPECT_EQ(doub1, 12.0);
70 EXPECT_EQ(doub2, 12e2);
71 EXPECT_EQ(std::numeric_limits<double>::infinity(),
72 parseJson("Infinity").asDouble());
73 EXPECT_EQ(-std::numeric_limits<double>::infinity(),
74 parseJson("-Infinity").asDouble());
75 EXPECT_TRUE(std::isnan(parseJson("NaN").asDouble()));
78 EXPECT_THROW(parseJson("infinity"), std::runtime_error);
79 EXPECT_THROW(parseJson("inf"), std::runtime_error);
80 EXPECT_THROW(parseJson("nan"), std::runtime_error);
82 auto array = parseJson(
83 "[12,false, false , null , [12e4,32, [], 12]]");
84 EXPECT_EQ(array.size(), 5);
85 if (array.size() == 5) {
86 EXPECT_EQ(boost::prior(array.end())->size(), 4);
89 EXPECT_THROW(parseJson("\n[12,\n\nnotvalidjson"),
92 EXPECT_THROW(parseJson("12e2e2"),
95 EXPECT_THROW(parseJson("{\"foo\":12,\"bar\":42} \"something\""),
98 dynamic value = dynamic::object
104 dynamic::object("a", "b")
114 // Print then parse and get the same thing, hopefully.
115 EXPECT_EQ(value, parseJson(toJson(value)));
118 // Test an object with non-string values.
119 dynamic something = parseJson(
120 "{\"old_value\":40,\"changed\":true,\"opened\":false}");
121 dynamic expected = dynamic::object
125 EXPECT_EQ(something, expected);
128 TEST(Json, ParseTrailingComma) {
129 folly::json::serialization_opts on, off;
130 on.allow_trailing_comma = true;
131 off.allow_trailing_comma = false;
133 dynamic arr { 1, 2 };
134 EXPECT_EQ(arr, parseJson("[1, 2]", on));
135 EXPECT_EQ(arr, parseJson("[1, 2,]", on));
136 EXPECT_EQ(arr, parseJson("[1, 2, ]", on));
137 EXPECT_EQ(arr, parseJson("[1, 2 , ]", on));
138 EXPECT_EQ(arr, parseJson("[1, 2 ,]", on));
139 EXPECT_THROW(parseJson("[1, 2,]", off), std::runtime_error);
141 dynamic obj = dynamic::object("a", 1);
142 EXPECT_EQ(obj, parseJson("{\"a\": 1}", on));
143 EXPECT_EQ(obj, parseJson("{\"a\": 1,}", on));
144 EXPECT_EQ(obj, parseJson("{\"a\": 1, }", on));
145 EXPECT_EQ(obj, parseJson("{\"a\": 1 , }", on));
146 EXPECT_EQ(obj, parseJson("{\"a\": 1 ,}", on));
147 EXPECT_THROW(parseJson("{\"a\":1,}", off), std::runtime_error);
150 TEST(Json, JavascriptSafe) {
151 auto badDouble = (1ll << 63ll) + 1;
152 dynamic badDyn = badDouble;
153 EXPECT_EQ(folly::toJson(badDouble), folly::to<folly::fbstring>(badDouble));
154 folly::json::serialization_opts opts;
155 opts.javascript_safe = true;
156 EXPECT_ANY_THROW(folly::json::serialize(badDouble, opts));
158 auto okDouble = 1ll << 63ll;
159 dynamic okDyn = okDouble;
160 EXPECT_EQ(folly::toJson(okDouble), folly::to<folly::fbstring>(okDouble));
163 TEST(Json, Produce) {
164 auto value = parseJson(R"( "f\"oo" )");
165 EXPECT_EQ(toJson(value), R"("f\"oo")");
166 value = parseJson("\"Control code: \001 \002 \x1f\"");
167 EXPECT_EQ(toJson(value), R"("Control code: \u0001 \u0002 \u001f")");
169 // We're not allowed to have non-string keys in json.
170 EXPECT_THROW(toJson(dynamic::object("abc", "xyz")(42.33, "asd")),
174 TEST(Json, JsonEscape) {
175 folly::json::serialization_opts opts;
177 folly::json::serialize("\b\f\n\r\x01\t\\\"/\v\a", opts),
178 R"("\b\f\n\r\u0001\t\\\"/\u000b\u0007")");
181 TEST(Json, JsonNonAsciiEncoding) {
182 folly::json::serialization_opts opts;
183 opts.encode_non_ascii = true;
186 EXPECT_EQ(folly::json::serialize("\x1f", opts), R"("\u001f")");
187 EXPECT_EQ(folly::json::serialize("\xc2\xa2", opts), R"("\u00a2")");
188 EXPECT_EQ(folly::json::serialize("\xe2\x82\xac", opts), R"("\u20ac")");
190 // multiple unicode encodings
192 folly::json::serialize("\x1f\xe2\x82\xac", opts),
193 R"("\u001f\u20ac")");
195 folly::json::serialize("\x1f\xc2\xa2\xe2\x82\xac", opts),
196 R"("\u001f\u00a2\u20ac")");
198 folly::json::serialize("\xc2\x80\xef\xbf\xbf", opts),
199 R"("\u0080\uffff")");
201 folly::json::serialize("\xe0\xa0\x80\xdf\xbf", opts),
202 R"("\u0800\u07ff")");
204 // first possible sequence of a certain length
205 EXPECT_EQ(folly::json::serialize("\xc2\x80", opts), R"("\u0080")");
206 EXPECT_EQ(folly::json::serialize("\xe0\xa0\x80", opts), R"("\u0800")");
208 // last possible sequence of a certain length
209 EXPECT_EQ(folly::json::serialize("\xdf\xbf", opts), R"("\u07ff")");
210 EXPECT_EQ(folly::json::serialize("\xef\xbf\xbf", opts), R"("\uffff")");
212 // other boundary conditions
213 EXPECT_EQ(folly::json::serialize("\xed\x9f\xbf", opts), R"("\ud7ff")");
214 EXPECT_EQ(folly::json::serialize("\xee\x80\x80", opts), R"("\ue000")");
215 EXPECT_EQ(folly::json::serialize("\xef\xbf\xbd", opts), R"("\ufffd")");
217 // incomplete sequences
218 EXPECT_ANY_THROW(folly::json::serialize("a\xed\x9f", opts));
219 EXPECT_ANY_THROW(folly::json::serialize("b\xee\x80", opts));
220 EXPECT_ANY_THROW(folly::json::serialize("c\xef\xbf", opts));
223 EXPECT_ANY_THROW(folly::json::serialize("\xfe", opts));
224 EXPECT_ANY_THROW(folly::json::serialize("\xff", opts));
226 // Sample overlong sequences
227 EXPECT_ANY_THROW(folly::json::serialize("\xc0\xaf", opts));
228 EXPECT_ANY_THROW(folly::json::serialize("\xe0\x80\xaf", opts));
230 // Maximum overlong sequences
231 EXPECT_ANY_THROW(folly::json::serialize("\xc1\xbf", opts));
232 EXPECT_ANY_THROW(folly::json::serialize("\x30\x9f\xbf", opts));
234 // illegal code positions
235 EXPECT_ANY_THROW(folly::json::serialize("\xed\xa0\x80", opts));
236 EXPECT_ANY_THROW(folly::json::serialize("\xed\xbf\xbf", opts));
238 // Overlong representation of NUL character
239 EXPECT_ANY_THROW(folly::json::serialize("\xc0\x80", opts));
240 EXPECT_ANY_THROW(folly::json::serialize("\xe0\x80\x80", opts));
242 // Longer than 3 byte encodings
243 EXPECT_ANY_THROW(folly::json::serialize("\xf4\x8f\xbf\xbf", opts));
244 EXPECT_ANY_THROW(folly::json::serialize("\xed\xaf\xbf\xed\xbf\xbf", opts));
247 TEST(Json, UTF8Retention) {
249 // test retention with valid utf8 strings
250 folly::fbstring input = "\u2665";
251 folly::fbstring jsonInput = folly::toJson(input);
252 folly::fbstring output = folly::parseJson(jsonInput).asString();
253 folly::fbstring jsonOutput = folly::toJson(output);
255 EXPECT_EQ(input, output);
256 EXPECT_EQ(jsonInput, jsonOutput);
258 // test retention with invalid utf8 - note that non-ascii chars are retained
259 // as is, and no unicode encoding is attempted so no exception is thrown.
261 folly::toJson("a\xe0\xa0\x80z\xc0\x80"),
262 "\"a\xe0\xa0\x80z\xc0\x80\""
266 TEST(Json, UTF8EncodeNonAsciiRetention) {
268 folly::json::serialization_opts opts;
269 opts.encode_non_ascii = true;
271 // test encode_non_ascii valid utf8 strings
272 folly::fbstring input = "\u2665";
273 folly::fbstring jsonInput = folly::json::serialize(input, opts);
274 folly::fbstring output = folly::parseJson(jsonInput).asString();
275 folly::fbstring jsonOutput = folly::json::serialize(output, opts);
277 EXPECT_EQ(input, output);
278 EXPECT_EQ(jsonInput, jsonOutput);
280 // test encode_non_ascii with invalid utf8 - note that an attempt to encode
281 // non-ascii to unicode will result is a utf8 validation and throw exceptions.
282 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts));
283 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xe0\x80\x80", opts));
286 TEST(Json, UTF8Validation) {
287 folly::json::serialization_opts opts;
288 opts.validate_utf8 = true;
290 // test validate_utf8 valid utf8 strings - note that we only validate the
291 // for utf8 but don't encode non-ascii to unicode so they are retained as is.
292 EXPECT_EQ(folly::json::serialize("a\xc2\x80z", opts), "\"a\xc2\x80z\"");
294 folly::json::serialize("a\xe0\xa0\x80z", opts),
295 "\"a\xe0\xa0\x80z\"");
297 folly::json::serialize("a\xe0\xa0\x80m\xc2\x80z", opts),
298 "\"a\xe0\xa0\x80m\xc2\x80z\"");
300 // test validate_utf8 with invalid utf8
301 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts));
302 EXPECT_ANY_THROW(folly::json::serialize("a\xe0\xa0\x80z\xe0\x80\x80", opts));
304 opts.skip_invalid_utf8 = true;
305 EXPECT_EQ(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts),
306 "\"a\xe0\xa0\x80z\ufffd\ufffd\"");
307 EXPECT_EQ(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80\x80", opts),
308 "\"a\xe0\xa0\x80z\ufffd\ufffd\ufffd\"");
309 EXPECT_EQ(folly::json::serialize("z\xc0\x80z\xe0\xa0\x80", opts),
310 "\"z\ufffd\ufffdz\xe0\xa0\x80\"");
312 opts.encode_non_ascii = true;
313 EXPECT_EQ(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80", opts),
314 "\"a\\u0800z\\ufffd\\ufffd\"");
315 EXPECT_EQ(folly::json::serialize("a\xe0\xa0\x80z\xc0\x80\x80", opts),
316 "\"a\\u0800z\\ufffd\\ufffd\\ufffd\"");
317 EXPECT_EQ(folly::json::serialize("z\xc0\x80z\xe0\xa0\x80", opts),
318 "\"z\\ufffd\\ufffdz\\u0800\"");
323 TEST(Json, ParseNonStringKeys) {
325 EXPECT_EQ("a", parseJson("{\"a\":[]}").items().begin()->first.asString());
327 // check that we don't allow non-string keys as this violates the
328 // strict JSON spec (though it is emitted by the output of
329 // folly::dynamic with operator <<).
330 EXPECT_THROW(parseJson("{1:[]}"), std::runtime_error);
332 // check that we can parse colloquial JSON if the option is set
333 folly::json::serialization_opts opts;
334 opts.allow_non_string_keys = true;
336 auto val = parseJson("{1:[]}", opts);
337 EXPECT_EQ(1, val.items().begin()->first.asInt());
340 // test we can still read in strings
341 auto sval = parseJson("{\"a\":[]}", opts);
342 EXPECT_EQ("a", sval.items().begin()->first.asString());
344 // test we can read in doubles
345 auto dval = parseJson("{1.5:[]}", opts);
346 EXPECT_EQ(1.5, dval.items().begin()->first.asDouble());
349 TEST(Json, SortKeys) {
350 folly::json::serialization_opts opts_on, opts_off;
351 opts_on.sort_keys = true;
352 opts_off.sort_keys = false;
354 dynamic value = dynamic::object
360 dynamic::object("a", "b")
370 std::string sorted_keys =
371 R"({"a":[{"a":"b","c":"d"},12.5,"Yo Dawg",["heh"],null],)"
372 R"("another":32.2,"foo":"bar","junk":12})";
374 EXPECT_EQ(value, parseJson(folly::json::serialize(value, opts_on)));
375 EXPECT_EQ(value, parseJson(folly::json::serialize(value, opts_off)));
377 EXPECT_EQ(sorted_keys, folly::json::serialize(value, opts_on));
380 int main(int argc, char** argv) {
381 testing::InitGoogleTest(&argc, argv);
382 gflags::ParseCommandLineFlags(&argc, &argv, true);
383 return RUN_ALL_TESTS();