2 * Copyright 2017 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include <folly/json.h>
19 #include <boost/next_prior.hpp>
20 #include <boost/algorithm/string.hpp>
22 #include <folly/Conv.h>
23 #include <folly/Portability.h>
24 #include <folly/Range.h>
25 #include <folly/String.h>
26 #include <folly/Unicode.h>
27 #include <folly/portability/Constexpr.h>
31 //////////////////////////////////////////////////////////////////////
39 unsigned* indentLevel,
40 serialization_opts const* opts)
41 : out_(out), indentLevel_(indentLevel), opts_(*opts) {}
43 void operator()(dynamic const& v) const {
46 if (!opts_.allow_nan_inf &&
47 (std::isnan(v.asDouble()) || std::isinf(v.asDouble()))) {
48 throw std::runtime_error("folly::toJson: JSON object value was a "
51 toAppend(v.asDouble(), &out_, opts_.double_mode, opts_.double_num_digits);
53 case dynamic::INT64: {
54 auto intval = v.asInt();
55 if (opts_.javascript_safe) {
56 // Use folly::to to check that this integer can be represented
57 // as a double without loss of precision.
58 intval = int64_t(to<double>(intval));
60 toAppend(intval, &out_);
64 out_ += v.asBool() ? "true" : "false";
70 escapeString(v.asString(), out_, opts_);
79 CHECK(0) << "Bad type " << v.type();
84 void printKV(const std::pair<const dynamic, dynamic>& p) const {
85 if (!opts_.allow_non_string_keys && !p.first.isString()) {
86 throw std::runtime_error("folly::toJson: JSON object key was not a "
94 template <typename Iterator>
95 void printKVPairs(Iterator begin, Iterator end) const {
97 for (++begin; begin != end; ++begin) {
104 void printObject(dynamic const& o) const {
113 if (opts_.sort_keys) {
114 std::vector<std::pair<dynamic, dynamic>> items(
115 o.items().begin(), o.items().end());
116 std::sort(items.begin(), items.end(), [](auto const& a, auto const& b) {
117 // Only compare keys. No ordering among identical keys.
118 return a.first < b.first;
120 printKVPairs(items.begin(), items.end());
122 printKVPairs(o.items().begin(), o.items().end());
129 void printArray(dynamic const& a) const {
139 for (auto& val : range(boost::next(a.begin()), a.end())) {
150 void outdent() const {
156 void indent() const {
162 void newline() const {
164 out_ += to<std::string>('\n', std::string(*indentLevel_ * 2, ' '));
168 void mapColon() const {
169 out_ += indentLevel_ ? " : " : ":";
174 unsigned* const indentLevel_;
175 serialization_opts const& opts_;
178 //////////////////////////////////////////////////////////////////////
180 struct ParseError : std::runtime_error {
181 explicit ParseError(int line)
182 : std::runtime_error(to<std::string>("json parse error on line ", line))
185 explicit ParseError(int line, std::string const& context,
186 std::string const& expected)
187 : std::runtime_error(to<std::string>("json parse error on line ", line,
188 !context.empty() ? to<std::string>(" near `", context, '\'')
193 explicit ParseError(std::string const& msg)
194 : std::runtime_error("json parse error: " + msg)
198 // Wraps our input buffer with some helper functions.
200 explicit Input(StringPiece range, json::serialization_opts const* opts)
208 Input(Input const&) = delete;
209 Input& operator=(Input const&) = delete;
211 char const* begin() const { return range_.begin(); }
213 // Parse ahead for as long as the supplied predicate is satisfied,
214 // returning a range of what was skipped.
215 template<class Predicate>
216 StringPiece skipWhile(const Predicate& p) {
217 std::size_t skipped = 0;
218 for (; skipped < range_.size(); ++skipped) {
219 if (!p(range_[skipped])) {
222 if (range_[skipped] == '\n') {
226 auto ret = range_.subpiece(0, skipped);
227 range_.advance(skipped);
232 StringPiece skipDigits() {
233 return skipWhile([] (char c) { return c >= '0' && c <= '9'; });
236 StringPiece skipMinusAndDigits() {
237 bool firstChar = true;
238 return skipWhile([&firstChar] (char c) {
239 bool result = (c >= '0' && c <= '9') || (firstChar && c == '-');
245 void skipWhitespace() {
246 range_ = folly::skipWhitespace(range_);
250 void expect(char c) {
252 throw ParseError(lineNum_, context(),
253 to<std::string>("expected '", c, '\''));
258 std::size_t size() const {
259 return range_.size();
262 int operator*() const {
274 return to<T>(&range_);
275 } catch (std::exception const& e) {
280 bool consume(StringPiece str) {
281 if (boost::starts_with(range_, str)) {
282 range_.advance(str.size());
289 std::string context() const {
290 return range_.subpiece(0, 16 /* arbitrary */).toString();
293 dynamic error(char const* what) const {
294 throw ParseError(lineNum_, context(), what);
297 json::serialization_opts const& getOpts() {
301 void incrementRecursionLevel() {
302 if (currentRecursionLevel_ > opts_.recursion_limit) {
303 error("recursion limit exceeded");
305 currentRecursionLevel_++;
308 void decrementRecursionLevel() {
309 currentRecursionLevel_--;
313 void storeCurrent() {
314 current_ = range_.empty() ? EOF : range_.front();
319 json::serialization_opts const& opts_;
322 unsigned int currentRecursionLevel_{0};
325 class RecursionGuard {
327 explicit RecursionGuard(Input& in) : in_(in) {
328 in_.incrementRecursionLevel();
332 in_.decrementRecursionLevel();
339 dynamic parseValue(Input& in);
340 std::string parseString(Input& in);
341 dynamic parseNumber(Input& in);
343 dynamic parseObject(Input& in) {
347 dynamic ret = dynamic::object;
356 if (in.getOpts().allow_trailing_comma && *in == '}') {
359 if (*in == '\"') { // string
360 auto key = parseString(in);
364 ret.insert(std::move(key), parseValue(in));
365 } else if (!in.getOpts().allow_non_string_keys) {
366 in.error("expected string for object key name");
368 auto key = parseValue(in);
372 ret.insert(std::move(key), parseValue(in));
387 dynamic parseArray(Input& in) {
391 dynamic ret = dynamic::array;
400 if (in.getOpts().allow_trailing_comma && *in == ']') {
403 ret.push_back(parseValue(in));
416 dynamic parseNumber(Input& in) {
417 bool const negative = (*in == '-');
418 if (negative && in.consume("-Infinity")) {
419 if (in.getOpts().parse_numbers_as_strings) {
422 return -std::numeric_limits<double>::infinity();
426 auto integral = in.skipMinusAndDigits();
427 if (negative && integral.size() < 2) {
428 in.error("expected digits after `-'");
431 auto const wasE = *in == 'e' || *in == 'E';
433 constexpr const char* maxInt = "9223372036854775807";
434 constexpr const char* minInt = "-9223372036854775808";
435 constexpr auto maxIntLen = constexpr_strlen(maxInt);
436 constexpr auto minIntLen = constexpr_strlen(minInt);
438 if (*in != '.' && !wasE && in.getOpts().parse_numbers_as_strings) {
442 if (*in != '.' && !wasE) {
443 if (LIKELY(!in.getOpts().double_fallback || integral.size() < maxIntLen) ||
444 (!negative && integral.size() == maxIntLen && integral <= maxInt) ||
445 (negative && integral.size() == minIntLen && integral <= minInt)) {
446 auto val = to<int64_t>(integral);
450 auto val = to<double>(integral);
456 auto end = !wasE ? (++in, in.skipDigits().end()) : in.begin();
457 if (*in == 'e' || *in == 'E') {
459 if (*in == '+' || *in == '-') {
462 auto expPart = in.skipDigits();
465 auto fullNum = range(integral.begin(), end);
466 if (in.getOpts().parse_numbers_as_strings) {
469 auto val = to<double>(fullNum);
473 std::string decodeUnicodeEscape(Input& in) {
474 auto hexVal = [&] (int c) -> uint16_t {
476 c >= '0' && c <= '9' ? c - '0' :
477 c >= 'a' && c <= 'f' ? c - 'a' + 10 :
478 c >= 'A' && c <= 'F' ? c - 'A' + 10 :
479 (in.error("invalid hex digit"), 0));
482 auto readHex = [&]() -> uint16_t {
484 in.error("expected 4 hex digits");
487 uint16_t ret = hexVal(*in) * 4096;
489 ret += hexVal(*in) * 256;
491 ret += hexVal(*in) * 16;
499 * If the value encoded is in the surrogate pair range, we need to
500 * make sure there is another escape that we can use also.
502 uint32_t codePoint = readHex();
503 if (codePoint >= 0xd800 && codePoint <= 0xdbff) {
504 if (!in.consume("\\u")) {
505 in.error("expected another unicode escape for second half of "
508 uint16_t second = readHex();
509 if (second >= 0xdc00 && second <= 0xdfff) {
510 codePoint = 0x10000 + ((codePoint & 0x3ff) << 10) +
513 in.error("second character in surrogate pair is invalid");
515 } else if (codePoint >= 0xdc00 && codePoint <= 0xdfff) {
516 in.error("invalid unicode code point (in range [0xdc00,0xdfff])");
519 return codePointToUtf8(codePoint);
522 std::string parseString(Input& in) {
528 auto range = in.skipWhile(
529 [] (char c) { return c != '\"' && c != '\\'; }
531 ret.append(range.begin(), range.end());
540 case '\"': ret.push_back('\"'); ++in; break;
541 case '\\': ret.push_back('\\'); ++in; break;
542 case '/': ret.push_back('/'); ++in; break;
543 case 'b': ret.push_back('\b'); ++in; break;
544 case 'f': ret.push_back('\f'); ++in; break;
545 case 'n': ret.push_back('\n'); ++in; break;
546 case 'r': ret.push_back('\r'); ++in; break;
547 case 't': ret.push_back('\t'); ++in; break;
548 case 'u': ++in; ret += decodeUnicodeEscape(in); break;
550 in.error(to<std::string>("unknown escape ", *in, " in string").c_str());
555 in.error("unterminated string");
559 * Apparently we're actually supposed to ban all control
560 * characters from strings. This seems unnecessarily
561 * restrictive, so we're only banning zero bytes. (Since the
562 * string is presumed to be UTF-8 encoded it's fine to just
565 in.error("null byte in string");
568 ret.push_back(char(*in));
575 dynamic parseValue(Input& in) {
576 RecursionGuard guard(in);
579 return *in == '[' ? parseArray(in) :
580 *in == '{' ? parseObject(in) :
581 *in == '\"' ? parseString(in) :
582 (*in == '-' || (*in >= '0' && *in <= '9')) ? parseNumber(in) :
583 in.consume("true") ? true :
584 in.consume("false") ? false :
585 in.consume("null") ? nullptr :
586 in.consume("Infinity") ?
587 (in.getOpts().parse_numbers_as_strings ? (dynamic)"Infinity" :
588 (dynamic)std::numeric_limits<double>::infinity()) :
590 (in.getOpts().parse_numbers_as_strings ? (dynamic)"NaN" :
591 (dynamic)std::numeric_limits<double>::quiet_NaN()) :
592 in.error("expected json value");
597 //////////////////////////////////////////////////////////////////////
599 std::string serialize(dynamic const& dyn, serialization_opts const& opts) {
601 unsigned indentLevel = 0;
602 Printer p(ret, opts.pretty_formatting ? &indentLevel : nullptr, &opts);
607 // Escape a string so that it is legal to print it in JSON text.
611 const serialization_opts& opts) {
612 auto hexDigit = [] (uint8_t c) -> char {
613 return c < 10 ? c + '0' : c - 10 + 'a';
618 auto* p = reinterpret_cast<const unsigned char*>(input.begin());
619 auto* q = reinterpret_cast<const unsigned char*>(input.begin());
620 auto* e = reinterpret_cast<const unsigned char*>(input.end());
623 // Since non-ascii encoding inherently does utf8 validation
624 // we explicitly validate utf8 only if non-ascii encoding is disabled.
625 if ((opts.validate_utf8 || opts.skip_invalid_utf8)
626 && !opts.encode_non_ascii) {
627 // to achieve better spatial and temporal coherence
628 // we do utf8 validation progressively along with the
629 // string-escaping instead of two separate passes
631 // as the encoding progresses, q will stay at or ahead of p
634 // as p catches up with q, move q forward
636 // calling utf8_decode has the side effect of
637 // checking that utf8 encodings are valid
638 char32_t v = utf8ToCodePoint(q, e, opts.skip_invalid_utf8);
639 if (opts.skip_invalid_utf8 && v == U'\ufffd') {
640 out.append(u8"\ufffd");
646 if (opts.encode_non_ascii && (*p & 0x80)) {
647 // note that this if condition captures utf8 chars
648 // with value > 127, so size > 1 byte
649 char32_t v = utf8ToCodePoint(p, e, opts.skip_invalid_utf8);
651 out.push_back(hexDigit(uint8_t(v >> 12)));
652 out.push_back(hexDigit((v >> 8) & 0x0f));
653 out.push_back(hexDigit((v >> 4) & 0x0f));
654 out.push_back(hexDigit(v & 0x0f));
655 } else if (*p == '\\' || *p == '\"') {
657 out.push_back(char(*p++));
658 } else if (*p <= 0x1f) {
660 case '\b': out.append("\\b"); p++; break;
661 case '\f': out.append("\\f"); p++; break;
662 case '\n': out.append("\\n"); p++; break;
663 case '\r': out.append("\\r"); p++; break;
664 case '\t': out.append("\\t"); p++; break;
666 // note that this if condition captures non readable chars
667 // with value < 32, so size = 1 byte (e.g control chars).
669 out.push_back(hexDigit((*p & 0xf0) >> 4));
670 out.push_back(hexDigit(*p & 0xf));
681 std::string stripComments(StringPiece jsonC) {
688 } state = State::None;
690 for (size_t i = 0; i < jsonC.size(); ++i) {
691 auto s = jsonC.subpiece(i);
694 if (s.startsWith("/*")) {
695 state = State::InlineComment;
698 } else if (s.startsWith("//")) {
699 state = State::LineComment;
702 } else if (s[0] == '\"') {
703 state = State::InString;
705 result.push_back(s[0]);
707 case State::InString:
709 if (UNLIKELY(s.size() == 1)) {
710 throw std::logic_error("Invalid JSONC: string is not terminated");
712 result.push_back(s[0]);
713 result.push_back(s[1]);
716 } else if (s[0] == '\"') {
719 result.push_back(s[0]);
721 case State::InlineComment:
722 if (s.startsWith("*/")) {
727 case State::LineComment:
729 // skip the line break. It doesn't matter.
734 throw std::logic_error("Unknown comment state");
742 //////////////////////////////////////////////////////////////////////
744 dynamic parseJson(StringPiece range) {
745 return parseJson(range, json::serialization_opts());
750 json::serialization_opts const& opts) {
752 json::Input in(range, &opts);
754 auto ret = parseValue(in);
756 if (in.size() && *in != '\0') {
757 in.error("parsing didn't consume all input");
762 std::string toJson(dynamic const& dyn) {
763 return json::serialize(dyn, json::serialization_opts());
766 std::string toPrettyJson(dynamic const& dyn) {
767 json::serialization_opts opts;
768 opts.pretty_formatting = true;
769 return json::serialize(dyn, opts);
772 //////////////////////////////////////////////////////////////////////
773 // dynamic::print_as_pseudo_json() is implemented here for header
774 // ordering reasons (most of the dynamic implementation is in
775 // dynamic-inl.h, which we don't want to include json.h).
777 void dynamic::print_as_pseudo_json(std::ostream& out) const {
778 json::serialization_opts opts;
779 opts.allow_non_string_keys = true;
780 opts.allow_nan_inf = true;
781 out << json::serialize(*this, opts);
784 void PrintTo(const dynamic& dyn, std::ostream* os) {
785 json::serialization_opts opts;
786 opts.allow_nan_inf = true;
787 opts.allow_non_string_keys = true;
788 opts.pretty_formatting = true;
789 opts.sort_keys = true;
790 *os << json::serialize(dyn, opts);
793 //////////////////////////////////////////////////////////////////////