X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=folly%2Fjson.cpp;h=105b1ca7ff9264574fdd1ee0c9fb024ece10fa0b;hb=e33794f7c6728fc0748d3241a27316e62c4c1e35;hp=bf5320d27559e301404607d7f36de5c1b3b911f2;hpb=27494a20393fa45072e7d526d358835f3abe312a;p=folly.git diff --git a/folly/json.cpp b/folly/json.cpp index bf5320d2..105b1ca7 100644 --- a/folly/json.cpp +++ b/folly/json.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2012 Facebook, Inc. + * Copyright 2016 Facebook, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,17 @@ * limitations under the License. */ -#include "folly/json.h" +#include #include #include #include -#include "folly/Range.h" -#include "folly/Unicode.h" -#include "folly/Conv.h" +#include +#include +#include +#include +#include +#include namespace folly { @@ -30,7 +33,10 @@ namespace folly { namespace json { namespace { -char32_t decodeUtf8(const char*& p, const char* const e) { +char32_t decodeUtf8( + const unsigned char*& p, + const unsigned char* const e, + bool skipOnError) { /* The following encodings are valid, except for the 5 and 6 byte * combinations: * 0xxxxxxx @@ -41,7 +47,10 @@ char32_t decodeUtf8(const char*& p, const char* const e) { * 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx */ + auto skip = [&] { ++p; return U'\ufffd'; }; + if (p >= e) { + if (skipOnError) return skip(); throw std::runtime_error("folly::decodeUtf8 empty/invalid string"); } @@ -62,8 +71,8 @@ char32_t decodeUtf8(const char*& p, const char* const e) { uint32_t d = fst; if ((fst & 0xC0) != 0xC0) { - throw std::runtime_error( - to("folly::decodeUtf8 i=0 d=", d)); + if (skipOnError) return skip(); + throw std::runtime_error(to("folly::decodeUtf8 i=0 d=", d)); } fst <<= 1; @@ -72,6 +81,7 @@ char32_t decodeUtf8(const char*& p, const char* const e) { unsigned char tmp = p[i]; if ((tmp & 0xC0) != 0x80) { + if (skipOnError) return skip(); throw std::runtime_error( to("folly::decodeUtf8 i=", i, " tmp=", (uint32_t)tmp)); } @@ -84,6 +94,7 @@ char32_t decodeUtf8(const char*& p, const char* const e) { // overlong, could have been encoded with i bytes if ((d & ~bitMask[i - 1]) == 0) { + if (skipOnError) return skip(); throw std::runtime_error( to("folly::decodeUtf8 i=", i, " d=", d)); } @@ -91,6 +102,7 @@ char32_t decodeUtf8(const char*& p, const char* const e) { // check for surrogates only needed for 3 bytes if (i == 2) { if ((d >= 0xD800 && d <= 0xDFFF) || d > 0x10FFFF) { + if (skipOnError) return skip(); throw std::runtime_error( to("folly::decodeUtf8 i=", i, " d=", d)); } @@ -101,85 +113,26 @@ char32_t decodeUtf8(const char*& p, const char* const e) { } } + if (skipOnError) return skip(); throw std::runtime_error("folly::decodeUtf8 encoding length maxed out"); } -// Escape a string so that it is legal to print it in JSON text. -void escapeString(StringPiece input, - fbstring& out, - const serialization_opts& opts) { - auto hexDigit = [] (int c) -> char { - return c < 10 ? c + '0' : c - 10 + 'a'; - }; - - out.reserve(out.size() + input.size() + 2); - out.push_back('\"'); - - const char* p = input.begin(); - const char* q = input.begin(); - const char* const e = input.end(); - - while (p < e) { - // Since non-ascii encoding inherently does utf8 validation - // we explicitly validate utf8 only if non-ascii encoding is disabled. - if (opts.validate_utf8 && !opts.encode_non_ascii) { - // to achieve better spatial and temporal coherence - // we do utf8 validation progressively along with the - // string-escaping instead of two separate passes - - // as the encoding progresses, q will stay at or ahead of p - CHECK(q >= p); - - // as p catches up with q, move q forward - if (q == p) { - // calling utf8_decode has the side effect of - // checking that utf8 encodings are valid - decodeUtf8(q, e); - } - } - - if (opts.encode_non_ascii && (*p & 0x80)) { - char32_t v = decodeUtf8(p, e); - out.append("\\u"); - out.push_back(hexDigit(v >> 12)); - out.push_back(hexDigit((v >> 8) & 0x0f)); - out.push_back(hexDigit((v >> 4) & 0x0f)); - out.push_back(hexDigit(v & 0x0f)); - continue; - } - if (*p == '\\' || *p == '\"') { - out.push_back('\\'); - out.push_back(*p++); - continue; - } - if (*p <= 0x1f) { - // note that this if condition captures both control characters - // and extended ascii characters - out.append("\\u00"); - out.push_back(hexDigit((*p & 0xf0) >> 4)); - out.push_back(hexDigit(*p & 0xf)); - p++; - continue; - } - out.push_back(*p++); - } - - out.push_back('\"'); -} - struct Printer { - explicit Printer(fbstring& out, - unsigned* indentLevel, - serialization_opts const* opts) - : out_(out) - , indentLevel_(indentLevel) - , opts_(*opts) - {} + explicit Printer( + std::string& out, + unsigned* indentLevel, + serialization_opts const* opts) + : out_(out), indentLevel_(indentLevel), opts_(*opts) {} void operator()(dynamic const& v) const { switch (v.type()) { case dynamic::DOUBLE: - toAppend(v.asDouble(), &out_); + if (!opts_.allow_nan_inf && + (std::isnan(v.asDouble()) || std::isinf(v.asDouble()))) { + throw std::runtime_error("folly::toJson: JSON object value was a " + "NaN or INF"); + } + toAppend(v.asDouble(), &out_, opts_.double_mode, opts_.double_num_digits); break; case dynamic::INT64: { auto intval = v.asInt(); @@ -212,7 +165,7 @@ struct Printer { } private: - void printKV(const std::pair& p) const { + void printKV(const std::pair& p) const { if (!opts_.allow_non_string_keys && !p.first.isString()) { throw std::runtime_error("folly::toJson: JSON object key was not a " "string"); @@ -222,6 +175,16 @@ private: (*this)(p.second); } + template + void printKVPairs(Iterator begin, Iterator end) const { + printKV(*begin); + for (++begin; begin != end; ++begin) { + out_ += ','; + newline(); + printKV(*begin); + } + } + void printObject(dynamic const& o) const { if (o.empty()) { out_ += "{}"; @@ -231,12 +194,13 @@ private: out_ += '{'; indent(); newline(); - auto it = o.items().begin(); - printKV(*it); - for (++it; it != o.items().end(); ++it) { - out_ += ','; - newline(); - printKV(*it); + if (opts_.sort_keys) { + std::vector> items( + o.items().begin(), o.items().end()); + std::sort(items.begin(), items.end()); + printKVPairs(items.begin(), items.end()); + } else { + printKVPairs(o.items().begin(), o.items().end()); } outdent(); newline(); @@ -253,7 +217,7 @@ private: indent(); newline(); (*this)(a[0]); - for (auto& val : makeRange(boost::next(a.begin()), a.end())) { + for (auto& val : range(boost::next(a.begin()), a.end())) { out_ += ','; newline(); (*this)(val); @@ -278,7 +242,7 @@ private: void newline() const { if (indentLevel_) { - out_ += to('\n', fbstring(*indentLevel_ * 2, ' ')); + out_ += to('\n', std::string(*indentLevel_ * 2, ' ')); } } @@ -287,36 +251,37 @@ private: } private: - fbstring& out_; - unsigned* const indentLevel_; - serialization_opts const& opts_; + std::string& out_; + unsigned* const indentLevel_; + serialization_opts const& opts_; }; -////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////// -struct ParseError : std::runtime_error { - explicit ParseError(int line) - : std::runtime_error(to("json parse error on line ", line)) - {} - - explicit ParseError(int line, std::string const& context, - std::string const& expected) - : std::runtime_error(to("json parse error on line ", line, - !context.empty() ? to(" near `", context, '\'') - : "", - ": ", expected)) - {} - - explicit ParseError(std::string const& what) - : std::runtime_error("json parse error: " + what) - {} -}; + struct ParseError : std::runtime_error { + explicit ParseError(int line) + : std::runtime_error(to("json parse error on line ", line)) + {} + + explicit ParseError(int line, std::string const& context, + std::string const& expected) + : std::runtime_error(to("json parse error on line ", line, + !context.empty() ? to(" near `", context, '\'') + : "", + ": ", expected)) + {} + + explicit ParseError(std::string const& msg) + : std::runtime_error("json parse error: " + msg) + {} + }; // Wraps our input buffer with some helper functions. struct Input { - explicit Input(StringPiece range) - : range_(range) - , lineNum_(0) + explicit Input(StringPiece range, json::serialization_opts const* opts) + : range_(range) + , opts_(*opts) + , lineNum_(0) { storeCurrent(); } @@ -349,21 +314,17 @@ struct Input { return skipWhile([] (char c) { return c >= '0' && c <= '9'; }); } - void skipWhitespace() { - // Spaces other than ' ' characters are less common but should be - // checked. This configuration where we loop on the ' ' - // separately from oddspaces was empirically fastest. - auto oddspace = [] (char c) { - return c == '\n' || c == '\t' || c == '\r'; - }; + StringPiece skipMinusAndDigits() { + bool firstChar = true; + return skipWhile([&firstChar] (char c) { + bool result = (c >= '0' && c <= '9') || (firstChar && c == '-'); + firstChar = false; + return result; + }); + } - loop: - for (; !range_.empty() && range_.front() == ' '; range_.pop_front()) { - } - if (!range_.empty() && oddspace(range_.front())) { - range_.pop_front(); - goto loop; - } + void skipWhitespace() { + range_ = folly::skipWhitespace(range_); storeCurrent(); } @@ -414,19 +375,51 @@ struct Input { throw ParseError(lineNum_, context(), what); } -private: + json::serialization_opts const& getOpts() { + return opts_; + } + + void incrementRecursionLevel() { + if (currentRecursionLevel_ > opts_.recursion_limit) { + error("recursion limit exceeded"); + } + currentRecursionLevel_++; + } + + void decrementRecursionLevel() { + currentRecursionLevel_--; + } + + private: void storeCurrent() { current_ = range_.empty() ? EOF : range_.front(); } private: StringPiece range_; + json::serialization_opts const& opts_; unsigned lineNum_; int current_; + unsigned int currentRecursionLevel_{0}; +}; + +class RecursionGuard { + public: + explicit RecursionGuard(Input& in) : in_(in) { + in_.incrementRecursionLevel(); + } + + ~RecursionGuard() { + in_.decrementRecursionLevel(); + } + + private: + Input& in_; }; dynamic parseValue(Input& in); -fbstring parseString(Input& in); +std::string parseString(Input& in); +dynamic parseNumber(Input& in); dynamic parseObject(Input& in) { assert(*in == '{'); @@ -441,14 +434,25 @@ dynamic parseObject(Input& in) { } for (;;) { - if (*in != '\"') { + if (in.getOpts().allow_trailing_comma && *in == '}') { + break; + } + if (*in == '\"') { // string + auto key = parseString(in); + in.skipWhitespace(); + in.expect(':'); + in.skipWhitespace(); + ret.insert(std::move(key), parseValue(in)); + } else if (!in.getOpts().allow_non_string_keys) { in.error("expected string for object key name"); + } else { + auto key = parseValue(in); + in.skipWhitespace(); + in.expect(':'); + in.skipWhitespace(); + ret.insert(std::move(key), parseValue(in)); } - auto key = parseString(in); - in.skipWhitespace(); - in.expect(':'); - in.skipWhitespace(); - ret.insert(std::move(key), parseValue(in)); + in.skipWhitespace(); if (*in != ',') { break; @@ -465,7 +469,7 @@ dynamic parseArray(Input& in) { assert(*in == '['); ++in; - dynamic ret = {}; + dynamic ret = dynamic::array; in.skipWhitespace(); if (*in == ']') { @@ -474,6 +478,9 @@ dynamic parseArray(Input& in) { } for (;;) { + if (in.getOpts().allow_trailing_comma && *in == ']') { + break; + } ret.push_back(parseValue(in)); in.skipWhitespace(); if (*in != ',') { @@ -489,25 +496,42 @@ dynamic parseArray(Input& in) { dynamic parseNumber(Input& in) { bool const negative = (*in == '-'); - if (negative) { - ++in; - if (in.consume("Infinity")) { + if (negative && in.consume("-Infinity")) { + if (in.getOpts().parse_numbers_as_strings) { + return "-Infinity"; + } else { return -std::numeric_limits::infinity(); } } - auto integral = in.skipDigits(); - if (integral.empty()) { + auto integral = in.skipMinusAndDigits(); + if (negative && integral.size() < 2) { in.error("expected digits after `-'"); } + auto const wasE = *in == 'e' || *in == 'E'; + + constexpr const char* maxInt = "9223372036854775807"; + constexpr const char* minInt = "-9223372036854775808"; + constexpr auto maxIntLen = constexpr_strlen(maxInt); + constexpr auto minIntLen = constexpr_strlen(minInt); + + if (*in != '.' && !wasE && in.getOpts().parse_numbers_as_strings) { + return integral; + } + if (*in != '.' && !wasE) { - auto val = to(integral); - if (negative) { - val = -val; + if (LIKELY(!in.getOpts().double_fallback || integral.size() < maxIntLen) || + (!negative && integral.size() == maxIntLen && integral <= maxInt) || + (negative && integral.size() == minIntLen && integral <= minInt)) { + auto val = to(integral); + in.skipWhitespace(); + return val; + } else { + auto val = to(integral); + in.skipWhitespace(); + return val; } - in.skipWhitespace(); - return val; } auto end = !wasE ? (++in, in.skipDigits().end()) : in.begin(); @@ -519,16 +543,15 @@ dynamic parseNumber(Input& in) { auto expPart = in.skipDigits(); end = expPart.end(); } - auto fullNum = makeRange(integral.begin(), end); - - auto val = to(fullNum); - if (negative) { - val *= -1; + auto fullNum = range(integral.begin(), end); + if (in.getOpts().parse_numbers_as_strings) { + return fullNum; } + auto val = to(fullNum); return val; } -fbstring decodeUnicodeEscape(Input& in) { +std::string decodeUnicodeEscape(Input& in) { auto hexVal = [&] (char c) -> unsigned { return c >= '0' && c <= '9' ? c - '0' : c >= 'a' && c <= 'f' ? c - 'a' + 10 : @@ -536,7 +559,7 @@ fbstring decodeUnicodeEscape(Input& in) { (in.error("invalid hex digit"), 0); }; - auto readHex = [&] { + auto readHex = [&]() -> uint16_t { if (in.size() < 4) { in.error("expected 4 hex digits"); } @@ -576,11 +599,11 @@ fbstring decodeUnicodeEscape(Input& in) { return codePointToUtf8(codePoint); } -fbstring parseString(Input& in) { +std::string parseString(Input& in) { assert(*in == '\"'); ++in; - fbstring ret; + std::string ret; for (;;) { auto range = in.skipWhile( [] (char c) { return c != '\"' && c != '\\'; } @@ -603,8 +626,8 @@ fbstring parseString(Input& in) { case 'r': ret.push_back('\r'); ++in; break; case 't': ret.push_back('\t'); ++in; break; case 'u': ++in; ret += decodeUnicodeEscape(in); break; - default: in.error(to("unknown escape ", *in, - " in string").c_str()); + default: + in.error(to("unknown escape ", *in, " in string").c_str()); } continue; } @@ -630,6 +653,8 @@ fbstring parseString(Input& in) { } dynamic parseValue(Input& in) { + RecursionGuard guard(in); + in.skipWhitespace(); return *in == '[' ? parseArray(in) : *in == '{' ? parseObject(in) : @@ -638,8 +663,12 @@ dynamic parseValue(Input& in) { in.consume("true") ? true : in.consume("false") ? false : in.consume("null") ? nullptr : - in.consume("Infinity") ? std::numeric_limits::infinity() : - in.consume("NaN") ? std::numeric_limits::quiet_NaN() : + in.consume("Infinity") ? + (in.getOpts().parse_numbers_as_strings ? (dynamic)"Infinity" : + (dynamic)std::numeric_limits::infinity()) : + in.consume("NaN") ? + (in.getOpts().parse_numbers_as_strings ? (dynamic)"NaN" : + (dynamic)std::numeric_limits::quiet_NaN()) : in.error("expected json value"); } @@ -647,34 +676,174 @@ dynamic parseValue(Input& in) { ////////////////////////////////////////////////////////////////////// -fbstring serialize(dynamic const& dyn, serialization_opts const& opts) { - fbstring ret; +std::string serialize(dynamic const& dyn, serialization_opts const& opts) { + std::string ret; unsigned indentLevel = 0; Printer p(ret, opts.pretty_formatting ? &indentLevel : nullptr, &opts); p(dyn); return ret; } +// Escape a string so that it is legal to print it in JSON text. +void escapeString( + StringPiece input, + std::string& out, + const serialization_opts& opts) { + auto hexDigit = [] (int c) -> char { + return c < 10 ? c + '0' : c - 10 + 'a'; + }; + + out.push_back('\"'); + + auto* p = reinterpret_cast(input.begin()); + auto* q = reinterpret_cast(input.begin()); + auto* e = reinterpret_cast(input.end()); + + while (p < e) { + // Since non-ascii encoding inherently does utf8 validation + // we explicitly validate utf8 only if non-ascii encoding is disabled. + if ((opts.validate_utf8 || opts.skip_invalid_utf8) + && !opts.encode_non_ascii) { + // to achieve better spatial and temporal coherence + // we do utf8 validation progressively along with the + // string-escaping instead of two separate passes + + // as the encoding progresses, q will stay at or ahead of p + CHECK(q >= p); + + // as p catches up with q, move q forward + if (q == p) { + // calling utf8_decode has the side effect of + // checking that utf8 encodings are valid + char32_t v = decodeUtf8(q, e, opts.skip_invalid_utf8); + if (opts.skip_invalid_utf8 && v == U'\ufffd') { + out.append(u8"\ufffd"); + p = q; + continue; + } + } + } + if (opts.encode_non_ascii && (*p & 0x80)) { + // note that this if condition captures utf8 chars + // with value > 127, so size > 1 byte + char32_t v = decodeUtf8(p, e, opts.skip_invalid_utf8); + out.append("\\u"); + out.push_back(hexDigit(v >> 12)); + out.push_back(hexDigit((v >> 8) & 0x0f)); + out.push_back(hexDigit((v >> 4) & 0x0f)); + out.push_back(hexDigit(v & 0x0f)); + } else if (*p == '\\' || *p == '\"') { + out.push_back('\\'); + out.push_back(*p++); + } else if (*p <= 0x1f) { + switch (*p) { + case '\b': out.append("\\b"); p++; break; + case '\f': out.append("\\f"); p++; break; + case '\n': out.append("\\n"); p++; break; + case '\r': out.append("\\r"); p++; break; + case '\t': out.append("\\t"); p++; break; + default: + // note that this if condition captures non readable chars + // with value < 32, so size = 1 byte (e.g control chars). + out.append("\\u00"); + out.push_back(hexDigit((*p & 0xf0) >> 4)); + out.push_back(hexDigit(*p & 0xf)); + p++; + } + } else { + out.push_back(*p++); + } + } + + out.push_back('\"'); +} + +std::string stripComments(StringPiece jsonC) { + std::string result; + enum class State { + None, + InString, + InlineComment, + LineComment + } state = State::None; + + for (size_t i = 0; i < jsonC.size(); ++i) { + auto s = jsonC.subpiece(i); + switch (state) { + case State::None: + if (s.startsWith("/*")) { + state = State::InlineComment; + ++i; + continue; + } else if (s.startsWith("//")) { + state = State::LineComment; + ++i; + continue; + } else if (s[0] == '\"') { + state = State::InString; + } + result.push_back(s[0]); + break; + case State::InString: + if (s[0] == '\\') { + if (UNLIKELY(s.size() == 1)) { + throw std::logic_error("Invalid JSONC: string is not terminated"); + } + result.push_back(s[0]); + result.push_back(s[1]); + ++i; + continue; + } else if (s[0] == '\"') { + state = State::None; + } + result.push_back(s[0]); + break; + case State::InlineComment: + if (s.startsWith("*/")) { + state = State::None; + ++i; + } + break; + case State::LineComment: + if (s[0] == '\n') { + // skip the line break. It doesn't matter. + state = State::None; + } + break; + default: + throw std::logic_error("Unknown comment state"); + } + } + return result; +} + } ////////////////////////////////////////////////////////////////////// dynamic parseJson(StringPiece range) { - json::Input in(range); + return parseJson(range, json::serialization_opts()); +} + +dynamic parseJson( + StringPiece range, + json::serialization_opts const& opts) { + + json::Input in(range, &opts); auto ret = parseValue(in); in.skipWhitespace(); - if (*in != '\0' && in.size()) { + if (in.size() && *in != '\0') { in.error("parsing didn't consume all input"); } return ret; } -fbstring toJson(dynamic const& dyn) { +std::string toJson(dynamic const& dyn) { return json::serialize(dyn, json::serialization_opts()); } -fbstring toPrettyJson(dynamic const& dyn) { +std::string toPrettyJson(dynamic const& dyn) { json::serialization_opts opts; opts.pretty_formatting = true; return json::serialize(dyn, opts); @@ -688,9 +857,19 @@ fbstring toPrettyJson(dynamic const& dyn) { void dynamic::print_as_pseudo_json(std::ostream& out) const { json::serialization_opts opts; opts.allow_non_string_keys = true; + opts.allow_nan_inf = true; out << json::serialize(*this, opts); } +void PrintTo(const dynamic& dyn, std::ostream* os) { + json::serialization_opts opts; + opts.allow_nan_inf = true; + opts.allow_non_string_keys = true; + opts.pretty_formatting = true; + opts.sort_keys = true; + *os << json::serialize(dyn, opts); +} + ////////////////////////////////////////////////////////////////////// }