2 * Copyright 2016 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include <folly/io/Cursor.h>
19 using namespace folly;
20 using folly::io::QueueAppender;
21 using folly::bser::serialization_opts;
26 const uint8_t kMagic[2] = {0, 1};
28 static void bserEncode(dynamic const& dyn,
29 QueueAppender& appender,
30 const serialization_opts& opts);
32 serialization_opts::serialization_opts()
33 : sort_keys(false), growth_increment(8192) {}
35 static const dynamic* getTemplate(const serialization_opts& opts,
36 dynamic const& dynArray) {
37 if (!opts.templates.hasValue()) {
40 const auto& templates = opts.templates.value();
41 const auto it = templates.find(&dynArray);
42 if (it == templates.end()) {
48 static void bserEncodeInt(int64_t ival, QueueAppender& appender) {
49 /* Return the smallest size int that can store the value */
51 ((ival == ((int8_t)ival)) ? 1 : (ival == ((int16_t)ival))
53 : (ival == ((int32_t)ival)) ? 4 : 8);
57 appender.write((int8_t)BserType::Int8);
58 appender.write(int8_t(ival));
61 appender.write((int8_t)BserType::Int16);
62 appender.write(int16_t(ival));
65 appender.write((int8_t)BserType::Int32);
66 appender.write(int32_t(ival));
69 appender.write((int8_t)BserType::Int64);
73 throw std::runtime_error("impossible integer size");
77 static void bserEncodeString(folly::StringPiece str, QueueAppender& appender) {
78 appender.write((int8_t)BserType::String);
79 bserEncodeInt(str.size(), appender);
80 appender.push((uint8_t*)str.data(), str.size());
83 static void bserEncodeArraySimple(dynamic const& dyn,
84 QueueAppender& appender,
85 const serialization_opts& opts) {
86 appender.write((int8_t)BserType::Array);
87 bserEncodeInt(dyn.size(), appender);
88 for (const auto& ele : dyn) {
89 bserEncode(ele, appender, opts);
93 static void bserEncodeArray(dynamic const& dyn,
94 QueueAppender& appender,
95 const serialization_opts& opts) {
97 auto templ = getTemplate(opts, dyn);
98 if (UNLIKELY(templ != nullptr)) {
99 appender.write((int8_t)BserType::Template);
101 // Emit the list of property names
102 bserEncodeArraySimple(*templ, appender, opts);
104 // The number of objects in the array
105 bserEncodeInt(dyn.size(), appender);
107 // For each object in the array
108 for (const auto& ele : dyn) {
109 // For each key in the template
110 for (const auto& name : *templ) {
111 if (auto found = ele.get_ptr(name)) {
112 if (found->isNull()) {
113 // Prefer to Skip rather than encode a null value for
114 // compatibility with the other bser implementations
115 appender.write((int8_t)BserType::Skip);
117 bserEncode(*found, appender, opts);
120 appender.write((int8_t)BserType::Skip);
127 bserEncodeArraySimple(dyn, appender, opts);
130 static void bserEncodeObject(dynamic const& dyn,
131 QueueAppender& appender,
132 const serialization_opts& opts) {
133 appender.write((int8_t)BserType::Object);
134 bserEncodeInt(dyn.size(), appender);
136 if (opts.sort_keys) {
137 std::vector<std::pair<dynamic, dynamic>> sorted(dyn.items().begin(),
139 std::sort(sorted.begin(), sorted.end());
140 for (const auto& item : sorted) {
141 bserEncode(item.first, appender, opts);
142 bserEncode(item.second, appender, opts);
145 for (const auto& item : dyn.items()) {
146 bserEncode(item.first, appender, opts);
147 bserEncode(item.second, appender, opts);
152 static void bserEncode(dynamic const& dyn,
153 QueueAppender& appender,
154 const serialization_opts& opts) {
155 switch (dyn.type()) {
156 case dynamic::Type::NULLT:
157 appender.write((int8_t)BserType::Null);
159 case dynamic::Type::BOOL:
161 (int8_t)(dyn.getBool() ? BserType::True : BserType::False));
163 case dynamic::Type::DOUBLE: {
164 double dval = dyn.getDouble();
165 appender.write((int8_t)BserType::Real);
166 appender.write(dval);
169 case dynamic::Type::INT64:
170 bserEncodeInt(dyn.getInt(), appender);
172 case dynamic::Type::OBJECT:
173 bserEncodeObject(dyn, appender, opts);
175 case dynamic::Type::ARRAY:
176 bserEncodeArray(dyn, appender, opts);
178 case dynamic::Type::STRING:
179 bserEncodeString(dyn.getString(), appender);
184 std::unique_ptr<folly::IOBuf> toBserIOBuf(folly::dynamic const& dyn,
185 const serialization_opts& opts) {
186 IOBufQueue q(IOBufQueue::cacheChainLength());
187 uint8_t hdrbuf[sizeof(kMagic) + 1 + sizeof(int64_t)];
189 // Reserve some headroom for the overall PDU size; we'll fill this in
190 // after we've serialized the data and know the length
191 auto firstbuf = IOBuf::create(opts.growth_increment);
192 firstbuf->advance(sizeof(hdrbuf));
193 q.append(std::move(firstbuf));
196 QueueAppender appender(&q, opts.growth_increment);
197 bserEncode(dyn, appender, opts);
199 // compute the length
200 auto len = q.chainLength();
201 if (len > std::numeric_limits<int64_t>::max()) {
202 throw std::range_error(folly::to<std::string>(
203 "serialized data size ", len, " is too large to represent as BSER"));
206 // This is a bit verbose, but it computes a header that is appropriate
207 // to the size of the serialized data
209 memcpy(hdrbuf, kMagic, sizeof(kMagic));
210 size_t hdrlen = sizeof(kMagic) + 1;
211 auto magicptr = hdrbuf + sizeof(kMagic);
212 auto lenptr = hdrbuf + hdrlen;
214 if (len > std::numeric_limits<int32_t>::max()) {
215 *magicptr = (int8_t)BserType::Int64;
216 *(int64_t*)lenptr = (int64_t)len;
217 hdrlen += sizeof(int64_t);
218 } else if (len > std::numeric_limits<int16_t>::max()) {
219 *magicptr = (int8_t)BserType::Int32;
220 *(int32_t*)lenptr = (int32_t)len;
221 hdrlen += sizeof(int32_t);
222 } else if (len > std::numeric_limits<int8_t>::max()) {
223 *magicptr = (int8_t)BserType::Int16;
224 *(int16_t*)lenptr = (int16_t)len;
225 hdrlen += sizeof(int16_t);
227 *magicptr = (int8_t)BserType::Int8;
228 *(int8_t*)lenptr = (int8_t)len;
229 hdrlen += sizeof(int8_t);
232 // and place the data in the headroom
233 q.prepend(hdrbuf, hdrlen);
238 fbstring toBser(dynamic const& dyn, const serialization_opts& opts) {
239 auto buf = toBserIOBuf(dyn, opts);
240 return buf->moveToFbString();