2 * Copyright 2017 Facebook, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 #include <folly/io/compression/Zlib.h>
21 #include <folly/Conv.h>
22 #include <folly/Optional.h>
23 #include <folly/Range.h>
24 #include <folly/ScopeGuard.h>
25 #include <folly/io/Compression.h>
26 #include <folly/io/Cursor.h>
27 #include <folly/io/compression/Utils.h>
29 using folly::io::compression::detail::dataStartsWithLE;
30 using folly::io::compression::detail::prefixToStringLE;
38 bool isValidStrategy(int strategy) {
39 std::array<int, 5> strategies{{
46 return std::any_of(strategies.begin(), strategies.end(), [&](int i) {
51 int getWindowBits(Options::Format format, int windowSize) {
53 case Options::Format::ZLIB:
55 case Options::Format::GZIP:
56 return windowSize + 16;
57 case Options::Format::RAW:
59 case Options::Format::AUTO:
60 return windowSize + 32;
66 CodecType getCodecType(Options options) {
67 if (options.windowSize == 15 && options.format == Options::Format::ZLIB) {
68 return CodecType::ZLIB;
70 options.windowSize == 15 && options.format == Options::Format::GZIP) {
71 return CodecType::GZIP;
73 return CodecType::USER_DEFINED;
77 class ZlibStreamCodec final : public StreamCodec {
79 static std::unique_ptr<Codec> createCodec(Options options, int level);
80 static std::unique_ptr<StreamCodec> createStream(Options options, int level);
82 explicit ZlibStreamCodec(Options options, int level);
83 ~ZlibStreamCodec() override;
85 std::vector<std::string> validPrefixes() const override;
86 bool canUncompress(const IOBuf* data, Optional<uint64_t> uncompressedLength)
90 uint64_t doMaxCompressedLength(uint64_t uncompressedLength) const override;
92 void doResetStream() override;
93 bool doCompressStream(
95 MutableByteRange& output,
96 StreamCodec::FlushOp flush) override;
97 bool doUncompressStream(
99 MutableByteRange& output,
100 StreamCodec::FlushOp flush) override;
102 void resetDeflateStream();
103 void resetInflateStream();
107 Optional<z_stream> deflateStream_{};
108 Optional<z_stream> inflateStream_{};
110 bool needReset_{true};
112 static constexpr uint16_t kGZIPMagicLE = 0x8B1F;
114 std::vector<std::string> ZlibStreamCodec::validPrefixes() const {
115 if (type() == CodecType::ZLIB) {
116 // Zlib streams start with a 2 byte header.
123 // We won't restrict the values of any sub-fields except as described below.
125 // The lowest 4 bits of CMF is the compression method (CM).
126 // CM == 0x8 is the deflate compression method, which is currently the only
127 // supported compression method, so any valid prefix must have CM == 0x8.
129 // The lowest 5 bits of FLG is FCHECK.
130 // FCHECK must be such that the two header bytes are a multiple of 31 when
131 // interpreted as a big endian 16-bit number.
132 std::vector<std::string> result;
133 // 16 values for the first byte, 8 values for the second byte.
134 // There are also 4 combinations where both 0x00 and 0x1F work as FCHECK.
136 // Select all values for the CMF byte that use the deflate algorithm 0x8.
137 for (uint32_t first = 0x0800; first <= 0xF800; first += 0x1000) {
138 // Select all values for the FLG, but leave FCHECK as 0 since it's fixed.
139 for (uint32_t second = 0x00; second <= 0xE0; second += 0x20) {
140 uint16_t prefix = first | second;
142 prefix += 31 - (prefix % 31);
143 result.push_back(prefixToStringLE(Endian::big(prefix)));
144 // zlib won't produce this, but it is a valid prefix.
145 if ((prefix & 0x1F) == 31) {
147 result.push_back(prefixToStringLE(Endian::big(prefix)));
152 } else if (type() == CodecType::GZIP) {
153 // The gzip frame starts with 2 magic bytes.
154 return {prefixToStringLE(kGZIPMagicLE)};
160 bool ZlibStreamCodec::canUncompress(const IOBuf* data, Optional<uint64_t>)
162 if (type() == CodecType::ZLIB) {
165 if (!cursor.tryReadBE(value)) {
168 // zlib compressed if using deflate and is a multiple of 31.
169 return (value & 0x0F00) == 0x0800 && value % 31 == 0;
170 } else if (type() == CodecType::GZIP) {
171 return dataStartsWithLE(data, kGZIPMagicLE);
177 uint64_t ZlibStreamCodec::doMaxCompressedLength(
178 uint64_t uncompressedLength) const {
179 return deflateBound(nullptr, uncompressedLength);
182 std::unique_ptr<Codec> ZlibStreamCodec::createCodec(
185 return std::make_unique<ZlibStreamCodec>(options, level);
188 std::unique_ptr<StreamCodec> ZlibStreamCodec::createStream(
191 return std::make_unique<ZlibStreamCodec>(options, level);
194 ZlibStreamCodec::ZlibStreamCodec(Options options, int level)
195 : StreamCodec(getCodecType(options)) {
197 case COMPRESSION_LEVEL_FASTEST:
200 case COMPRESSION_LEVEL_DEFAULT:
201 level = Z_DEFAULT_COMPRESSION;
203 case COMPRESSION_LEVEL_BEST:
207 auto inBounds = [](int value, int low, int high) {
208 return (value >= low) && (value <= high);
211 if (level != Z_DEFAULT_COMPRESSION && !inBounds(level, 0, 9)) {
212 throw std::invalid_argument(
213 to<std::string>("ZlibStreamCodec: invalid level: ", level));
218 // Although zlib allows a windowSize of 8..15, a value of 8 is not
219 // properly supported and is treated as a value of 9. This means data deflated
220 // with windowSize==8 can not be re-inflated with windowSize==8. windowSize==8
221 // is also not supported for gzip and raw deflation.
222 // Hence, the codec supports only 9..15.
223 if (!inBounds(options_.windowSize, 9, 15)) {
224 throw std::invalid_argument(to<std::string>(
225 "ZlibStreamCodec: invalid windowSize option: ", options.windowSize));
227 if (!inBounds(options_.memLevel, 1, 9)) {
228 throw std::invalid_argument(to<std::string>(
229 "ZlibStreamCodec: invalid memLevel option: ", options.memLevel));
231 if (!isValidStrategy(options_.strategy)) {
232 throw std::invalid_argument(to<std::string>(
233 "ZlibStreamCodec: invalid strategy: ", options.strategy));
237 ZlibStreamCodec::~ZlibStreamCodec() {
238 if (deflateStream_) {
239 deflateEnd(deflateStream_.get_pointer());
240 deflateStream_.clear();
242 if (inflateStream_) {
243 inflateEnd(inflateStream_.get_pointer());
244 inflateStream_.clear();
248 void ZlibStreamCodec::doResetStream() {
252 void ZlibStreamCodec::resetDeflateStream() {
253 if (deflateStream_) {
254 int const rc = deflateReset(deflateStream_.get_pointer());
256 deflateStream_.clear();
257 throw std::runtime_error(
258 to<std::string>("ZlibStreamCodec: deflateReset error: ", rc));
262 deflateStream_ = z_stream{};
264 // The automatic header detection format is only for inflation.
265 // Use zlib for deflation if the format is auto.
266 int const windowBits = getWindowBits(
267 options_.format == Options::Format::AUTO ? Options::Format::ZLIB
269 options_.windowSize);
271 int const rc = deflateInit2(
272 deflateStream_.get_pointer(),
279 deflateStream_.clear();
280 throw std::runtime_error(
281 to<std::string>("ZlibStreamCodec: deflateInit error: ", rc));
285 void ZlibStreamCodec::resetInflateStream() {
286 if (inflateStream_) {
287 int const rc = inflateReset(inflateStream_.get_pointer());
289 inflateStream_.clear();
290 throw std::runtime_error(
291 to<std::string>("ZlibStreamCodec: inflateReset error: ", rc));
295 inflateStream_ = z_stream{};
296 int const rc = inflateInit2(
297 inflateStream_.get_pointer(),
298 getWindowBits(options_.format, options_.windowSize));
300 inflateStream_.clear();
301 throw std::runtime_error(
302 to<std::string>("ZlibStreamCodec: inflateInit error: ", rc));
306 static int zlibTranslateFlush(StreamCodec::FlushOp flush) {
308 case StreamCodec::FlushOp::NONE:
310 case StreamCodec::FlushOp::FLUSH:
312 case StreamCodec::FlushOp::END:
315 throw std::invalid_argument("ZlibStreamCodec: Invalid flush");
319 static int zlibThrowOnError(int rc) {
326 throw std::runtime_error(to<std::string>("ZlibStreamCodec: error: ", rc));
330 bool ZlibStreamCodec::doCompressStream(
332 MutableByteRange& output,
333 StreamCodec::FlushOp flush) {
335 resetDeflateStream();
338 DCHECK(deflateStream_.hasValue());
339 // zlib will return Z_STREAM_ERROR if output.data() is null.
340 if (output.data() == nullptr) {
343 deflateStream_->next_in = const_cast<uint8_t*>(input.data());
344 deflateStream_->avail_in = input.size();
345 deflateStream_->next_out = output.data();
346 deflateStream_->avail_out = output.size();
348 input.uncheckedAdvance(input.size() - deflateStream_->avail_in);
349 output.uncheckedAdvance(output.size() - deflateStream_->avail_out);
351 int const rc = zlibThrowOnError(
352 deflate(deflateStream_.get_pointer(), zlibTranslateFlush(flush)));
354 case StreamCodec::FlushOp::NONE:
356 case StreamCodec::FlushOp::FLUSH:
357 return deflateStream_->avail_in == 0 && deflateStream_->avail_out != 0;
358 case StreamCodec::FlushOp::END:
359 return rc == Z_STREAM_END;
361 throw std::invalid_argument("ZlibStreamCodec: Invalid flush");
365 bool ZlibStreamCodec::doUncompressStream(
367 MutableByteRange& output,
368 StreamCodec::FlushOp flush) {
370 resetInflateStream();
373 DCHECK(inflateStream_.hasValue());
374 // zlib will return Z_STREAM_ERROR if output.data() is null.
375 if (output.data() == nullptr) {
378 inflateStream_->next_in = const_cast<uint8_t*>(input.data());
379 inflateStream_->avail_in = input.size();
380 inflateStream_->next_out = output.data();
381 inflateStream_->avail_out = output.size();
383 input.advance(input.size() - inflateStream_->avail_in);
384 output.advance(output.size() - inflateStream_->avail_out);
386 int const rc = zlibThrowOnError(
387 inflate(inflateStream_.get_pointer(), zlibTranslateFlush(flush)));
388 return rc == Z_STREAM_END;
393 Options defaultGzipOptions() {
394 return Options(Options::Format::GZIP);
397 Options defaultZlibOptions() {
398 return Options(Options::Format::ZLIB);
401 std::unique_ptr<Codec> getCodec(Options options, int level) {
402 return ZlibStreamCodec::createCodec(options, level);
405 std::unique_ptr<StreamCodec> getStreamCodec(Options options, int level) {
406 return ZlibStreamCodec::createStream(options, level);
413 #endif // FOLLY_HAVE_LIBZ