Summary:
The `LZMA2` codec doesn't allow decompression without passing the uncompressed size.
Every other codec that supports streaming decompression without the uncompressed size by streaming into an IOBuf chain.
Benchmarks show that reducing the `defaultBufferLength` to 256 KiB doesn't slow down decompression of large files, but it speeds up decompression of small files (< 1000 bytes).
Reviewed By: yfeldblum
Differential Revision:
D4751571
fbshipit-source-id:
39dbe6754a1ecdc2b7ba3107e9face926d4c98ca
}
bool LZMA2Codec::doNeedsUncompressedLength() const {
}
bool LZMA2Codec::doNeedsUncompressedLength() const {
}
uint64_t LZMA2Codec::doMaxUncompressedLength() const {
}
uint64_t LZMA2Codec::doMaxUncompressedLength() const {
SCOPE_EXIT { lzma_end(&stream); };
// Max 64MiB in one go
SCOPE_EXIT { lzma_end(&stream); };
// Max 64MiB in one go
- constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
- constexpr uint32_t defaultBufferLength = uint32_t(4) << 20; // 4MiB
+ constexpr uint32_t maxSingleStepLength = uint32_t(64) << 20; // 64MiB
+ constexpr uint32_t defaultBufferLength = uint32_t(256) << 10; // 256 KiB
folly::io::Cursor cursor(data);
folly::io::Cursor cursor(data);
- uint64_t actualUncompressedLength;
- actualUncompressedLength = decodeVarintFromCursor(cursor);
+ const uint64_t actualUncompressedLength = decodeVarintFromCursor(cursor);
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
uncompressedLength != actualUncompressedLength) {
throw std::runtime_error("LZMA2Codec: invalid uncompressed length");
}
if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
uncompressedLength != actualUncompressedLength) {
throw std::runtime_error("LZMA2Codec: invalid uncompressed length");
}
- } else {
- actualUncompressedLength = uncompressedLength;
- DCHECK_NE(actualUncompressedLength, UNKNOWN_UNCOMPRESSED_LENGTH);
+ uncompressedLength = actualUncompressedLength;
}
auto out = addOutputBuffer(
&stream,
}
auto out = addOutputBuffer(
&stream,
- (actualUncompressedLength <= maxSingleStepLength ?
- actualUncompressedLength :
- defaultBufferLength));
+ ((uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
+ uncompressedLength <= maxSingleStepLength)
+ ? uncompressedLength
+ : defaultBufferLength));
bool streamEnd = false;
auto buf = cursor.peekBytes();
bool streamEnd = false;
auto buf = cursor.peekBytes();
out->prev()->trimEnd(stream.avail_out);
out->prev()->trimEnd(stream.avail_out);
- if (actualUncompressedLength != stream.total_out) {
- throw std::runtime_error(to<std::string>(
- "LZMA2Codec: invalid uncompressed length"));
+ if (uncompressedLength != UNKNOWN_UNCOMPRESSED_LENGTH &&
+ uncompressedLength != stream.total_out) {
+ throw std::runtime_error(
+ to<std::string>("LZMA2Codec: invalid uncompressed length"));
{ CodecType::SNAPPY, false },
{ CodecType::ZLIB, false },
{ CodecType::LZ4_VARINT_SIZE, false },
{ CodecType::SNAPPY, false },
{ CodecType::ZLIB, false },
{ CodecType::LZ4_VARINT_SIZE, false },
- { CodecType::LZMA2, true },
+ { CodecType::LZMA2, false },
{ CodecType::LZMA2_VARINT_SIZE, false },
{ CodecType::ZSTD, false },
{ CodecType::GZIP, false },
{ CodecType::LZMA2_VARINT_SIZE, false },
{ CodecType::ZSTD, false },
{ CodecType::GZIP, false },
supportedCodecs({
CodecType::SNAPPY,
CodecType::ZLIB,
supportedCodecs({
CodecType::SNAPPY,
CodecType::ZLIB,
CodecType::ZSTD,
CodecType::LZ4_FRAME,
})));
CodecType::ZSTD,
CodecType::LZ4_FRAME,
})));