Skip to content

Commit

Permalink
feat: autodetect frame blocksize
Browse files Browse the repository at this point in the history 
The default blocksize of `FrameInfo` is now auto instead of 64kb, it will detect the blocksize
depending of the size of the first `write` call. This increases
compression ratio and speed for use cases where the data is larger than
64kb.
  • Loading branch information
@PSeitz
PSeitz committed Feb 8, 2023
1 parent b20c518 commit bcc0d62
Show file tree
Hide file tree
Showing 4 changed files with 124 additions and 39 deletions.
10 changes: 5 additions & 5 deletions README.md
View file
Expand Up @@ -9,14 +9,14 @@
Fastest LZ4 implementation in Rust. Originally based on [redox-os' lz4 compression](https://crates.io/crates/lz4-compress), but now a complete rewrite.
The results in the table are from a benchmark in this project (66Kb JSON) with the block format.

AMD Ryzen 7 5800H, rustc 1.57.0-nightly (5ecc8ad84 2021-09-19), Linux Mint.
AMD Ryzen 7 5900HX, rustc 1.67.0 (fc594f156 2023-01-24), Manjaro.

| Compressor | Compression | Decompression | Ratio |
|----------------------|-------------|---------------|---------------|
| lz4_flex unsafe | 1897 MiB/s | 7123 MiB/s | 0.2289 |
| lz4_flex unsafe w. checked_decode | 1897 MiB/s | 6637 MiB/s | 0.2289 |
| lz4_flex safe | 1591 MiB/s | 5163 MiB/s | 0.2289 |
| lzzz (lz4 1.9.3) | 2235 MiB/s | 7001 MiB/s | 0.2283 |
| lz4_flex unsafe | 2169 MiB/s | 8266 MiB/s | 0.2289 |
| lz4_flex unsafe w. checked_decode | 2169 MiB/s | 7019 MiB/s | 0.2289 |
| lz4_flex safe | 1730 MiB/s | 5925 MiB/s | 0.2289 |
| lzzz (lz4 1.9.3) | 2292 MiB/s | 7196 MiB/s | 0.2283 |
| lz4_fear | 886 MiB/s | 1359 MiB/s | 0.2283 |
| snap | 1886 MiB/s | 1649 MiB/s | 0.2242 |

Expand Down
46 changes: 28 additions & 18 deletions src/frame/compress.rs
View file
Expand Up @@ -13,8 +13,11 @@ use crate::{
sink::vec_sink_for_compression,
};

use super::header::{BlockInfo, BlockMode, FrameInfo, BLOCK_INFO_SIZE, MAX_FRAME_INFO_SIZE};
use super::Error;
use super::{
header::{BlockInfo, BlockMode, FrameInfo, BLOCK_INFO_SIZE, MAX_FRAME_INFO_SIZE},
BlockSize,
};
use crate::block::WINDOW_SIZE;

/// A writer for compressing a LZ4 stream.
Expand Down Expand Up @@ -84,10 +87,9 @@ pub struct FrameEncoder<W: io::Write> {
}

impl<W: io::Write> FrameEncoder<W> {
/// Creates a new Encoder with the specified FrameInfo.
pub fn with_frame_info(frame_info: FrameInfo, wtr: W) -> Self {
let max_block_size = frame_info.block_size.get_size();
let src_size = if frame_info.block_mode == BlockMode::Linked {
fn init(&mut self) {
let max_block_size = self.frame_info.block_size.get_size();
let src_size = if self.frame_info.block_mode == BlockMode::Linked {
// In linked mode we consume the input (bumping src_start) but leave the
// beginning of src to be used as a prefix in subsequent blocks.
// That is at least until we have at least `max_block_size + WINDOW_SIZE`
Expand All @@ -99,21 +101,26 @@ impl<W: io::Write> FrameEncoder<W> {
} else {
max_block_size
};
let src = Vec::with_capacity(src_size);
let dst = Vec::with_capacity(crate::block::compress::get_maximum_output_size(
max_block_size,
));

// 16 KB hash table for matches, same as the reference implementation.
//let (dict_size, dict_bitshift) = (4 * 1024, 4);
// Since this method is called potentially multiple times, don't reserve _additional_
// capacity if not required.
self.src
.reserve(src_size.saturating_sub(self.src.capacity()));
self.dst.reserve(
crate::block::compress::get_maximum_output_size(max_block_size)
.saturating_sub(self.dst.capacity()),
);
}

/// Creates a new Encoder with the specified FrameInfo.
pub fn with_frame_info(frame_info: FrameInfo, wtr: W) -> Self {
FrameEncoder {
src,
src: Vec::new(),
w: wtr,
// 16 KB hash table for matches, same as the reference implementation.
compression_table: HashTable4K::new(),
content_hasher: XxHash32::with_seed(0),
content_len: 0,
dst,
dst: Vec::new(),
is_frame_open: false,
frame_info,
src_start: 0,
Expand Down Expand Up @@ -195,8 +202,12 @@ impl<W: io::Write> FrameEncoder<W> {

/// Begin the frame by writing the frame header.
/// It'll also setup the encoder for compressing blocks for the the new frame.
fn begin_frame(&mut self) -> io::Result<()> {
fn begin_frame(&mut self, buf_len: usize) -> io::Result<()> {
self.is_frame_open = true;
if self.frame_info.block_size == BlockSize::Auto {
self.frame_info.block_size = BlockSize::from_buf_length(buf_len);
}
self.init();
let mut frame_info_buffer = [0u8; MAX_FRAME_INFO_SIZE];
let size = self.frame_info.write(&mut frame_info_buffer)?;
self.w.write_all(&frame_info_buffer[..size])?;
Expand Down Expand Up @@ -336,13 +347,12 @@ impl<W: io::Write> FrameEncoder<W> {
impl<W: io::Write> io::Write for FrameEncoder<W> {
fn write(&mut self, mut buf: &[u8]) -> io::Result<usize> {
if !self.is_frame_open && !buf.is_empty() {
self.begin_frame()?;
self.begin_frame(buf.len())?;
}
let buf_len = buf.len();
let max_block_size = self.frame_info.block_size.get_size();
while !buf.is_empty() {
let src_filled = self.src_end - self.src_start;
let max_fill_len = max_block_size - src_filled;
let max_fill_len = self.frame_info.block_size.get_size() - src_filled;
if max_fill_len == 0 {
// make space by writing next block
self.write_block()?;
Expand Down
17 changes: 16 additions & 1 deletion src/frame/header.rs
View file
Expand Up @@ -37,6 +37,8 @@ pub(crate) const BLOCK_INFO_SIZE: usize = 4;
#[derive(Clone, Copy, PartialEq, Debug)]
/// Different predefines blocksizes to choose when compressing data.
pub enum BlockSize {
/// Will detect optimal frame size based on the size of the first write call
Auto = 0,
/// The default block size.
Max64KB = 4,
/// 256KB block size.
Expand All @@ -51,13 +53,26 @@ pub enum BlockSize {

impl Default for BlockSize {
fn default() -> Self {
BlockSize::Max64KB
BlockSize::Auto
}
}

impl BlockSize {
/// Try to find optimal size based on passed buffer length.
pub(crate) fn from_buf_length(buf_len: usize) -> Self {
let mut blocksize = BlockSize::Max4MB;

for candidate in [BlockSize::Max256KB, BlockSize::Max64KB] {
if buf_len > candidate.get_size() {
return blocksize;
}
blocksize = candidate;
}
return BlockSize::Max64KB;
}
pub(crate) fn get_size(&self) -> usize {
match self {
BlockSize::Auto => unreachable!(),
BlockSize::Max64KB => 64 * 1024,
BlockSize::Max256KB => 256 * 1024,
BlockSize::Max1MB => 1024 * 1024,
Expand Down
90 changes: 75 additions & 15 deletions tests/tests.rs
View file
Expand Up @@ -10,7 +10,7 @@ use lz4_compress::compress as lz4_rust_compress;
use lz4_flex::frame::BlockMode;
use lz4_flex::{
block::{compress_prepend_size, decompress_size_prepended},
compress, decompress,
compress as compress_block, decompress,
};

const COMPRESSION1K: &[u8] = include_bytes!("../benches/compression_1k.txt");
Expand Down Expand Up @@ -76,7 +76,7 @@ pub fn lz4_flex_frame_decompress(input: &[u8]) -> Result<Vec<u8>, lz4_flex::fram
fn test_roundtrip(bytes: impl AsRef<[u8]>) {
let bytes = bytes.as_ref();
// compress with rust, decompress with rust
let compressed_flex = compress(bytes);
let compressed_flex = compress_block(bytes);
let decompressed = decompress(&compressed_flex, bytes.len()).unwrap();
assert_eq!(decompressed, bytes);

Expand Down Expand Up @@ -115,7 +115,7 @@ fn lz4_cpp_compatibility(bytes: &[u8]) {
}

// compress with rust, decompress with lz4 cpp
let compressed_flex = compress(bytes);
let compressed_flex = compress_block(bytes);
let decompressed = lz4_cpp_block_decompress(&compressed_flex, bytes.len()).unwrap();
assert_eq!(decompressed, bytes);

Expand Down Expand Up @@ -155,20 +155,42 @@ fn compare_compression() {
}

#[test]
fn test_minimum_compression_ratio() {
let compressed = compress(COMPRESSION34K);
fn test_minimum_compression_ratio_block() {
let compressed = compress_block(COMPRESSION34K);
let ratio = compressed.len() as f64 / COMPRESSION34K.len() as f64;
assert_lt!(ratio, 0.585); // TODO check why compression is not deterministic (fails in ci for
// 0.58)
let compressed = compress(COMPRESSION65);
let compressed = compress_block(COMPRESSION65);
let ratio = compressed.len() as f64 / COMPRESSION65.len() as f64;
assert_lt!(ratio, 0.574);

let compressed = compress(COMPRESSION66JSON);
let compressed = compress_block(COMPRESSION66JSON);
let ratio = compressed.len() as f64 / COMPRESSION66JSON.len() as f64;
assert_lt!(ratio, 0.229);
}

#[cfg(feature = "frame")]
#[test]
fn test_minimum_compression_ratio_frame() {
use lz4_flex::frame::FrameInfo;

let get_ratio = |input| {
let compressed = lz4_flex_frame_compress_with(FrameInfo::new(), input).unwrap();

let ratio = compressed.len() as f64 / input.len() as f64;
ratio
};

let ratio = get_ratio(COMPRESSION34K);
assert_lt!(ratio, 0.585);

let ratio = get_ratio(COMPRESSION65);
assert_lt!(ratio, 0.574);

let ratio = get_ratio(COMPRESSION66JSON);
assert_lt!(ratio, 0.235);
}

use lz_fear::raw::compress2;
use lz_fear::raw::U16Table;
use lz_fear::raw::U32Table;
Expand All @@ -184,10 +206,12 @@ fn compress_lz4_fear(input: &[u8]) -> Vec<u8> {
}

fn print_compression_ration(input: &'static [u8], name: &str) {
let compressed = compress(input);
println!("\nComparing for {}", name);
let name = "";
let compressed = compress_block(input);
// println!("{:?}", compressed);
println!(
"lz4_flex Compression Ratio {:?} {:?}",
"lz4_flex block Compression Ratio {:?} {:?}",
name,
compressed.len() as f64 / input.len() as f64
);
Expand All @@ -197,7 +221,7 @@ fn print_compression_ration(input: &'static [u8], name: &str) {
let compressed = lz4_cpp_block_compress(input).unwrap();
// println!("{:?}", compressed);
println!(
"Lz4 Cpp Compression Ratio {:?} {:?}",
"Lz4 Cpp block Compression Ratio {:?} {:?}",
name,
compressed.len() as f64 / input.len() as f64
);
Expand All @@ -206,15 +230,15 @@ fn print_compression_ration(input: &'static [u8], name: &str) {
assert_eq!(decompressed, input);
let compressed = lz4_rust_compress(input);
println!(
"lz4_rust_compress Compression Ratio {:?} {:?}",
"lz4_rust_compress block Compression Ratio {:?} {:?}",
name,
compressed.len() as f64 / input.len() as f64
);

assert_eq!(decompressed, input);
let compressed = compress_lz4_fear(input);
println!(
"lz4_fear_compress Compression Ratio {:?} {:?}",
"lz4_fear_compress block Compression Ratio {:?} {:?}",
name,
compressed.len() as f64 / input.len() as f64
);
Expand All @@ -225,6 +249,42 @@ fn print_compression_ration(input: &'static [u8], name: &str) {
name,
compressed.len() as f64 / input.len() as f64
);

#[cfg(feature = "frame")]
{
let mut frame_info = lz4_flex::frame::FrameInfo::new();
frame_info.block_mode = BlockMode::Independent;
//frame_info.block_size = lz4_flex::frame::BlockSize::Max4MB;
let compressed = lz4_flex_frame_compress_with(frame_info, input).unwrap();
println!(
"lz4_flex frame indep Compression Ratio {:?} {:?}",
name,
compressed.len() as f64 / input.len() as f64
);

let mut frame_info = lz4_flex::frame::FrameInfo::new();
frame_info.block_mode = BlockMode::Linked;
let compressed = lz4_flex_frame_compress_with(frame_info, input).unwrap();
println!(
"lz4_flex frame linked Compression Ratio {:?} {:?}",
name,
compressed.len() as f64 / input.len() as f64
);

let compressed = lz4_cpp_frame_compress(input, true).unwrap();
println!(
"lz4 cpp frame indep Compression Ratio {:?} {:?}",
name,
compressed.len() as f64 / input.len() as f64
);

let compressed = lz4_cpp_frame_compress(input, false).unwrap();
println!(
"lz4 cpp frame linked Compression Ratio {:?} {:?}",
name,
compressed.len() as f64 / input.len() as f64
);
}
}

// #[test]
Expand Down Expand Up @@ -430,7 +490,7 @@ fn compression_works() {
The len method has a default implementation, so you usually shouldn't implement it. However, you may be able to provide a more performant implementation than the default, so overriding it in this case makes sense."#;

test_roundtrip(s);
assert!(compress(s.as_bytes()).len() < s.len());
assert!(compress_block(s.as_bytes()).len() < s.len());
}

// #[test]
Expand Down Expand Up @@ -649,12 +709,12 @@ mod test_compression {
print_ratio(
"Ratio 1k flex",
COMPRESSION1K.len(),
compress(COMPRESSION1K).len(),
compress_block(COMPRESSION1K).len(),
);
print_ratio(
"Ratio 34k flex",
COMPRESSION34K.len(),
compress(COMPRESSION34K).len(),
compress_block(COMPRESSION34K).len(),
);
}

Expand Down

0 comments on commit bcc0d62

Please sign in to comment.