diff --git a/README.md b/README.md index 35d0b9b..5590cd2 100644 --- a/README.md +++ b/README.md @@ -9,14 +9,14 @@ Fastest LZ4 implementation in Rust. Originally based on [redox-os' lz4 compression](https://crates.io/crates/lz4-compress), but now a complete rewrite. The results in the table are from a benchmark in this project (66Kb JSON) with the block format. -AMD Ryzen 7 5800H, rustc 1.57.0-nightly (5ecc8ad84 2021-09-19), Linux Mint. +AMD Ryzen 7 5900HX, rustc 1.67.0 (fc594f156 2023-01-24), Manjaro. | Compressor | Compression | Decompression | Ratio | |----------------------|-------------|---------------|---------------| -| lz4_flex unsafe | 1897 MiB/s | 7123 MiB/s | 0.2289 | -| lz4_flex unsafe w. checked_decode | 1897 MiB/s | 6637 MiB/s | 0.2289 | -| lz4_flex safe | 1591 MiB/s | 5163 MiB/s | 0.2289 | -| lzzz (lz4 1.9.3) | 2235 MiB/s | 7001 MiB/s | 0.2283 | +| lz4_flex unsafe | 2169 MiB/s | 8266 MiB/s | 0.2289 | +| lz4_flex unsafe w. checked_decode | 2169 MiB/s | 7019 MiB/s | 0.2289 | +| lz4_flex safe | 1730 MiB/s | 5925 MiB/s | 0.2289 | +| lzzz (lz4 1.9.3) | 2292 MiB/s | 7196 MiB/s | 0.2283 | | lz4_fear | 886 MiB/s | 1359 MiB/s | 0.2283 | | snap | 1886 MiB/s | 1649 MiB/s | 0.2242 | diff --git a/src/frame/compress.rs b/src/frame/compress.rs index f147549..7b3d649 100644 --- a/src/frame/compress.rs +++ b/src/frame/compress.rs @@ -13,8 +13,11 @@ use crate::{ sink::vec_sink_for_compression, }; -use super::header::{BlockInfo, BlockMode, FrameInfo, BLOCK_INFO_SIZE, MAX_FRAME_INFO_SIZE}; use super::Error; +use super::{ + header::{BlockInfo, BlockMode, FrameInfo, BLOCK_INFO_SIZE, MAX_FRAME_INFO_SIZE}, + BlockSize, +}; use crate::block::WINDOW_SIZE; /// A writer for compressing a LZ4 stream. @@ -84,10 +87,9 @@ pub struct FrameEncoder { } impl FrameEncoder { - /// Creates a new Encoder with the specified FrameInfo. - pub fn with_frame_info(frame_info: FrameInfo, wtr: W) -> Self { - let max_block_size = frame_info.block_size.get_size(); - let src_size = if frame_info.block_mode == BlockMode::Linked { + fn init(&mut self) { + let max_block_size = self.frame_info.block_size.get_size(); + let src_size = if self.frame_info.block_mode == BlockMode::Linked { // In linked mode we consume the input (bumping src_start) but leave the // beginning of src to be used as a prefix in subsequent blocks. // That is at least until we have at least `max_block_size + WINDOW_SIZE` @@ -99,21 +101,26 @@ impl FrameEncoder { } else { max_block_size }; - let src = Vec::with_capacity(src_size); - let dst = Vec::with_capacity(crate::block::compress::get_maximum_output_size( - max_block_size, - )); - - // 16 KB hash table for matches, same as the reference implementation. - //let (dict_size, dict_bitshift) = (4 * 1024, 4); + // Since this method is called potentially multiple times, don't reserve _additional_ + // capacity if not required. + self.src + .reserve(src_size.saturating_sub(self.src.capacity())); + self.dst.reserve( + crate::block::compress::get_maximum_output_size(max_block_size) + .saturating_sub(self.dst.capacity()), + ); + } + /// Creates a new Encoder with the specified FrameInfo. + pub fn with_frame_info(frame_info: FrameInfo, wtr: W) -> Self { FrameEncoder { - src, + src: Vec::new(), w: wtr, + // 16 KB hash table for matches, same as the reference implementation. compression_table: HashTable4K::new(), content_hasher: XxHash32::with_seed(0), content_len: 0, - dst, + dst: Vec::new(), is_frame_open: false, frame_info, src_start: 0, @@ -195,8 +202,12 @@ impl FrameEncoder { /// Begin the frame by writing the frame header. /// It'll also setup the encoder for compressing blocks for the the new frame. - fn begin_frame(&mut self) -> io::Result<()> { + fn begin_frame(&mut self, buf_len: usize) -> io::Result<()> { self.is_frame_open = true; + if self.frame_info.block_size == BlockSize::Auto { + self.frame_info.block_size = BlockSize::from_buf_length(buf_len); + } + self.init(); let mut frame_info_buffer = [0u8; MAX_FRAME_INFO_SIZE]; let size = self.frame_info.write(&mut frame_info_buffer)?; self.w.write_all(&frame_info_buffer[..size])?; @@ -336,13 +347,12 @@ impl FrameEncoder { impl io::Write for FrameEncoder { fn write(&mut self, mut buf: &[u8]) -> io::Result { if !self.is_frame_open && !buf.is_empty() { - self.begin_frame()?; + self.begin_frame(buf.len())?; } let buf_len = buf.len(); - let max_block_size = self.frame_info.block_size.get_size(); while !buf.is_empty() { let src_filled = self.src_end - self.src_start; - let max_fill_len = max_block_size - src_filled; + let max_fill_len = self.frame_info.block_size.get_size() - src_filled; if max_fill_len == 0 { // make space by writing next block self.write_block()?; diff --git a/src/frame/header.rs b/src/frame/header.rs index d3e81cc..84fb66f 100644 --- a/src/frame/header.rs +++ b/src/frame/header.rs @@ -37,6 +37,8 @@ pub(crate) const BLOCK_INFO_SIZE: usize = 4; #[derive(Clone, Copy, PartialEq, Debug)] /// Different predefines blocksizes to choose when compressing data. pub enum BlockSize { + /// Will detect optimal frame size based on the size of the first write call + Auto = 0, /// The default block size. Max64KB = 4, /// 256KB block size. @@ -51,13 +53,26 @@ pub enum BlockSize { impl Default for BlockSize { fn default() -> Self { - BlockSize::Max64KB + BlockSize::Auto } } impl BlockSize { + /// Try to find optimal size based on passed buffer length. + pub(crate) fn from_buf_length(buf_len: usize) -> Self { + let mut blocksize = BlockSize::Max4MB; + + for candidate in [BlockSize::Max256KB, BlockSize::Max64KB] { + if buf_len > candidate.get_size() { + return blocksize; + } + blocksize = candidate; + } + return BlockSize::Max64KB; + } pub(crate) fn get_size(&self) -> usize { match self { + BlockSize::Auto => unreachable!(), BlockSize::Max64KB => 64 * 1024, BlockSize::Max256KB => 256 * 1024, BlockSize::Max1MB => 1024 * 1024, diff --git a/tests/tests.rs b/tests/tests.rs index 33460d9..a9de5a0 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -10,7 +10,7 @@ use lz4_compress::compress as lz4_rust_compress; use lz4_flex::frame::BlockMode; use lz4_flex::{ block::{compress_prepend_size, decompress_size_prepended}, - compress, decompress, + compress as compress_block, decompress, }; const COMPRESSION1K: &[u8] = include_bytes!("../benches/compression_1k.txt"); @@ -76,7 +76,7 @@ pub fn lz4_flex_frame_decompress(input: &[u8]) -> Result, lz4_flex::fram fn test_roundtrip(bytes: impl AsRef<[u8]>) { let bytes = bytes.as_ref(); // compress with rust, decompress with rust - let compressed_flex = compress(bytes); + let compressed_flex = compress_block(bytes); let decompressed = decompress(&compressed_flex, bytes.len()).unwrap(); assert_eq!(decompressed, bytes); @@ -115,7 +115,7 @@ fn lz4_cpp_compatibility(bytes: &[u8]) { } // compress with rust, decompress with lz4 cpp - let compressed_flex = compress(bytes); + let compressed_flex = compress_block(bytes); let decompressed = lz4_cpp_block_decompress(&compressed_flex, bytes.len()).unwrap(); assert_eq!(decompressed, bytes); @@ -155,20 +155,42 @@ fn compare_compression() { } #[test] -fn test_minimum_compression_ratio() { - let compressed = compress(COMPRESSION34K); +fn test_minimum_compression_ratio_block() { + let compressed = compress_block(COMPRESSION34K); let ratio = compressed.len() as f64 / COMPRESSION34K.len() as f64; assert_lt!(ratio, 0.585); // TODO check why compression is not deterministic (fails in ci for // 0.58) - let compressed = compress(COMPRESSION65); + let compressed = compress_block(COMPRESSION65); let ratio = compressed.len() as f64 / COMPRESSION65.len() as f64; assert_lt!(ratio, 0.574); - let compressed = compress(COMPRESSION66JSON); + let compressed = compress_block(COMPRESSION66JSON); let ratio = compressed.len() as f64 / COMPRESSION66JSON.len() as f64; assert_lt!(ratio, 0.229); } +#[cfg(feature = "frame")] +#[test] +fn test_minimum_compression_ratio_frame() { + use lz4_flex::frame::FrameInfo; + + let get_ratio = |input| { + let compressed = lz4_flex_frame_compress_with(FrameInfo::new(), input).unwrap(); + + let ratio = compressed.len() as f64 / input.len() as f64; + ratio + }; + + let ratio = get_ratio(COMPRESSION34K); + assert_lt!(ratio, 0.585); + + let ratio = get_ratio(COMPRESSION65); + assert_lt!(ratio, 0.574); + + let ratio = get_ratio(COMPRESSION66JSON); + assert_lt!(ratio, 0.235); +} + use lz_fear::raw::compress2; use lz_fear::raw::U16Table; use lz_fear::raw::U32Table; @@ -184,10 +206,12 @@ fn compress_lz4_fear(input: &[u8]) -> Vec { } fn print_compression_ration(input: &'static [u8], name: &str) { - let compressed = compress(input); + println!("\nComparing for {}", name); + let name = ""; + let compressed = compress_block(input); // println!("{:?}", compressed); println!( - "lz4_flex Compression Ratio {:?} {:?}", + "lz4_flex block Compression Ratio {:?} {:?}", name, compressed.len() as f64 / input.len() as f64 ); @@ -197,7 +221,7 @@ fn print_compression_ration(input: &'static [u8], name: &str) { let compressed = lz4_cpp_block_compress(input).unwrap(); // println!("{:?}", compressed); println!( - "Lz4 Cpp Compression Ratio {:?} {:?}", + "Lz4 Cpp block Compression Ratio {:?} {:?}", name, compressed.len() as f64 / input.len() as f64 ); @@ -206,7 +230,7 @@ fn print_compression_ration(input: &'static [u8], name: &str) { assert_eq!(decompressed, input); let compressed = lz4_rust_compress(input); println!( - "lz4_rust_compress Compression Ratio {:?} {:?}", + "lz4_rust_compress block Compression Ratio {:?} {:?}", name, compressed.len() as f64 / input.len() as f64 ); @@ -214,7 +238,7 @@ fn print_compression_ration(input: &'static [u8], name: &str) { assert_eq!(decompressed, input); let compressed = compress_lz4_fear(input); println!( - "lz4_fear_compress Compression Ratio {:?} {:?}", + "lz4_fear_compress block Compression Ratio {:?} {:?}", name, compressed.len() as f64 / input.len() as f64 ); @@ -225,6 +249,42 @@ fn print_compression_ration(input: &'static [u8], name: &str) { name, compressed.len() as f64 / input.len() as f64 ); + + #[cfg(feature = "frame")] + { + let mut frame_info = lz4_flex::frame::FrameInfo::new(); + frame_info.block_mode = BlockMode::Independent; + //frame_info.block_size = lz4_flex::frame::BlockSize::Max4MB; + let compressed = lz4_flex_frame_compress_with(frame_info, input).unwrap(); + println!( + "lz4_flex frame indep Compression Ratio {:?} {:?}", + name, + compressed.len() as f64 / input.len() as f64 + ); + + let mut frame_info = lz4_flex::frame::FrameInfo::new(); + frame_info.block_mode = BlockMode::Linked; + let compressed = lz4_flex_frame_compress_with(frame_info, input).unwrap(); + println!( + "lz4_flex frame linked Compression Ratio {:?} {:?}", + name, + compressed.len() as f64 / input.len() as f64 + ); + + let compressed = lz4_cpp_frame_compress(input, true).unwrap(); + println!( + "lz4 cpp frame indep Compression Ratio {:?} {:?}", + name, + compressed.len() as f64 / input.len() as f64 + ); + + let compressed = lz4_cpp_frame_compress(input, false).unwrap(); + println!( + "lz4 cpp frame linked Compression Ratio {:?} {:?}", + name, + compressed.len() as f64 / input.len() as f64 + ); + } } // #[test] @@ -430,7 +490,7 @@ fn compression_works() { The len method has a default implementation, so you usually shouldn't implement it. However, you may be able to provide a more performant implementation than the default, so overriding it in this case makes sense."#; test_roundtrip(s); - assert!(compress(s.as_bytes()).len() < s.len()); + assert!(compress_block(s.as_bytes()).len() < s.len()); } // #[test] @@ -649,12 +709,12 @@ mod test_compression { print_ratio( "Ratio 1k flex", COMPRESSION1K.len(), - compress(COMPRESSION1K).len(), + compress_block(COMPRESSION1K).len(), ); print_ratio( "Ratio 34k flex", COMPRESSION34K.len(), - compress(COMPRESSION34K).len(), + compress_block(COMPRESSION34K).len(), ); }