feat(compress): move external block summary to Rust
The explicit-delimiter sequence path still calculated each block summary in C, with a separate AVX2 implementation and scalar fallback. Move that pure scan to zstd_compress_stats.rs and keep a C wrapper for the existing orchestration. The Rust leaf preserves delimiter inclusion, literal and match totals, and the external-sequences error when no delimiter is present, while the C ABI is pinned with a repr(C) layout check. Test Plan: - `cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression` -- 227 passed - `cargo clippy --manifest-path rust/Cargo.toml --no-default-features --features compression` plus benches/tests -- passed before and after formatting - `cargo +nightly fmt --manifest-path rust/Cargo.toml --all` -- passed - `make -B -C lib -j2 lib` -- passed - `make -C tests -j2 test-zstream` -- passed, including 84 deterministic and 15,464 randomized cases
This commit is contained in:
@@ -158,6 +158,15 @@ pub struct ZSTD_Sequence {
|
||||
pub rep: u32,
|
||||
}
|
||||
|
||||
/// ABI-compatible `BlockSummary` from `zstd_compress_internal.h`.
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct BlockSummary {
|
||||
pub nbSequences: usize,
|
||||
pub blockSize: usize,
|
||||
pub litSize: usize,
|
||||
}
|
||||
|
||||
/// Converts public sequences to the internal no-repcodes `SeqDef` format.
|
||||
///
|
||||
/// The return value is a side-band marker: zero means that every length fits
|
||||
@@ -198,6 +207,45 @@ pub unsafe extern "C" fn ZSTD_rust_convertSequencesNoRepcodes(
|
||||
long_length
|
||||
}
|
||||
|
||||
/// Finds the first explicit block delimiter and totals the sequences before
|
||||
/// and including it. This is the Rust leaf for C's
|
||||
/// `ZSTD_get1BlockSummary()`; the delimiter is identified by zero match
|
||||
/// length and contributes its literal length to the totals.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_get1BlockSummary(
|
||||
seqs: *const ZSTD_Sequence,
|
||||
nb_seqs: usize,
|
||||
) -> BlockSummary {
|
||||
let invalid = || BlockSummary {
|
||||
nbSequences: ERROR(ZstdErrorCode::ExternalSequencesInvalid),
|
||||
blockSize: 0,
|
||||
litSize: 0,
|
||||
};
|
||||
|
||||
if nb_seqs == 0 || seqs.is_null() {
|
||||
return invalid();
|
||||
}
|
||||
|
||||
let seqs = unsafe { std::slice::from_raw_parts(seqs, nb_seqs) };
|
||||
let mut total_match_size = 0usize;
|
||||
let mut lit_size = 0usize;
|
||||
|
||||
for (index, sequence) in seqs.iter().enumerate() {
|
||||
total_match_size = total_match_size.wrapping_add(sequence.matchLength as usize);
|
||||
lit_size = lit_size.wrapping_add(sequence.litLength as usize);
|
||||
if sequence.matchLength == 0 {
|
||||
debug_assert_eq!(sequence.offset, 0);
|
||||
return BlockSummary {
|
||||
nbSequences: index + 1,
|
||||
blockSize: lit_size.wrapping_add(total_match_size),
|
||||
litSize: lit_size,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
invalid()
|
||||
}
|
||||
|
||||
/// Converts a raw sequence offset to the stored offBase representation.
|
||||
///
|
||||
/// This is the Rust leaf for C's `ZSTD_finalizeOffBase()`. The repcode
|
||||
@@ -1802,6 +1850,9 @@ mod tests {
|
||||
fn c_leaf_layouts_match_supported_abis() {
|
||||
assert_eq!(size_of::<SeqDef>(), 8);
|
||||
assert_eq!(align_of::<SeqDef>(), align_of::<u32>());
|
||||
assert_eq!(size_of::<BlockSummary>(), 3 * size_of::<usize>());
|
||||
assert_eq!(offset_of!(BlockSummary, blockSize), size_of::<usize>());
|
||||
assert_eq!(offset_of!(BlockSummary, litSize), 2 * size_of::<usize>());
|
||||
assert_eq!(offset_of!(SeqStore_t, sequencesStart), 0);
|
||||
assert_eq!(
|
||||
offset_of!(SeqStore_t, longLengthPos),
|
||||
@@ -1892,6 +1943,57 @@ mod tests {
|
||||
assert_eq!(result, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn block_summary_stops_at_and_includes_the_first_delimiter() {
|
||||
let sequences = [
|
||||
ZSTD_Sequence {
|
||||
offset: 7,
|
||||
litLength: 3,
|
||||
matchLength: 4,
|
||||
rep: 0,
|
||||
},
|
||||
ZSTD_Sequence {
|
||||
offset: 0,
|
||||
litLength: 5,
|
||||
matchLength: 0,
|
||||
rep: 0,
|
||||
},
|
||||
ZSTD_Sequence {
|
||||
offset: 11,
|
||||
litLength: 99,
|
||||
matchLength: 100,
|
||||
rep: 0,
|
||||
},
|
||||
];
|
||||
|
||||
let summary = unsafe { ZSTD_rust_get1BlockSummary(sequences.as_ptr(), sequences.len()) };
|
||||
|
||||
assert_eq!(summary.nbSequences, 2);
|
||||
assert_eq!(summary.blockSize, 12);
|
||||
assert_eq!(summary.litSize, 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn block_summary_rejects_missing_or_empty_delimiters() {
|
||||
let sequences = [ZSTD_Sequence {
|
||||
offset: 7,
|
||||
litLength: 3,
|
||||
matchLength: 4,
|
||||
rep: 0,
|
||||
}];
|
||||
let missing = unsafe { ZSTD_rust_get1BlockSummary(sequences.as_ptr(), sequences.len()) };
|
||||
assert_eq!(
|
||||
missing.nbSequences,
|
||||
ERROR(ZstdErrorCode::ExternalSequencesInvalid)
|
||||
);
|
||||
|
||||
let empty = unsafe { ZSTD_rust_get1BlockSummary(std::ptr::null(), 0) };
|
||||
assert_eq!(
|
||||
empty.nbSequences,
|
||||
ERROR(ZstdErrorCode::ExternalSequencesInvalid)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compressed_block_state_reset_restores_repcodes_and_repeat_modes() {
|
||||
let mut block_state =
|
||||
|
||||
Reference in New Issue
Block a user