feat(compress): move external block summary to Rust
The explicit-delimiter sequence path still calculated each block summary in C, with a separate AVX2 implementation and scalar fallback. Move that pure scan to zstd_compress_stats.rs and keep a C wrapper for the existing orchestration. The Rust leaf preserves delimiter inclusion, literal and match totals, and the external-sequences error when no delimiter is present, while the C ABI is pinned with a repr(C) layout check. Test Plan: - `cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression` -- 227 passed - `cargo clippy --manifest-path rust/Cargo.toml --no-default-features --features compression` plus benches/tests -- passed before and after formatting - `cargo +nightly fmt --manifest-path rust/Cargo.toml --all` -- passed - `make -B -C lib -j2 lib` -- passed - `make -C tests -j2 test-zstream` -- passed, including 84 deterministic and 15,464 randomized cases
This commit is contained in:
@@ -268,6 +268,8 @@ size_t ZSTD_rust_fastSequenceLengthSum(const ZSTD_Sequence* seqBuf,
|
|||||||
size_t seqBufSize);
|
size_t seqBufSize);
|
||||||
size_t ZSTD_rust_convertSequencesNoRepcodes(
|
size_t ZSTD_rust_convertSequencesNoRepcodes(
|
||||||
SeqDef* dstSeqs, const ZSTD_Sequence* inSeqs, size_t nbSequences);
|
SeqDef* dstSeqs, const ZSTD_Sequence* inSeqs, size_t nbSequences);
|
||||||
|
BlockSummary ZSTD_rust_get1BlockSummary(const ZSTD_Sequence* seqs,
|
||||||
|
size_t nbSeqs);
|
||||||
int ZSTD_rust_isRLE(const BYTE* src, size_t length);
|
int ZSTD_rust_isRLE(const BYTE* src, size_t length);
|
||||||
int ZSTD_rust_maybeRLE(const SeqStore_t* seqStore);
|
int ZSTD_rust_maybeRLE(const SeqStore_t* seqStore);
|
||||||
size_t ZSTD_rust_postProcessSequenceProducerResult(
|
size_t ZSTD_rust_postProcessSequenceProducerResult(
|
||||||
@@ -299,6 +301,8 @@ size_t ZSTD_rust_optimalBlockSize(const void* src, size_t srcSize,
|
|||||||
void* workspace, size_t workspaceSize);
|
void* workspace, size_t workspaceSize);
|
||||||
|
|
||||||
typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
|
typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
|
||||||
|
typedef char ZSTD_rust_stats_block_summary_layout[
|
||||||
|
(sizeof(BlockSummary) == 3 * sizeof(size_t)) ? 1 : -1];
|
||||||
typedef char ZSTD_rust_stats_seqstore_long_length_pos[
|
typedef char ZSTD_rust_stats_seqstore_long_length_pos[
|
||||||
(offsetof(SeqStore_t, longLengthPos) == 9 * sizeof(size_t) + 4) ? 1 : -1];
|
(offsetof(SeqStore_t, longLengthPos) == 9 * sizeof(size_t) + 4) ? 1 : -1];
|
||||||
typedef char ZSTD_rust_stats_seqstore_layout[
|
typedef char ZSTD_rust_stats_seqstore_layout[
|
||||||
@@ -5843,88 +5847,11 @@ size_t ZSTD_convertBlockSequences(ZSTD_CCtx* cctx,
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
#if defined(ZSTD_ARCH_X86_AVX2)
|
|
||||||
|
|
||||||
BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs)
|
BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs)
|
||||||
{
|
{
|
||||||
size_t i;
|
return ZSTD_rust_get1BlockSummary(seqs, nbSeqs);
|
||||||
__m256i const zeroVec = _mm256_setzero_si256();
|
|
||||||
__m256i sumVec = zeroVec; /* accumulates match+lit in 32-bit lanes */
|
|
||||||
ZSTD_ALIGNED(32) U32 tmp[8]; /* temporary buffer for reduction */
|
|
||||||
size_t mSum = 0, lSum = 0;
|
|
||||||
ZSTD_STATIC_ASSERT(sizeof(ZSTD_Sequence) == 16);
|
|
||||||
|
|
||||||
/* Process 2 structs (32 bytes) at a time */
|
|
||||||
for (i = 0; i + 2 <= nbSeqs; i += 2) {
|
|
||||||
/* Load two consecutive ZSTD_Sequence (8×4 = 32 bytes) */
|
|
||||||
__m256i data = _mm256_loadu_si256((const __m256i*)(const void*)&seqs[i]);
|
|
||||||
/* check end of block signal */
|
|
||||||
__m256i cmp = _mm256_cmpeq_epi32(data, zeroVec);
|
|
||||||
int cmp_res = _mm256_movemask_epi8(cmp);
|
|
||||||
/* indices for match lengths correspond to bits [8..11], [24..27]
|
|
||||||
* => combined mask = 0x0F000F00 */
|
|
||||||
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, matchLength) == 8);
|
|
||||||
if (cmp_res & 0x0F000F00) break;
|
|
||||||
/* Accumulate in sumVec */
|
|
||||||
sumVec = _mm256_add_epi32(sumVec, data);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Horizontal reduction */
|
|
||||||
_mm256_store_si256((__m256i*)tmp, sumVec);
|
|
||||||
lSum = tmp[1] + tmp[5];
|
|
||||||
mSum = tmp[2] + tmp[6];
|
|
||||||
|
|
||||||
/* Handle the leftover */
|
|
||||||
for (; i < nbSeqs; i++) {
|
|
||||||
lSum += seqs[i].litLength;
|
|
||||||
mSum += seqs[i].matchLength;
|
|
||||||
if (seqs[i].matchLength == 0) break; /* end of block */
|
|
||||||
}
|
|
||||||
|
|
||||||
if (i==nbSeqs) {
|
|
||||||
/* reaching end of sequences: end of block signal was not present */
|
|
||||||
BlockSummary bs;
|
|
||||||
bs.nbSequences = ERROR(externalSequences_invalid);
|
|
||||||
return bs;
|
|
||||||
}
|
|
||||||
{ BlockSummary bs;
|
|
||||||
bs.nbSequences = i+1;
|
|
||||||
bs.blockSize = lSum + mSum;
|
|
||||||
bs.litSize = lSum;
|
|
||||||
return bs;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#else
|
|
||||||
|
|
||||||
BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs)
|
|
||||||
{
|
|
||||||
size_t totalMatchSize = 0;
|
|
||||||
size_t litSize = 0;
|
|
||||||
size_t n;
|
|
||||||
assert(seqs);
|
|
||||||
for (n=0; n<nbSeqs; n++) {
|
|
||||||
totalMatchSize += seqs[n].matchLength;
|
|
||||||
litSize += seqs[n].litLength;
|
|
||||||
if (seqs[n].matchLength == 0) {
|
|
||||||
assert(seqs[n].offset == 0);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (n==nbSeqs) {
|
|
||||||
BlockSummary bs;
|
|
||||||
bs.nbSequences = ERROR(externalSequences_invalid);
|
|
||||||
return bs;
|
|
||||||
}
|
|
||||||
{ BlockSummary bs;
|
|
||||||
bs.nbSequences = n+1;
|
|
||||||
bs.blockSize = litSize + totalMatchSize;
|
|
||||||
bs.litSize = litSize;
|
|
||||||
return bs;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
|
|
||||||
static size_t
|
static size_t
|
||||||
ZSTD_compressSequencesAndLiterals_internal(ZSTD_CCtx* cctx,
|
ZSTD_compressSequencesAndLiterals_internal(ZSTD_CCtx* cctx,
|
||||||
|
|||||||
@@ -158,6 +158,15 @@ pub struct ZSTD_Sequence {
|
|||||||
pub rep: u32,
|
pub rep: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// ABI-compatible `BlockSummary` from `zstd_compress_internal.h`.
|
||||||
|
#[repr(C)]
|
||||||
|
#[derive(Clone, Copy, Debug, Default)]
|
||||||
|
pub struct BlockSummary {
|
||||||
|
pub nbSequences: usize,
|
||||||
|
pub blockSize: usize,
|
||||||
|
pub litSize: usize,
|
||||||
|
}
|
||||||
|
|
||||||
/// Converts public sequences to the internal no-repcodes `SeqDef` format.
|
/// Converts public sequences to the internal no-repcodes `SeqDef` format.
|
||||||
///
|
///
|
||||||
/// The return value is a side-band marker: zero means that every length fits
|
/// The return value is a side-band marker: zero means that every length fits
|
||||||
@@ -198,6 +207,45 @@ pub unsafe extern "C" fn ZSTD_rust_convertSequencesNoRepcodes(
|
|||||||
long_length
|
long_length
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Finds the first explicit block delimiter and totals the sequences before
|
||||||
|
/// and including it. This is the Rust leaf for C's
|
||||||
|
/// `ZSTD_get1BlockSummary()`; the delimiter is identified by zero match
|
||||||
|
/// length and contributes its literal length to the totals.
|
||||||
|
#[no_mangle]
|
||||||
|
pub unsafe extern "C" fn ZSTD_rust_get1BlockSummary(
|
||||||
|
seqs: *const ZSTD_Sequence,
|
||||||
|
nb_seqs: usize,
|
||||||
|
) -> BlockSummary {
|
||||||
|
let invalid = || BlockSummary {
|
||||||
|
nbSequences: ERROR(ZstdErrorCode::ExternalSequencesInvalid),
|
||||||
|
blockSize: 0,
|
||||||
|
litSize: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
if nb_seqs == 0 || seqs.is_null() {
|
||||||
|
return invalid();
|
||||||
|
}
|
||||||
|
|
||||||
|
let seqs = unsafe { std::slice::from_raw_parts(seqs, nb_seqs) };
|
||||||
|
let mut total_match_size = 0usize;
|
||||||
|
let mut lit_size = 0usize;
|
||||||
|
|
||||||
|
for (index, sequence) in seqs.iter().enumerate() {
|
||||||
|
total_match_size = total_match_size.wrapping_add(sequence.matchLength as usize);
|
||||||
|
lit_size = lit_size.wrapping_add(sequence.litLength as usize);
|
||||||
|
if sequence.matchLength == 0 {
|
||||||
|
debug_assert_eq!(sequence.offset, 0);
|
||||||
|
return BlockSummary {
|
||||||
|
nbSequences: index + 1,
|
||||||
|
blockSize: lit_size.wrapping_add(total_match_size),
|
||||||
|
litSize: lit_size,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
invalid()
|
||||||
|
}
|
||||||
|
|
||||||
/// Converts a raw sequence offset to the stored offBase representation.
|
/// Converts a raw sequence offset to the stored offBase representation.
|
||||||
///
|
///
|
||||||
/// This is the Rust leaf for C's `ZSTD_finalizeOffBase()`. The repcode
|
/// This is the Rust leaf for C's `ZSTD_finalizeOffBase()`. The repcode
|
||||||
@@ -1802,6 +1850,9 @@ mod tests {
|
|||||||
fn c_leaf_layouts_match_supported_abis() {
|
fn c_leaf_layouts_match_supported_abis() {
|
||||||
assert_eq!(size_of::<SeqDef>(), 8);
|
assert_eq!(size_of::<SeqDef>(), 8);
|
||||||
assert_eq!(align_of::<SeqDef>(), align_of::<u32>());
|
assert_eq!(align_of::<SeqDef>(), align_of::<u32>());
|
||||||
|
assert_eq!(size_of::<BlockSummary>(), 3 * size_of::<usize>());
|
||||||
|
assert_eq!(offset_of!(BlockSummary, blockSize), size_of::<usize>());
|
||||||
|
assert_eq!(offset_of!(BlockSummary, litSize), 2 * size_of::<usize>());
|
||||||
assert_eq!(offset_of!(SeqStore_t, sequencesStart), 0);
|
assert_eq!(offset_of!(SeqStore_t, sequencesStart), 0);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
offset_of!(SeqStore_t, longLengthPos),
|
offset_of!(SeqStore_t, longLengthPos),
|
||||||
@@ -1892,6 +1943,57 @@ mod tests {
|
|||||||
assert_eq!(result, 0);
|
assert_eq!(result, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn block_summary_stops_at_and_includes_the_first_delimiter() {
|
||||||
|
let sequences = [
|
||||||
|
ZSTD_Sequence {
|
||||||
|
offset: 7,
|
||||||
|
litLength: 3,
|
||||||
|
matchLength: 4,
|
||||||
|
rep: 0,
|
||||||
|
},
|
||||||
|
ZSTD_Sequence {
|
||||||
|
offset: 0,
|
||||||
|
litLength: 5,
|
||||||
|
matchLength: 0,
|
||||||
|
rep: 0,
|
||||||
|
},
|
||||||
|
ZSTD_Sequence {
|
||||||
|
offset: 11,
|
||||||
|
litLength: 99,
|
||||||
|
matchLength: 100,
|
||||||
|
rep: 0,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
let summary = unsafe { ZSTD_rust_get1BlockSummary(sequences.as_ptr(), sequences.len()) };
|
||||||
|
|
||||||
|
assert_eq!(summary.nbSequences, 2);
|
||||||
|
assert_eq!(summary.blockSize, 12);
|
||||||
|
assert_eq!(summary.litSize, 8);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn block_summary_rejects_missing_or_empty_delimiters() {
|
||||||
|
let sequences = [ZSTD_Sequence {
|
||||||
|
offset: 7,
|
||||||
|
litLength: 3,
|
||||||
|
matchLength: 4,
|
||||||
|
rep: 0,
|
||||||
|
}];
|
||||||
|
let missing = unsafe { ZSTD_rust_get1BlockSummary(sequences.as_ptr(), sequences.len()) };
|
||||||
|
assert_eq!(
|
||||||
|
missing.nbSequences,
|
||||||
|
ERROR(ZstdErrorCode::ExternalSequencesInvalid)
|
||||||
|
);
|
||||||
|
|
||||||
|
let empty = unsafe { ZSTD_rust_get1BlockSummary(std::ptr::null(), 0) };
|
||||||
|
assert_eq!(
|
||||||
|
empty.nbSequences,
|
||||||
|
ERROR(ZstdErrorCode::ExternalSequencesInvalid)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn compressed_block_state_reset_restores_repcodes_and_repeat_modes() {
|
fn compressed_block_state_reset_restores_repcodes_and_repeat_modes() {
|
||||||
let mut block_state =
|
let mut block_state =
|
||||||
|
|||||||
Reference in New Issue
Block a user