ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move external block summary to Rust

Browse Source 
The explicit-delimiter sequence path still calculated each block summary in C,
with a separate AVX2 implementation and scalar fallback. Move that pure scan
to zstd_compress_stats.rs and keep a C wrapper for the existing orchestration.
The Rust leaf preserves delimiter inclusion, literal and match totals, and the
external-sequences error when no delimiter is present, while the C ABI is pinned
with a repr(C) layout check.

Test Plan:
- `cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression` -- 227 passed
- `cargo clippy --manifest-path rust/Cargo.toml --no-default-features --features compression` plus benches/tests -- passed before and after formatting
- `cargo +nightly fmt --manifest-path rust/Cargo.toml --all` -- passed
- `make -B -C lib -j2 lib` -- passed
- `make -C tests -j2 test-zstream` -- passed, including 84 deterministic and 15,464 randomized cases
This commit is contained in:
ddidderr
2026-07-18 06:56:38 +02:00
parent 18f5419cc6
commit d334aaafa3
2 changed files with 107 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+5 -78
View File
@@ -268,6 +268,8 @@ size_t ZSTD_rust_fastSequenceLengthSum(const ZSTD_Sequence* seqBuf,
size_t seqBufSize);
size_t ZSTD_rust_convertSequencesNoRepcodes(
SeqDef* dstSeqs, const ZSTD_Sequence* inSeqs, size_t nbSequences);
BlockSummary ZSTD_rust_get1BlockSummary(const ZSTD_Sequence* seqs,
size_t nbSeqs);
int ZSTD_rust_isRLE(const BYTE* src, size_t length);
int ZSTD_rust_maybeRLE(const SeqStore_t* seqStore);
size_t ZSTD_rust_postProcessSequenceProducerResult(
@@ -299,6 +301,8 @@ size_t ZSTD_rust_optimalBlockSize(const void* src, size_t srcSize,
void* workspace, size_t workspaceSize);
typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
typedef char ZSTD_rust_stats_block_summary_layout[
(sizeof(BlockSummary) == 3 * sizeof(size_t)) ? 1 : -1];
typedef char ZSTD_rust_stats_seqstore_long_length_pos[
(offsetof(SeqStore_t, longLengthPos) == 9 * sizeof(size_t) + 4) ? 1 : -1];
typedef char ZSTD_rust_stats_seqstore_layout[
@@ -5843,88 +5847,11 @@ size_t ZSTD_convertBlockSequences(ZSTD_CCtx* cctx,
return 0;
}
#if defined(ZSTD_ARCH_X86_AVX2)
BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs)
{
size_t i;
__m256i const zeroVec = _mm256_setzero_si256();
__m256i sumVec = zeroVec; /* accumulates match+lit in 32-bit lanes */
ZSTD_ALIGNED(32) U32 tmp[8]; /* temporary buffer for reduction */
size_t mSum = 0, lSum = 0;
ZSTD_STATIC_ASSERT(sizeof(ZSTD_Sequence) == 16);
/* Process 2 structs (32 bytes) at a time */
for (i = 0; i + 2 <= nbSeqs; i += 2) {
/* Load two consecutive ZSTD_Sequence (8×4 = 32 bytes) */
__m256i data = _mm256_loadu_si256((const __m256i*)(const void*)&seqs[i]);
/* check end of block signal */
__m256i cmp = _mm256_cmpeq_epi32(data, zeroVec);
int cmp_res = _mm256_movemask_epi8(cmp);
/* indices for match lengths correspond to bits [8..11], [24..27]
* => combined mask = 0x0F000F00 */
ZSTD_STATIC_ASSERT(offsetof(ZSTD_Sequence, matchLength) == 8);
if (cmp_res & 0x0F000F00) break;
/* Accumulate in sumVec */
sumVec = _mm256_add_epi32(sumVec, data);
}
/* Horizontal reduction */
_mm256_store_si256((__m256i*)tmp, sumVec);
lSum = tmp[1] + tmp[5];
mSum = tmp[2] + tmp[6];
/* Handle the leftover */
for (; i < nbSeqs; i++) {
lSum += seqs[i].litLength;
mSum += seqs[i].matchLength;
if (seqs[i].matchLength == 0) break; /* end of block */
}
if (i==nbSeqs) {
/* reaching end of sequences: end of block signal was not present */
BlockSummary bs;
bs.nbSequences = ERROR(externalSequences_invalid);
return bs;
}
{ BlockSummary bs;
bs.nbSequences = i+1;
bs.blockSize = lSum + mSum;
bs.litSize = lSum;
return bs;
}
return ZSTD_rust_get1BlockSummary(seqs, nbSeqs);
}
#else
BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs)
{
size_t totalMatchSize = 0;
size_t litSize = 0;
size_t n;
assert(seqs);
for (n=0; n<nbSeqs; n++) {
totalMatchSize += seqs[n].matchLength;
litSize += seqs[n].litLength;
if (seqs[n].matchLength == 0) {
assert(seqs[n].offset == 0);
break;
}
}
if (n==nbSeqs) {
BlockSummary bs;
bs.nbSequences = ERROR(externalSequences_invalid);
return bs;
}
{ BlockSummary bs;
bs.nbSequences = n+1;
bs.blockSize = litSize + totalMatchSize;
bs.litSize = litSize;
return bs;
}
}
#endif
static size_t
ZSTD_compressSequencesAndLiterals_internal(ZSTD_CCtx* cctx,