ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move seq-store leaves to Rust

Browse Source 
Move compressed-block state reset, final-literal storage, sequence-store reset, sequence-length summation, RLE detection, and RLE heuristics into the Rust compression statistics module. Preserve the C helper surface and add focused pointer/layout and boundary tests.

Test Plan: cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression; cargo clippy --manifest-path rust/Cargo.toml; cargo clippy --manifest-path rust/Cargo.toml --benches; cargo clippy --manifest-path rust/Cargo.toml --tests; make -B -C lib -j2 lib; make -B -C programs -j2 zstd zstd-small zstd-frugal; make -B -C tests -j2 test-cli-tests; make -B -C tests -j2 test-zstream
This commit is contained in:
ddidderr
2026-07-18 03:55:57 +02:00
parent 8ac9855309
commit 1edec2cc0e
2 changed files with 304 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+14 -43
View File
@@ -233,6 +233,14 @@ size_t ZSTD_rust_buildBlockEntropyStats(
size_t ZSTD_rust_copyBlockSequences(
SeqCollector* seqCollector, const SeqStore_t* seqStore,
const U32 prevRepcodes[ZSTD_REP_NUM]);
void ZSTD_rust_resetCompressedBlockState(ZSTD_compressedBlockState_t* bs);
void ZSTD_rust_storeLastLiterals(SeqStore_t* seqStorePtr,
const BYTE* anchor, size_t lastLLSize);
void ZSTD_rust_resetSeqStore(SeqStore_t* ssPtr);
size_t ZSTD_rust_fastSequenceLengthSum(const ZSTD_Sequence* seqBuf,
size_t seqBufSize);
int ZSTD_rust_isRLE(const BYTE* src, size_t length);
int ZSTD_rust_maybeRLE(const SeqStore_t* seqStore);
typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
typedef char ZSTD_rust_stats_seqstore_long_length_pos[
@@ -1373,13 +1381,7 @@ static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1,
void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs)
{
int i;
for (i = 0; i < ZSTD_REP_NUM; ++i)
bs->rep[i] = repStartValue[i];
bs->entropy.huf.repeatMode = HUF_repeat_none;
bs->entropy.fse.offcode_repeatMode = FSE_repeat_none;
bs->entropy.fse.matchlength_repeatMode = FSE_repeat_none;
bs->entropy.fse.litlength_repeatMode = FSE_repeat_none;
ZSTD_rust_resetCompressedBlockState(bs);
}
/*! ZSTD_invalidateMatchState()
@@ -2307,15 +2309,12 @@ ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_Para
static void ZSTD_storeLastLiterals(SeqStore_t* seqStorePtr,
const BYTE* anchor, size_t lastLLSize)
{
ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize;
ZSTD_rust_storeLastLiterals(seqStorePtr, anchor, lastLLSize);
}
void ZSTD_resetSeqStore(SeqStore_t* ssPtr)
{
ssPtr->lit = ssPtr->litStart;
ssPtr->sequences = ssPtr->sequencesStart;
ssPtr->longLengthType = ZSTD_llt_none;
ZSTD_rust_resetSeqStore(ssPtr);
}
/* ZSTD_postProcessSequenceProducerResult() :
@@ -2374,14 +2373,7 @@ static size_t ZSTD_postProcessSequenceProducerResult(
* Removing the early exit allows the compiler to auto-vectorize (https://godbolt.org/z/cY1cajz9P).
* This function can be deleted and replaced by determine_blockSize after we resolve issue #3456. */
static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seqBufSize) {
size_t matchLenSum, litLenSum, i;
matchLenSum = 0;
litLenSum = 0;
for (i = 0; i < seqBufSize; i++) {
litLenSum += seqBuf[i].litLength;
matchLenSum += seqBuf[i].matchLength;
}
return litLenSum + matchLenSum;
return ZSTD_rust_fastSequenceLengthSum(seqBuf, seqBufSize);
}
/**
@@ -2626,25 +2618,7 @@ size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
/* Unrolled loop to read four size_ts of input at a time. Returns 1 if is RLE, 0 if not. */
static int ZSTD_isRLE(const BYTE* src, size_t length) {
const BYTE* ip = src;
const BYTE value = ip[0];
const size_t valueST = (size_t)((U64)value * 0x0101010101010101ULL);
const size_t unrollSize = sizeof(size_t) * 4;
const size_t unrollMask = unrollSize - 1;
const size_t prefixLength = length & unrollMask;
size_t i;
if (length == 1) return 1;
/* Check if prefix is RLE first before using unrolled loop */
if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) {
return 0;
}
for (i = prefixLength; i != length; i += unrollSize) {
size_t u;
for (u = 0; u < unrollSize; u += sizeof(size_t)) {
if (MEM_readST(ip + i + u) != valueST) {
return 0;
} } }
return 1;
return ZSTD_rust_isRLE(src, length);
}
/* Returns true if the given block may be RLE.
@@ -2653,10 +2627,7 @@ static int ZSTD_isRLE(const BYTE* src, size_t length) {
*/
static int ZSTD_maybeRLE(SeqStore_t const* seqStore)
{
size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart);
return nbSeqs < 4 && nbLits < 10;
return ZSTD_rust_maybeRLE(seqStore);
}
static void