ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move explicit sequence transfer to Rust

Browse Source 
Move ZSTD_transferSequences_wBlockDelim's delimiter scan, literal copying, SeqDef storage, validation, and repcode bookkeeping into the Rust compression statistics module. Keep ZSTD_CCtx and block-state ownership in C through a narrow scalar/pointer shim, with ABI layout assertions for ZSTD_SequencePosition.

Test Plan:

- cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression

- cargo clippy --manifest-path rust/Cargo.toml --no-default-features --features compression --benches --tests

- make -B -C lib -j2 lib

- make -C tests -j2 test-zstream
This commit is contained in:
ddidderr
2026-07-18 07:24:18 +02:00
parent 511882e213
commit 77841fcfd2
2 changed files with 574 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+20 -71
View File
@@ -303,6 +303,14 @@ size_t ZSTD_rust_determineBlockSize(int mode, size_t blockSize, size_t remaining
size_t ZSTD_rust_validateSequence(U32 offBase, U32 matchLength, U32 minMatch,
size_t posInSrc, U32 windowLog, size_t dictSize,
int useSequenceProducer);
size_t ZSTD_rust_transferSequencesWBlockDelim(
SeqStore_t* seqStore, ZSTD_SequencePosition* seqPos,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
const BYTE* src, size_t blockSize, int externalRepSearch,
const U32 prevRepcodes[ZSTD_REP_NUM],
U32 nextRepcodes[ZSTD_REP_NUM], U32 dictSize,
int validateSequences, U32 minMatch, U32 windowLog,
int useSequenceProducer);
size_t ZSTD_rust_optimalBlockSize(const void* src, size_t srcSize,
size_t blockSizeMax, int splitLevel,
int strategy, S64 savings,
@@ -330,6 +338,9 @@ typedef char ZSTD_rust_stats_entropy_metadata_layout[
(offsetof(ZSTD_entropyCTablesMetadata_t, fseMetadata)
== sizeof(ZSTD_hufCTablesMetadata_t)) ? 1 : -1];
typedef char ZSTD_rust_stats_sequence_layout[(sizeof(ZSTD_Sequence) == 16) ? 1 : -1];
typedef char ZSTD_rust_stats_sequence_position_layout[
(sizeof(ZSTD_SequencePosition) == 2 * sizeof(U32) + sizeof(size_t)
&& offsetof(ZSTD_SequencePosition, posInSrc) == 2 * sizeof(U32)) ? 1 : -1];
typedef char ZSTD_rust_stats_seqcollector_layout[
(offsetof(SeqCollector, seqStart) == sizeof(size_t)
&& offsetof(SeqCollector, seqIndex) == 2 * sizeof(size_t)
@@ -5244,15 +5255,8 @@ ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx,
const void* src, size_t blockSize,
ZSTD_ParamSwitch_e externalRepSearch)
{
U32 idx = seqPos->idx;
U32 const startIdx = idx;
BYTE const* ip = (BYTE const*)(src);
const BYTE* const iend = ip + blockSize;
Repcodes_t updatedRepcodes;
U32 dictSize;
DEBUGLOG(5, "ZSTD_transferSequences_wBlockDelim (blockSize = %zu)", blockSize);
if (cctx->cdict) {
dictSize = (U32)cctx->cdict->dictContentSize;
} else if (cctx->prefixDict.dict) {
@@ -5260,70 +5264,15 @@ ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx,
} else {
dictSize = 0;
}
ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t));
for (; idx < inSeqsSize && (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0); ++idx) {
U32 const litLength = inSeqs[idx].litLength;
U32 const matchLength = inSeqs[idx].matchLength;
U32 offBase;
if (externalRepSearch == ZSTD_ps_disable) {
offBase = OFFSET_TO_OFFBASE(inSeqs[idx].offset);
} else {
U32 const ll0 = (litLength == 0);
offBase = ZSTD_finalizeOffBase(inSeqs[idx].offset, updatedRepcodes.rep, ll0);
ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0);
}
DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
if (cctx->appliedParams.validateSequences) {
seqPos->posInSrc += litLength + matchLength;
FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch,
seqPos->posInSrc,
cctx->appliedParams.cParams.windowLog, dictSize,
ZSTD_hasExtSeqProd(&cctx->appliedParams)),
"Sequence validation failed");
}
RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
"Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
ip += matchLength + litLength;
}
RETURN_ERROR_IF(idx == inSeqsSize, externalSequences_invalid, "Block delimiter not found.");
/* If we skipped repcode search while parsing, we need to update repcodes now */
assert(externalRepSearch != ZSTD_ps_auto);
assert(idx >= startIdx);
if (externalRepSearch == ZSTD_ps_disable && idx != startIdx) {
U32* const rep = updatedRepcodes.rep;
U32 lastSeqIdx = idx - 1; /* index of last non-block-delimiter sequence */
if (lastSeqIdx >= startIdx + 2) {
rep[2] = inSeqs[lastSeqIdx - 2].offset;
rep[1] = inSeqs[lastSeqIdx - 1].offset;
rep[0] = inSeqs[lastSeqIdx].offset;
} else if (lastSeqIdx == startIdx + 1) {
rep[2] = rep[0];
rep[1] = inSeqs[lastSeqIdx - 1].offset;
rep[0] = inSeqs[lastSeqIdx].offset;
} else {
assert(lastSeqIdx == startIdx);
rep[2] = rep[1];
rep[1] = rep[0];
rep[0] = inSeqs[lastSeqIdx].offset;
}
}
ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t));
if (inSeqs[idx].litLength) {
DEBUGLOG(6, "Storing last literals of size: %u", inSeqs[idx].litLength);
ZSTD_storeLastLiterals(&cctx->seqStore, ip, inSeqs[idx].litLength);
ip += inSeqs[idx].litLength;
seqPos->posInSrc += inSeqs[idx].litLength;
}
RETURN_ERROR_IF(ip != iend, externalSequences_invalid, "Blocksize doesn't agree with block delimiter!");
seqPos->idx = idx+1;
return blockSize;
return ZSTD_rust_transferSequencesWBlockDelim(
&cctx->seqStore, seqPos, inSeqs, inSeqsSize,
(const BYTE*)src, blockSize, (int)externalRepSearch,
cctx->blockState.prevCBlock->rep,
cctx->blockState.nextCBlock->rep, dictSize,
cctx->appliedParams.validateSequences,
cctx->appliedParams.cParams.minMatch,
cctx->appliedParams.cParams.windowLog,
ZSTD_hasExtSeqProd(&cctx->appliedParams));
}
/*