ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move sequence leaves to Rust

Browse Source 
Move sequence-store byte accounting, chunk derivation, and repcode resolution into the Rust compression module. Move MT raw-sequence buffer conversions into Rust while preserving the existing C adapters and ABI layouts.

Test Plan:\n- cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression\n- cargo clippy --manifest-path rust/Cargo.toml\n- cargo clippy --manifest-path rust/Cargo.toml --benches\n- cargo clippy --manifest-path rust/Cargo.toml --tests\n- make -B -C lib -j2 lib\n- make -B -C tests -j2 test-cli-tests
This commit is contained in:
ddidderr
2026-07-18 04:05:48 +02:00
parent 1edec2cc0e
commit bdb14b4fbf
4 changed files with 460 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+12 -59
View File
@@ -241,6 +241,13 @@ size_t ZSTD_rust_fastSequenceLengthSum(const ZSTD_Sequence* seqBuf,
size_t seqBufSize);
int ZSTD_rust_isRLE(const BYTE* src, size_t length);
int ZSTD_rust_maybeRLE(const SeqStore_t* seqStore);
size_t ZSTD_rust_countSeqStoreLiteralsBytes(const SeqStore_t* seqStore);
size_t ZSTD_rust_countSeqStoreMatchBytes(const SeqStore_t* seqStore);
void ZSTD_rust_deriveSeqStoreChunk(SeqStore_t* resultSeqStore,
const SeqStore_t* originalSeqStore,
size_t startIdx, size_t endIdx);
U32 ZSTD_rust_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM],
U32 offBase, U32 ll0);
typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
typedef char ZSTD_rust_stats_seqstore_long_length_pos[
@@ -2817,31 +2824,13 @@ ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(SeqStore_t* seqStore, ZSTD_CC
/* Returns literals bytes represented in a seqStore */
static size_t ZSTD_countSeqStoreLiteralsBytes(const SeqStore_t* const seqStore)
{
size_t literalsBytes = 0;
size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
size_t i;
for (i = 0; i < nbSeqs; ++i) {
SeqDef const seq = seqStore->sequencesStart[i];
literalsBytes += seq.litLength;
if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) {
literalsBytes += 0x10000;
} }
return literalsBytes;
return ZSTD_rust_countSeqStoreLiteralsBytes(seqStore);
}
/* Returns match bytes represented in a seqStore */
static size_t ZSTD_countSeqStoreMatchBytes(const SeqStore_t* const seqStore)
{
size_t matchBytes = 0;
size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
size_t i;
for (i = 0; i < nbSeqs; ++i) {
SeqDef seq = seqStore->sequencesStart[i];
matchBytes += seq.mlBase + MINMATCH;
if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) {
matchBytes += 0x10000;
} }
return matchBytes;
return ZSTD_rust_countSeqStoreMatchBytes(seqStore);
}
/* Derives the seqStore that is a chunk of the originalSeqStore from [startIdx, endIdx).
@@ -2851,32 +2840,8 @@ static void ZSTD_deriveSeqStoreChunk(SeqStore_t* resultSeqStore,
const SeqStore_t* originalSeqStore,
size_t startIdx, size_t endIdx)
{
*resultSeqStore = *originalSeqStore;
if (startIdx > 0) {
resultSeqStore->sequences = originalSeqStore->sequencesStart + startIdx;
resultSeqStore->litStart += ZSTD_countSeqStoreLiteralsBytes(resultSeqStore);
}
/* Move longLengthPos into the correct position if necessary */
if (originalSeqStore->longLengthType != ZSTD_llt_none) {
if (originalSeqStore->longLengthPos < startIdx || originalSeqStore->longLengthPos > endIdx) {
resultSeqStore->longLengthType = ZSTD_llt_none;
} else {
resultSeqStore->longLengthPos -= (U32)startIdx;
}
}
resultSeqStore->sequencesStart = originalSeqStore->sequencesStart + startIdx;
resultSeqStore->sequences = originalSeqStore->sequencesStart + endIdx;
if (endIdx == (size_t)(originalSeqStore->sequences - originalSeqStore->sequencesStart)) {
/* This accounts for possible last literals if the derived chunk reaches the end of the block */
assert(resultSeqStore->lit == originalSeqStore->lit);
} else {
size_t const literalsBytes = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore);
resultSeqStore->lit = resultSeqStore->litStart + literalsBytes;
}
resultSeqStore->llCode += startIdx;
resultSeqStore->mlCode += startIdx;
resultSeqStore->ofCode += startIdx;
ZSTD_rust_deriveSeqStoreChunk(resultSeqStore, originalSeqStore,
startIdx, endIdx);
}
/**
@@ -2886,20 +2851,8 @@ static void ZSTD_deriveSeqStoreChunk(SeqStore_t* resultSeqStore,
static U32
ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, const U32 ll0)
{
U32 const adjustedRepCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; /* [ 0 - 3 ] */
assert(OFFBASE_IS_REPCODE(offBase));
if (adjustedRepCode == ZSTD_REP_NUM) {
assert(ll0);
/* litlength == 0 and offCode == 2 implies selection of first repcode - 1
* This is only valid if it results in a valid offset value, aka > 0.
* Note : it may happen that `rep[0]==1` in exceptional circumstances.
* In which case this function will return 0, which is an invalid offset.
* It's not an issue though, since this value will be
* compared and discarded within ZSTD_seqStore_resolveOffCodes().
*/
return rep[0] - 1;
}
return rep[adjustedRepCode];
return ZSTD_rust_resolveRepcodeToRawOffset(rep, offBase, ll0);
}
/**