ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move sequence leaves to Rust

Browse Source 
Move sequence-store byte accounting, chunk derivation, and repcode resolution into the Rust compression module. Move MT raw-sequence buffer conversions into Rust while preserving the existing C adapters and ABI layouts.

Test Plan:\n- cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression\n- cargo clippy --manifest-path rust/Cargo.toml\n- cargo clippy --manifest-path rust/Cargo.toml --benches\n- cargo clippy --manifest-path rust/Cargo.toml --tests\n- make -B -C lib -j2 lib\n- make -B -C tests -j2 test-cli-tests
This commit is contained in:
ddidderr
2026-07-18 04:05:48 +02:00
parent 1edec2cc0e
commit bdb14b4fbf
4 changed files with 460 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+12 -59
View File
@@ -241,6 +241,13 @@ size_t ZSTD_rust_fastSequenceLengthSum(const ZSTD_Sequence* seqBuf,
size_t seqBufSize);
int ZSTD_rust_isRLE(const BYTE* src, size_t length);
int ZSTD_rust_maybeRLE(const SeqStore_t* seqStore);
size_t ZSTD_rust_countSeqStoreLiteralsBytes(const SeqStore_t* seqStore);
size_t ZSTD_rust_countSeqStoreMatchBytes(const SeqStore_t* seqStore);
void ZSTD_rust_deriveSeqStoreChunk(SeqStore_t* resultSeqStore,
const SeqStore_t* originalSeqStore,
size_t startIdx, size_t endIdx);
U32 ZSTD_rust_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM],
U32 offBase, U32 ll0);
typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
typedef char ZSTD_rust_stats_seqstore_long_length_pos[
@@ -2817,31 +2824,13 @@ ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(SeqStore_t* seqStore, ZSTD_CC
/* Returns literals bytes represented in a seqStore */
static size_t ZSTD_countSeqStoreLiteralsBytes(const SeqStore_t* const seqStore)
{
size_t literalsBytes = 0;
size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
size_t i;
for (i = 0; i < nbSeqs; ++i) {
SeqDef const seq = seqStore->sequencesStart[i];
literalsBytes += seq.litLength;
if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) {
literalsBytes += 0x10000;
} }
return literalsBytes;
return ZSTD_rust_countSeqStoreLiteralsBytes(seqStore);
}
/* Returns match bytes represented in a seqStore */
static size_t ZSTD_countSeqStoreMatchBytes(const SeqStore_t* const seqStore)
{
size_t matchBytes = 0;
size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
size_t i;
for (i = 0; i < nbSeqs; ++i) {
SeqDef seq = seqStore->sequencesStart[i];
matchBytes += seq.mlBase + MINMATCH;
if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) {
matchBytes += 0x10000;
} }
return matchBytes;
return ZSTD_rust_countSeqStoreMatchBytes(seqStore);
}
/* Derives the seqStore that is a chunk of the originalSeqStore from [startIdx, endIdx).
@@ -2851,32 +2840,8 @@ static void ZSTD_deriveSeqStoreChunk(SeqStore_t* resultSeqStore,
const SeqStore_t* originalSeqStore,
size_t startIdx, size_t endIdx)
{
*resultSeqStore = *originalSeqStore;
if (startIdx > 0) {
resultSeqStore->sequences = originalSeqStore->sequencesStart + startIdx;
resultSeqStore->litStart += ZSTD_countSeqStoreLiteralsBytes(resultSeqStore);
}
/* Move longLengthPos into the correct position if necessary */
if (originalSeqStore->longLengthType != ZSTD_llt_none) {
if (originalSeqStore->longLengthPos < startIdx || originalSeqStore->longLengthPos > endIdx) {
resultSeqStore->longLengthType = ZSTD_llt_none;
} else {
resultSeqStore->longLengthPos -= (U32)startIdx;
}
}
resultSeqStore->sequencesStart = originalSeqStore->sequencesStart + startIdx;
resultSeqStore->sequences = originalSeqStore->sequencesStart + endIdx;
if (endIdx == (size_t)(originalSeqStore->sequences - originalSeqStore->sequencesStart)) {
/* This accounts for possible last literals if the derived chunk reaches the end of the block */
assert(resultSeqStore->lit == originalSeqStore->lit);
} else {
size_t const literalsBytes = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore);
resultSeqStore->lit = resultSeqStore->litStart + literalsBytes;
}
resultSeqStore->llCode += startIdx;
resultSeqStore->mlCode += startIdx;
resultSeqStore->ofCode += startIdx;
ZSTD_rust_deriveSeqStoreChunk(resultSeqStore, originalSeqStore,
startIdx, endIdx);
}
/**
@@ -2886,20 +2851,8 @@ static void ZSTD_deriveSeqStoreChunk(SeqStore_t* resultSeqStore,
static U32
ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, const U32 ll0)
{
U32 const adjustedRepCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; /* [ 0 - 3 ] */
assert(OFFBASE_IS_REPCODE(offBase));
if (adjustedRepCode == ZSTD_REP_NUM) {
assert(ll0);
/* litlength == 0 and offCode == 2 implies selection of first repcode - 1
* This is only valid if it results in a valid offset value, aka > 0.
* Note : it may happen that `rep[0]==1` in exceptional circumstances.
* In which case this function will return 0, which is an invalid offset.
* It's not an issue though, since this value will be
* compared and discarded within ZSTD_seqStore_resolveOffCodes().
*/
return rep[0] - 1;
}
return rep[adjustedRepCode];
return ZSTD_rust_resolveRepcodeToRawOffset(rep, offBase, ll0);
}
/**
lib/compress/zstdmt_compress.c
+37 -7
View File
@@ -115,6 +115,25 @@ void ZSTDMT_rust_buffer_pool_release(ZSTDMT_RustBufferPool* pool,
ZSTDMT_RustBuffer ZSTDMT_rust_buffer_pool_resize(ZSTDMT_RustBufferPool* pool,
ZSTDMT_RustBuffer buffer);
typedef struct {
rawSeq* seq;
size_t pos;
size_t posInSequence;
size_t size;
size_t capacity;
} ZSTDMT_RustRawSeqStore;
typedef char ZSTDMT_rust_raw_seq_layout[
(sizeof(rawSeq) == 3 * sizeof(U32)) ? 1 : -1];
typedef char ZSTDMT_rust_raw_seq_store_layout[
(sizeof(ZSTDMT_RustRawSeqStore) == sizeof(RawSeqStore_t)
&& offsetof(ZSTDMT_RustRawSeqStore, pos) == offsetof(RawSeqStore_t, pos)
&& offsetof(ZSTDMT_RustRawSeqStore, capacity) == offsetof(RawSeqStore_t, capacity))
? 1 : -1];
ZSTDMT_RustRawSeqStore ZSTDMT_rust_bufferToSeq(ZSTDMT_RustBuffer buffer);
ZSTDMT_RustBuffer ZSTDMT_rust_seqToBuffer(ZSTDMT_RustRawSeqStore seq);
unsigned ZSTDMT_rust_computeTargetJobLog(unsigned windowLog, unsigned chainLog,
int strategy, int enableLdm);
int ZSTDMT_rust_overlapLog(int overlapLog, int strategy);
@@ -234,18 +253,29 @@ static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool)
static RawSeqStore_t bufferToSeq(Buffer buffer)
{
RawSeqStore_t seq = kNullRawSeqStore;
seq.seq = (rawSeq*)buffer.start;
seq.capacity = buffer.capacity / sizeof(rawSeq);
ZSTDMT_RustRawSeqStore const rustSeq =
ZSTDMT_rust_bufferToSeq((ZSTDMT_RustBuffer){ buffer.start, buffer.capacity });
RawSeqStore_t seq = {
(rawSeq*)rustSeq.seq,
rustSeq.pos,
rustSeq.posInSequence,
rustSeq.size,
rustSeq.capacity
};
return seq;
}
static Buffer seqToBuffer(RawSeqStore_t seq)
{
Buffer buffer;
buffer.start = seq.seq;
buffer.capacity = seq.capacity * sizeof(rawSeq);
return buffer;
ZSTDMT_RustRawSeqStore const rustSeq = {
seq.seq,
seq.pos,
seq.posInSequence,
seq.size,
seq.capacity
};
ZSTDMT_RustBuffer const rustBuffer = ZSTDMT_rust_seqToBuffer(rustSeq);
return (Buffer){ rustBuffer.start, rustBuffer.capacity };
}
static RawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool)