ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(ldm): move sequence-store emission into Rust

Browse Source 
Remove the LDM-specific C sequence-store callback and reuse the existing Rust
sequence-store leaf from the fast matcher through a crate-local opaque wrapper.
The LDM C shim now retains only private match-state extraction and compressor
dispatch, with explicit SeqDef and SeqStore_t layout checks protecting the
cross-module pointer contract. The Rust boundary documentation now reflects
that sequence-store encoding is Rust-owned.

Test Plan:
- ulimit -v 41943040 && cargo fmt --manifest-path rust/Cargo.toml && git diff --check
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml zstd_ldm::tests --lib (7 passed)
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml store_seq_preserves_literal --lib (1 passed)
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --lib (683 passed)
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040 && make -B -C programs -j1 zstd (passed; existing fileio const-cast warnings)
- ulimit -v 41943040 && make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s (84 tests and both short fuzz rounds passed; existing zstreamtest warning)
This commit is contained in:
ddidderr
2026-07-19 18:27:52 +02:00
parent db8c58ac97
commit d38c1e2a62
4 changed files with 121 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_ldm.c
+85 -13
View File
@@ -8,8 +8,9 @@
* You may select, at your option, one of the above-listed licenses.
*/
/* The LDM algorithms live in rust/src/zstd_ldm.rs. This file retains private
* match-state extraction and block-compressor dispatch only. */
/* The LDM algorithms and sequence-store encoding live in
* rust/src/zstd_ldm.rs. This file retains private match-state extraction and
* block-compressor dispatch only. */
#include "zstd_ldm.h"
#include "../common/debug.h"
@@ -17,6 +18,26 @@
#include "zstd_double_fast.h"
#include "zstd_ldm_geartab.h"
typedef struct {
U32 offBase;
U16 litLength;
U16 mlBase;
} ZSTD_rust_seq_def_layout;
typedef struct {
SeqDef* sequencesStart;
SeqDef* sequences;
BYTE* litStart;
BYTE* lit;
BYTE* llCode;
BYTE* mlCode;
BYTE* ofCode;
size_t maxNbSeq;
size_t maxNbLit;
int longLengthType;
U32 longLengthPos;
} ZSTD_rust_seq_store_layout;
typedef struct {
U32 offset;
U32 checksum;
@@ -56,6 +77,10 @@ typedef struct {
typedef char ZSTD_rust_ldm_entry_layout_check[
sizeof(ldmEntry_t) == sizeof(ZSTD_rust_ldm_entry_layout) ? 1 : -1];
typedef char ZSTD_rust_seq_def_layout_check[
sizeof(SeqDef) == sizeof(ZSTD_rust_seq_def_layout) ? 1 : -1];
typedef char ZSTD_rust_seq_store_layout_check[
sizeof(SeqStore_t) == sizeof(ZSTD_rust_seq_store_layout) ? 1 : -1];
typedef char ZSTD_rust_raw_seq_layout_check[
sizeof(rawSeq) == sizeof(ZSTD_rust_raw_seq_layout) ? 1 : -1];
typedef char ZSTD_rust_raw_seq_store_layout_check[
@@ -79,6 +104,64 @@ ZSTD_RUST_LDM_OFFSET_CHECK(
ldmEntry_t, checksum,
ZSTD_rust_ldm_entry_layout, checksum);
ZSTD_RUST_LDM_OFFSET_CHECK(
ZSTD_rust_seq_def_off_base_check,
SeqDef, offBase,
ZSTD_rust_seq_def_layout, offBase);
ZSTD_RUST_LDM_OFFSET_CHECK(
ZSTD_rust_seq_def_lit_length_check,
SeqDef, litLength,
ZSTD_rust_seq_def_layout, litLength);
ZSTD_RUST_LDM_OFFSET_CHECK(
ZSTD_rust_seq_def_ml_base_check,
SeqDef, mlBase,
ZSTD_rust_seq_def_layout, mlBase);
ZSTD_RUST_LDM_OFFSET_CHECK(
ZSTD_rust_seq_store_sequences_start_check,
SeqStore_t, sequencesStart,
ZSTD_rust_seq_store_layout, sequencesStart);
ZSTD_RUST_LDM_OFFSET_CHECK(
ZSTD_rust_seq_store_sequences_check,
SeqStore_t, sequences,
ZSTD_rust_seq_store_layout, sequences);
ZSTD_RUST_LDM_OFFSET_CHECK(
ZSTD_rust_seq_store_lit_start_check,
SeqStore_t, litStart,
ZSTD_rust_seq_store_layout, litStart);
ZSTD_RUST_LDM_OFFSET_CHECK(
ZSTD_rust_seq_store_lit_check,
SeqStore_t, lit,
ZSTD_rust_seq_store_layout, lit);
ZSTD_RUST_LDM_OFFSET_CHECK(
ZSTD_rust_seq_store_ll_code_check,
SeqStore_t, llCode,
ZSTD_rust_seq_store_layout, llCode);
ZSTD_RUST_LDM_OFFSET_CHECK(
ZSTD_rust_seq_store_ml_code_check,
SeqStore_t, mlCode,
ZSTD_rust_seq_store_layout, mlCode);
ZSTD_RUST_LDM_OFFSET_CHECK(
ZSTD_rust_seq_store_of_code_check,
SeqStore_t, ofCode,
ZSTD_rust_seq_store_layout, ofCode);
ZSTD_RUST_LDM_OFFSET_CHECK(
ZSTD_rust_seq_store_max_nb_seq_check,
SeqStore_t, maxNbSeq,
ZSTD_rust_seq_store_layout, maxNbSeq);
ZSTD_RUST_LDM_OFFSET_CHECK(
ZSTD_rust_seq_store_max_nb_lit_check,
SeqStore_t, maxNbLit,
ZSTD_rust_seq_store_layout, maxNbLit);
ZSTD_RUST_LDM_OFFSET_CHECK(
ZSTD_rust_seq_store_long_length_type_check,
SeqStore_t, longLengthType,
ZSTD_rust_seq_store_layout, longLengthType);
ZSTD_RUST_LDM_OFFSET_CHECK(
ZSTD_rust_seq_store_long_length_pos_check,
SeqStore_t, longLengthPos,
ZSTD_rust_seq_store_layout, longLengthPos);
ZSTD_RUST_LDM_OFFSET_CHECK(
ZSTD_rust_raw_seq_offset_check,
rawSeq, offset,
@@ -192,9 +275,6 @@ void ZSTD_ldm_rust_prepareBlock(void* blockContext, const void* anchor);
size_t ZSTD_ldm_rust_compressLiterals(
void* blockContext, void* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize);
void ZSTD_ldm_rust_storeSeq(
void* seqStore, size_t litLength, const void* literals, const void* litLimit,
U32 offBase, size_t matchLength);
void ZSTD_ldm_rust_setLdmSeqStore(void* blockContext, const void* rawSeqStore);
const U64* ZSTD_ldm_rust_gearTable(void)
@@ -250,14 +330,6 @@ size_t ZSTD_ldm_rust_compressLiterals(
return context->blockCompressor(context->ms, (SeqStore_t*)seqStore, rep, src, srcSize);
}
void ZSTD_ldm_rust_storeSeq(
void* seqStore, size_t litLength, const void* literals, const void* litLimit,
U32 offBase, size_t matchLength)
{
ZSTD_storeSeq((SeqStore_t*)seqStore, litLength, (const BYTE*)literals,
(const BYTE*)litLimit, offBase, matchLength);
}
void ZSTD_ldm_rust_setLdmSeqStore(void* blockContext, const void* rawSeqStore)
{
ZSTD_rust_ldm_block_context* const context = (ZSTD_rust_ldm_block_context*)blockContext;