feat(ldm): move sequence-store emission into Rust

Remove the LDM-specific C sequence-store callback and reuse the existing Rust
sequence-store leaf from the fast matcher through a crate-local opaque wrapper.
The LDM C shim now retains only private match-state extraction and compressor
dispatch, with explicit SeqDef and SeqStore_t layout checks protecting the
cross-module pointer contract. The Rust boundary documentation now reflects
that sequence-store encoding is Rust-owned.

Test Plan:
- ulimit -v 41943040 && cargo fmt --manifest-path rust/Cargo.toml && git diff --check
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml zstd_ldm::tests --lib (7 passed)
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml store_seq_preserves_literal --lib (1 passed)
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --lib (683 passed)
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040 && make -B -C programs -j1 zstd (passed; existing fileio const-cast warnings)
- ulimit -v 41943040 && make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s (84 tests and both short fuzz rounds passed; existing zstreamtest warning)

lib/compress/zstd_ldm.c

@@ -8,8 +8,9 @@
  * You may select, at your option, one of the above-listed licenses.
  */
 
-/* The LDM algorithms live in rust/src/zstd_ldm.rs. This file retains private
+/* The LDM algorithms and sequence-store encoding live in
- * match-state extraction and block-compressor dispatch only. */
+ * rust/src/zstd_ldm.rs. This file retains private match-state extraction and
+ * block-compressor dispatch only. */
 #include "zstd_ldm.h"
 
 #include "../common/debug.h"
@@ -17,6 +18,26 @@
 #include "zstd_double_fast.h"
 #include "zstd_ldm_geartab.h"
 
+typedef struct {
+    U32 offBase;
+    U16 litLength;
+    U16 mlBase;
+} ZSTD_rust_seq_def_layout;
+
+typedef struct {
+    SeqDef* sequencesStart;
+    SeqDef* sequences;
+    BYTE* litStart;
+    BYTE* lit;
+    BYTE* llCode;
+    BYTE* mlCode;
+    BYTE* ofCode;
+    size_t maxNbSeq;
+    size_t maxNbLit;
+    int longLengthType;
+    U32 longLengthPos;
+} ZSTD_rust_seq_store_layout;
+
 typedef struct {
     U32 offset;
     U32 checksum;
@@ -56,6 +77,10 @@ typedef struct {
 
 typedef char ZSTD_rust_ldm_entry_layout_check[
         sizeof(ldmEntry_t) == sizeof(ZSTD_rust_ldm_entry_layout) ? 1 : -1];
+typedef char ZSTD_rust_seq_def_layout_check[
+        sizeof(SeqDef) == sizeof(ZSTD_rust_seq_def_layout) ? 1 : -1];
+typedef char ZSTD_rust_seq_store_layout_check[
+        sizeof(SeqStore_t) == sizeof(ZSTD_rust_seq_store_layout) ? 1 : -1];
 typedef char ZSTD_rust_raw_seq_layout_check[
         sizeof(rawSeq) == sizeof(ZSTD_rust_raw_seq_layout) ? 1 : -1];
 typedef char ZSTD_rust_raw_seq_store_layout_check[
@@ -79,6 +104,64 @@ ZSTD_RUST_LDM_OFFSET_CHECK(
         ldmEntry_t, checksum,
         ZSTD_rust_ldm_entry_layout, checksum);
 
+ZSTD_RUST_LDM_OFFSET_CHECK(
+        ZSTD_rust_seq_def_off_base_check,
+        SeqDef, offBase,
+        ZSTD_rust_seq_def_layout, offBase);
+ZSTD_RUST_LDM_OFFSET_CHECK(
+        ZSTD_rust_seq_def_lit_length_check,
+        SeqDef, litLength,
+        ZSTD_rust_seq_def_layout, litLength);
+ZSTD_RUST_LDM_OFFSET_CHECK(
+        ZSTD_rust_seq_def_ml_base_check,
+        SeqDef, mlBase,
+        ZSTD_rust_seq_def_layout, mlBase);
+
+ZSTD_RUST_LDM_OFFSET_CHECK(
+        ZSTD_rust_seq_store_sequences_start_check,
+        SeqStore_t, sequencesStart,
+        ZSTD_rust_seq_store_layout, sequencesStart);
+ZSTD_RUST_LDM_OFFSET_CHECK(
+        ZSTD_rust_seq_store_sequences_check,
+        SeqStore_t, sequences,
+        ZSTD_rust_seq_store_layout, sequences);
+ZSTD_RUST_LDM_OFFSET_CHECK(
+        ZSTD_rust_seq_store_lit_start_check,
+        SeqStore_t, litStart,
+        ZSTD_rust_seq_store_layout, litStart);
+ZSTD_RUST_LDM_OFFSET_CHECK(
+        ZSTD_rust_seq_store_lit_check,
+        SeqStore_t, lit,
+        ZSTD_rust_seq_store_layout, lit);
+ZSTD_RUST_LDM_OFFSET_CHECK(
+        ZSTD_rust_seq_store_ll_code_check,
+        SeqStore_t, llCode,
+        ZSTD_rust_seq_store_layout, llCode);
+ZSTD_RUST_LDM_OFFSET_CHECK(
+        ZSTD_rust_seq_store_ml_code_check,
+        SeqStore_t, mlCode,
+        ZSTD_rust_seq_store_layout, mlCode);
+ZSTD_RUST_LDM_OFFSET_CHECK(
+        ZSTD_rust_seq_store_of_code_check,
+        SeqStore_t, ofCode,
+        ZSTD_rust_seq_store_layout, ofCode);
+ZSTD_RUST_LDM_OFFSET_CHECK(
+        ZSTD_rust_seq_store_max_nb_seq_check,
+        SeqStore_t, maxNbSeq,
+        ZSTD_rust_seq_store_layout, maxNbSeq);
+ZSTD_RUST_LDM_OFFSET_CHECK(
+        ZSTD_rust_seq_store_max_nb_lit_check,
+        SeqStore_t, maxNbLit,
+        ZSTD_rust_seq_store_layout, maxNbLit);
+ZSTD_RUST_LDM_OFFSET_CHECK(
+        ZSTD_rust_seq_store_long_length_type_check,
+        SeqStore_t, longLengthType,
+        ZSTD_rust_seq_store_layout, longLengthType);
+ZSTD_RUST_LDM_OFFSET_CHECK(
+        ZSTD_rust_seq_store_long_length_pos_check,
+        SeqStore_t, longLengthPos,
+        ZSTD_rust_seq_store_layout, longLengthPos);
+
 ZSTD_RUST_LDM_OFFSET_CHECK(
         ZSTD_rust_raw_seq_offset_check,
         rawSeq, offset,
@@ -192,9 +275,6 @@ void ZSTD_ldm_rust_prepareBlock(void* blockContext, const void* anchor);
 size_t ZSTD_ldm_rust_compressLiterals(
         void* blockContext, void* seqStore, U32 rep[ZSTD_REP_NUM],
         const void* src, size_t srcSize);
-void ZSTD_ldm_rust_storeSeq(
-        void* seqStore, size_t litLength, const void* literals, const void* litLimit,
-        U32 offBase, size_t matchLength);
 void ZSTD_ldm_rust_setLdmSeqStore(void* blockContext, const void* rawSeqStore);
 
 const U64* ZSTD_ldm_rust_gearTable(void)
@@ -250,14 +330,6 @@ size_t ZSTD_ldm_rust_compressLiterals(
     return context->blockCompressor(context->ms, (SeqStore_t*)seqStore, rep, src, srcSize);
 }
 
-void ZSTD_ldm_rust_storeSeq(
-        void* seqStore, size_t litLength, const void* literals, const void* litLimit,
-        U32 offBase, size_t matchLength)
-{
-    ZSTD_storeSeq((SeqStore_t*)seqStore, litLength, (const BYTE*)literals,
-                  (const BYTE*)litLimit, offBase, matchLength);
-}
-
 void ZSTD_ldm_rust_setLdmSeqStore(void* blockContext, const void* rawSeqStore)
 {
     ZSTD_rust_ldm_block_context* const context = (ZSTD_rust_ldm_block_context*)blockContext;

rust/README.md

@@ -44,9 +44,10 @@ zstd ABI:
     and the outer block-dispatch decision. `zstd_compress` also owns ordinary
     sequence-block entropy emission, sequence collection, the legacy RLE
     compatibility gate, and sequence-store construction; C supplies only the
-    private matchfinder, LDM, external-sequence-producer, and state-preparation
+    private matchfinder, LDM block-preparation/consumption, external-sequence-
-    callbacks. Rust also owns the frame-chunk block loop, including block
+    producer, and state-preparation callbacks. Rust also owns the frame-chunk
-    sizing, target/split/internal dispatch, output accounting, and frame-state
+    block loop, including block sizing, target/split/internal dispatch, output
+    accounting, and frame-state
     updates; C supplies the private block-compression callbacks. Rust also
     owns the single-threaded buffered/stable stream state machine, including
     direct versus buffered output, pending-output draining, and frame reset
@@ -78,7 +79,8 @@ zstd ABI:
     `zstd_opt` owns the dynamic-programming price model, optimal parse, and
     sequence emission.
   - `zstd_ldm` implements long-distance-match parameter selection, table
-    maintenance, sequence generation, and sequence consumption.
+    maintenance, sequence generation, sequence consumption, and sequence-store
+    encoding.
 - Dictionary building
   - `divsufsort` constructs the suffix array that drives the legacy `ZDICT`
     trainer (`ZDICT_trainFromBuffer_legacy`); `dict_builder_zdict`,

rust/src/zstd_fast.rs

@@ -293,6 +293,28 @@ unsafe fn store_seq(
     seq_store.sequences = sequence.wrapping_add(1);
 }
 
+/// Reuse the sequence-store leaf from another Rust compression module without
+/// exposing this module's C-layout projection in its public API.
+pub(crate) unsafe fn store_seq_opaque(
+    seq_store: *mut c_void,
+    lit_length: usize,
+    literals: *const c_void,
+    lit_limit: *const c_void,
+    off_base: u32,
+    match_length: usize,
+) {
+    unsafe {
+        store_seq(
+            seq_store.cast::<SeqStore_t>(),
+            lit_length,
+            literals.cast::<u8>(),
+            lit_limit.cast::<u8>(),
+            off_base,
+            match_length,
+        )
+    }
+}
+
 #[inline]
 unsafe fn match4_found(
     current: *const u8,
@@ -1576,11 +1598,11 @@ mod tests {
         };
 
         unsafe {
-            store_seq(
+            store_seq_opaque(
-                &mut store,
+                (&mut store as *mut SeqStore_t).cast(),
                 source.len(),
-                source.as_ptr(),
+                source.as_ptr().cast(),
-                source.as_ptr().add(source.len()),
+                source.as_ptr().add(source.len()).cast(),
                 13,
                 3 + 0x1_0000,
             );

rust/src/zstd_ldm.rs

@@ -12,6 +12,7 @@
 use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
 use crate::mem::{MEM_64bits, MEM_isLittleEndian, MEM_read16, MEM_read32, MEM_readST};
 use crate::xxhash::XXH64;
+use crate::zstd_fast::store_seq_opaque;
 use std::ffi::c_void;
 use std::mem::size_of;
 use std::os::raw::c_int;
@@ -98,14 +99,6 @@ unsafe extern "C" {
         src: *const c_void,
         src_size: usize,
     ) -> usize;
-    fn ZSTD_ldm_rust_storeSeq(
-        seq_store: *mut c_void,
-        lit_length: usize,
-        literals: *const c_void,
-        lit_limit: *const c_void,
-        off_base: u32,
-        match_length: usize,
-    );
     fn ZSTD_ldm_rust_setLdmSeqStore(context: *mut c_void, raw_seq_store: *const c_void);
 }
 
@@ -990,11 +983,11 @@ pub unsafe extern "C" fn ZSTD_rust_ldm_blockCompress(
             *reps.add(2) = *reps.add(1);
             *reps.add(1) = *reps;
             *reps = sequence.offset;
-            ZSTD_ldm_rust_storeSeq(
+            store_seq_opaque(
                 seq_store,
                 new_lit_length,
-                input_position.wrapping_sub(new_lit_length).cast::<c_void>(),
+                input_position.wrapping_sub(new_lit_length).cast(),
-                input_end.cast::<c_void>(),
+                input_end.cast(),
                 sequence.offset.wrapping_add(ZSTD_REP_NUM as u32),
                 sequence.match_length as usize,
             );