feat(compress): move Double-Fast dictionary fill to Rust

Replace the dictionary-content Double-Fast callback with a live-field
projection and call the existing Rust table-fill leaf directly. The
projection keeps hash tables, window base, match-state index, and parameters
live through the window/publication callbacks, preserving the timing behavior
of the former C adapter. Preserve the excluded-DFast assertion path and the
34-word dictionary bridge layout.

Test Plan:
- git diff --check
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --manifest-path rust/Cargo.toml --tests
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -A clippy::manual-bits -D warnings
- ulimit -v 41943040; make -j1
- capped strategy-2 compress/decompress round trip with cmp
- Full original suite deferred to the next serialized verification step

lib/compress/zstd_compress.c

@@ -2805,8 +2805,45 @@ typedef char ZSTD_rust_load_dictionary_fast_table_layout[
          && sizeof(ZSTD_rust_loadDictionaryContentFastTableState)
                 == 5 * sizeof(void*) + 2 * sizeof(int))
         ? 1 : -1];
-typedef void (*ZSTD_rust_loadDictionaryContent_fillTable_f)(
+/* The Double-Fast dictionary-table leaf is also implemented in Rust. Keep
-        void* context, const void* iend, int dtlm, int tfp);
+ * pointers to the live match-state fields because window/publish callbacks
+ * run before the Rust orchestrator selects this branch. */
+typedef struct {
+    U32** hashLong;
+    U32** hashSmall;
+    const BYTE** base;
+    const U32* nextToUpdate;
+    const U32* hashLog;
+    const U32* chainLog;
+    const U32* minMatch;
+    int fullTableLoad;
+    int forCDict;
+    int available;
+} ZSTD_rust_loadDictionaryContentDoubleFastTableState;
+typedef char ZSTD_rust_load_dictionary_double_fast_table_layout[
+        (offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hashLong) == 0
+         && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hashSmall)
+                == sizeof(void*)
+         && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, base)
+                == 2 * sizeof(void*)
+         && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, nextToUpdate)
+                == 3 * sizeof(void*)
+         && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hashLog)
+                == 4 * sizeof(void*)
+         && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, chainLog)
+                == 5 * sizeof(void*)
+         && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, minMatch)
+                == 6 * sizeof(void*)
+         && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, fullTableLoad)
+                == 7 * sizeof(void*)
+         && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, forCDict)
+                == 7 * sizeof(void*) + sizeof(int)
+         && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, available)
+                == 7 * sizeof(void*) + 2 * sizeof(int)
+         && sizeof(ZSTD_rust_loadDictionaryContentDoubleFastTableState)
+                == ((7 * sizeof(void*) + 3 * sizeof(int) + sizeof(void*) - 1)
+                    / sizeof(void*) * sizeof(void*)))
+        ? 1 : -1];
 typedef void (*ZSTD_rust_loadDictionaryContent_loadMatch_f)(
         void* context, const void* ip);
 typedef void (*ZSTD_rust_loadDictionaryContent_loadTree_f)(
@@ -2841,7 +2878,7 @@ typedef struct {
     ZSTD_rust_loadDictionaryContent_fillLdm_f fillLdm;
     ZSTD_rust_loadDictionaryContent_overflowCorrect_f overflowCorrect;
     const ZSTD_rust_loadDictionaryContentFastTableState* fastTable;
-    ZSTD_rust_loadDictionaryContent_fillTable_f fillDoubleHashTable;
+    const ZSTD_rust_loadDictionaryContentDoubleFastTableState* doubleFastTable;
     ZSTD_rust_loadDictionaryContent_loadMatch_f loadDedicated;
     ZSTD_rust_loadDictionaryContent_loadMatch_f loadRow;
     ZSTD_rust_loadDictionaryContent_loadMatch_f loadChain;
@@ -2865,6 +2902,9 @@ ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_callback_offset,
 ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_fast_table_offset,
                            offsetof(ZSTD_rust_loadDictionaryContentState,
                                     fastTable) == 25 * sizeof(void*));
+ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_double_fast_table_offset,
+                           offsetof(ZSTD_rust_loadDictionaryContentState,
+                                    doubleFastTable) == 26 * sizeof(void*));
 ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_final_callback_offset,
                            offsetof(ZSTD_rust_loadDictionaryContentState,
                                     publishFinalIndex) == 31 * sizeof(void*));
@@ -6891,21 +6931,6 @@ static void ZSTD_loadDictionaryContent_overflowCorrect(
                                  context->params, ip, iend);
 }
 
-static void ZSTD_loadDictionaryContent_fillDoubleHashTable(
-        void* opaque, const void* iend, int dtlm, int tfp)
-{
-    ZSTD_loadDictionaryContent_context const* const context =
-            (const ZSTD_loadDictionaryContent_context*)opaque;
-#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR
-    ZSTD_fillDoubleHashTable(context->matchState, (const BYTE*)iend,
-                             (ZSTD_dictTableLoadMethod_e)dtlm,
-                             (ZSTD_tableFillPurpose_e)tfp);
-#else
-    (void)context; (void)iend; (void)dtlm; (void)tfp;
-    assert(0); /* shouldn't be called: cparams should've been adjusted. */
-#endif
-}
-
 static void ZSTD_loadDictionaryContent_loadDedicated(void* opaque, const void* ip)
 {
     ZSTD_loadDictionaryContent_context const* const context =
@@ -7002,6 +7027,7 @@ static size_t ZSTD_loadDictionaryContent_callback(
 {
     ZSTD_loadDictionaryContent_context context;
     ZSTD_rust_loadDictionaryContentFastTableState fastTable;
+    ZSTD_rust_loadDictionaryContentDoubleFastTableState doubleFastTable;
     ZSTD_rust_loadDictionaryContentState state;
     ZSTD_MatchState_t* const ms = (ZSTD_MatchState_t*)matchState;
     ldmState_t* const ls = (ldmState_t*)ldmState;
@@ -7025,6 +7051,20 @@ static size_t ZSTD_loadDictionaryContent_callback(
     fastTable.minMatch = &ms->cParams.minMatch;
     fastTable.fullTableLoad = dtlm == ZSTD_dtlm_full;
     fastTable.forCDict = tfp == ZSTD_tfp_forCDict;
+    doubleFastTable.hashLong = &ms->hashTable;
+    doubleFastTable.hashSmall = &ms->chainTable;
+    doubleFastTable.base = &ms->window.base;
+    doubleFastTable.nextToUpdate = &ms->nextToUpdate;
+    doubleFastTable.hashLog = &ms->cParams.hashLog;
+    doubleFastTable.chainLog = &ms->cParams.chainLog;
+    doubleFastTable.minMatch = &ms->cParams.minMatch;
+    doubleFastTable.fullTableLoad = dtlm == ZSTD_dtlm_full;
+    doubleFastTable.forCDict = tfp == ZSTD_tfp_forCDict;
+#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR
+    doubleFastTable.available = 1;
+#else
+    doubleFastTable.available = 0;
+#endif
 
     state.callbackContext = &context;
     state.currentMax = ZSTD_CURRENT_MAX;
@@ -7052,7 +7092,7 @@ static size_t ZSTD_loadDictionaryContent_callback(
     state.fillLdm = ZSTD_loadDictionaryContent_fillLdm;
     state.overflowCorrect = ZSTD_loadDictionaryContent_overflowCorrect;
     state.fastTable = &fastTable;
-    state.fillDoubleHashTable = ZSTD_loadDictionaryContent_fillDoubleHashTable;
+    state.doubleFastTable = &doubleFastTable;
     state.loadDedicated = ZSTD_loadDictionaryContent_loadDedicated;
     state.loadRow = ZSTD_loadDictionaryContent_loadRow;
     state.loadChain = ZSTD_loadDictionaryContent_loadChain;

rust/src/zstd_compress_dictionary.rs

@@ -28,6 +28,7 @@ use crate::zstd_compress_params_api::ZSTD_CCtx_params;
 use crate::zstd_compress_stats::{
     ZSTD_compressedBlockState_t, ZSTD_rust_resetCompressedBlockState,
 };
+use crate::zstd_double_fast::ZSTD_rust_fillDoubleHashTable;
 use crate::zstd_fast::ZSTD_rust_fillHashTable;
 use std::ffi::c_void;
 use std::mem::{offset_of, size_of};
@@ -156,8 +157,21 @@ struct ZSTD_rust_loadDictionaryContentFastTableState {
     for_cdict: c_int,
 }
 
-type LoadDictionaryContentFillTableFn =
+/// Direct projection for the Double-Fast dictionary-table leaf.
-    unsafe extern "C" fn(context: *mut c_void, iend: *const c_void, dtlm: c_int, tfp: c_int);
+#[repr(C)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+struct ZSTD_rust_loadDictionaryContentDoubleFastTableState {
+    hash_long: *const *mut c_uint,
+    hash_small: *const *mut c_uint,
+    base: *const *const u8,
+    next_to_update: *const c_uint,
+    hash_log: *const c_uint,
+    chain_log: *const c_uint,
+    min_match: *const c_uint,
+    full_table_load: c_int,
+    for_cdict: c_int,
+    available: c_int,
+}
 type LoadDictionaryContentLoadMatchFn =
     unsafe extern "C" fn(context: *mut c_void, ip: *const c_void);
 type LoadDictionaryContentLoadTreeFn =
@@ -200,7 +214,7 @@ pub struct ZSTD_rust_loadDictionaryContentState {
     fill_ldm: LoadDictionaryContentFillLdmFn,
     overflow_correct: LoadDictionaryContentOverflowCorrectFn,
     fast_table: *const ZSTD_rust_loadDictionaryContentFastTableState,
-    fill_double_hash_table: LoadDictionaryContentFillTableFn,
+    double_fast_table: *const ZSTD_rust_loadDictionaryContentDoubleFastTableState,
     load_dedicated: LoadDictionaryContentLoadMatchFn,
     load_row: LoadDictionaryContentLoadMatchFn,
     load_chain: LoadDictionaryContentLoadMatchFn,
@@ -218,7 +232,6 @@ const _: () = {
     assert!(size_of::<LoadDictionaryContentPublishMatchStateFn>() == size_of::<usize>());
     assert!(size_of::<LoadDictionaryContentFillLdmFn>() == size_of::<usize>());
     assert!(size_of::<LoadDictionaryContentOverflowCorrectFn>() == size_of::<usize>());
-    assert!(size_of::<LoadDictionaryContentFillTableFn>() == size_of::<usize>());
     assert!(offset_of!(ZSTD_rust_loadDictionaryContentFastTableState, hash_table) == 0);
     assert!(offset_of!(ZSTD_rust_loadDictionaryContentFastTableState, base) == size_of::<usize>());
     assert!(
@@ -249,6 +262,49 @@ const _: () = {
         size_of::<ZSTD_rust_loadDictionaryContentFastTableState>()
             == 5 * size_of::<usize>() + 2 * size_of::<c_int>()
     );
+    assert!(offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hash_long) == 0);
+    assert!(
+        offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hash_small)
+            == size_of::<usize>()
+    );
+    assert!(
+        offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, base)
+            == 2 * size_of::<usize>()
+    );
+    assert!(
+        offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, next_to_update)
+            == 3 * size_of::<usize>()
+    );
+    assert!(
+        offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hash_log)
+            == 4 * size_of::<usize>()
+    );
+    assert!(
+        offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, chain_log)
+            == 5 * size_of::<usize>()
+    );
+    assert!(
+        offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, min_match)
+            == 6 * size_of::<usize>()
+    );
+    assert!(
+        offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, full_table_load)
+            == 7 * size_of::<usize>()
+    );
+    assert!(
+        offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, for_cdict)
+            == 7 * size_of::<usize>() + size_of::<c_int>()
+    );
+    assert!(
+        offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, available)
+            == 7 * size_of::<usize>() + 2 * size_of::<c_int>()
+    );
+    assert!(
+        size_of::<ZSTD_rust_loadDictionaryContentDoubleFastTableState>()
+            == (7 * size_of::<usize>() + 3 * size_of::<c_int>())
+                .div_ceil(size_of::<usize>())
+                * size_of::<usize>()
+    );
     assert!(size_of::<LoadDictionaryContentLoadMatchFn>() == size_of::<usize>());
     assert!(size_of::<LoadDictionaryContentLoadTreeFn>() == size_of::<usize>());
     assert!(size_of::<LoadDictionaryContentPublishFinalIndexFn>() == size_of::<usize>());
@@ -261,6 +317,10 @@ const _: () = {
     assert!(
         offset_of!(ZSTD_rust_loadDictionaryContentState, fast_table) == 25 * size_of::<usize>()
     );
+    assert!(
+        offset_of!(ZSTD_rust_loadDictionaryContentState, double_fast_table)
+            == 26 * size_of::<usize>()
+    );
     assert!(
         offset_of!(ZSTD_rust_loadDictionaryContentState, publish_final_index)
             == 31 * size_of::<usize>()
@@ -385,6 +445,31 @@ unsafe fn fill_fast_dictionary_table(
     }
 }
 
+#[inline]
+unsafe fn fill_double_fast_dictionary_table(
+    double_fast_table: &ZSTD_rust_loadDictionaryContentDoubleFastTableState,
+    end: *const c_void,
+) -> bool {
+    if double_fast_table.available == 0 {
+        return false;
+    }
+    unsafe {
+        ZSTD_rust_fillDoubleHashTable(
+            *double_fast_table.hash_long,
+            *double_fast_table.hash_small,
+            *double_fast_table.base,
+            *double_fast_table.next_to_update,
+            end,
+            *double_fast_table.hash_log,
+            *double_fast_table.chain_log,
+            *double_fast_table.min_match,
+            double_fast_table.full_table_load,
+            double_fast_table.for_cdict,
+        )
+    }
+    true
+}
+
 unsafe fn load_dictionary_content(
     state: &ZSTD_rust_loadDictionaryContentState,
     src: *const c_void,
@@ -465,12 +550,10 @@ unsafe fn load_dictionary_content(
             fill_fast_dictionary_table(fast_table, iend.cast())
         },
         DictionaryTablePolicy::DoubleFast => unsafe {
-            (state.fill_double_hash_table)(
+            let double_fast_table = &*state.double_fast_table;
-                state.callback_context,
+            if !fill_double_fast_dictionary_table(double_fast_table, iend.cast()) {
-                iend.cast(),
+                (state.assert_invalid_strategy)(state.callback_context);
-                state.dtlm as c_int,
+            }
-                state.tfp as c_int,
-            )
         },
         DictionaryTablePolicy::Dedicated => unsafe {
             (state.load_dedicated)(state.callback_context, hash_ip.cast())
@@ -6753,4 +6836,47 @@ mod tests {
         assert_eq!(projection.full_table_load, 1);
         assert_eq!(projection.for_cdict, 1);
     }
+
+    #[test]
+    fn double_fast_dictionary_table_projection_dispatches_to_rust_leaf() {
+        let hash_log = 4u32;
+        let chain_log = 4u32;
+        let mut hash_long = vec![0u32; 1usize << (hash_log + 8)];
+        let mut hash_small = vec![0u32; 1usize << (chain_log + 8)];
+        let mut input = [0u8; 64];
+        for (index, byte) in input.iter_mut().enumerate() {
+            *byte = (index as u8).wrapping_mul(29).wrapping_add(7);
+        }
+        let hash_long_ptr = hash_long.as_mut_ptr();
+        let hash_small_ptr = hash_small.as_mut_ptr();
+        let base_ptr = input.as_ptr();
+        let next_to_update = 17u32;
+        let min_match = 4u32;
+        let projection = ZSTD_rust_loadDictionaryContentDoubleFastTableState {
+            hash_long: &hash_long_ptr,
+            hash_small: &hash_small_ptr,
+            base: &base_ptr,
+            next_to_update: &next_to_update,
+            hash_log: &hash_log,
+            chain_log: &chain_log,
+            min_match: &min_match,
+            full_table_load: 1,
+            for_cdict: 1,
+            available: 1,
+        };
+
+        assert!(unsafe {
+            fill_double_fast_dictionary_table(
+                &projection,
+                input.as_ptr().wrapping_add(input.len()).cast(),
+            )
+        });
+
+        assert!(hash_long.iter().any(|&entry| entry != 0));
+        assert!(hash_small.iter().any(|&entry| entry != 0));
+        assert_eq!(unsafe { *projection.next_to_update }, 17);
+        assert_eq!(projection.full_table_load, 1);
+        assert_eq!(projection.for_cdict, 1);
+        assert_eq!(projection.available, 1);
+    }
 }