From f89ae7589871b064325f9fb145247598786552dc Mon Sep 17 00:00:00 2001 From: ddidderr Date: Tue, 21 Jul 2026 21:33:37 +0200 Subject: [PATCH] feat(compress): move Double-Fast dictionary fill to Rust Replace the dictionary-content Double-Fast callback with a live-field projection and call the existing Rust table-fill leaf directly. The projection keeps hash tables, window base, match-state index, and parameters live through the window/publication callbacks, preserving the timing behavior of the former C adapter. Preserve the excluded-DFast assertion path and the 34-word dictionary bridge layout. Test Plan: - git diff --check - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --manifest-path rust/Cargo.toml --tests - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -A clippy::manual-bits -D warnings - ulimit -v 41943040; make -j1 - capped strategy-2 compress/decompress round trip with cmp - Full original suite deferred to the next serialized verification step --- lib/compress/zstd_compress.c | 78 ++++++++++---- rust/src/zstd_compress_dictionary.rs | 146 +++++++++++++++++++++++++-- 2 files changed, 195 insertions(+), 29 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index d794b309b..b39e6302b 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2805,8 +2805,45 @@ typedef char ZSTD_rust_load_dictionary_fast_table_layout[ && sizeof(ZSTD_rust_loadDictionaryContentFastTableState) == 5 * sizeof(void*) + 2 * sizeof(int)) ? 1 : -1]; -typedef void (*ZSTD_rust_loadDictionaryContent_fillTable_f)( - void* context, const void* iend, int dtlm, int tfp); +/* The Double-Fast dictionary-table leaf is also implemented in Rust. Keep + * pointers to the live match-state fields because window/publish callbacks + * run before the Rust orchestrator selects this branch. */ +typedef struct { + U32** hashLong; + U32** hashSmall; + const BYTE** base; + const U32* nextToUpdate; + const U32* hashLog; + const U32* chainLog; + const U32* minMatch; + int fullTableLoad; + int forCDict; + int available; +} ZSTD_rust_loadDictionaryContentDoubleFastTableState; +typedef char ZSTD_rust_load_dictionary_double_fast_table_layout[ + (offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hashLong) == 0 + && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hashSmall) + == sizeof(void*) + && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, base) + == 2 * sizeof(void*) + && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, nextToUpdate) + == 3 * sizeof(void*) + && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hashLog) + == 4 * sizeof(void*) + && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, chainLog) + == 5 * sizeof(void*) + && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, minMatch) + == 6 * sizeof(void*) + && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, fullTableLoad) + == 7 * sizeof(void*) + && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, forCDict) + == 7 * sizeof(void*) + sizeof(int) + && offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, available) + == 7 * sizeof(void*) + 2 * sizeof(int) + && sizeof(ZSTD_rust_loadDictionaryContentDoubleFastTableState) + == ((7 * sizeof(void*) + 3 * sizeof(int) + sizeof(void*) - 1) + / sizeof(void*) * sizeof(void*))) + ? 1 : -1]; typedef void (*ZSTD_rust_loadDictionaryContent_loadMatch_f)( void* context, const void* ip); typedef void (*ZSTD_rust_loadDictionaryContent_loadTree_f)( @@ -2841,7 +2878,7 @@ typedef struct { ZSTD_rust_loadDictionaryContent_fillLdm_f fillLdm; ZSTD_rust_loadDictionaryContent_overflowCorrect_f overflowCorrect; const ZSTD_rust_loadDictionaryContentFastTableState* fastTable; - ZSTD_rust_loadDictionaryContent_fillTable_f fillDoubleHashTable; + const ZSTD_rust_loadDictionaryContentDoubleFastTableState* doubleFastTable; ZSTD_rust_loadDictionaryContent_loadMatch_f loadDedicated; ZSTD_rust_loadDictionaryContent_loadMatch_f loadRow; ZSTD_rust_loadDictionaryContent_loadMatch_f loadChain; @@ -2865,6 +2902,9 @@ ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_callback_offset, ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_fast_table_offset, offsetof(ZSTD_rust_loadDictionaryContentState, fastTable) == 25 * sizeof(void*)); +ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_double_fast_table_offset, + offsetof(ZSTD_rust_loadDictionaryContentState, + doubleFastTable) == 26 * sizeof(void*)); ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_final_callback_offset, offsetof(ZSTD_rust_loadDictionaryContentState, publishFinalIndex) == 31 * sizeof(void*)); @@ -6891,21 +6931,6 @@ static void ZSTD_loadDictionaryContent_overflowCorrect( context->params, ip, iend); } -static void ZSTD_loadDictionaryContent_fillDoubleHashTable( - void* opaque, const void* iend, int dtlm, int tfp) -{ - ZSTD_loadDictionaryContent_context const* const context = - (const ZSTD_loadDictionaryContent_context*)opaque; -#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR - ZSTD_fillDoubleHashTable(context->matchState, (const BYTE*)iend, - (ZSTD_dictTableLoadMethod_e)dtlm, - (ZSTD_tableFillPurpose_e)tfp); -#else - (void)context; (void)iend; (void)dtlm; (void)tfp; - assert(0); /* shouldn't be called: cparams should've been adjusted. */ -#endif -} - static void ZSTD_loadDictionaryContent_loadDedicated(void* opaque, const void* ip) { ZSTD_loadDictionaryContent_context const* const context = @@ -7002,6 +7027,7 @@ static size_t ZSTD_loadDictionaryContent_callback( { ZSTD_loadDictionaryContent_context context; ZSTD_rust_loadDictionaryContentFastTableState fastTable; + ZSTD_rust_loadDictionaryContentDoubleFastTableState doubleFastTable; ZSTD_rust_loadDictionaryContentState state; ZSTD_MatchState_t* const ms = (ZSTD_MatchState_t*)matchState; ldmState_t* const ls = (ldmState_t*)ldmState; @@ -7025,6 +7051,20 @@ static size_t ZSTD_loadDictionaryContent_callback( fastTable.minMatch = &ms->cParams.minMatch; fastTable.fullTableLoad = dtlm == ZSTD_dtlm_full; fastTable.forCDict = tfp == ZSTD_tfp_forCDict; + doubleFastTable.hashLong = &ms->hashTable; + doubleFastTable.hashSmall = &ms->chainTable; + doubleFastTable.base = &ms->window.base; + doubleFastTable.nextToUpdate = &ms->nextToUpdate; + doubleFastTable.hashLog = &ms->cParams.hashLog; + doubleFastTable.chainLog = &ms->cParams.chainLog; + doubleFastTable.minMatch = &ms->cParams.minMatch; + doubleFastTable.fullTableLoad = dtlm == ZSTD_dtlm_full; + doubleFastTable.forCDict = tfp == ZSTD_tfp_forCDict; +#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR + doubleFastTable.available = 1; +#else + doubleFastTable.available = 0; +#endif state.callbackContext = &context; state.currentMax = ZSTD_CURRENT_MAX; @@ -7052,7 +7092,7 @@ static size_t ZSTD_loadDictionaryContent_callback( state.fillLdm = ZSTD_loadDictionaryContent_fillLdm; state.overflowCorrect = ZSTD_loadDictionaryContent_overflowCorrect; state.fastTable = &fastTable; - state.fillDoubleHashTable = ZSTD_loadDictionaryContent_fillDoubleHashTable; + state.doubleFastTable = &doubleFastTable; state.loadDedicated = ZSTD_loadDictionaryContent_loadDedicated; state.loadRow = ZSTD_loadDictionaryContent_loadRow; state.loadChain = ZSTD_loadDictionaryContent_loadChain; diff --git a/rust/src/zstd_compress_dictionary.rs b/rust/src/zstd_compress_dictionary.rs index bb99d5dd5..a523f296b 100644 --- a/rust/src/zstd_compress_dictionary.rs +++ b/rust/src/zstd_compress_dictionary.rs @@ -28,6 +28,7 @@ use crate::zstd_compress_params_api::ZSTD_CCtx_params; use crate::zstd_compress_stats::{ ZSTD_compressedBlockState_t, ZSTD_rust_resetCompressedBlockState, }; +use crate::zstd_double_fast::ZSTD_rust_fillDoubleHashTable; use crate::zstd_fast::ZSTD_rust_fillHashTable; use std::ffi::c_void; use std::mem::{offset_of, size_of}; @@ -156,8 +157,21 @@ struct ZSTD_rust_loadDictionaryContentFastTableState { for_cdict: c_int, } -type LoadDictionaryContentFillTableFn = - unsafe extern "C" fn(context: *mut c_void, iend: *const c_void, dtlm: c_int, tfp: c_int); +/// Direct projection for the Double-Fast dictionary-table leaf. +#[repr(C)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ZSTD_rust_loadDictionaryContentDoubleFastTableState { + hash_long: *const *mut c_uint, + hash_small: *const *mut c_uint, + base: *const *const u8, + next_to_update: *const c_uint, + hash_log: *const c_uint, + chain_log: *const c_uint, + min_match: *const c_uint, + full_table_load: c_int, + for_cdict: c_int, + available: c_int, +} type LoadDictionaryContentLoadMatchFn = unsafe extern "C" fn(context: *mut c_void, ip: *const c_void); type LoadDictionaryContentLoadTreeFn = @@ -200,7 +214,7 @@ pub struct ZSTD_rust_loadDictionaryContentState { fill_ldm: LoadDictionaryContentFillLdmFn, overflow_correct: LoadDictionaryContentOverflowCorrectFn, fast_table: *const ZSTD_rust_loadDictionaryContentFastTableState, - fill_double_hash_table: LoadDictionaryContentFillTableFn, + double_fast_table: *const ZSTD_rust_loadDictionaryContentDoubleFastTableState, load_dedicated: LoadDictionaryContentLoadMatchFn, load_row: LoadDictionaryContentLoadMatchFn, load_chain: LoadDictionaryContentLoadMatchFn, @@ -218,7 +232,6 @@ const _: () = { assert!(size_of::() == size_of::()); assert!(size_of::() == size_of::()); assert!(size_of::() == size_of::()); - assert!(size_of::() == size_of::()); assert!(offset_of!(ZSTD_rust_loadDictionaryContentFastTableState, hash_table) == 0); assert!(offset_of!(ZSTD_rust_loadDictionaryContentFastTableState, base) == size_of::()); assert!( @@ -249,6 +262,49 @@ const _: () = { size_of::() == 5 * size_of::() + 2 * size_of::() ); + assert!(offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hash_long) == 0); + assert!( + offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hash_small) + == size_of::() + ); + assert!( + offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, base) + == 2 * size_of::() + ); + assert!( + offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, next_to_update) + == 3 * size_of::() + ); + assert!( + offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hash_log) + == 4 * size_of::() + ); + assert!( + offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, chain_log) + == 5 * size_of::() + ); + assert!( + offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, min_match) + == 6 * size_of::() + ); + assert!( + offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, full_table_load) + == 7 * size_of::() + ); + assert!( + offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, for_cdict) + == 7 * size_of::() + size_of::() + ); + assert!( + offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, available) + == 7 * size_of::() + 2 * size_of::() + ); + assert!( + size_of::() + == (7 * size_of::() + 3 * size_of::()) + .div_ceil(size_of::()) + * size_of::() + ); assert!(size_of::() == size_of::()); assert!(size_of::() == size_of::()); assert!(size_of::() == size_of::()); @@ -261,6 +317,10 @@ const _: () = { assert!( offset_of!(ZSTD_rust_loadDictionaryContentState, fast_table) == 25 * size_of::() ); + assert!( + offset_of!(ZSTD_rust_loadDictionaryContentState, double_fast_table) + == 26 * size_of::() + ); assert!( offset_of!(ZSTD_rust_loadDictionaryContentState, publish_final_index) == 31 * size_of::() @@ -385,6 +445,31 @@ unsafe fn fill_fast_dictionary_table( } } +#[inline] +unsafe fn fill_double_fast_dictionary_table( + double_fast_table: &ZSTD_rust_loadDictionaryContentDoubleFastTableState, + end: *const c_void, +) -> bool { + if double_fast_table.available == 0 { + return false; + } + unsafe { + ZSTD_rust_fillDoubleHashTable( + *double_fast_table.hash_long, + *double_fast_table.hash_small, + *double_fast_table.base, + *double_fast_table.next_to_update, + end, + *double_fast_table.hash_log, + *double_fast_table.chain_log, + *double_fast_table.min_match, + double_fast_table.full_table_load, + double_fast_table.for_cdict, + ) + } + true +} + unsafe fn load_dictionary_content( state: &ZSTD_rust_loadDictionaryContentState, src: *const c_void, @@ -465,12 +550,10 @@ unsafe fn load_dictionary_content( fill_fast_dictionary_table(fast_table, iend.cast()) }, DictionaryTablePolicy::DoubleFast => unsafe { - (state.fill_double_hash_table)( - state.callback_context, - iend.cast(), - state.dtlm as c_int, - state.tfp as c_int, - ) + let double_fast_table = &*state.double_fast_table; + if !fill_double_fast_dictionary_table(double_fast_table, iend.cast()) { + (state.assert_invalid_strategy)(state.callback_context); + } }, DictionaryTablePolicy::Dedicated => unsafe { (state.load_dedicated)(state.callback_context, hash_ip.cast()) @@ -6753,4 +6836,47 @@ mod tests { assert_eq!(projection.full_table_load, 1); assert_eq!(projection.for_cdict, 1); } + + #[test] + fn double_fast_dictionary_table_projection_dispatches_to_rust_leaf() { + let hash_log = 4u32; + let chain_log = 4u32; + let mut hash_long = vec![0u32; 1usize << (hash_log + 8)]; + let mut hash_small = vec![0u32; 1usize << (chain_log + 8)]; + let mut input = [0u8; 64]; + for (index, byte) in input.iter_mut().enumerate() { + *byte = (index as u8).wrapping_mul(29).wrapping_add(7); + } + let hash_long_ptr = hash_long.as_mut_ptr(); + let hash_small_ptr = hash_small.as_mut_ptr(); + let base_ptr = input.as_ptr(); + let next_to_update = 17u32; + let min_match = 4u32; + let projection = ZSTD_rust_loadDictionaryContentDoubleFastTableState { + hash_long: &hash_long_ptr, + hash_small: &hash_small_ptr, + base: &base_ptr, + next_to_update: &next_to_update, + hash_log: &hash_log, + chain_log: &chain_log, + min_match: &min_match, + full_table_load: 1, + for_cdict: 1, + available: 1, + }; + + assert!(unsafe { + fill_double_fast_dictionary_table( + &projection, + input.as_ptr().wrapping_add(input.len()).cast(), + ) + }); + + assert!(hash_long.iter().any(|&entry| entry != 0)); + assert!(hash_small.iter().any(|&entry| entry != 0)); + assert_eq!(unsafe { *projection.next_to_update }, 17); + assert_eq!(projection.full_table_load, 1); + assert_eq!(projection.for_cdict, 1); + assert_eq!(projection.available, 1); + } }