diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 5e25c3501..7749f5328 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2776,6 +2776,37 @@ typedef void (*ZSTD_rust_loadDictionaryContent_fillLdm_f)( void* context, const void* ip, const void* iend); typedef void (*ZSTD_rust_loadDictionaryContent_overflowCorrect_f)( void* context, const void* ip, const void* iend); +/* The Fast dictionary-table leaf is implemented in Rust. Keep only the + * fields needed by that leaf in the ABI projection; the complete + * ZSTD_MatchState_t layout remains private to C. */ +typedef struct { + U32* hashTable; + const BYTE* base; + U32 nextToUpdate; + U32 hashLog; + U32 minMatch; + int fullTableLoad; + int forCDict; +} ZSTD_rust_loadDictionaryContentFastTableState; +typedef char ZSTD_rust_load_dictionary_fast_table_layout[ + (offsetof(ZSTD_rust_loadDictionaryContentFastTableState, hashTable) == 0 + && offsetof(ZSTD_rust_loadDictionaryContentFastTableState, base) + == sizeof(void*) + && offsetof(ZSTD_rust_loadDictionaryContentFastTableState, nextToUpdate) + == 2 * sizeof(void*) + && offsetof(ZSTD_rust_loadDictionaryContentFastTableState, hashLog) + == 2 * sizeof(void*) + sizeof(U32) + && offsetof(ZSTD_rust_loadDictionaryContentFastTableState, minMatch) + == 2 * sizeof(void*) + 2 * sizeof(U32) + && offsetof(ZSTD_rust_loadDictionaryContentFastTableState, fullTableLoad) + == 2 * sizeof(void*) + 3 * sizeof(U32) + && offsetof(ZSTD_rust_loadDictionaryContentFastTableState, forCDict) + == 2 * sizeof(void*) + 3 * sizeof(U32) + sizeof(int) + && sizeof(ZSTD_rust_loadDictionaryContentFastTableState) + == ((offsetof(ZSTD_rust_loadDictionaryContentFastTableState, forCDict) + + sizeof(int) + sizeof(void*) - 1) + / sizeof(void*) * sizeof(void*))) + ? 1 : -1]; typedef void (*ZSTD_rust_loadDictionaryContent_fillTable_f)( void* context, const void* iend, int dtlm, int tfp); typedef void (*ZSTD_rust_loadDictionaryContent_loadMatch_f)( @@ -2811,7 +2842,7 @@ typedef struct { ZSTD_rust_loadDictionaryContent_publishMatchState_f publishMatchState; ZSTD_rust_loadDictionaryContent_fillLdm_f fillLdm; ZSTD_rust_loadDictionaryContent_overflowCorrect_f overflowCorrect; - ZSTD_rust_loadDictionaryContent_fillTable_f fillHashTable; + const ZSTD_rust_loadDictionaryContentFastTableState* fastTable; ZSTD_rust_loadDictionaryContent_fillTable_f fillDoubleHashTable; ZSTD_rust_loadDictionaryContent_loadMatch_f loadDedicated; ZSTD_rust_loadDictionaryContent_loadMatch_f loadRow; @@ -2833,6 +2864,9 @@ ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_scalar_offset, ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_callback_offset, offsetof(ZSTD_rust_loadDictionaryContentState, assertCParams) == 18 * sizeof(void*)); +ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_fast_table_offset, + offsetof(ZSTD_rust_loadDictionaryContentState, + fastTable) == 25 * sizeof(void*)); ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_final_callback_offset, offsetof(ZSTD_rust_loadDictionaryContentState, publishFinalIndex) == 31 * sizeof(void*)); @@ -6858,16 +6892,6 @@ static void ZSTD_loadDictionaryContent_overflowCorrect( context->params, ip, iend); } -static void ZSTD_loadDictionaryContent_fillHashTable( - void* opaque, const void* iend, int dtlm, int tfp) -{ - ZSTD_loadDictionaryContent_context const* const context = - (const ZSTD_loadDictionaryContent_context*)opaque; - ZSTD_fillHashTable(context->matchState, (const BYTE*)iend, - (ZSTD_dictTableLoadMethod_e)dtlm, - (ZSTD_tableFillPurpose_e)tfp); -} - static void ZSTD_loadDictionaryContent_fillDoubleHashTable( void* opaque, const void* iend, int dtlm, int tfp) { @@ -6978,6 +7002,7 @@ static size_t ZSTD_loadDictionaryContent_callback( int dtlm, int tfp) { ZSTD_loadDictionaryContent_context context; + ZSTD_rust_loadDictionaryContentFastTableState fastTable; ZSTD_rust_loadDictionaryContentState state; ZSTD_MatchState_t* const ms = (ZSTD_MatchState_t*)matchState; ldmState_t* const ls = (ldmState_t*)ldmState; @@ -6989,6 +7014,16 @@ static size_t ZSTD_loadDictionaryContent_callback( context.workspace = ws; context.params = cctxParams; + assert((tfp == ZSTD_tfp_forCDict && dtlm == ZSTD_dtlm_full) + || (tfp != ZSTD_tfp_forCDict && dtlm == ZSTD_dtlm_fast)); + fastTable.hashTable = ms->hashTable; + fastTable.base = ms->window.base; + fastTable.nextToUpdate = ms->nextToUpdate; + fastTable.hashLog = ms->cParams.hashLog; + fastTable.minMatch = ms->cParams.minMatch; + fastTable.fullTableLoad = dtlm == ZSTD_dtlm_full; + fastTable.forCDict = tfp == ZSTD_tfp_forCDict; + state.callbackContext = &context; state.currentMax = ZSTD_CURRENT_MAX; state.windowStartIndex = ZSTD_WINDOW_START_INDEX; @@ -7014,7 +7049,7 @@ static size_t ZSTD_loadDictionaryContent_callback( state.publishMatchState = ZSTD_loadDictionaryContent_publishMatchState; state.fillLdm = ZSTD_loadDictionaryContent_fillLdm; state.overflowCorrect = ZSTD_loadDictionaryContent_overflowCorrect; - state.fillHashTable = ZSTD_loadDictionaryContent_fillHashTable; + state.fastTable = &fastTable; state.fillDoubleHashTable = ZSTD_loadDictionaryContent_fillDoubleHashTable; state.loadDedicated = ZSTD_loadDictionaryContent_loadDedicated; state.loadRow = ZSTD_loadDictionaryContent_loadRow; diff --git a/rust/src/zstd_compress_dictionary.rs b/rust/src/zstd_compress_dictionary.rs index f24566e3f..d97a899b2 100644 --- a/rust/src/zstd_compress_dictionary.rs +++ b/rust/src/zstd_compress_dictionary.rs @@ -28,6 +28,7 @@ use crate::zstd_compress_params_api::ZSTD_CCtx_params; use crate::zstd_compress_stats::{ ZSTD_compressedBlockState_t, ZSTD_rust_resetCompressedBlockState, }; +use crate::zstd_fast::ZSTD_rust_fillHashTable; use std::ffi::c_void; use std::mem::{offset_of, size_of}; use std::os::raw::{c_int, c_short, c_uint}; @@ -138,6 +139,23 @@ type LoadDictionaryContentFillLdmFn = unsafe extern "C" fn(context: *mut c_void, ip: *const c_void, iend: *const c_void); type LoadDictionaryContentOverflowCorrectFn = unsafe extern "C" fn(context: *mut c_void, ip: *const c_void, iend: *const c_void); +/// Direct projection for the Fast dictionary-table leaf. +/// +/// C copies `nextToUpdate` rather than exposing a pointer into the private +/// match state. `fullTableLoad` and `forCDict` are normalized from the C enum +/// inputs after their original validity assertion has run. +#[repr(C)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ZSTD_rust_loadDictionaryContentFastTableState { + hash_table: *mut c_uint, + base: *const u8, + next_to_update: c_uint, + hash_log: c_uint, + min_match: c_uint, + full_table_load: c_int, + for_cdict: c_int, +} + type LoadDictionaryContentFillTableFn = unsafe extern "C" fn(context: *mut c_void, iend: *const c_void, dtlm: c_int, tfp: c_int); type LoadDictionaryContentLoadMatchFn = @@ -149,9 +167,10 @@ type LoadDictionaryContentPublishFinalIndexFn = /// Rust-owned policy projection for `ZSTD_loadDictionaryContent()`. /// -/// All configuration-dependent C layouts stay behind callbacks. The scalar -/// fields are copied by the C adapter so Rust owns suffix selection, window -/// and LDM ordering, index publication, and strategy dispatch without seeing +/// All remaining configuration-dependent C layouts stay behind callbacks. The +/// Fast table leaf uses the explicit projection above; the scalar fields are +/// copied by the C adapter so Rust owns suffix selection, window and LDM +/// ordering, index publication, and strategy dispatch without seeing /// `ZSTD_MatchState_t`, `ldmState_t`, or workspace internals. #[repr(C)] pub struct ZSTD_rust_loadDictionaryContentState { @@ -180,7 +199,7 @@ pub struct ZSTD_rust_loadDictionaryContentState { publish_match_state: LoadDictionaryContentPublishMatchStateFn, fill_ldm: LoadDictionaryContentFillLdmFn, overflow_correct: LoadDictionaryContentOverflowCorrectFn, - fill_hash_table: LoadDictionaryContentFillTableFn, + fast_table: *const ZSTD_rust_loadDictionaryContentFastTableState, fill_double_hash_table: LoadDictionaryContentFillTableFn, load_dedicated: LoadDictionaryContentLoadMatchFn, load_row: LoadDictionaryContentLoadMatchFn, @@ -200,6 +219,39 @@ const _: () = { assert!(size_of::() == size_of::()); assert!(size_of::() == size_of::()); assert!(size_of::() == size_of::()); + assert!(offset_of!(ZSTD_rust_loadDictionaryContentFastTableState, hash_table) == 0); + assert!(offset_of!(ZSTD_rust_loadDictionaryContentFastTableState, base) == size_of::()); + assert!( + offset_of!( + ZSTD_rust_loadDictionaryContentFastTableState, + next_to_update + ) == 2 * size_of::() + ); + assert!( + offset_of!(ZSTD_rust_loadDictionaryContentFastTableState, hash_log) + == 2 * size_of::() + size_of::() + ); + assert!( + offset_of!(ZSTD_rust_loadDictionaryContentFastTableState, min_match) + == 2 * size_of::() + 2 * size_of::() + ); + assert!( + offset_of!( + ZSTD_rust_loadDictionaryContentFastTableState, + full_table_load + ) == 2 * size_of::() + 3 * size_of::() + ); + assert!( + offset_of!(ZSTD_rust_loadDictionaryContentFastTableState, for_cdict) + == 2 * size_of::() + 3 * size_of::() + size_of::() + ); + assert!( + size_of::() + == (offset_of!(ZSTD_rust_loadDictionaryContentFastTableState, for_cdict) + + size_of::()) + .div_ceil(size_of::()) + * size_of::() + ); assert!(size_of::() == size_of::()); assert!(size_of::() == size_of::()); assert!(size_of::() == size_of::()); @@ -209,6 +261,9 @@ const _: () = { offset_of!(ZSTD_rust_loadDictionaryContentState, assert_c_params) == 18 * size_of::() ); + assert!( + offset_of!(ZSTD_rust_loadDictionaryContentState, fast_table) == 25 * size_of::() + ); assert!( offset_of!(ZSTD_rust_loadDictionaryContentState, publish_final_index) == 31 * size_of::() @@ -390,11 +445,16 @@ unsafe fn load_dictionary_content( state.use_row_match_finder, ) { DictionaryTablePolicy::Fast => unsafe { - (state.fill_hash_table)( - state.callback_context, + let fast_table = &*state.fast_table; + ZSTD_rust_fillHashTable( + fast_table.hash_table, + fast_table.base, + fast_table.next_to_update, iend.cast(), - state.dtlm as c_int, - state.tfp as c_int, + fast_table.hash_log, + fast_table.min_match, + fast_table.full_table_load, + fast_table.for_cdict, ) }, DictionaryTablePolicy::DoubleFast => unsafe { @@ -6398,4 +6458,41 @@ mod tests { DictionaryTablePolicy::Tree ); } + + #[test] + fn fast_dictionary_table_projection_dispatches_to_rust_leaf() { + let hash_log = 4u32; + let mut hash_table = vec![0u32; 1usize << (hash_log + 8)]; + let mut input = [0u8; 64]; + for (index, byte) in input.iter_mut().enumerate() { + *byte = (index as u8).wrapping_mul(37).wrapping_add(11); + } + let projection = ZSTD_rust_loadDictionaryContentFastTableState { + hash_table: hash_table.as_mut_ptr(), + base: input.as_ptr(), + next_to_update: 17, + hash_log, + min_match: 4, + full_table_load: 1, + for_cdict: 1, + }; + + unsafe { + ZSTD_rust_fillHashTable( + projection.hash_table, + projection.base, + projection.next_to_update, + input.as_ptr().wrapping_add(input.len()).cast(), + projection.hash_log, + projection.min_match, + projection.full_table_load, + projection.for_cdict, + ) + }; + + assert!(hash_table.iter().any(|&entry| entry != 0)); + assert_eq!(projection.next_to_update, 17); + assert_eq!(projection.full_table_load, 1); + assert_eq!(projection.for_cdict, 1); + } }