From cf6a3166931c9a559b74b0962825905739bd072d Mon Sep 17 00:00:00 2001 From: ddidderr Date: Tue, 21 Jul 2026 23:01:31 +0200 Subject: [PATCH] feat(compress): move dictionary state publication to Rust Move LDM loadedDictEnd and final nextToUpdate publication out of the C dictionary-content callbacks. Rust now writes through compact projections with live window bases, while C retains the configuration-sensitive window, matchfinder, and LDM operations. Test Plan: - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --tests - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --tests -- -A clippy::manual-bits -D warnings - ulimit -v 41943040; make -j1 - ulimit -v 41943040; make -j1 -C tests test --- lib/compress/zstd_compress.c | 52 +++++------ rust/src/zstd_compress_dictionary.rs | 133 ++++++++++++++++++++++++--- 2 files changed, 144 insertions(+), 41 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3f46ecf49..0f8d6959a 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2767,12 +2767,22 @@ typedef void (*ZSTD_rust_loadDictionaryContent_assertWindowEmpty_f)( void* context, int ldm); typedef void (*ZSTD_rust_loadDictionaryContent_windowUpdate_f)( void* context, int ldm, const void* src, size_t srcSize); -typedef void (*ZSTD_rust_loadDictionaryContent_setLdmLoadedDictEnd_f)( - void* context, const void* iend, int forceWindow); typedef void (*ZSTD_rust_loadDictionaryContent_fillLdm_f)( void* context, const void* ip, const void* iend); typedef void (*ZSTD_rust_loadDictionaryContent_overflowCorrect_f)( void* context, const void* ip, const void* iend); +typedef struct { + const BYTE** base; + U32* loadedDictEnd; +} ZSTD_rust_loadDictionaryContentLdmPublicationState; +typedef char ZSTD_rust_load_dictionary_ldm_publication_layout[ + (offsetof(ZSTD_rust_loadDictionaryContentLdmPublicationState, base) == 0 + && offsetof(ZSTD_rust_loadDictionaryContentLdmPublicationState, + loadedDictEnd) + == sizeof(void*) + && sizeof(ZSTD_rust_loadDictionaryContentLdmPublicationState) + == 2 * sizeof(void*)) + ? 1 : -1]; typedef struct { const BYTE** base; U32* nextToUpdate; @@ -2866,8 +2876,6 @@ typedef void (*ZSTD_rust_loadDictionaryContent_loadMatch_f)( void* context, const void* ip); typedef void (*ZSTD_rust_loadDictionaryContent_loadTree_f)( void* context, const void* ip, const void* iend); -typedef void (*ZSTD_rust_loadDictionaryContent_publishFinalIndex_f)( - void* context, const void* iend); typedef struct { void* callbackContext; @@ -2891,7 +2899,7 @@ typedef struct { ZSTD_rust_loadDictionaryContent_assert_f assertCParams; ZSTD_rust_loadDictionaryContent_assertWindowEmpty_f assertWindowEmpty; ZSTD_rust_loadDictionaryContent_windowUpdate_f windowUpdate; - ZSTD_rust_loadDictionaryContent_setLdmLoadedDictEnd_f setLdmLoadedDictEnd; + const ZSTD_rust_loadDictionaryContentLdmPublicationState* setLdmLoadedDictEnd; const ZSTD_rust_loadDictionaryContentMatchStatePublicationState* publishMatchState; ZSTD_rust_loadDictionaryContent_fillLdm_f fillLdm; ZSTD_rust_loadDictionaryContent_overflowCorrect_f overflowCorrect; @@ -2901,7 +2909,7 @@ typedef struct { ZSTD_rust_loadDictionaryContent_loadMatch_f loadRow; ZSTD_rust_loadDictionaryContent_loadMatch_f loadChain; ZSTD_rust_loadDictionaryContent_loadTree_f loadTree; - ZSTD_rust_loadDictionaryContent_publishFinalIndex_f publishFinalIndex; + const ZSTD_rust_loadDictionaryContentMatchStatePublicationState* publishFinalIndex; ZSTD_rust_loadDictionaryContent_assert_f assertLazyConfiguration; ZSTD_rust_loadDictionaryContent_assert_f assertInvalidStrategy; } ZSTD_rust_loadDictionaryContentState; @@ -2917,6 +2925,12 @@ ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_scalar_offset, ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_callback_offset, offsetof(ZSTD_rust_loadDictionaryContentState, assertCParams) == 18 * sizeof(void*)); +ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_ldm_publication_offset, + offsetof(ZSTD_rust_loadDictionaryContentState, + setLdmLoadedDictEnd) == 21 * sizeof(void*)); +ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_match_publication_offset, + offsetof(ZSTD_rust_loadDictionaryContentState, + publishMatchState) == 22 * sizeof(void*)); ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_fast_table_offset, offsetof(ZSTD_rust_loadDictionaryContentState, fastTable) == 25 * sizeof(void*)); @@ -6904,17 +6918,6 @@ static void ZSTD_loadDictionaryContent_windowUpdate( } } -static void ZSTD_loadDictionaryContent_setLdmLoadedDictEnd( - void* opaque, const void* iend, int forceWindow) -{ - ZSTD_loadDictionaryContent_context const* const context = - (const ZSTD_loadDictionaryContent_context*)opaque; - assert(context->ldmState != NULL); - context->ldmState->loadedDictEnd = forceWindow - ? 0 - : (U32)((const BYTE*)iend - context->ldmState->window.base); -} - static void ZSTD_loadDictionaryContent_fillLdm( void* opaque, const void* ip, const void* iend) { @@ -7000,14 +7003,6 @@ static void ZSTD_loadDictionaryContent_loadTree( #endif } -static void ZSTD_loadDictionaryContent_publishFinalIndex(void* opaque, const void* iend) -{ - ZSTD_loadDictionaryContent_context const* const context = - (const ZSTD_loadDictionaryContent_context*)opaque; - context->matchState->nextToUpdate = - (U32)((const BYTE*)iend - context->matchState->window.base); -} - static void ZSTD_loadDictionaryContent_assertLazyConfiguration(void* opaque) { ZSTD_loadDictionaryContent_context const* const context = @@ -7030,6 +7025,7 @@ static size_t ZSTD_loadDictionaryContent_callback( int dtlm, int tfp) { ZSTD_loadDictionaryContent_context context; + ZSTD_rust_loadDictionaryContentLdmPublicationState ldmPublication; ZSTD_rust_loadDictionaryContentMatchStatePublicationState matchStatePublication; ZSTD_rust_loadDictionaryContentFastTableState fastTable; ZSTD_rust_loadDictionaryContentDoubleFastTableState doubleFastTable; @@ -7048,6 +7044,8 @@ static size_t ZSTD_loadDictionaryContent_callback( matchStatePublication.nextToUpdate = &ms->nextToUpdate; matchStatePublication.loadedDictEnd = &ms->loadedDictEnd; matchStatePublication.forceNonContiguous = &ms->forceNonContiguous; + ldmPublication.base = ls == NULL ? NULL : &ls->window.base; + ldmPublication.loadedDictEnd = ls == NULL ? NULL : &ls->loadedDictEnd; assert((tfp == ZSTD_tfp_forCDict && dtlm == ZSTD_dtlm_full) || (tfp != ZSTD_tfp_forCDict && dtlm == ZSTD_dtlm_fast)); @@ -7097,7 +7095,7 @@ static size_t ZSTD_loadDictionaryContent_callback( state.assertCParams = ZSTD_loadDictionaryContent_assertCParams; state.assertWindowEmpty = ZSTD_loadDictionaryContent_assertWindowEmpty; state.windowUpdate = ZSTD_loadDictionaryContent_windowUpdate; - state.setLdmLoadedDictEnd = ZSTD_loadDictionaryContent_setLdmLoadedDictEnd; + state.setLdmLoadedDictEnd = &ldmPublication; state.publishMatchState = &matchStatePublication; state.fillLdm = ZSTD_loadDictionaryContent_fillLdm; state.overflowCorrect = ZSTD_loadDictionaryContent_overflowCorrect; @@ -7107,7 +7105,7 @@ static size_t ZSTD_loadDictionaryContent_callback( state.loadRow = ZSTD_loadDictionaryContent_loadRow; state.loadChain = ZSTD_loadDictionaryContent_loadChain; state.loadTree = ZSTD_loadDictionaryContent_loadTree; - state.publishFinalIndex = ZSTD_loadDictionaryContent_publishFinalIndex; + state.publishFinalIndex = &matchStatePublication; state.assertLazyConfiguration = ZSTD_loadDictionaryContent_assertLazyConfiguration; state.assertInvalidStrategy = ZSTD_loadDictionaryContent_assertInvalidStrategy; return ZSTD_rust_loadDictionaryContent(&state, src, srcSize); diff --git a/rust/src/zstd_compress_dictionary.rs b/rust/src/zstd_compress_dictionary.rs index 4b515376f..d417f8805 100644 --- a/rust/src/zstd_compress_dictionary.rs +++ b/rust/src/zstd_compress_dictionary.rs @@ -127,13 +127,32 @@ type LoadDictionaryContentAssertWindowEmptyFn = unsafe extern "C" fn(context: *mut c_void, ldm: c_int); type LoadDictionaryContentWindowUpdateFn = unsafe extern "C" fn(context: *mut c_void, ldm: c_int, src: *const c_void, src_size: usize); -type LoadDictionaryContentSetLdmLoadedDictEndFn = - unsafe extern "C" fn(context: *mut c_void, iend: *const c_void, force_window: c_int); type LoadDictionaryContentFillLdmFn = unsafe extern "C" fn(context: *mut c_void, ip: *const c_void, iend: *const c_void); type LoadDictionaryContentOverflowCorrectFn = unsafe extern "C" fn(context: *mut c_void, ip: *const c_void, iend: *const c_void); +#[repr(C)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ZSTD_rust_loadDictionaryContentLdmPublicationState { + base: *const *const u8, + loaded_dict_end: *mut c_uint, +} + +const _: () = { + assert!(offset_of!(ZSTD_rust_loadDictionaryContentLdmPublicationState, base) == 0); + assert!( + offset_of!( + ZSTD_rust_loadDictionaryContentLdmPublicationState, + loaded_dict_end + ) == size_of::() + ); + assert!( + size_of::() + == 2 * size_of::() + ); +}; + /// Live match-state fields used when the dictionary loader publishes the /// current prefix. The base is indirect because window updates may replace it /// before this publication point. @@ -209,8 +228,6 @@ type LoadDictionaryContentLoadMatchFn = unsafe extern "C" fn(context: *mut c_void, ip: *const c_void); type LoadDictionaryContentLoadTreeFn = unsafe extern "C" fn(context: *mut c_void, ip: *const c_void, iend: *const c_void); -type LoadDictionaryContentPublishFinalIndexFn = - unsafe extern "C" fn(context: *mut c_void, iend: *const c_void); /// Rust-owned policy projection for `ZSTD_loadDictionaryContent()`. /// @@ -242,7 +259,7 @@ pub struct ZSTD_rust_loadDictionaryContentState { assert_c_params: LoadDictionaryContentAssertFn, assert_window_empty: LoadDictionaryContentAssertWindowEmptyFn, window_update: LoadDictionaryContentWindowUpdateFn, - set_ldm_loaded_dict_end: LoadDictionaryContentSetLdmLoadedDictEndFn, + set_ldm_loaded_dict_end: *const ZSTD_rust_loadDictionaryContentLdmPublicationState, publish_match_state: *const ZSTD_rust_loadDictionaryContentMatchStatePublicationState, fill_ldm: LoadDictionaryContentFillLdmFn, overflow_correct: LoadDictionaryContentOverflowCorrectFn, @@ -252,7 +269,7 @@ pub struct ZSTD_rust_loadDictionaryContentState { load_row: LoadDictionaryContentLoadMatchFn, load_chain: LoadDictionaryContentLoadMatchFn, load_tree: LoadDictionaryContentLoadTreeFn, - publish_final_index: LoadDictionaryContentPublishFinalIndexFn, + publish_final_index: *const ZSTD_rust_loadDictionaryContentMatchStatePublicationState, assert_lazy_configuration: LoadDictionaryContentAssertFn, assert_invalid_strategy: LoadDictionaryContentAssertFn, } @@ -261,7 +278,6 @@ const _: () = { assert!(size_of::() == size_of::()); assert!(size_of::() == size_of::()); assert!(size_of::() == size_of::()); - assert!(size_of::() == size_of::()); assert!(size_of::() == size_of::()); assert!(size_of::() == size_of::()); assert!(offset_of!(ZSTD_rust_loadDictionaryContentFastTableState, hash_table) == 0); @@ -339,13 +355,16 @@ const _: () = { ); assert!(size_of::() == size_of::()); assert!(size_of::() == size_of::()); - assert!(size_of::() == size_of::()); assert!(offset_of!(ZSTD_rust_loadDictionaryContentState, callback_context) == 0); assert!(offset_of!(ZSTD_rust_loadDictionaryContentState, current_max) == size_of::()); assert!( offset_of!(ZSTD_rust_loadDictionaryContentState, assert_c_params) == 18 * size_of::() ); + assert!( + offset_of!(ZSTD_rust_loadDictionaryContentState, set_ldm_loaded_dict_end) + == 21 * size_of::() + ); assert!( offset_of!(ZSTD_rust_loadDictionaryContentState, publish_match_state) == 22 * size_of::() @@ -526,12 +545,39 @@ unsafe fn publish_dictionary_match_state( } } +#[inline] +unsafe fn publish_dictionary_ldm_loaded_dict_end( + publication: &ZSTD_rust_loadDictionaryContentLdmPublicationState, + iend: *const u8, + force_window: usize, +) { + let base = unsafe { *publication.base }; + unsafe { + *publication.loaded_dict_end = if force_window != 0 { + 0 + } else { + iend.offset_from(base) as c_uint + }; + } +} + +#[inline] +unsafe fn publish_dictionary_final_index( + publication: &ZSTD_rust_loadDictionaryContentMatchStatePublicationState, + iend: *const u8, +) { + let base = unsafe { *publication.base }; + unsafe { + *publication.next_to_update = iend.offset_from(base) as c_uint; + } +} + unsafe fn load_dictionary_content( state: &ZSTD_rust_loadDictionaryContentState, src: *const c_void, src_size: usize, ) -> usize { - if state.publish_match_state.is_null() { + if state.publish_match_state.is_null() || state.publish_final_index.is_null() { return ERROR(ZstdErrorCode::Generic); } let match_state_publication = unsafe { &*state.publish_match_state }; @@ -563,12 +609,16 @@ unsafe fn load_dictionary_content( unsafe { (state.window_update)(state.callback_context, 0, ip.cast(), loaded_src_size) }; if load_ldm_dict { + if state.set_ldm_loaded_dict_end.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + let ldm_publication = unsafe { &*state.set_ldm_loaded_dict_end }; unsafe { (state.window_update)(state.callback_context, 1, ip.cast(), loaded_src_size); - (state.set_ldm_loaded_dict_end)( - state.callback_context, - iend.cast(), - (state.force_window != 0) as c_int, + publish_dictionary_ldm_loaded_dict_end( + ldm_publication, + iend, + state.force_window, ); (state.fill_ldm)(state.callback_context, ip.cast(), iend.cast()); } @@ -634,7 +684,8 @@ unsafe fn load_dictionary_content( }, } - unsafe { (state.publish_final_index)(state.callback_context, iend.cast()) }; + let final_index_publication = unsafe { &*state.publish_final_index }; + unsafe { publish_dictionary_final_index(final_index_publication, iend) }; 0 } @@ -6910,6 +6961,60 @@ mod tests { assert_eq!(force_non_contiguous, 0); } + #[test] + fn dictionary_ldm_and_final_index_projections_use_live_bases() { + let input = [0u8; 64]; + let mut ldm_base_ptr = input.as_ptr(); + let mut ldm_loaded_dict_end = 0u32; + let ldm_projection = ZSTD_rust_loadDictionaryContentLdmPublicationState { + base: &ldm_base_ptr, + loaded_dict_end: &mut ldm_loaded_dict_end, + }; + + unsafe { + publish_dictionary_ldm_loaded_dict_end( + &ldm_projection, + input.as_ptr().add(32), + 0, + ); + assert_eq!(ldm_loaded_dict_end, 32); + ptr::write(&mut ldm_base_ptr, input.as_ptr().add(4)); + assert_eq!(*ldm_projection.base, input.as_ptr().add(4)); + publish_dictionary_ldm_loaded_dict_end( + &ldm_projection, + input.as_ptr().add(32), + 0, + ); + publish_dictionary_ldm_loaded_dict_end( + &ldm_projection, + input.as_ptr().add(32), + 1, + ); + } + assert_eq!(ldm_loaded_dict_end, 0); + + let mut match_base_ptr = input.as_ptr(); + let mut next_to_update = 0u32; + let mut loaded_dict_end = 0u32; + let mut force_non_contiguous = 0i32; + let match_projection = ZSTD_rust_loadDictionaryContentMatchStatePublicationState { + base: &match_base_ptr, + next_to_update: &mut next_to_update, + loaded_dict_end: &mut loaded_dict_end, + force_non_contiguous: &mut force_non_contiguous, + }; + + unsafe { + ptr::write(&mut match_base_ptr, input.as_ptr().add(6)); + assert_eq!(*match_projection.base, input.as_ptr().add(6)); + publish_dictionary_final_index( + &match_projection, + input.as_ptr().add(38), + ); + } + assert_eq!(next_to_update, 32); + } + #[test] fn fast_dictionary_table_projection_dispatches_to_rust_leaf() { let hash_log = 4u32;