diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 46bce0e88..935274622 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1414,6 +1414,80 @@ typedef char ZSTD_rust_external_sequence_store_state_layout[ == 5 * sizeof(void*)) ? 1 : -1]; +typedef size_t (*ZSTD_rust_initCDictAssignContent_f)( + void* context, const void* dict, size_t dictSize, int dictLoadMethod); +typedef void* (*ZSTD_rust_initCDictReserveEntropy_f)(void* context); +typedef void (*ZSTD_rust_initCDictResetBlockState_f)(void* context); +typedef size_t (*ZSTD_rust_initCDictResetMatchState_f)( + void* context, const ZSTD_compressionParameters* cParams, + int useRowMatchFinder); +typedef size_t (*ZSTD_rust_initCDictInsertDictionary_f)( + void* context, const void* params, const void* dict, + size_t dictSize, int dictContentType); +typedef struct { + void* callbackContext; + void* params; + const ZSTD_compressionParameters* cParams; + ZSTD_compressionParameters* matchStateCParams; + int* dedicatedDictSearch; + const int* enableDedicatedDictSearch; + const ZSTD_ParamSwitch_e* useRowMatchFinder; + const void** dictContent; + size_t* dictContentSize; + int* dictContentType; + U32** entropyWorkspace; + U32* dictID; + int* compressionLevel; + int* contentSizeFlag; + ZSTD_rust_initCDictAssignContent_f assignContent; + ZSTD_rust_initCDictReserveEntropy_f reserveEntropy; + ZSTD_rust_initCDictResetBlockState_f resetBlockState; + ZSTD_rust_initCDictResetMatchState_f resetMatchState; + ZSTD_rust_initCDictInsertDictionary_f insertDictionary; +} ZSTD_rust_initCDictState; +size_t ZSTD_rust_initCDict( + const ZSTD_rust_initCDictState* state, + const void* dict, size_t dictSize, + int dictLoadMethod, int dictContentType); +typedef char ZSTD_rust_init_cdict_state_layout[ + (offsetof(ZSTD_rust_initCDictState, callbackContext) == 0 + && offsetof(ZSTD_rust_initCDictState, params) == sizeof(void*) + && offsetof(ZSTD_rust_initCDictState, cParams) == 2 * sizeof(void*) + && offsetof(ZSTD_rust_initCDictState, matchStateCParams) + == 3 * sizeof(void*) + && offsetof(ZSTD_rust_initCDictState, dedicatedDictSearch) + == 4 * sizeof(void*) + && offsetof(ZSTD_rust_initCDictState, enableDedicatedDictSearch) + == 5 * sizeof(void*) + && offsetof(ZSTD_rust_initCDictState, useRowMatchFinder) + == 6 * sizeof(void*) + && offsetof(ZSTD_rust_initCDictState, dictContent) + == 7 * sizeof(void*) + && offsetof(ZSTD_rust_initCDictState, dictContentSize) + == 8 * sizeof(void*) + && offsetof(ZSTD_rust_initCDictState, dictContentType) + == 9 * sizeof(void*) + && offsetof(ZSTD_rust_initCDictState, entropyWorkspace) + == 10 * sizeof(void*) + && offsetof(ZSTD_rust_initCDictState, dictID) + == 11 * sizeof(void*) + && offsetof(ZSTD_rust_initCDictState, compressionLevel) + == 12 * sizeof(void*) + && offsetof(ZSTD_rust_initCDictState, contentSizeFlag) + == 13 * sizeof(void*) + && offsetof(ZSTD_rust_initCDictState, assignContent) + == 14 * sizeof(void*) + && offsetof(ZSTD_rust_initCDictState, reserveEntropy) + == 15 * sizeof(void*) + && offsetof(ZSTD_rust_initCDictState, resetBlockState) + == 16 * sizeof(void*) + && offsetof(ZSTD_rust_initCDictState, resetMatchState) + == 17 * sizeof(void*) + && offsetof(ZSTD_rust_initCDictState, insertDictionary) + == 18 * sizeof(void*) + && sizeof(ZSTD_rust_initCDictState) == 19 * sizeof(void*)) + ? 1 : -1]; + /* The sequence-compression loop receives only the state it actually reads or * updates. In particular, neither ZSTD_CCtx nor a C function pointer crosses * the Rust ABI. */ @@ -4345,6 +4419,57 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_loadDictionaryContent_callback); } +static size_t ZSTD_rust_initCDict_assignContent( + void* context, const void* dict, size_t dictSize, int dictLoadMethod) +{ + ZSTD_CDict* const cdict = (ZSTD_CDict*)context; + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { + cdict->dictContent = dict; + } else { + void* const internalBuffer = ZSTD_cwksp_reserve_object( + &cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*))); + RETURN_ERROR_IF(!internalBuffer, memory_allocation, "NULL pointer!"); + cdict->dictContent = internalBuffer; + ZSTD_memcpy(internalBuffer, dict, dictSize); + } + return 0; +} + +static void* ZSTD_rust_initCDict_reserveEntropy(void* context) +{ + ZSTD_CDict* const cdict = (ZSTD_CDict*)context; + return ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); +} + +static void ZSTD_rust_initCDict_resetBlockState(void* context) +{ + ZSTD_CDict* const cdict = (ZSTD_CDict*)context; + ZSTD_reset_compressedBlockState(&cdict->cBlockState); +} + +static size_t ZSTD_rust_initCDict_resetMatchState( + void* context, const ZSTD_compressionParameters* cParams, + int useRowMatchFinder) +{ + ZSTD_CDict* const cdict = (ZSTD_CDict*)context; + return ZSTD_reset_matchState( + &cdict->matchState, &cdict->workspace, cParams, + (ZSTD_ParamSwitch_e)useRowMatchFinder, + ZSTDcrp_makeClean, ZSTDirp_reset, ZSTD_resetTarget_CDict); +} + +static size_t ZSTD_rust_initCDict_insertDictionary( + void* context, const void* params, const void* dict, + size_t dictSize, int dictContentType) +{ + ZSTD_CDict* const cdict = (ZSTD_CDict*)context; + return ZSTD_compress_insertDictionary( + &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace, + (const ZSTD_CCtx_params*)params, dict, dictSize, + (ZSTD_dictContentType_e)dictContentType, ZSTD_dtlm_full, + ZSTD_tfp_forCDict, cdict->entropyWorkspace); +} + #define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) #define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL) @@ -4763,50 +4888,31 @@ static size_t ZSTD_initCDict_internal( ZSTD_dictContentType_e dictContentType, ZSTD_CCtx_params params) { + ZSTD_rust_initCDictState state; DEBUGLOG(3, "ZSTD_initCDict_internal (dictContentType:%u)", (unsigned)dictContentType); assert(!ZSTD_checkCParams(params.cParams)); - cdict->matchState.cParams = params.cParams; - cdict->matchState.dedicatedDictSearch = params.enableDedicatedDictSearch; - if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { - cdict->dictContent = dictBuffer; - } else { - void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*))); - RETURN_ERROR_IF(!internalBuffer, memory_allocation, "NULL pointer!"); - cdict->dictContent = internalBuffer; - ZSTD_memcpy(internalBuffer, dictBuffer, dictSize); - } - cdict->dictContentSize = dictSize; - cdict->dictContentType = dictContentType; - - cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); - - - /* Reset the state to no dictionary */ - ZSTD_reset_compressedBlockState(&cdict->cBlockState); - FORWARD_IF_ERROR(ZSTD_reset_matchState( - &cdict->matchState, - &cdict->workspace, - ¶ms.cParams, - params.useRowMatchFinder, - ZSTDcrp_makeClean, - ZSTDirp_reset, - ZSTD_resetTarget_CDict), ""); - /* (Maybe) load the dictionary - * Skips loading the dictionary if it is < 8 bytes. - */ - { params.compressionLevel = ZSTD_CLEVEL_DEFAULT; - params.fParams.contentSizeFlag = 1; - { size_t const dictID = ZSTD_compress_insertDictionary( - &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace, - ¶ms, cdict->dictContent, cdict->dictContentSize, - dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace); - FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); - assert(dictID <= (size_t)(U32)-1); - cdict->dictID = (U32)dictID; - } - } - - return 0; + state.callbackContext = cdict; + state.params = ¶ms; + state.cParams = ¶ms.cParams; + state.matchStateCParams = &cdict->matchState.cParams; + state.dedicatedDictSearch = &cdict->matchState.dedicatedDictSearch; + state.enableDedicatedDictSearch = ¶ms.enableDedicatedDictSearch; + state.useRowMatchFinder = ¶ms.useRowMatchFinder; + state.dictContent = &cdict->dictContent; + state.dictContentSize = &cdict->dictContentSize; + state.dictContentType = (int*)&cdict->dictContentType; + state.entropyWorkspace = &cdict->entropyWorkspace; + state.dictID = &cdict->dictID; + state.compressionLevel = ¶ms.compressionLevel; + state.contentSizeFlag = ¶ms.fParams.contentSizeFlag; + state.assignContent = ZSTD_rust_initCDict_assignContent; + state.reserveEntropy = ZSTD_rust_initCDict_reserveEntropy; + state.resetBlockState = ZSTD_rust_initCDict_resetBlockState; + state.resetMatchState = ZSTD_rust_initCDict_resetMatchState; + state.insertDictionary = ZSTD_rust_initCDict_insertDictionary; + return ZSTD_rust_initCDict( + &state, dictBuffer, dictSize, + (int)dictLoadMethod, (int)dictContentType); } static ZSTD_CDict* diff --git a/rust/src/zstd_compress_dictionary.rs b/rust/src/zstd_compress_dictionary.rs index d825e4f8a..1f6eb1286 100644 --- a/rust/src/zstd_compress_dictionary.rs +++ b/rust/src/zstd_compress_dictionary.rs @@ -232,6 +232,178 @@ pub unsafe extern "C" fn ZSTD_rust_compressBeginUsingCDict( } } +type InitCDictAssignContentFn = + unsafe extern "C" fn(*mut c_void, *const c_void, usize, c_int) -> usize; +type InitCDictReserveEntropyFn = unsafe extern "C" fn(*mut c_void) -> *mut c_void; +type InitCDictResetBlockStateFn = unsafe extern "C" fn(*mut c_void); +type InitCDictResetMatchStateFn = unsafe extern "C" fn(*mut c_void, *const c_void, c_int) -> usize; +type InitCDictInsertDictionaryFn = + unsafe extern "C" fn(*mut c_void, *const c_void, *const c_void, usize, c_int) -> usize; + +/// Projection for CDict content/state initialization. +/// +/// Rust owns the initialization order and scalar field policy. C callbacks +/// retain workspace allocation, match-state reset, and dictionary insertion +/// because those operations use private CDict layouts. +#[repr(C)] +pub struct ZSTD_rust_initCDictState { + callback_context: *mut c_void, + params: *const c_void, + c_params: *const ZSTD_compressionParameters, + match_state_c_params: *mut ZSTD_compressionParameters, + dedicated_dict_search: *mut c_int, + enable_dedicated_dict_search: *const c_int, + use_row_match_finder: *const c_int, + dict_content: *mut *const c_void, + dict_content_size: *mut usize, + dict_content_type: *mut c_int, + entropy_workspace: *mut *mut c_void, + dict_id: *mut c_uint, + compression_level: *mut c_int, + content_size_flag: *mut c_int, + assign_content: Option, + reserve_entropy: Option, + reset_block_state: Option, + reset_match_state: Option, + insert_dictionary: Option, +} + +const _: () = { + assert!(size_of::() == size_of::()); + assert!(size_of::() == size_of::()); + assert!(size_of::() == size_of::()); + assert!(size_of::() == size_of::()); + assert!(size_of::() == size_of::()); + assert!(offset_of!(ZSTD_rust_initCDictState, callback_context) == 0); + assert!(offset_of!(ZSTD_rust_initCDictState, params) == size_of::()); + assert!(offset_of!(ZSTD_rust_initCDictState, c_params) == 2 * size_of::()); + assert!(offset_of!(ZSTD_rust_initCDictState, match_state_c_params) == 3 * size_of::()); + assert!(offset_of!(ZSTD_rust_initCDictState, dedicated_dict_search) == 4 * size_of::()); + assert!( + offset_of!(ZSTD_rust_initCDictState, enable_dedicated_dict_search) + == 5 * size_of::() + ); + assert!(offset_of!(ZSTD_rust_initCDictState, use_row_match_finder) == 6 * size_of::()); + assert!(offset_of!(ZSTD_rust_initCDictState, dict_content) == 7 * size_of::()); + assert!(offset_of!(ZSTD_rust_initCDictState, dict_content_size) == size_of::<[usize; 8]>()); + assert!(offset_of!(ZSTD_rust_initCDictState, dict_content_type) == 9 * size_of::()); + assert!(offset_of!(ZSTD_rust_initCDictState, entropy_workspace) == 10 * size_of::()); + assert!(offset_of!(ZSTD_rust_initCDictState, dict_id) == 11 * size_of::()); + assert!(offset_of!(ZSTD_rust_initCDictState, compression_level) == 12 * size_of::()); + assert!(offset_of!(ZSTD_rust_initCDictState, content_size_flag) == 13 * size_of::()); + assert!(offset_of!(ZSTD_rust_initCDictState, assign_content) == 14 * size_of::()); + assert!(offset_of!(ZSTD_rust_initCDictState, reserve_entropy) == 15 * size_of::()); + assert!(offset_of!(ZSTD_rust_initCDictState, reset_block_state) == 16 * size_of::()); + assert!(offset_of!(ZSTD_rust_initCDictState, reset_match_state) == 17 * size_of::()); + assert!(offset_of!(ZSTD_rust_initCDictState, insert_dictionary) == 18 * size_of::()); + assert!(size_of::() == size_of::<[usize; 19]>()); +}; + +const CDICT_DEFAULT_CLEVEL: c_int = 3; + +/// Initialize a private CDict through narrow C callbacks. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_initCDict( + state: *const ZSTD_rust_initCDictState, + dict: *const c_void, + dict_size: usize, + dict_load_method: c_int, + dict_content_type: c_int, +) -> usize { + if state.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + let state = unsafe { &*state }; + let Some(assign_content) = state.assign_content else { + return ERROR(ZstdErrorCode::Generic); + }; + let Some(reserve_entropy) = state.reserve_entropy else { + return ERROR(ZstdErrorCode::Generic); + }; + let Some(reset_block_state) = state.reset_block_state else { + return ERROR(ZstdErrorCode::Generic); + }; + let Some(reset_match_state) = state.reset_match_state else { + return ERROR(ZstdErrorCode::Generic); + }; + let Some(insert_dictionary) = state.insert_dictionary else { + return ERROR(ZstdErrorCode::Generic); + }; + if state.callback_context.is_null() + || state.params.is_null() + || state.c_params.is_null() + || state.match_state_c_params.is_null() + || state.dedicated_dict_search.is_null() + || state.enable_dedicated_dict_search.is_null() + || state.use_row_match_finder.is_null() + || state.dict_content.is_null() + || state.dict_content_size.is_null() + || state.dict_content_type.is_null() + || state.entropy_workspace.is_null() + || state.dict_id.is_null() + || state.compression_level.is_null() + || state.content_size_flag.is_null() + { + return ERROR(ZstdErrorCode::Generic); + } + + let assign_result = + unsafe { assign_content(state.callback_context, dict, dict_size, dict_load_method) }; + if ERR_isError(assign_result) { + return assign_result; + } + + unsafe { + *state.match_state_c_params = *state.c_params; + *state.dedicated_dict_search = *state.enable_dedicated_dict_search; + *state.dict_content_size = dict_size; + *state.dict_content_type = dict_content_type; + } + + let entropy_workspace = unsafe { reserve_entropy(state.callback_context) }; + if entropy_workspace.is_null() { + return ERROR(ZstdErrorCode::MemoryAllocation); + } + unsafe { + *state.entropy_workspace = entropy_workspace; + } + + unsafe { reset_block_state(state.callback_context) }; + let reset_match_result = unsafe { + reset_match_state( + state.callback_context, + state.c_params.cast(), + *state.use_row_match_finder, + ) + }; + if ERR_isError(reset_match_result) { + return reset_match_result; + } + + unsafe { + *state.compression_level = CDICT_DEFAULT_CLEVEL; + *state.content_size_flag = 1; + } + let dict_content = unsafe { *state.dict_content }; + let dict_id = unsafe { + insert_dictionary( + state.callback_context, + state.params, + dict_content, + dict_size, + dict_content_type, + ) + }; + if ERR_isError(dict_id) { + return dict_id; + } + if dict_id > c_uint::MAX as usize { + return ERROR(ZstdErrorCode::Generic); + } + unsafe { *state.dict_id = dict_id as c_uint }; + 0 +} + /// Scalar projections for the public CDict query helpers. #[repr(C)] pub struct ZSTD_rust_cdictQueryState { @@ -746,6 +918,188 @@ mod tests { } } + struct InitCDictProbe { + events: Vec<&'static str>, + assigned_dict: *const c_void, + assigned_size: usize, + assigned_load_method: c_int, + reset_c_params: ZSTD_compressionParameters, + reset_use_row_match_finder: c_int, + inserted_params: *const c_void, + inserted_dict: *const c_void, + inserted_size: usize, + inserted_content_type: c_int, + assign_result: usize, + entropy_workspace: *mut c_void, + reset_match_result: usize, + insert_result: usize, + } + + impl Default for InitCDictProbe { + fn default() -> Self { + Self { + events: Vec::new(), + assigned_dict: ptr::null(), + assigned_size: 0, + assigned_load_method: 0, + reset_c_params: ZSTD_compressionParameters::default(), + reset_use_row_match_finder: 0, + inserted_params: ptr::null(), + inserted_dict: ptr::null(), + inserted_size: 0, + inserted_content_type: 0, + assign_result: 0, + entropy_workspace: ptr::null_mut(), + reset_match_result: 0, + insert_result: 0, + } + } + } + + unsafe fn init_cdict_probe(context: *mut c_void) -> &'static mut InitCDictProbe { + unsafe { &mut *context.cast::() } + } + + unsafe extern "C" fn init_cdict_assign_content( + context: *mut c_void, + dict: *const c_void, + dict_size: usize, + dict_load_method: c_int, + ) -> usize { + let probe = unsafe { init_cdict_probe(context) }; + probe.events.push("assign"); + probe.assigned_dict = dict; + probe.assigned_size = dict_size; + probe.assigned_load_method = dict_load_method; + probe.assign_result + } + + unsafe extern "C" fn init_cdict_reserve_entropy(context: *mut c_void) -> *mut c_void { + let probe = unsafe { init_cdict_probe(context) }; + probe.events.push("reserve"); + probe.entropy_workspace + } + + unsafe extern "C" fn init_cdict_reset_block_state(context: *mut c_void) { + unsafe { init_cdict_probe(context) } + .events + .push("reset-block"); + } + + unsafe extern "C" fn init_cdict_reset_match_state( + context: *mut c_void, + c_params: *const c_void, + use_row_match_finder: c_int, + ) -> usize { + let probe = unsafe { init_cdict_probe(context) }; + probe.events.push("reset-match"); + probe.reset_c_params = unsafe { *c_params.cast::() }; + probe.reset_use_row_match_finder = use_row_match_finder; + probe.reset_match_result + } + + unsafe extern "C" fn init_cdict_insert_dictionary( + context: *mut c_void, + params: *const c_void, + dict: *const c_void, + dict_size: usize, + dict_content_type: c_int, + ) -> usize { + let probe = unsafe { init_cdict_probe(context) }; + probe.events.push("insert"); + probe.inserted_params = params; + probe.inserted_dict = dict; + probe.inserted_size = dict_size; + probe.inserted_content_type = dict_content_type; + probe.insert_result + } + + #[test] + fn cdict_init_orders_callbacks_and_publishes_state() { + let dictionary = [1u8, 2, 3, 4]; + let mut probe = InitCDictProbe { + entropy_workspace: 0x1000usize as *mut c_void, + insert_result: 0x1234, + ..Default::default() + }; + let c_params = ZSTD_compressionParameters { + windowLog: 21, + chainLog: 18, + hashLog: 19, + searchLog: 4, + minMatch: 5, + targetLength: 16, + strategy: 3, + }; + let mut match_state_c_params = ZSTD_compressionParameters::default(); + let enable_dedicated_dict_search = 1; + let use_row_match_finder = 2; + let mut dedicated_dict_search = 0; + let mut dict_content = dictionary.as_ptr().cast::(); + let mut dict_content_size = 0; + let mut dict_content_type = 0; + let mut entropy_workspace = ptr::null_mut(); + let mut dict_id = 0; + let mut compression_level = 99; + let mut content_size_flag = 0; + let params = 0x2000usize as *const c_void; + let callback_context = (&mut probe as *mut InitCDictProbe).cast(); + let state = ZSTD_rust_initCDictState { + callback_context, + params, + c_params: &c_params, + match_state_c_params: &mut match_state_c_params, + dedicated_dict_search: &mut dedicated_dict_search, + enable_dedicated_dict_search: &enable_dedicated_dict_search, + use_row_match_finder: &use_row_match_finder, + dict_content: &mut dict_content, + dict_content_size: &mut dict_content_size, + dict_content_type: &mut dict_content_type, + entropy_workspace: &mut entropy_workspace, + dict_id: &mut dict_id, + compression_level: &mut compression_level, + content_size_flag: &mut content_size_flag, + assign_content: Some(init_cdict_assign_content), + reserve_entropy: Some(init_cdict_reserve_entropy), + reset_block_state: Some(init_cdict_reset_block_state), + reset_match_state: Some(init_cdict_reset_match_state), + insert_dictionary: Some(init_cdict_insert_dictionary), + }; + + let result = unsafe { + ZSTD_rust_initCDict( + &state, + dictionary.as_ptr().cast(), + dictionary.len(), + ZSTD_DLM_BY_REF, + ZSTD_DCT_RAW_CONTENT, + ) + }; + + assert_eq!(result, 0); + assert_eq!( + probe.events, + ["assign", "reserve", "reset-block", "reset-match", "insert"] + ); + assert_eq!(probe.assigned_dict, dictionary.as_ptr().cast()); + assert_eq!(probe.assigned_size, dictionary.len()); + assert_eq!(probe.assigned_load_method, ZSTD_DLM_BY_REF); + assert_eq!(probe.reset_c_params, c_params); + assert_eq!(probe.reset_use_row_match_finder, use_row_match_finder); + assert_eq!(probe.inserted_params, params); + assert_eq!(probe.inserted_dict, dictionary.as_ptr().cast()); + assert_eq!(probe.inserted_size, dictionary.len()); + assert_eq!(probe.inserted_content_type, ZSTD_DCT_RAW_CONTENT); + assert_eq!(match_state_c_params, c_params); + assert_eq!(dedicated_dict_search, enable_dedicated_dict_search); + assert_eq!(dict_content_size, dictionary.len()); + assert_eq!(dict_content_type, ZSTD_DCT_RAW_CONTENT); + assert_eq!(entropy_workspace, probe.entropy_workspace); + assert_eq!(dict_id, probe.insert_result as c_uint); + assert_eq!(compression_level, 3); + assert_eq!(content_size_flag, 1); + } + unsafe fn cctx_policy_probe(context: *mut c_void) -> &'static mut CctxPolicyProbe { unsafe { &mut *context.cast::() } }