diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index fb1076e80..ec3e1a435 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -462,8 +462,6 @@ typedef size_t (*ZSTD_rust_copyCCtxReset_f)( const ZSTD_frameParameters* fParams, U64 pledgedSrcSize, int zbuff); typedef void (*ZSTD_rust_copyCCtxMarkTables_f)(void* context); -typedef void (*ZSTD_rust_copyCCtxCopyTables_f)( - void* context, const void* srcCCtx); typedef struct { void* callbackContext; const void* srcCCtx; @@ -474,8 +472,18 @@ typedef struct { const void* sourceCustomMem; ZSTD_rust_copyCCtxReset_f reset; ZSTD_rust_copyCCtxMarkTables_f markTablesDirty; - ZSTD_rust_copyCCtxCopyTables_f copyTables; ZSTD_rust_copyCCtxMarkTables_f markTablesClean; + U32** destinationHashTable; + const U32* sourceHashTable; + const U32* sourceHashLog; + U32** destinationChainTable; + const U32* sourceChainTable; + const U32* sourceChainLog; + U32** destinationHashTable3; + const U32* sourceHashTable3; + const U32* sourceHashLog3; + const int* sourceStrategy; + const int* sourceUseRowMatchFinder; void* destinationWindow; const void* sourceWindow; U32* destinationNextToUpdate; @@ -533,7 +541,7 @@ typedef char ZSTD_rust_copy_cctx_internal_state_layout[ == 3 * sizeof(void*) + sizeof(U64) && offsetof(ZSTD_rust_copyCCtxInternalState, zbuff) == 3 * sizeof(void*) + sizeof(U64) - + 19 * sizeof(void*) + + 29 * sizeof(void*) && sizeof(ZSTD_rust_copyCCtxInternalState) == ((offsetof(ZSTD_rust_copyCCtxInternalState, zbuff) + sizeof(int) + sizeof(void*) - 1) / sizeof(void*)) @@ -4782,31 +4790,6 @@ static void ZSTD_rust_copyCCtx_mark_tables_dirty(void* context) ZSTD_cwksp_mark_tables_dirty(&dst->workspace); } -static void ZSTD_rust_copyCCtx_copy_tables( - void* context, const void* srcCCtx) -{ - ZSTD_CCtx* const dst = (ZSTD_CCtx*)context; - const ZSTD_CCtx* const src = (const ZSTD_CCtx*)srcCCtx; - size_t const chainSize = ZSTD_allocateChainTable( - src->appliedParams.cParams.strategy, - src->appliedParams.useRowMatchFinder, - 0 /* forDDSDict */) - ? ((size_t)1 << src->appliedParams.cParams.chainLog) - : 0; - size_t const hSize = (size_t)1 << src->appliedParams.cParams.hashLog; - U32 const h3log = src->blockState.matchState.hashLog3; - size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; - ZSTD_memcpy(dst->blockState.matchState.hashTable, - src->blockState.matchState.hashTable, - hSize * sizeof(U32)); - ZSTD_memcpy(dst->blockState.matchState.chainTable, - src->blockState.matchState.chainTable, - chainSize * sizeof(U32)); - ZSTD_memcpy(dst->blockState.matchState.hashTable3, - src->blockState.matchState.hashTable3, - h3Size * sizeof(U32)); -} - static void ZSTD_rust_copyCCtx_mark_tables_clean(void* context) { ZSTD_CCtx* const dst = (ZSTD_CCtx*)context; @@ -4819,7 +4802,15 @@ static size_t ZSTD_rust_copyCCtx_internal_callback( U64 pledgedSrcSize, int zbuff) { ZSTD_rust_copyCCtxInternalState state; + ZSTD_CCtx* const dst = (ZSTD_CCtx*)context; + const ZSTD_CCtx* const src = (const ZSTD_CCtx*)srcCCtx; int const sourceStage = (int)((const ZSTD_CCtx*)srcCCtx)->stage; + U32 const sourceHashLog = src->appliedParams.cParams.hashLog; + U32 const sourceChainLog = src->appliedParams.cParams.chainLog; + U32 const sourceHashLog3 = src->blockState.matchState.hashLog3; + int const sourceStrategy = (int)src->appliedParams.cParams.strategy; + int const sourceUseRowMatchFinder = + (int)src->appliedParams.useRowMatchFinder; state.callbackContext = context; state.srcCCtx = srcCCtx; state.fParams = fParams; @@ -4831,8 +4822,18 @@ static size_t ZSTD_rust_copyCCtx_internal_callback( &((const ZSTD_CCtx*)srcCCtx)->customMem; state.reset = ZSTD_rust_copyCCtx_reset; state.markTablesDirty = ZSTD_rust_copyCCtx_mark_tables_dirty; - state.copyTables = ZSTD_rust_copyCCtx_copy_tables; state.markTablesClean = ZSTD_rust_copyCCtx_mark_tables_clean; + state.destinationHashTable = &dst->blockState.matchState.hashTable; + state.sourceHashTable = src->blockState.matchState.hashTable; + state.sourceHashLog = &sourceHashLog; + state.destinationChainTable = &dst->blockState.matchState.chainTable; + state.sourceChainTable = src->blockState.matchState.chainTable; + state.sourceChainLog = &sourceChainLog; + state.destinationHashTable3 = &dst->blockState.matchState.hashTable3; + state.sourceHashTable3 = src->blockState.matchState.hashTable3; + state.sourceHashLog3 = &sourceHashLog3; + state.sourceStrategy = &sourceStrategy; + state.sourceUseRowMatchFinder = &sourceUseRowMatchFinder; state.destinationWindow = &((ZSTD_CCtx*)context)->blockState.matchState.window; state.sourceWindow = diff --git a/rust/src/zstd_compress.rs b/rust/src/zstd_compress.rs index be18a278c..ce9a7208f 100644 --- a/rust/src/zstd_compress.rs +++ b/rust/src/zstd_compress.rs @@ -1680,7 +1680,6 @@ type CopyCCtxResetFn = unsafe extern "C" fn( c_int, ) -> usize; type CopyCCtxMarkTablesFn = unsafe extern "C" fn(*mut c_void); -type CopyCCtxCopyTablesFn = unsafe extern "C" fn(*mut c_void, *const c_void); #[repr(C)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -1733,8 +1732,18 @@ pub struct ZSTD_rust_copyCCtxInternalState { source_custom_mem: *const c_void, reset: Option, mark_tables_dirty: Option, - copy_tables: Option, mark_tables_clean: Option, + destination_hash_table: *mut *mut c_uint, + source_hash_table: *const c_uint, + source_hash_log: *const c_uint, + destination_chain_table: *mut *mut c_uint, + source_chain_table: *const c_uint, + source_chain_log: *const c_uint, + destination_hash_table3: *mut *mut c_uint, + source_hash_table3: *const c_uint, + source_hash_log3: *const c_uint, + source_strategy: *const c_int, + source_use_row_match_finder: *const c_int, destination_window: *mut c_void, source_window: *const c_void, destination_next_to_update: *mut c_uint, @@ -1753,7 +1762,6 @@ pub struct ZSTD_rust_copyCCtxInternalState { const _: () = { assert!(size_of::() == size_of::()); assert!(size_of::() == size_of::()); - assert!(size_of::() == size_of::()); assert!(offset_of!(ZSTD_rust_copyCCtxInternalState, callback_context) == 0); assert!(offset_of!(ZSTD_rust_copyCCtxInternalState, src_cctx) == size_of::()); assert!(offset_of!(ZSTD_rust_copyCCtxInternalState, f_params) == 2 * size_of::()); @@ -1766,7 +1774,7 @@ const _: () = { ); assert!( offset_of!(ZSTD_rust_copyCCtxInternalState, zbuff) - == 3 * size_of::() + size_of::() + 19 * size_of::() + == 3 * size_of::() + size_of::() + 29 * size_of::() ); assert!( size_of::() @@ -1786,10 +1794,9 @@ pub unsafe extern "C" fn ZSTD_rust_copyCCtxInternal( return ERROR(ZstdErrorCode::Generic); } let state = unsafe { &*state }; - let (Some(reset), Some(mark_tables_dirty), Some(copy_tables), Some(mark_tables_clean)) = ( + let (Some(reset), Some(mark_tables_dirty), Some(mark_tables_clean)) = ( state.reset, state.mark_tables_dirty, - state.copy_tables, state.mark_tables_clean, ) else { return ERROR(ZstdErrorCode::Generic); @@ -1800,6 +1807,14 @@ pub unsafe extern "C" fn ZSTD_rust_copyCCtxInternal( || state.source_stage.is_null() || state.destination_custom_mem.is_null() || state.source_custom_mem.is_null() + || state.destination_hash_table.is_null() + || state.destination_chain_table.is_null() + || state.destination_hash_table3.is_null() + || state.source_hash_log.is_null() + || state.source_chain_log.is_null() + || state.source_hash_log3.is_null() + || state.source_strategy.is_null() + || state.source_use_row_match_finder.is_null() || state.destination_window.is_null() || state.source_window.is_null() || state.destination_next_to_update.is_null() @@ -1820,6 +1835,36 @@ pub unsafe extern "C" fn ZSTD_rust_copyCCtxInternal( return ERROR(ZstdErrorCode::StageWrong); } + let source_hash_log = unsafe { *state.source_hash_log }; + let source_chain_log = unsafe { *state.source_chain_log }; + let source_hash_log3 = unsafe { *state.source_hash_log3 }; + let source_strategy = unsafe { *state.source_strategy }; + let source_use_row_match_finder = unsafe { *state.source_use_row_match_finder }; + let Some(hash_table_size) = 1usize.checked_shl(source_hash_log) else { + return ERROR(ZstdErrorCode::Generic); + }; + let chain_table_size = if ZSTD_rust_params_allocateChainTable( + source_strategy, + source_use_row_match_finder, + 0, + ) != 0 + { + let Some(size) = 1usize.checked_shl(source_chain_log) else { + return ERROR(ZstdErrorCode::Generic); + }; + size + } else { + 0 + }; + let hash_table3_size = if source_hash_log3 != 0 { + let Some(size) = 1usize.checked_shl(source_hash_log3) else { + return ERROR(ZstdErrorCode::Generic); + }; + size + } else { + 0 + }; + unsafe { ptr::copy( state.source_custom_mem.cast::(), @@ -1836,8 +1881,40 @@ pub unsafe extern "C" fn ZSTD_rust_copyCCtxInternal( if ERR_isError(reset_error) { return reset_error; } + let destination_hash_table = *state.destination_hash_table; + let destination_chain_table = *state.destination_chain_table; + let destination_hash_table3 = *state.destination_hash_table3; + if (hash_table_size != 0 + && (destination_hash_table.is_null() || state.source_hash_table.is_null())) + || (chain_table_size != 0 + && (destination_chain_table.is_null() || state.source_chain_table.is_null())) + || (hash_table3_size != 0 + && (destination_hash_table3.is_null() || state.source_hash_table3.is_null())) + { + return ERROR(ZstdErrorCode::Generic); + } mark_tables_dirty(state.callback_context); - copy_tables(state.callback_context, state.src_cctx); + if hash_table_size != 0 { + ptr::copy_nonoverlapping( + state.source_hash_table, + destination_hash_table, + hash_table_size, + ); + } + if chain_table_size != 0 { + ptr::copy_nonoverlapping( + state.source_chain_table, + destination_chain_table, + chain_table_size, + ); + } + if hash_table3_size != 0 { + ptr::copy_nonoverlapping( + state.source_hash_table3, + destination_hash_table3, + hash_table3_size, + ); + } mark_tables_clean(state.callback_context); ptr::copy( state.source_window.cast::(), @@ -13923,14 +14000,6 @@ mod tests { .push("dirty"); } - unsafe extern "C" fn copy_cctx_internal_test_copy_tables( - context: *mut c_void, - _src_cctx: *const c_void, - ) { - let context = unsafe { copy_cctx_internal_test_context(context) }; - context.events.push("tables"); - } - unsafe extern "C" fn copy_cctx_internal_test_mark_tables_clean(context: *mut c_void) { unsafe { copy_cctx_internal_test_context(context) } .events @@ -13944,6 +14013,17 @@ mod tests { source_stage: *const c_int, destination_custom_mem: *mut c_void, source_custom_mem: *const c_void, + destination_hash_table: *mut *mut c_uint, + source_hash_table: *const c_uint, + source_hash_log: *const c_uint, + destination_chain_table: *mut *mut c_uint, + source_chain_table: *const c_uint, + source_chain_log: *const c_uint, + destination_hash_table3: *mut *mut c_uint, + source_hash_table3: *const c_uint, + source_hash_log3: *const c_uint, + source_strategy: *const c_int, + source_use_row_match_finder: *const c_int, destination_window: *mut c_void, source_window: *const c_void, destination_next_to_update: *mut c_uint, @@ -13967,8 +14047,18 @@ mod tests { source_custom_mem, reset: Some(copy_cctx_internal_test_reset), mark_tables_dirty: Some(copy_cctx_internal_test_mark_tables_dirty), - copy_tables: Some(copy_cctx_internal_test_copy_tables), mark_tables_clean: Some(copy_cctx_internal_test_mark_tables_clean), + destination_hash_table, + source_hash_table, + source_hash_log, + destination_chain_table, + source_chain_table, + source_chain_log, + destination_hash_table3, + source_hash_table3, + source_hash_log3, + source_strategy, + source_use_row_match_finder, destination_window, source_window, destination_next_to_update, @@ -13996,6 +14086,20 @@ mod tests { let source_stage = ZSTD_COMPRESSION_STAGE_INIT; let source_custom_mem = [0x11usize, 0x22, 0x33]; let mut destination_custom_mem = [0usize; 3]; + let source_hash_table = [11u32, 12, 13, 14]; + let mut destination_hash_table = [0u32; 4]; + let mut destination_hash_table_ptr = destination_hash_table.as_mut_ptr(); + let source_hash_log = 2; + let source_chain_table = [21u32, 22]; + let mut destination_chain_table = [0u32; 2]; + let mut destination_chain_table_ptr = destination_chain_table.as_mut_ptr(); + let source_chain_log = 1; + let source_hash_table3 = [31u32, 32]; + let mut destination_hash_table3 = [0u32; 2]; + let mut destination_hash_table3_ptr = destination_hash_table3.as_mut_ptr(); + let source_hash_log3 = 1; + let source_strategy = ZSTD_DFAST; + let source_use_row_match_finder = ZSTD_RUST_PS_DISABLE; let mut source_block_state = unsafe { MaybeUninit::::zeroed().assume_init() }; source_block_state.rep = [15, 16, 17]; @@ -14036,6 +14140,17 @@ mod tests { &source_stage, (&mut destination_custom_mem as *mut [usize; 3]).cast(), (&source_custom_mem as *const [usize; 3]).cast(), + &mut destination_hash_table_ptr, + source_hash_table.as_ptr(), + &source_hash_log, + &mut destination_chain_table_ptr, + source_chain_table.as_ptr(), + &source_chain_log, + &mut destination_hash_table3_ptr, + source_hash_table3.as_ptr(), + &source_hash_log3, + &source_strategy, + &source_use_row_match_finder, (&mut destination_window as *mut ZSTD_rust_copyWindowState).cast(), (&source_window as *const ZSTD_rust_copyWindowState).cast(), &mut destination_next_to_update, @@ -14053,11 +14168,14 @@ mod tests { let result = unsafe { ZSTD_rust_copyCCtxInternal(&state) }; assert_eq!(result, 0); - assert_eq!(context.events, ["reset", "dirty", "tables", "clean"]); + assert_eq!(context.events, ["reset", "dirty", "clean"]); assert_eq!(context.frame_params, f_params); assert_eq!(context.pledged_src_size, 123); assert_eq!(context.zbuff, 7); assert_eq!(destination_custom_mem, source_custom_mem); + assert_eq!(destination_hash_table, source_hash_table); + assert_eq!(destination_chain_table, source_chain_table); + assert_eq!(destination_hash_table3, source_hash_table3); assert_eq!(destination_window, source_window); assert_eq!(destination_next_to_update, source_next_to_update); assert_eq!(destination_loaded_dict_end, source_loaded_dict_end); @@ -14100,6 +14218,17 @@ mod tests { (&source_custom_mem as *const [usize; 3]).cast(), ptr::dangling_mut(), ptr::dangling(), + ptr::dangling(), + ptr::dangling_mut(), + ptr::dangling(), + ptr::dangling(), + ptr::dangling_mut(), + ptr::dangling(), + ptr::dangling(), + ptr::dangling(), + ptr::dangling(), + ptr::dangling_mut(), + ptr::dangling(), ptr::dangling_mut(), ptr::dangling(), ptr::dangling_mut(), @@ -14128,6 +14257,11 @@ mod tests { let source_stage = ZSTD_COMPRESSION_STAGE_INIT; let source_custom_mem = [0usize; 3]; let mut destination_custom_mem = [0usize; 3]; + let source_hash_log = 0; + let source_chain_log = 0; + let source_hash_log3 = 0; + let source_strategy = ZSTD_DFAST; + let source_use_row_match_finder = ZSTD_RUST_PS_DISABLE; let mut destination_block_state = unsafe { MaybeUninit::::zeroed().assume_init() }; let mut destination_block_state_slot = @@ -14147,6 +14281,17 @@ mod tests { (&source_custom_mem as *const [usize; 3]).cast(), ptr::dangling_mut(), ptr::dangling(), + &source_hash_log, + ptr::dangling_mut(), + ptr::dangling(), + &source_chain_log, + ptr::dangling_mut(), + ptr::dangling(), + &source_hash_log3, + &source_strategy, + &source_use_row_match_finder, + ptr::dangling_mut(), + ptr::dangling(), ptr::dangling_mut(), ptr::dangling(), ptr::dangling_mut(),