diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index ec3e1a435..a8a0d9799 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2511,9 +2511,23 @@ typedef struct { U64 pledgedSrcSize; ZSTD_rust_resetCCtxByCopyingCDictReset_f reset; ZSTD_rust_resetCCtxByCopyingCDictMarkTables_f markTablesDirty; - ZSTD_rust_resetCCtxByCopyingCDictState_f copyTables; ZSTD_rust_resetCCtxByCopyingCDictState_f zeroHashTable3; ZSTD_rust_resetCCtxByCopyingCDictMarkTables_f markTablesClean; + U32** destinationHashTable; + const U32* sourceHashTable; + const U32* sourceHashLog; + U32** destinationChainTable; + const U32* sourceChainTable; + const U32* sourceChainLog; + const int* sourceStrategy; + const int* sourceUseRowMatchFinder; + const int* sourceIndicesTagged; + const int* destinationStrategy; + const int* destinationUseRowMatchFinder; + BYTE** destinationTagTable; + const BYTE* sourceTagTable; + U64* destinationHashSalt; + const U64* sourceHashSalt; void* destinationWindow; const void* sourceWindow; U32* destinationNextToUpdate; @@ -2541,7 +2555,7 @@ typedef char ZSTD_rust_reset_cctx_by_copying_cdict_state_layout[ && offsetof(ZSTD_rust_resetCCtxByCopyingCDictState, reset) == 3 * sizeof(void*) + sizeof(U64) && offsetof(ZSTD_rust_resetCCtxByCopyingCDictState, zbuff) - == 3 * sizeof(void*) + sizeof(U64) + 17 * sizeof(void*) + == 3 * sizeof(void*) + sizeof(U64) + 31 * sizeof(void*) && sizeof(ZSTD_rust_resetCCtxByCopyingCDictState) == ((offsetof(ZSTD_rust_resetCCtxByCopyingCDictState, zbuff) + sizeof(int) + sizeof(void*) - 1) / sizeof(void*)) @@ -3089,14 +3103,6 @@ static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) { return ZSTD_rust_params_rowMatchFinderSupported((int)strategy); } -/* Returns true if the strategy and useRowMatchFinder mode indicate that we will use the row based matchfinder - * for this compression. - */ -static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_ParamSwitch_e mode) { - assert(mode != ZSTD_ps_auto); - return ZSTD_rust_params_rowMatchFinderUsed((int)strategy, (int)mode); -} - /* Returns row matchfinder usage given an initial mode and cParams */ static ZSTD_ParamSwitch_e ZSTD_resolveRowMatchFinderMode(ZSTD_ParamSwitch_e mode, const ZSTD_compressionParameters* const cParams) { @@ -3111,15 +3117,6 @@ static ZSTD_ParamSwitch_e ZSTD_resolveBlockSplitterMode(ZSTD_ParamSwitch_e mode, (int)mode, *cParams); } -/* Returns 1 if the arguments indicate that we should allocate a chainTable, 0 otherwise */ -static int ZSTD_allocateChainTable(const ZSTD_strategy strategy, - const ZSTD_ParamSwitch_e useRowMatchFinder, - const U32 forDDSDict) { - assert(useRowMatchFinder != ZSTD_ps_auto); - return ZSTD_rust_params_allocateChainTable( - (int)strategy, (int)useRowMatchFinder, (int)forDDSDict); -} - /* Returns ZSTD_ps_enable if compression parameters are such that we should * enable long distance matching (wlog >= 27, strategy >= btopt). * Returns ZSTD_ps_disable otherwise. @@ -4563,13 +4560,6 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, return ZSTD_rust_resetCCtxByAttachingCDict(&state); } -static void ZSTD_copyCDictTableIntoCCtx(U32* dst, U32 const* src, size_t tableSize, - ZSTD_compressionParameters const* cParams) { - ZSTD_STATIC_ASSERT(ZSTD_SHORT_CACHE_TAG_BITS == 8); - ZSTD_rust_copyCDictTableIntoCCtx(dst, src, tableSize, - ZSTD_CDictIndicesAreTagged(cParams)); -} - static size_t ZSTD_rust_resetCCtx_byCopyingCDict_reset( void* context, const void* cdictOpaque, const void* paramsOpaque, U64 pledgedSrcSize, int zbuff) @@ -4610,40 +4600,6 @@ static void ZSTD_rust_resetCCtx_byCopyingCDict_mark_tables_dirty(void* context) ZSTD_cwksp_mark_tables_dirty(&cctx->workspace); } -static void ZSTD_rust_resetCCtx_byCopyingCDict_copy_tables( - void* context, const void* cdictOpaque) -{ - ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context; - const ZSTD_CDict* const cdict = (const ZSTD_CDict*)cdictOpaque; - const ZSTD_compressionParameters* const cdict_cParams = - &cdict->matchState.cParams; - size_t const chainSize = ZSTD_allocateChainTable( - cdict_cParams->strategy, cdict->useRowMatchFinder, - 0 /* DDS guaranteed disabled */) - ? ((size_t)1 << cdict_cParams->chainLog) - : 0; - size_t const hSize = (size_t)1 << cdict_cParams->hashLog; - - ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.hashTable, - cdict->matchState.hashTable, - hSize, cdict_cParams); - - /* Do not copy cdict's chainTable if cctx will not use a chainTable. */ - if (ZSTD_allocateChainTable(cctx->appliedParams.cParams.strategy, - cctx->appliedParams.useRowMatchFinder, - 0 /* forDDSDict */)) { - ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.chainTable, - cdict->matchState.chainTable, - chainSize, cdict_cParams); - } - if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy, - cdict->useRowMatchFinder)) { - ZSTD_memcpy(cctx->blockState.matchState.tagTable, - cdict->matchState.tagTable, hSize); - cctx->blockState.matchState.hashSalt = cdict->matchState.hashSalt; - } -} - static void ZSTD_rust_resetCCtx_byCopyingCDict_zero_hash_table3( void* context, const void* cdictOpaque) { @@ -4668,6 +4624,14 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) { + const ZSTD_compressionParameters* const cdict_cParams = + &cdict->matchState.cParams; + U32 const sourceHashLog = cdict_cParams->hashLog; + U32 const sourceChainLog = cdict_cParams->chainLog; + int const sourceStrategy = (int)cdict_cParams->strategy; + int const sourceUseRowMatchFinder = (int)cdict->useRowMatchFinder; + int const sourceIndicesTagged = + ZSTD_CDictIndicesAreTagged(cdict_cParams); ZSTD_rust_resetCCtxByCopyingCDictState state; state.callbackContext = cctx; state.cdict = cdict; @@ -4676,11 +4640,31 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, state.reset = ZSTD_rust_resetCCtx_byCopyingCDict_reset; state.markTablesDirty = ZSTD_rust_resetCCtx_byCopyingCDict_mark_tables_dirty; - state.copyTables = ZSTD_rust_resetCCtx_byCopyingCDict_copy_tables; state.zeroHashTable3 = ZSTD_rust_resetCCtx_byCopyingCDict_zero_hash_table3; state.markTablesClean = ZSTD_rust_resetCCtx_byCopyingCDict_mark_tables_clean; + state.destinationHashTable = + &cctx->blockState.matchState.hashTable; + state.sourceHashTable = cdict->matchState.hashTable; + state.sourceHashLog = &sourceHashLog; + state.destinationChainTable = + &cctx->blockState.matchState.chainTable; + state.sourceChainTable = cdict->matchState.chainTable; + state.sourceChainLog = &sourceChainLog; + state.sourceStrategy = &sourceStrategy; + state.sourceUseRowMatchFinder = &sourceUseRowMatchFinder; + state.sourceIndicesTagged = &sourceIndicesTagged; + state.destinationStrategy = + (const int*)&cctx->appliedParams.cParams.strategy; + state.destinationUseRowMatchFinder = + (const int*)&cctx->appliedParams.useRowMatchFinder; + state.destinationTagTable = + &cctx->blockState.matchState.tagTable; + state.sourceTagTable = cdict->matchState.tagTable; + state.destinationHashSalt = + &cctx->blockState.matchState.hashSalt; + state.sourceHashSalt = &cdict->matchState.hashSalt; state.destinationWindow = &cctx->blockState.matchState.window; state.sourceWindow = &cdict->matchState.window; state.destinationNextToUpdate = &cctx->blockState.matchState.nextToUpdate; diff --git a/rust/src/zstd_compress_dictionary.rs b/rust/src/zstd_compress_dictionary.rs index b3cd1897e..eb7ba786f 100644 --- a/rust/src/zstd_compress_dictionary.rs +++ b/rust/src/zstd_compress_dictionary.rs @@ -15,10 +15,12 @@ use crate::entropy_common::FSE_readNCount; use crate::errors::{ERR_isError, ZstdErrorCode, ERROR}; use crate::fse_compress::FSE_buildCTable_wksp; use crate::huf_compress::HUF_readCTable; +use crate::zstd_compress::ZSTD_rust_copyCDictTableIntoCCtx; use crate::zstd_compress_params::{ ZSTD_compressionParameters, ZSTD_frameParameters, ZSTD_parameters, - ZSTD_rust_params_defaultCLevel, ZSTD_rust_params_getCParams, - ZSTD_rust_params_getParamsInternal, ZSTD_rust_params_shouldAttachDict, + ZSTD_rust_params_allocateChainTable, ZSTD_rust_params_defaultCLevel, + ZSTD_rust_params_getCParams, ZSTD_rust_params_getParamsInternal, + ZSTD_rust_params_rowMatchFinderUsed, ZSTD_rust_params_shouldAttachDict, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_RUST_CPM_CREATE_CDICT, ZSTD_RUST_CPM_NO_ATTACH_DICT, }; use crate::zstd_compress_params_api::ZSTD_CCtx_params; @@ -1337,9 +1339,9 @@ const _: () = { /// Projection for copying a prepared CDict into a working CCtx. /// -/// Rust owns the reset/copy ordering and stops on reset allocation failure. -/// C retains the private CCtx/CDict layout, table-size policy, and table -/// pointer arithmetic behind callbacks. +/// Rust owns the reset/copy ordering, table-size policy, and table copies, and +/// stops on reset allocation failure. C retains the private CCtx/CDict layout +/// behind explicit table-field projections and callbacks. #[repr(C)] pub struct ZSTD_rust_resetCCtxByCopyingCDictState { callback_context: *mut c_void, @@ -1348,9 +1350,23 @@ pub struct ZSTD_rust_resetCCtxByCopyingCDictState { pledged_src_size: u64, reset: Option, mark_tables_dirty: Option, - copy_tables: Option, zero_hash_table3: Option, mark_tables_clean: Option, + destination_hash_table: *mut *mut c_uint, + source_hash_table: *const c_uint, + source_hash_log: *const c_uint, + destination_chain_table: *mut *mut c_uint, + source_chain_table: *const c_uint, + source_chain_log: *const c_uint, + source_strategy: *const c_int, + source_use_row_match_finder: *const c_int, + source_indices_tagged: *const c_int, + destination_strategy: *const c_int, + destination_use_row_match_finder: *const c_int, + destination_tag_table: *mut *mut u8, + source_tag_table: *const u8, + destination_hash_salt: *mut u64, + source_hash_salt: *const u64, destination_window: *mut c_void, source_window: *const c_void, destination_next_to_update: *mut c_uint, @@ -1383,7 +1399,7 @@ const _: () = { ); assert!( offset_of!(ZSTD_rust_resetCCtxByCopyingCDictState, zbuff) - == 3 * size_of::() + size_of::() + size_of::<[usize; 17]>() + == 3 * size_of::() + size_of::() + size_of::<[usize; 31]>() ); assert!( size_of::() @@ -1394,7 +1410,8 @@ const _: () = { }; /// Run the private CDict-copy operation through C-owned layout callbacks and -/// copy the compressed-block state directly in Rust. +/// explicit table-field projections, then copy the compressed-block state +/// directly in Rust. #[no_mangle] pub unsafe extern "C" fn ZSTD_rust_resetCCtxByCopyingCDict( state: *const ZSTD_rust_resetCCtxByCopyingCDictState, @@ -1403,25 +1420,29 @@ pub unsafe extern "C" fn ZSTD_rust_resetCCtxByCopyingCDict( return ERROR(ZstdErrorCode::Generic); } let state = unsafe { &*state }; - let ( - Some(reset), - Some(mark_tables_dirty), - Some(copy_tables), - Some(zero_hash_table3), - Some(mark_tables_clean), - ) = ( + let (Some(reset), Some(mark_tables_dirty), Some(zero_hash_table3), Some(mark_tables_clean)) = ( state.reset, state.mark_tables_dirty, - state.copy_tables, state.zero_hash_table3, state.mark_tables_clean, - ) - else { + ) else { return ERROR(ZstdErrorCode::Generic); }; if state.callback_context.is_null() || state.cdict.is_null() || state.params.is_null() + || state.destination_hash_table.is_null() + || state.source_hash_log.is_null() + || state.destination_chain_table.is_null() + || state.source_chain_log.is_null() + || state.source_strategy.is_null() + || state.source_use_row_match_finder.is_null() + || state.source_indices_tagged.is_null() + || state.destination_strategy.is_null() + || state.destination_use_row_match_finder.is_null() + || state.destination_tag_table.is_null() + || state.destination_hash_salt.is_null() + || state.source_hash_salt.is_null() || state.destination_window.is_null() || state.source_window.is_null() || state.destination_next_to_update.is_null() @@ -1438,6 +1459,30 @@ pub unsafe extern "C" fn ZSTD_rust_resetCCtxByCopyingCDict( return ERROR(ZstdErrorCode::Generic); } + let source_hash_log = unsafe { *state.source_hash_log }; + let source_chain_log = unsafe { *state.source_chain_log }; + let source_strategy = unsafe { *state.source_strategy }; + let source_use_row_match_finder = unsafe { *state.source_use_row_match_finder }; + let source_indices_tagged = unsafe { *state.source_indices_tagged }; + let Some(hash_table_size) = 1usize.checked_shl(source_hash_log) else { + return ERROR(ZstdErrorCode::Generic); + }; + let source_chain_table_size = if ZSTD_rust_params_allocateChainTable( + source_strategy, + source_use_row_match_finder, + 0, + ) != 0 + { + let Some(size) = 1usize.checked_shl(source_chain_log) else { + return ERROR(ZstdErrorCode::Generic); + }; + size + } else { + 0 + }; + let row_match_finder_used = + ZSTD_rust_params_rowMatchFinderUsed(source_strategy, source_use_row_match_finder) != 0; + unsafe { let reset_error = reset( state.callback_context, @@ -1449,8 +1494,52 @@ pub unsafe extern "C" fn ZSTD_rust_resetCCtxByCopyingCDict( if ERR_isError(reset_error) { return reset_error; } + let destination_strategy = *state.destination_strategy; + let destination_use_row_match_finder = *state.destination_use_row_match_finder; + let destination_chain_table_size = if ZSTD_rust_params_allocateChainTable( + destination_strategy, + destination_use_row_match_finder, + 0, + ) != 0 + { + source_chain_table_size + } else { + 0 + }; + let destination_hash_table = *state.destination_hash_table; + let destination_chain_table = *state.destination_chain_table; + let destination_tag_table = *state.destination_tag_table; + if (destination_hash_table.is_null() || state.source_hash_table.is_null()) + || (destination_chain_table_size != 0 + && (destination_chain_table.is_null() || state.source_chain_table.is_null())) + || (row_match_finder_used + && (destination_tag_table.is_null() || state.source_tag_table.is_null())) + { + return ERROR(ZstdErrorCode::Generic); + } mark_tables_dirty(state.callback_context); - copy_tables(state.callback_context, state.cdict); + ZSTD_rust_copyCDictTableIntoCCtx( + destination_hash_table, + state.source_hash_table, + hash_table_size, + source_indices_tagged, + ); + if destination_chain_table_size != 0 { + ZSTD_rust_copyCDictTableIntoCCtx( + destination_chain_table, + state.source_chain_table, + destination_chain_table_size, + source_indices_tagged, + ); + } + if row_match_finder_used { + ptr::copy_nonoverlapping( + state.source_tag_table, + destination_tag_table, + hash_table_size, + ); + *state.destination_hash_salt = *state.source_hash_salt; + } zero_hash_table3(state.callback_context, state.cdict); mark_tables_clean(state.callback_context); ptr::copy( @@ -2768,6 +2857,24 @@ mod tests { params: *const c_void, pledged_src_size: u64, zbuff: c_int, + source_hash_table: [c_uint; 4], + destination_hash_table: [c_uint; 4], + destination_hash_table_slot: *mut c_uint, + source_chain_table: [c_uint; 4], + destination_chain_table: [c_uint; 4], + destination_chain_table_slot: *mut c_uint, + source_tag_table: [u8; 4], + destination_tag_table: [u8; 4], + destination_tag_table_slot: *mut u8, + source_hash_salt: u64, + destination_hash_salt: u64, + source_hash_log: c_uint, + source_chain_log: c_uint, + source_strategy: c_int, + source_use_row_match_finder: c_int, + source_indices_tagged: c_int, + destination_strategy: c_int, + destination_use_row_match_finder: c_int, } unsafe fn reset_cctx_by_copying_cdict_probe( @@ -2798,15 +2905,6 @@ mod tests { .push("dirty"); } - unsafe extern "C" fn reset_cctx_by_copying_cdict_copy_tables( - context: *mut c_void, - _cdict: *const c_void, - ) { - unsafe { reset_cctx_by_copying_cdict_probe(context) } - .events - .push("tables"); - } - unsafe extern "C" fn reset_cctx_by_copying_cdict_zero_hash_table3( context: *mut c_void, _cdict: *const c_void, @@ -2839,6 +2937,16 @@ mod tests { destination_block_state: *mut *mut ZSTD_compressedBlockState_t, source_block_state: *const ZSTD_compressedBlockState_t, ) -> ZSTD_rust_resetCCtxByCopyingCDictState { + probe.destination_hash_table_slot = probe.destination_hash_table.as_mut_ptr(); + probe.destination_chain_table_slot = probe.destination_chain_table.as_mut_ptr(); + probe.destination_tag_table_slot = probe.destination_tag_table.as_mut_ptr(); + probe.source_hash_log = 2; + probe.source_chain_log = 2; + probe.source_strategy = 2; + probe.source_use_row_match_finder = 2; + probe.source_indices_tagged = 1; + probe.destination_strategy = 2; + probe.destination_use_row_match_finder = 2; ZSTD_rust_resetCCtxByCopyingCDictState { callback_context: (probe as *mut ResetCCtxByCopyingCDictProbe).cast(), cdict, @@ -2846,9 +2954,23 @@ mod tests { pledged_src_size: 123, reset: Some(reset_cctx_by_copying_cdict_reset), mark_tables_dirty: Some(reset_cctx_by_copying_cdict_mark_dirty), - copy_tables: Some(reset_cctx_by_copying_cdict_copy_tables), zero_hash_table3: Some(reset_cctx_by_copying_cdict_zero_hash_table3), mark_tables_clean: Some(reset_cctx_by_copying_cdict_mark_clean), + destination_hash_table: &mut probe.destination_hash_table_slot, + source_hash_table: probe.source_hash_table.as_ptr(), + source_hash_log: &probe.source_hash_log, + destination_chain_table: &mut probe.destination_chain_table_slot, + source_chain_table: probe.source_chain_table.as_ptr(), + source_chain_log: &probe.source_chain_log, + source_strategy: &probe.source_strategy, + source_use_row_match_finder: &probe.source_use_row_match_finder, + source_indices_tagged: &probe.source_indices_tagged, + destination_strategy: &probe.destination_strategy, + destination_use_row_match_finder: &probe.destination_use_row_match_finder, + destination_tag_table: &mut probe.destination_tag_table_slot, + source_tag_table: probe.source_tag_table.as_ptr(), + destination_hash_salt: &mut probe.destination_hash_salt, + source_hash_salt: &probe.source_hash_salt, destination_window, source_window, destination_next_to_update, @@ -2867,7 +2989,11 @@ mod tests { #[test] fn reset_cctx_by_copying_cdict_runs_callbacks_in_original_order() { - let mut probe = ResetCCtxByCopyingCDictProbe::default(); + let mut probe = ResetCCtxByCopyingCDictProbe { + source_hash_table: [0x0000_0101, 0x0000_0202, 0x0000_0303, 0x0000_0404], + source_chain_table: [0x0000_1101, 0x0000_2202, 0x0000_3303, 0x0000_4404], + ..Default::default() + }; let cdict = 0x4000usize as *const c_void; let params = 0x3000usize as *const c_void; let mut source_block_state = @@ -2924,10 +3050,11 @@ mod tests { let result = unsafe { ZSTD_rust_resetCCtxByCopyingCDict(&state) }; assert_eq!(result, 0); - assert_eq!( - probe.events, - ["reset", "dirty", "tables", "zero-h3", "clean"] - ); + assert_eq!(probe.events, ["reset", "dirty", "zero-h3", "clean"]); + assert_eq!(probe.destination_hash_table, [1, 2, 3, 4]); + assert_eq!(probe.destination_chain_table, [0x11, 0x22, 0x33, 0x44]); + assert_eq!(probe.destination_tag_table, [0; 4]); + assert_eq!(probe.destination_hash_salt, 0); assert_eq!(probe.cdict, cdict); assert_eq!(probe.params, params); assert_eq!(probe.pledged_src_size, 123); @@ -2946,6 +3073,27 @@ mod tests { destination_block_state.entropy.fse.offcode_repeatMode, source_block_state.entropy.fse.offcode_repeatMode ); + + probe.events.clear(); + probe.source_hash_table = [11, 22, 33, 44]; + probe.source_tag_table = [5, 6, 7, 8]; + probe.destination_hash_table = [0; 4]; + probe.destination_tag_table = [0; 4]; + probe.destination_hash_salt = 0; + probe.source_hash_salt = 0x0123_4567_89ab_cdef; + probe.source_strategy = 3; + probe.source_use_row_match_finder = 1; + probe.source_indices_tagged = 0; + probe.destination_strategy = 3; + probe.destination_use_row_match_finder = 1; + + let result = unsafe { ZSTD_rust_resetCCtxByCopyingCDict(&state) }; + + assert_eq!(result, 0); + assert_eq!(probe.events, ["reset", "dirty", "zero-h3", "clean"]); + assert_eq!(probe.destination_hash_table, probe.source_hash_table); + assert_eq!(probe.destination_tag_table, probe.source_tag_table); + assert_eq!(probe.destination_hash_salt, probe.source_hash_salt); } #[test]