diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index bafe4c47e..7a595c1ee 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -614,6 +614,65 @@ U64 ZSTD_rust_advanceHashSalt(U64 hashSalt, U64 hashSaltEntropy); int ZSTD_rust_indexTooCloseToMax(size_t nextSrcBaseOffset); int ZSTD_rust_dictTooBig(size_t loadedDictSize); +/* Match-state reset orchestration lives in Rust. The callbacks keep the + * workspace implementation, window representation, and private pointer + * fields in C. */ +typedef void (*ZSTD_rust_resetMatchStateCallback_f)(void* context); +typedef void (*ZSTD_rust_resetMatchStateSetPointer_f)( + void* context, int pointerKind, void* pointer); +typedef void* (*ZSTD_rust_resetMatchStateReserve_f)( + void* context, int reserveKind, size_t size); +typedef int (*ZSTD_rust_resetMatchStateReserveFailed_f)(void* context); +typedef void (*ZSTD_rust_resetMatchStateZero_f)( + void* context, void* pointer, size_t size); +typedef struct { + void* callbackContext; + ZSTD_compressionParameters cParams; + int dedicatedDictSearch; + int useRowMatchFinder; + int compResetPolicy; + int indexResetPolicy; + int resetTarget; + unsigned hashLog3Max; + size_t litFreqSize; + size_t litLengthFreqSize; + size_t matchLengthFreqSize; + size_t offCodeFreqSize; + size_t matchSize; + size_t optimalSize; + void* hashLog3; + void* rowHashLog; + void* hashSalt; + void* lazySkipping; + void* cParamsOut; + ZSTD_rust_resetMatchStateSetPointer_f setPointer; + ZSTD_rust_resetMatchStateCallback_f windowInit; + ZSTD_rust_resetMatchStateCallback_f markTablesDirty; + ZSTD_rust_resetMatchStateCallback_f invalidateMatchState; + ZSTD_rust_resetMatchStateCallback_f clearTables; + ZSTD_rust_resetMatchStateCallback_f cleanTables; + ZSTD_rust_resetMatchStateCallback_f advanceHashSalt; + ZSTD_rust_resetMatchStateReserve_f reserve; + ZSTD_rust_resetMatchStateReserveFailed_f reserveFailed; + ZSTD_rust_resetMatchStateZero_f zero; +} ZSTD_rust_resetMatchStateState; +size_t ZSTD_rust_resetMatchState(const ZSTD_rust_resetMatchStateState* state); +typedef char ZSTD_rust_reset_match_state_layout[ + (offsetof(ZSTD_rust_resetMatchStateState, callbackContext) == 0 + && offsetof(ZSTD_rust_resetMatchStateState, cParams) == sizeof(void*) + && offsetof(ZSTD_rust_resetMatchStateState, litFreqSize) + > offsetof(ZSTD_rust_resetMatchStateState, hashLog3Max) + && offsetof(ZSTD_rust_resetMatchStateState, hashLog3) + == offsetof(ZSTD_rust_resetMatchStateState, optimalSize) + + sizeof(size_t) + && offsetof(ZSTD_rust_resetMatchStateState, setPointer) + == offsetof(ZSTD_rust_resetMatchStateState, cParamsOut) + + sizeof(void*) + && sizeof(ZSTD_rust_resetMatchStateState) + == offsetof(ZSTD_rust_resetMatchStateState, zero) + + sizeof(void*)) + ? 1 : -1]; + /* The frame-chunk loop is Rust-owned. Its callbacks keep the private * ZSTD_CCtx and match-state layout in C: Rust only drives the block loop and * passes this context back to these C-owned state-preparation/dispatch seams. */ @@ -3308,6 +3367,147 @@ static void ZSTD_advanceHashSalt(ZSTD_MatchState_t* ms) { ms->hashSalt = ZSTD_rust_advanceHashSalt(ms->hashSalt, (U64)ms->hashSaltEntropy); } +typedef struct { + ZSTD_MatchState_t* ms; + ZSTD_cwksp* ws; +} ZSTD_rust_resetMatchStateContext; + +enum { + ZSTD_RUST_RESET_MATCH_RESERVE_TABLE = 0, + ZSTD_RUST_RESET_MATCH_RESERVE_ALIGNED_INIT_ONCE = 1, + ZSTD_RUST_RESET_MATCH_RESERVE_ALIGNED64 = 2 +}; + +enum { + ZSTD_RUST_RESET_MATCH_POINTER_HASH_TABLE = 0, + ZSTD_RUST_RESET_MATCH_POINTER_CHAIN_TABLE = 1, + ZSTD_RUST_RESET_MATCH_POINTER_HASH_TABLE3 = 2, + ZSTD_RUST_RESET_MATCH_POINTER_TAG_TABLE = 3, + ZSTD_RUST_RESET_MATCH_POINTER_LIT_FREQ = 4, + ZSTD_RUST_RESET_MATCH_POINTER_LIT_LENGTH_FREQ = 5, + ZSTD_RUST_RESET_MATCH_POINTER_MATCH_LENGTH_FREQ = 6, + ZSTD_RUST_RESET_MATCH_POINTER_OFF_CODE_FREQ = 7, + ZSTD_RUST_RESET_MATCH_POINTER_MATCH_TABLE = 8, + ZSTD_RUST_RESET_MATCH_POINTER_PRICE_TABLE = 9 +}; + +static void ZSTD_rust_resetMatchState_windowInit(void* opaque) +{ + ZSTD_rust_resetMatchStateContext* const context = + (ZSTD_rust_resetMatchStateContext*)opaque; + ZSTD_window_init(&context->ms->window); +} + +static void ZSTD_rust_resetMatchState_markTablesDirty(void* opaque) +{ + ZSTD_rust_resetMatchStateContext* const context = + (ZSTD_rust_resetMatchStateContext*)opaque; + ZSTD_cwksp_mark_tables_dirty(context->ws); +} + +static void ZSTD_rust_resetMatchState_invalidate(void* opaque) +{ + ZSTD_rust_resetMatchStateContext* const context = + (ZSTD_rust_resetMatchStateContext*)opaque; + ZSTD_invalidateMatchState(context->ms); +} + +static void ZSTD_rust_resetMatchState_clearTables(void* opaque) +{ + ZSTD_rust_resetMatchStateContext* const context = + (ZSTD_rust_resetMatchStateContext*)opaque; + ZSTD_cwksp_clear_tables(context->ws); +} + +static void ZSTD_rust_resetMatchState_cleanTables(void* opaque) +{ + ZSTD_rust_resetMatchStateContext* const context = + (ZSTD_rust_resetMatchStateContext*)opaque; + ZSTD_cwksp_clean_tables(context->ws); +} + +static void ZSTD_rust_resetMatchState_advanceHashSalt(void* opaque) +{ + ZSTD_rust_resetMatchStateContext* const context = + (ZSTD_rust_resetMatchStateContext*)opaque; + ZSTD_advanceHashSalt(context->ms); +} + +static void* ZSTD_rust_resetMatchState_reserve( + void* opaque, int reserveKind, size_t size) +{ + ZSTD_rust_resetMatchStateContext* const context = + (ZSTD_rust_resetMatchStateContext*)opaque; + switch (reserveKind) { + case ZSTD_RUST_RESET_MATCH_RESERVE_TABLE: + return ZSTD_cwksp_reserve_table(context->ws, size); + case ZSTD_RUST_RESET_MATCH_RESERVE_ALIGNED_INIT_ONCE: + return ZSTD_cwksp_reserve_aligned_init_once(context->ws, size); + case ZSTD_RUST_RESET_MATCH_RESERVE_ALIGNED64: + return ZSTD_cwksp_reserve_aligned64(context->ws, size); + default: + assert(0); + return NULL; + } +} + +static int ZSTD_rust_resetMatchState_reserveFailed(void* opaque) +{ + ZSTD_rust_resetMatchStateContext* const context = + (ZSTD_rust_resetMatchStateContext*)opaque; + return ZSTD_cwksp_reserve_failed(context->ws); +} + +static void ZSTD_rust_resetMatchState_setPointer( + void* opaque, int pointerKind, void* pointer) +{ + ZSTD_rust_resetMatchStateContext* const context = + (ZSTD_rust_resetMatchStateContext*)opaque; + ZSTD_MatchState_t* const ms = context->ms; + switch (pointerKind) { + case ZSTD_RUST_RESET_MATCH_POINTER_HASH_TABLE: + ms->hashTable = (U32*)pointer; + break; + case ZSTD_RUST_RESET_MATCH_POINTER_CHAIN_TABLE: + ms->chainTable = (U32*)pointer; + break; + case ZSTD_RUST_RESET_MATCH_POINTER_HASH_TABLE3: + ms->hashTable3 = (U32*)pointer; + break; + case ZSTD_RUST_RESET_MATCH_POINTER_TAG_TABLE: + ms->tagTable = (BYTE*)pointer; + break; + case ZSTD_RUST_RESET_MATCH_POINTER_LIT_FREQ: + ms->opt.litFreq = (unsigned*)pointer; + break; + case ZSTD_RUST_RESET_MATCH_POINTER_LIT_LENGTH_FREQ: + ms->opt.litLengthFreq = (unsigned*)pointer; + break; + case ZSTD_RUST_RESET_MATCH_POINTER_MATCH_LENGTH_FREQ: + ms->opt.matchLengthFreq = (unsigned*)pointer; + break; + case ZSTD_RUST_RESET_MATCH_POINTER_OFF_CODE_FREQ: + ms->opt.offCodeFreq = (unsigned*)pointer; + break; + case ZSTD_RUST_RESET_MATCH_POINTER_MATCH_TABLE: + ms->opt.matchTable = (ZSTD_match_t*)pointer; + break; + case ZSTD_RUST_RESET_MATCH_POINTER_PRICE_TABLE: + ms->opt.priceTable = (ZSTD_optimal_t*)pointer; + break; + default: + assert(0); + break; + } +} + +static void ZSTD_rust_resetMatchState_zero( + void* opaque, void* pointer, size_t size) +{ + (void)opaque; + ZSTD_memset(pointer, 0, size); +} + static size_t ZSTD_reset_matchState(ZSTD_MatchState_t* ms, ZSTD_cwksp* ws, @@ -3317,82 +3517,40 @@ ZSTD_reset_matchState(ZSTD_MatchState_t* ms, const ZSTD_indexResetPolicy_e forceResetIndex, const ZSTD_resetTarget_e forWho) { - /* disable chain table allocation for fast or row-based strategies */ - size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, - ms->dedicatedDictSearch && (forWho == ZSTD_resetTarget_CDict)) - ? ((size_t)1 << cParams->chainLog) - : 0; - size_t const hSize = ((size_t)1) << cParams->hashLog; - U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; - size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; - - DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); - assert(useRowMatchFinder != ZSTD_ps_auto); - if (forceResetIndex == ZSTDirp_reset) { - ZSTD_window_init(&ms->window); - ZSTD_cwksp_mark_tables_dirty(ws); - } - - ms->hashLog3 = hashLog3; - ms->lazySkipping = 0; - - ZSTD_invalidateMatchState(ms); - - assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */ - - ZSTD_cwksp_clear_tables(ws); - - DEBUGLOG(5, "reserving table space"); - /* table Space */ - ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32)); - ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32)); - ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32)); - RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, - "failed a workspace allocation in ZSTD_reset_matchState"); - - DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty); - if (crp!=ZSTDcrp_leaveDirty) { - /* reset tables only */ - ZSTD_cwksp_clean_tables(ws); - } - - if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) { - /* Row match finder needs an additional table of hashes ("tags") */ - size_t const tagTableSize = hSize; - /* We want to generate a new salt in case we reset a Cctx, but we always want to use - * 0 when we reset a Cdict */ - if(forWho == ZSTD_resetTarget_CCtx) { - ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned_init_once(ws, tagTableSize); - ZSTD_advanceHashSalt(ms); - } else { - /* When we are not salting we want to always memset the memory */ - ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned64(ws, tagTableSize); - ZSTD_memset(ms->tagTable, 0, tagTableSize); - ms->hashSalt = 0; - } - { /* Switch to 32-entry rows if searchLog is 5 (or more) */ - U32 const rowLog = BOUNDED(4, cParams->searchLog, 6); - assert(cParams->hashLog >= rowLog); - ms->rowHashLog = cParams->hashLog - rowLog; - } - } - - /* opt parser space */ - if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { - DEBUGLOG(4, "reserving optimal parser space"); - ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxLL+1) * sizeof(unsigned)); - ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxML+1) * sizeof(unsigned)); - ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxOff+1) * sizeof(unsigned)); - ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_match_t)); - ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t)); - } - - ms->cParams = *cParams; - - RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, - "failed a workspace allocation in ZSTD_reset_matchState"); - return 0; + ZSTD_rust_resetMatchStateContext context; + ZSTD_rust_resetMatchStateState state; + context.ms = ms; + context.ws = ws; + state.callbackContext = &context; + state.cParams = *cParams; + state.dedicatedDictSearch = ms->dedicatedDictSearch; + state.useRowMatchFinder = (int)useRowMatchFinder; + state.compResetPolicy = (int)crp; + state.indexResetPolicy = (int)forceResetIndex; + state.resetTarget = (int)forWho; + state.hashLog3Max = ZSTD_HASHLOG3_MAX; + state.litFreqSize = (1 << Litbits) * sizeof(unsigned); + state.litLengthFreqSize = (MaxLL + 1) * sizeof(unsigned); + state.matchLengthFreqSize = (MaxML + 1) * sizeof(unsigned); + state.offCodeFreqSize = (MaxOff + 1) * sizeof(unsigned); + state.matchSize = ZSTD_OPT_SIZE * sizeof(ZSTD_match_t); + state.optimalSize = ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t); + state.hashLog3 = &ms->hashLog3; + state.rowHashLog = &ms->rowHashLog; + state.hashSalt = &ms->hashSalt; + state.lazySkipping = &ms->lazySkipping; + state.cParamsOut = &ms->cParams; + state.setPointer = ZSTD_rust_resetMatchState_setPointer; + state.windowInit = ZSTD_rust_resetMatchState_windowInit; + state.markTablesDirty = ZSTD_rust_resetMatchState_markTablesDirty; + state.invalidateMatchState = ZSTD_rust_resetMatchState_invalidate; + state.clearTables = ZSTD_rust_resetMatchState_clearTables; + state.cleanTables = ZSTD_rust_resetMatchState_cleanTables; + state.advanceHashSalt = ZSTD_rust_resetMatchState_advanceHashSalt; + state.reserve = ZSTD_rust_resetMatchState_reserve; + state.reserveFailed = ZSTD_rust_resetMatchState_reserveFailed; + state.zero = ZSTD_rust_resetMatchState_zero; + return ZSTD_rust_resetMatchState(&state); } /* ZSTD_indexTooCloseToMax() : diff --git a/rust/README.md b/rust/README.md index 2be045db0..20e014515 100644 --- a/rust/README.md +++ b/rust/README.md @@ -166,10 +166,10 @@ CDict advanced private workspace construction, private static-CCtx and static-CDict workspace construction and dictionary-content allocation/loading, and advanced-CDict dictionary-content loading remain in C. Rust now owns advanced-CDict custom-memory validation, workspace-size query/allocation, -allocation/create/init cleanup ordering, and the CCtx workspace-size formula; -C retains private layout-size inputs, workspace layout, and allocator -callbacks. Reset policy, -private CCtx/matchfinder/workspace operations, and codec/adaptive-policy +allocation/create/init cleanup ordering, the CCtx workspace-size formula, and +the match-state reset policy/order; C retains private layout-size inputs, +workspace layout, and allocator callbacks. Private CCtx reset/matchfinder/ +workspace operations and codec/adaptive-policy callbacks remain in C. CDict initialization ordering and scalar publication, shared compression-begin dictionary selection, CDict reset attach-versus-copy selection, and CDict-begin parameter selection, initialization ordering, and diff --git a/rust/src/zstd_compress.rs b/rust/src/zstd_compress.rs index 18873d6d9..62b1a0b78 100644 --- a/rust/src/zstd_compress.rs +++ b/rust/src/zstd_compress.rs @@ -24,11 +24,12 @@ use crate::zstd_compress_frame::{ use crate::zstd_compress_literals::min_gain; use crate::zstd_compress_params::{ ZSTD_compressionParameters, ZSTD_frameParameters, ZSTD_parameters, ZSTD_rustMatchStateSizing, - ZSTD_rust_params_adjustCParams, ZSTD_rust_params_checkCParams, ZSTD_rust_params_defaultCLevel, + ZSTD_rust_params_adjustCParams, ZSTD_rust_params_allocateChainTable, + ZSTD_rust_params_checkCParams, ZSTD_rust_params_defaultCLevel, ZSTD_rust_params_estimateMatchStateSize, ZSTD_rust_params_getParamsInternal, ZSTD_rust_params_maxNbSeq, ZSTD_rust_params_resolveMaxBlockSize, - ZSTD_rust_params_selectCParams, ZSTD_RUST_CPM_NO_ATTACH_DICT, ZSTD_RUST_PS_AUTO, - ZSTD_RUST_PS_DISABLE, ZSTD_RUST_PS_ENABLE, + ZSTD_rust_params_rowMatchFinderUsed, ZSTD_rust_params_selectCParams, + ZSTD_RUST_CPM_NO_ATTACH_DICT, ZSTD_RUST_PS_AUTO, ZSTD_RUST_PS_DISABLE, ZSTD_RUST_PS_ENABLE, }; use crate::zstd_compress_params_api::{ ZSTD_CCtxParams_setParameter, ZSTD_CCtx_params, ZSTD_rust_isUpdateAuthorized, @@ -5221,6 +5222,356 @@ pub extern "C" fn ZSTD_rust_dictTooBig(loaded_dict_size: usize) -> c_int { dict_too_big(loaded_dict_size) as c_int } +type ResetMatchStateCallback = unsafe extern "C" fn(*mut c_void); +type ResetMatchStateSetPointer = unsafe extern "C" fn(*mut c_void, c_int, *mut c_void); +type ResetMatchStateReserve = unsafe extern "C" fn(*mut c_void, c_int, usize) -> *mut c_void; +type ResetMatchStateReserveFailed = unsafe extern "C" fn(*mut c_void) -> c_int; +type ResetMatchStateZero = unsafe extern "C" fn(*mut c_void, *mut c_void, usize); + +const RESET_MATCH_RESERVE_TABLE: c_int = 0; +const RESET_MATCH_RESERVE_ALIGNED_INIT_ONCE: c_int = 1; +const RESET_MATCH_RESERVE_ALIGNED64: c_int = 2; + +const RESET_MATCH_POINTER_HASH_TABLE: c_int = 0; +const RESET_MATCH_POINTER_CHAIN_TABLE: c_int = 1; +const RESET_MATCH_POINTER_HASH_TABLE3: c_int = 2; +const RESET_MATCH_POINTER_TAG_TABLE: c_int = 3; +const RESET_MATCH_POINTER_LIT_FREQ: c_int = 4; +const RESET_MATCH_POINTER_LIT_LENGTH_FREQ: c_int = 5; +const RESET_MATCH_POINTER_MATCH_LENGTH_FREQ: c_int = 6; +const RESET_MATCH_POINTER_OFF_CODE_FREQ: c_int = 7; +const RESET_MATCH_POINTER_MATCH_TABLE: c_int = 8; +const RESET_MATCH_POINTER_PRICE_TABLE: c_int = 9; + +/// Private C layout pointers and callback seams for match-state reset. +/// +/// Rust owns the reset policy and allocation order. C retains the workspace, +/// window, and private match-state pointer representations behind callbacks. +#[repr(C)] +pub struct ZSTD_rust_resetMatchStateState { + callback_context: *mut c_void, + c_params: ZSTD_compressionParameters, + dedicated_dict_search: c_int, + use_row_match_finder: c_int, + comp_reset_policy: c_int, + index_reset_policy: c_int, + reset_target: c_int, + hash_log3_max: c_uint, + lit_freq_size: usize, + lit_length_freq_size: usize, + match_length_freq_size: usize, + off_code_freq_size: usize, + match_size: usize, + optimal_size: usize, + hash_log3: *mut c_uint, + row_hash_log: *mut c_uint, + hash_salt: *mut u64, + lazy_skipping: *mut c_int, + c_params_out: *mut ZSTD_compressionParameters, + set_pointer: Option, + window_init: Option, + mark_tables_dirty: Option, + invalidate_match_state: Option, + clear_tables: Option, + clean_tables: Option, + advance_hash_salt: Option, + reserve: Option, + reserve_failed: Option, + zero: Option, +} + +const _: () = { + assert!(size_of::() == size_of::()); + assert!(size_of::() == size_of::()); + assert!(size_of::() == size_of::()); + assert!(size_of::() == size_of::()); + assert!(size_of::() == size_of::()); + assert!(offset_of!(ZSTD_rust_resetMatchStateState, callback_context) == 0); + assert!(offset_of!(ZSTD_rust_resetMatchStateState, c_params) == size_of::()); + assert!( + offset_of!(ZSTD_rust_resetMatchStateState, lit_freq_size) + > offset_of!(ZSTD_rust_resetMatchStateState, hash_log3_max) + ); + assert!( + offset_of!(ZSTD_rust_resetMatchStateState, hash_log3) + == offset_of!(ZSTD_rust_resetMatchStateState, optimal_size) + size_of::() + ); + assert!( + offset_of!(ZSTD_rust_resetMatchStateState, set_pointer) + == offset_of!(ZSTD_rust_resetMatchStateState, c_params_out) + size_of::() + ); + assert!( + size_of::() + == offset_of!(ZSTD_rust_resetMatchStateState, zero) + size_of::() + ); +}; + +#[inline] +fn reset_match_state_table_size(log: u32) -> Option { + 1usize.checked_shl(log) +} + +/// Reset and allocate one private match-state projection. +/// +/// The callback context deliberately remains opaque here. This keeps C's +/// workspace phases and match-state layout out of the Rust ABI while making +/// the policy/order independently testable. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_resetMatchState( + state: *const ZSTD_rust_resetMatchStateState, +) -> usize { + if state.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + let state = unsafe { &*state }; + let Some(set_pointer) = state.set_pointer else { + return ERROR(ZstdErrorCode::Generic); + }; + let Some(window_init) = state.window_init else { + return ERROR(ZstdErrorCode::Generic); + }; + let Some(mark_tables_dirty) = state.mark_tables_dirty else { + return ERROR(ZstdErrorCode::Generic); + }; + let Some(invalidate_match_state) = state.invalidate_match_state else { + return ERROR(ZstdErrorCode::Generic); + }; + let Some(clear_tables) = state.clear_tables else { + return ERROR(ZstdErrorCode::Generic); + }; + let Some(clean_tables) = state.clean_tables else { + return ERROR(ZstdErrorCode::Generic); + }; + let Some(advance_hash_salt) = state.advance_hash_salt else { + return ERROR(ZstdErrorCode::Generic); + }; + let Some(reserve) = state.reserve else { + return ERROR(ZstdErrorCode::Generic); + }; + let Some(reserve_failed) = state.reserve_failed else { + return ERROR(ZstdErrorCode::Generic); + }; + let Some(zero) = state.zero else { + return ERROR(ZstdErrorCode::Generic); + }; + if state.callback_context.is_null() + || state.hash_log3.is_null() + || state.row_hash_log.is_null() + || state.hash_salt.is_null() + || state.lazy_skipping.is_null() + || state.c_params_out.is_null() + { + return ERROR(ZstdErrorCode::Generic); + } + if state.use_row_match_finder == ZSTD_RUST_PS_AUTO + || !matches!(state.comp_reset_policy, 0..=1) + || !matches!(state.index_reset_policy, 0..=1) + || !matches!(state.reset_target, 0..=1) + { + return ERROR(ZstdErrorCode::Generic); + } + + let chain_size = if ZSTD_rust_params_allocateChainTable( + state.c_params.strategy, + state.use_row_match_finder, + c_int::from(state.dedicated_dict_search != 0 && state.reset_target == 0), + ) != 0 + { + let Some(size) = reset_match_state_table_size(state.c_params.chainLog) else { + return ERROR(ZstdErrorCode::Generic); + }; + size + } else { + 0 + }; + let Some(hash_size) = reset_match_state_table_size(state.c_params.hashLog) else { + return ERROR(ZstdErrorCode::Generic); + }; + let hash_log3 = if state.reset_target == 1 && state.c_params.minMatch == 3 { + state.hash_log3_max.min(state.c_params.windowLog) + } else { + 0 + }; + let hash3_size = if hash_log3 == 0 { + 0 + } else { + let Some(size) = reset_match_state_table_size(hash_log3) else { + return ERROR(ZstdErrorCode::Generic); + }; + size + }; + + if state.index_reset_policy == 1 { + unsafe { + window_init(state.callback_context); + mark_tables_dirty(state.callback_context); + } + } + + unsafe { + *state.hash_log3 = hash_log3; + *state.lazy_skipping = 0; + invalidate_match_state(state.callback_context); + debug_assert_eq!(reserve_failed(state.callback_context), 0); + clear_tables(state.callback_context); + } + + unsafe { + set_pointer( + state.callback_context, + RESET_MATCH_POINTER_HASH_TABLE, + reserve( + state.callback_context, + RESET_MATCH_RESERVE_TABLE, + hash_size.wrapping_mul(size_of::()), + ), + ); + set_pointer( + state.callback_context, + RESET_MATCH_POINTER_CHAIN_TABLE, + reserve( + state.callback_context, + RESET_MATCH_RESERVE_TABLE, + chain_size.wrapping_mul(size_of::()), + ), + ); + set_pointer( + state.callback_context, + RESET_MATCH_POINTER_HASH_TABLE3, + reserve( + state.callback_context, + RESET_MATCH_RESERVE_TABLE, + hash3_size.wrapping_mul(size_of::()), + ), + ); + } + if unsafe { reserve_failed(state.callback_context) } != 0 { + return ERROR(ZstdErrorCode::MemoryAllocation); + } + + if state.comp_reset_policy != 1 { + unsafe { clean_tables(state.callback_context) }; + } + + if ZSTD_rust_params_rowMatchFinderUsed(state.c_params.strategy, state.use_row_match_finder) != 0 + { + if state.reset_target == 1 { + let tag_table = unsafe { + reserve( + state.callback_context, + RESET_MATCH_RESERVE_ALIGNED_INIT_ONCE, + hash_size, + ) + }; + unsafe { + set_pointer( + state.callback_context, + RESET_MATCH_POINTER_TAG_TABLE, + tag_table, + ); + advance_hash_salt(state.callback_context); + } + } else { + let tag_table = unsafe { + reserve( + state.callback_context, + RESET_MATCH_RESERVE_ALIGNED64, + hash_size, + ) + }; + unsafe { + set_pointer( + state.callback_context, + RESET_MATCH_POINTER_TAG_TABLE, + tag_table, + ); + } + if unsafe { reserve_failed(state.callback_context) } != 0 { + return ERROR(ZstdErrorCode::MemoryAllocation); + } + unsafe { + zero(state.callback_context, tag_table, hash_size); + *state.hash_salt = 0; + } + } + if unsafe { reserve_failed(state.callback_context) } != 0 { + return ERROR(ZstdErrorCode::MemoryAllocation); + } + let row_log = state.c_params.searchLog.clamp(4, 6); + debug_assert!(state.c_params.hashLog >= row_log); + unsafe { + *state.row_hash_log = state.c_params.hashLog.wrapping_sub(row_log); + } + } + + if state.reset_target == 1 && state.c_params.strategy >= 7 { + unsafe { + set_pointer( + state.callback_context, + RESET_MATCH_POINTER_LIT_FREQ, + reserve( + state.callback_context, + RESET_MATCH_RESERVE_ALIGNED64, + state.lit_freq_size, + ), + ); + set_pointer( + state.callback_context, + RESET_MATCH_POINTER_LIT_LENGTH_FREQ, + reserve( + state.callback_context, + RESET_MATCH_RESERVE_ALIGNED64, + state.lit_length_freq_size, + ), + ); + set_pointer( + state.callback_context, + RESET_MATCH_POINTER_MATCH_LENGTH_FREQ, + reserve( + state.callback_context, + RESET_MATCH_RESERVE_ALIGNED64, + state.match_length_freq_size, + ), + ); + set_pointer( + state.callback_context, + RESET_MATCH_POINTER_OFF_CODE_FREQ, + reserve( + state.callback_context, + RESET_MATCH_RESERVE_ALIGNED64, + state.off_code_freq_size, + ), + ); + set_pointer( + state.callback_context, + RESET_MATCH_POINTER_MATCH_TABLE, + reserve( + state.callback_context, + RESET_MATCH_RESERVE_ALIGNED64, + state.match_size, + ), + ); + set_pointer( + state.callback_context, + RESET_MATCH_POINTER_PRICE_TABLE, + reserve( + state.callback_context, + RESET_MATCH_RESERVE_ALIGNED64, + state.optimal_size, + ), + ); + } + } + + unsafe { + *state.c_params_out = state.c_params; + } + if unsafe { reserve_failed(state.callback_context) } != 0 { + return ERROR(ZstdErrorCode::MemoryAllocation); + } + 0 +} + #[inline] fn sizeof_local_dict(dict_buffer_present: c_int, dict_size: usize, cdict_size: usize) -> usize { let buffer_size = if dict_buffer_present != 0 { @@ -6821,6 +7172,348 @@ mod tests { const ZSTD_BTOPT: c_int = 7; const ZSTD_BTULTRA2: c_int = 9; + #[derive(Default)] + struct ResetMatchStateTestContext { + events: Vec, + reserves: Vec<(c_int, usize)>, + pointers: Vec<(c_int, *mut c_void)>, + zeroed: Vec<(usize, usize)>, + storage: [usize; 32], + fail_after: usize, + } + + unsafe fn reset_match_state_test_context( + context: *mut c_void, + ) -> &'static mut ResetMatchStateTestContext { + unsafe { &mut *context.cast::() } + } + + unsafe extern "C" fn reset_match_state_test_callback(context: *mut c_void) { + let context = unsafe { reset_match_state_test_context(context) }; + context.events.push(1); + } + + unsafe extern "C" fn reset_match_state_test_mark_dirty(context: *mut c_void) { + let context = unsafe { reset_match_state_test_context(context) }; + context.events.push(2); + } + + unsafe extern "C" fn reset_match_state_test_invalidate(context: *mut c_void) { + let context = unsafe { reset_match_state_test_context(context) }; + context.events.push(3); + } + + unsafe extern "C" fn reset_match_state_test_clear(context: *mut c_void) { + let context = unsafe { reset_match_state_test_context(context) }; + context.events.push(4); + } + + unsafe extern "C" fn reset_match_state_test_clean(context: *mut c_void) { + let context = unsafe { reset_match_state_test_context(context) }; + context.events.push(5); + } + + unsafe extern "C" fn reset_match_state_test_advance(context: *mut c_void) { + let context = unsafe { reset_match_state_test_context(context) }; + context.events.push(6); + } + + unsafe extern "C" fn reset_match_state_test_reserve( + context: *mut c_void, + reserve_kind: c_int, + size: usize, + ) -> *mut c_void { + let context = unsafe { reset_match_state_test_context(context) }; + context.events.push(100 + reserve_kind); + let index = context.reserves.len(); + context.reserves.push((reserve_kind, size)); + if context.fail_after != 0 && index + 1 >= context.fail_after { + ptr::null_mut() + } else { + unsafe { context.storage.as_mut_ptr().add(index).cast() } + } + } + + unsafe extern "C" fn reset_match_state_test_reserve_failed(context: *mut c_void) -> c_int { + let context = unsafe { reset_match_state_test_context(context) }; + c_int::from(context.fail_after != 0 && context.reserves.len() >= context.fail_after) + } + + unsafe extern "C" fn reset_match_state_test_set_pointer( + context: *mut c_void, + pointer_kind: c_int, + pointer: *mut c_void, + ) { + let context = unsafe { reset_match_state_test_context(context) }; + context.events.push(200 + pointer_kind); + context.pointers.push((pointer_kind, pointer)); + } + + unsafe extern "C" fn reset_match_state_test_zero( + context: *mut c_void, + pointer: *mut c_void, + size: usize, + ) { + let context = unsafe { reset_match_state_test_context(context) }; + context.events.push(7); + context.zeroed.push((pointer as usize, size)); + } + + #[allow(clippy::too_many_arguments)] + fn reset_match_state_test_state( + context: &mut ResetMatchStateTestContext, + c_params: ZSTD_compressionParameters, + dedicated_dict_search: c_int, + use_row_match_finder: c_int, + comp_reset_policy: c_int, + index_reset_policy: c_int, + reset_target: c_int, + hash_log3: &mut c_uint, + row_hash_log: &mut c_uint, + hash_salt: &mut u64, + lazy_skipping: &mut c_int, + c_params_out: &mut ZSTD_compressionParameters, + ) -> ZSTD_rust_resetMatchStateState { + ZSTD_rust_resetMatchStateState { + callback_context: (context as *mut ResetMatchStateTestContext).cast(), + c_params, + dedicated_dict_search, + use_row_match_finder, + comp_reset_policy, + index_reset_policy, + reset_target, + hash_log3_max: 17, + lit_freq_size: 256 * size_of::(), + lit_length_freq_size: 36 * size_of::(), + match_length_freq_size: 53 * size_of::(), + off_code_freq_size: 32 * size_of::(), + match_size: 4099 * 2 * size_of::(), + optimal_size: 4099 * 8 * size_of::(), + hash_log3, + row_hash_log, + hash_salt, + lazy_skipping, + c_params_out, + set_pointer: Some(reset_match_state_test_set_pointer), + window_init: Some(reset_match_state_test_callback), + mark_tables_dirty: Some(reset_match_state_test_mark_dirty), + invalidate_match_state: Some(reset_match_state_test_invalidate), + clear_tables: Some(reset_match_state_test_clear), + clean_tables: Some(reset_match_state_test_clean), + advance_hash_salt: Some(reset_match_state_test_advance), + reserve: Some(reset_match_state_test_reserve), + reserve_failed: Some(reset_match_state_test_reserve_failed), + zero: Some(reset_match_state_test_zero), + } + } + + fn reset_match_state_test_params( + strategy: c_int, + search_log: u32, + min_match: u32, + ) -> ZSTD_compressionParameters { + ZSTD_compressionParameters { + windowLog: 20, + chainLog: 11, + hashLog: 12, + searchLog: search_log, + minMatch: min_match, + targetLength: 16, + strategy, + } + } + + #[test] + fn reset_match_state_orders_cctx_opt_allocations() { + let c_params = reset_match_state_test_params(ZSTD_BTOPT, 5, 3); + let mut context = ResetMatchStateTestContext::default(); + let mut hash_log3 = 99; + let mut row_hash_log = 99; + let mut hash_salt = 7; + let mut lazy_skipping = 9; + let mut c_params_out = ZSTD_compressionParameters::default(); + let state = reset_match_state_test_state( + &mut context, + c_params, + 0, + ZSTD_RUST_PS_DISABLE, + 0, + 1, + 1, + &mut hash_log3, + &mut row_hash_log, + &mut hash_salt, + &mut lazy_skipping, + &mut c_params_out, + ); + + let result = unsafe { ZSTD_rust_resetMatchState(&state) }; + + assert_eq!(result, 0); + assert_eq!(hash_log3, 17); + assert_eq!(row_hash_log, 99); + assert_eq!(hash_salt, 7); + assert_eq!(lazy_skipping, 0); + assert_eq!(c_params_out, c_params); + assert_eq!( + context.events, + [ + 1, 2, 3, 4, 100, 200, 100, 201, 100, 202, 5, 102, 204, 102, 205, 102, 206, 102, + 207, 102, 208, 102, 209, + ] + ); + assert_eq!( + context.reserves, + [ + (0, (1 << 12) * size_of::()), + (0, (1 << 11) * size_of::()), + (0, (1 << 17) * size_of::()), + (2, 256 * size_of::()), + (2, 36 * size_of::()), + (2, 53 * size_of::()), + (2, 32 * size_of::()), + (2, 4099 * 2 * size_of::()), + (2, 4099 * 8 * size_of::()), + ] + ); + assert_eq!(context.pointers.len(), 9); + assert!(context.zeroed.is_empty()); + } + + #[test] + fn reset_match_state_salts_cctx_row_tables_without_zeroing() { + let c_params = reset_match_state_test_params(ZSTD_GREEDY, 6, 4); + let mut context = ResetMatchStateTestContext::default(); + let mut hash_log3 = 99; + let mut row_hash_log = 99; + let mut hash_salt = 7; + let mut lazy_skipping = 9; + let mut c_params_out = ZSTD_compressionParameters::default(); + let state = reset_match_state_test_state( + &mut context, + c_params, + 0, + ZSTD_RUST_PS_ENABLE, + 0, + 0, + 1, + &mut hash_log3, + &mut row_hash_log, + &mut hash_salt, + &mut lazy_skipping, + &mut c_params_out, + ); + + let result = unsafe { ZSTD_rust_resetMatchState(&state) }; + + assert_eq!(result, 0); + assert_eq!(hash_log3, 0); + assert_eq!(row_hash_log, 6); + assert_eq!(hash_salt, 7); + assert_eq!(c_params_out, c_params); + assert_eq!( + context.events, + [3, 4, 100, 200, 100, 201, 100, 202, 5, 101, 203, 6] + ); + assert_eq!( + context.reserves, + [ + (0, (1 << 12) * size_of::()), + (0, 0), + (0, 0), + (1, 1 << 12), + ] + ); + assert_eq!(context.pointers.len(), 4); + assert!(context.zeroed.is_empty()); + } + + #[test] + fn reset_match_state_uses_cdict_row_policy_without_opt_storage() { + let c_params = reset_match_state_test_params(ZSTD_GREEDY, 6, 4); + let mut context = ResetMatchStateTestContext::default(); + let mut hash_log3 = 99; + let mut row_hash_log = 99; + let mut hash_salt = 7; + let mut lazy_skipping = 9; + let mut c_params_out = ZSTD_compressionParameters::default(); + let state = reset_match_state_test_state( + &mut context, + c_params, + 1, + ZSTD_RUST_PS_ENABLE, + 1, + 0, + 0, + &mut hash_log3, + &mut row_hash_log, + &mut hash_salt, + &mut lazy_skipping, + &mut c_params_out, + ); + + let result = unsafe { ZSTD_rust_resetMatchState(&state) }; + + assert_eq!(result, 0); + assert_eq!(hash_log3, 0); + assert_eq!(row_hash_log, 6); + assert_eq!(hash_salt, 0); + assert_eq!(c_params_out, c_params); + assert_eq!( + context.events, + [3, 4, 100, 200, 100, 201, 100, 202, 102, 203, 7] + ); + assert_eq!( + context.reserves, + [ + (0, (1 << 12) * size_of::()), + (0, (1 << 11) * size_of::()), + (0, 0), + (2, 1 << 12), + ] + ); + assert_eq!(context.pointers.len(), 4); + assert_eq!(context.zeroed.len(), 1); + assert_eq!(context.zeroed[0].1, 1 << 12); + } + + #[test] + fn reset_match_state_propagates_table_allocation_failure() { + let c_params = reset_match_state_test_params(ZSTD_GREEDY, 4, 4); + let mut context = ResetMatchStateTestContext { + fail_after: 1, + ..ResetMatchStateTestContext::default() + }; + let mut hash_log3 = 99; + let mut row_hash_log = 99; + let mut hash_salt = 7; + let mut lazy_skipping = 9; + let mut c_params_out = ZSTD_compressionParameters::default(); + let state = reset_match_state_test_state( + &mut context, + c_params, + 0, + ZSTD_RUST_PS_DISABLE, + 0, + 1, + 1, + &mut hash_log3, + &mut row_hash_log, + &mut hash_salt, + &mut lazy_skipping, + &mut c_params_out, + ); + + let result = unsafe { ZSTD_rust_resetMatchState(&state) }; + + assert_eq!(result, ERROR(ZstdErrorCode::MemoryAllocation)); + assert_eq!(context.events, [1, 2, 3, 4, 100, 200, 100, 201, 100, 202]); + assert_eq!(context.pointers.len(), 3); + assert_eq!(hash_log3, 0); + assert_eq!(lazy_skipping, 0); + assert_eq!(c_params_out, ZSTD_compressionParameters::default()); + } + #[test] fn reference_external_sequences_resets_store_state() { let mut source = [0u8; 3];