diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index efe420893..b255d7fac 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -677,7 +677,8 @@ typedef struct { ZSTD_rust_overflowCallback_f markTablesDirty; ZSTD_rust_overflowReduceIndex_f reduceIndex; ZSTD_rust_overflowCallback_f markTablesClean; - ZSTD_rust_overflowCallback_f invalidateDictionary; + U32* loadedDictEnd; + const ZSTD_MatchState_t** dictMatchState; } ZSTD_rust_overflowCorrectState; void ZSTD_rust_overflowCorrectIfNeeded( const ZSTD_rust_overflowCorrectState* state, @@ -690,8 +691,9 @@ typedef char ZSTD_rust_overflow_correct_state_layout[ && offsetof(ZSTD_rust_overflowCorrectState, markTablesDirty) == 4 * sizeof(void*) && offsetof(ZSTD_rust_overflowCorrectState, reduceIndex) == 5 * sizeof(void*) && offsetof(ZSTD_rust_overflowCorrectState, markTablesClean) == 6 * sizeof(void*) - && offsetof(ZSTD_rust_overflowCorrectState, invalidateDictionary) == 7 * sizeof(void*) - && sizeof(ZSTD_rust_overflowCorrectState) == 8 * sizeof(void*)) + && offsetof(ZSTD_rust_overflowCorrectState, loadedDictEnd) == 7 * sizeof(void*) + && offsetof(ZSTD_rust_overflowCorrectState, dictMatchState) == 8 * sizeof(void*) + && sizeof(ZSTD_rust_overflowCorrectState) == 9 * sizeof(void*)) ? 1 : -1]; void ZSTD_rust_copyCDictTableIntoCCtx(U32* dst, U32 const* src, size_t tableSize, int tagged); @@ -768,15 +770,23 @@ typedef void (*ZSTD_rust_frameChunkPrepareWindow_f)(void* context, const void* src, size_t blockSize, U32 maxDist); -typedef void (*ZSTD_rust_frameChunkPrepareClamp_f)(void* context); +typedef struct { + U32* nextToUpdate; + const U32* lowLimit; +} ZSTD_rust_frameChunkClampState; typedef struct { void* callbackContext; U32 maxDist; ZSTD_rust_frameChunkPrepareOverflow_f correctOverflow; ZSTD_rust_frameChunkPrepareWindow_f checkDictValidity; ZSTD_rust_frameChunkPrepareWindow_f enforceMaxDist; - ZSTD_rust_frameChunkPrepareClamp_f clampNextToUpdate; + const ZSTD_rust_frameChunkClampState* clampState; } ZSTD_rust_frameChunkPrepareState; +typedef char ZSTD_rust_frame_chunk_clamp_state_layout[ + (offsetof(ZSTD_rust_frameChunkClampState, nextToUpdate) == 0 + && offsetof(ZSTD_rust_frameChunkClampState, lowLimit) == sizeof(void*) + && sizeof(ZSTD_rust_frameChunkClampState) == 2 * sizeof(void*)) + ? 1 : -1]; typedef char ZSTD_rust_frame_chunk_prepare_state_layout[ (offsetof(ZSTD_rust_frameChunkPrepareState, callbackContext) == 0 && offsetof(ZSTD_rust_frameChunkPrepareState, maxDist) == sizeof(void*) @@ -786,7 +796,7 @@ typedef char ZSTD_rust_frame_chunk_prepare_state_layout[ == 3 * sizeof(void*) && offsetof(ZSTD_rust_frameChunkPrepareState, enforceMaxDist) == 4 * sizeof(void*) - && offsetof(ZSTD_rust_frameChunkPrepareState, clampNextToUpdate) + && offsetof(ZSTD_rust_frameChunkPrepareState, clampState) == 5 * sizeof(void*) && sizeof(ZSTD_rust_frameChunkPrepareState) == 6 * sizeof(void*)) ? 1 : -1]; @@ -5405,14 +5415,6 @@ static void ZSTD_rust_overflowCorrect_markTablesClean(void* context) ZSTD_cwksp_mark_tables_clean(state->workspace); } -static void ZSTD_rust_overflowCorrect_invalidateDictionary(void* context) -{ - ZSTD_rust_overflowCorrectContext const* const state = - (const ZSTD_rust_overflowCorrectContext*)context; - state->matchState->loadedDictEnd = 0; - state->matchState->dictMatchState = NULL; -} - static void ZSTD_overflowCorrectIfNeeded(ZSTD_MatchState_t* ms, ZSTD_cwksp* ws, ZSTD_CCtx_params const* params, @@ -5434,7 +5436,8 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_MatchState_t* ms, state.markTablesDirty = ZSTD_rust_overflowCorrect_markTablesDirty; state.reduceIndex = ZSTD_rust_overflowCorrect_reduceIndex; state.markTablesClean = ZSTD_rust_overflowCorrect_markTablesClean; - state.invalidateDictionary = ZSTD_rust_overflowCorrect_invalidateDictionary; + state.loadedDictEnd = &ms->loadedDictEnd; + state.dictMatchState = &ms->dictMatchState; ZSTD_rust_overflowCorrectIfNeeded(&state, ip, iend); } @@ -5476,15 +5479,6 @@ static void ZSTD_rust_frameChunk_enforceMaxDist(void* context, (void)blockSize; } -static void ZSTD_rust_frameChunk_clampNextToUpdate(void* context) -{ - ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context; - ZSTD_MatchState_t* const ms = &cctx->blockState.matchState; - /* Ensure hash/chain table insertion resumes no sooner than lowlimit. */ - if (ms->nextToUpdate < ms->window.lowLimit) - ms->nextToUpdate = ms->window.lowLimit; -} - static size_t ZSTD_rust_frameChunk_compressTarget( void* context, void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) @@ -5531,12 +5525,16 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx, { ZSTD_rust_frameChunkState state; ZSTD_rust_frameChunkPrepareState prepareState; + ZSTD_rust_frameChunkClampState clampState; + ZSTD_MatchState_t* const ms = &cctx->blockState.matchState; prepareState.callbackContext = cctx; prepareState.maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; prepareState.correctOverflow = ZSTD_rust_frameChunk_correctOverflow; prepareState.checkDictValidity = ZSTD_rust_frameChunk_checkDictValidity; prepareState.enforceMaxDist = ZSTD_rust_frameChunk_enforceMaxDist; - prepareState.clampNextToUpdate = ZSTD_rust_frameChunk_clampNextToUpdate; + clampState.nextToUpdate = &ms->nextToUpdate; + clampState.lowLimit = &ms->window.lowLimit; + prepareState.clampState = &clampState; state.callbackContext = cctx; state.tmpWorkspace = cctx->tmpWorkspace; state.checksumState = &cctx->xxhState; diff --git a/rust/src/zstd_compress.rs b/rust/src/zstd_compress.rs index 7d9c23751..6f97646be 100644 --- a/rust/src/zstd_compress.rs +++ b/rust/src/zstd_compress.rs @@ -173,7 +173,6 @@ const FSE_REPEAT_VALID: c_int = 2; type FrameChunkPrepareOverflowFn = unsafe extern "C" fn(*mut c_void, *const c_void, usize); type FrameChunkPrepareWindowFn = unsafe extern "C" fn(*mut c_void, *const c_void, usize, c_uint); -type FrameChunkPrepareClampFn = unsafe extern "C" fn(*mut c_void); type FrameChunkCompressFn = unsafe extern "C" fn(*mut c_void, *mut c_void, usize, *const c_void, usize, c_uint) -> usize; type FrameChunkChecksumFn = unsafe extern "C" fn(*mut c_void, *const c_void, usize); @@ -507,9 +506,21 @@ pub struct ZSTD_rust_frameChunkPrepareState { correct_overflow: FrameChunkPrepareOverflowFn, check_dict_validity: FrameChunkPrepareWindowFn, enforce_max_dist: FrameChunkPrepareWindowFn, - clamp_next_to_update: FrameChunkPrepareClampFn, + clamp_state: *const ZSTD_rust_frameChunkClampState, } +#[repr(C)] +pub struct ZSTD_rust_frameChunkClampState { + next_to_update: *mut c_uint, + low_limit: *const c_uint, +} + +const _: () = { + assert!(offset_of!(ZSTD_rust_frameChunkClampState, next_to_update) == 0); + assert!(offset_of!(ZSTD_rust_frameChunkClampState, low_limit) == size_of::()); + assert!(size_of::() == 2 * size_of::()); +}; + const _: () = { assert!(offset_of!(ZSTD_rust_frameChunkPrepareState, callback_context) == 0); assert!(offset_of!(ZSTD_rust_frameChunkPrepareState, max_dist) == size_of::()); @@ -522,10 +533,7 @@ const _: () = { assert!( offset_of!(ZSTD_rust_frameChunkPrepareState, enforce_max_dist) == 4 * size_of::() ); - assert!( - offset_of!(ZSTD_rust_frameChunkPrepareState, clamp_next_to_update) - == 5 * size_of::() - ); + assert!(offset_of!(ZSTD_rust_frameChunkPrepareState, clamp_state) == 5 * size_of::()); assert!(size_of::() == 6 * size_of::()); }; @@ -616,7 +624,19 @@ unsafe fn prepare_frame_chunk_block( unsafe { (state.correct_overflow)(state.callback_context, src, block_size) }; unsafe { (state.check_dict_validity)(state.callback_context, src, block_size, state.max_dist) }; unsafe { (state.enforce_max_dist)(state.callback_context, src, block_size, state.max_dist) }; - unsafe { (state.clamp_next_to_update)(state.callback_context) }; + if state.clamp_state.is_null() { + return; + } + let clamp_state = unsafe { &*state.clamp_state }; + if clamp_state.next_to_update.is_null() || clamp_state.low_limit.is_null() { + return; + } + unsafe { + /* Ensure hash/chain table insertion resumes no sooner than lowlimit. */ + if *clamp_state.next_to_update < *clamp_state.low_limit { + *clamp_state.next_to_update = *clamp_state.low_limit; + } + } } #[allow(clippy::too_many_arguments)] @@ -5480,7 +5500,8 @@ pub struct ZSTD_rust_overflowCorrectState { mark_tables_dirty: Option, reduce_index: Option, mark_tables_clean: Option, - invalidate_dictionary: Option, + loaded_dict_end: *mut c_uint, + dict_match_state: *mut *const c_void, } const _: () = { @@ -5499,10 +5520,11 @@ const _: () = { assert!( offset_of!(ZSTD_rust_overflowCorrectState, mark_tables_clean) == 6 * size_of::() ); + assert!(offset_of!(ZSTD_rust_overflowCorrectState, loaded_dict_end) == 7 * size_of::()); assert!( - offset_of!(ZSTD_rust_overflowCorrectState, invalidate_dictionary) == 7 * size_of::() + offset_of!(ZSTD_rust_overflowCorrectState, dict_match_state) == size_of::<[usize; 8]>() ); - assert!(size_of::() == size_of::<[usize; 8]>()); + assert!(size_of::() == size_of::<[usize; 9]>()); }; /// Apply one overflow correction while keeping all private codec state in C. @@ -5516,7 +5538,11 @@ pub unsafe extern "C" fn ZSTD_rust_overflowCorrectIfNeeded( return; } let state = unsafe { &*state }; - if state.callback_context.is_null() || state.next_to_update.is_null() { + if state.callback_context.is_null() + || state.next_to_update.is_null() + || state.loaded_dict_end.is_null() + || state.dict_match_state.is_null() + { return; } let ( @@ -5525,14 +5551,12 @@ pub unsafe extern "C" fn ZSTD_rust_overflowCorrectIfNeeded( Some(mark_tables_dirty), Some(reduce_index), Some(mark_tables_clean), - Some(invalidate_dictionary), ) = ( state.need_correction, state.correct_overflow, state.mark_tables_dirty, state.reduce_index, state.mark_tables_clean, - state.invalidate_dictionary, ) else { return; @@ -5547,7 +5571,8 @@ pub unsafe extern "C" fn ZSTD_rust_overflowCorrectIfNeeded( reduce_index(state.callback_context, correction); mark_tables_clean(state.callback_context); *state.next_to_update = (*state.next_to_update).saturating_sub(correction); - invalidate_dictionary(state.callback_context); + *state.loaded_dict_end = 0; + *state.dict_match_state = ptr::null(); } } @@ -10215,6 +10240,9 @@ mod tests { prepare_order_len: usize, prepare_max_dist: c_uint, prepare_state: Option, + clamp_state: Option, + clamp_next_to_update: c_uint, + clamp_low_limit: c_uint, target_calls: usize, split_calls: usize, internal_calls: usize, @@ -10272,10 +10300,6 @@ mod tests { unsafe { frame_chunk_test_context(context) }.prepare_max_dist = max_dist; } - unsafe extern "C" fn frame_chunk_test_clamp_next_to_update(context: *mut c_void) { - unsafe { frame_chunk_test_record_prepare(context, 4, 0) }; - } - unsafe extern "C" fn frame_chunk_test_target( context: *mut c_void, _dst: *mut c_void, @@ -10345,13 +10369,20 @@ mod tests { let context_ptr = context as *mut FrameChunkTestContext; let callback_context = context_ptr.cast::(); unsafe { + (*context_ptr).clamp_state = Some(ZSTD_rust_frameChunkClampState { + next_to_update: &mut (*context_ptr).clamp_next_to_update, + low_limit: &(*context_ptr).clamp_low_limit, + }); (*context_ptr).prepare_state = Some(ZSTD_rust_frameChunkPrepareState { callback_context, max_dist: 64, correct_overflow: frame_chunk_test_correct_overflow, check_dict_validity: frame_chunk_test_check_dict_validity, enforce_max_dist: frame_chunk_test_enforce_max_dist, - clamp_next_to_update: frame_chunk_test_clamp_next_to_update, + clamp_state: (*context_ptr) + .clamp_state + .as_ref() + .map_or(ptr::null(), |state| state as *const _), }); } let prepare_state = unsafe { @@ -10387,6 +10418,8 @@ mod tests { fn frame_chunk_internal_path_emits_blocks_and_updates_state_once() { let mut context = FrameChunkTestContext { internal_result: 2, + clamp_next_to_update: 1, + clamp_low_limit: 4, ..FrameChunkTestContext::default() }; let mut is_first_block = 1; @@ -10409,8 +10442,9 @@ mod tests { assert_eq!(result, 10); assert_eq!(context.prepare_calls, 2); assert_eq!(context.prepared_sizes[..2], [4, 4]); - assert_eq!(context.prepare_order[..8], [1, 2, 3, 4, 1, 2, 3, 4]); + assert_eq!(context.prepare_order[..6], [1, 2, 3, 1, 2, 3]); assert_eq!(context.prepare_max_dist, 64); + assert_eq!(context.clamp_next_to_update, 4); assert_eq!(context.internal_calls, 2); assert_eq!(context.last_blocks[..2], [0, 1]); assert_eq!(context.checksum_calls, 1); @@ -11456,6 +11490,8 @@ mod tests { events: Vec<&'static str>, should_correct: c_int, correction: c_uint, + loaded_dict_end: c_uint, + dict_match_state: *const c_void, } unsafe fn overflow_correction_test_context( @@ -11504,12 +11540,6 @@ mod tests { .push("clean"); } - unsafe extern "C" fn overflow_correction_test_invalidate(context: *mut c_void) { - unsafe { overflow_correction_test_context(context) } - .events - .push("invalidate"); - } - fn overflow_correction_test_state( context: &mut OverflowCorrectionTestContext, next_to_update: &mut c_uint, @@ -11522,7 +11552,8 @@ mod tests { mark_tables_dirty: Some(overflow_correction_test_mark_dirty), reduce_index: Some(overflow_correction_test_reduce), mark_tables_clean: Some(overflow_correction_test_mark_clean), - invalidate_dictionary: Some(overflow_correction_test_invalidate), + loaded_dict_end: &mut context.loaded_dict_end, + dict_match_state: &mut context.dict_match_state, } } @@ -11531,6 +11562,8 @@ mod tests { let mut context = OverflowCorrectionTestContext { should_correct: 1, correction: 10, + loaded_dict_end: 33, + dict_match_state: ptr::dangling(), ..Default::default() }; let mut next_to_update = 7; @@ -11543,8 +11576,10 @@ mod tests { assert_eq!(next_to_update, 0); assert_eq!( context.events, - ["need", "correct", "dirty", "reduce", "clean", "invalidate"] + ["need", "correct", "dirty", "reduce", "clean"] ); + assert_eq!(context.loaded_dict_end, 0); + assert!(context.dict_match_state.is_null()); } #[test]