diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c14e2d532..9c6b2c6fe 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2401,6 +2401,43 @@ typedef char ZSTD_rust_reset_cctx_by_copying_cdict_state_layout[ + sizeof(int) + sizeof(void*) - 1) / sizeof(void*)) * sizeof(void*)) ? 1 : -1]; +typedef size_t (*ZSTD_rust_resetCCtxByAttachingCDictReset_f)( + void* context, const void* cdict, const void* params, + U64 pledgedSrcSize, int zbuff); +typedef void (*ZSTD_rust_resetCCtxByAttachingCDictAttach_f)( + void* context, const void* cdict); +typedef void (*ZSTD_rust_resetCCtxByAttachingCDictState_f)( + void* context, const void* cdict); +typedef struct { + void* callbackContext; + const void* cdict; + const void* params; + U64 pledgedSrcSize; + ZSTD_rust_resetCCtxByAttachingCDictReset_f reset; + ZSTD_rust_resetCCtxByAttachingCDictAttach_f attach; + ZSTD_rust_resetCCtxByAttachingCDictState_f copyDictState; + ZSTD_rust_resetCCtxByAttachingCDictState_f copyBlockState; + int zbuff; +} ZSTD_rust_resetCCtxByAttachingCDictState; +size_t ZSTD_rust_resetCCtxByAttachingCDict( + const ZSTD_rust_resetCCtxByAttachingCDictState* state); +typedef char ZSTD_rust_reset_cctx_by_attaching_cdict_state_layout[ + (offsetof(ZSTD_rust_resetCCtxByAttachingCDictState, callbackContext) == 0 + && offsetof(ZSTD_rust_resetCCtxByAttachingCDictState, cdict) + == sizeof(void*) + && offsetof(ZSTD_rust_resetCCtxByAttachingCDictState, params) + == 2 * sizeof(void*) + && offsetof(ZSTD_rust_resetCCtxByAttachingCDictState, pledgedSrcSize) + == 3 * sizeof(void*) + && offsetof(ZSTD_rust_resetCCtxByAttachingCDictState, reset) + == 3 * sizeof(void*) + sizeof(U64) + && offsetof(ZSTD_rust_resetCCtxByAttachingCDictState, zbuff) + == 3 * sizeof(void*) + sizeof(U64) + 4 * sizeof(void*) + && sizeof(ZSTD_rust_resetCCtxByAttachingCDictState) + == ((offsetof(ZSTD_rust_resetCCtxByAttachingCDictState, zbuff) + + sizeof(int) + sizeof(void*) - 1) / sizeof(void*)) + * sizeof(void*)) + ? 1 : -1]; /* The sequence-compression loop receives only the state it actually reads or * updates. In particular, neither ZSTD_CCtx nor a C function pointer crosses @@ -4286,6 +4323,90 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); } +static size_t ZSTD_rust_resetCCtx_byAttachingCDict_reset( + void* context, const void* cdictOpaque, const void* paramsOpaque, + U64 pledgedSrcSize, int zbuff) +{ + ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context; + const ZSTD_CDict* const cdict = (const ZSTD_CDict*)cdictOpaque; + ZSTD_CCtx_params params = *(const ZSTD_CCtx_params*)paramsOpaque; + ZSTD_compressionParameters adjusted_cdict_cParams = + cdict->matchState.cParams; + unsigned const windowLog = params.cParams.windowLog; + + DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%llu", + (unsigned long long)pledgedSrcSize); + assert(windowLog != 0); + /* Resize working context table params for input only, since the dict + * has its own tables. */ + if (cdict->matchState.dedicatedDictSearch) { + ZSTD_dedicatedDictSearch_revertCParams(&adjusted_cdict_cParams); + } + params.cParams = ZSTD_adjustCParams_internal( + adjusted_cdict_cParams, pledgedSrcSize, + cdict->dictContentSize, ZSTD_cpm_attachDict, + params.useRowMatchFinder); + params.cParams.windowLog = windowLog; + params.useRowMatchFinder = cdict->useRowMatchFinder; + { size_t const resetError = ZSTD_resetCCtx_internal( + cctx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, + ZSTDcrp_makeClean, + (ZSTD_buffered_policy_e)zbuff); + if (ZSTD_isError(resetError)) return resetError; + } + assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy); + return 0; +} + +static void ZSTD_rust_resetCCtx_byAttachingCDict_attach( + void* context, const void* cdictOpaque) +{ + ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context; + const ZSTD_CDict* const cdict = (const ZSTD_CDict*)cdictOpaque; + U32 const cdictEnd = (U32)(cdict->matchState.window.nextSrc + - cdict->matchState.window.base); + U32 const cdictLen = cdictEnd - cdict->matchState.window.dictLimit; + + if (cdictLen == 0) { + DEBUGLOG(4, "skipping attaching empty dictionary"); + return; + } + + DEBUGLOG(4, "attaching dictionary into context"); + cctx->blockState.matchState.dictMatchState = &cdict->matchState; + if (cctx->blockState.matchState.window.dictLimit < cdictEnd) { + cctx->blockState.matchState.window.nextSrc = + cctx->blockState.matchState.window.base + cdictEnd; + ZSTD_rust_windowClear( + (size_t)(cctx->blockState.matchState.window.nextSrc + - cctx->blockState.matchState.window.base), + &cctx->blockState.matchState.window.lowLimit, + &cctx->blockState.matchState.window.dictLimit); + } + /* loadedDictEnd is expressed within the active context referential. */ + cctx->blockState.matchState.loadedDictEnd = + cctx->blockState.matchState.window.dictLimit; +} + +static void ZSTD_rust_resetCCtx_byAttachingCDict_copy_dict_state( + void* context, const void* cdictOpaque) +{ + ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context; + const ZSTD_CDict* const cdict = (const ZSTD_CDict*)cdictOpaque; + cctx->dictID = cdict->dictID; + cctx->dictContentSize = cdict->dictContentSize; +} + +static void ZSTD_rust_resetCCtx_byAttachingCDict_copy_block_state( + void* context, const void* cdictOpaque) +{ + ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context; + const ZSTD_CDict* const cdict = (const ZSTD_CDict*)cdictOpaque; + ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, + sizeof(cdict->cBlockState)); +} + static size_t ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, @@ -4293,63 +4414,17 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) { - DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%llu", - (unsigned long long)pledgedSrcSize); - { - ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams; - unsigned const windowLog = params.cParams.windowLog; - assert(windowLog != 0); - /* Resize working context table params for input only, since the dict - * has its own tables. */ - /* pledgedSrcSize == 0 means 0! */ - - if (cdict->matchState.dedicatedDictSearch) { - ZSTD_dedicatedDictSearch_revertCParams(&adjusted_cdict_cParams); - } - - params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize, - cdict->dictContentSize, ZSTD_cpm_attachDict, - params.useRowMatchFinder); - params.cParams.windowLog = windowLog; - params.useRowMatchFinder = cdict->useRowMatchFinder; /* cdict overrides */ - FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, - /* loadedDictSize */ 0, - ZSTDcrp_makeClean, zbuff), ""); - assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy); - } - - { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc - - cdict->matchState.window.base); - const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; - if (cdictLen == 0) { - /* don't even attach dictionaries with no contents */ - DEBUGLOG(4, "skipping attaching empty dictionary"); - } else { - DEBUGLOG(4, "attaching dictionary into context"); - cctx->blockState.matchState.dictMatchState = &cdict->matchState; - - /* prep working match state so dict matches never have negative indices - * when they are translated to the working context's index space. */ - if (cctx->blockState.matchState.window.dictLimit < cdictEnd) { - cctx->blockState.matchState.window.nextSrc = - cctx->blockState.matchState.window.base + cdictEnd; - ZSTD_rust_windowClear( - (size_t)(cctx->blockState.matchState.window.nextSrc - - cctx->blockState.matchState.window.base), - &cctx->blockState.matchState.window.lowLimit, - &cctx->blockState.matchState.window.dictLimit); - } - /* loadedDictEnd is expressed within the referential of the active context */ - cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit; - } } - - cctx->dictID = cdict->dictID; - cctx->dictContentSize = cdict->dictContentSize; - - /* copy block state */ - ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); - - return 0; + ZSTD_rust_resetCCtxByAttachingCDictState state; + state.callbackContext = cctx; + state.cdict = cdict; + state.params = ¶ms; + state.pledgedSrcSize = pledgedSrcSize; + state.reset = ZSTD_rust_resetCCtx_byAttachingCDict_reset; + state.attach = ZSTD_rust_resetCCtx_byAttachingCDict_attach; + state.copyDictState = ZSTD_rust_resetCCtx_byAttachingCDict_copy_dict_state; + state.copyBlockState = ZSTD_rust_resetCCtx_byAttachingCDict_copy_block_state; + state.zbuff = (int)zbuff; + return ZSTD_rust_resetCCtxByAttachingCDict(&state); } static void ZSTD_copyCDictTableIntoCCtx(U32* dst, U32 const* src, size_t tableSize, diff --git a/rust/src/zstd_compress_dictionary.rs b/rust/src/zstd_compress_dictionary.rs index 0ad18f666..d47304c32 100644 --- a/rust/src/zstd_compress_dictionary.rs +++ b/rust/src/zstd_compress_dictionary.rs @@ -1405,6 +1405,94 @@ pub unsafe extern "C" fn ZSTD_rust_resetCCtxByCopyingCDict( 0 } +type ResetCCtxByAttachingCDictResetFn = + unsafe extern "C" fn(*mut c_void, *const c_void, *const c_void, u64, c_int) -> usize; +type ResetCCtxByAttachingCDictAttachFn = unsafe extern "C" fn(*mut c_void, *const c_void); +type ResetCCtxByAttachingCDictStateFn = unsafe extern "C" fn(*mut c_void, *const c_void); + +/// Projection for attaching a prepared CDict to a working CCtx. +/// +/// Rust owns reset, attach, and metadata-copy ordering. C retains parameter +/// adjustment, private window linkage, and the CCtx/CDict field layout. +#[repr(C)] +pub struct ZSTD_rust_resetCCtxByAttachingCDictState { + callback_context: *mut c_void, + cdict: *const c_void, + params: *const c_void, + pledged_src_size: u64, + reset: Option, + attach: Option, + copy_dict_state: Option, + copy_block_state: Option, + zbuff: c_int, +} + +const _: () = { + assert!(size_of::() == size_of::()); + assert!(size_of::() == size_of::()); + assert!(size_of::() == size_of::()); + assert!(offset_of!(ZSTD_rust_resetCCtxByAttachingCDictState, callback_context) == 0); + assert!(offset_of!(ZSTD_rust_resetCCtxByAttachingCDictState, cdict) == size_of::()); + assert!(offset_of!(ZSTD_rust_resetCCtxByAttachingCDictState, params) == 2 * size_of::()); + assert!( + offset_of!(ZSTD_rust_resetCCtxByAttachingCDictState, pledged_src_size) + == 3 * size_of::() + ); + assert!( + offset_of!(ZSTD_rust_resetCCtxByAttachingCDictState, reset) + == 3 * size_of::() + size_of::() + ); + assert!( + offset_of!(ZSTD_rust_resetCCtxByAttachingCDictState, zbuff) + == 3 * size_of::() + size_of::() + size_of::<[usize; 4]>() + ); + assert!( + size_of::() + == (offset_of!(ZSTD_rust_resetCCtxByAttachingCDictState, zbuff) + size_of::()) + .div_ceil(size_of::()) + * size_of::() + ); +}; + +/// Run the private CDict-attachment operation through C-owned layout callbacks. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_resetCCtxByAttachingCDict( + state: *const ZSTD_rust_resetCCtxByAttachingCDictState, +) -> usize { + if state.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + let state = unsafe { &*state }; + let (Some(reset), Some(attach), Some(copy_dict_state), Some(copy_block_state)) = ( + state.reset, + state.attach, + state.copy_dict_state, + state.copy_block_state, + ) else { + return ERROR(ZstdErrorCode::Generic); + }; + if state.callback_context.is_null() || state.cdict.is_null() || state.params.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + + unsafe { + let reset_error = reset( + state.callback_context, + state.cdict, + state.params, + state.pledged_src_size, + state.zbuff, + ); + if ERR_isError(reset_error) { + return reset_error; + } + attach(state.callback_context, state.cdict); + copy_dict_state(state.callback_context, state.cdict); + copy_block_state(state.callback_context, state.cdict); + } + 0 +} + /// Scalar projections for the public CDict query helpers. #[repr(C)] pub struct ZSTD_rust_cdictQueryState { @@ -2676,6 +2764,118 @@ mod tests { assert_eq!(probe.events, ["reset"]); } + #[derive(Default)] + struct ResetCCtxByAttachingCDictProbe { + events: Vec<&'static str>, + reset_result: usize, + cdict: *const c_void, + params: *const c_void, + pledged_src_size: u64, + zbuff: c_int, + } + + unsafe fn reset_cctx_by_attaching_cdict_probe( + context: *mut c_void, + ) -> &'static mut ResetCCtxByAttachingCDictProbe { + unsafe { &mut *context.cast::() } + } + + unsafe extern "C" fn reset_cctx_by_attaching_cdict_reset( + context: *mut c_void, + cdict: *const c_void, + params: *const c_void, + pledged_src_size: u64, + zbuff: c_int, + ) -> usize { + let probe = unsafe { reset_cctx_by_attaching_cdict_probe(context) }; + probe.events.push("reset"); + probe.cdict = cdict; + probe.params = params; + probe.pledged_src_size = pledged_src_size; + probe.zbuff = zbuff; + probe.reset_result + } + + unsafe extern "C" fn reset_cctx_by_attaching_cdict_attach( + context: *mut c_void, + _cdict: *const c_void, + ) { + unsafe { reset_cctx_by_attaching_cdict_probe(context) } + .events + .push("attach"); + } + + unsafe extern "C" fn reset_cctx_by_attaching_cdict_copy_dict_state( + context: *mut c_void, + _cdict: *const c_void, + ) { + unsafe { reset_cctx_by_attaching_cdict_probe(context) } + .events + .push("dict"); + } + + unsafe extern "C" fn reset_cctx_by_attaching_cdict_copy_block_state( + context: *mut c_void, + _cdict: *const c_void, + ) { + unsafe { reset_cctx_by_attaching_cdict_probe(context) } + .events + .push("block"); + } + + fn reset_cctx_by_attaching_cdict_test_state( + probe: &mut ResetCCtxByAttachingCDictProbe, + cdict: *const c_void, + params: *const c_void, + ) -> ZSTD_rust_resetCCtxByAttachingCDictState { + ZSTD_rust_resetCCtxByAttachingCDictState { + callback_context: (probe as *mut ResetCCtxByAttachingCDictProbe).cast(), + cdict, + params, + pledged_src_size: 123, + reset: Some(reset_cctx_by_attaching_cdict_reset), + attach: Some(reset_cctx_by_attaching_cdict_attach), + copy_dict_state: Some(reset_cctx_by_attaching_cdict_copy_dict_state), + copy_block_state: Some(reset_cctx_by_attaching_cdict_copy_block_state), + zbuff: 7, + } + } + + #[test] + fn reset_cctx_by_attaching_cdict_runs_callbacks_in_original_order() { + let mut probe = ResetCCtxByAttachingCDictProbe::default(); + let cdict = 0x4000usize as *const c_void; + let params = 0x3000usize as *const c_void; + let state = reset_cctx_by_attaching_cdict_test_state(&mut probe, cdict, params); + + let result = unsafe { ZSTD_rust_resetCCtxByAttachingCDict(&state) }; + + assert_eq!(result, 0); + assert_eq!(probe.events, ["reset", "attach", "dict", "block"]); + assert_eq!(probe.cdict, cdict); + assert_eq!(probe.params, params); + assert_eq!(probe.pledged_src_size, 123); + assert_eq!(probe.zbuff, 7); + } + + #[test] + fn reset_cctx_by_attaching_cdict_stops_after_reset_error() { + let mut probe = ResetCCtxByAttachingCDictProbe { + reset_result: ERROR(ZstdErrorCode::MemoryAllocation), + ..Default::default() + }; + let state = reset_cctx_by_attaching_cdict_test_state( + &mut probe, + 0x4000usize as *const c_void, + 0x3000usize as *const c_void, + ); + + let result = unsafe { ZSTD_rust_resetCCtxByAttachingCDict(&state) }; + + assert_eq!(result, ERROR(ZstdErrorCode::MemoryAllocation)); + assert_eq!(probe.events, ["reset"]); + } + unsafe fn cctx_policy_probe(context: *mut c_void) -> &'static mut CctxPolicyProbe { unsafe { &mut *context.cast::() } }