diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 7a595c1ee..759dd3e52 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1195,6 +1195,85 @@ typedef struct { size_t optimalTSize; size_t asanRedzoneSize; } ZSTD_rustCCtxWorkspaceSizing; +typedef struct { + size_t windowSize; + size_t blockSize; + size_t maxNbSeq; + size_t buffInSize; + size_t buffOutSize; + size_t maxNbLdmSeq; + size_t maxNbExternalSeq; + size_t neededSpace; + int needsIndexReset; +} ZSTD_rustCCtxResetPlan; +typedef char ZSTD_rust_cctx_workspace_sizing_layout[ + (offsetof(ZSTD_rustCCtxWorkspaceSizing, cctxSize) == 0 + && offsetof(ZSTD_rustCCtxWorkspaceSizing, compressedBlockStateSize) + == sizeof(size_t) + && offsetof(ZSTD_rustCCtxWorkspaceSizing, seqDefSize) + == 2 * sizeof(size_t) + && offsetof(ZSTD_rustCCtxWorkspaceSizing, rawSeqSize) + == 3 * sizeof(size_t) + && offsetof(ZSTD_rustCCtxWorkspaceSizing, externalSequenceSize) + == 4 * sizeof(size_t) + && offsetof(ZSTD_rustCCtxWorkspaceSizing, tmpWorkspaceSize) + == 5 * sizeof(size_t) + && offsetof(ZSTD_rustCCtxWorkspaceSizing, wildcopyOverlength) + == 6 * sizeof(size_t) + && offsetof(ZSTD_rustCCtxWorkspaceSizing, matchTSize) + == 7 * sizeof(size_t) + && offsetof(ZSTD_rustCCtxWorkspaceSizing, optimalTSize) + == 8 * sizeof(size_t) + && offsetof(ZSTD_rustCCtxWorkspaceSizing, asanRedzoneSize) + == 9 * sizeof(size_t) + && sizeof(ZSTD_rustCCtxWorkspaceSizing) == 10 * sizeof(size_t)) + ? 1 : -1]; +typedef struct { + ZSTD_compressionParameters cParams; + int ldmEnable; + U32 ldmHashLog; + U32 ldmBucketSizeLog; + U32 ldmMinMatchLength; + int isStatic; + int useRowMatchFinder; + int inBufferBuffered; + int outBufferBuffered; + int useSequenceProducer; + int initialized; + int indexTooClose; + int dictTooBig; + U64 pledgedSrcSize; + size_t maxBlockSize; + const ZSTD_rustCCtxWorkspaceSizing* sizing; + ZSTD_rustCCtxResetPlan* plan; +} ZSTD_rustCCtxResetState; +typedef char ZSTD_rust_cctx_reset_state_layout[ + (offsetof(ZSTD_rustCCtxResetState, cParams) == 0 + && offsetof(ZSTD_rustCCtxResetState, ldmEnable) + == sizeof(ZSTD_compressionParameters) + && offsetof(ZSTD_rustCCtxResetState, pledgedSrcSize) + > offsetof(ZSTD_rustCCtxResetState, dictTooBig) + && offsetof(ZSTD_rustCCtxResetState, maxBlockSize) + == offsetof(ZSTD_rustCCtxResetState, pledgedSrcSize) + sizeof(U64) + && offsetof(ZSTD_rustCCtxResetState, sizing) + == offsetof(ZSTD_rustCCtxResetState, maxBlockSize) + sizeof(size_t) + && offsetof(ZSTD_rustCCtxResetState, plan) + == offsetof(ZSTD_rustCCtxResetState, sizing) + sizeof(void*) + && sizeof(ZSTD_rustCCtxResetState) + == offsetof(ZSTD_rustCCtxResetState, plan) + sizeof(void*)) + ? 1 : -1]; +typedef char ZSTD_rust_cctx_reset_plan_layout[ + (offsetof(ZSTD_rustCCtxResetPlan, windowSize) == 0 + && offsetof(ZSTD_rustCCtxResetPlan, blockSize) == sizeof(size_t) + && offsetof(ZSTD_rustCCtxResetPlan, maxNbSeq) == 2 * sizeof(size_t) + && offsetof(ZSTD_rustCCtxResetPlan, buffInSize) == 3 * sizeof(size_t) + && offsetof(ZSTD_rustCCtxResetPlan, buffOutSize) == 4 * sizeof(size_t) + && offsetof(ZSTD_rustCCtxResetPlan, maxNbLdmSeq) == 5 * sizeof(size_t) + && offsetof(ZSTD_rustCCtxResetPlan, maxNbExternalSeq) == 6 * sizeof(size_t) + && offsetof(ZSTD_rustCCtxResetPlan, neededSpace) == 7 * sizeof(size_t) + && offsetof(ZSTD_rustCCtxResetPlan, needsIndexReset) == 8 * sizeof(size_t) + && sizeof(ZSTD_rustCCtxResetPlan) == 9 * sizeof(size_t)) + ? 1 : -1]; size_t ZSTD_rust_estimateCCtxWorkspaceSize( ZSTD_compressionParameters cParams, int ldmEnable, U32 ldmHashLog, U32 ldmBucketSizeLog, @@ -1204,6 +1283,7 @@ size_t ZSTD_rust_estimateCCtxWorkspaceSize( U64 pledgedSrcSize, int useSequenceProducer, size_t maxBlockSize, const ZSTD_rustCCtxWorkspaceSizing* sizing); +size_t ZSTD_rust_planCCtxReset(const ZSTD_rustCCtxResetState* state); size_t ZSTD_rust_maxEstimateCCtxSize(size_t estimate0, size_t estimate1, size_t estimate2, size_t estimate3); ZSTD_inBuffer ZSTD_rust_inBufferForEndFlush(int inBufferMode, @@ -3115,12 +3195,6 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, forCCtx, &sizing); } -/* Helper function for calculating memory requirements. - * Gives a tighter bound than ZSTD_sequenceBound() by taking minMatch into account. */ -static size_t ZSTD_maxNbSeq(size_t blockSize, unsigned minMatch, int useSequenceProducer) { - return ZSTD_rust_params_maxNbSeq(blockSize, minMatch, useSequenceProducer); -} - static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( const ZSTD_compressionParameters* cParams, const ldmParams_t* ldmParams, @@ -3613,28 +3687,56 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, assert(params->ldmParams.hashRateLog < 32); } - { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize)); - size_t const blockSize = MIN(params->maxBlockSize, windowSize); - size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, ZSTD_hasExtSeqProd(params)); - size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered) - ? ZSTD_compressBound(blockSize) + 1 - : 0; - size_t const buffInSize = (zbuff == ZSTDb_buffered && params->inBufferMode == ZSTD_bm_buffered) - ? windowSize + blockSize - : 0; - size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize); + { ZSTD_rustCCtxWorkspaceSizing const sizing = { + sizeof(ZSTD_CCtx), + sizeof(ZSTD_compressedBlockState_t), + sizeof(SeqDef), + sizeof(rawSeq), + sizeof(ZSTD_Sequence), + TMP_WORKSPACE_SIZE, + WILDCOPY_OVERLENGTH, + sizeof(ZSTD_match_t), + sizeof(ZSTD_optimal_t), + ZSTD_RUST_ASAN_REDZONE_SIZE + }; + ZSTD_rustCCtxResetPlan resetPlan; + ZSTD_rustCCtxResetState resetState; + size_t blockSize; + size_t maxNbSeq; + size_t buffOutSize; + size_t buffInSize; + size_t maxNbLdmSeq; + size_t neededSpace; + ZSTD_indexResetPolicy_e needsIndexReset; + resetState.cParams = params->cParams; + resetState.ldmEnable = (int)params->ldmParams.enableLdm; + resetState.ldmHashLog = params->ldmParams.hashLog; + resetState.ldmBucketSizeLog = params->ldmParams.bucketSizeLog; + resetState.ldmMinMatchLength = params->ldmParams.minMatchLength; + resetState.isStatic = zc->staticSize != 0; + resetState.useRowMatchFinder = (int)params->useRowMatchFinder; + resetState.inBufferBuffered = + zbuff == ZSTDb_buffered && params->inBufferMode == ZSTD_bm_buffered; + resetState.outBufferBuffered = + zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered; + resetState.useSequenceProducer = ZSTD_hasExtSeqProd(params); + resetState.initialized = zc->initialized != 0; + resetState.indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window); + resetState.dictTooBig = ZSTD_dictTooBig(loadedDictSize); + resetState.pledgedSrcSize = pledgedSrcSize; + resetState.maxBlockSize = params->maxBlockSize; + resetState.sizing = &sizing; + resetState.plan = &resetPlan; - int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window); - int const dictTooBig = ZSTD_dictTooBig(loadedDictSize); - ZSTD_indexResetPolicy_e needsIndexReset = - (indexTooClose || dictTooBig || !zc->initialized) ? ZSTDirp_reset : ZSTDirp_continue; + FORWARD_IF_ERROR(ZSTD_rust_planCCtxReset(&resetState), "cctx reset plan failed!"); - size_t const neededSpace = - ZSTD_estimateCCtxSize_usingCCtxParams_internal( - ¶ms->cParams, ¶ms->ldmParams, zc->staticSize != 0, params->useRowMatchFinder, - buffInSize, buffOutSize, pledgedSrcSize, ZSTD_hasExtSeqProd(params), params->maxBlockSize); - - FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!"); + blockSize = resetPlan.blockSize; + maxNbSeq = resetPlan.maxNbSeq; + buffOutSize = resetPlan.buffOutSize; + buffInSize = resetPlan.buffInSize; + maxNbLdmSeq = resetPlan.maxNbLdmSeq; + neededSpace = resetPlan.neededSpace; + needsIndexReset = (ZSTD_indexResetPolicy_e)resetPlan.needsIndexReset; if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0); @@ -3643,7 +3745,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); int resizeWorkspace = workspaceTooSmall || workspaceWasteful; DEBUGLOG(4, "Need %zu B workspace", neededSpace); - DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); + DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", resetPlan.windowSize, blockSize); if (resizeWorkspace) { DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", @@ -3718,7 +3820,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, /* reserve space for block-level external sequences */ if (ZSTD_hasExtSeqProd(params)) { - size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); + size_t const maxNbExternalSeq = resetPlan.maxNbExternalSeq; zc->extSeqBufCapacity = maxNbExternalSeq; zc->extSeqBuf = (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned64(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence)); diff --git a/rust/README.md b/rust/README.md index 20e014515..46cb01e57 100644 --- a/rust/README.md +++ b/rust/README.md @@ -167,9 +167,10 @@ static-CDict workspace construction and dictionary-content allocation/loading, and advanced-CDict dictionary-content loading remain in C. Rust now owns advanced-CDict custom-memory validation, workspace-size query/allocation, allocation/create/init cleanup ordering, the CCtx workspace-size formula, and -the match-state reset policy/order; C retains private layout-size inputs, -workspace layout, and allocator callbacks. Private CCtx reset/matchfinder/ -workspace operations and codec/adaptive-policy +the scalar CCtx-reset plan and match-state reset policy/order; C retains +private layout-size inputs, workspace resize/layout, private field publication, +and allocator callbacks. Private CCtx reset/matchfinder/workspace operations +and codec/adaptive-policy callbacks remain in C. CDict initialization ordering and scalar publication, shared compression-begin dictionary selection, CDict reset attach-versus-copy selection, and CDict-begin parameter selection, initialization ordering, and diff --git a/rust/src/zstd_compress.rs b/rust/src/zstd_compress.rs index 62b1a0b78..56c4eb86d 100644 --- a/rust/src/zstd_compress.rs +++ b/rust/src/zstd_compress.rs @@ -5849,6 +5849,177 @@ pub unsafe extern "C" fn ZSTD_rust_estimateCCtxWorkspaceSize( ) } +/// Scalar outputs computed while resetting a private C compression context. +/// +/// C still owns workspace resizing and all private reservations. Rust owns +/// this policy projection so the reset formula is testable without exposing +/// `ZSTD_CCtx` or `ZSTD_cwksp` layouts across the ABI. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ZSTD_rustCCtxResetPlan { + pub windowSize: usize, + pub blockSize: usize, + pub maxNbSeq: usize, + pub buffInSize: usize, + pub buffOutSize: usize, + pub maxNbLdmSeq: usize, + pub maxNbExternalSeq: usize, + pub neededSpace: usize, + pub needsIndexReset: c_int, +} + +/// Scalar inputs needed to plan a private C compression-context reset. +/// +/// The C adapter supplies resolved parameters, private object sizes, and the +/// already-reduced index/dictionary predicates. No private C pointer or +/// layout is passed through this record. +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub struct ZSTD_rustCCtxResetState { + pub cParams: ZSTD_compressionParameters, + pub ldmEnable: c_int, + pub ldmHashLog: c_uint, + pub ldmBucketSizeLog: c_uint, + pub ldmMinMatchLength: c_uint, + pub isStatic: c_int, + pub useRowMatchFinder: c_int, + pub inBufferBuffered: c_int, + pub outBufferBuffered: c_int, + pub useSequenceProducer: c_int, + pub initialized: c_int, + pub indexTooClose: c_int, + pub dictTooBig: c_int, + pub pledgedSrcSize: u64, + pub maxBlockSize: usize, + pub sizing: *const ZSTD_rustCCtxWorkspaceSizing, + pub plan: *mut ZSTD_rustCCtxResetPlan, +} + +const _: () = { + assert!(size_of::() == 9 * size_of::()); + assert!(offset_of!(ZSTD_rustCCtxResetPlan, windowSize) == 0); + assert!(offset_of!(ZSTD_rustCCtxResetPlan, blockSize) == size_of::()); + assert!(offset_of!(ZSTD_rustCCtxResetPlan, maxNbSeq) == 2 * size_of::()); + assert!(offset_of!(ZSTD_rustCCtxResetPlan, buffInSize) == 3 * size_of::()); + assert!(offset_of!(ZSTD_rustCCtxResetPlan, buffOutSize) == 4 * size_of::()); + assert!(offset_of!(ZSTD_rustCCtxResetPlan, maxNbLdmSeq) == 5 * size_of::()); + assert!(offset_of!(ZSTD_rustCCtxResetPlan, maxNbExternalSeq) == 6 * size_of::()); + assert!(offset_of!(ZSTD_rustCCtxResetPlan, neededSpace) == 7 * size_of::()); + assert!(offset_of!(ZSTD_rustCCtxResetPlan, needsIndexReset) == size_of::<[usize; 8]>()); + assert!(offset_of!(ZSTD_rustCCtxResetState, cParams) == 0); + assert!( + offset_of!(ZSTD_rustCCtxResetState, ldmEnable) == size_of::() + ); + assert!( + offset_of!(ZSTD_rustCCtxResetState, pledgedSrcSize) + > offset_of!(ZSTD_rustCCtxResetState, dictTooBig) + ); + assert!( + offset_of!(ZSTD_rustCCtxResetState, maxBlockSize) + == offset_of!(ZSTD_rustCCtxResetState, pledgedSrcSize) + size_of::() + ); + assert!( + offset_of!(ZSTD_rustCCtxResetState, sizing) + == offset_of!(ZSTD_rustCCtxResetState, maxBlockSize) + size_of::() + ); + assert!( + offset_of!(ZSTD_rustCCtxResetState, plan) + == offset_of!(ZSTD_rustCCtxResetState, sizing) + size_of::() + ); + assert!( + size_of::() + == offset_of!(ZSTD_rustCCtxResetState, plan) + size_of::() + ); +}; + +/// Compute the scalar portion of `ZSTD_resetCCtx_internal()`. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_planCCtxReset(state: *const ZSTD_rustCCtxResetState) -> usize { + if state.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + let state = unsafe { &*state }; + if state.sizing.is_null() || state.plan.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + + let ldm_enabled = state.ldmEnable == ZSTD_RUST_PS_ENABLE; + if ldm_enabled && state.ldmMinMatchLength == 0 { + return ERROR(ZstdErrorCode::Generic); + } + + let window_limit = 1u64 + .checked_shl(state.cParams.windowLog) + .unwrap_or(u64::MAX); + let window_size = (window_limit + .min(state.pledgedSrcSize) + .min(usize::MAX as u64) as usize) + .max(1); + let block_size = state.maxBlockSize.min(window_size); + let max_nb_seq = ZSTD_rust_params_maxNbSeq( + block_size, + state.cParams.minMatch, + state.useSequenceProducer, + ); + let buff_out_size = if state.outBufferBuffered != 0 { + crate::zstd_compress_api::ZSTD_compressBound(block_size).wrapping_add(1) + } else { + 0 + }; + let buff_in_size = if state.inBufferBuffered != 0 { + window_size.wrapping_add(block_size) + } else { + 0 + }; + let max_nb_ldm_seq = if ldm_enabled { + block_size / state.ldmMinMatchLength as usize + } else { + 0 + }; + let max_nb_external_seq = if state.useSequenceProducer != 0 { + crate::zstd_compress_api::ZSTD_sequenceBound(block_size) + } else { + 0 + }; + let needed_space = unsafe { + ZSTD_rust_estimateCCtxWorkspaceSize( + state.cParams, + state.ldmEnable, + state.ldmHashLog, + state.ldmBucketSizeLog, + state.ldmMinMatchLength, + state.isStatic, + state.useRowMatchFinder, + buff_in_size, + buff_out_size, + state.pledgedSrcSize, + state.useSequenceProducer, + state.maxBlockSize, + state.sizing, + ) + }; + if ERR_isError(needed_space) { + return needed_space; + } + + unsafe { + *state.plan = ZSTD_rustCCtxResetPlan { + windowSize: window_size, + blockSize: block_size, + maxNbSeq: max_nb_seq, + buffInSize: buff_in_size, + buffOutSize: buff_out_size, + maxNbLdmSeq: max_nb_ldm_seq, + maxNbExternalSeq: max_nb_external_seq, + neededSpace: needed_space, + needsIndexReset: c_int::from( + state.indexTooClose != 0 || state.dictTooBig != 0 || state.initialized == 0, + ), + }; + } + 0 +} + #[inline] fn max_estimate_cctx_size( estimate0: usize, @@ -13668,6 +13839,142 @@ mod tests { } } + fn cctx_reset_test_state( + sizing: &ZSTD_rustCCtxWorkspaceSizing, + plan: &mut ZSTD_rustCCtxResetPlan, + pledged_src_size: u64, + max_block_size: usize, + ldm_enable: c_int, + ldm_min_match_length: u32, + in_buffer_buffered: c_int, + out_buffer_buffered: c_int, + use_sequence_producer: c_int, + initialized: c_int, + index_too_close: c_int, + dict_too_big: c_int, + ) -> ZSTD_rustCCtxResetState { + ZSTD_rustCCtxResetState { + cParams: cctx_workspace_test_cparams(), + ldmEnable: ldm_enable, + ldmHashLog: 4, + ldmBucketSizeLog: 2, + ldmMinMatchLength: ldm_min_match_length, + isStatic: 0, + useRowMatchFinder: ZSTD_RUST_PS_DISABLE, + inBufferBuffered: in_buffer_buffered, + outBufferBuffered: out_buffer_buffered, + useSequenceProducer: use_sequence_producer, + initialized, + indexTooClose: index_too_close, + dictTooBig: dict_too_big, + pledgedSrcSize: pledged_src_size, + maxBlockSize: max_block_size, + sizing, + plan, + } + } + + #[test] + fn cctx_reset_plan_computes_window_buffers_and_sequence_capacities() { + let sizing = cctx_workspace_test_sizing(); + let mut plan = ZSTD_rustCCtxResetPlan::default(); + let state = cctx_reset_test_state( + &sizing, + &mut plan, + 1000, + 512, + ZSTD_RUST_PS_DISABLE, + 0, + 1, + 1, + 1, + 1, + 0, + 0, + ); + + assert_eq!(unsafe { ZSTD_rust_planCCtxReset(&state) }, 0); + assert_eq!(plan.windowSize, 1000); + assert_eq!(plan.blockSize, 512); + assert_eq!(plan.maxNbSeq, 512 / 3); + assert_eq!(plan.buffInSize, 1000 + 512); + assert_eq!( + plan.buffOutSize, + crate::zstd_compress_api::ZSTD_compressBound(512) + 1 + ); + assert_eq!( + plan.maxNbExternalSeq, + crate::zstd_compress_api::ZSTD_sequenceBound(512) + ); + assert_eq!(plan.maxNbLdmSeq, 0); + assert_eq!(plan.needsIndexReset, 0); + assert!(!ERR_isError(plan.neededSpace)); + assert_ne!(plan.neededSpace, 0); + } + + #[test] + fn cctx_reset_plan_tracks_ldm_and_index_reset_policy() { + let sizing = cctx_workspace_test_sizing(); + let mut plan = ZSTD_rustCCtxResetPlan::default(); + let mut state = cctx_reset_test_state( + &sizing, + &mut plan, + u64::MAX, + 2048, + ZSTD_RUST_PS_ENABLE, + 64, + 0, + 0, + 0, + 0, + 0, + 0, + ); + + assert_eq!(unsafe { ZSTD_rust_planCCtxReset(&state) }, 0); + assert_eq!(plan.windowSize, 1 << 10); + assert_eq!(plan.blockSize, 1 << 10); + assert_eq!(plan.maxNbLdmSeq, (1 << 10) / 64); + assert_eq!(plan.buffInSize, 0); + assert_eq!(plan.buffOutSize, 0); + assert_eq!(plan.needsIndexReset, 1); + + state.initialized = 1; + state.indexTooClose = 1; + assert_eq!(unsafe { ZSTD_rust_planCCtxReset(&state) }, 0); + assert_eq!(plan.needsIndexReset, 1); + + state.indexTooClose = 0; + state.dictTooBig = 1; + assert_eq!(unsafe { ZSTD_rust_planCCtxReset(&state) }, 0); + assert_eq!(plan.needsIndexReset, 1); + } + + #[test] + fn cctx_reset_plan_rejects_unadjusted_enabled_ldm() { + let sizing = cctx_workspace_test_sizing(); + let mut plan = ZSTD_rustCCtxResetPlan::default(); + let state = cctx_reset_test_state( + &sizing, + &mut plan, + 1000, + 512, + ZSTD_RUST_PS_ENABLE, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + ); + + assert_eq!( + unsafe { ZSTD_rust_planCCtxReset(&state) }, + ERROR(ZstdErrorCode::Generic) + ); + } + #[test] fn estimate_cctx_workspace_size_keeps_static_and_buffer_components_separate() { let sizing = cctx_workspace_test_sizing();