From 4409ef1e163977edf5650b6ec431f05dad46a6a8 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Mon, 20 Jul 2026 09:49:37 +0200 Subject: [PATCH] refactor(compress): move CCtx estimate tier policy to Rust Move the fixed source-size tier traversal used by ZSTD_estimateCCtxSize_internal() into Rust. C retains the authoritative parameter lookup and per-tier workspace estimator behind callbacks, while Rust owns the tier ordering and maximum-selection policy. The projection includes ABI layout assertions and tests for callback order, level propagation, and error-valued maxima. Test Plan: - `ulimit -v 41943040; cargo +nightly fmt --manifest-path rust/Cargo.toml --all -- --check` - `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings` - `ulimit -v 41943040; make -j1` - `ulimit -v 41943040; make -j1 -C tests test` - `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings` - `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/cli/Cargo.toml --all-targets` The integrated checks ran at the combined working-tree tip under a serial 40 GiB virtual-memory limit. Standalone root Rust unit linking remains unavailable because the crate imports C-owned bridge symbols without a Cargo build/link setup. --- lib/compress/zstd_compress.c | 54 +++++++++++++--- rust/src/zstd_compress.rs | 117 +++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 10 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 7ad728ab0..68d568d0c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1880,6 +1880,28 @@ size_t ZSTD_rust_estimateCCtxWorkspaceSize( size_t ZSTD_rust_planCCtxReset(const ZSTD_rustCCtxResetState* state); size_t ZSTD_rust_maxEstimateCCtxSize(size_t estimate0, size_t estimate1, size_t estimate2, size_t estimate3); +typedef ZSTD_compressionParameters (*ZSTD_rust_estimateCCtxSizeGetCParams_f)( + void* context, int compressionLevel, U64 srcSizeHint); +typedef size_t (*ZSTD_rust_estimateCCtxSizeEstimate_f)( + void* context, ZSTD_compressionParameters cParams); +typedef struct { + void* callbackContext; + int compressionLevel; + ZSTD_rust_estimateCCtxSizeGetCParams_f getCParams; + ZSTD_rust_estimateCCtxSizeEstimate_f estimate; +} ZSTD_rust_estimateCCtxSizeState; +typedef char ZSTD_rust_estimate_cctx_size_state_layout[ + (offsetof(ZSTD_rust_estimateCCtxSizeState, callbackContext) == 0 + && offsetof(ZSTD_rust_estimateCCtxSizeState, compressionLevel) + == sizeof(void*) + && offsetof(ZSTD_rust_estimateCCtxSizeState, getCParams) + == 2 * sizeof(void*) + && offsetof(ZSTD_rust_estimateCCtxSizeState, estimate) + == 3 * sizeof(void*) + && sizeof(ZSTD_rust_estimateCCtxSizeState) == 4 * sizeof(void*)) + ? 1 : -1]; +size_t ZSTD_rust_estimateCCtxSizeInternal( + const ZSTD_rust_estimateCCtxSizeState* state); ZSTD_inBuffer ZSTD_rust_inBufferForEndFlush(int inBufferMode, const void* expectedSrc, size_t expectedSize, @@ -4114,18 +4136,30 @@ size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) } } +static ZSTD_compressionParameters ZSTD_rust_estimateCCtxSize_getCParams( + void* context, int compressionLevel, U64 srcSizeHint) +{ + (void)context; + return ZSTD_getCParams_internal( + compressionLevel, srcSizeHint, 0, ZSTD_cpm_noAttachDict); +} + +static size_t ZSTD_rust_estimateCCtxSize_estimate( + void* context, ZSTD_compressionParameters cParams) +{ + (void)context; + return ZSTD_estimateCCtxSize_usingCParams(cParams); +} + static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel) { - int tier = 0; - size_t estimates[4]; - static const unsigned long long srcSizeTiers[4] = {16 KB, 128 KB, 256 KB, ZSTD_CONTENTSIZE_UNKNOWN}; - for (; tier < 4; ++tier) { - /* Choose the set of cParams for a given level across all srcSizes that give the largest cctxSize */ - ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeTiers[tier], 0, ZSTD_cpm_noAttachDict); - estimates[tier] = ZSTD_estimateCCtxSize_usingCParams(cParams); - } - return ZSTD_rust_maxEstimateCCtxSize( - estimates[0], estimates[1], estimates[2], estimates[3]); + ZSTD_rust_estimateCCtxSizeState const state = { + NULL, + compressionLevel, + ZSTD_rust_estimateCCtxSize_getCParams, + ZSTD_rust_estimateCCtxSize_estimate + }; + return ZSTD_rust_estimateCCtxSizeInternal(&state); } size_t ZSTD_estimateCCtxSize(int compressionLevel) diff --git a/rust/src/zstd_compress.rs b/rust/src/zstd_compress.rs index 81decb941..5e92636ef 100644 --- a/rust/src/zstd_compress.rs +++ b/rust/src/zstd_compress.rs @@ -8588,6 +8588,57 @@ fn max_estimate_cctx_size( estimate0.max(estimate1).max(estimate2).max(estimate3) } +type EstimateCCtxSizeGetCParamsFn = + unsafe extern "C" fn(*mut c_void, c_int, u64) -> ZSTD_compressionParameters; +type EstimateCCtxSizeEstimateFn = + unsafe extern "C" fn(*mut c_void, ZSTD_compressionParameters) -> usize; + +/// Rust owns the fixed source-size tier policy for CCtx estimates. C keeps +/// the configuration-sensitive parameter lookup and workspace estimator behind +/// callbacks, so this seam does not expose any private context layout. +#[repr(C)] +pub struct ZSTD_rust_estimateCCtxSizeState { + callback_context: *mut c_void, + compression_level: c_int, + get_c_params: EstimateCCtxSizeGetCParamsFn, + estimate: EstimateCCtxSizeEstimateFn, +} + +const _: () = { + assert!(offset_of!(ZSTD_rust_estimateCCtxSizeState, callback_context) == 0); + assert!(offset_of!(ZSTD_rust_estimateCCtxSizeState, compression_level) == size_of::()); + assert!(offset_of!(ZSTD_rust_estimateCCtxSizeState, get_c_params) == 2 * size_of::()); + assert!(offset_of!(ZSTD_rust_estimateCCtxSizeState, estimate) == 3 * size_of::()); + assert!(size_of::() == 4 * size_of::()); +}; + +/// Evaluate the four source-size tiers used by `ZSTD_estimateCCtxSize()` and +/// return the largest raw result, including size_t-encoded error values. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_estimateCCtxSizeInternal( + state: *const ZSTD_rust_estimateCCtxSizeState, +) -> usize { + if state.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + + let state = unsafe { &*state }; + let source_size_tiers = [16 * 1024, 128 * 1024, 256 * 1024, ZSTD_CONTENTSIZE_UNKNOWN]; + let mut estimates = [0usize; 4]; + for (estimate, &source_size_hint) in estimates.iter_mut().zip(source_size_tiers.iter()) { + let c_params = unsafe { + (state.get_c_params)( + state.callback_context, + state.compression_level, + source_size_hint, + ) + }; + *estimate = unsafe { (state.estimate)(state.callback_context, c_params) }; + } + + max_estimate_cctx_size(estimates[0], estimates[1], estimates[2], estimates[3]) +} + /// Return the largest raw estimate, including any C size_t error values. #[no_mangle] pub extern "C" fn ZSTD_rust_maxEstimateCCtxSize( @@ -10177,6 +10228,41 @@ mod tests { const ZSTD_BTOPT: c_int = 7; const ZSTD_BTULTRA2: c_int = 9; + struct EstimateCCtxSizeTestContext { + compression_levels: Vec, + source_size_hints: Vec, + parameter_tiers: Vec, + } + + unsafe extern "C" fn estimate_cctx_size_test_get_c_params( + context: *mut c_void, + compression_level: c_int, + source_size_hint: u64, + ) -> ZSTD_compressionParameters { + let context = unsafe { &mut *(context.cast::()) }; + context.compression_levels.push(compression_level); + context.source_size_hints.push(source_size_hint); + ZSTD_compressionParameters { + windowLog: context.source_size_hints.len() as u32, + ..ZSTD_compressionParameters::default() + } + } + + unsafe extern "C" fn estimate_cctx_size_test_estimate( + context: *mut c_void, + c_params: ZSTD_compressionParameters, + ) -> usize { + let context = unsafe { &mut *(context.cast::()) }; + context.parameter_tiers.push(c_params.windowLog); + match c_params.windowLog { + 1 => 17, + 2 => 91, + 3 => 23, + 4 => 67, + _ => ERROR(ZstdErrorCode::Generic), + } + } + struct GenerateSequencesTestContext { events: Vec<&'static str>, target_c_block_size: c_int, @@ -19906,6 +19992,37 @@ mod tests { ); } + #[test] + fn estimate_cctx_size_internal_preserves_tier_order_and_max_policy() { + let mut context = EstimateCCtxSizeTestContext { + compression_levels: Vec::new(), + source_size_hints: Vec::new(), + parameter_tiers: Vec::new(), + }; + let state = ZSTD_rust_estimateCCtxSizeState { + callback_context: (&mut context as *mut EstimateCCtxSizeTestContext).cast(), + compression_level: 7, + get_c_params: estimate_cctx_size_test_get_c_params, + estimate: estimate_cctx_size_test_estimate, + }; + + assert_eq!(unsafe { ZSTD_rust_estimateCCtxSizeInternal(&state) }, 91); + assert_eq!(context.compression_levels, vec![7; 4]); + assert_eq!( + context.source_size_hints, + vec![16 * 1024, 128 * 1024, 256 * 1024, ZSTD_CONTENTSIZE_UNKNOWN] + ); + assert_eq!(context.parameter_tiers, vec![1, 2, 3, 4]); + } + + #[test] + fn estimate_cctx_size_internal_rejects_a_null_state() { + assert_eq!( + unsafe { ZSTD_rust_estimateCCtxSizeInternal(ptr::null()) }, + ERROR(ZstdErrorCode::Generic) + ); + } + #[test] fn public_one_shot_abi_is_c_compatible() { let entry: unsafe extern "C" fn(*mut c_void, usize, *const c_void, usize, c_int) -> usize =