From 2125f9822df6cb378ba886a450e07844c3774596 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Sun, 19 Jul 2026 21:56:28 +0200 Subject: [PATCH] feat(compress): move static CDict sizing into Rust Route ZSTD_initStaticCDict() workspace-size calculation through the Rust CDict sizing policy, including the by-reference/by-copy, resolved row match-finder, and dedicated-search branches. Keep private layout constants at the C boundary and retain C ownership of static workspace construction and dictionary initialization. Remove the obsolete C match-state sizing adapter now that no C path calls it, avoiding a dead implementation after the sizing migration. Test Plan: - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040 && make -j1 - ulimit -v 41943040 && make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s - ulimit -v 41943040 && make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests - ulimit -v 41943040 && cargo fmt --manifest-path rust/Cargo.toml -- --check - git diff --check --- lib/compress/zstd_compress.c | 58 +++++++------------------------- rust/src/zstd_compress_params.rs | 25 +++++++++++++- 2 files changed, 37 insertions(+), 46 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index fab1f7eda..d1eafb43d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1796,17 +1796,6 @@ ZSTD_RUST_CCTX_PARAMS_ASSERT(ZSTD_rust_cctx_params_repcode_offset, offsetof(ZSTD_CCtx_params, searchForExternalRepcodes) == (sizeof(void*) == 8 ? 216 : 176)); #undef ZSTD_RUST_CCTX_PARAMS_ASSERT -typedef struct { - U32 hashLog3Max; - size_t matchTSize; - size_t optimalTSize; - size_t asanRedzoneSize; -} ZSTD_rustMatchStateSizing; -size_t ZSTD_rust_params_estimateMatchStateSize( - ZSTD_compressionParameters cParams, int useRowMatchFinder, - int enableDedicatedDictSearch, U32 forCCtx, - const ZSTD_rustMatchStateSizing* sizing); - typedef struct { size_t cdictSize; size_t hufWorkspaceSize; @@ -3586,33 +3575,6 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( ZSTD_getCParamsExclusionMask()); } -static size_t -ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, - const ZSTD_ParamSwitch_e useRowMatchFinder, - const int enableDedicatedDictSearch, - const U32 forCCtx) -{ - ZSTD_rustMatchStateSizing sizing; - sizing.hashLog3Max = ZSTD_HASHLOG3_MAX; - sizing.matchTSize = sizeof(ZSTD_match_t); - sizing.optimalTSize = sizeof(ZSTD_optimal_t); - sizing.asanRedzoneSize = ZSTD_RUST_ASAN_REDZONE_SIZE; - - /* The Rust leaf hardcodes the frozen format bounds and workspace rules; - * keep them checked against the private headers here. */ - ZSTD_STATIC_ASSERT(MaxML == 52 && MaxLL == 35 && MaxOff == 31); - ZSTD_STATIC_ASSERT(Litbits == 8 && ZSTD_OPT_SIZE == 4099); - ZSTD_STATIC_ASSERT(ZSTD_CWKSP_ALIGNMENT_BYTES == 64); - /* tables are guaranteed to be sized in multiples of 64 bytes (or 16 uint32_t) */ - ZSTD_STATIC_ASSERT(ZSTD_HASHLOG_MIN >= 4 && ZSTD_WINDOWLOG_MIN >= 4 && ZSTD_CHAINLOG_MIN >= 4); - assert(useRowMatchFinder != ZSTD_ps_auto); - - return ZSTD_rust_params_estimateMatchStateSize(*cParams, - (int)useRowMatchFinder, - enableDedicatedDictSearch, - forCCtx, &sizing); -} - static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( const ZSTD_compressionParameters* cParams, const ldmParams_t* ldmParams, @@ -6938,13 +6900,19 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_compressionParameters cParams) { ZSTD_ParamSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams); - /* enableDedicatedDictSearch == 1 ensures matchstate is not too small in case this CDict will be used for DDS + row hash */ - size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0); - size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) - + (dictLoadMethod == ZSTD_dlm_byRef ? 0 - : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))) - + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) - + matchStateSize; + ZSTD_rustCDictSizing const sizing = { + sizeof(ZSTD_CDict), + HUF_WORKSPACE_SIZE, + ZSTD_HASHLOG3_MAX, + sizeof(ZSTD_match_t), + sizeof(ZSTD_optimal_t), + ZSTD_RUST_ASAN_REDZONE_SIZE + }; + /* Dedicated search keeps the match state large enough for a later DDS + * plus row-hash use of this static CDict. */ + size_t const neededSize = ZSTD_rust_params_estimateCDictWorkspaceSize( + dictSize, cParams, (int)dictLoadMethod, + useRowMatchFinder, 1, &sizing); ZSTD_rust_initStaticCDictContext context; ZSTD_rust_initStaticCDictState state; diff --git a/rust/src/zstd_compress_params.rs b/rust/src/zstd_compress_params.rs index 28bfdd3e1..a79b46643 100644 --- a/rust/src/zstd_compress_params.rs +++ b/rust/src/zstd_compress_params.rs @@ -1218,7 +1218,7 @@ pub struct ZSTD_rustMatchStateSizing { pub asanRedzoneSize: usize, } -/// C-only layout inputs for `ZSTD_estimateCDictSize_advanced()`. +/// C-only layout inputs for CDict workspace-size formulas. #[repr(C)] #[derive(Clone, Copy, Debug, Default)] pub struct ZSTD_rustCDictSizing { @@ -2689,6 +2689,29 @@ mod tests { }, by_ref ); + + let resolved_row_match_finder = resolve_row_match_finder(ZSTD_RUST_PS_AUTO, cparams); + let static_cdict = estimate_cdict_size_from_cparams( + 123, + cparams, + 1, + resolved_row_match_finder, + true, + sizing, + ); + assert_eq!( + unsafe { + ZSTD_rust_params_estimateCDictWorkspaceSize( + 123, + cparams, + 1, + resolved_row_match_finder, + 1, + &sizing, + ) + }, + static_cdict + ); } #[test]