From 4a52d64f993d6b05dea820deb316415185e7b618 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Sun, 19 Jul 2026 21:49:50 +0200 Subject: [PATCH] feat(compress): move advanced CDict sizing arithmetic into Rust Move the advanced-CDict workspace-size formula and its by-reference, by-copy, row-match-finder, and dedicated-search branches into the Rust compression-parameter layer. Keep private C layout constants and the already-resolved mode at the C boundary, where the allocator still owns private object layout details. Add a focused test covering the load-method and dedicated-search deltas and the exported advanced-sizing boundary. Test Plan: - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040 && make -j1 - ulimit -v 41943040 && make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s - ulimit -v 41943040 && make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests - ulimit -v 41943040 && cargo fmt --manifest-path rust/Cargo.toml -- --check - git diff --check --- lib/compress/zstd_compress.c | 25 ++++++--- rust/src/zstd_compress_params.rs | 87 ++++++++++++++++++++++++++++++-- 2 files changed, 100 insertions(+), 12 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 088d4e021..fab1f7eda 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1818,6 +1818,10 @@ typedef struct { size_t ZSTD_rust_params_estimateCDictSizeFromCParams( size_t dictSize, ZSTD_compressionParameters cParams, int dictLoadMethod, const ZSTD_rustCDictSizing* sizing); +size_t ZSTD_rust_params_estimateCDictWorkspaceSize( + size_t dictSize, ZSTD_compressionParameters cParams, + int dictLoadMethod, int useRowMatchFinder, + int enableDedicatedDictSearch, const ZSTD_rustCDictSizing* sizing); /* Sequence statistics and seqStore entropy compression live in Rust * (rust/src/zstd_compress_stats.rs), which also exports ZSTD_seqToCodes() @@ -6681,14 +6685,19 @@ static size_t ZSTD_rust_createCDictAdvanced_workspaceSize( { (void)context; if (cParams == NULL) return 0; - return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) - + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) - + ZSTD_sizeof_matchState(cParams, - (ZSTD_ParamSwitch_e)useRowMatchFinder, - enableDedicatedDictSearch, - /* forCCtx */ 0) - + (dictLoadMethod == ZSTD_dlm_byRef ? 0 - : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))); + { ZSTD_rustCDictSizing const sizing = { + sizeof(ZSTD_CDict), + HUF_WORKSPACE_SIZE, + ZSTD_HASHLOG3_MAX, + sizeof(ZSTD_match_t), + sizeof(ZSTD_optimal_t), + ZSTD_RUST_ASAN_REDZONE_SIZE + }; + return ZSTD_rust_params_estimateCDictWorkspaceSize( + dictSize, *cParams, dictLoadMethod, + useRowMatchFinder, enableDedicatedDictSearch, + &sizing); + } } static void* ZSTD_rust_createCDictAdvanced_allocate( diff --git a/rust/src/zstd_compress_params.rs b/rust/src/zstd_compress_params.rs index e095fc466..28bfdd3e1 100644 --- a/rust/src/zstd_compress_params.rs +++ b/rust/src/zstd_compress_params.rs @@ -1457,6 +1457,8 @@ fn estimate_cdict_size_from_cparams( dict_size: usize, cparams: ZSTD_compressionParameters, dict_load_method: c_int, + use_row_match_finder: c_int, + enable_dedicated_dict_search: bool, sizing: ZSTD_rustCDictSizing, ) -> usize { if check_cparams(cparams) != 0 { @@ -1468,7 +1470,6 @@ fn estimate_cdict_size_from_cparams( optimalTSize: sizing.optimalTSize, asanRedzoneSize: sizing.asanRedzoneSize, }; - let row_match_finder = resolve_row_match_finder(ZSTD_RUST_PS_AUTO, cparams); let copied_dict_space = if dict_load_method == 1 { 0 } else { @@ -1484,8 +1485,8 @@ fn estimate_cdict_size_from_cparams( )) .wrapping_add(estimate_match_state_size( cparams, - row_match_finder, - true, + use_row_match_finder, + enable_dedicated_dict_search, false, match_state_sizing, )) @@ -1507,7 +1508,42 @@ pub unsafe extern "C" fn ZSTD_rust_params_estimateCDictSizeFromCParams( if sizing.is_null() { return 0; } - estimate_cdict_size_from_cparams(dictSize, cparams, dictLoadMethod, unsafe { *sizing }) + let use_row_match_finder = resolve_row_match_finder(ZSTD_RUST_PS_AUTO, cparams); + estimate_cdict_size_from_cparams( + dictSize, + cparams, + dictLoadMethod, + use_row_match_finder, + true, + unsafe { *sizing }, + ) +} + +/// Workspace-size policy for an advanced CDict construction. +/// +/// C supplies the private layout-size inputs and the already-resolved row +/// match-finder mode. Rust owns the size arithmetic and preserves the +/// dedicated-search and by-reference/by-copy branches. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_params_estimateCDictWorkspaceSize( + dictSize: usize, + cparams: ZSTD_compressionParameters, + dictLoadMethod: c_int, + useRowMatchFinder: c_int, + enableDedicatedDictSearch: c_int, + sizing: *const ZSTD_rustCDictSizing, +) -> usize { + if sizing.is_null() { + return 0; + } + estimate_cdict_size_from_cparams( + dictSize, + cparams, + dictLoadMethod, + useRowMatchFinder, + enableDedicatedDictSearch != 0, + unsafe { *sizing }, + ) } #[cfg(test)] @@ -2612,6 +2648,49 @@ mod tests { assert_eq!(ZSTD_rust_params_resolveMaxBlockSize(0), 128 * 1024); } + #[test] + fn cdict_workspace_size_preserves_load_and_dedicated_search_branches() { + let sizing = ZSTD_rustCDictSizing { + cdictSize: 64, + hufWorkspaceSize: 32, + hashLog3Max: 17, + matchTSize: 16, + optimalTSize: 32, + asanRedzoneSize: 0, + }; + let cparams = ZSTD_compressionParameters { + windowLog: 10, + chainLog: 10, + hashLog: 10, + searchLog: 1, + minMatch: 4, + targetLength: 0, + strategy: ZSTD_FAST, + }; + let by_ref = + estimate_cdict_size_from_cparams(123, cparams, 1, ZSTD_RUST_PS_DISABLE, false, sizing); + let by_copy = + estimate_cdict_size_from_cparams(123, cparams, 0, ZSTD_RUST_PS_DISABLE, false, sizing); + let dedicated = + estimate_cdict_size_from_cparams(123, cparams, 1, ZSTD_RUST_PS_DISABLE, true, sizing); + + assert_eq!(by_copy - by_ref, 128); + assert_eq!(dedicated - by_ref, (1 << 10) * size_of::()); + assert_eq!( + unsafe { + ZSTD_rust_params_estimateCDictWorkspaceSize( + 123, + cparams, + 1, + ZSTD_RUST_PS_DISABLE, + 0, + &sizing, + ) + }, + by_ref + ); + } + #[test] fn block_size_uses_the_smaller_configured_limit() { assert_eq!(ZSTD_rust_params_getBlockSize(64 * 1024, 17), 64 * 1024);