feat(compress): move advanced CDict sizing arithmetic into Rust

Move the advanced-CDict workspace-size formula and its by-reference,
by-copy, row-match-finder, and dedicated-search branches into the Rust
compression-parameter layer. Keep private C layout constants and the
already-resolved mode at the C boundary, where the allocator still owns
private object layout details.

Add a focused test covering the load-method and dedicated-search deltas
and the exported advanced-sizing boundary.

Test Plan:
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040 && make -j1
- ulimit -v 41943040 && make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s
- ulimit -v 41943040 && make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests
- ulimit -v 41943040 && cargo fmt --manifest-path rust/Cargo.toml -- --check
- git diff --check
This commit is contained in:
2026-07-19 21:49:50 +02:00
parent 0be6d23512
commit 4a52d64f99
2 changed files with 100 additions and 12 deletions
+17 -8
View File
@@ -1818,6 +1818,10 @@ typedef struct {
size_t ZSTD_rust_params_estimateCDictSizeFromCParams(
size_t dictSize, ZSTD_compressionParameters cParams,
int dictLoadMethod, const ZSTD_rustCDictSizing* sizing);
size_t ZSTD_rust_params_estimateCDictWorkspaceSize(
size_t dictSize, ZSTD_compressionParameters cParams,
int dictLoadMethod, int useRowMatchFinder,
int enableDedicatedDictSearch, const ZSTD_rustCDictSizing* sizing);
/* Sequence statistics and seqStore entropy compression live in Rust
* (rust/src/zstd_compress_stats.rs), which also exports ZSTD_seqToCodes()
@@ -6681,14 +6685,19 @@ static size_t ZSTD_rust_createCDictAdvanced_workspaceSize(
{
(void)context;
if (cParams == NULL) return 0;
return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
+ ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
+ ZSTD_sizeof_matchState(cParams,
(ZSTD_ParamSwitch_e)useRowMatchFinder,
enableDedicatedDictSearch,
/* forCCtx */ 0)
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))));
{ ZSTD_rustCDictSizing const sizing = {
sizeof(ZSTD_CDict),
HUF_WORKSPACE_SIZE,
ZSTD_HASHLOG3_MAX,
sizeof(ZSTD_match_t),
sizeof(ZSTD_optimal_t),
ZSTD_RUST_ASAN_REDZONE_SIZE
};
return ZSTD_rust_params_estimateCDictWorkspaceSize(
dictSize, *cParams, dictLoadMethod,
useRowMatchFinder, enableDedicatedDictSearch,
&sizing);
}
}
static void* ZSTD_rust_createCDictAdvanced_allocate(
+83 -4
View File
@@ -1457,6 +1457,8 @@ fn estimate_cdict_size_from_cparams(
dict_size: usize,
cparams: ZSTD_compressionParameters,
dict_load_method: c_int,
use_row_match_finder: c_int,
enable_dedicated_dict_search: bool,
sizing: ZSTD_rustCDictSizing,
) -> usize {
if check_cparams(cparams) != 0 {
@@ -1468,7 +1470,6 @@ fn estimate_cdict_size_from_cparams(
optimalTSize: sizing.optimalTSize,
asanRedzoneSize: sizing.asanRedzoneSize,
};
let row_match_finder = resolve_row_match_finder(ZSTD_RUST_PS_AUTO, cparams);
let copied_dict_space = if dict_load_method == 1 {
0
} else {
@@ -1484,8 +1485,8 @@ fn estimate_cdict_size_from_cparams(
))
.wrapping_add(estimate_match_state_size(
cparams,
row_match_finder,
true,
use_row_match_finder,
enable_dedicated_dict_search,
false,
match_state_sizing,
))
@@ -1507,7 +1508,42 @@ pub unsafe extern "C" fn ZSTD_rust_params_estimateCDictSizeFromCParams(
if sizing.is_null() {
return 0;
}
estimate_cdict_size_from_cparams(dictSize, cparams, dictLoadMethod, unsafe { *sizing })
let use_row_match_finder = resolve_row_match_finder(ZSTD_RUST_PS_AUTO, cparams);
estimate_cdict_size_from_cparams(
dictSize,
cparams,
dictLoadMethod,
use_row_match_finder,
true,
unsafe { *sizing },
)
}
/// Workspace-size policy for an advanced CDict construction.
///
/// C supplies the private layout-size inputs and the already-resolved row
/// match-finder mode. Rust owns the size arithmetic and preserves the
/// dedicated-search and by-reference/by-copy branches.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_params_estimateCDictWorkspaceSize(
dictSize: usize,
cparams: ZSTD_compressionParameters,
dictLoadMethod: c_int,
useRowMatchFinder: c_int,
enableDedicatedDictSearch: c_int,
sizing: *const ZSTD_rustCDictSizing,
) -> usize {
if sizing.is_null() {
return 0;
}
estimate_cdict_size_from_cparams(
dictSize,
cparams,
dictLoadMethod,
useRowMatchFinder,
enableDedicatedDictSearch != 0,
unsafe { *sizing },
)
}
#[cfg(test)]
@@ -2612,6 +2648,49 @@ mod tests {
assert_eq!(ZSTD_rust_params_resolveMaxBlockSize(0), 128 * 1024);
}
#[test]
fn cdict_workspace_size_preserves_load_and_dedicated_search_branches() {
let sizing = ZSTD_rustCDictSizing {
cdictSize: 64,
hufWorkspaceSize: 32,
hashLog3Max: 17,
matchTSize: 16,
optimalTSize: 32,
asanRedzoneSize: 0,
};
let cparams = ZSTD_compressionParameters {
windowLog: 10,
chainLog: 10,
hashLog: 10,
searchLog: 1,
minMatch: 4,
targetLength: 0,
strategy: ZSTD_FAST,
};
let by_ref =
estimate_cdict_size_from_cparams(123, cparams, 1, ZSTD_RUST_PS_DISABLE, false, sizing);
let by_copy =
estimate_cdict_size_from_cparams(123, cparams, 0, ZSTD_RUST_PS_DISABLE, false, sizing);
let dedicated =
estimate_cdict_size_from_cparams(123, cparams, 1, ZSTD_RUST_PS_DISABLE, true, sizing);
assert_eq!(by_copy - by_ref, 128);
assert_eq!(dedicated - by_ref, (1 << 10) * size_of::<u32>());
assert_eq!(
unsafe {
ZSTD_rust_params_estimateCDictWorkspaceSize(
123,
cparams,
1,
ZSTD_RUST_PS_DISABLE,
0,
&sizing,
)
},
by_ref
);
}
#[test]
fn block_size_uses_the_smaller_configured_limit() {
assert_eq!(ZSTD_rust_params_getBlockSize(64 * 1024, 17), 64 * 1024);