feat(compress): move advanced CDict sizing arithmetic into Rust
Move the advanced-CDict workspace-size formula and its by-reference, by-copy, row-match-finder, and dedicated-search branches into the Rust compression-parameter layer. Keep private C layout constants and the already-resolved mode at the C boundary, where the allocator still owns private object layout details. Add a focused test covering the load-method and dedicated-search deltas and the exported advanced-sizing boundary. Test Plan: - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040 && make -j1 - ulimit -v 41943040 && make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s - ulimit -v 41943040 && make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests - ulimit -v 41943040 && cargo fmt --manifest-path rust/Cargo.toml -- --check - git diff --check
This commit is contained in:
@@ -1818,6 +1818,10 @@ typedef struct {
|
||||
size_t ZSTD_rust_params_estimateCDictSizeFromCParams(
|
||||
size_t dictSize, ZSTD_compressionParameters cParams,
|
||||
int dictLoadMethod, const ZSTD_rustCDictSizing* sizing);
|
||||
size_t ZSTD_rust_params_estimateCDictWorkspaceSize(
|
||||
size_t dictSize, ZSTD_compressionParameters cParams,
|
||||
int dictLoadMethod, int useRowMatchFinder,
|
||||
int enableDedicatedDictSearch, const ZSTD_rustCDictSizing* sizing);
|
||||
|
||||
/* Sequence statistics and seqStore entropy compression live in Rust
|
||||
* (rust/src/zstd_compress_stats.rs), which also exports ZSTD_seqToCodes()
|
||||
@@ -6681,14 +6685,19 @@ static size_t ZSTD_rust_createCDictAdvanced_workspaceSize(
|
||||
{
|
||||
(void)context;
|
||||
if (cParams == NULL) return 0;
|
||||
return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
|
||||
+ ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
|
||||
+ ZSTD_sizeof_matchState(cParams,
|
||||
(ZSTD_ParamSwitch_e)useRowMatchFinder,
|
||||
enableDedicatedDictSearch,
|
||||
/* forCCtx */ 0)
|
||||
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0
|
||||
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))));
|
||||
{ ZSTD_rustCDictSizing const sizing = {
|
||||
sizeof(ZSTD_CDict),
|
||||
HUF_WORKSPACE_SIZE,
|
||||
ZSTD_HASHLOG3_MAX,
|
||||
sizeof(ZSTD_match_t),
|
||||
sizeof(ZSTD_optimal_t),
|
||||
ZSTD_RUST_ASAN_REDZONE_SIZE
|
||||
};
|
||||
return ZSTD_rust_params_estimateCDictWorkspaceSize(
|
||||
dictSize, *cParams, dictLoadMethod,
|
||||
useRowMatchFinder, enableDedicatedDictSearch,
|
||||
&sizing);
|
||||
}
|
||||
}
|
||||
|
||||
static void* ZSTD_rust_createCDictAdvanced_allocate(
|
||||
|
||||
@@ -1457,6 +1457,8 @@ fn estimate_cdict_size_from_cparams(
|
||||
dict_size: usize,
|
||||
cparams: ZSTD_compressionParameters,
|
||||
dict_load_method: c_int,
|
||||
use_row_match_finder: c_int,
|
||||
enable_dedicated_dict_search: bool,
|
||||
sizing: ZSTD_rustCDictSizing,
|
||||
) -> usize {
|
||||
if check_cparams(cparams) != 0 {
|
||||
@@ -1468,7 +1470,6 @@ fn estimate_cdict_size_from_cparams(
|
||||
optimalTSize: sizing.optimalTSize,
|
||||
asanRedzoneSize: sizing.asanRedzoneSize,
|
||||
};
|
||||
let row_match_finder = resolve_row_match_finder(ZSTD_RUST_PS_AUTO, cparams);
|
||||
let copied_dict_space = if dict_load_method == 1 {
|
||||
0
|
||||
} else {
|
||||
@@ -1484,8 +1485,8 @@ fn estimate_cdict_size_from_cparams(
|
||||
))
|
||||
.wrapping_add(estimate_match_state_size(
|
||||
cparams,
|
||||
row_match_finder,
|
||||
true,
|
||||
use_row_match_finder,
|
||||
enable_dedicated_dict_search,
|
||||
false,
|
||||
match_state_sizing,
|
||||
))
|
||||
@@ -1507,7 +1508,42 @@ pub unsafe extern "C" fn ZSTD_rust_params_estimateCDictSizeFromCParams(
|
||||
if sizing.is_null() {
|
||||
return 0;
|
||||
}
|
||||
estimate_cdict_size_from_cparams(dictSize, cparams, dictLoadMethod, unsafe { *sizing })
|
||||
let use_row_match_finder = resolve_row_match_finder(ZSTD_RUST_PS_AUTO, cparams);
|
||||
estimate_cdict_size_from_cparams(
|
||||
dictSize,
|
||||
cparams,
|
||||
dictLoadMethod,
|
||||
use_row_match_finder,
|
||||
true,
|
||||
unsafe { *sizing },
|
||||
)
|
||||
}
|
||||
|
||||
/// Workspace-size policy for an advanced CDict construction.
|
||||
///
|
||||
/// C supplies the private layout-size inputs and the already-resolved row
|
||||
/// match-finder mode. Rust owns the size arithmetic and preserves the
|
||||
/// dedicated-search and by-reference/by-copy branches.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_params_estimateCDictWorkspaceSize(
|
||||
dictSize: usize,
|
||||
cparams: ZSTD_compressionParameters,
|
||||
dictLoadMethod: c_int,
|
||||
useRowMatchFinder: c_int,
|
||||
enableDedicatedDictSearch: c_int,
|
||||
sizing: *const ZSTD_rustCDictSizing,
|
||||
) -> usize {
|
||||
if sizing.is_null() {
|
||||
return 0;
|
||||
}
|
||||
estimate_cdict_size_from_cparams(
|
||||
dictSize,
|
||||
cparams,
|
||||
dictLoadMethod,
|
||||
useRowMatchFinder,
|
||||
enableDedicatedDictSearch != 0,
|
||||
unsafe { *sizing },
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -2612,6 +2648,49 @@ mod tests {
|
||||
assert_eq!(ZSTD_rust_params_resolveMaxBlockSize(0), 128 * 1024);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cdict_workspace_size_preserves_load_and_dedicated_search_branches() {
|
||||
let sizing = ZSTD_rustCDictSizing {
|
||||
cdictSize: 64,
|
||||
hufWorkspaceSize: 32,
|
||||
hashLog3Max: 17,
|
||||
matchTSize: 16,
|
||||
optimalTSize: 32,
|
||||
asanRedzoneSize: 0,
|
||||
};
|
||||
let cparams = ZSTD_compressionParameters {
|
||||
windowLog: 10,
|
||||
chainLog: 10,
|
||||
hashLog: 10,
|
||||
searchLog: 1,
|
||||
minMatch: 4,
|
||||
targetLength: 0,
|
||||
strategy: ZSTD_FAST,
|
||||
};
|
||||
let by_ref =
|
||||
estimate_cdict_size_from_cparams(123, cparams, 1, ZSTD_RUST_PS_DISABLE, false, sizing);
|
||||
let by_copy =
|
||||
estimate_cdict_size_from_cparams(123, cparams, 0, ZSTD_RUST_PS_DISABLE, false, sizing);
|
||||
let dedicated =
|
||||
estimate_cdict_size_from_cparams(123, cparams, 1, ZSTD_RUST_PS_DISABLE, true, sizing);
|
||||
|
||||
assert_eq!(by_copy - by_ref, 128);
|
||||
assert_eq!(dedicated - by_ref, (1 << 10) * size_of::<u32>());
|
||||
assert_eq!(
|
||||
unsafe {
|
||||
ZSTD_rust_params_estimateCDictWorkspaceSize(
|
||||
123,
|
||||
cparams,
|
||||
1,
|
||||
ZSTD_RUST_PS_DISABLE,
|
||||
0,
|
||||
&sizing,
|
||||
)
|
||||
},
|
||||
by_ref
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn block_size_uses_the_smaller_configured_limit() {
|
||||
assert_eq!(ZSTD_rust_params_getBlockSize(64 * 1024, 17), 64 * 1024);
|
||||
|
||||
Reference in New Issue
Block a user