feat(compress): move advanced CDict sizing arithmetic into Rust
Move the advanced-CDict workspace-size formula and its by-reference, by-copy, row-match-finder, and dedicated-search branches into the Rust compression-parameter layer. Keep private C layout constants and the already-resolved mode at the C boundary, where the allocator still owns private object layout details. Add a focused test covering the load-method and dedicated-search deltas and the exported advanced-sizing boundary. Test Plan: - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040 && make -j1 - ulimit -v 41943040 && make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s - ulimit -v 41943040 && make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests - ulimit -v 41943040 && cargo fmt --manifest-path rust/Cargo.toml -- --check - git diff --check
This commit is contained in:
@@ -1818,6 +1818,10 @@ typedef struct {
|
|||||||
size_t ZSTD_rust_params_estimateCDictSizeFromCParams(
|
size_t ZSTD_rust_params_estimateCDictSizeFromCParams(
|
||||||
size_t dictSize, ZSTD_compressionParameters cParams,
|
size_t dictSize, ZSTD_compressionParameters cParams,
|
||||||
int dictLoadMethod, const ZSTD_rustCDictSizing* sizing);
|
int dictLoadMethod, const ZSTD_rustCDictSizing* sizing);
|
||||||
|
size_t ZSTD_rust_params_estimateCDictWorkspaceSize(
|
||||||
|
size_t dictSize, ZSTD_compressionParameters cParams,
|
||||||
|
int dictLoadMethod, int useRowMatchFinder,
|
||||||
|
int enableDedicatedDictSearch, const ZSTD_rustCDictSizing* sizing);
|
||||||
|
|
||||||
/* Sequence statistics and seqStore entropy compression live in Rust
|
/* Sequence statistics and seqStore entropy compression live in Rust
|
||||||
* (rust/src/zstd_compress_stats.rs), which also exports ZSTD_seqToCodes()
|
* (rust/src/zstd_compress_stats.rs), which also exports ZSTD_seqToCodes()
|
||||||
@@ -6681,14 +6685,19 @@ static size_t ZSTD_rust_createCDictAdvanced_workspaceSize(
|
|||||||
{
|
{
|
||||||
(void)context;
|
(void)context;
|
||||||
if (cParams == NULL) return 0;
|
if (cParams == NULL) return 0;
|
||||||
return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
|
{ ZSTD_rustCDictSizing const sizing = {
|
||||||
+ ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
|
sizeof(ZSTD_CDict),
|
||||||
+ ZSTD_sizeof_matchState(cParams,
|
HUF_WORKSPACE_SIZE,
|
||||||
(ZSTD_ParamSwitch_e)useRowMatchFinder,
|
ZSTD_HASHLOG3_MAX,
|
||||||
enableDedicatedDictSearch,
|
sizeof(ZSTD_match_t),
|
||||||
/* forCCtx */ 0)
|
sizeof(ZSTD_optimal_t),
|
||||||
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0
|
ZSTD_RUST_ASAN_REDZONE_SIZE
|
||||||
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))));
|
};
|
||||||
|
return ZSTD_rust_params_estimateCDictWorkspaceSize(
|
||||||
|
dictSize, *cParams, dictLoadMethod,
|
||||||
|
useRowMatchFinder, enableDedicatedDictSearch,
|
||||||
|
&sizing);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void* ZSTD_rust_createCDictAdvanced_allocate(
|
static void* ZSTD_rust_createCDictAdvanced_allocate(
|
||||||
|
|||||||
@@ -1457,6 +1457,8 @@ fn estimate_cdict_size_from_cparams(
|
|||||||
dict_size: usize,
|
dict_size: usize,
|
||||||
cparams: ZSTD_compressionParameters,
|
cparams: ZSTD_compressionParameters,
|
||||||
dict_load_method: c_int,
|
dict_load_method: c_int,
|
||||||
|
use_row_match_finder: c_int,
|
||||||
|
enable_dedicated_dict_search: bool,
|
||||||
sizing: ZSTD_rustCDictSizing,
|
sizing: ZSTD_rustCDictSizing,
|
||||||
) -> usize {
|
) -> usize {
|
||||||
if check_cparams(cparams) != 0 {
|
if check_cparams(cparams) != 0 {
|
||||||
@@ -1468,7 +1470,6 @@ fn estimate_cdict_size_from_cparams(
|
|||||||
optimalTSize: sizing.optimalTSize,
|
optimalTSize: sizing.optimalTSize,
|
||||||
asanRedzoneSize: sizing.asanRedzoneSize,
|
asanRedzoneSize: sizing.asanRedzoneSize,
|
||||||
};
|
};
|
||||||
let row_match_finder = resolve_row_match_finder(ZSTD_RUST_PS_AUTO, cparams);
|
|
||||||
let copied_dict_space = if dict_load_method == 1 {
|
let copied_dict_space = if dict_load_method == 1 {
|
||||||
0
|
0
|
||||||
} else {
|
} else {
|
||||||
@@ -1484,8 +1485,8 @@ fn estimate_cdict_size_from_cparams(
|
|||||||
))
|
))
|
||||||
.wrapping_add(estimate_match_state_size(
|
.wrapping_add(estimate_match_state_size(
|
||||||
cparams,
|
cparams,
|
||||||
row_match_finder,
|
use_row_match_finder,
|
||||||
true,
|
enable_dedicated_dict_search,
|
||||||
false,
|
false,
|
||||||
match_state_sizing,
|
match_state_sizing,
|
||||||
))
|
))
|
||||||
@@ -1507,7 +1508,42 @@ pub unsafe extern "C" fn ZSTD_rust_params_estimateCDictSizeFromCParams(
|
|||||||
if sizing.is_null() {
|
if sizing.is_null() {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
estimate_cdict_size_from_cparams(dictSize, cparams, dictLoadMethod, unsafe { *sizing })
|
let use_row_match_finder = resolve_row_match_finder(ZSTD_RUST_PS_AUTO, cparams);
|
||||||
|
estimate_cdict_size_from_cparams(
|
||||||
|
dictSize,
|
||||||
|
cparams,
|
||||||
|
dictLoadMethod,
|
||||||
|
use_row_match_finder,
|
||||||
|
true,
|
||||||
|
unsafe { *sizing },
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Workspace-size policy for an advanced CDict construction.
|
||||||
|
///
|
||||||
|
/// C supplies the private layout-size inputs and the already-resolved row
|
||||||
|
/// match-finder mode. Rust owns the size arithmetic and preserves the
|
||||||
|
/// dedicated-search and by-reference/by-copy branches.
|
||||||
|
#[no_mangle]
|
||||||
|
pub unsafe extern "C" fn ZSTD_rust_params_estimateCDictWorkspaceSize(
|
||||||
|
dictSize: usize,
|
||||||
|
cparams: ZSTD_compressionParameters,
|
||||||
|
dictLoadMethod: c_int,
|
||||||
|
useRowMatchFinder: c_int,
|
||||||
|
enableDedicatedDictSearch: c_int,
|
||||||
|
sizing: *const ZSTD_rustCDictSizing,
|
||||||
|
) -> usize {
|
||||||
|
if sizing.is_null() {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
estimate_cdict_size_from_cparams(
|
||||||
|
dictSize,
|
||||||
|
cparams,
|
||||||
|
dictLoadMethod,
|
||||||
|
useRowMatchFinder,
|
||||||
|
enableDedicatedDictSearch != 0,
|
||||||
|
unsafe { *sizing },
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -2612,6 +2648,49 @@ mod tests {
|
|||||||
assert_eq!(ZSTD_rust_params_resolveMaxBlockSize(0), 128 * 1024);
|
assert_eq!(ZSTD_rust_params_resolveMaxBlockSize(0), 128 * 1024);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cdict_workspace_size_preserves_load_and_dedicated_search_branches() {
|
||||||
|
let sizing = ZSTD_rustCDictSizing {
|
||||||
|
cdictSize: 64,
|
||||||
|
hufWorkspaceSize: 32,
|
||||||
|
hashLog3Max: 17,
|
||||||
|
matchTSize: 16,
|
||||||
|
optimalTSize: 32,
|
||||||
|
asanRedzoneSize: 0,
|
||||||
|
};
|
||||||
|
let cparams = ZSTD_compressionParameters {
|
||||||
|
windowLog: 10,
|
||||||
|
chainLog: 10,
|
||||||
|
hashLog: 10,
|
||||||
|
searchLog: 1,
|
||||||
|
minMatch: 4,
|
||||||
|
targetLength: 0,
|
||||||
|
strategy: ZSTD_FAST,
|
||||||
|
};
|
||||||
|
let by_ref =
|
||||||
|
estimate_cdict_size_from_cparams(123, cparams, 1, ZSTD_RUST_PS_DISABLE, false, sizing);
|
||||||
|
let by_copy =
|
||||||
|
estimate_cdict_size_from_cparams(123, cparams, 0, ZSTD_RUST_PS_DISABLE, false, sizing);
|
||||||
|
let dedicated =
|
||||||
|
estimate_cdict_size_from_cparams(123, cparams, 1, ZSTD_RUST_PS_DISABLE, true, sizing);
|
||||||
|
|
||||||
|
assert_eq!(by_copy - by_ref, 128);
|
||||||
|
assert_eq!(dedicated - by_ref, (1 << 10) * size_of::<u32>());
|
||||||
|
assert_eq!(
|
||||||
|
unsafe {
|
||||||
|
ZSTD_rust_params_estimateCDictWorkspaceSize(
|
||||||
|
123,
|
||||||
|
cparams,
|
||||||
|
1,
|
||||||
|
ZSTD_RUST_PS_DISABLE,
|
||||||
|
0,
|
||||||
|
&sizing,
|
||||||
|
)
|
||||||
|
},
|
||||||
|
by_ref
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn block_size_uses_the_smaller_configured_limit() {
|
fn block_size_uses_the_smaller_configured_limit() {
|
||||||
assert_eq!(ZSTD_rust_params_getBlockSize(64 * 1024, 17), 64 * 1024);
|
assert_eq!(ZSTD_rust_params_getBlockSize(64 * 1024, 17), 64 * 1024);
|
||||||
|
|||||||
Reference in New Issue
Block a user