feat(compress): move static CDict sizing into Rust
Route ZSTD_initStaticCDict() workspace-size calculation through the Rust CDict sizing policy, including the by-reference/by-copy, resolved row match-finder, and dedicated-search branches. Keep private layout constants at the C boundary and retain C ownership of static workspace construction and dictionary initialization. Remove the obsolete C match-state sizing adapter now that no C path calls it, avoiding a dead implementation after the sizing migration. Test Plan: - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040 && make -j1 - ulimit -v 41943040 && make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s - ulimit -v 41943040 && make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests - ulimit -v 41943040 && cargo fmt --manifest-path rust/Cargo.toml -- --check - git diff --check
This commit is contained in:
@@ -1796,17 +1796,6 @@ ZSTD_RUST_CCTX_PARAMS_ASSERT(ZSTD_rust_cctx_params_repcode_offset,
|
||||
offsetof(ZSTD_CCtx_params, searchForExternalRepcodes) == (sizeof(void*) == 8 ? 216 : 176));
|
||||
#undef ZSTD_RUST_CCTX_PARAMS_ASSERT
|
||||
|
||||
typedef struct {
|
||||
U32 hashLog3Max;
|
||||
size_t matchTSize;
|
||||
size_t optimalTSize;
|
||||
size_t asanRedzoneSize;
|
||||
} ZSTD_rustMatchStateSizing;
|
||||
size_t ZSTD_rust_params_estimateMatchStateSize(
|
||||
ZSTD_compressionParameters cParams, int useRowMatchFinder,
|
||||
int enableDedicatedDictSearch, U32 forCCtx,
|
||||
const ZSTD_rustMatchStateSizing* sizing);
|
||||
|
||||
typedef struct {
|
||||
size_t cdictSize;
|
||||
size_t hufWorkspaceSize;
|
||||
@@ -3586,33 +3575,6 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
|
||||
ZSTD_getCParamsExclusionMask());
|
||||
}
|
||||
|
||||
static size_t
|
||||
ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
|
||||
const ZSTD_ParamSwitch_e useRowMatchFinder,
|
||||
const int enableDedicatedDictSearch,
|
||||
const U32 forCCtx)
|
||||
{
|
||||
ZSTD_rustMatchStateSizing sizing;
|
||||
sizing.hashLog3Max = ZSTD_HASHLOG3_MAX;
|
||||
sizing.matchTSize = sizeof(ZSTD_match_t);
|
||||
sizing.optimalTSize = sizeof(ZSTD_optimal_t);
|
||||
sizing.asanRedzoneSize = ZSTD_RUST_ASAN_REDZONE_SIZE;
|
||||
|
||||
/* The Rust leaf hardcodes the frozen format bounds and workspace rules;
|
||||
* keep them checked against the private headers here. */
|
||||
ZSTD_STATIC_ASSERT(MaxML == 52 && MaxLL == 35 && MaxOff == 31);
|
||||
ZSTD_STATIC_ASSERT(Litbits == 8 && ZSTD_OPT_SIZE == 4099);
|
||||
ZSTD_STATIC_ASSERT(ZSTD_CWKSP_ALIGNMENT_BYTES == 64);
|
||||
/* tables are guaranteed to be sized in multiples of 64 bytes (or 16 uint32_t) */
|
||||
ZSTD_STATIC_ASSERT(ZSTD_HASHLOG_MIN >= 4 && ZSTD_WINDOWLOG_MIN >= 4 && ZSTD_CHAINLOG_MIN >= 4);
|
||||
assert(useRowMatchFinder != ZSTD_ps_auto);
|
||||
|
||||
return ZSTD_rust_params_estimateMatchStateSize(*cParams,
|
||||
(int)useRowMatchFinder,
|
||||
enableDedicatedDictSearch,
|
||||
forCCtx, &sizing);
|
||||
}
|
||||
|
||||
static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
|
||||
const ZSTD_compressionParameters* cParams,
|
||||
const ldmParams_t* ldmParams,
|
||||
@@ -6938,13 +6900,19 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
|
||||
ZSTD_compressionParameters cParams)
|
||||
{
|
||||
ZSTD_ParamSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams);
|
||||
/* enableDedicatedDictSearch == 1 ensures matchstate is not too small in case this CDict will be used for DDS + row hash */
|
||||
size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0);
|
||||
size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
|
||||
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0
|
||||
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))))
|
||||
+ ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
|
||||
+ matchStateSize;
|
||||
ZSTD_rustCDictSizing const sizing = {
|
||||
sizeof(ZSTD_CDict),
|
||||
HUF_WORKSPACE_SIZE,
|
||||
ZSTD_HASHLOG3_MAX,
|
||||
sizeof(ZSTD_match_t),
|
||||
sizeof(ZSTD_optimal_t),
|
||||
ZSTD_RUST_ASAN_REDZONE_SIZE
|
||||
};
|
||||
/* Dedicated search keeps the match state large enough for a later DDS
|
||||
* plus row-hash use of this static CDict. */
|
||||
size_t const neededSize = ZSTD_rust_params_estimateCDictWorkspaceSize(
|
||||
dictSize, cParams, (int)dictLoadMethod,
|
||||
useRowMatchFinder, 1, &sizing);
|
||||
ZSTD_rust_initStaticCDictContext context;
|
||||
ZSTD_rust_initStaticCDictState state;
|
||||
|
||||
|
||||
@@ -1218,7 +1218,7 @@ pub struct ZSTD_rustMatchStateSizing {
|
||||
pub asanRedzoneSize: usize,
|
||||
}
|
||||
|
||||
/// C-only layout inputs for `ZSTD_estimateCDictSize_advanced()`.
|
||||
/// C-only layout inputs for CDict workspace-size formulas.
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct ZSTD_rustCDictSizing {
|
||||
@@ -2689,6 +2689,29 @@ mod tests {
|
||||
},
|
||||
by_ref
|
||||
);
|
||||
|
||||
let resolved_row_match_finder = resolve_row_match_finder(ZSTD_RUST_PS_AUTO, cparams);
|
||||
let static_cdict = estimate_cdict_size_from_cparams(
|
||||
123,
|
||||
cparams,
|
||||
1,
|
||||
resolved_row_match_finder,
|
||||
true,
|
||||
sizing,
|
||||
);
|
||||
assert_eq!(
|
||||
unsafe {
|
||||
ZSTD_rust_params_estimateCDictWorkspaceSize(
|
||||
123,
|
||||
cparams,
|
||||
1,
|
||||
resolved_row_match_finder,
|
||||
1,
|
||||
&sizing,
|
||||
)
|
||||
},
|
||||
static_cdict
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user