feat(compress): move static CDict sizing into Rust
Route ZSTD_initStaticCDict() workspace-size calculation through the Rust CDict sizing policy, including the by-reference/by-copy, resolved row match-finder, and dedicated-search branches. Keep private layout constants at the C boundary and retain C ownership of static workspace construction and dictionary initialization. Remove the obsolete C match-state sizing adapter now that no C path calls it, avoiding a dead implementation after the sizing migration. Test Plan: - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040 && make -j1 - ulimit -v 41943040 && make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s - ulimit -v 41943040 && make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests - ulimit -v 41943040 && cargo fmt --manifest-path rust/Cargo.toml -- --check - git diff --check
This commit is contained in:
@@ -1796,17 +1796,6 @@ ZSTD_RUST_CCTX_PARAMS_ASSERT(ZSTD_rust_cctx_params_repcode_offset,
|
|||||||
offsetof(ZSTD_CCtx_params, searchForExternalRepcodes) == (sizeof(void*) == 8 ? 216 : 176));
|
offsetof(ZSTD_CCtx_params, searchForExternalRepcodes) == (sizeof(void*) == 8 ? 216 : 176));
|
||||||
#undef ZSTD_RUST_CCTX_PARAMS_ASSERT
|
#undef ZSTD_RUST_CCTX_PARAMS_ASSERT
|
||||||
|
|
||||||
typedef struct {
|
|
||||||
U32 hashLog3Max;
|
|
||||||
size_t matchTSize;
|
|
||||||
size_t optimalTSize;
|
|
||||||
size_t asanRedzoneSize;
|
|
||||||
} ZSTD_rustMatchStateSizing;
|
|
||||||
size_t ZSTD_rust_params_estimateMatchStateSize(
|
|
||||||
ZSTD_compressionParameters cParams, int useRowMatchFinder,
|
|
||||||
int enableDedicatedDictSearch, U32 forCCtx,
|
|
||||||
const ZSTD_rustMatchStateSizing* sizing);
|
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
size_t cdictSize;
|
size_t cdictSize;
|
||||||
size_t hufWorkspaceSize;
|
size_t hufWorkspaceSize;
|
||||||
@@ -3586,33 +3575,6 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
|
|||||||
ZSTD_getCParamsExclusionMask());
|
ZSTD_getCParamsExclusionMask());
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t
|
|
||||||
ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
|
|
||||||
const ZSTD_ParamSwitch_e useRowMatchFinder,
|
|
||||||
const int enableDedicatedDictSearch,
|
|
||||||
const U32 forCCtx)
|
|
||||||
{
|
|
||||||
ZSTD_rustMatchStateSizing sizing;
|
|
||||||
sizing.hashLog3Max = ZSTD_HASHLOG3_MAX;
|
|
||||||
sizing.matchTSize = sizeof(ZSTD_match_t);
|
|
||||||
sizing.optimalTSize = sizeof(ZSTD_optimal_t);
|
|
||||||
sizing.asanRedzoneSize = ZSTD_RUST_ASAN_REDZONE_SIZE;
|
|
||||||
|
|
||||||
/* The Rust leaf hardcodes the frozen format bounds and workspace rules;
|
|
||||||
* keep them checked against the private headers here. */
|
|
||||||
ZSTD_STATIC_ASSERT(MaxML == 52 && MaxLL == 35 && MaxOff == 31);
|
|
||||||
ZSTD_STATIC_ASSERT(Litbits == 8 && ZSTD_OPT_SIZE == 4099);
|
|
||||||
ZSTD_STATIC_ASSERT(ZSTD_CWKSP_ALIGNMENT_BYTES == 64);
|
|
||||||
/* tables are guaranteed to be sized in multiples of 64 bytes (or 16 uint32_t) */
|
|
||||||
ZSTD_STATIC_ASSERT(ZSTD_HASHLOG_MIN >= 4 && ZSTD_WINDOWLOG_MIN >= 4 && ZSTD_CHAINLOG_MIN >= 4);
|
|
||||||
assert(useRowMatchFinder != ZSTD_ps_auto);
|
|
||||||
|
|
||||||
return ZSTD_rust_params_estimateMatchStateSize(*cParams,
|
|
||||||
(int)useRowMatchFinder,
|
|
||||||
enableDedicatedDictSearch,
|
|
||||||
forCCtx, &sizing);
|
|
||||||
}
|
|
||||||
|
|
||||||
static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
|
static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
|
||||||
const ZSTD_compressionParameters* cParams,
|
const ZSTD_compressionParameters* cParams,
|
||||||
const ldmParams_t* ldmParams,
|
const ldmParams_t* ldmParams,
|
||||||
@@ -6938,13 +6900,19 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
|
|||||||
ZSTD_compressionParameters cParams)
|
ZSTD_compressionParameters cParams)
|
||||||
{
|
{
|
||||||
ZSTD_ParamSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams);
|
ZSTD_ParamSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams);
|
||||||
/* enableDedicatedDictSearch == 1 ensures matchstate is not too small in case this CDict will be used for DDS + row hash */
|
ZSTD_rustCDictSizing const sizing = {
|
||||||
size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0);
|
sizeof(ZSTD_CDict),
|
||||||
size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
|
HUF_WORKSPACE_SIZE,
|
||||||
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0
|
ZSTD_HASHLOG3_MAX,
|
||||||
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))))
|
sizeof(ZSTD_match_t),
|
||||||
+ ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
|
sizeof(ZSTD_optimal_t),
|
||||||
+ matchStateSize;
|
ZSTD_RUST_ASAN_REDZONE_SIZE
|
||||||
|
};
|
||||||
|
/* Dedicated search keeps the match state large enough for a later DDS
|
||||||
|
* plus row-hash use of this static CDict. */
|
||||||
|
size_t const neededSize = ZSTD_rust_params_estimateCDictWorkspaceSize(
|
||||||
|
dictSize, cParams, (int)dictLoadMethod,
|
||||||
|
useRowMatchFinder, 1, &sizing);
|
||||||
ZSTD_rust_initStaticCDictContext context;
|
ZSTD_rust_initStaticCDictContext context;
|
||||||
ZSTD_rust_initStaticCDictState state;
|
ZSTD_rust_initStaticCDictState state;
|
||||||
|
|
||||||
|
|||||||
@@ -1218,7 +1218,7 @@ pub struct ZSTD_rustMatchStateSizing {
|
|||||||
pub asanRedzoneSize: usize,
|
pub asanRedzoneSize: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// C-only layout inputs for `ZSTD_estimateCDictSize_advanced()`.
|
/// C-only layout inputs for CDict workspace-size formulas.
|
||||||
#[repr(C)]
|
#[repr(C)]
|
||||||
#[derive(Clone, Copy, Debug, Default)]
|
#[derive(Clone, Copy, Debug, Default)]
|
||||||
pub struct ZSTD_rustCDictSizing {
|
pub struct ZSTD_rustCDictSizing {
|
||||||
@@ -2689,6 +2689,29 @@ mod tests {
|
|||||||
},
|
},
|
||||||
by_ref
|
by_ref
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let resolved_row_match_finder = resolve_row_match_finder(ZSTD_RUST_PS_AUTO, cparams);
|
||||||
|
let static_cdict = estimate_cdict_size_from_cparams(
|
||||||
|
123,
|
||||||
|
cparams,
|
||||||
|
1,
|
||||||
|
resolved_row_match_finder,
|
||||||
|
true,
|
||||||
|
sizing,
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
unsafe {
|
||||||
|
ZSTD_rust_params_estimateCDictWorkspaceSize(
|
||||||
|
123,
|
||||||
|
cparams,
|
||||||
|
1,
|
||||||
|
resolved_row_match_finder,
|
||||||
|
1,
|
||||||
|
&sizing,
|
||||||
|
)
|
||||||
|
},
|
||||||
|
static_cdict
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
Reference in New Issue
Block a user