ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(compress): move estimator policy to Rust

Browse Source 
The public CCtx and CStream size-estimation APIs still selected row-matchfinder modes and walked compression levels in C, leaving a policy-level implementation behind the Rust workspace-sizing bridge. That also made the C layer responsible for the subtle raw-size_t MAX behavior used when an estimator returns an encoded error.

Move the row-mode selection and shared monotonic memory-budget policy into Rust. Keep the C callbacks responsible for constructing private ZSTD_CCtx_params values and for the distinct CCtx/CStream sizing formulas, so no private parameter layout crosses the ABI. Preserve the signed MIN(compressionLevel, 1) start and raw maxima exactly.

Test Plan:

- cargo +nightly fmt --manifest-path rust/Cargo.toml --all -- --check

- git diff --check

- ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1
This commit is contained in:
ddidderr
2026-07-21 11:11:46 +02:00
parent ee1429cf79
commit 1a3971ff10
2 changed files with 301 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+92 -41
View File
@@ -2179,6 +2179,40 @@ size_t ZSTD_rust_estimateCCtxWorkspaceSize(
size_t ZSTD_rust_planCCtxReset(const ZSTD_rustCCtxResetState* state);
size_t ZSTD_rust_maxEstimateCCtxSize(size_t estimate0, size_t estimate1,
size_t estimate2, size_t estimate3);
typedef size_t (*ZSTD_rust_estimateCSizeUsingCParams_f)(
void* context, int useRowMatchFinder);
typedef struct {
void* callbackContext;
int strategy;
ZSTD_rust_estimateCSizeUsingCParams_f estimate;
} ZSTD_rust_estimateCSizeUsingCParamsState;
size_t ZSTD_rust_estimateCSizeUsingCParams(
const ZSTD_rust_estimateCSizeUsingCParamsState* state);
typedef char ZSTD_rust_estimate_csize_using_cparams_state_layout[
(offsetof(ZSTD_rust_estimateCSizeUsingCParamsState, callbackContext) == 0
&& offsetof(ZSTD_rust_estimateCSizeUsingCParamsState, strategy)
== sizeof(void*)
&& offsetof(ZSTD_rust_estimateCSizeUsingCParamsState, estimate)
== 2 * sizeof(void*)
&& sizeof(ZSTD_rust_estimateCSizeUsingCParamsState)
== 3 * sizeof(void*))
? 1 : -1];
typedef size_t (*ZSTD_rust_estimateCSize_f)(void* context, int compressionLevel);
typedef struct {
void* callbackContext;
int compressionLevel;
ZSTD_rust_estimateCSize_f estimate;
} ZSTD_rust_estimateCSizeState;
size_t ZSTD_rust_estimateCSizeMonotonic(
const ZSTD_rust_estimateCSizeState* state);
typedef char ZSTD_rust_estimate_csize_state_layout[
(offsetof(ZSTD_rust_estimateCSizeState, callbackContext) == 0
&& offsetof(ZSTD_rust_estimateCSizeState, compressionLevel)
== sizeof(void*)
&& offsetof(ZSTD_rust_estimateCSizeState, estimate)
== 2 * sizeof(void*)
&& sizeof(ZSTD_rust_estimateCSizeState) == 3 * sizeof(void*))
? 1 : -1];
typedef ZSTD_compressionParameters (*ZSTD_rust_estimateCCtxSizeGetCParams_f)(
void* context, int compressionLevel, U64 srcSizeHint);
typedef size_t (*ZSTD_rust_estimateCCtxSizeEstimate_f)(
@@ -4391,21 +4425,24 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
&cParams, &params->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_hasExtSeqProd(params), params->maxBlockSize);
}
static size_t ZSTD_rust_estimateCCtxSize_usingCParams_estimate(
void* context, int useRowMatchFinder)
{
ZSTD_compressionParameters const* const cParams =
(const ZSTD_compressionParameters*)context;
ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(*cParams);
initialParams.useRowMatchFinder = (ZSTD_ParamSwitch_e)useRowMatchFinder;
return ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams);
}
size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
{
ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams);
if (ZSTD_rowMatchFinderSupported((int)cParams.strategy)) {
/* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */
size_t noRowCCtxSize;
size_t rowCCtxSize;
initialParams.useRowMatchFinder = ZSTD_ps_disable;
noRowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams);
initialParams.useRowMatchFinder = ZSTD_ps_enable;
rowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams);
return MAX(noRowCCtxSize, rowCCtxSize);
} else {
return ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams);
}
ZSTD_rust_estimateCSizeUsingCParamsState const state = {
&cParams,
(int)cParams.strategy,
ZSTD_rust_estimateCCtxSize_usingCParams_estimate
};
return ZSTD_rust_estimateCSizeUsingCParams(&state);
}
static ZSTD_compressionParameters ZSTD_rust_estimateCCtxSize_getCParams(
@@ -4434,16 +4471,21 @@ static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel)
return ZSTD_rust_estimateCCtxSizeInternal(&state);
}
static size_t ZSTD_rust_estimateCCtxSize_estimateLevel(
void* context, int compressionLevel)
{
(void)context;
return ZSTD_estimateCCtxSize_internal(compressionLevel);
}
size_t ZSTD_estimateCCtxSize(int compressionLevel)
{
int level;
size_t memBudget = 0;
for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) {
/* Ensure monotonically increasing memory usage as compression level increases */
size_t const newMB = ZSTD_estimateCCtxSize_internal(level);
if (newMB > memBudget) memBudget = newMB;
}
return memBudget;
ZSTD_rust_estimateCSizeState const state = {
NULL,
compressionLevel,
ZSTD_rust_estimateCCtxSize_estimateLevel
};
return ZSTD_rust_estimateCSizeMonotonic(&state);
}
size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
@@ -4468,21 +4510,24 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
}
}
static size_t ZSTD_rust_estimateCStreamSize_usingCParams_estimate(
void* context, int useRowMatchFinder)
{
ZSTD_compressionParameters const* const cParams =
(const ZSTD_compressionParameters*)context;
ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(*cParams);
initialParams.useRowMatchFinder = (ZSTD_ParamSwitch_e)useRowMatchFinder;
return ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams);
}
size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams)
{
ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams);
if (ZSTD_rowMatchFinderSupported((int)cParams.strategy)) {
/* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */
size_t noRowCCtxSize;
size_t rowCCtxSize;
initialParams.useRowMatchFinder = ZSTD_ps_disable;
noRowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams);
initialParams.useRowMatchFinder = ZSTD_ps_enable;
rowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams);
return MAX(noRowCCtxSize, rowCCtxSize);
} else {
return ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams);
}
ZSTD_rust_estimateCSizeUsingCParamsState const state = {
&cParams,
(int)cParams.strategy,
ZSTD_rust_estimateCStreamSize_usingCParams_estimate
};
return ZSTD_rust_estimateCSizeUsingCParams(&state);
}
static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel)
@@ -4491,15 +4536,21 @@ static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel)
return ZSTD_estimateCStreamSize_usingCParams(cParams);
}
static size_t ZSTD_rust_estimateCStreamSize_estimateLevel(
void* context, int compressionLevel)
{
(void)context;
return ZSTD_estimateCStreamSize_internal(compressionLevel);
}
size_t ZSTD_estimateCStreamSize(int compressionLevel)
{
int level;
size_t memBudget = 0;
for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) {
size_t const newMB = ZSTD_estimateCStreamSize_internal(level);
if (newMB > memBudget) memBudget = newMB;
}
return memBudget;
ZSTD_rust_estimateCSizeState const state = {
NULL,
compressionLevel,
ZSTD_rust_estimateCStreamSize_estimateLevel
};
return ZSTD_rust_estimateCSizeMonotonic(&state);
}
/* ZSTD_getFrameProgression():