feat(compress): move CCtx workspace estimator into Rust

The CCtx workspace estimator still assembled its private sizing formula in C,
even though the match-state and component-sum leaves already lived in Rust.
That left buffering, LDM, static-context, and external-sequence sizing policy
outside the rewrite boundary.

Move the complete size_t formula into a Rust entry point. C now supplies only
private object sizes, sanitizer redzone policy, and LDM scalar fields through a
small ABI record. Rust retains the C wrapping/alignment behavior and focused
branch coverage while the C adapter preserves the existing public estimator
surface.

Test Plan:
- cargo fmt --manifest-path rust/Cargo.toml -- --check
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --release estimate_cctx_workspace_size
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --release (667 passed)
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --release --all-targets -- -D warnings
- ulimit -v 41943040; make -B -C programs -j1 zstd
- ulimit -v 41943040; make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s (84 tests and both short fuzzer rounds)
This commit is contained in:
2026-07-19 17:21:02 +02:00
parent e6101bb1af
commit 09334d2065
3 changed files with 306 additions and 44 deletions
+47 -38
View File
@@ -1124,15 +1124,27 @@ size_t ZSTD_rust_sizeofLocalDict(int dictBufferPresent, size_t dictSize,
size_t cdictSize);
size_t ZSTD_rust_sizeofCCtx(size_t objectSize, size_t workspaceSize,
size_t localDictSize, size_t mtctxSize);
size_t ZSTD_rust_estimateWorkspaceSize(size_t cctxSpace,
size_t tmpWorkSpace,
size_t blockStateSpace,
size_t ldmSpace,
size_t ldmSeqSpace,
size_t matchStateSize,
size_t tokenSpace,
size_t bufferSpace,
size_t externalSeqSpace);
typedef struct {
size_t cctxSize;
size_t compressedBlockStateSize;
size_t seqDefSize;
size_t rawSeqSize;
size_t externalSequenceSize;
size_t tmpWorkspaceSize;
size_t wildcopyOverlength;
size_t matchTSize;
size_t optimalTSize;
size_t asanRedzoneSize;
} ZSTD_rustCCtxWorkspaceSizing;
size_t ZSTD_rust_estimateCCtxWorkspaceSize(
ZSTD_compressionParameters cParams,
int ldmEnable, U32 ldmHashLog, U32 ldmBucketSizeLog,
U32 ldmMinMatchLength,
int isStatic, int useRowMatchFinder,
size_t buffInSize, size_t buffOutSize,
U64 pledgedSrcSize, int useSequenceProducer,
size_t maxBlockSize,
const ZSTD_rustCCtxWorkspaceSizing* sizing);
size_t ZSTD_rust_maxEstimateCCtxSize(size_t estimate0, size_t estimate1,
size_t estimate2, size_t estimate3);
ZSTD_inBuffer ZSTD_rust_inBufferForEndFlush(int inBufferMode,
@@ -3061,35 +3073,32 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
int useSequenceProducer,
size_t maxBlockSize)
{
size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize);
size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize);
size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useSequenceProducer);
size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
+ ZSTD_cwksp_aligned64_alloc_size(maxNbSeq * sizeof(SeqDef))
+ 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
size_t const tmpWorkSpace = ZSTD_cwksp_alloc_size(TMP_WORKSPACE_SIZE);
size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t));
size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1);
size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams);
size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize);
size_t const ldmSeqSpace = ldmParams->enableLdm == ZSTD_ps_enable ?
ZSTD_cwksp_aligned64_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0;
size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize)
+ ZSTD_cwksp_alloc_size(buffOutSize);
size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0;
size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
size_t const externalSeqSpace = useSequenceProducer
? ZSTD_cwksp_aligned64_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
: 0;
size_t const neededSpace = ZSTD_rust_estimateWorkspaceSize(
cctxSpace, tmpWorkSpace, blockStateSpace, ldmSpace, ldmSeqSpace,
matchStateSize, tokenSpace, bufferSpace, externalSeqSpace);
ZSTD_rustCCtxWorkspaceSizing const sizing = {
sizeof(ZSTD_CCtx),
sizeof(ZSTD_compressedBlockState_t),
sizeof(SeqDef),
sizeof(rawSeq),
sizeof(ZSTD_Sequence),
TMP_WORKSPACE_SIZE,
WILDCOPY_OVERLENGTH,
sizeof(ZSTD_match_t),
sizeof(ZSTD_optimal_t),
ZSTD_RUST_ASAN_REDZONE_SIZE
};
size_t const neededSpace = ZSTD_rust_estimateCCtxWorkspaceSize(
*cParams,
(int)ldmParams->enableLdm,
ldmParams->hashLog,
ldmParams->bucketSizeLog,
ldmParams->minMatchLength,
isStatic,
(int)useRowMatchFinder,
buffInSize,
buffOutSize,
pledgedSrcSize,
useSequenceProducer,
maxBlockSize,
&sizing);
DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace);
return neededSpace;