feat(compress): move CCtx workspace estimator into Rust
The CCtx workspace estimator still assembled its private sizing formula in C, even though the match-state and component-sum leaves already lived in Rust. That left buffering, LDM, static-context, and external-sequence sizing policy outside the rewrite boundary. Move the complete size_t formula into a Rust entry point. C now supplies only private object sizes, sanitizer redzone policy, and LDM scalar fields through a small ABI record. Rust retains the C wrapping/alignment behavior and focused branch coverage while the C adapter preserves the existing public estimator surface. Test Plan: - cargo fmt --manifest-path rust/Cargo.toml -- --check - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --release estimate_cctx_workspace_size - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --release (667 passed) - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --release --all-targets -- -D warnings - ulimit -v 41943040; make -B -C programs -j1 zstd - ulimit -v 41943040; make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s (84 tests and both short fuzzer rounds)
This commit is contained in:
@@ -1124,15 +1124,27 @@ size_t ZSTD_rust_sizeofLocalDict(int dictBufferPresent, size_t dictSize,
|
||||
size_t cdictSize);
|
||||
size_t ZSTD_rust_sizeofCCtx(size_t objectSize, size_t workspaceSize,
|
||||
size_t localDictSize, size_t mtctxSize);
|
||||
size_t ZSTD_rust_estimateWorkspaceSize(size_t cctxSpace,
|
||||
size_t tmpWorkSpace,
|
||||
size_t blockStateSpace,
|
||||
size_t ldmSpace,
|
||||
size_t ldmSeqSpace,
|
||||
size_t matchStateSize,
|
||||
size_t tokenSpace,
|
||||
size_t bufferSpace,
|
||||
size_t externalSeqSpace);
|
||||
typedef struct {
|
||||
size_t cctxSize;
|
||||
size_t compressedBlockStateSize;
|
||||
size_t seqDefSize;
|
||||
size_t rawSeqSize;
|
||||
size_t externalSequenceSize;
|
||||
size_t tmpWorkspaceSize;
|
||||
size_t wildcopyOverlength;
|
||||
size_t matchTSize;
|
||||
size_t optimalTSize;
|
||||
size_t asanRedzoneSize;
|
||||
} ZSTD_rustCCtxWorkspaceSizing;
|
||||
size_t ZSTD_rust_estimateCCtxWorkspaceSize(
|
||||
ZSTD_compressionParameters cParams,
|
||||
int ldmEnable, U32 ldmHashLog, U32 ldmBucketSizeLog,
|
||||
U32 ldmMinMatchLength,
|
||||
int isStatic, int useRowMatchFinder,
|
||||
size_t buffInSize, size_t buffOutSize,
|
||||
U64 pledgedSrcSize, int useSequenceProducer,
|
||||
size_t maxBlockSize,
|
||||
const ZSTD_rustCCtxWorkspaceSizing* sizing);
|
||||
size_t ZSTD_rust_maxEstimateCCtxSize(size_t estimate0, size_t estimate1,
|
||||
size_t estimate2, size_t estimate3);
|
||||
ZSTD_inBuffer ZSTD_rust_inBufferForEndFlush(int inBufferMode,
|
||||
@@ -3061,35 +3073,32 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
|
||||
int useSequenceProducer,
|
||||
size_t maxBlockSize)
|
||||
{
|
||||
size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize);
|
||||
size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize);
|
||||
size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useSequenceProducer);
|
||||
size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
|
||||
+ ZSTD_cwksp_aligned64_alloc_size(maxNbSeq * sizeof(SeqDef))
|
||||
+ 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
|
||||
size_t const tmpWorkSpace = ZSTD_cwksp_alloc_size(TMP_WORKSPACE_SIZE);
|
||||
size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t));
|
||||
size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1);
|
||||
|
||||
size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams);
|
||||
size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize);
|
||||
size_t const ldmSeqSpace = ldmParams->enableLdm == ZSTD_ps_enable ?
|
||||
ZSTD_cwksp_aligned64_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0;
|
||||
|
||||
|
||||
size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize)
|
||||
+ ZSTD_cwksp_alloc_size(buffOutSize);
|
||||
|
||||
size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0;
|
||||
|
||||
size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
|
||||
size_t const externalSeqSpace = useSequenceProducer
|
||||
? ZSTD_cwksp_aligned64_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
|
||||
: 0;
|
||||
|
||||
size_t const neededSpace = ZSTD_rust_estimateWorkspaceSize(
|
||||
cctxSpace, tmpWorkSpace, blockStateSpace, ldmSpace, ldmSeqSpace,
|
||||
matchStateSize, tokenSpace, bufferSpace, externalSeqSpace);
|
||||
ZSTD_rustCCtxWorkspaceSizing const sizing = {
|
||||
sizeof(ZSTD_CCtx),
|
||||
sizeof(ZSTD_compressedBlockState_t),
|
||||
sizeof(SeqDef),
|
||||
sizeof(rawSeq),
|
||||
sizeof(ZSTD_Sequence),
|
||||
TMP_WORKSPACE_SIZE,
|
||||
WILDCOPY_OVERLENGTH,
|
||||
sizeof(ZSTD_match_t),
|
||||
sizeof(ZSTD_optimal_t),
|
||||
ZSTD_RUST_ASAN_REDZONE_SIZE
|
||||
};
|
||||
size_t const neededSpace = ZSTD_rust_estimateCCtxWorkspaceSize(
|
||||
*cParams,
|
||||
(int)ldmParams->enableLdm,
|
||||
ldmParams->hashLog,
|
||||
ldmParams->bucketSizeLog,
|
||||
ldmParams->minMatchLength,
|
||||
isStatic,
|
||||
(int)useRowMatchFinder,
|
||||
buffInSize,
|
||||
buffOutSize,
|
||||
pledgedSrcSize,
|
||||
useSequenceProducer,
|
||||
maxBlockSize,
|
||||
&sizing);
|
||||
|
||||
DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace);
|
||||
return neededSpace;
|
||||
|
||||
Reference in New Issue
Block a user