ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move optimal block policy to Rust

Browse Source 
Move the frame chunk pre-split policy behind a scalar Rust ABI while retaining C ownership of ZSTD_CCtx and workspace extraction.

Test Plan: cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression; cargo clippy --manifest-path rust/Cargo.toml; cargo clippy --manifest-path rust/Cargo.toml --benches; cargo clippy --manifest-path rust/Cargo.toml --tests; make -B -C lib -j2 lib; make -B -C tests -j2 test-zstream (84 deterministic, 6697 and 9217 fuzzer cases).
This commit is contained in:
ddidderr
2026-07-18 05:38:54 +02:00
parent 1a05952013
commit 327272f8b3
2 changed files with 120 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+7 -28
View File
@@ -268,6 +268,10 @@ size_t ZSTD_rust_determineBlockSize(int mode, size_t blockSize, size_t remaining
size_t ZSTD_rust_validateSequence(U32 offBase, U32 matchLength, U32 minMatch,
size_t posInSrc, U32 windowLog, size_t dictSize,
int useSequenceProducer);
size_t ZSTD_rust_optimalBlockSize(const void* src, size_t srcSize,
size_t blockSizeMax, int splitLevel,
int strategy, S64 savings,
void* workspace, size_t workspaceSize);
typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
typedef char ZSTD_rust_stats_seqstore_long_length_pos[
@@ -3239,34 +3243,9 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_MatchState_t* ms,
static size_t ZSTD_optimalBlockSize(ZSTD_CCtx* cctx, const void* src, size_t srcSize, size_t blockSizeMax, int splitLevel, ZSTD_strategy strat, S64 savings)
{
/* split level based on compression strategy, from `fast` to `btultra2` */
static const int splitLevels[] = { 0, 0, 1, 2, 2, 3, 3, 4, 4, 4 };
/* note: conservatively only split full blocks (128 KB) currently.
* While it's possible to go lower, let's keep it simple for a first implementation.
* Besides, benefits of splitting are reduced when blocks are already small.
*/
if (srcSize < 128 KB || blockSizeMax < 128 KB)
return MIN(srcSize, blockSizeMax);
/* do not split incompressible data though:
* require verified savings to allow pre-splitting.
* Note: as a consequence, the first full block is not split.
*/
if (savings < 3) {
DEBUGLOG(6, "don't attempt splitting: savings (%i) too low", (int)savings);
return 128 KB;
}
/* apply @splitLevel, or use default value (which depends on @strat).
* note that splitting heuristic is still conditioned by @savings >= 3,
* so the first block will not reach this code path */
if (splitLevel == 1) return 128 KB;
if (splitLevel == 0) {
assert(ZSTD_fast <= strat && strat <= ZSTD_btultra2);
splitLevel = splitLevels[strat];
} else {
assert(2 <= splitLevel && splitLevel <= 6);
splitLevel -= 2;
}
return ZSTD_splitBlock(src, blockSizeMax, splitLevel, cctx->tmpWorkspace, cctx->tmpWkspSize);
return ZSTD_rust_optimalBlockSize(src, srcSize, blockSizeMax, splitLevel,
(int)strat, savings,
cctx->tmpWorkspace, cctx->tmpWkspSize);
}
/*! ZSTD_compress_frameChunk() :