ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move MT sizing policy to Rust

Browse Source 
Move target job-log, overlap-log, and overlap-size policy into the Rust multithreaded compression module, retaining the C context projection and logging wrapper. Add policy tests for LDM, strategy defaults, bounds, and overlap sizing.

Test Plan: cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression; cargo clippy --manifest-path rust/Cargo.toml; cargo clippy --manifest-path rust/Cargo.toml --benches; cargo clippy --manifest-path rust/Cargo.toml --tests; make -B -C lib -j2 lib; make -B -C programs -j2 zstd zstd-small zstd-frugal; make -B -C tests -j2 test-cli-tests; make -B -C tests -j2 test-zstream
This commit is contained in:
ddidderr
2026-07-18 03:40:12 +02:00
parent 386ec3d0a7
commit 429d5176e9
2 changed files with 213 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstdmt_compress.c
+19 -52
View File
@@ -115,6 +115,12 @@ void ZSTDMT_rust_buffer_pool_release(ZSTDMT_RustBufferPool* pool,
ZSTDMT_RustBuffer ZSTDMT_rust_buffer_pool_resize(ZSTDMT_RustBufferPool* pool,
ZSTDMT_RustBuffer buffer);
unsigned ZSTDMT_rust_computeTargetJobLog(unsigned windowLog, unsigned chainLog,
int strategy, int enableLdm);
int ZSTDMT_rust_overlapLog(int overlapLog, int strategy);
size_t ZSTDMT_rust_computeOverlapSize(unsigned windowLog, unsigned chainLog,
int strategy, int overlapLog, int enableLdm);
typedef struct ZSTDMT_bufferPool_s {
ZSTDMT_RustBufferPool* rustPool;
ZSTD_customMem cMem;
@@ -1103,63 +1109,24 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx)
static unsigned ZSTDMT_computeTargetJobLog(const ZSTD_CCtx_params* params)
{
unsigned jobLog;
if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
/* In Long Range Mode, the windowLog is typically oversized.
* In which case, it's preferable to determine the jobSize
* based on cycleLog instead. */
jobLog = MAX(21, ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy) + 3);
} else {
jobLog = MAX(20, params->cParams.windowLog + 2);
}
return MIN(jobLog, (unsigned)ZSTDMT_JOBLOG_MAX);
}
static int ZSTDMT_overlapLog_default(ZSTD_strategy strat)
{
switch(strat)
{
case ZSTD_btultra2:
return 9;
case ZSTD_btultra:
case ZSTD_btopt:
return 8;
case ZSTD_btlazy2:
case ZSTD_lazy2:
return 7;
case ZSTD_lazy:
case ZSTD_greedy:
case ZSTD_dfast:
case ZSTD_fast:
default:;
}
return 6;
}
static int ZSTDMT_overlapLog(int ovlog, ZSTD_strategy strat)
{
assert(0 <= ovlog && ovlog <= 9);
if (ovlog == 0) return ZSTDMT_overlapLog_default(strat);
return ovlog;
return ZSTDMT_rust_computeTargetJobLog(params->cParams.windowLog,
params->cParams.chainLog,
(int)params->cParams.strategy,
(int)params->ldmParams.enableLdm);
}
static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params)
{
int const overlapRLog = 9 - ZSTDMT_overlapLog(params->overlapLog, params->cParams.strategy);
int ovLog = (overlapRLog >= 8) ? 0 : (params->cParams.windowLog - overlapRLog);
assert(0 <= overlapRLog && overlapRLog <= 8);
if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
/* In Long Range Mode, the windowLog is typically oversized.
* In which case, it's preferable to determine the jobSize
* based on chainLog instead.
* Then, ovLog becomes a fraction of the jobSize, rather than windowSize */
ovLog = MIN(params->cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2)
- overlapRLog;
}
assert(0 <= ovLog && ovLog <= ZSTD_WINDOWLOG_MAX);
size_t overlapSize;
assert(0 <= params->overlapLog && params->overlapLog <= 9);
overlapSize = ZSTDMT_rust_computeOverlapSize(params->cParams.windowLog,
params->cParams.chainLog,
(int)params->cParams.strategy,
params->overlapLog,
(int)params->ldmParams.enableLdm);
DEBUGLOG(4, "overlapLog : %i", params->overlapLog);
DEBUGLOG(4, "overlap size : %i", 1 << ovLog);
return (ovLog==0) ? 0 : (size_t)1 << ovLog;
DEBUGLOG(4, "overlap size : %i", (int)(overlapSize == 0 ? 1 : overlapSize));
return overlapSize;
}
/* ====================================== */