ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move MT stream initialization policy into Rust

Browse Source 
Move the high-level ZSTDMT_initCStream_internal setup policy into Rust. Rust
now owns worker-count resizing decisions, job-size normalization, unfinished-
job draining order, overlap and section sizing, rsync setup, buffer sizing,
and stream reset sequencing through a scalar projection and callbacks.

Keep MT contexts, pools, job resources, dictionaries, buffers, synchronization,
and serial state private to C. C callbacks perform those private mutations while
Rust controls the transparent initialization flow and can test its normalization
and ordering independently of the private layouts.

Test Plan:
- cargo test --manifest-path rust/Cargo.toml --all-targets -- --test-threads=1
- cargo test --manifest-path rust/cli/Cargo.toml --all-targets -- --test-threads=1
- run the legacy Rust feature matrix and all six library/CLI clippy gates with
  -D warnings
- run lib and program native rebuilds plus test-cli-tests,
  test-rust-lib-smoke, and test-zstd with make -j1
- run fuzzer, zstream, and decode-corpus stress gates serially with
  ulimit -v 41943040

Commit is intentionally unsigned because GPG pinentry hangs in this
non-interactive environment.
This commit is contained in:
ddidderr
2026-07-19 09:43:28 +02:00
parent a9589d2d7d
commit 15d42bcc95
2 changed files with 704 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstdmt_compress.c
+222 -130
View File
@@ -371,6 +371,44 @@ void ZSTDMT_rust_findSynchronizationPoint(const void* inputSrc, size_t inputSize
U64 ZSTDMT_rust_rollingHashPrimePower(U32 length);
size_t ZSTDMT_rust_nextInputSizeHint(size_t targetSectionSize,
size_t inBuffFilled);
typedef struct {
unsigned requestedNbWorkers;
unsigned currentNbWorkers;
size_t jobSize;
size_t jobSizeMin;
size_t jobSizeMax;
int enableLdm;
unsigned windowLog;
unsigned chainLog;
int strategy;
int overlapLog;
int rsyncable;
size_t roundBuffCapacity;
unsigned allJobsCompleted;
} ZSTDMT_RustInitCStreamProjection;
typedef size_t (*ZSTDMT_initResizeFn)(void* opaque, unsigned nbWorkers);
typedef void (*ZSTDMT_initDrainFn)(void* opaque);
typedef void (*ZSTDMT_initApplyParametersFn)(void* opaque, size_t jobSize);
typedef size_t (*ZSTDMT_initDictionaryFn)(void* opaque);
typedef void (*ZSTDMT_initSetSizeFn)(void* opaque, size_t size);
typedef void (*ZSTDMT_initSetRsyncFn)(void* opaque, U64 hitMask, U64 primePower);
typedef void (*ZSTDMT_initSetBufferSizeFn)(void* opaque, size_t size);
typedef size_t (*ZSTDMT_initResizeRoundBufferFn)(void* opaque, size_t capacity);
typedef void (*ZSTDMT_initResetStreamFn)(void* opaque);
typedef size_t (*ZSTDMT_initSerialResetFn)(void* opaque, size_t targetSectionSize);
size_t ZSTDMT_rust_initCStream(
const ZSTDMT_RustInitCStreamProjection* projection, void* opaque,
ZSTDMT_initResizeFn resize, ZSTDMT_initDrainFn drain,
ZSTDMT_initApplyParametersFn applyParameters,
ZSTDMT_initDictionaryFn prepareDictionary,
ZSTDMT_initSetSizeFn setTargetPrefixSize,
ZSTDMT_initSetSizeFn setTargetSectionSize,
ZSTDMT_initSetRsyncFn setRsync,
ZSTDMT_initSetBufferSizeFn setBufferSize,
ZSTDMT_initResizeRoundBufferFn resizeRoundBuffer,
ZSTDMT_initResetStreamFn resetStream,
ZSTDMT_initDictionaryFn updateDictionary,
ZSTDMT_initSerialResetFn serialReset);
typedef struct {
size_t consumed;
size_t cSize;
@@ -1524,26 +1562,159 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx)
/* ===== Multi-threaded compression ===== */
/* ------------------------------------------ */
static unsigned ZSTDMT_computeTargetJobLog(const ZSTD_CCtx_params* params)
typedef struct {
ZSTDMT_CCtx* mtctx;
ZSTD_CCtx_params params;
const void* dict;
size_t dictSize;
ZSTD_dictContentType_e dictContentType;
const ZSTD_CDict* cdict;
unsigned long long pledgedSrcSize;
} ZSTDMT_initCStreamState;
static size_t ZSTDMT_initCStreamResize(void* opaque, unsigned nbWorkers)
{
return ZSTDMT_rust_computeTargetJobLog(params->cParams.windowLog,
params->cParams.chainLog,
(int)params->cParams.strategy,
(int)params->ldmParams.enableLdm);
ZSTDMT_initCStreamState* const state = (ZSTDMT_initCStreamState*)opaque;
return ZSTDMT_resize(state->mtctx, nbWorkers);
}
static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params)
static void ZSTDMT_initCStreamDrain(void* opaque)
{
size_t overlapSize;
assert(0 <= params->overlapLog && params->overlapLog <= 9);
overlapSize = ZSTDMT_rust_computeOverlapSize(params->cParams.windowLog,
params->cParams.chainLog,
(int)params->cParams.strategy,
params->overlapLog,
(int)params->ldmParams.enableLdm);
DEBUGLOG(4, "overlapLog : %i", params->overlapLog);
DEBUGLOG(4, "overlap size : %i", (int)(overlapSize == 0 ? 1 : overlapSize));
return overlapSize;
ZSTDMT_initCStreamState* const state = (ZSTDMT_initCStreamState*)opaque;
ZSTDMT_CCtx* const mtctx = state->mtctx;
ZSTDMT_waitForAllJobsCompleted(mtctx);
ZSTDMT_releaseAllJobResources(mtctx);
mtctx->allJobsCompleted = 1;
}
static void ZSTDMT_initCStreamApplyParameters(void* opaque, size_t jobSize)
{
ZSTDMT_initCStreamState* const state = (ZSTDMT_initCStreamState*)opaque;
state->params.jobSize = jobSize;
state->mtctx->params = state->params;
state->mtctx->frameContentSize = state->pledgedSrcSize;
}
static size_t ZSTDMT_initCStreamPrepareDictionary(void* opaque)
{
ZSTDMT_initCStreamState* const state = (ZSTDMT_initCStreamState*)opaque;
ZSTDMT_CCtx* const mtctx = state->mtctx;
ZSTD_freeCDict(mtctx->cdictLocal);
if (state->dict) {
mtctx->cdictLocal = ZSTD_createCDict_advanced(
state->dict, state->dictSize, ZSTD_dlm_byCopy,
state->dictContentType, state->params.cParams, mtctx->cMem);
mtctx->cdict = mtctx->cdictLocal;
if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation);
} else {
mtctx->cdictLocal = NULL;
mtctx->cdict = state->cdict;
}
return 0;
}
static void ZSTDMT_initCStreamSetTargetPrefixSize(void* opaque, size_t size)
{
ZSTDMT_initCStreamState* const state = (ZSTDMT_initCStreamState*)opaque;
state->mtctx->targetPrefixSize = size;
DEBUGLOG(4, "overlapLog=%i => %u KB", state->params.overlapLog, (U32)(size >> 10));
}
static void ZSTDMT_initCStreamSetTargetSectionSize(void* opaque, size_t size)
{
ZSTDMT_initCStreamState* const state = (ZSTDMT_initCStreamState*)opaque;
state->mtctx->targetSectionSize = size;
}
static void ZSTDMT_initCStreamSetRsync(void* opaque, U64 hitMask, U64 primePower)
{
ZSTDMT_initCStreamState* const state = (ZSTDMT_initCStreamState*)opaque;
state->mtctx->rsync.hash = 0;
state->mtctx->rsync.hitMask = hitMask;
state->mtctx->rsync.primePower = primePower;
DEBUGLOG(4, "rsyncLog = %u", ZSTD_highbit32((U32)(hitMask + 1)));
}
static void ZSTDMT_initCStreamSetBufferSize(void* opaque, size_t size)
{
ZSTDMT_initCStreamState* const state = (ZSTDMT_initCStreamState*)opaque;
DEBUGLOG(4, "Job Size : %u KB (note : set to %u)",
(U32)(state->mtctx->targetSectionSize >> 10),
(U32)state->params.jobSize);
DEBUGLOG(4, "inBuff Size : %u KB", (U32)(state->mtctx->targetSectionSize >> 10));
ZSTDMT_setBufferSize(state->mtctx->bufPool, size);
}
static size_t ZSTDMT_initCStreamResizeRoundBuffer(void* opaque, size_t capacity)
{
ZSTDMT_initCStreamState* const state = (ZSTDMT_initCStreamState*)opaque;
ZSTDMT_CCtx* const mtctx = state->mtctx;
if (mtctx->roundBuff.capacity < capacity) {
if (mtctx->roundBuff.buffer)
ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem);
mtctx->roundBuff.buffer = (BYTE*)ZSTD_customMalloc(capacity, mtctx->cMem);
if (mtctx->roundBuff.buffer == NULL) {
mtctx->roundBuff.capacity = 0;
return ERROR(memory_allocation);
}
mtctx->roundBuff.capacity = capacity;
}
return 0;
}
static void ZSTDMT_initCStreamResetStream(void* opaque)
{
ZSTDMT_initCStreamState* const state = (ZSTDMT_initCStreamState*)opaque;
ZSTDMT_CCtx* const mtctx = state->mtctx;
DEBUGLOG(4, "roundBuff capacity : %u KB", (U32)(mtctx->roundBuff.capacity >> 10));
mtctx->roundBuff.pos = 0;
mtctx->inBuff.buffer = g_nullBuffer;
mtctx->inBuff.filled = 0;
mtctx->inBuff.prefix = kNullRange;
mtctx->doneJobID = 0;
mtctx->nextJobID = 0;
mtctx->frameEnded = 0;
mtctx->allJobsCompleted = 0;
mtctx->consumed = 0;
mtctx->produced = 0;
}
static size_t ZSTDMT_initCStreamUpdateDictionary(void* opaque)
{
ZSTDMT_initCStreamState* const state = (ZSTDMT_initCStreamState*)opaque;
ZSTDMT_CCtx* const mtctx = state->mtctx;
ZSTD_freeCDict(mtctx->cdictLocal);
mtctx->cdictLocal = NULL;
mtctx->cdict = NULL;
if (state->dict) {
if (state->dictContentType == ZSTD_dct_rawContent) {
mtctx->inBuff.prefix.start = (const BYTE*)state->dict;
mtctx->inBuff.prefix.size = state->dictSize;
} else {
/* note : a loadPrefix becomes an internal CDict */
mtctx->cdictLocal = ZSTD_createCDict_advanced(
state->dict, state->dictSize, ZSTD_dlm_byRef,
state->dictContentType, state->params.cParams, mtctx->cMem);
mtctx->cdict = mtctx->cdictLocal;
if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation);
}
} else {
mtctx->cdict = state->cdict;
}
return 0;
}
static size_t ZSTDMT_initCStreamSerialReset(void* opaque, size_t targetSectionSize)
{
ZSTDMT_initCStreamState* const state = (ZSTDMT_initCStreamState*)opaque;
if (ZSTDMT_serialState_reset(
&state->mtctx->serial, state->mtctx->seqPool, state->params,
targetSectionSize, state->dict, state->dictSize,
state->dictContentType))
return ERROR(memory_allocation);
return 0;
}
/* ====================================== */
@@ -1556,6 +1727,9 @@ size_t ZSTDMT_initCStream_internal(
const ZSTD_CDict* cdict, ZSTD_CCtx_params params,
unsigned long long pledgedSrcSize)
{
ZSTDMT_initCStreamState state;
ZSTDMT_RustInitCStreamProjection projection;
DEBUGLOG(4, "ZSTDMT_initCStream_internal (pledgedSrcSize=%u, nbWorkers=%u, cctxPool=%u)",
(U32)pledgedSrcSize, params.nbWorkers, mtctx->cctxPool->totalCCtx);
@@ -1563,121 +1737,39 @@ size_t ZSTDMT_initCStream_internal(
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
assert(!((dict) && (cdict))); /* either dict or cdict, not both */
/* init */
if (params.nbWorkers != mtctx->params.nbWorkers)
FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, (unsigned)params.nbWorkers) , "");
state = (ZSTDMT_initCStreamState){
mtctx, params, dict, dictSize, dictContentType, cdict, pledgedSrcSize
};
projection = (ZSTDMT_RustInitCStreamProjection){
(unsigned)params.nbWorkers,
(unsigned)mtctx->params.nbWorkers,
params.jobSize,
ZSTDMT_JOBSIZE_MIN,
(size_t)ZSTDMT_JOBSIZE_MAX,
params.ldmParams.enableLdm,
params.cParams.windowLog,
params.cParams.chainLog,
params.cParams.strategy,
params.overlapLog,
params.rsyncable,
mtctx->roundBuff.capacity,
mtctx->allJobsCompleted
};
if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN;
if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX;
if (mtctx->allJobsCompleted == 0) { /* previous compression not correctly finished */
ZSTDMT_waitForAllJobsCompleted(mtctx);
ZSTDMT_releaseAllJobResources(mtctx);
mtctx->allJobsCompleted = 1;
}
mtctx->params = params;
mtctx->frameContentSize = pledgedSrcSize;
ZSTD_freeCDict(mtctx->cdictLocal);
if (dict) {
mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
ZSTD_dlm_byCopy, dictContentType, /* note : a loadPrefix becomes an internal CDict */
params.cParams, mtctx->cMem);
mtctx->cdict = mtctx->cdictLocal;
if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation);
} else {
mtctx->cdictLocal = NULL;
mtctx->cdict = cdict;
}
mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(&params);
DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10));
mtctx->targetSectionSize = params.jobSize;
if (mtctx->targetSectionSize == 0) {
mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(&params);
}
assert(mtctx->targetSectionSize <= (size_t)ZSTDMT_JOBSIZE_MAX);
if (params.rsyncable) {
/* Aim for the targetsectionSize as the average job size. */
U32 const jobSizeKB = (U32)(mtctx->targetSectionSize >> 10);
U32 const rsyncBits = (assert(jobSizeKB >= 1), ZSTD_highbit32(jobSizeKB) + 10);
/* We refuse to create jobs < RSYNC_MIN_BLOCK_SIZE bytes, so make sure our
* expected job size is at least 4x larger. */
assert(rsyncBits >= RSYNC_MIN_BLOCK_LOG + 2);
DEBUGLOG(4, "rsyncLog = %u", rsyncBits);
mtctx->rsync.hash = 0;
mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1;
mtctx->rsync.primePower = ZSTDMT_rust_rollingHashPrimePower(RSYNC_LENGTH);
}
if (mtctx->targetSectionSize < mtctx->targetPrefixSize) mtctx->targetSectionSize = mtctx->targetPrefixSize; /* job size must be >= overlap size */
DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), (U32)params.jobSize);
DEBUGLOG(4, "inBuff Size : %u KB", (U32)(mtctx->targetSectionSize>>10));
ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize));
{
/* If ldm is enabled we need windowSize space. */
size_t const windowSize = mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable ? (1U << mtctx->params.cParams.windowLog) : 0;
/* Two buffers of slack, plus extra space for the overlap
* This is the minimum slack that LDM works with. One extra because
* flush might waste up to targetSectionSize-1 bytes. Another extra
* for the overlap (if > 0), then one to fill which doesn't overlap
* with the LDM window.
*/
size_t const nbSlackBuffers = 2 + (mtctx->targetPrefixSize > 0);
size_t const slackSize = mtctx->targetSectionSize * nbSlackBuffers;
/* Compute the total size, and always have enough slack */
size_t const nbWorkers = MAX(mtctx->params.nbWorkers, 1);
size_t const sectionsSize = mtctx->targetSectionSize * nbWorkers;
size_t const capacity = MAX(windowSize, sectionsSize) + slackSize;
if (mtctx->roundBuff.capacity < capacity) {
if (mtctx->roundBuff.buffer)
ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem);
mtctx->roundBuff.buffer = (BYTE*)ZSTD_customMalloc(capacity, mtctx->cMem);
if (mtctx->roundBuff.buffer == NULL) {
mtctx->roundBuff.capacity = 0;
return ERROR(memory_allocation);
}
mtctx->roundBuff.capacity = capacity;
}
}
DEBUGLOG(4, "roundBuff capacity : %u KB", (U32)(mtctx->roundBuff.capacity>>10));
mtctx->roundBuff.pos = 0;
mtctx->inBuff.buffer = g_nullBuffer;
mtctx->inBuff.filled = 0;
mtctx->inBuff.prefix = kNullRange;
mtctx->doneJobID = 0;
mtctx->nextJobID = 0;
mtctx->frameEnded = 0;
mtctx->allJobsCompleted = 0;
mtctx->consumed = 0;
mtctx->produced = 0;
/* update dictionary */
ZSTD_freeCDict(mtctx->cdictLocal);
mtctx->cdictLocal = NULL;
mtctx->cdict = NULL;
if (dict) {
if (dictContentType == ZSTD_dct_rawContent) {
mtctx->inBuff.prefix.start = (const BYTE*)dict;
mtctx->inBuff.prefix.size = dictSize;
} else {
/* note : a loadPrefix becomes an internal CDict */
mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
ZSTD_dlm_byRef, dictContentType,
params.cParams, mtctx->cMem);
mtctx->cdict = mtctx->cdictLocal;
if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation);
}
} else {
mtctx->cdict = cdict;
}
if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize,
dict, dictSize, dictContentType))
return ERROR(memory_allocation);
return 0;
return ZSTDMT_rust_initCStream(
&projection, &state,
ZSTDMT_initCStreamResize,
ZSTDMT_initCStreamDrain,
ZSTDMT_initCStreamApplyParameters,
ZSTDMT_initCStreamPrepareDictionary,
ZSTDMT_initCStreamSetTargetPrefixSize,
ZSTDMT_initCStreamSetTargetSectionSize,
ZSTDMT_initCStreamSetRsync,
ZSTDMT_initCStreamSetBufferSize,
ZSTDMT_initCStreamResizeRoundBuffer,
ZSTDMT_initCStreamResetStream,
ZSTDMT_initCStreamUpdateDictionary,
ZSTDMT_initCStreamSerialReset);
}