ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(mt): move LDM window wait policy into Rust

Browse Source 
Move the MT reusable-input wait decision and lock/overlap/condition-wait
ordering into Rust through a small callback projection. C retains ownership
of the pthread synchronization objects, private LDM window, and diagnostics.
Add layout assertions and callback-order tests for enabled and disabled LDM.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1; cargo test --manifest-path rust/Cargo.toml
- ulimit -v 41943040; CARGO_BUILD_JOBS=1; cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040; make -B -C programs -j1 zstd
- ulimit -v 41943040; make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s
This commit is contained in:
ddidderr
2026-07-19 19:41:35 +02:00
parent bd9ca5115d
commit 082c5a7dbe
2 changed files with 232 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstdmt_compress.c
+68 -9
View File
@@ -368,6 +368,33 @@ void ZSTDMT_rust_serialStateGenSequences(
ZSTDMT_serialGenerateLdmFn generateLdm,
ZSTDMT_serialUpdateChecksumFn updateChecksum,
ZSTDMT_serialAdvanceFn advance);
typedef void (*ZSTDMT_waitForLdmLockFn)(void* opaque);
typedef int (*ZSTDMT_waitForLdmOverlapFn)(
void* opaque, void* bufferStart, size_t bufferCapacity);
typedef void (*ZSTDMT_waitForLdmWaitFn)(void* opaque);
typedef void (*ZSTDMT_waitForLdmUnlockFn)(void* opaque);
typedef struct {
void* callbackContext;
void* bufferStart;
size_t bufferCapacity;
int ldmEnabled;
ZSTDMT_waitForLdmLockFn lock;
ZSTDMT_waitForLdmOverlapFn overlaps;
ZSTDMT_waitForLdmWaitFn wait;
ZSTDMT_waitForLdmUnlockFn unlock;
} ZSTDMT_RustWaitForLdmState;
typedef char ZSTDMT_rust_wait_for_ldm_state_layout[
(offsetof(ZSTDMT_RustWaitForLdmState, callbackContext) == 0
&& offsetof(ZSTDMT_RustWaitForLdmState, bufferStart) == sizeof(void*)
&& offsetof(ZSTDMT_RustWaitForLdmState, bufferCapacity) == 2 * sizeof(void*)
&& offsetof(ZSTDMT_RustWaitForLdmState, ldmEnabled) == 3 * sizeof(void*)
&& offsetof(ZSTDMT_RustWaitForLdmState, lock) == 4 * sizeof(void*)
&& offsetof(ZSTDMT_RustWaitForLdmState, overlaps) == 5 * sizeof(void*)
&& offsetof(ZSTDMT_RustWaitForLdmState, wait) == 6 * sizeof(void*)
&& offsetof(ZSTDMT_RustWaitForLdmState, unlock) == 7 * sizeof(void*)
&& sizeof(ZSTDMT_RustWaitForLdmState) == 8 * sizeof(void*)) ? 1 : -1];
void ZSTDMT_rust_waitForLdmComplete(
const ZSTDMT_RustWaitForLdmState* state);
unsigned ZSTDMT_rust_computeTargetJobLog(unsigned windowLog, unsigned chainLog,
int strategy, int enableLdm);
@@ -2102,22 +2129,54 @@ static int ZSTDMT_doesOverlapWindow(Buffer buffer, ZSTD_window_t window)
window.lowLimit);
}
static void ZSTDMT_waitForLdmLock(void* opaque)
{
ZSTDMT_CCtx* const mtctx = (ZSTDMT_CCtx*)opaque;
ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->serial.ldmWindowMutex);
}
static int ZSTDMT_waitForLdmOverlap(
void* opaque, void* bufferStart, size_t bufferCapacity)
{
ZSTDMT_CCtx* const mtctx = (ZSTDMT_CCtx*)opaque;
Buffer const buffer = { bufferStart, bufferCapacity };
return ZSTDMT_doesOverlapWindow(buffer, mtctx->serial.ldmWindow);
}
static void ZSTDMT_waitForLdmWait(void* opaque)
{
ZSTDMT_CCtx* const mtctx = (ZSTDMT_CCtx*)opaque;
DEBUGLOG(5, "Waiting for LDM to finish...");
ZSTD_pthread_cond_wait(&mtctx->serial.ldmWindowCond,
&mtctx->serial.ldmWindowMutex);
}
static void ZSTDMT_waitForLdmUnlock(void* opaque)
{
ZSTDMT_CCtx* const mtctx = (ZSTDMT_CCtx*)opaque;
DEBUGLOG(6, "Done waiting for LDM to finish");
ZSTD_pthread_mutex_unlock(&mtctx->serial.ldmWindowMutex);
}
static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, Buffer buffer)
{
if (mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable) {
ZSTD_pthread_mutex_t* mutex = &mtctx->serial.ldmWindowMutex;
int const ldmEnabled = mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable;
ZSTDMT_RustWaitForLdmState state;
if (ldmEnabled) {
DEBUGLOG(5, "ZSTDMT_waitForLdmComplete");
DEBUGLOG(5, "source [0x%zx, 0x%zx)",
(size_t)buffer.start,
(size_t)buffer.start + buffer.capacity);
ZSTD_PTHREAD_MUTEX_LOCK(mutex);
while (ZSTDMT_doesOverlapWindow(buffer, mtctx->serial.ldmWindow)) {
DEBUGLOG(5, "Waiting for LDM to finish...");
ZSTD_pthread_cond_wait(&mtctx->serial.ldmWindowCond, mutex);
}
DEBUGLOG(6, "Done waiting for LDM to finish");
ZSTD_pthread_mutex_unlock(mutex);
}
state.callbackContext = mtctx;
state.bufferStart = buffer.start;
state.bufferCapacity = buffer.capacity;
state.ldmEnabled = ldmEnabled;
state.lock = ZSTDMT_waitForLdmLock;
state.overlaps = ZSTDMT_waitForLdmOverlap;
state.wait = ZSTDMT_waitForLdmWait;
state.unlock = ZSTDMT_waitForLdmUnlock;
ZSTDMT_rust_waitForLdmComplete(&state);
}
/**