ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(mt): move failed-job serial cleanup into Rust

Browse Source 
Move the failed-job serial-state orchestration out of zstdmt_compress.c.
Rust now owns the lock, skip decision, serial-counter publication,
broadcast, LDM cleanup ordering, and final unlock. C retains the pthread
objects, private LDM window, and error/debug leaves behind callbacks. Add
ABI layout checks and focused tests for both skipped predecessors and later
jobs that only need the lock/unlock pair.

Test Plan:
- cargo fmt --manifest-path rust/Cargo.toml -- --check
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml zstdmt_compress::tests::serial_ensure_finished -- --nocapture
- ulimit -v 41943040 && make -j1
- ulimit -v 41943040 && make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests
This commit is contained in:
ddidderr
2026-07-19 20:58:18 +02:00
parent 1d8bbb421d
commit 591794e15a
2 changed files with 313 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstdmt_compress.c
+82 -19
View File
@@ -407,6 +407,40 @@ typedef struct {
} ZSTDMT_RustSerialEnsureFinishedResult;
ZSTDMT_RustSerialEnsureFinishedResult ZSTDMT_rust_serialStateEnsureFinished(
unsigned nextJobID, unsigned jobID);
typedef void (*ZSTDMT_serialStateCallbackFn)(void* opaque);
typedef void (*ZSTDMT_serialStateSkipFn)(
void* opaque, unsigned jobID, size_t cSize);
typedef struct {
void* callbackContext;
unsigned* nextJobID;
ZSTDMT_serialStateCallbackFn lock;
ZSTDMT_serialStateCallbackFn broadcast;
ZSTDMT_serialStateCallbackFn ldmLock;
ZSTDMT_serialStateCallbackFn clearLdmWindow;
ZSTDMT_serialStateCallbackFn ldmSignal;
ZSTDMT_serialStateCallbackFn ldmUnlock;
ZSTDMT_serialStateCallbackFn unlock;
ZSTDMT_serialStateSkipFn onSkip;
size_t cSize;
unsigned jobID;
} ZSTDMT_RustSerialEnsureFinishedState;
typedef char ZSTDMT_rust_serial_ensure_finished_state_layout[
(offsetof(ZSTDMT_RustSerialEnsureFinishedState, callbackContext) == 0
&& offsetof(ZSTDMT_RustSerialEnsureFinishedState, nextJobID) == sizeof(void*)
&& offsetof(ZSTDMT_RustSerialEnsureFinishedState, lock) == 2 * sizeof(void*)
&& offsetof(ZSTDMT_RustSerialEnsureFinishedState, broadcast) == 3 * sizeof(void*)
&& offsetof(ZSTDMT_RustSerialEnsureFinishedState, ldmLock) == 4 * sizeof(void*)
&& offsetof(ZSTDMT_RustSerialEnsureFinishedState, clearLdmWindow) == 5 * sizeof(void*)
&& offsetof(ZSTDMT_RustSerialEnsureFinishedState, ldmSignal) == 6 * sizeof(void*)
&& offsetof(ZSTDMT_RustSerialEnsureFinishedState, ldmUnlock) == 7 * sizeof(void*)
&& offsetof(ZSTDMT_RustSerialEnsureFinishedState, unlock) == 8 * sizeof(void*)
&& offsetof(ZSTDMT_RustSerialEnsureFinishedState, onSkip) == 9 * sizeof(void*)
&& offsetof(ZSTDMT_RustSerialEnsureFinishedState, cSize) == 10 * sizeof(void*)
&& offsetof(ZSTDMT_RustSerialEnsureFinishedState, jobID)
== 10 * sizeof(void*) + sizeof(size_t)
&& sizeof(ZSTDMT_RustSerialEnsureFinishedState) == 12 * sizeof(void*)) ? 1 : -1];
void ZSTDMT_rust_serialStateEnsureFinishedOrchestrated(
const ZSTDMT_RustSerialEnsureFinishedState* state);
typedef void (*ZSTDMT_waitForJobCompleteFn)(
void* opaque, unsigned jobID, unsigned doneJobID);
unsigned ZSTDMT_rust_waitForAllJobsCompleted(
@@ -1207,6 +1241,40 @@ static void ZSTDMT_serialState_advance(void* opaque)
ZSTDMT_rust_serialStateAdvance(&state);
}
static void ZSTDMT_serialState_ldmLock(void* opaque)
{
SerialState* const serialState = (SerialState*)opaque;
ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex);
}
static void ZSTDMT_serialState_clearLdmWindow(void* opaque)
{
SerialState* const serialState = (SerialState*)opaque;
ZSTD_rust_windowClear((size_t)(serialState->ldmWindow.nextSrc -
serialState->ldmWindow.base),
&serialState->ldmWindow.lowLimit,
&serialState->ldmWindow.dictLimit);
}
static void ZSTDMT_serialState_ldmSignal(void* opaque)
{
SerialState* const serialState = (SerialState*)opaque;
ZSTD_pthread_cond_signal(&serialState->ldmWindowCond);
}
static void ZSTDMT_serialState_ldmUnlock(void* opaque)
{
SerialState* const serialState = (SerialState*)opaque;
ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex);
}
static void ZSTDMT_serialState_onSkip(void* opaque, unsigned jobID, size_t cSize)
{
(void)opaque;
assert(ZSTD_isError(cSize)); (void)cSize;
DEBUGLOG(5, "Skipping past job %u because of error", jobID);
}
static void
ZSTDMT_serialState_applySequences(const SerialState* serialState, /* just for an assert() check */
ZSTD_CCtx* jobCCtx,
@@ -1223,25 +1291,20 @@ ZSTDMT_serialState_applySequences(const SerialState* serialState, /* just for an
static void ZSTDMT_serialState_ensureFinished(SerialState* serialState,
unsigned jobID, size_t cSize)
{
ZSTDMT_RustSerialEnsureFinishedResult result;
ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex);
result = ZSTDMT_rust_serialStateEnsureFinished(
serialState->nextJobID, jobID);
if (result.skip) {
assert(ZSTD_isError(cSize)); (void)cSize;
DEBUGLOG(5, "Skipping past job %u because of error", jobID);
serialState->nextJobID = result.nextJobID;
ZSTD_pthread_cond_broadcast(&serialState->cond);
ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex);
ZSTD_rust_windowClear((size_t)(serialState->ldmWindow.nextSrc -
serialState->ldmWindow.base),
&serialState->ldmWindow.lowLimit,
&serialState->ldmWindow.dictLimit);
ZSTD_pthread_cond_signal(&serialState->ldmWindowCond);
ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex);
}
ZSTD_pthread_mutex_unlock(&serialState->mutex);
ZSTDMT_RustSerialEnsureFinishedState state;
state.callbackContext = serialState;
state.nextJobID = &serialState->nextJobID;
state.lock = ZSTDMT_serialState_lock;
state.broadcast = ZSTDMT_serialState_broadcast;
state.ldmLock = ZSTDMT_serialState_ldmLock;
state.clearLdmWindow = ZSTDMT_serialState_clearLdmWindow;
state.ldmSignal = ZSTDMT_serialState_ldmSignal;
state.ldmUnlock = ZSTDMT_serialState_ldmUnlock;
state.unlock = ZSTDMT_serialState_unlock;
state.onSkip = ZSTDMT_serialState_onSkip;
state.cSize = cSize;
state.jobID = jobID;
ZSTDMT_rust_serialStateEnsureFinishedOrchestrated(&state);
}