ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(compress): move MT stream loop to Rust

Browse Source 
Move the multithreaded compressStream2 outer coordinator into the Rust compression module. The C bridge now owns only the private ZSTDMT call and consumed/produced counters, while Rust preserves the progress loop, completion ordering, and error handoff.

Keep the MT context mutations, diagnostics, assertions, and buffer-expectation publication in C because they depend on the private context layout. Add callback-driven tests for continue/break, error completion, end trace/reset ordering, and malformed states.

Test Plan:

- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo +nightly fmt --manifest-path rust/Cargo.toml --all -- --check

- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings

- ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1

- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/cli/Cargo.toml --all-targets

- ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1 -C tests test

- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings
This commit is contained in:
ddidderr
2026-07-21 12:14:35 +02:00
parent 1530b4bf4f
commit 5bc03f4e04
2 changed files with 431 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+58 -34
View File
@@ -1012,6 +1012,36 @@ typedef char ZSTD_rust_compress_stream2_mt_completion_state_layout[
&& sizeof(ZSTD_rust_compressStream2MTCompletionState)
== 2 * sizeof(void*))
? 1 : -1];
typedef size_t (*ZSTD_rust_compressStream2MTStep_f)(
void* context, ZSTD_outBuffer* output, ZSTD_inBuffer* input, int endOp);
typedef struct {
void* callbackContext;
ZSTD_rust_compressStream2MTStep_f step;
ZSTD_outBuffer* output;
ZSTD_inBuffer* input;
const ZSTD_rust_compressStream2MTCompletionCallbacks* completionCallbacks;
int endOp;
} ZSTD_rust_compressStream2MTCoordinatorState;
size_t ZSTD_rust_compressStream2MTCoordinator(
const ZSTD_rust_compressStream2MTCoordinatorState* state);
typedef char ZSTD_rust_compress_stream2_mt_coordinator_state_layout[
(offsetof(ZSTD_rust_compressStream2MTCoordinatorState, callbackContext)
== 0
&& offsetof(ZSTD_rust_compressStream2MTCoordinatorState, step)
== sizeof(void*)
&& offsetof(ZSTD_rust_compressStream2MTCoordinatorState, output)
== 2 * sizeof(void*)
&& offsetof(ZSTD_rust_compressStream2MTCoordinatorState, input)
== 3 * sizeof(void*)
&& offsetof(ZSTD_rust_compressStream2MTCoordinatorState,
completionCallbacks)
== 4 * sizeof(void*)
&& offsetof(ZSTD_rust_compressStream2MTCoordinatorState, endOp)
== 5 * sizeof(void*)
&& sizeof(ZSTD_rust_compressStream2MTStep_f) == sizeof(void*)
&& sizeof(ZSTD_rust_compressStream2MTCoordinatorState)
== 6 * sizeof(void*))
? 1 : -1];
int ZSTD_rust_simpleCompress2Level(const void* cctx, size_t srcSize);
typedef int (*ZSTD_rust_simpleCompress2Level_f)(const void* cctx, size_t srcSize);
typedef struct {
@@ -8674,6 +8704,19 @@ static size_t ZSTD_rust_compressStream2MT_reset(void* context)
{
return ZSTD_CCtx_reset((ZSTD_CCtx*)context, ZSTD_reset_session_only);
}
static size_t ZSTD_rust_compressStream2MT_step(
void* context, ZSTD_outBuffer* output, ZSTD_inBuffer* input, int endOp)
{
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context;
size_t const ipos = input->pos;
size_t const opos = output->pos;
size_t const flushMin = ZSTDMT_compressStream_generic(
cctx->mtctx, output, input, (ZSTD_EndDirective)endOp);
cctx->consumedSrcSize += (U64)(input->pos - ipos);
cctx->producedCSize += (U64)(output->pos - opos);
return flushMin;
}
#endif /* ZSTD_MULTITHREAD */
size_t ZSTD_compressStream2_c( ZSTD_CCtx* cctx,
@@ -8781,41 +8824,22 @@ size_t ZSTD_compressStream2_c( ZSTD_CCtx* cctx,
input->pos -= cctx->stableIn_notConsumed;
cctx->stableIn_notConsumed = 0;
}
for (;;) {
size_t const ipos = input->pos;
size_t const opos = output->pos;
flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
cctx->consumedSrcSize += (U64)(input->pos - ipos);
cctx->producedCSize += (U64)(output->pos - opos);
{ ZSTD_rust_compressStream2MTLoopPolicyState const state = {
(int)endOp,
flushMin,
input->pos,
input->size,
ipos,
output->pos,
output->size,
opos
};
int const loopPolicy =
ZSTD_rust_compressStream2MTLoopPolicy(&state);
{ ZSTD_rust_compressStream2MTCompletionCallbacks const callbacks = {
cctx,
ZSTD_rust_compressStream2MT_trace,
ZSTD_rust_compressStream2MT_reset
};
ZSTD_rust_compressStream2MTCompletionState const completionState = {
loopPolicy,
&callbacks
};
ZSTD_rust_compressStream2MTCompletion(&completionState);
}
FORWARD_IF_ERROR(flushMin, "ZSTDMT_compressStream_generic failed");
if (loopPolicy == ZSTD_RUST_COMPRESS_STREAM2_MT_LOOP_POLICY_BREAK
|| loopPolicy == ZSTD_RUST_COMPRESS_STREAM2_MT_LOOP_POLICY_END_COMPLETE)
break;
}
{ ZSTD_rust_compressStream2MTCompletionCallbacks const callbacks = {
cctx,
ZSTD_rust_compressStream2MT_trace,
ZSTD_rust_compressStream2MT_reset
};
ZSTD_rust_compressStream2MTCoordinatorState const state = {
cctx,
ZSTD_rust_compressStream2MT_step,
output,
input,
&callbacks,
(int)endOp
};
flushMin = ZSTD_rust_compressStream2MTCoordinator(&state);
}
FORWARD_IF_ERROR(flushMin, "ZSTDMT_compressStream_generic failed");
DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic");
/* Either we don't require maximum forward progress, we've finished the
* flush, or we are out of output space.