ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(mt): move worker finish publication to Rust

Browse Source 
Replace the C worker finish callbacks that publish errors, final block sizes,
consumed bytes, and completion signals with a Rust-owned publication path. The
nested projection keeps the job counters and pthread operations private to C,
while Rust preserves the error/success ordering and size_t wrapping behavior.
The consumed-size assertion remains a C validation callback at the original
point before publication.

Test Plan:
- git diff --cached --check
- capped cargo check --tests
- capped cargo clippy --tests -- -A clippy::manual-bits -D warnings
- capped make -j1
- capped make -j1 -C tests test
- standalone hybrid cargo test remains unavailable because its link lacks the
  pre-existing ZSTD_rust_dctx_trace_view, ZSTD_rust_dctx_view, and
  ZSTD_rust_block_context_init symbols
This commit is contained in:
ddidderr
2026-07-21 22:27:57 +02:00
parent 0b18e73d4b
commit 6b8051b98a
2 changed files with 262 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstdmt_compress.c
+59 -59
View File
@@ -300,16 +300,48 @@ typedef char ZSTDMT_compression_job_progress_projection_layout[
void ZSTDMT_rust_compressionJobProgress(
void* opaque, size_t cSize, size_t consumed);
typedef struct {
void* callbackContext;
size_t* cSize;
size_t* consumed;
size_t srcSize;
ZSTDMT_compressionJobVoidFn lock;
ZSTDMT_compressionJobVoidFn signal;
ZSTDMT_compressionJobVoidFn unlock;
ZSTDMT_compressionJobSizeFn validateConsumed;
} ZSTDMT_RustCompressionJobFinishPublicationProjection;
typedef char ZSTDMT_compression_job_finish_publication_projection_layout[
(offsetof(ZSTDMT_RustCompressionJobFinishPublicationProjection,
callbackContext) == 0
&& offsetof(ZSTDMT_RustCompressionJobFinishPublicationProjection, cSize)
== sizeof(void*)
&& offsetof(ZSTDMT_RustCompressionJobFinishPublicationProjection,
consumed)
== sizeof(void*) + sizeof(size_t)
&& offsetof(ZSTDMT_RustCompressionJobFinishPublicationProjection,
srcSize)
== sizeof(void*) + 2 * sizeof(size_t)
&& offsetof(ZSTDMT_RustCompressionJobFinishPublicationProjection, lock)
== sizeof(void*) + 3 * sizeof(size_t)
&& offsetof(ZSTDMT_RustCompressionJobFinishPublicationProjection, signal)
== sizeof(void*) + 3 * sizeof(size_t) + sizeof(void*)
&& offsetof(ZSTDMT_RustCompressionJobFinishPublicationProjection,
unlock)
== sizeof(void*) + 3 * sizeof(size_t) + 2 * sizeof(void*)
&& offsetof(ZSTDMT_RustCompressionJobFinishPublicationProjection,
validateConsumed)
== sizeof(void*) + 3 * sizeof(size_t) + 3 * sizeof(void*)
&& sizeof(ZSTDMT_RustCompressionJobFinishPublicationProjection)
== sizeof(void*) + 3 * sizeof(size_t) + 4 * sizeof(void*))
? 1 : -1];
typedef struct {
void* callbackContext;
size_t srcSize;
ZSTDMT_compressionJobVoidFn ensureFinished;
ZSTDMT_compressionJobSizeFn publishError;
ZSTDMT_compressionJobSizeFn publishLastBlockSize;
ZSTDMT_RustCompressionJobFinishPublicationProjection publication;
ZSTDMT_compressionJobVoidFn releaseSeq;
ZSTDMT_compressionJobVoidFn releaseCCtx;
ZSTDMT_compressionJobSizeFn publishConsumed;
ZSTDMT_compressionJobVoidFn signal;
} ZSTDMT_RustCompressionJobFinishProjection;
typedef char ZSTDMT_compression_job_finish_projection_layout[
(offsetof(ZSTDMT_RustCompressionJobFinishProjection, callbackContext) == 0
@@ -317,19 +349,17 @@ typedef char ZSTDMT_compression_job_finish_projection_layout[
== sizeof(void*)
&& offsetof(ZSTDMT_RustCompressionJobFinishProjection, ensureFinished)
== 2 * sizeof(void*)
&& offsetof(ZSTDMT_RustCompressionJobFinishProjection, publishError)
&& offsetof(ZSTDMT_RustCompressionJobFinishProjection, publication)
== 3 * sizeof(void*)
&& offsetof(ZSTDMT_RustCompressionJobFinishProjection, publishLastBlockSize)
== 4 * sizeof(void*)
&& offsetof(ZSTDMT_RustCompressionJobFinishProjection, releaseSeq)
== 5 * sizeof(void*)
== 3 * sizeof(void*)
+ sizeof(ZSTDMT_RustCompressionJobFinishPublicationProjection)
&& offsetof(ZSTDMT_RustCompressionJobFinishProjection, releaseCCtx)
== 6 * sizeof(void*)
&& offsetof(ZSTDMT_RustCompressionJobFinishProjection, publishConsumed)
== 7 * sizeof(void*)
&& offsetof(ZSTDMT_RustCompressionJobFinishProjection, signal)
== 8 * sizeof(void*)
&& sizeof(ZSTDMT_RustCompressionJobFinishProjection) == 9 * sizeof(void*))
== 4 * sizeof(void*)
+ sizeof(ZSTDMT_RustCompressionJobFinishPublicationProjection)
&& sizeof(ZSTDMT_RustCompressionJobFinishProjection)
== 5 * sizeof(void*)
+ sizeof(ZSTDMT_RustCompressionJobFinishPublicationProjection))
? 1 : -1];
typedef struct {
@@ -2243,28 +2273,6 @@ static void ZSTDMT_compressionJobEnsureFinished(void* opaque)
DEBUGLOG(5, "Finished with source: %zx", (size_t)job->src.start);
}
static void ZSTDMT_compressionJobPublishError(void* opaque, size_t error)
{
ZSTDMT_compressionJobState* const state =
(ZSTDMT_compressionJobState*)opaque;
ZSTDMT_jobDescription* const job = state->job;
ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex);
job->cSize = error;
ZSTD_pthread_mutex_unlock(&job->job_mutex);
}
static void ZSTDMT_compressionJobPublishLastBlockSize(void* opaque, size_t lastCBlockSize)
{
ZSTDMT_compressionJobState* const state =
(ZSTDMT_compressionJobState*)opaque;
ZSTDMT_jobDescription* const job = state->job;
ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex);
job->cSize += lastCBlockSize;
ZSTD_pthread_mutex_unlock(&job->job_mutex);
}
static void ZSTDMT_compressionJobReleaseSeq(void* opaque)
{
ZSTDMT_compressionJobState* const state =
@@ -2283,27 +2291,11 @@ static void ZSTDMT_compressionJobReleaseCCtx(void* opaque)
ZSTDMT_releaseCCtx(job->cctxPool, state->cctx);
}
static void ZSTDMT_compressionJobPublishConsumed(void* opaque, size_t srcSize)
static void ZSTDMT_compressionJobValidateConsumed(void* opaque, size_t srcSize)
{
ZSTDMT_compressionJobState* const state =
(ZSTDMT_compressionJobState*)opaque;
ZSTDMT_jobDescription* const job = state->job;
ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)opaque;
assert(srcSize == job->src.size);
ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex);
job->consumed = srcSize; /* when job->consumed == job->src.size , compression job is presumed completed */
ZSTD_pthread_mutex_unlock(&job->job_mutex);
}
static void ZSTDMT_compressionJobSignal(void* opaque)
{
ZSTDMT_compressionJobState* const state =
(ZSTDMT_compressionJobState*)opaque;
ZSTDMT_jobDescription* const job = state->job;
ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex);
ZSTD_pthread_cond_signal(&job->job_cond);
ZSTD_pthread_mutex_unlock(&job->job_mutex);
}
/* ZSTDMT_compressionJob() is a POOL_function type. Rust owns the stage
@@ -2313,6 +2305,7 @@ static void ZSTDMT_compressionJob(void* jobDescription)
ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription;
ZSTDMT_compressionJobState state;
ZSTDMT_RustCompressionJobSequenceState sequenceState;
ZSTDMT_RustCompressionJobFinishPublicationProjection publication;
ZSTDMT_RustCompressionJobFinishProjection finishProjection;
ZSTDMT_RustCompressionJobProjection projection;
@@ -2331,16 +2324,23 @@ static void ZSTDMT_compressionJob(void* jobDescription)
&state.frameHeaderState,
&sequenceState
};
publication = (ZSTDMT_RustCompressionJobFinishPublicationProjection){
job,
&job->cSize,
&job->consumed,
job->src.size,
ZSTDMT_compressionJobProgressLock,
ZSTDMT_compressionJobProgressSignal,
ZSTDMT_compressionJobProgressUnlock,
ZSTDMT_compressionJobValidateConsumed
};
finishProjection = (ZSTDMT_RustCompressionJobFinishProjection){
&state,
job->src.size,
ZSTDMT_compressionJobEnsureFinished,
ZSTDMT_compressionJobPublishError,
ZSTDMT_compressionJobPublishLastBlockSize,
publication,
ZSTDMT_compressionJobReleaseSeq,
ZSTDMT_compressionJobReleaseCCtx,
ZSTDMT_compressionJobPublishConsumed,
ZSTDMT_compressionJobSignal
ZSTDMT_compressionJobReleaseCCtx
};
ZSTDMT_rust_compressionJob(