ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(compress): move MT checksum serialization to Rust

Browse Source 
Keep XXH64 state ownership and checksum calculation in the C MT scheduler,
but project the final four-byte frame write through Rust.  Rust now owns the
little-endian serialization and destination-capacity policy, with ABI layout
assertions and flush-state error propagation preserving the existing cleanup
path.  Focused tests cover byte order, exact capacity, invalid destinations,
and callback error handling.

Test Plan:
- `ulimit -v 41943040; cargo +nightly fmt --manifest-path rust/Cargo.toml --all -- --check`
- `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings`
- `ulimit -v 41943040; make -j1`
- `ulimit -v 41943040; make -j1 -C tests test`
- `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings`
- `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/cli/Cargo.toml --all-targets`

The integrated checks ran under a serial 40 GiB virtual-memory limit. The
full upstream suite passed, including large-file, multithreaded, API, fuzzer,
and zstream coverage. Standalone root Rust unit linking remains unavailable
because the crate imports C-owned bridge symbols without a Cargo build/link
setup.
This commit is contained in:
ddidderr
2026-07-20 09:29:17 +02:00
parent b550b14ceb
commit dd87b328a4
2 changed files with 190 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstdmt_compress.c
+33 -5
View File
@@ -285,6 +285,24 @@ typedef struct {
size_t dstFlushed;
unsigned frameChecksumNeeded;
} ZSTDMT_RustFlushJobProjection;
typedef struct {
void* dst;
size_t dstCapacity;
size_t cSize;
U32 checksum;
} ZSTDMT_RustFrameChecksumProjection;
typedef char ZSTDMT_frame_checksum_projection_layout[
(offsetof(ZSTDMT_RustFrameChecksumProjection, dst) == 0
&& offsetof(ZSTDMT_RustFrameChecksumProjection, dstCapacity)
== sizeof(void*)
&& offsetof(ZSTDMT_RustFrameChecksumProjection, cSize)
== 2 * sizeof(void*)
&& offsetof(ZSTDMT_RustFrameChecksumProjection, checksum)
== 3 * sizeof(void*)
&& sizeof(U32) == 4
&& sizeof(ZSTDMT_RustFrameChecksumProjection)
== 4 * sizeof(void*))
? 1 : -1];
typedef struct {
unsigned doneJobID;
unsigned nextJobID;
@@ -319,6 +337,8 @@ ZSTDMT_RustFlushProducedResult ZSTDMT_rust_flushProduced(
ZSTDMT_flushUpdateJobFn updateJob,
ZSTDMT_flushCompleteJobFn completeJob,
ZSTDMT_flushErrorFn onError);
size_t ZSTDMT_rust_writeFrameChecksum(
const ZSTDMT_RustFrameChecksumProjection* projection);
/* The Rust outer scheduler sees only this scalar snapshot. The MT context,
* reusable input buffer, worker pool, and all synchronization remain private
@@ -2786,15 +2806,23 @@ static void ZSTDMT_addFrameChecksum(void* opaque, unsigned jobID,
ZSTDMT_CCtx* const mtctx = (ZSTDMT_CCtx*)opaque;
ZSTDMT_jobDescription* const job = &mtctx->jobs[jobID];
U32 const checksum = (U32)XXH64_digest(&mtctx->serial.xxhState);
ZSTDMT_RustFrameChecksumProjection const checksumProjection = {
job->dstBuff.start,
job->dstBuff.capacity,
job->cSize,
checksum
};
size_t const cSize = ZSTDMT_rust_writeFrameChecksum(&checksumProjection);
assert(projection->frameChecksumNeeded);
assert(projection->consumed == projection->srcSize);
assert(projection->cSize == job->cSize);
MEM_writeLE32((char*)job->dstBuff.start + job->cSize, checksum);
job->cSize += 4;
job->frameChecksumNeeded = 0;
projection->cSize = job->cSize;
projection->frameChecksumNeeded = 0;
job->cSize = cSize;
projection->cSize = cSize;
if (!ZSTD_isError(cSize)) {
job->frameChecksumNeeded = 0;
projection->frameChecksumNeeded = 0;
}
}
static void ZSTDMT_updateFlushJob(void* opaque, unsigned jobID, size_t dstFlushed)