ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(mt): move frame checksum finalization to Rust

Browse Source 
The MT flush path already delegated checksum serialization to Rust but still
computed XXH64_digest in C. Pass the live C-owned XXH64 state through the
checked projection and let Rust finalize the digest before the existing
little-endian write. C retains serial state storage and job/flush mutation.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --tests (rust)
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --tests -- -A clippy::manual-bits -D warnings (rust)
- ulimit -v 41943040; make -j1
- ulimit -v 41943040; make -j1 -C tests test
- Focused cargo test compiles but remains link-blocked by existing C-owned decompression-view symbols.
This commit is contained in:
ddidderr
2026-07-22 00:13:58 +02:00
parent b72c51216b
commit e3d1f7057d
2 changed files with 47 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstdmt_compress.c
+3 -5
View File
@@ -466,7 +466,7 @@ typedef struct {
void* dst;
size_t dstCapacity;
size_t cSize;
U32 checksum;
const XXH64_state_t* checksumState;
} ZSTDMT_RustFrameChecksumProjection;
typedef char ZSTDMT_frame_checksum_projection_layout[
(offsetof(ZSTDMT_RustFrameChecksumProjection, dst) == 0
@@ -474,9 +474,8 @@ typedef char ZSTDMT_frame_checksum_projection_layout[
== sizeof(void*)
&& offsetof(ZSTDMT_RustFrameChecksumProjection, cSize)
== 2 * sizeof(void*)
&& offsetof(ZSTDMT_RustFrameChecksumProjection, checksum)
&& offsetof(ZSTDMT_RustFrameChecksumProjection, checksumState)
== 3 * sizeof(void*)
&& sizeof(U32) == 4
&& sizeof(ZSTDMT_RustFrameChecksumProjection)
== 4 * sizeof(void*))
? 1 : -1];
@@ -3440,12 +3439,11 @@ static void ZSTDMT_addFrameChecksum(void* opaque, unsigned jobID,
{
ZSTDMT_CCtx* const mtctx = (ZSTDMT_CCtx*)opaque;
ZSTDMT_jobDescription* const job = &mtctx->jobs[jobID];
U32 const checksum = (U32)XXH64_digest(&mtctx->serial.xxhState);
ZSTDMT_RustFrameChecksumProjection const checksumProjection = {
job->dstBuff.start,
job->dstBuff.capacity,
job->cSize,
checksum
&mtctx->serial.xxhState
};
size_t const cSize = ZSTDMT_rust_writeFrameChecksum(&checksumProjection);
rust/src/zstdmt_compress.rs
+44 -14
View File
@@ -23,7 +23,7 @@ use std::sync::Mutex;
use crate::bits::ZSTD_highbit32;
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
use crate::mem::MEM_writeLE32;
use crate::xxhash::{XXH64_state_t, XXH64_update};
use crate::xxhash::{XXH64_digest, XXH64_state_t, XXH64_update};
use crate::zstd_compress::{ZSTD_frameProgression, ZSTD_rust_invalidateRepCodes};
use crate::zstd_compress_frame::ZSTD_rust_writeFrameHeader;
use crate::zstd_compress_params::{
@@ -1893,23 +1893,22 @@ pub struct ZSTDMT_flushJobProjection {
}
/// Narrow projection for serializing the C-owned frame checksum. C retains
/// the serial XXH64 state and computes the checksum; Rust owns only the
/// deterministic little-endian write and destination policy.
/// the serial XXH64 state; Rust owns digest finalization, the deterministic
/// little-endian write, and destination policy.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTDMT_frameChecksumProjection {
pub dst: *mut c_void,
pub dstCapacity: usize,
pub cSize: usize,
pub checksum: c_uint,
pub checksumState: *const XXH64_state_t,
}
const _: () = {
assert!(offset_of!(ZSTDMT_frameChecksumProjection, dst) == 0);
assert!(offset_of!(ZSTDMT_frameChecksumProjection, dstCapacity) == size_of::<usize>());
assert!(offset_of!(ZSTDMT_frameChecksumProjection, cSize) == 2 * size_of::<usize>());
assert!(offset_of!(ZSTDMT_frameChecksumProjection, checksum) == 3 * size_of::<usize>());
assert!(size_of::<c_uint>() == 4);
assert!(offset_of!(ZSTDMT_frameChecksumProjection, checksumState) == 3 * size_of::<usize>());
assert!(size_of::<ZSTDMT_frameChecksumProjection>() == 4 * size_of::<usize>());
};
@@ -1921,11 +1920,15 @@ fn write_frame_checksum(projection: &ZSTDMT_frameChecksumProjection) -> usize {
{
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
if projection.checksumState.is_null() {
return ERROR(ZstdErrorCode::Generic);
}
let checksum = unsafe { XXH64_digest(projection.checksumState) as c_uint };
unsafe {
MEM_writeLE32(
projection.dst.cast::<u8>().add(projection.cSize).cast(),
projection.checksum,
checksum,
);
}
projection.cSize + 4
@@ -5966,7 +5969,7 @@ pub unsafe extern "C" fn ZSTDMT_rust_cctx_pool_release(pool: *mut RustCCtxPool,
#[cfg(test)]
mod tests {
use super::*;
use crate::xxhash::{XXH64, XXH64_digest, XXH64_reset};
use crate::xxhash::{XXH64_digest, XXH64_reset, XXH64_update, XXH64};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::{cell::RefCell, rc::Rc};
@@ -9321,46 +9324,73 @@ mod tests {
#[test]
fn frame_checksum_is_serialized_little_endian() {
let mut dst = [0xa5u8; 8];
let input = b"mt frame checksum";
let mut checksum_state = unsafe { MaybeUninit::<XXH64_state_t>::zeroed().assume_init() };
unsafe {
assert_eq!(XXH64_reset(&mut checksum_state, 0), 0);
assert_eq!(
XXH64_update(&mut checksum_state, input.as_ptr().cast(), input.len()),
0
);
}
let checksum = unsafe { XXH64(input.as_ptr().cast(), input.len(), 0) as u32 };
let projection = ZSTDMT_frameChecksumProjection {
dst: dst.as_mut_ptr().cast(),
dstCapacity: dst.len(),
cSize: 4,
checksum: 0x7856_3412,
checksumState: &checksum_state,
};
assert_eq!(write_frame_checksum(&projection), 8);
assert_eq!(&dst[..4], &[0xa5; 4]);
assert_eq!(&dst[4..], &[0x12, 0x34, 0x56, 0x78]);
assert_eq!(&dst[4..], &checksum.to_le_bytes());
}
#[test]
fn frame_checksum_accepts_exact_destination_capacity() {
let mut dst = [0xa5u8; 6];
let input = b"exact capacity";
let mut checksum_state = unsafe { MaybeUninit::<XXH64_state_t>::zeroed().assume_init() };
unsafe {
assert_eq!(XXH64_reset(&mut checksum_state, 0), 0);
assert_eq!(
XXH64_update(&mut checksum_state, input.as_ptr().cast(), input.len()),
0
);
}
let projection = ZSTDMT_frameChecksumProjection {
dst: dst.as_mut_ptr().cast(),
dstCapacity: dst.len(),
cSize: 2,
checksum: 0x1122_3344,
checksumState: &checksum_state,
};
assert_eq!(write_frame_checksum(&projection), dst.len());
assert_eq!(&dst, &[0xa5, 0xa5, 0x44, 0x33, 0x22, 0x11]);
assert_eq!(&dst[..2], &[0xa5, 0xa5]);
assert_eq!(
&dst[2..],
&(unsafe { XXH64(input.as_ptr().cast(), input.len(), 0) as u32 }).to_le_bytes()
);
}
#[test]
fn frame_checksum_rejects_too_small_or_null_destination() {
let mut dst = [0xa5u8; 8];
let mut checksum_state = unsafe { MaybeUninit::<XXH64_state_t>::zeroed().assume_init() };
unsafe {
assert_eq!(XXH64_reset(&mut checksum_state, 0), 0);
}
let too_small = ZSTDMT_frameChecksumProjection {
dst: dst.as_mut_ptr().cast(),
dstCapacity: 7,
cSize: 4,
checksum: 0x0102_0304,
checksumState: &checksum_state,
};
let null_destination = ZSTDMT_frameChecksumProjection {
dst: ptr::null_mut(),
dstCapacity: 4,
cSize: 0,
checksum: 0x0102_0304,
checksumState: &checksum_state,
};
assert_eq!(