From e3d1f7057dd8cddf28bb3d40fba1ea3dc3f94de4 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Wed, 22 Jul 2026 00:13:58 +0200 Subject: [PATCH] feat(mt): move frame checksum finalization to Rust The MT flush path already delegated checksum serialization to Rust but still computed XXH64_digest in C. Pass the live C-owned XXH64 state through the checked projection and let Rust finalize the digest before the existing little-endian write. C retains serial state storage and job/flush mutation. Test Plan: - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --tests (rust) - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --tests -- -A clippy::manual-bits -D warnings (rust) - ulimit -v 41943040; make -j1 - ulimit -v 41943040; make -j1 -C tests test - Focused cargo test compiles but remains link-blocked by existing C-owned decompression-view symbols. --- lib/compress/zstdmt_compress.c | 8 ++--- rust/src/zstdmt_compress.rs | 58 ++++++++++++++++++++++++++-------- 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index b1af71a8d..5af6f388f 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -466,7 +466,7 @@ typedef struct { void* dst; size_t dstCapacity; size_t cSize; - U32 checksum; + const XXH64_state_t* checksumState; } ZSTDMT_RustFrameChecksumProjection; typedef char ZSTDMT_frame_checksum_projection_layout[ (offsetof(ZSTDMT_RustFrameChecksumProjection, dst) == 0 @@ -474,9 +474,8 @@ typedef char ZSTDMT_frame_checksum_projection_layout[ == sizeof(void*) && offsetof(ZSTDMT_RustFrameChecksumProjection, cSize) == 2 * sizeof(void*) - && offsetof(ZSTDMT_RustFrameChecksumProjection, checksum) + && offsetof(ZSTDMT_RustFrameChecksumProjection, checksumState) == 3 * sizeof(void*) - && sizeof(U32) == 4 && sizeof(ZSTDMT_RustFrameChecksumProjection) == 4 * sizeof(void*)) ? 1 : -1]; @@ -3440,12 +3439,11 @@ static void ZSTDMT_addFrameChecksum(void* opaque, unsigned jobID, { ZSTDMT_CCtx* const mtctx = (ZSTDMT_CCtx*)opaque; ZSTDMT_jobDescription* const job = &mtctx->jobs[jobID]; - U32 const checksum = (U32)XXH64_digest(&mtctx->serial.xxhState); ZSTDMT_RustFrameChecksumProjection const checksumProjection = { job->dstBuff.start, job->dstBuff.capacity, job->cSize, - checksum + &mtctx->serial.xxhState }; size_t const cSize = ZSTDMT_rust_writeFrameChecksum(&checksumProjection); diff --git a/rust/src/zstdmt_compress.rs b/rust/src/zstdmt_compress.rs index 20f2b656d..eaa7fc456 100644 --- a/rust/src/zstdmt_compress.rs +++ b/rust/src/zstdmt_compress.rs @@ -23,7 +23,7 @@ use std::sync::Mutex; use crate::bits::ZSTD_highbit32; use crate::errors::{ERR_isError, ZstdErrorCode, ERROR}; use crate::mem::MEM_writeLE32; -use crate::xxhash::{XXH64_state_t, XXH64_update}; +use crate::xxhash::{XXH64_digest, XXH64_state_t, XXH64_update}; use crate::zstd_compress::{ZSTD_frameProgression, ZSTD_rust_invalidateRepCodes}; use crate::zstd_compress_frame::ZSTD_rust_writeFrameHeader; use crate::zstd_compress_params::{ @@ -1893,23 +1893,22 @@ pub struct ZSTDMT_flushJobProjection { } /// Narrow projection for serializing the C-owned frame checksum. C retains -/// the serial XXH64 state and computes the checksum; Rust owns only the -/// deterministic little-endian write and destination policy. +/// the serial XXH64 state; Rust owns digest finalization, the deterministic +/// little-endian write, and destination policy. #[repr(C)] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct ZSTDMT_frameChecksumProjection { pub dst: *mut c_void, pub dstCapacity: usize, pub cSize: usize, - pub checksum: c_uint, + pub checksumState: *const XXH64_state_t, } const _: () = { assert!(offset_of!(ZSTDMT_frameChecksumProjection, dst) == 0); assert!(offset_of!(ZSTDMT_frameChecksumProjection, dstCapacity) == size_of::()); assert!(offset_of!(ZSTDMT_frameChecksumProjection, cSize) == 2 * size_of::()); - assert!(offset_of!(ZSTDMT_frameChecksumProjection, checksum) == 3 * size_of::()); - assert!(size_of::() == 4); + assert!(offset_of!(ZSTDMT_frameChecksumProjection, checksumState) == 3 * size_of::()); assert!(size_of::() == 4 * size_of::()); }; @@ -1921,11 +1920,15 @@ fn write_frame_checksum(projection: &ZSTDMT_frameChecksumProjection) -> usize { { return ERROR(ZstdErrorCode::DstSizeTooSmall); } + if projection.checksumState.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + let checksum = unsafe { XXH64_digest(projection.checksumState) as c_uint }; unsafe { MEM_writeLE32( projection.dst.cast::().add(projection.cSize).cast(), - projection.checksum, + checksum, ); } projection.cSize + 4 @@ -5966,7 +5969,7 @@ pub unsafe extern "C" fn ZSTDMT_rust_cctx_pool_release(pool: *mut RustCCtxPool, #[cfg(test)] mod tests { use super::*; - use crate::xxhash::{XXH64, XXH64_digest, XXH64_reset}; + use crate::xxhash::{XXH64_digest, XXH64_reset, XXH64_update, XXH64}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::{cell::RefCell, rc::Rc}; @@ -9321,46 +9324,73 @@ mod tests { #[test] fn frame_checksum_is_serialized_little_endian() { let mut dst = [0xa5u8; 8]; + let input = b"mt frame checksum"; + let mut checksum_state = unsafe { MaybeUninit::::zeroed().assume_init() }; + unsafe { + assert_eq!(XXH64_reset(&mut checksum_state, 0), 0); + assert_eq!( + XXH64_update(&mut checksum_state, input.as_ptr().cast(), input.len()), + 0 + ); + } + let checksum = unsafe { XXH64(input.as_ptr().cast(), input.len(), 0) as u32 }; let projection = ZSTDMT_frameChecksumProjection { dst: dst.as_mut_ptr().cast(), dstCapacity: dst.len(), cSize: 4, - checksum: 0x7856_3412, + checksumState: &checksum_state, }; assert_eq!(write_frame_checksum(&projection), 8); assert_eq!(&dst[..4], &[0xa5; 4]); - assert_eq!(&dst[4..], &[0x12, 0x34, 0x56, 0x78]); + assert_eq!(&dst[4..], &checksum.to_le_bytes()); } #[test] fn frame_checksum_accepts_exact_destination_capacity() { let mut dst = [0xa5u8; 6]; + let input = b"exact capacity"; + let mut checksum_state = unsafe { MaybeUninit::::zeroed().assume_init() }; + unsafe { + assert_eq!(XXH64_reset(&mut checksum_state, 0), 0); + assert_eq!( + XXH64_update(&mut checksum_state, input.as_ptr().cast(), input.len()), + 0 + ); + } let projection = ZSTDMT_frameChecksumProjection { dst: dst.as_mut_ptr().cast(), dstCapacity: dst.len(), cSize: 2, - checksum: 0x1122_3344, + checksumState: &checksum_state, }; assert_eq!(write_frame_checksum(&projection), dst.len()); - assert_eq!(&dst, &[0xa5, 0xa5, 0x44, 0x33, 0x22, 0x11]); + assert_eq!(&dst[..2], &[0xa5, 0xa5]); + assert_eq!( + &dst[2..], + &(unsafe { XXH64(input.as_ptr().cast(), input.len(), 0) as u32 }).to_le_bytes() + ); } #[test] fn frame_checksum_rejects_too_small_or_null_destination() { let mut dst = [0xa5u8; 8]; + let mut checksum_state = unsafe { MaybeUninit::::zeroed().assume_init() }; + unsafe { + assert_eq!(XXH64_reset(&mut checksum_state, 0), 0); + } let too_small = ZSTDMT_frameChecksumProjection { dst: dst.as_mut_ptr().cast(), dstCapacity: 7, cSize: 4, - checksum: 0x0102_0304, + checksumState: &checksum_state, }; let null_destination = ZSTDMT_frameChecksumProjection { dst: ptr::null_mut(), dstCapacity: 4, cSize: 0, - checksum: 0x0102_0304, + checksumState: &checksum_state, }; assert_eq!(