ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move sequence checksums into Rust

Browse Source 
Project the sequence API's XXH64 state directly through the Rust ABI boundary
so Rust owns checksum updates, digest ordering, and little-endian frame
checksum serialization.  Keep the C callback limited to private CCtx and
sequence-state initialization, remove the three checksum callback shims, and
update the cross-language layout assertions for the smaller state.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo fmt --all -- --check
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --lib zstd_compress::tests::sequence_api -- --nocapture
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --all-targets -- -D warnings
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test
- ulimit -v 41943040; make -j1
- ulimit -v 41943040; make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s
- ulimit -v 41943040; make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests
This commit is contained in:
ddidderr
2026-07-20 00:23:13 +02:00
parent 4a8345f596
commit cce9254703
2 changed files with 41 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/zstd_compress.rs
+26 -33
View File
@@ -15,7 +15,8 @@
use crate::common::MINMATCH;
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
use crate::xxhash::{XXH64_digest, XXH64_state_t};
use crate::mem::MEM_writeLE32;
use crate::xxhash::{XXH64_digest, XXH64_state_t, XXH64_update};
#[cfg(not(test))]
use crate::zstd_compress_api::ZSTD_compressBound;
use crate::zstd_compress_frame::{
@@ -3739,25 +3740,21 @@ const _: () = {
type SequenceApiInitFn =
unsafe extern "C" fn(*mut c_void, usize, *mut ZSTD_rust_sequenceApiState) -> usize;
type SequenceApiUpdateChecksumFn = unsafe extern "C" fn(*mut c_void, *const c_void, usize);
type SequenceApiDigestChecksumFn = unsafe extern "C" fn(*mut c_void) -> c_uint;
type SequenceApiWriteChecksumFn = unsafe extern "C" fn(*mut c_void, *mut c_void, c_uint);
/// Explicit projection for the public sequence-compression API orchestration.
///
/// Rust owns validation ordering, frame-header/checksum sequencing, and
/// output accounting. Frame-header parameters are projected as scalars; C
/// retains the private CCtx, sequence-store, block, and checksum layouts
/// through the two block-state projections and callbacks.
/// retains the private CCtx and sequence-store/block layouts through the
/// initialization callback and two block-state projections. The XXH64 state
/// is projected directly so checksum updates and serialization stay in Rust.
#[repr(C)]
pub struct ZSTD_rust_sequenceApiState {
callback_context: *mut c_void,
sequence_state: *mut ZSTD_rust_sequenceCompressionState,
sequence_literals_state: *mut ZSTD_rust_sequenceLiteralsState,
init: SequenceApiInitFn,
update_checksum: SequenceApiUpdateChecksumFn,
digest_checksum: SequenceApiDigestChecksumFn,
write_checksum: SequenceApiWriteChecksumFn,
checksum_state: *mut XXH64_state_t,
checksum_flag: c_int,
block_delimiters: c_int,
validate_sequences: c_int,
@@ -3775,40 +3772,38 @@ const _: () = {
offset_of!(ZSTD_rust_sequenceApiState, sequence_literals_state) == 2 * size_of::<usize>()
);
assert!(offset_of!(ZSTD_rust_sequenceApiState, init) == 3 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_sequenceApiState, update_checksum) == 4 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_sequenceApiState, digest_checksum) == 5 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_sequenceApiState, write_checksum) == 6 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_sequenceApiState, checksum_flag) == size_of::<[usize; 7]>());
assert!(offset_of!(ZSTD_rust_sequenceApiState, checksum_state) == 4 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_sequenceApiState, checksum_flag) == size_of::<[usize; 5]>());
assert!(
offset_of!(ZSTD_rust_sequenceApiState, block_delimiters)
== size_of::<[usize; 7]>() + size_of::<c_int>()
== size_of::<[usize; 5]>() + size_of::<c_int>()
);
assert!(
offset_of!(ZSTD_rust_sequenceApiState, validate_sequences)
== size_of::<[usize; 7]>() + size_of::<[c_int; 2]>()
== size_of::<[usize; 5]>() + size_of::<[c_int; 2]>()
);
assert!(
offset_of!(ZSTD_rust_sequenceApiState, no_dict_id_flag)
== size_of::<[usize; 7]>() + size_of::<[c_int; 3]>()
== size_of::<[usize; 5]>() + size_of::<[c_int; 3]>()
);
assert!(
offset_of!(ZSTD_rust_sequenceApiState, content_size_flag)
== size_of::<[usize; 7]>() + size_of::<[c_int; 4]>()
== size_of::<[usize; 5]>() + size_of::<[c_int; 4]>()
);
assert!(
offset_of!(ZSTD_rust_sequenceApiState, format)
== size_of::<[usize; 7]>() + size_of::<[c_int; 5]>()
== size_of::<[usize; 5]>() + size_of::<[c_int; 5]>()
);
assert!(
offset_of!(ZSTD_rust_sequenceApiState, window_log)
== size_of::<[usize; 7]>() + size_of::<[c_int; 6]>()
== size_of::<[usize; 5]>() + size_of::<[c_int; 6]>()
);
assert!(
offset_of!(ZSTD_rust_sequenceApiState, dict_id)
== size_of::<[usize; 7]>() + size_of::<[c_int; 6]>() + size_of::<c_uint>()
== size_of::<[usize; 5]>() + size_of::<[c_int; 6]>() + size_of::<c_uint>()
);
assert!(
size_of::<ZSTD_rust_sequenceApiState>() == if size_of::<usize>() == 8 { 88 } else { 60 }
size_of::<ZSTD_rust_sequenceApiState>() == if size_of::<usize>() == 8 { 72 } else { 52 }
);
};
@@ -8263,9 +8258,7 @@ unsafe fn sequence_api_prepare(
|| state.sequence_state.is_null()
|| state.sequence_literals_state.is_null()
|| state.init as usize == 0
|| state.update_checksum as usize == 0
|| state.digest_checksum as usize == 0
|| state.write_checksum as usize == 0
|| state.checksum_state.is_null()
{
return Err(ERROR(ZstdErrorCode::Generic));
}
@@ -8337,13 +8330,11 @@ unsafe fn sequence_api_append_frame_checksum(
/* Keep the original ordering: digest the checksum before checking the
* remaining destination capacity. */
let checksum = unsafe { (state.digest_checksum)(state.callback_context) };
let checksum = unsafe { XXH64_digest(state.checksum_state) as c_uint };
if dst_capacity < 4 {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
unsafe {
(state.write_checksum)(state.callback_context, op.cast(), checksum);
}
unsafe { MEM_writeLE32(op.cast(), checksum) };
*c_size = c_size.wrapping_add(4);
0
}
@@ -8351,10 +8342,10 @@ unsafe fn sequence_api_append_frame_checksum(
/// Rust-owned orchestration for `ZSTD_compressSequences`.
///
/// The C wrapper provides the post-initialization block-state projection and
/// callbacks for the private CCtx/checksum operations; frame-header fields are
/// projected as scalars. Rust preserves the public ordering: initialize,
/// write the frame header, update the input checksum, emit blocks, then append
/// the frame checksum.
/// callback for private CCtx initialization; frame-header fields and the
/// checksum state are projected directly. Rust preserves the public ordering:
/// initialize, write the frame header, update the input checksum, emit blocks,
/// then append the frame checksum.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressSequences(
state: *mut ZSTD_rust_sequenceApiState,
@@ -8385,7 +8376,9 @@ pub unsafe extern "C" fn ZSTD_rust_compressSequences(
}
if plan.update_input_checksum && src_size != 0 {
unsafe { (state.update_checksum)(state.callback_context, src, src_size) };
unsafe {
let _ = XXH64_update(state.checksum_state, src, src_size);
}
}
let block_size = unsafe {