ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move frame chunk checksum into Rust

Browse Source 
Project the frame-chunk XXH64 state as a typed pointer so Rust owns the
checksum update while preserving the existing pre-block ordering and error
behavior.  Remove the now-redundant C checksum callback and shrink the
cross-language frame-chunk state layout assertions.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo fmt --all -- --check
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --lib zstd_compress::tests::frame_chunk -- --nocapture
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --all-targets -- -D warnings
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test
- ulimit -v 41943040; make -j1
- ulimit -v 41943040; make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s
- ulimit -v 41943040; make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests
This commit is contained in:
ddidderr
2026-07-20 00:34:32 +02:00
parent 0a75273a50
commit 30c5bc9f90
2 changed files with 31 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+2 -13
View File
@@ -821,13 +821,10 @@ typedef size_t (*ZSTD_rust_frameChunkCompress_f)(void* context,
const void* src,
size_t srcSize,
U32 lastBlock);
typedef void (*ZSTD_rust_frameChunkChecksum_f)(void* state,
const void* src,
size_t srcSize);
typedef struct {
void* callbackContext;
void* tmpWorkspace;
void* checksumState;
XXH64_state_t* checksumState;
int* isFirstBlock;
ZSTD_compressionStage_e* stage;
size_t tmpWkspSize;
@@ -843,7 +840,6 @@ typedef struct {
ZSTD_rust_frameChunkCompress_f compressTarget;
ZSTD_rust_frameChunkCompress_f compressSplit;
ZSTD_rust_frameChunkCompress_f compressInternal;
ZSTD_rust_frameChunkChecksum_f updateChecksum;
} ZSTD_rust_frameChunkState;
size_t ZSTD_rust_compressFrameChunk(
const ZSTD_rust_frameChunkState* state,
@@ -874,7 +870,7 @@ typedef char ZSTD_rust_frame_chunk_state_layout[
&& offsetof(ZSTD_rust_frameChunkState, prepareState)
== 7 * sizeof(void*) + sizeof(S64) + 6 * sizeof(int)
&& sizeof(ZSTD_rust_frameChunkState)
== 12 * sizeof(void*) + sizeof(S64) + 6 * sizeof(int))
== 11 * sizeof(void*) + sizeof(S64) + 6 * sizeof(int))
? 1 : -1];
/* The high-level continue/block entry points are Rust-owned. This projection
@@ -5575,12 +5571,6 @@ static size_t ZSTD_rust_frameChunk_compressInternal(
1 /* frame */);
}
static void ZSTD_rust_frameChunk_updateChecksum(
void* state, const void* src, size_t srcSize)
{
(void)XXH64_update((XXH64_state_t*)state, src, srcSize);
}
/*! ZSTD_compress_frameChunk() :
* Compress a chunk of data into one or multiple blocks.
* All blocks will be terminated, all input will be consumed.
@@ -5623,7 +5613,6 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
state.compressTarget = ZSTD_rust_frameChunk_compressTarget;
state.compressSplit = ZSTD_rust_frameChunk_compressSplit;
state.compressInternal = ZSTD_rust_frameChunk_compressInternal;
state.updateChecksum = ZSTD_rust_frameChunk_updateChecksum;
return ZSTD_rust_compressFrameChunk(
&state, dst, dstCapacity, src, srcSize, lastFrameChunk);
}
rust/src/zstd_compress.rs
+29 -22
View File
@@ -16,6 +16,8 @@
use crate::common::MINMATCH;
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
use crate::mem::MEM_writeLE32;
#[cfg(test)]
use crate::xxhash::XXH64_reset;
use crate::xxhash::{XXH64_digest, XXH64_state_t, XXH64_update};
#[cfg(not(test))]
use crate::zstd_compress_api::ZSTD_compressBound;
@@ -178,7 +180,6 @@ type FrameChunkPrepareOverflowFn = unsafe extern "C" fn(*mut c_void, *const c_vo
type FrameChunkPrepareWindowFn = unsafe extern "C" fn(*mut c_void, *const c_void, usize, c_uint);
type FrameChunkCompressFn =
unsafe extern "C" fn(*mut c_void, *mut c_void, usize, *const c_void, usize, c_uint) -> usize;
type FrameChunkChecksumFn = unsafe extern "C" fn(*mut c_void, *const c_void, usize);
type BuildSeqStoreSkipFn = unsafe extern "C" fn(*mut c_void, usize);
type BuildSeqStorePrepareFn = unsafe extern "C" fn(*mut c_void, *const c_void, usize);
@@ -550,7 +551,7 @@ const _: () = {
pub struct ZSTD_rust_frameChunkState {
callback_context: *mut c_void,
tmp_workspace: *mut c_void,
checksum_state: *mut c_void,
checksum_state: *mut XXH64_state_t,
is_first_block: *mut c_int,
stage: *mut c_int,
tmp_wksp_size: usize,
@@ -566,7 +567,6 @@ pub struct ZSTD_rust_frameChunkState {
compress_target: FrameChunkCompressFn,
compress_split: FrameChunkCompressFn,
compress_internal: FrameChunkCompressFn,
update_checksum: FrameChunkChecksumFn,
}
const _: () = {
@@ -608,7 +608,7 @@ const _: () = {
);
assert!(
size_of::<ZSTD_rust_frameChunkState>()
== 12 * size_of::<usize>() + size_of::<c_longlong>() + 6 * size_of::<c_int>()
== 11 * size_of::<usize>() + size_of::<c_longlong>() + 6 * size_of::<c_int>()
);
};
@@ -659,7 +659,9 @@ unsafe fn compress_frame_chunk_body_with(
if state.checksum_state.is_null() {
return ERROR(ZstdErrorCode::Generic);
}
unsafe { (state.update_checksum)(state.checksum_state, src, src_size) };
unsafe {
let _ = XXH64_update(state.checksum_state, src, src_size);
}
}
let mut remaining = src_size;
@@ -10844,8 +10846,7 @@ mod tests {
split_calls: usize,
internal_calls: usize,
last_blocks: [c_uint; 4],
checksum_calls: usize,
checksum_size: usize,
checksum_state: Option<XXH64_state_t>,
target_result: usize,
split_result: usize,
internal_result: usize,
@@ -10945,16 +10946,6 @@ mod tests {
context.internal_result
}
unsafe extern "C" fn frame_chunk_test_checksum(
context: *mut c_void,
_src: *const c_void,
src_size: usize,
) {
let context = unsafe { frame_chunk_test_context(context) };
context.checksum_calls += 1;
context.checksum_size = context.checksum_size.wrapping_add(src_size);
}
fn frame_chunk_test_state(
context: &mut FrameChunkTestContext,
is_first_block: &mut c_int,
@@ -10965,6 +10956,18 @@ mod tests {
) -> ZSTD_rust_frameChunkState {
let context_ptr = context as *mut FrameChunkTestContext;
let callback_context = context_ptr.cast::<c_void>();
context.checksum_state = Some(XXH64_state_t {
total_len: 0,
v: [0; 4],
mem64: [0; 4],
memsize: 0,
reserved32: 0,
reserved64: 0,
});
let checksum_state = context.checksum_state.as_mut().unwrap();
unsafe {
let _ = XXH64_reset(checksum_state, 0);
}
unsafe {
(*context_ptr).clamp_state = Some(ZSTD_rust_frameChunkClampState {
next_to_update: &mut (*context_ptr).clamp_next_to_update,
@@ -10991,7 +10994,7 @@ mod tests {
ZSTD_rust_frameChunkState {
callback_context,
tmp_workspace: ptr::null_mut(),
checksum_state: callback_context,
checksum_state,
is_first_block,
stage,
tmp_wksp_size: 0,
@@ -11007,7 +11010,6 @@ mod tests {
compress_target: frame_chunk_test_target,
compress_split: frame_chunk_test_split,
compress_internal: frame_chunk_test_internal,
update_checksum: frame_chunk_test_checksum,
}
}
@@ -11044,8 +11046,10 @@ mod tests {
assert_eq!(context.clamp_next_to_update, 4);
assert_eq!(context.internal_calls, 2);
assert_eq!(context.last_blocks[..2], [0, 1]);
assert_eq!(context.checksum_calls, 1);
assert_eq!(context.checksum_size, source.len());
assert_eq!(
context.checksum_state.as_ref().unwrap().total_len,
source.len() as u64
);
assert_eq!(is_first_block, 0);
assert_eq!(stage, 77);
assert_ne!(output[..3], [0xa5; 3]);
@@ -11141,7 +11145,10 @@ mod tests {
};
assert_eq!(result, ERROR(ZstdErrorCode::DstSizeTooSmall));
assert_eq!(context.checksum_calls, 1);
assert_eq!(
context.checksum_state.as_ref().unwrap().total_len,
source.len() as u64
);
assert_eq!(context.prepare_calls, 0);
assert_eq!(context.internal_calls, 0);
assert_eq!(is_first_block, 1);