ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): project end continue state into Rust

Browse Source 
Share the synchronous compressContinue projection with the end-of-frame
orchestration so Rust calls the migrated continue path directly. Remove the
redundant C end callback while retaining the C trace callback and the
context-sensitive compression leaves.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo fmt --all -- --check
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --lib zstd_compress::tests::compress_end -- --nocapture
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --all-targets -- -D warnings
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test
- ulimit -v 41943040; make -j1
- ulimit -v 41943040; make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s
- ulimit -v 41943040; make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests
This commit is contained in:
ddidderr
2026-07-20 00:57:46 +02:00
parent 58e6d2eef7
commit 0a69df20a3
2 changed files with 289 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/zstd_compress.rs
+179 -41
View File
@@ -1146,27 +1146,18 @@ pub unsafe extern "C" fn ZSTD_rust_compressContinue(
}
}
type CompressEndContinueFn = unsafe extern "C" fn(
*mut c_void,
*mut c_void,
usize,
*const c_void,
usize,
c_uint,
c_uint,
) -> usize;
type CompressEndTraceFn = unsafe extern "C" fn(*mut c_void, usize);
/// Explicit projection for the public end-of-frame orchestration.
///
/// Rust owns callback ordering, output offset/capacity accounting, and
/// pledged-size validation. The opaque callback context retains only the
/// private CCtx-dependent continue and trace operations in C; Rust serializes
/// the epilogue from explicit frame and checksum projections.
/// pledged-size validation. The continue state is projected explicitly and
/// the opaque callback context retains only the private trace operation in C;
/// Rust serializes the epilogue from explicit frame and checksum projections.
#[repr(C)]
pub struct ZSTD_rust_compressEndState {
callback_context: *mut c_void,
compress_continue: CompressEndContinueFn,
compress_continue_state: *const ZSTD_rust_compressContinueState,
trace: CompressEndTraceFn,
consumed_src_size: *const u64,
pledged_src_size_plus_one: u64,
@@ -1181,7 +1172,7 @@ pub struct ZSTD_rust_compressEndState {
const _: () = {
assert!(offset_of!(ZSTD_rust_compressEndState, callback_context) == 0);
assert!(offset_of!(ZSTD_rust_compressEndState, compress_continue) == size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressEndState, compress_continue_state) == size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressEndState, trace) == 2 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressEndState, consumed_src_size) == 3 * size_of::<usize>());
assert!(
@@ -1229,14 +1220,17 @@ unsafe fn compress_end_body_with(
src: *const c_void,
src_size: usize,
) -> usize {
if state.consumed_src_size.is_null() || state.stage.is_null() || state.checksum_state.is_null()
if state.compress_continue_state.is_null()
|| state.consumed_src_size.is_null()
|| state.stage.is_null()
|| state.checksum_state.is_null()
{
return ERROR(ZstdErrorCode::Generic);
}
let c_size = unsafe {
(state.compress_continue)(
state.callback_context,
ZSTD_rust_compressContinue(
state.compress_continue_state,
dst,
dst_capacity,
src,
@@ -10155,6 +10149,16 @@ mod tests {
continue_frame: c_uint,
continue_last_frame_chunk: c_uint,
stage: c_int,
window: ZSTD_rust_windowUpdateState,
force_non_contiguous: c_int,
next_to_update: c_uint,
window_projection: ZSTD_rust_compressContinueWindowProjection,
frame_chunk_clamp_state: Option<ZSTD_rust_frameChunkClampState>,
frame_chunk_prepare_state: Option<ZSTD_rust_frameChunkPrepareState>,
frame_chunk_state: Option<ZSTD_rust_frameChunkState>,
continue_state: Option<ZSTD_rust_compressContinueState>,
is_first_block: c_int,
produced_c_size: u64,
}
static COMPRESS_END_TEST_CHECKSUM_STATE: XXH64_state_t = XXH64_state_t {
@@ -10172,19 +10176,33 @@ mod tests {
unsafe { &mut *context.cast::<CompressEndTestContext>() }
}
unsafe extern "C" fn compress_end_test_continue(
unsafe extern "C" fn compress_end_test_frame_prepare_overflow(
_context: *mut c_void,
_src: *const c_void,
_block_size: usize,
) {
}
unsafe extern "C" fn compress_end_test_frame_prepare_window(
_context: *mut c_void,
_src: *const c_void,
_block_size: usize,
_max_dist: c_uint,
) {
}
unsafe extern "C" fn compress_end_test_frame_target(
context: *mut c_void,
_dst: *mut c_void,
_dst_capacity: usize,
_src: *const c_void,
_src_size: usize,
frame: c_uint,
last_frame_chunk: c_uint,
last_block: c_uint,
) -> usize {
let context = unsafe { compress_end_test_context(context) };
context.events.push("continue");
context.continue_frame = frame;
context.continue_last_frame_chunk = last_frame_chunk;
context.continue_frame = 1;
context.continue_last_frame_chunk = last_block;
context.continue_result
}
@@ -10196,14 +10214,105 @@ mod tests {
fn compress_end_test_state(
context: &mut CompressEndTestContext,
consumed_src_size: &u64,
consumed_src_size: &mut u64,
pledged_src_size_plus_one: u64,
content_size_flag: c_int,
) -> ZSTD_rust_compressEndState {
context.stage = 1;
let base = WINDOW_INIT_SENTINEL.as_ptr().cast::<c_void>();
context.stage = ZSTD_COMPRESSION_STAGE_ONGOING;
context.window = ZSTD_rust_windowUpdateState {
nextSrc: base,
base,
dictBase: base,
dictLimit: ZSTD_WINDOW_START_INDEX,
lowLimit: ZSTD_WINDOW_START_INDEX,
};
context.force_non_contiguous = 0;
context.next_to_update = ZSTD_WINDOW_START_INDEX;
context.is_first_block = 1;
context.produced_c_size = 0;
context.window_projection = ZSTD_rust_compressContinueWindowProjection {
next_src: ptr::addr_of_mut!(context.window.nextSrc),
base: ptr::addr_of_mut!(context.window.base),
dict_base: ptr::addr_of_mut!(context.window.dictBase),
dict_limit: ptr::addr_of_mut!(context.window.dictLimit),
low_limit: ptr::addr_of_mut!(context.window.lowLimit),
force_non_contiguous: ptr::addr_of_mut!(context.force_non_contiguous),
next_to_update: ptr::addr_of_mut!(context.next_to_update),
};
let callback_context = (context as *mut CompressEndTestContext).cast();
context.frame_chunk_clamp_state = Some(ZSTD_rust_frameChunkClampState {
next_to_update: ptr::addr_of_mut!(context.next_to_update),
low_limit: ptr::addr_of!(context.window.lowLimit),
});
let frame_chunk_clamp_state = context
.frame_chunk_clamp_state
.as_ref()
.map_or(ptr::null(), |state| state as *const _);
context.frame_chunk_prepare_state = Some(ZSTD_rust_frameChunkPrepareState {
callback_context,
max_dist: 64,
correct_overflow: compress_end_test_frame_prepare_overflow,
check_dict_validity: compress_end_test_frame_prepare_window,
enforce_max_dist: compress_end_test_frame_prepare_window,
clamp_state: frame_chunk_clamp_state,
});
let frame_chunk_prepare_state = context
.frame_chunk_prepare_state
.as_ref()
.map_or(ptr::null(), |state| state as *const _);
context.frame_chunk_state = Some(ZSTD_rust_frameChunkState {
callback_context,
tmp_workspace: ptr::null_mut(),
checksum_state: ptr::null_mut(),
is_first_block: ptr::addr_of_mut!(context.is_first_block),
stage: ptr::addr_of_mut!(context.stage),
tmp_wksp_size: 0,
block_size_max: 1,
savings: 0,
pre_block_splitter_level: 1,
strategy: ZSTD_FAST,
use_target_c_block_size: 1,
block_splitter_enabled: 0,
checksum_flag: 0,
ending_stage: ZSTD_COMPRESSION_STAGE_ENDING,
prepare_state: frame_chunk_prepare_state,
compress_target: compress_end_test_frame_target,
compress_split: compress_end_test_frame_target,
compress_internal: compress_end_test_frame_target,
});
let frame_chunk_state = context
.frame_chunk_state
.as_ref()
.map_or(ptr::null(), |state| state as *const _);
context.continue_state = Some(ZSTD_rust_compressContinueState {
callback_context,
window_state: &context.window_projection,
ldm_window_state: ptr::null(),
overflow_state: ptr::null(),
frame_chunk_state,
compress_block: compress_end_test_frame_target,
stage: ptr::addr_of_mut!(context.stage),
consumed_src_size: consumed_src_size as *mut u64,
produced_c_size: ptr::addr_of_mut!(context.produced_c_size),
pledged_src_size_plus_one,
block_size_max: 1,
check_block_size: 0,
no_dict_id_flag: 0,
checksum_flag: 0,
content_size_flag: 0,
format: 0,
window_log: 20,
dict_id: 0,
ldm_enabled: 0,
});
let continue_state = context
.continue_state
.as_ref()
.map_or(ptr::null(), |state| state as *const _);
ZSTD_rust_compressEndState {
callback_context: (context as *mut CompressEndTestContext).cast(),
compress_continue: compress_end_test_continue,
callback_context,
compress_continue_state: continue_state,
trace: compress_end_test_trace,
consumed_src_size: consumed_src_size as *const u64,
pledged_src_size_plus_one,
@@ -10224,20 +10333,28 @@ mod tests {
continue_result: 3,
..CompressEndTestContext::default()
};
let consumed_src_size = 7;
let state = compress_end_test_state(&mut context, &consumed_src_size, 8, 1);
let mut consumed_src_size = 7;
let state = compress_end_test_state(&mut context, &mut consumed_src_size, 9, 1);
let source = [0u8; 1];
let result = unsafe {
ZSTD_rust_compressEnd(&state, dst.as_mut_ptr().cast(), dst.len(), ptr::null(), 0)
ZSTD_rust_compressEnd(
&state,
dst.as_mut_ptr().cast(),
dst.len(),
source.as_ptr().cast(),
source.len(),
)
};
assert_eq!(result, 16);
assert_eq!(result, 7);
assert_eq!(context.events, ["continue", "trace"]);
assert_eq!(
(context.continue_frame, context.continue_last_frame_chunk),
(1, 1)
);
assert_eq!(context.trace_extra, 13);
assert_eq!(context.trace_extra, 4);
assert_eq!((consumed_src_size, context.produced_c_size), (8, 3));
assert_eq!(context.stage, 0);
}
@@ -10248,11 +10365,18 @@ mod tests {
continue_result: ERROR(ZstdErrorCode::MemoryAllocation),
..CompressEndTestContext::default()
};
let consumed_src_size = 7;
let state = compress_end_test_state(&mut context, &consumed_src_size, 8, 0);
let mut consumed_src_size = 7;
let state = compress_end_test_state(&mut context, &mut consumed_src_size, 8, 0);
let source = [0u8; 1];
let result = unsafe {
ZSTD_rust_compressEnd(&state, dst.as_mut_ptr().cast(), dst.len(), ptr::null(), 0)
ZSTD_rust_compressEnd(
&state,
dst.as_mut_ptr().cast(),
dst.len(),
source.as_ptr().cast(),
source.len(),
)
};
assert_eq!(result, ERROR(ZstdErrorCode::MemoryAllocation));
@@ -10261,16 +10385,23 @@ mod tests {
#[test]
fn compress_end_stops_before_validation_and_trace_when_epilogue_fails() {
let mut dst = [0u8; 8];
let mut dst = [0u8; 6];
let mut context = CompressEndTestContext {
continue_result: 3,
..CompressEndTestContext::default()
};
let consumed_src_size = 7;
let state = compress_end_test_state(&mut context, &consumed_src_size, 99, 1);
let mut consumed_src_size = 7;
let state = compress_end_test_state(&mut context, &mut consumed_src_size, 99, 1);
let source = [0u8; 1];
let result = unsafe {
ZSTD_rust_compressEnd(&state, dst.as_mut_ptr().cast(), dst.len(), ptr::null(), 0)
ZSTD_rust_compressEnd(
&state,
dst.as_mut_ptr().cast(),
dst.len(),
source.as_ptr().cast(),
source.len(),
)
};
assert_eq!(result, ERROR(ZstdErrorCode::DstSizeTooSmall));
@@ -10284,11 +10415,18 @@ mod tests {
continue_result: 3,
..CompressEndTestContext::default()
};
let consumed_src_size = 7;
let state = compress_end_test_state(&mut context, &consumed_src_size, 99, 1);
let mut consumed_src_size = 7;
let state = compress_end_test_state(&mut context, &mut consumed_src_size, 99, 1);
let source = [0u8; 1];
let result = unsafe {
ZSTD_rust_compressEnd(&state, dst.as_mut_ptr().cast(), dst.len(), ptr::null(), 0)
ZSTD_rust_compressEnd(
&state,
dst.as_mut_ptr().cast(),
dst.len(),
source.as_ptr().cast(),
source.len(),
)
};
assert_eq!(result, ERROR(ZstdErrorCode::SrcSizeWrong));