ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move frame progression input policy to Rust

Browse Source 
Move buffered-input presence, offset subtraction, and block-size validation into
Rust while keeping the private MT progression query behind its C callback. The
C entry point now projects only the consumed/produced counters, input-buffer
fields, and worker dispatch inputs. Rust preserves the null-buffer behavior and
aborts on the same impossible offset or oversized-buffer invariants as the
original C assertions.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --manifest-path rust/Cargo.toml --tests
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1
This commit is contained in:
ddidderr
2026-07-21 18:51:04 +02:00
parent 4fc1a43977
commit 42e6662a6d
2 changed files with 117 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+21 -11
View File
@@ -757,10 +757,24 @@ ZSTD_frameProgression ZSTD_rust_frameProgression(U64 consumedSrcSize,
U64 producedCSize);
typedef ZSTD_frameProgression (*ZSTD_rust_frameProgressionMTorST_f)(
void* context);
typedef struct {
const void* inBuff;
size_t inBuffPos;
size_t inToCompress;
} ZSTD_rust_frameProgressionInputState;
typedef char ZSTD_rust_frame_progression_input_state_layout[
(offsetof(ZSTD_rust_frameProgressionInputState, inBuff) == 0
&& offsetof(ZSTD_rust_frameProgressionInputState, inBuffPos)
== sizeof(void*)
&& offsetof(ZSTD_rust_frameProgressionInputState, inToCompress)
== sizeof(void*) + sizeof(size_t)
&& sizeof(ZSTD_rust_frameProgressionInputState)
== sizeof(void*) + 2 * sizeof(size_t))
? 1 : -1];
typedef struct {
U64 consumedSrcSize;
U64 producedCSize;
size_t buffered;
ZSTD_rust_frameProgressionInputState input;
void* mtContext;
ZSTD_rust_frameProgressionMTorST_f mtFrameProgression;
int nbWorkers;
@@ -770,16 +784,16 @@ typedef char ZSTD_rust_frame_progression_mt_or_st_state_layout[
&& offsetof(ZSTD_rust_frameProgressionMTorSTState, consumedSrcSize) == 0
&& offsetof(ZSTD_rust_frameProgressionMTorSTState, producedCSize)
== sizeof(U64)
&& offsetof(ZSTD_rust_frameProgressionMTorSTState, buffered)
&& offsetof(ZSTD_rust_frameProgressionMTorSTState, input)
== 2 * sizeof(U64)
&& offsetof(ZSTD_rust_frameProgressionMTorSTState, mtContext)
== 2 * sizeof(U64) + sizeof(size_t)
== 2 * sizeof(U64) + sizeof(void*) + 2 * sizeof(size_t)
&& offsetof(ZSTD_rust_frameProgressionMTorSTState, mtFrameProgression)
== 2 * sizeof(U64) + sizeof(size_t) + sizeof(void*)
== 2 * sizeof(U64) + 2 * sizeof(void*) + 2 * sizeof(size_t)
&& offsetof(ZSTD_rust_frameProgressionMTorSTState, nbWorkers)
== 2 * sizeof(U64) + sizeof(size_t) + 2 * sizeof(void*)
== 2 * sizeof(U64) + 3 * sizeof(void*) + 2 * sizeof(size_t)
&& sizeof(ZSTD_rust_frameProgressionMTorSTState)
== (sizeof(void*) == 8 ? 48 : 32))
== (sizeof(void*) == 8 ? 64 : 40))
? 1 : -1];
ZSTD_frameProgression ZSTD_rust_frameProgressionMTorST(
const ZSTD_rust_frameProgressionMTorSTState* state);
@@ -4835,18 +4849,14 @@ static ZSTD_frameProgression ZSTD_rust_frameProgression_MT(void* context)
ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx)
{
size_t const buffered = (cctx->inBuff == NULL) ? 0 :
cctx->inBuffPos - cctx->inToCompress;
ZSTD_rust_frameProgressionMTorSTState state = {
cctx->consumedSrcSize,
cctx->producedCSize,
buffered,
{ cctx->inBuff, cctx->inBuffPos, cctx->inToCompress },
NULL,
NULL,
0
};
if (buffered) assert(cctx->inBuffPos >= cctx->inToCompress);
assert(buffered <= ZSTD_BLOCKSIZE_MAX);
#ifdef ZSTD_MULTITHREAD
state.mtContext = cctx->mtctx;
state.mtFrameProgression = ZSTD_rust_frameProgression_MT;
rust/src/zstd_compress.rs
+95 -16
View File
@@ -8010,16 +8010,56 @@ pub extern "C" fn ZSTD_rust_frameProgression(
type FrameProgressionMTorSTFn = unsafe extern "C" fn(*mut c_void) -> ZSTD_frameProgression;
/// Project the C stream-buffer fields needed to calculate buffered input.
///
/// The pointer is used only as the C-side presence flag. Rust owns the
/// null-buffer policy, offset subtraction, and the block-size assertion.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZSTD_rust_frameProgressionInputState {
in_buff: *const c_void,
in_buff_pos: usize,
in_to_compress: usize,
}
const _: () = {
assert!(offset_of!(ZSTD_rust_frameProgressionInputState, in_buff) == 0);
assert!(
offset_of!(ZSTD_rust_frameProgressionInputState, in_buff_pos) == size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_frameProgressionInputState, in_to_compress)
== 2 * size_of::<usize>()
);
assert!(size_of::<ZSTD_rust_frameProgressionInputState>() == 3 * size_of::<usize>());
};
#[inline]
fn frame_progression_buffered_input(state: &ZSTD_rust_frameProgressionInputState) -> usize {
if state.in_buff.is_null() {
return 0;
}
/* C performs the size_t subtraction before evaluating its assertions. */
let buffered = state.in_buff_pos.wrapping_sub(state.in_to_compress);
if buffered != 0 && state.in_buff_pos < state.in_to_compress {
std::process::abort();
}
if buffered > ZSTD_BLOCKSIZE_MAX {
std::process::abort();
}
buffered
}
/// Projection for the MT-versus-single-thread frame-progression dispatch.
///
/// Rust owns only the worker-count branch. The single-thread progression
/// inputs are scalar, while the private MT context remains opaque behind its
/// C callback.
/// Rust owns the buffered-input policy and worker-count branch. The private
/// MT context remains opaque behind its C callback.
#[repr(C)]
pub struct ZSTD_rust_frameProgressionMTorSTState {
consumed_src_size: u64,
produced_c_size: u64,
buffered: usize,
input: ZSTD_rust_frameProgressionInputState,
mt_context: *mut c_void,
mt_frame_progression: Option<FrameProgressionMTorSTFn>,
nb_workers: c_int,
@@ -8030,22 +8070,24 @@ const _: () = {
assert!(size_of::<Option<FrameProgressionMTorSTFn>>() == size_of::<usize>());
assert!(offset_of!(ZSTD_rust_frameProgressionMTorSTState, consumed_src_size) == 0);
assert!(offset_of!(ZSTD_rust_frameProgressionMTorSTState, produced_c_size) == size_of::<u64>());
assert!(offset_of!(ZSTD_rust_frameProgressionMTorSTState, buffered) == 2 * size_of::<u64>());
assert!(
offset_of!(ZSTD_rust_frameProgressionMTorSTState, input) == 2 * size_of::<u64>()
);
assert!(
offset_of!(ZSTD_rust_frameProgressionMTorSTState, mt_context)
== 2 * size_of::<u64>() + size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_frameProgressionMTorSTState, mt_frame_progression)
== 2 * size_of::<u64>() + 2 * size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_frameProgressionMTorSTState, nb_workers)
== 2 * size_of::<u64>() + 3 * size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_frameProgressionMTorSTState, mt_frame_progression)
== 2 * size_of::<u64>() + 4 * size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_frameProgressionMTorSTState, nb_workers)
== 2 * size_of::<u64>() + 5 * size_of::<usize>()
);
assert!(
size_of::<ZSTD_rust_frameProgressionMTorSTState>()
== if size_of::<usize>() == 8 { 48 } else { 32 }
== if size_of::<usize>() == 8 { 64 } else { 40 }
);
};
@@ -8058,6 +8100,7 @@ pub unsafe extern "C" fn ZSTD_rust_frameProgressionMTorST(
let Some(state) = (unsafe { state.as_ref() }) else {
return ZSTD_frameProgression::default();
};
let buffered = frame_progression_buffered_input(&state.input);
if state.nb_workers > 0 {
let Some(mt_frame_progression) = state.mt_frame_progression else {
return ZSTD_frameProgression::default();
@@ -8066,7 +8109,7 @@ pub unsafe extern "C" fn ZSTD_rust_frameProgressionMTorST(
}
frame_progression(
state.consumed_src_size,
state.buffered,
buffered,
state.produced_c_size,
)
}
@@ -17783,6 +17826,38 @@ mod tests {
assert_eq!(progression.nbActiveWorkers, 0);
}
#[test]
fn frame_progression_buffered_input_ignores_offsets_without_a_buffer() {
let state = ZSTD_rust_frameProgressionInputState {
in_buff: ptr::null(),
in_buff_pos: usize::MAX,
in_to_compress: usize::MAX - 1,
};
assert_eq!(frame_progression_buffered_input(&state), 0);
}
#[test]
fn frame_progression_buffered_input_uses_valid_offsets_through_block_limit() {
let marker = 0u8;
let state = ZSTD_rust_frameProgressionInputState {
in_buff: (&marker as *const u8).cast(),
in_buff_pos: 37,
in_to_compress: 11,
};
assert_eq!(frame_progression_buffered_input(&state), 26);
let boundary_state = ZSTD_rust_frameProgressionInputState {
in_buff: (&marker as *const u8).cast(),
in_buff_pos: ZSTD_BLOCKSIZE_MAX + 11,
in_to_compress: 11,
};
assert_eq!(
frame_progression_buffered_input(&boundary_state),
ZSTD_BLOCKSIZE_MAX
);
}
unsafe extern "C" fn frame_progression_mt_or_st_test_callback(
context: *mut c_void,
) -> ZSTD_frameProgression {
@@ -17805,7 +17880,11 @@ mod tests {
ZSTD_rust_frameProgressionMTorSTState {
consumed_src_size: 11,
produced_c_size: 53,
buffered: 37,
input: ZSTD_rust_frameProgressionInputState {
in_buff: (calls as *mut usize).cast(),
in_buff_pos: 37,
in_to_compress: 0,
},
mt_context: calls as *mut usize as *mut c_void,
mt_frame_progression: Some(frame_progression_mt_or_st_test_callback),
nb_workers,