ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): project sequence API frame header in Rust

Browse Source 
Remove the sequence-compression API's C frame-header callback and carry the
validated frame parameters as scalar state.  Rust now serializes headers for
both sequence API variants through the same frame-header leaf, while C keeps
only CCtx initialization and checksum callbacks behind the boundary.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --lib zstd_compress::tests::sequence_api -- --nocapture
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --all-targets -- -D warnings
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test
- ulimit -v 41943040; make -j1
- ulimit -v 41943040; make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s
- ulimit -v 41943040; make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests
This commit is contained in:
ddidderr
2026-07-20 00:12:48 +02:00
parent 534d7c9264
commit 4a8345f596
2 changed files with 83 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+36 -32
View File
@@ -2742,16 +2742,14 @@ typedef char ZSTD_rust_sequence_literals_state_layout[
== 6 * sizeof(void*) + 4 * sizeof(int) + 3 * sizeof(void*))
? 1 : -1];
/* The public sequence APIs keep their CCtx initialization, frame header, and
* checksum state in C, but Rust owns the ordering and output accounting. The
* two block-loop projections above are populated by the initialization
* callback after the private CCtx has been initialized. */
/* The public sequence APIs keep their CCtx initialization and checksum state
* in C, while Rust owns frame-header serialization, ordering, and output
* accounting. The two block-loop projections above are populated by the
* initialization callback after the private CCtx has been initialized. */
typedef struct ZSTD_rust_sequenceApiState_s ZSTD_rust_sequenceApiState;
typedef size_t (*ZSTD_rust_sequenceApiInit_f)(
void* context, size_t pledgedSrcSize,
ZSTD_rust_sequenceApiState* state);
typedef size_t (*ZSTD_rust_sequenceApiWriteFrameHeader_f)(
void* context, void* dst, size_t dstCapacity, size_t pledgedSrcSize);
typedef void (*ZSTD_rust_sequenceApiUpdateChecksum_f)(
void* context, const void* src, size_t srcSize);
typedef U32 (*ZSTD_rust_sequenceApiDigestChecksum_f)(void* context);
@@ -2763,13 +2761,17 @@ struct ZSTD_rust_sequenceApiState_s {
ZSTD_rust_sequenceCompressionState* sequenceState;
ZSTD_rust_sequenceLiteralsState* sequenceLiteralsState;
ZSTD_rust_sequenceApiInit_f init;
ZSTD_rust_sequenceApiWriteFrameHeader_f writeFrameHeader;
ZSTD_rust_sequenceApiUpdateChecksum_f updateChecksum;
ZSTD_rust_sequenceApiDigestChecksum_f digestChecksum;
ZSTD_rust_sequenceApiWriteChecksum_f writeChecksum;
int checksumFlag;
int blockDelimiters;
int validateSequences;
int noDictIDFlag;
int contentSizeFlag;
int format;
U32 windowLog;
U32 dictID;
};
size_t ZSTD_rust_compressSequences(
ZSTD_rust_sequenceApiState* state,
@@ -2790,22 +2792,30 @@ typedef char ZSTD_rust_sequence_api_state_layout[
== 2 * sizeof(void*)
&& offsetof(ZSTD_rust_sequenceApiState, init)
== 3 * sizeof(void*)
&& offsetof(ZSTD_rust_sequenceApiState, writeFrameHeader)
== 4 * sizeof(void*)
&& offsetof(ZSTD_rust_sequenceApiState, updateChecksum)
== 5 * sizeof(void*)
== 4 * sizeof(void*)
&& offsetof(ZSTD_rust_sequenceApiState, digestChecksum)
== 6 * sizeof(void*)
== 5 * sizeof(void*)
&& offsetof(ZSTD_rust_sequenceApiState, writeChecksum)
== 7 * sizeof(void*)
== 6 * sizeof(void*)
&& offsetof(ZSTD_rust_sequenceApiState, checksumFlag)
== 8 * sizeof(void*)
== 7 * sizeof(void*)
&& offsetof(ZSTD_rust_sequenceApiState, blockDelimiters)
== 8 * sizeof(void*) + sizeof(int)
== 7 * sizeof(void*) + sizeof(int)
&& offsetof(ZSTD_rust_sequenceApiState, validateSequences)
== 8 * sizeof(void*) + 2 * sizeof(int)
== 7 * sizeof(void*) + 2 * sizeof(int)
&& offsetof(ZSTD_rust_sequenceApiState, noDictIDFlag)
== 7 * sizeof(void*) + 3 * sizeof(int)
&& offsetof(ZSTD_rust_sequenceApiState, contentSizeFlag)
== 7 * sizeof(void*) + 4 * sizeof(int)
&& offsetof(ZSTD_rust_sequenceApiState, format)
== 7 * sizeof(void*) + 5 * sizeof(int)
&& offsetof(ZSTD_rust_sequenceApiState, windowLog)
== 7 * sizeof(void*) + 6 * sizeof(int)
&& offsetof(ZSTD_rust_sequenceApiState, dictID)
== 7 * sizeof(void*) + 6 * sizeof(int) + sizeof(U32)
&& sizeof(ZSTD_rust_sequenceApiState)
== (sizeof(void*) == 8 ? 80 : 44))
== (sizeof(void*) == 8 ? 88 : 60))
? 1 : -1];
typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
@@ -8019,24 +8029,14 @@ static size_t ZSTD_rust_sequenceApi_init(
state->checksumFlag = cctx->appliedParams.fParams.checksumFlag;
state->blockDelimiters = (int)cctx->appliedParams.blockDelimiters;
state->validateSequences = cctx->appliedParams.validateSequences;
state->noDictIDFlag = cctx->appliedParams.fParams.noDictIDFlag;
state->contentSizeFlag = cctx->appliedParams.fParams.contentSizeFlag;
state->format = (int)cctx->appliedParams.format;
state->windowLog = cctx->appliedParams.cParams.windowLog;
state->dictID = cctx->dictID;
return 0;
}
static size_t ZSTD_rust_sequenceApi_writeFrameHeader(
void* context, void* dst, size_t dstCapacity, size_t pledgedSrcSize)
{
ZSTD_CCtx const* const cctx = (ZSTD_CCtx const*)context;
return ZSTD_rust_writeFrameHeader(
dst, dstCapacity,
cctx->appliedParams.fParams.noDictIDFlag,
cctx->appliedParams.fParams.checksumFlag,
cctx->appliedParams.fParams.contentSizeFlag,
(int)cctx->appliedParams.format,
cctx->appliedParams.cParams.windowLog,
pledgedSrcSize,
cctx->dictID);
}
static void ZSTD_rust_sequenceApi_updateChecksum(
void* context, const void* src, size_t srcSize)
{
@@ -8068,13 +8068,17 @@ static void ZSTD_rust_sequenceApi_initState(
state->sequenceState = sequenceState;
state->sequenceLiteralsState = sequenceLiteralsState;
state->init = ZSTD_rust_sequenceApi_init;
state->writeFrameHeader = ZSTD_rust_sequenceApi_writeFrameHeader;
state->updateChecksum = ZSTD_rust_sequenceApi_updateChecksum;
state->digestChecksum = ZSTD_rust_sequenceApi_digestChecksum;
state->writeChecksum = ZSTD_rust_sequenceApi_writeChecksum;
state->checksumFlag = 0;
state->blockDelimiters = 0;
state->validateSequences = 0;
state->noDictIDFlag = 0;
state->contentSizeFlag = 0;
state->format = 0;
state->windowLog = 0;
state->dictID = 0;
}
size_t ZSTD_compressSequences(ZSTD_CCtx* cctx,
rust/src/zstd_compress.rs
+47 -20
View File
@@ -3739,8 +3739,6 @@ const _: () = {
type SequenceApiInitFn =
unsafe extern "C" fn(*mut c_void, usize, *mut ZSTD_rust_sequenceApiState) -> usize;
type SequenceApiWriteFrameHeaderFn =
unsafe extern "C" fn(*mut c_void, *mut c_void, usize, usize) -> usize;
type SequenceApiUpdateChecksumFn = unsafe extern "C" fn(*mut c_void, *const c_void, usize);
type SequenceApiDigestChecksumFn = unsafe extern "C" fn(*mut c_void) -> c_uint;
type SequenceApiWriteChecksumFn = unsafe extern "C" fn(*mut c_void, *mut c_void, c_uint);
@@ -3748,21 +3746,26 @@ type SequenceApiWriteChecksumFn = unsafe extern "C" fn(*mut c_void, *mut c_void,
/// Explicit projection for the public sequence-compression API orchestration.
///
/// Rust owns validation ordering, frame-header/checksum sequencing, and
/// output accounting. C retains the private CCtx, sequence-store, block,
/// and checksum layouts through the two block-state projections and callbacks.
/// output accounting. Frame-header parameters are projected as scalars; C
/// retains the private CCtx, sequence-store, block, and checksum layouts
/// through the two block-state projections and callbacks.
#[repr(C)]
pub struct ZSTD_rust_sequenceApiState {
callback_context: *mut c_void,
sequence_state: *mut ZSTD_rust_sequenceCompressionState,
sequence_literals_state: *mut ZSTD_rust_sequenceLiteralsState,
init: SequenceApiInitFn,
write_frame_header: SequenceApiWriteFrameHeaderFn,
update_checksum: SequenceApiUpdateChecksumFn,
digest_checksum: SequenceApiDigestChecksumFn,
write_checksum: SequenceApiWriteChecksumFn,
checksum_flag: c_int,
block_delimiters: c_int,
validate_sequences: c_int,
no_dict_id_flag: c_int,
content_size_flag: c_int,
format: c_int,
window_log: c_uint,
dict_id: c_uint,
}
const _: () = {
@@ -3772,21 +3775,40 @@ const _: () = {
offset_of!(ZSTD_rust_sequenceApiState, sequence_literals_state) == 2 * size_of::<usize>()
);
assert!(offset_of!(ZSTD_rust_sequenceApiState, init) == 3 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_sequenceApiState, write_frame_header) == 4 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_sequenceApiState, update_checksum) == 5 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_sequenceApiState, digest_checksum) == 6 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_sequenceApiState, write_checksum) == 7 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_sequenceApiState, checksum_flag) == size_of::<[usize; 8]>());
assert!(offset_of!(ZSTD_rust_sequenceApiState, update_checksum) == 4 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_sequenceApiState, digest_checksum) == 5 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_sequenceApiState, write_checksum) == 6 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_sequenceApiState, checksum_flag) == size_of::<[usize; 7]>());
assert!(
offset_of!(ZSTD_rust_sequenceApiState, block_delimiters)
== size_of::<[usize; 8]>() + size_of::<c_int>()
== size_of::<[usize; 7]>() + size_of::<c_int>()
);
assert!(
offset_of!(ZSTD_rust_sequenceApiState, validate_sequences)
== size_of::<[usize; 8]>() + 2 * size_of::<c_int>()
== size_of::<[usize; 7]>() + size_of::<[c_int; 2]>()
);
assert!(
size_of::<ZSTD_rust_sequenceApiState>() == if size_of::<usize>() == 8 { 80 } else { 44 }
offset_of!(ZSTD_rust_sequenceApiState, no_dict_id_flag)
== size_of::<[usize; 7]>() + size_of::<[c_int; 3]>()
);
assert!(
offset_of!(ZSTD_rust_sequenceApiState, content_size_flag)
== size_of::<[usize; 7]>() + size_of::<[c_int; 4]>()
);
assert!(
offset_of!(ZSTD_rust_sequenceApiState, format)
== size_of::<[usize; 7]>() + size_of::<[c_int; 5]>()
);
assert!(
offset_of!(ZSTD_rust_sequenceApiState, window_log)
== size_of::<[usize; 7]>() + size_of::<[c_int; 6]>()
);
assert!(
offset_of!(ZSTD_rust_sequenceApiState, dict_id)
== size_of::<[usize; 7]>() + size_of::<[c_int; 6]>() + size_of::<c_uint>()
);
assert!(
size_of::<ZSTD_rust_sequenceApiState>() == if size_of::<usize>() == 8 { 88 } else { 60 }
);
};
@@ -8241,7 +8263,6 @@ unsafe fn sequence_api_prepare(
|| state.sequence_state.is_null()
|| state.sequence_literals_state.is_null()
|| state.init as usize == 0
|| state.write_frame_header as usize == 0
|| state.update_checksum as usize == 0
|| state.digest_checksum as usize == 0
|| state.write_checksum as usize == 0
@@ -8276,11 +8297,16 @@ unsafe fn sequence_api_write_frame_header(
pledged_src_size: usize,
) -> usize {
let frame_header_size = unsafe {
(state.write_frame_header)(
state.callback_context,
ZSTD_rust_writeFrameHeader(
(*op).cast(),
*dst_capacity,
pledged_src_size,
state.no_dict_id_flag,
state.checksum_flag,
state.content_size_flag,
state.format,
state.window_log,
pledged_src_size as u64,
state.dict_id,
)
};
if ERR_isError(frame_header_size) {
@@ -8325,9 +8351,10 @@ unsafe fn sequence_api_append_frame_checksum(
/// Rust-owned orchestration for `ZSTD_compressSequences`.
///
/// The C wrapper provides the post-initialization block-state projection and
/// callbacks for the private CCtx/header/checksum operations. Rust preserves
/// the public ordering: initialize, write the frame header, update the input
/// checksum, emit blocks, then append the frame checksum.
/// callbacks for the private CCtx/checksum operations; frame-header fields are
/// projected as scalars. Rust preserves the public ordering: initialize,
/// write the frame header, update the input checksum, emit blocks, then append
/// the frame checksum.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressSequences(
state: *mut ZSTD_rust_sequenceApiState,