From 4a8345f596fe31f1a3093c1ffcee076ef9eb6e22 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Mon, 20 Jul 2026 00:12:48 +0200 Subject: [PATCH] feat(compress): project sequence API frame header in Rust Remove the sequence-compression API's C frame-header callback and carry the validated frame parameters as scalar state. Rust now serializes headers for both sequence API variants through the same frame-header leaf, while C keeps only CCtx initialization and checksum callbacks behind the boundary. Test Plan: - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --lib zstd_compress::tests::sequence_api -- --nocapture - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --all-targets -- -D warnings - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test - ulimit -v 41943040; make -j1 - ulimit -v 41943040; make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s - ulimit -v 41943040; make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests --- lib/compress/zstd_compress.c | 68 +++++++++++++++++++----------------- rust/src/zstd_compress.rs | 67 ++++++++++++++++++++++++----------- 2 files changed, 83 insertions(+), 52 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 468e9634d..946e55e52 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2742,16 +2742,14 @@ typedef char ZSTD_rust_sequence_literals_state_layout[ == 6 * sizeof(void*) + 4 * sizeof(int) + 3 * sizeof(void*)) ? 1 : -1]; -/* The public sequence APIs keep their CCtx initialization, frame header, and - * checksum state in C, but Rust owns the ordering and output accounting. The - * two block-loop projections above are populated by the initialization - * callback after the private CCtx has been initialized. */ +/* The public sequence APIs keep their CCtx initialization and checksum state + * in C, while Rust owns frame-header serialization, ordering, and output + * accounting. The two block-loop projections above are populated by the + * initialization callback after the private CCtx has been initialized. */ typedef struct ZSTD_rust_sequenceApiState_s ZSTD_rust_sequenceApiState; typedef size_t (*ZSTD_rust_sequenceApiInit_f)( void* context, size_t pledgedSrcSize, ZSTD_rust_sequenceApiState* state); -typedef size_t (*ZSTD_rust_sequenceApiWriteFrameHeader_f)( - void* context, void* dst, size_t dstCapacity, size_t pledgedSrcSize); typedef void (*ZSTD_rust_sequenceApiUpdateChecksum_f)( void* context, const void* src, size_t srcSize); typedef U32 (*ZSTD_rust_sequenceApiDigestChecksum_f)(void* context); @@ -2763,13 +2761,17 @@ struct ZSTD_rust_sequenceApiState_s { ZSTD_rust_sequenceCompressionState* sequenceState; ZSTD_rust_sequenceLiteralsState* sequenceLiteralsState; ZSTD_rust_sequenceApiInit_f init; - ZSTD_rust_sequenceApiWriteFrameHeader_f writeFrameHeader; ZSTD_rust_sequenceApiUpdateChecksum_f updateChecksum; ZSTD_rust_sequenceApiDigestChecksum_f digestChecksum; ZSTD_rust_sequenceApiWriteChecksum_f writeChecksum; int checksumFlag; int blockDelimiters; int validateSequences; + int noDictIDFlag; + int contentSizeFlag; + int format; + U32 windowLog; + U32 dictID; }; size_t ZSTD_rust_compressSequences( ZSTD_rust_sequenceApiState* state, @@ -2790,22 +2792,30 @@ typedef char ZSTD_rust_sequence_api_state_layout[ == 2 * sizeof(void*) && offsetof(ZSTD_rust_sequenceApiState, init) == 3 * sizeof(void*) - && offsetof(ZSTD_rust_sequenceApiState, writeFrameHeader) - == 4 * sizeof(void*) && offsetof(ZSTD_rust_sequenceApiState, updateChecksum) - == 5 * sizeof(void*) + == 4 * sizeof(void*) && offsetof(ZSTD_rust_sequenceApiState, digestChecksum) - == 6 * sizeof(void*) + == 5 * sizeof(void*) && offsetof(ZSTD_rust_sequenceApiState, writeChecksum) - == 7 * sizeof(void*) + == 6 * sizeof(void*) && offsetof(ZSTD_rust_sequenceApiState, checksumFlag) - == 8 * sizeof(void*) + == 7 * sizeof(void*) && offsetof(ZSTD_rust_sequenceApiState, blockDelimiters) - == 8 * sizeof(void*) + sizeof(int) + == 7 * sizeof(void*) + sizeof(int) && offsetof(ZSTD_rust_sequenceApiState, validateSequences) - == 8 * sizeof(void*) + 2 * sizeof(int) + == 7 * sizeof(void*) + 2 * sizeof(int) + && offsetof(ZSTD_rust_sequenceApiState, noDictIDFlag) + == 7 * sizeof(void*) + 3 * sizeof(int) + && offsetof(ZSTD_rust_sequenceApiState, contentSizeFlag) + == 7 * sizeof(void*) + 4 * sizeof(int) + && offsetof(ZSTD_rust_sequenceApiState, format) + == 7 * sizeof(void*) + 5 * sizeof(int) + && offsetof(ZSTD_rust_sequenceApiState, windowLog) + == 7 * sizeof(void*) + 6 * sizeof(int) + && offsetof(ZSTD_rust_sequenceApiState, dictID) + == 7 * sizeof(void*) + 6 * sizeof(int) + sizeof(U32) && sizeof(ZSTD_rust_sequenceApiState) - == (sizeof(void*) == 8 ? 80 : 44)) + == (sizeof(void*) == 8 ? 88 : 60)) ? 1 : -1]; typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1]; @@ -8019,24 +8029,14 @@ static size_t ZSTD_rust_sequenceApi_init( state->checksumFlag = cctx->appliedParams.fParams.checksumFlag; state->blockDelimiters = (int)cctx->appliedParams.blockDelimiters; state->validateSequences = cctx->appliedParams.validateSequences; + state->noDictIDFlag = cctx->appliedParams.fParams.noDictIDFlag; + state->contentSizeFlag = cctx->appliedParams.fParams.contentSizeFlag; + state->format = (int)cctx->appliedParams.format; + state->windowLog = cctx->appliedParams.cParams.windowLog; + state->dictID = cctx->dictID; return 0; } -static size_t ZSTD_rust_sequenceApi_writeFrameHeader( - void* context, void* dst, size_t dstCapacity, size_t pledgedSrcSize) -{ - ZSTD_CCtx const* const cctx = (ZSTD_CCtx const*)context; - return ZSTD_rust_writeFrameHeader( - dst, dstCapacity, - cctx->appliedParams.fParams.noDictIDFlag, - cctx->appliedParams.fParams.checksumFlag, - cctx->appliedParams.fParams.contentSizeFlag, - (int)cctx->appliedParams.format, - cctx->appliedParams.cParams.windowLog, - pledgedSrcSize, - cctx->dictID); -} - static void ZSTD_rust_sequenceApi_updateChecksum( void* context, const void* src, size_t srcSize) { @@ -8068,13 +8068,17 @@ static void ZSTD_rust_sequenceApi_initState( state->sequenceState = sequenceState; state->sequenceLiteralsState = sequenceLiteralsState; state->init = ZSTD_rust_sequenceApi_init; - state->writeFrameHeader = ZSTD_rust_sequenceApi_writeFrameHeader; state->updateChecksum = ZSTD_rust_sequenceApi_updateChecksum; state->digestChecksum = ZSTD_rust_sequenceApi_digestChecksum; state->writeChecksum = ZSTD_rust_sequenceApi_writeChecksum; state->checksumFlag = 0; state->blockDelimiters = 0; state->validateSequences = 0; + state->noDictIDFlag = 0; + state->contentSizeFlag = 0; + state->format = 0; + state->windowLog = 0; + state->dictID = 0; } size_t ZSTD_compressSequences(ZSTD_CCtx* cctx, diff --git a/rust/src/zstd_compress.rs b/rust/src/zstd_compress.rs index 5c7fd04c3..28ce0665c 100644 --- a/rust/src/zstd_compress.rs +++ b/rust/src/zstd_compress.rs @@ -3739,8 +3739,6 @@ const _: () = { type SequenceApiInitFn = unsafe extern "C" fn(*mut c_void, usize, *mut ZSTD_rust_sequenceApiState) -> usize; -type SequenceApiWriteFrameHeaderFn = - unsafe extern "C" fn(*mut c_void, *mut c_void, usize, usize) -> usize; type SequenceApiUpdateChecksumFn = unsafe extern "C" fn(*mut c_void, *const c_void, usize); type SequenceApiDigestChecksumFn = unsafe extern "C" fn(*mut c_void) -> c_uint; type SequenceApiWriteChecksumFn = unsafe extern "C" fn(*mut c_void, *mut c_void, c_uint); @@ -3748,21 +3746,26 @@ type SequenceApiWriteChecksumFn = unsafe extern "C" fn(*mut c_void, *mut c_void, /// Explicit projection for the public sequence-compression API orchestration. /// /// Rust owns validation ordering, frame-header/checksum sequencing, and -/// output accounting. C retains the private CCtx, sequence-store, block, -/// and checksum layouts through the two block-state projections and callbacks. +/// output accounting. Frame-header parameters are projected as scalars; C +/// retains the private CCtx, sequence-store, block, and checksum layouts +/// through the two block-state projections and callbacks. #[repr(C)] pub struct ZSTD_rust_sequenceApiState { callback_context: *mut c_void, sequence_state: *mut ZSTD_rust_sequenceCompressionState, sequence_literals_state: *mut ZSTD_rust_sequenceLiteralsState, init: SequenceApiInitFn, - write_frame_header: SequenceApiWriteFrameHeaderFn, update_checksum: SequenceApiUpdateChecksumFn, digest_checksum: SequenceApiDigestChecksumFn, write_checksum: SequenceApiWriteChecksumFn, checksum_flag: c_int, block_delimiters: c_int, validate_sequences: c_int, + no_dict_id_flag: c_int, + content_size_flag: c_int, + format: c_int, + window_log: c_uint, + dict_id: c_uint, } const _: () = { @@ -3772,21 +3775,40 @@ const _: () = { offset_of!(ZSTD_rust_sequenceApiState, sequence_literals_state) == 2 * size_of::() ); assert!(offset_of!(ZSTD_rust_sequenceApiState, init) == 3 * size_of::()); - assert!(offset_of!(ZSTD_rust_sequenceApiState, write_frame_header) == 4 * size_of::()); - assert!(offset_of!(ZSTD_rust_sequenceApiState, update_checksum) == 5 * size_of::()); - assert!(offset_of!(ZSTD_rust_sequenceApiState, digest_checksum) == 6 * size_of::()); - assert!(offset_of!(ZSTD_rust_sequenceApiState, write_checksum) == 7 * size_of::()); - assert!(offset_of!(ZSTD_rust_sequenceApiState, checksum_flag) == size_of::<[usize; 8]>()); + assert!(offset_of!(ZSTD_rust_sequenceApiState, update_checksum) == 4 * size_of::()); + assert!(offset_of!(ZSTD_rust_sequenceApiState, digest_checksum) == 5 * size_of::()); + assert!(offset_of!(ZSTD_rust_sequenceApiState, write_checksum) == 6 * size_of::()); + assert!(offset_of!(ZSTD_rust_sequenceApiState, checksum_flag) == size_of::<[usize; 7]>()); assert!( offset_of!(ZSTD_rust_sequenceApiState, block_delimiters) - == size_of::<[usize; 8]>() + size_of::() + == size_of::<[usize; 7]>() + size_of::() ); assert!( offset_of!(ZSTD_rust_sequenceApiState, validate_sequences) - == size_of::<[usize; 8]>() + 2 * size_of::() + == size_of::<[usize; 7]>() + size_of::<[c_int; 2]>() ); assert!( - size_of::() == if size_of::() == 8 { 80 } else { 44 } + offset_of!(ZSTD_rust_sequenceApiState, no_dict_id_flag) + == size_of::<[usize; 7]>() + size_of::<[c_int; 3]>() + ); + assert!( + offset_of!(ZSTD_rust_sequenceApiState, content_size_flag) + == size_of::<[usize; 7]>() + size_of::<[c_int; 4]>() + ); + assert!( + offset_of!(ZSTD_rust_sequenceApiState, format) + == size_of::<[usize; 7]>() + size_of::<[c_int; 5]>() + ); + assert!( + offset_of!(ZSTD_rust_sequenceApiState, window_log) + == size_of::<[usize; 7]>() + size_of::<[c_int; 6]>() + ); + assert!( + offset_of!(ZSTD_rust_sequenceApiState, dict_id) + == size_of::<[usize; 7]>() + size_of::<[c_int; 6]>() + size_of::() + ); + assert!( + size_of::() == if size_of::() == 8 { 88 } else { 60 } ); }; @@ -8241,7 +8263,6 @@ unsafe fn sequence_api_prepare( || state.sequence_state.is_null() || state.sequence_literals_state.is_null() || state.init as usize == 0 - || state.write_frame_header as usize == 0 || state.update_checksum as usize == 0 || state.digest_checksum as usize == 0 || state.write_checksum as usize == 0 @@ -8276,11 +8297,16 @@ unsafe fn sequence_api_write_frame_header( pledged_src_size: usize, ) -> usize { let frame_header_size = unsafe { - (state.write_frame_header)( - state.callback_context, + ZSTD_rust_writeFrameHeader( (*op).cast(), *dst_capacity, - pledged_src_size, + state.no_dict_id_flag, + state.checksum_flag, + state.content_size_flag, + state.format, + state.window_log, + pledged_src_size as u64, + state.dict_id, ) }; if ERR_isError(frame_header_size) { @@ -8325,9 +8351,10 @@ unsafe fn sequence_api_append_frame_checksum( /// Rust-owned orchestration for `ZSTD_compressSequences`. /// /// The C wrapper provides the post-initialization block-state projection and -/// callbacks for the private CCtx/header/checksum operations. Rust preserves -/// the public ordering: initialize, write the frame header, update the input -/// checksum, emit blocks, then append the frame checksum. +/// callbacks for the private CCtx/checksum operations; frame-header fields are +/// projected as scalars. Rust preserves the public ordering: initialize, +/// write the frame header, update the input checksum, emit blocks, then append +/// the frame checksum. #[no_mangle] pub unsafe extern "C" fn ZSTD_rust_compressSequences( state: *mut ZSTD_rust_sequenceApiState,