From 9605174324d46a40d45aec5be76c85d19470a672 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Mon, 20 Jul 2026 01:41:30 +0200 Subject: [PATCH] feat(compress): move frame chunk internal blocks into Rust Route frame-chunk internal blocks through the projected build-sequence-store and block-internal state, using a Rust-owned frame-chunk leaf while retaining the target and split callbacks in C. Refresh the per-block isFirstBlock snapshot for the direct path so its behavior remains identical to the former C adapter, and remove the obsolete C frame-chunk and block-internal adapters. Test Plan: - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo fmt --manifest-path rust/Cargo.toml --all -- --check - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --lib - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040; make -j1 - ulimit -v 41943040; make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s - ulimit -v 41943040; make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests --- lib/compress/zstd_compress.c | 50 +++-------- rust/src/zstd_compress.rs | 169 ++++++++++++++++++++++++++--------- 2 files changed, 136 insertions(+), 83 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 647394711..14536ee2a 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -815,6 +815,8 @@ typedef char ZSTD_rust_frame_chunk_prepare_state_layout[ == 5 * sizeof(void*) && sizeof(ZSTD_rust_frameChunkPrepareState) == 6 * sizeof(void*)) ? 1 : -1]; +typedef struct ZSTD_rust_compressContinueBlockState_s + ZSTD_rust_compressContinueBlockState; typedef size_t (*ZSTD_rust_frameChunkCompress_f)(void* context, void* dst, size_t dstCapacity, @@ -840,6 +842,7 @@ typedef struct { ZSTD_rust_frameChunkCompress_f compressTarget; ZSTD_rust_frameChunkCompress_f compressSplit; ZSTD_rust_frameChunkCompress_f compressInternal; + const ZSTD_rust_compressContinueBlockState* compressInternalState; } ZSTD_rust_frameChunkState; size_t ZSTD_rust_compressFrameChunk( const ZSTD_rust_frameChunkState* state, @@ -869,8 +872,10 @@ typedef char ZSTD_rust_frame_chunk_state_layout[ == 7 * sizeof(void*) + sizeof(S64) + 5 * sizeof(int) && offsetof(ZSTD_rust_frameChunkState, prepareState) == 7 * sizeof(void*) + sizeof(S64) + 6 * sizeof(int) + && offsetof(ZSTD_rust_frameChunkState, compressInternalState) + == 11 * sizeof(void*) + sizeof(S64) + 6 * sizeof(int) && sizeof(ZSTD_rust_frameChunkState) - == 11 * sizeof(void*) + sizeof(S64) + 6 * sizeof(int)) + == 12 * sizeof(void*) + sizeof(S64) + 6 * sizeof(int)) ? 1 : -1]; /* The high-level continue/block entry points are Rust-owned. This projection @@ -5363,32 +5368,6 @@ ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc, return cSize; } -static size_t -ZSTD_compressBlock_internal(ZSTD_CCtx* zc, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, U32 frame) -{ - ZSTD_rust_blockInternalState state; - size_t const bss = ZSTD_buildSeqStore(zc, src, srcSize); - DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", - (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, - (unsigned)zc->blockState.matchState.nextToUpdate); - FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); - - state.seqStore = &zc->seqStore; - state.prevCBlock = &zc->blockState.prevCBlock; - state.nextCBlock = &zc->blockState.nextCBlock; - state.tmpWorkspace = zc->tmpWorkspace; - state.tmpWkspSize = zc->tmpWkspSize; - state.seqCollector = &zc->seqCollector; - state.strategy = (int)zc->appliedParams.cParams.strategy; - state.disableLiteralCompression = ZSTD_literalsCompressionIsDisabled(&zc->appliedParams); - state.bmi2 = zc->bmi2; - state.isFirstBlock = zc->isFirstBlock; - return ZSTD_rust_compressBlockInternalAfterBuild( - &state, dst, dstCapacity, src, srcSize, frame, (int)bss); -} - static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -5425,10 +5404,10 @@ typedef struct { const ZSTD_CCtx_params* params; } ZSTD_rust_overflowCorrectContext; -typedef struct { +struct ZSTD_rust_compressContinueBlockState_s { const ZSTD_rust_buildSeqStoreState* buildSeqStoreState; const ZSTD_rust_blockInternalState* blockInternalState; -} ZSTD_rust_compressContinueBlockState; +}; typedef char ZSTD_rust_compress_continue_block_state_layout[ (sizeof(ZSTD_rust_compressContinueBlockState) == 2 * sizeof(void*)) ? 1 : -1]; @@ -5576,16 +5555,6 @@ static size_t ZSTD_rust_frameChunk_compressSplit( (ZSTD_CCtx*)context, dst, dstCapacity, src, srcSize, lastBlock); } -static size_t ZSTD_rust_frameChunk_compressInternal( - void* context, void* dst, size_t dstCapacity, - const void* src, size_t srcSize, U32 lastBlock) -{ - (void)lastBlock; - return ZSTD_compressBlock_internal( - (ZSTD_CCtx*)context, dst, dstCapacity, src, srcSize, - 1 /* frame */); -} - static void ZSTD_compressContinue_prepare( ZSTD_CCtx* cctx, size_t blockSizeMax, int checkBlockSize, ZSTD_rust_compressContinueContext* context) @@ -5634,7 +5603,7 @@ static void ZSTD_compressContinue_prepare( context->frameChunkState.prepareState = &context->frameChunkPrepareState; context->frameChunkState.compressTarget = ZSTD_rust_frameChunk_compressTarget; context->frameChunkState.compressSplit = ZSTD_rust_frameChunk_compressSplit; - context->frameChunkState.compressInternal = ZSTD_rust_frameChunk_compressInternal; + context->frameChunkState.compressInternal = NULL; ZSTD_initBuildSeqStoreState(cctx, &context->buildSeqStoreState); context->blockInternalState.seqStore = &cctx->seqStore; @@ -5650,6 +5619,7 @@ static void ZSTD_compressContinue_prepare( context->blockInternalState.isFirstBlock = cctx->isFirstBlock; context->blockState.buildSeqStoreState = &context->buildSeqStoreState; context->blockState.blockInternalState = &context->blockInternalState; + context->frameChunkState.compressInternalState = &context->blockState; context->overflowContext.matchState = ms; context->overflowContext.workspace = &cctx->workspace; diff --git a/rust/src/zstd_compress.rs b/rust/src/zstd_compress.rs index c79775bd7..b323d6fbf 100644 --- a/rust/src/zstd_compress.rs +++ b/rust/src/zstd_compress.rs @@ -541,6 +541,24 @@ const _: () = { assert!(size_of::() == 6 * size_of::()); }; +/// Projected inputs shared by the frame-chunk internal block path and the +/// deprecated block adapter. Sequence-store construction and the private +/// compressed-block storage remain behind these two C-owned projections. +#[repr(C)] +pub struct ZSTD_rust_compressContinueBlockState { + build_seq_store_state: *const ZSTD_rust_buildSeqStoreState, + block_internal_state: *const ZSTD_rust_blockInternalState, +} + +const _: () = { + assert!(offset_of!(ZSTD_rust_compressContinueBlockState, build_seq_store_state) == 0); + assert!( + offset_of!(ZSTD_rust_compressContinueBlockState, block_internal_state) + == size_of::() + ); + assert!(size_of::() == 2 * size_of::()); +}; + /// Explicit projection of the state used by `ZSTD_compress_frameChunk`. /// /// The Rust side owns the per-frame block loop and its savings/dispatch @@ -566,7 +584,8 @@ pub struct ZSTD_rust_frameChunkState { prepare_state: *const ZSTD_rust_frameChunkPrepareState, compress_target: FrameChunkCompressFn, compress_split: FrameChunkCompressFn, - compress_internal: FrameChunkCompressFn, + compress_internal: Option, + compress_internal_state: *const ZSTD_rust_compressContinueBlockState, } const _: () = { @@ -607,9 +626,13 @@ const _: () = { == 7 * size_of::() + size_of::() + 6 * size_of::() ); assert!( - size_of::() + offset_of!(ZSTD_rust_frameChunkState, compress_internal_state) == 11 * size_of::() + size_of::() + 6 * size_of::() ); + assert!( + size_of::() + == 12 * size_of::() + size_of::() + 6 * size_of::() + ); }; /// Rust implementation of `ZSTD_compress_frameChunk`. @@ -721,15 +744,32 @@ unsafe fn compress_frame_chunk_body_with( ) } } else { - let compressed_size = unsafe { - (state.compress_internal)( - state.callback_context, - op.add(ZSTD_BLOCK_HEADER_SIZE).cast(), - remaining_capacity - ZSTD_BLOCK_HEADER_SIZE, - ip.cast(), - block_size, - last_block, - ) + let compressed_size = if !state.compress_internal_state.is_null() { + unsafe { + ZSTD_rust_compressFrameChunkInternal( + state.compress_internal_state, + state.is_first_block, + op.add(ZSTD_BLOCK_HEADER_SIZE).cast(), + remaining_capacity - ZSTD_BLOCK_HEADER_SIZE, + ip.cast(), + block_size, + last_block, + ) + } + } else { + let Some(compress_internal) = state.compress_internal else { + return ERROR(ZstdErrorCode::Generic); + }; + unsafe { + compress_internal( + state.callback_context, + op.add(ZSTD_BLOCK_HEADER_SIZE).cast(), + remaining_capacity - ZSTD_BLOCK_HEADER_SIZE, + ip.cast(), + block_size, + last_block, + ) + } }; if ERR_isError(compressed_size) { return compressed_size; @@ -796,24 +836,6 @@ pub unsafe extern "C" fn ZSTD_rust_compressFrameChunk( type CompressContinueBlockFn = unsafe extern "C" fn(*mut c_void, *mut c_void, usize, *const c_void, usize, c_uint) -> usize; -/// Explicit projection for the deprecated block adapter. Rust builds the -/// sequence store and then enters the already-Rust-owned block body; the -/// projected states retain only the matchfinder and CCtx storage callbacks. -#[repr(C)] -struct ZSTD_rust_compressContinueBlockState { - build_seq_store_state: *const ZSTD_rust_buildSeqStoreState, - block_internal_state: *const ZSTD_rust_blockInternalState, -} - -const _: () = { - assert!(offset_of!(ZSTD_rust_compressContinueBlockState, build_seq_store_state) == 0); - assert!( - offset_of!(ZSTD_rust_compressContinueBlockState, block_internal_state) - == size_of::() - ); - assert!(size_of::() == 2 * size_of::()); -}; - /// Pointer projection for one C-owned `ZSTD_window_t` and the match-state /// fields which follow its update. Each field points directly at the C /// storage so later callbacks observe the updated window immediately. @@ -960,6 +982,51 @@ const ZSTD_COMPRESSION_STAGE_ONGOING: c_int = 2; #[cfg(test)] const ZSTD_COMPRESSION_STAGE_ENDING: c_int = 3; +#[inline] +unsafe fn compress_continue_block_body_with( + state: &ZSTD_rust_compressContinueBlockState, + dst: *mut c_void, + dst_capacity: usize, + src: *const c_void, + src_size: usize, + frame: c_uint, + is_first_block: *const c_int, +) -> usize { + if state.build_seq_store_state.is_null() || state.block_internal_state.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + let bss = unsafe { ZSTD_rust_buildSeqStore(state.build_seq_store_state, src, src_size) }; + if ERR_isError(bss) { + return bss; + } + if !is_first_block.is_null() { + let mut block_state = unsafe { ptr::read(state.block_internal_state) }; + block_state.is_first_block = unsafe { *is_first_block }; + return unsafe { + ZSTD_rust_compressBlockInternalAfterBuild( + &block_state, + dst, + dst_capacity, + src, + src_size, + frame, + bss as c_int, + ) + }; + } + unsafe { + ZSTD_rust_compressBlockInternalAfterBuild( + state.block_internal_state, + dst, + dst_capacity, + src, + src_size, + frame, + bss as c_int, + ) + } +} + #[no_mangle] pub unsafe extern "C" fn ZSTD_rust_compressContinueBlock( context: *mut c_void, @@ -973,22 +1040,33 @@ pub unsafe extern "C" fn ZSTD_rust_compressContinueBlock( return ERROR(ZstdErrorCode::Generic); } let state = unsafe { &*context.cast::() }; - if state.build_seq_store_state.is_null() || state.block_internal_state.is_null() { + unsafe { + compress_continue_block_body_with(state, dst, dst_capacity, src, src_size, 0, ptr::null()) + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_compressFrameChunkInternal( + state: *const ZSTD_rust_compressContinueBlockState, + is_first_block: *const c_int, + dst: *mut c_void, + dst_capacity: usize, + src: *const c_void, + src_size: usize, + _last_block: c_uint, +) -> usize { + if state.is_null() { return ERROR(ZstdErrorCode::Generic); } - let bss = unsafe { ZSTD_rust_buildSeqStore(state.build_seq_store_state, src, src_size) }; - if ERR_isError(bss) { - return bss; - } unsafe { - ZSTD_rust_compressBlockInternalAfterBuild( - state.block_internal_state, + compress_continue_block_body_with( + &*state, dst, dst_capacity, src, src_size, - 0, - bss as c_int, + 1, + is_first_block, ) } } @@ -10333,7 +10411,8 @@ mod tests { prepare_state: frame_chunk_prepare_state, compress_target: compress_end_test_frame_target, compress_split: compress_end_test_frame_target, - compress_internal: compress_end_test_frame_target, + compress_internal: Some(compress_end_test_frame_target), + compress_internal_state: ptr::null(), }); let frame_chunk_state = context .frame_chunk_state @@ -11208,7 +11287,8 @@ mod tests { prepare_state, compress_target: frame_chunk_test_target, compress_split: frame_chunk_test_split, - compress_internal: frame_chunk_test_internal, + compress_internal: Some(frame_chunk_test_internal), + compress_internal_state: ptr::null(), } } @@ -11547,7 +11627,8 @@ mod tests { prepare_state: frame_chunk_prepare_state, compress_target: compress_continue_test_frame, compress_split: compress_continue_test_frame, - compress_internal: compress_continue_test_frame, + compress_internal: Some(compress_continue_test_frame), + compress_internal_state: ptr::null(), }); let frame_chunk_state = context .frame_chunk_state @@ -12032,7 +12113,8 @@ mod tests { prepare_state: frame_chunk_prepare_state, compress_target: compress_stream_test_block, compress_split: compress_stream_test_block, - compress_internal: compress_stream_test_block, + compress_internal: Some(compress_stream_test_block), + compress_internal_state: ptr::null(), }); let continue_frame_chunk_state = context .continue_frame_chunk_state @@ -12056,7 +12138,8 @@ mod tests { prepare_state: frame_chunk_prepare_state, compress_target: compress_stream_test_end, compress_split: compress_stream_test_end, - compress_internal: compress_stream_test_end, + compress_internal: Some(compress_stream_test_end), + compress_internal_state: ptr::null(), }); let end_frame_chunk_state = context .end_frame_chunk_state