From 62732ff31df4098184b8ac11a5b2e46e06970c2c Mon Sep 17 00:00:00 2001 From: ddidderr Date: Sat, 18 Jul 2026 20:48:11 +0200 Subject: [PATCH] feat(compress): move split-block emission into Rust Move the post-split partition loop out of zstd_compress.c while keeping the private CCtx, matchfinder sequence-store construction, and split discovery in C. Rust now receives a layout-asserted projection containing sequence-store views, block-state slots, workspace and scalar policy, then mirrors the C loop's dRep/cRep histories, final-literal accounting, repeated single-block serialization, and final dRep publication. Remove the obsolete C sequence-store/count/chunk wrappers and the debug-only C size-estimation path that was coupled to the old loop. Add a focused Rust fixture covering partition payloads and final literals, and document the new ownership boundary for the split search and emission paths. Test Plan: - cargo test --manifest-path rust/Cargo.toml --lib -- --test-threads=1 (448 passed) - cargo clippy --manifest-path rust/Cargo.toml --lib -- -D warnings (passed) - cargo clippy --manifest-path rust/Cargo.toml -- -D warnings (passed) - cargo +nightly fmt --manifest-path rust/Cargo.toml -- --check (passed) - cargo clippy --tests/--benches remain blocked only by the pre-existing manual_repeat_n lint in one_shot_promotes_nonfirst_rle_blocks - make -B -C lib -j2 lib (passed) - make -B -C tests -j2 test-zstd (passed) - make -B -C tests -j2 test-cli-tests (41 passed) - ZSTREAM_TESTTIME=-T2s make -B -C tests -j2 test-zstream (passed) - FUZZERTEST=-T5s make -B -C tests -j2 test-fuzzer (252 passed) - make -B -C tests/fuzz -j2 all and sequence_compression_api (passed) --- lib/compress/zstd_compress.c | 268 ++++------------- rust/README.md | 4 + rust/src/zstd_compress.rs | 406 +++++++++++++++++++++++++- rust/src/zstd_compress_block_split.rs | 8 +- rust/src/zstd_compress_stats.rs | 7 +- 5 files changed, 461 insertions(+), 232 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 209e3d689..1014385a6 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -120,46 +120,50 @@ typedef char ZSTD_rust_target_cblock_state_layout[ && sizeof(ZSTD_rust_targetCBlockSizeState) == (sizeof(void*) == 8 ? 72 : 44)) ? 1 : -1]; -/* The single-block sequence-store body only needs this narrow projection of - * ZSTD_CCtx. Matchfinding, sequence-store construction, and split-block - * control remain in C. */ +/* The post-split partition loop only needs this projection of ZSTD_CCtx. + * Split discovery remains in the surrounding C function; the Rust body owns + * partition accounting, repcode simulation, and per-partition emission. */ typedef struct { const SeqStore_t* seqStore; - U32* dRep; - U32* cRep; + const U32* partitions; + SeqStore_t* nextSeqStore; + SeqStore_t* currSeqStore; ZSTD_compressedBlockState_t** prevCBlock; ZSTD_compressedBlockState_t** nextCBlock; void* tmpWorkspace; size_t tmpWkspSize; SeqCollector* seqCollector; + size_t blockSizeMax; int strategy; int disableLiteralCompression; int bmi2; int isFirstBlock; -} ZSTD_rust_seqStoreSingleBlockState; -size_t ZSTD_rust_compressSeqStoreSingleBlock( - const ZSTD_rust_seqStoreSingleBlockState* state, +} ZSTD_rust_splitBlockState; +size_t ZSTD_rust_compressBlockSplit( + const ZSTD_rust_splitBlockState* state, void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - U32 lastBlock, U32 isPartition); -typedef char ZSTD_rust_seqstore_single_block_state_layout[ - (offsetof(ZSTD_rust_seqStoreSingleBlockState, seqStore) == 0 - && offsetof(ZSTD_rust_seqStoreSingleBlockState, dRep) == sizeof(void*) - && offsetof(ZSTD_rust_seqStoreSingleBlockState, cRep) == 2 * sizeof(void*) - && offsetof(ZSTD_rust_seqStoreSingleBlockState, prevCBlock) == 3 * sizeof(void*) - && offsetof(ZSTD_rust_seqStoreSingleBlockState, nextCBlock) == 4 * sizeof(void*) - && offsetof(ZSTD_rust_seqStoreSingleBlockState, tmpWorkspace) == 5 * sizeof(void*) - && offsetof(ZSTD_rust_seqStoreSingleBlockState, tmpWkspSize) == 6 * sizeof(void*) - && offsetof(ZSTD_rust_seqStoreSingleBlockState, seqCollector) == 7 * sizeof(void*) - && offsetof(ZSTD_rust_seqStoreSingleBlockState, strategy) == 8 * sizeof(void*) - && offsetof(ZSTD_rust_seqStoreSingleBlockState, disableLiteralCompression) - == 8 * sizeof(void*) + sizeof(int) - && offsetof(ZSTD_rust_seqStoreSingleBlockState, bmi2) - == 8 * sizeof(void*) + 2 * sizeof(int) - && offsetof(ZSTD_rust_seqStoreSingleBlockState, isFirstBlock) - == 8 * sizeof(void*) + 3 * sizeof(int) - && sizeof(ZSTD_rust_seqStoreSingleBlockState) - == 8 * sizeof(void*) + 4 * sizeof(int)) + const void* src, size_t blockSize, + U32 lastBlock, size_t numSplits); +typedef char ZSTD_rust_split_block_state_layout[ + (offsetof(ZSTD_rust_splitBlockState, seqStore) == 0 + && offsetof(ZSTD_rust_splitBlockState, partitions) == sizeof(void*) + && offsetof(ZSTD_rust_splitBlockState, nextSeqStore) == 2 * sizeof(void*) + && offsetof(ZSTD_rust_splitBlockState, currSeqStore) == 3 * sizeof(void*) + && offsetof(ZSTD_rust_splitBlockState, prevCBlock) == 4 * sizeof(void*) + && offsetof(ZSTD_rust_splitBlockState, nextCBlock) == 5 * sizeof(void*) + && offsetof(ZSTD_rust_splitBlockState, tmpWorkspace) == 6 * sizeof(void*) + && offsetof(ZSTD_rust_splitBlockState, tmpWkspSize) == 7 * sizeof(void*) + && offsetof(ZSTD_rust_splitBlockState, seqCollector) == 8 * sizeof(void*) + && offsetof(ZSTD_rust_splitBlockState, blockSizeMax) == 9 * sizeof(void*) + && offsetof(ZSTD_rust_splitBlockState, strategy) == 10 * sizeof(void*) + && offsetof(ZSTD_rust_splitBlockState, disableLiteralCompression) + == 10 * sizeof(void*) + sizeof(int) + && offsetof(ZSTD_rust_splitBlockState, bmi2) + == 10 * sizeof(void*) + 2 * sizeof(int) + && offsetof(ZSTD_rust_splitBlockState, isFirstBlock) + == 10 * sizeof(void*) + 3 * sizeof(int) + && sizeof(ZSTD_rust_splitBlockState) + == 10 * sizeof(void*) + 4 * sizeof(int)) ? 1 : -1]; size_t ZSTD_rust_nextInputSizeHint(int inBufferMode, size_t blockSizeMax, @@ -417,14 +421,6 @@ size_t ZSTD_rust_buildBlockEntropyStats( int strategy, int disableLiteralCompression, ZSTD_entropyCTablesMetadata_t* entropyMetadata, void* workspace, size_t wkspSize); -size_t ZSTD_rust_estimateBlockSize( - const BYTE* literals, size_t litSize, - const BYTE* ofCodeTable, const BYTE* llCodeTable, - const BYTE* mlCodeTable, size_t nbSeq, - const ZSTD_entropyCTables_t* entropy, - const ZSTD_entropyCTablesMetadata_t* entropyMetadata, - void* workspace, size_t wkspSize, - int writeLitEntropy, int writeSeqEntropy); size_t ZSTD_rust_copyBlockSequences( SeqCollector* seqCollector, const SeqStore_t* seqStore, const U32 prevRepcodes[ZSTD_REP_NUM]); @@ -448,11 +444,6 @@ int ZSTD_rust_isRLE(const BYTE* src, size_t length); size_t ZSTD_rust_postProcessSequenceProducerResult( ZSTD_Sequence* outSeqs, size_t nbExternalSeqs, size_t outSeqsCapacity, size_t srcSize); -size_t ZSTD_rust_countSeqStoreLiteralsBytes(const SeqStore_t* seqStore); -size_t ZSTD_rust_countSeqStoreMatchBytes(const SeqStore_t* seqStore); -void ZSTD_rust_deriveSeqStoreChunk(SeqStore_t* resultSeqStore, - const SeqStore_t* originalSeqStore, - size_t startIdx, size_t endIdx); size_t ZSTD_rust_deriveBlockSplits( U32* partitions, U32 nbSeq, const SeqStore_t* originalSeqStore, @@ -2733,105 +2724,6 @@ size_t ZSTD_buildBlockEntropyStats( workspace, wkspSize); } -/* The block-size estimator is implemented in Rust; C retains the stateful - * block-splitting recursion and the entropy-table ownership. */ -static size_t -ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize, - const BYTE* ofCodeTable, - const BYTE* llCodeTable, - const BYTE* mlCodeTable, - size_t nbSeq, - const ZSTD_entropyCTables_t* entropy, - const ZSTD_entropyCTablesMetadata_t* entropyMetadata, - void* workspace, size_t wkspSize, - int writeLitEntropy, int writeSeqEntropy) -{ - return ZSTD_rust_estimateBlockSize( - literals, litSize, - ofCodeTable, llCodeTable, mlCodeTable, nbSeq, - entropy, entropyMetadata, - workspace, wkspSize, - writeLitEntropy, writeSeqEntropy); -} - -/* Builds entropy statistics and uses them for blocksize estimation. - * - * @return: estimated compressed size of the seqStore, or a zstd error. - */ -UNUSED_ATTR static size_t -ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(SeqStore_t* seqStore, ZSTD_CCtx* zc) -{ - ZSTD_entropyCTablesMetadata_t* const entropyMetadata = &zc->blockSplitCtx.entropyMetadata; - DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()"); - FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore, - &zc->blockState.prevCBlock->entropy, - &zc->blockState.nextCBlock->entropy, - &zc->appliedParams, - entropyMetadata, - zc->tmpWorkspace, zc->tmpWkspSize), ""); - return ZSTD_estimateBlockSize( - seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart), - seqStore->ofCode, seqStore->llCode, seqStore->mlCode, - (size_t)(seqStore->sequences - seqStore->sequencesStart), - &zc->blockState.nextCBlock->entropy, - entropyMetadata, - zc->tmpWorkspace, zc->tmpWkspSize, - (int)(entropyMetadata->hufMetadata.hType == set_compressed), 1); -} - -/* Returns literals bytes represented in a seqStore */ -static size_t ZSTD_countSeqStoreLiteralsBytes(const SeqStore_t* const seqStore) -{ - return ZSTD_rust_countSeqStoreLiteralsBytes(seqStore); -} - -/* Returns match bytes represented in a seqStore */ -static size_t ZSTD_countSeqStoreMatchBytes(const SeqStore_t* const seqStore) -{ - return ZSTD_rust_countSeqStoreMatchBytes(seqStore); -} - -/* Derives the seqStore that is a chunk of the originalSeqStore from [startIdx, endIdx). - * Stores the result in resultSeqStore. - */ -static void ZSTD_deriveSeqStoreChunk(SeqStore_t* resultSeqStore, - const SeqStore_t* originalSeqStore, - size_t startIdx, size_t endIdx) -{ - ZSTD_rust_deriveSeqStoreChunk(resultSeqStore, originalSeqStore, - startIdx, endIdx); -} - -/* ZSTD_compressSeqStore_singleBlock(): - * Compresses a seqStore into a block with a block header, into the buffer dst. - * - * Returns the total size of that block (including header) or a ZSTD error code. - */ -static size_t -ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, - const SeqStore_t* const seqStore, - Repcodes_t* const dRep, Repcodes_t* const cRep, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - U32 lastBlock, U32 isPartition) -{ - ZSTD_rust_seqStoreSingleBlockState state; - state.seqStore = seqStore; - state.dRep = dRep->rep; - state.cRep = cRep->rep; - state.prevCBlock = &zc->blockState.prevCBlock; - state.nextCBlock = &zc->blockState.nextCBlock; - state.tmpWorkspace = zc->tmpWorkspace; - state.tmpWkspSize = zc->tmpWkspSize; - state.seqCollector = &zc->seqCollector; - state.strategy = (int)zc->appliedParams.cParams.strategy; - state.disableLiteralCompression = ZSTD_literalsCompressionIsDisabled(&zc->appliedParams); - state.bmi2 = zc->bmi2; - state.isFirstBlock = zc->isFirstBlock; - return ZSTD_rust_compressSeqStoreSingleBlock( - &state, dst, dstCapacity, src, srcSize, lastBlock, isPartition); -} - /* Base recursive function. * Populates a table with intra-block partition indices that can improve compression ratio. * @@ -2865,91 +2757,29 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, const void* src, size_t blockSize, U32 lastBlock, U32 nbSeq) { - size_t cSize = 0; - const BYTE* ip = (const BYTE*)src; - BYTE* op = (BYTE*)dst; - size_t i = 0; - size_t srcBytesTotal = 0; + ZSTD_rust_splitBlockState state; U32* const partitions = zc->blockSplitCtx.partitions; /* splits plus the terminal boundary */ - SeqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore; - SeqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore; size_t const numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq); - /* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history - * may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two - * separate repcode histories that simulate repcode history on compression and decompression side, - * and use the histories to determine whether we must replace a particular repcode with its raw offset. - * - * 1) cRep gets updated for each partition, regardless of whether the block was emitted as uncompressed - * or RLE. This allows us to retrieve the offset value that an invalid repcode references within - * a nocompress/RLE block. - * 2) dRep gets updated only for compressed partitions, and when a repcode gets replaced, will use - * the replacement offset value rather than the original repcode to update the repcode history. - * dRep also will be the final repcode history sent to the next block. - * - * See ZSTD_seqStore_resolveOffCodes() for more details. - */ - Repcodes_t dRep; - Repcodes_t cRep; - ZSTD_memcpy(dRep.rep, zc->blockState.prevCBlock->rep, sizeof(Repcodes_t)); - ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(Repcodes_t)); - ZSTD_memset(nextSeqStore, 0, sizeof(SeqStore_t)); - DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate); - - if (numSplits == 0) { - size_t cSizeSingleBlock = - ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore, - &dRep, &cRep, - op, dstCapacity, - ip, blockSize, - lastBlock, 0 /* isPartition */); - FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!"); - DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits"); - assert(zc->blockSizeMax <= ZSTD_BLOCKSIZE_MAX); - assert(cSizeSingleBlock <= zc->blockSizeMax + ZSTD_blockHeaderSize); - return cSizeSingleBlock; - } - - ZSTD_deriveSeqStoreChunk(currSeqStore, &zc->seqStore, 0, partitions[0]); - for (i = 0; i <= numSplits; ++i) { - size_t cSizeChunk; - U32 const lastPartition = (i == numSplits); - U32 lastBlockEntireSrc = 0; - - size_t srcBytes = ZSTD_countSeqStoreLiteralsBytes(currSeqStore) + ZSTD_countSeqStoreMatchBytes(currSeqStore); - srcBytesTotal += srcBytes; - if (lastPartition) { - /* This is the final partition, need to account for possible last literals */ - srcBytes += blockSize - srcBytesTotal; - lastBlockEntireSrc = lastBlock; - } else { - ZSTD_deriveSeqStoreChunk(nextSeqStore, &zc->seqStore, partitions[i], partitions[i+1]); - } - - cSizeChunk = ZSTD_compressSeqStore_singleBlock(zc, currSeqStore, - &dRep, &cRep, - op, dstCapacity, - ip, srcBytes, - lastBlockEntireSrc, 1 /* isPartition */); - DEBUGLOG(5, "Estimated size: %zu vs %zu : actual size", - ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk); - FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!"); - - ip += srcBytes; - op += cSizeChunk; - dstCapacity -= cSizeChunk; - cSize += cSizeChunk; - *currSeqStore = *nextSeqStore; - assert(cSizeChunk <= zc->blockSizeMax + ZSTD_blockHeaderSize); - } - /* cRep and dRep may have diverged during the compression. - * If so, we use the dRep repcodes for the next block. - */ - ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(Repcodes_t)); - return cSize; + state.seqStore = &zc->seqStore; + state.partitions = partitions; + state.nextSeqStore = &zc->blockSplitCtx.nextSeqStore; + state.currSeqStore = &zc->blockSplitCtx.currSeqStore; + state.prevCBlock = &zc->blockState.prevCBlock; + state.nextCBlock = &zc->blockState.nextCBlock; + state.tmpWorkspace = zc->tmpWorkspace; + state.tmpWkspSize = zc->tmpWkspSize; + state.seqCollector = &zc->seqCollector; + state.blockSizeMax = zc->blockSizeMax; + state.strategy = (int)zc->appliedParams.cParams.strategy; + state.disableLiteralCompression = ZSTD_literalsCompressionIsDisabled(&zc->appliedParams); + state.bmi2 = zc->bmi2; + state.isFirstBlock = zc->isFirstBlock; + return ZSTD_rust_compressBlockSplit( + &state, dst, dstCapacity, src, blockSize, lastBlock, numSplits); } static size_t diff --git a/rust/README.md b/rust/README.md index f712a492c..ab3e1a687 100644 --- a/rust/README.md +++ b/rust/README.md @@ -38,6 +38,10 @@ zstd ABI: splitter, and exports collected sequences in the public `ZSTD_Sequence` format. Its C shims extract the sequence store, the entropy-table leaves, and the two `ZSTD_CCtx_params` scalars these paths read. + - `zstd_compress_block_split` searches for profitable sequence-store + partitions, while `zstd_compress` emits those partitions through the + Rust single-block serializer. C retains split discovery's context setup + and the outer block-dispatch decision. - `zstd_compress_frame` serializes frame headers, skippable frames, and the last empty block; it takes scalar frame parameters so the C-owned `ZSTD_CCtx_params` layout never crosses the language boundary. diff --git a/rust/src/zstd_compress.rs b/rust/src/zstd_compress.rs index 09ccd2a33..271602f36 100644 --- a/rust/src/zstd_compress.rs +++ b/rust/src/zstd_compress.rs @@ -29,9 +29,11 @@ use crate::zstd_compress_sequences::SeqDef; use crate::zstd_compress_stats::{ SeqCollector, SeqStore_t, ZSTD_Sequence, ZSTD_SequencePosition, ZSTD_compressedBlockState_t, ZSTD_entropyCTables_t, ZSTD_rust_confirmRepcodesAndEntropyTables, ZSTD_rust_copyBlockSequences, - ZSTD_rust_determineBlockSize, ZSTD_rust_entropyCompressSeqStore, ZSTD_rust_isRLE, - ZSTD_rust_maybeRLE, ZSTD_rust_resetSeqStore, ZSTD_rust_seqStore_resolveOffCodes, - ZSTD_rust_transferSequencesNoDelim, ZSTD_rust_transferSequencesWBlockDelim, + ZSTD_rust_countSeqStoreLiteralsBytes, ZSTD_rust_countSeqStoreMatchBytes, + ZSTD_rust_deriveSeqStoreChunk, ZSTD_rust_determineBlockSize, ZSTD_rust_entropyCompressSeqStore, + ZSTD_rust_isRLE, ZSTD_rust_maybeRLE, ZSTD_rust_resetSeqStore, + ZSTD_rust_seqStore_resolveOffCodes, ZSTD_rust_transferSequencesNoDelim, + ZSTD_rust_transferSequencesWBlockDelim, }; use crate::zstd_compress_superblock::ZSTD_rust_compressSuperBlock; use std::ffi::c_void; @@ -166,10 +168,9 @@ const _: () = { /// Explicit projection of the state used by `ZSTD_compressSeqStore_singleBlock`. /// -/// Sequence-store construction and the surrounding split-block bookkeeping -/// remain in C. Only the sequence store, simulated repcode histories, -/// compressed-block state slots, sequence collector, and scalar compression -/// settings cross the ABI. +/// Sequence-store construction and split discovery remain in C. Only the +/// sequence store, simulated repcode histories, compressed-block state slots, +/// sequence collector, and scalar compression settings cross the ABI. #[repr(C)] pub struct ZSTD_rust_seqStoreSingleBlockState { seq_store: *const SeqStore_t, @@ -222,6 +223,58 @@ const _: () = { ); }; +/// Explicit projection of the state used by the post-split partition loop. +/// +/// The C caller still owns the private compression context and derives the +/// partition boundaries. Rust owns the partition accounting, repcode +/// simulation, and repeated single-block dispatch. +#[repr(C)] +pub struct ZSTD_rust_splitBlockState { + seq_store: *const SeqStore_t, + partitions: *const u32, + next_seq_store: *mut SeqStore_t, + curr_seq_store: *mut SeqStore_t, + prev_c_block: *mut *mut ZSTD_compressedBlockState_t, + next_c_block: *mut *mut ZSTD_compressedBlockState_t, + tmp_workspace: *mut c_void, + tmp_wksp_size: usize, + seq_collector: *mut SeqCollector, + block_size_max: usize, + strategy: c_int, + disable_literal_compression: c_int, + bmi2: c_int, + is_first_block: c_int, +} + +const _: () = { + assert!(offset_of!(ZSTD_rust_splitBlockState, seq_store) == 0); + assert!(offset_of!(ZSTD_rust_splitBlockState, partitions) == size_of::()); + assert!(offset_of!(ZSTD_rust_splitBlockState, next_seq_store) == 2 * size_of::()); + assert!(offset_of!(ZSTD_rust_splitBlockState, curr_seq_store) == 3 * size_of::()); + assert!(offset_of!(ZSTD_rust_splitBlockState, prev_c_block) == 4 * size_of::()); + assert!(offset_of!(ZSTD_rust_splitBlockState, next_c_block) == 5 * size_of::()); + assert!(offset_of!(ZSTD_rust_splitBlockState, tmp_workspace) == 6 * size_of::()); + assert!(offset_of!(ZSTD_rust_splitBlockState, tmp_wksp_size) == 7 * size_of::()); + assert!(offset_of!(ZSTD_rust_splitBlockState, seq_collector) == usize::BITS as usize); + assert!(offset_of!(ZSTD_rust_splitBlockState, block_size_max) == 9 * size_of::()); + assert!(offset_of!(ZSTD_rust_splitBlockState, strategy) == 10 * size_of::()); + assert!( + offset_of!(ZSTD_rust_splitBlockState, disable_literal_compression) + == 10 * size_of::() + size_of::() + ); + assert!( + offset_of!(ZSTD_rust_splitBlockState, bmi2) + == 10 * size_of::() + 2 * size_of::() + ); + assert!( + offset_of!(ZSTD_rust_splitBlockState, is_first_block) + == 10 * size_of::() + 3 * size_of::() + ); + assert!( + size_of::() == 10 * size_of::() + 4 * size_of::() + ); +}; + /// Explicit projection of the state used by the target-sized block body. /// /// Sequence-store construction and the matchfinder remain in C. This state @@ -698,6 +751,213 @@ pub unsafe extern "C" fn ZSTD_rust_compressSeqStoreSingleBlock( } } +#[inline] +fn split_single_block_state( + split_state: &ZSTD_rust_splitBlockState, + seq_store: *const SeqStore_t, + d_rep: &mut [u32; ZSTD_REP_NUM], + c_rep: &mut [u32; ZSTD_REP_NUM], +) -> ZSTD_rust_seqStoreSingleBlockState { + ZSTD_rust_seqStoreSingleBlockState { + seq_store, + d_rep: d_rep.as_mut_ptr(), + c_rep: c_rep.as_mut_ptr(), + prev_c_block: split_state.prev_c_block, + next_c_block: split_state.next_c_block, + tmp_workspace: split_state.tmp_workspace, + tmp_wksp_size: split_state.tmp_wksp_size, + seq_collector: split_state.seq_collector, + strategy: split_state.strategy, + disable_literal_compression: split_state.disable_literal_compression, + bmi2: split_state.bmi2, + is_first_block: split_state.is_first_block, + } +} + +/// Rust implementation of the post-split partition loop. +/// +/// C retains `ZSTD_deriveBlockSplits()` and the private `ZSTD_CCtx`; Rust +/// receives only the sequence-store views and block-emission state needed to +/// reproduce the original loop. +#[allow(clippy::too_many_arguments)] +unsafe fn compress_block_split_body_with( + state: &ZSTD_rust_splitBlockState, + dst: *mut c_void, + dst_capacity: usize, + src: *const c_void, + block_size: usize, + last_block: c_uint, + num_splits: usize, + seams: SingleBlockSeams, +) -> usize { + if state.seq_store.is_null() + || state.partitions.is_null() + || state.next_seq_store.is_null() + || state.curr_seq_store.is_null() + || state.prev_c_block.is_null() + || state.next_c_block.is_null() + || state.seq_collector.is_null() + { + return ERROR(ZstdErrorCode::Generic); + } + + let prev_c_block = unsafe { *state.prev_c_block }; + let next_c_block = unsafe { *state.next_c_block }; + if prev_c_block.is_null() || next_c_block.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + + /* cRep and dRep start from the preceding block and diverge only when a + * partition is emitted as raw or RLE. */ + let mut d_rep = [0u32; ZSTD_REP_NUM]; + unsafe { + ptr::copy_nonoverlapping( + (*prev_c_block).rep.as_ptr(), + d_rep.as_mut_ptr(), + ZSTD_REP_NUM, + ); + } + let mut c_rep = d_rep; + unsafe { + ptr::write_bytes( + state.next_seq_store.cast::(), + 0, + size_of::(), + ); + } + + if num_splits == 0 { + let single_state = split_single_block_state(state, state.seq_store, &mut d_rep, &mut c_rep); + let c_size = unsafe { + compress_seq_store_single_block_body_with( + &single_state, + dst, + dst_capacity, + src, + block_size, + last_block, + 0, + seams, + ) + }; + debug_assert!(state.block_size_max <= ZSTD_BLOCKSIZE_MAX); + debug_assert!( + c_size <= state.block_size_max.wrapping_add(ZSTD_BLOCK_HEADER_SIZE) + || ERR_isError(c_size) + ); + return c_size; + } + + unsafe { + ZSTD_rust_deriveSeqStoreChunk( + state.curr_seq_store, + state.seq_store, + 0, + *state.partitions as usize, + ); + } + + let mut c_size = 0usize; + let mut src_bytes_total = 0usize; + let mut ip = src.cast::(); + let mut op = dst.cast::(); + let mut remaining_capacity = dst_capacity; + + for i in 0..=num_splits { + let last_partition = i == num_splits; + let mut last_block_entire_src = 0; + let partition_seq_store = state.curr_seq_store as *const SeqStore_t; + let partition_bytes = unsafe { + ZSTD_rust_countSeqStoreLiteralsBytes(partition_seq_store) + .wrapping_add(ZSTD_rust_countSeqStoreMatchBytes(partition_seq_store)) + }; + src_bytes_total = src_bytes_total.wrapping_add(partition_bytes); + let src_bytes = if last_partition { + last_block_entire_src = last_block; + partition_bytes.wrapping_add(block_size.wrapping_sub(src_bytes_total)) + } else { + unsafe { + ZSTD_rust_deriveSeqStoreChunk( + state.next_seq_store, + state.seq_store, + *state.partitions.add(i) as usize, + *state.partitions.add(i + 1) as usize, + ); + } + partition_bytes + }; + + let single_state = + split_single_block_state(state, partition_seq_store, &mut d_rep, &mut c_rep); + let c_size_chunk = unsafe { + compress_seq_store_single_block_body_with( + &single_state, + op.cast(), + remaining_capacity, + ip.cast(), + src_bytes, + last_block_entire_src, + 1, + seams, + ) + }; + if ERR_isError(c_size_chunk) { + return c_size_chunk; + } + + unsafe { + ip = ip.add(src_bytes); + op = op.add(c_size_chunk); + } + remaining_capacity = remaining_capacity.wrapping_sub(c_size_chunk); + c_size = c_size.wrapping_add(c_size_chunk); + unsafe { + ptr::copy_nonoverlapping(state.next_seq_store, state.curr_seq_store, 1); + } + debug_assert!(c_size_chunk <= state.block_size_max.wrapping_add(ZSTD_BLOCK_HEADER_SIZE)); + } + + let final_prev_c_block = unsafe { *state.prev_c_block }; + if final_prev_c_block.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + unsafe { + ptr::copy_nonoverlapping( + d_rep.as_ptr(), + (*final_prev_c_block).rep.as_mut_ptr(), + ZSTD_REP_NUM, + ); + } + c_size +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_compressBlockSplit( + state: *const ZSTD_rust_splitBlockState, + dst: *mut c_void, + dst_capacity: usize, + src: *const c_void, + block_size: usize, + last_block: c_uint, + num_splits: usize, +) -> usize { + if state.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + unsafe { + compress_block_split_body_with( + &*state, + dst, + dst_capacity, + src, + block_size, + last_block, + num_splits, + SingleBlockSeams::production(), + ) + } +} + /// Select the strategy used by the simple compression entry points. /// /// This is the Rust equivalent of the strategy portion of @@ -3397,6 +3657,138 @@ mod tests { } } + struct SplitBlockTestStore { + seq_store: SeqStore_t, + _sequences: Box<[SeqDef; 2]>, + _ll_code: Box<[u8; 2]>, + _ml_code: Box<[u8; 2]>, + _of_code: Box<[u8; 2]>, + _literals: Box<[u8; 6]>, + } + + fn split_block_test_seq_store() -> SplitBlockTestStore { + let mut sequences = Box::new([ + SeqDef { + offBase: 4, + litLength: 2, + mlBase: 0, + }, + SeqDef { + offBase: 5, + litLength: 2, + mlBase: 0, + }, + ]); + let mut ll_code = Box::new([1u8; 2]); + let mut ml_code = Box::new([1u8; 2]); + let mut of_code = Box::new([1u8; 2]); + let mut literals = Box::new([0u8; 6]); + let sequences_start = sequences.as_mut_ptr(); + let literals_start = literals.as_mut_ptr(); + let seq_store = SeqStore_t { + sequencesStart: sequences_start, + sequences: unsafe { sequences_start.add(sequences.len()) }, + litStart: literals_start, + lit: unsafe { literals_start.add(literals.len()) }, + llCode: ll_code.as_mut_ptr(), + mlCode: ml_code.as_mut_ptr(), + ofCode: of_code.as_mut_ptr(), + maxNbSeq: sequences.len(), + maxNbLit: literals.len(), + longLengthType: 0, + longLengthPos: 0, + }; + SplitBlockTestStore { + seq_store, + _sequences: sequences, + _ll_code: ll_code, + _ml_code: ml_code, + _of_code: of_code, + _literals: literals, + } + } + + #[allow(clippy::too_many_arguments)] + fn split_block_test_state( + seq_store: &SeqStore_t, + partitions: &[u32], + next_seq_store: &mut SeqStore_t, + curr_seq_store: &mut SeqStore_t, + prev_block: &mut ZSTD_compressedBlockState_t, + next_block: &mut ZSTD_compressedBlockState_t, + prev_c_block: &mut *mut ZSTD_compressedBlockState_t, + next_c_block: &mut *mut ZSTD_compressedBlockState_t, + seq_collector: &mut SeqCollector, + ) -> ZSTD_rust_splitBlockState { + *prev_c_block = prev_block; + *next_c_block = next_block; + ZSTD_rust_splitBlockState { + seq_store, + partitions: partitions.as_ptr(), + next_seq_store, + curr_seq_store, + prev_c_block, + next_c_block, + tmp_workspace: ptr::null_mut(), + tmp_wksp_size: 0, + seq_collector, + block_size_max: ZSTD_BLOCKSIZE_MAX, + strategy: 0, + disable_literal_compression: 0, + bmi2: 0, + is_first_block: 1, + } + } + + #[test] + fn split_block_body_emits_partitions_and_final_literals() { + let fixture = split_block_test_seq_store(); + let partitions = [1, 2]; + let mut next_seq_store = unsafe { MaybeUninit::::zeroed().assume_init() }; + let mut curr_seq_store = unsafe { MaybeUninit::::zeroed().assume_init() }; + let mut prev_block = zeroed_state(); + let mut next_block = zeroed_state(); + let mut prev_c_block = ptr::null_mut(); + let mut next_c_block = ptr::null_mut(); + let mut seq_collector = SeqCollector { + collectSequences: 0, + seqStart: ptr::null_mut(), + seqIndex: 0, + maxSequences: 0, + }; + let state = split_block_test_state( + &fixture.seq_store, + &partitions, + &mut next_seq_store, + &mut curr_seq_store, + &mut prev_block, + &mut next_block, + &mut prev_c_block, + &mut next_c_block, + &mut seq_collector, + ); + let source: Vec = (0..12).collect(); + let mut output = [0xa5u8; 32]; + + let result = unsafe { + compress_block_split_body_with( + &state, + output.as_mut_ptr().cast(), + output.len(), + source.as_ptr().cast(), + source.len(), + 1, + 1, + single_block_test_seams(), + ) + }; + + assert_eq!(result, 18); + assert_eq!(&output[ZSTD_BLOCK_HEADER_SIZE..8], &source[..5]); + assert_eq!(&output[8 + ZSTD_BLOCK_HEADER_SIZE..result], &source[5..]); + assert!(std::ptr::eq(prev_c_block, &prev_block)); + } + #[test] fn single_block_body_propagates_entropy_errors_without_serializing() { let (seq_store, _sequences, _literals) = target_block_test_seq_store(); diff --git a/rust/src/zstd_compress_block_split.rs b/rust/src/zstd_compress_block_split.rs index e360c7a80..d702bee27 100644 --- a/rust/src/zstd_compress_block_split.rs +++ b/rust/src/zstd_compress_block_split.rs @@ -1,9 +1,11 @@ //! Post-block-split partition search. //! //! The C compressor owns `ZSTD_CCtx`, block-state ownership, entropy-table -//! selection, and block emission. This module owns only the recursive search -//! over shallow `SeqStore_t` views. The estimator is supplied with projected -//! entropy state and parameter scalars rather than the private C context. +//! selection, outer block dispatch, and split discovery's context setup. This +//! module owns the recursive search over shallow `SeqStore_t` views. The +//! estimator is supplied with projected entropy state and parameter scalars +//! rather than the private C context; post-split partition emission lives in +//! `zstd_compress.rs`. use crate::errors::ERR_isError; use crate::zstd_compress_stats::{ diff --git a/rust/src/zstd_compress_stats.rs b/rust/src/zstd_compress_stats.rs index 58460870e..f56603a71 100644 --- a/rust/src/zstd_compress_stats.rs +++ b/rust/src/zstd_compress_stats.rs @@ -9,9 +9,10 @@ //! `ZSTD_copyBlockSequences()`. The `ZSTD_CCtx` and `ZSTD_CCtx_params` //! layouts stay private to C: the C shims extract the sequence store, the //! entropy-table leaves, and the two parameter scalars these paths read -//! (the compression strategy and the literals-compression switch). Block -//! dispatch and block-splitting recursion remain in C; the pure block-size -//! estimator is owned here. +//! (the compression strategy and the literals-compression switch). C retains +//! outer block dispatch and split discovery; Rust owns the pure block-size +//! estimator and the post-split partition emission loop in +//! `zstd_compress.rs`. use crate::bits::ZSTD_highbit32; use crate::common::{