diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 1c576b26f..bb37402fc 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -400,12 +400,6 @@ void ZSTD_rust_seqStore_resolveOffCodes(U32 dRep[ZSTD_REP_NUM], U32 ZSTD_rust_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], U32 offBase, U32 ll0); U32 ZSTD_rust_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0); -size_t ZSTD_rust_blockSizeExplicitDelimiter(const ZSTD_Sequence* inSeqs, - size_t inSeqsSize, U32 seqIdx); -size_t ZSTD_rust_determineBlockSize(int mode, size_t blockSize, size_t remaining, - const ZSTD_Sequence* inSeqs, size_t inSeqsSize, - U32 seqIdx); -int ZSTD_rust_selectSequenceCopier(int mode); size_t ZSTD_rust_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, const void* dict, size_t dictSize); size_t ZSTD_rust_transferSequencesWBlockDelim( @@ -416,19 +410,58 @@ size_t ZSTD_rust_transferSequencesWBlockDelim( U32 nextRepcodes[ZSTD_REP_NUM], U32 dictSize, int validateSequences, U32 minMatch, U32 windowLog, int useSequenceProducer); -size_t ZSTD_rust_transferSequencesNoDelim( - SeqStore_t* seqStore, ZSTD_SequencePosition* seqPos, - const ZSTD_Sequence* inSeqs, size_t inSeqsSize, - const BYTE* src, size_t blockSize, - const U32 prevRepcodes[ZSTD_REP_NUM], - U32 nextRepcodes[ZSTD_REP_NUM], U32 dictSize, - int validateSequences, U32 minMatch, U32 windowLog, - int useSequenceProducer); size_t ZSTD_rust_optimalBlockSize(const void* src, size_t srcSize, size_t blockSizeMax, int splitLevel, int strategy, S64 savings, void* workspace, size_t workspaceSize); +/* The sequence-compression loop receives only the state it actually reads or + * updates. In particular, neither ZSTD_CCtx nor a C function pointer crosses + * the Rust ABI. */ +typedef struct { + SeqStore_t* seqStore; + ZSTD_compressedBlockState_t** prevCBlock; + ZSTD_compressedBlockState_t** nextCBlock; + void* tmpWorkspace; + size_t tmpWkspSize; + size_t blockSizeMax; + int bmi2; + int blockDelimiters; + int strategy; + int disableLiteralCompression; + int searchForExternalRepcodes; + int validateSequences; + U32 minMatch; + U32 windowLog; + U32 dictSize; + int useSequenceProducer; + int* isFirstBlock; +} ZSTD_rust_sequenceCompressionState; + +size_t ZSTD_rust_compressSequencesInternal( + const ZSTD_rust_sequenceCompressionState* state, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize); + +typedef char ZSTD_rust_sequence_state_layout[ + (offsetof(ZSTD_rust_sequenceCompressionState, seqStore) == 0 + && offsetof(ZSTD_rust_sequenceCompressionState, prevCBlock) == sizeof(void*) + && offsetof(ZSTD_rust_sequenceCompressionState, nextCBlock) == 2 * sizeof(void*) + && offsetof(ZSTD_rust_sequenceCompressionState, tmpWorkspace) == 3 * sizeof(void*) + && offsetof(ZSTD_rust_sequenceCompressionState, tmpWkspSize) == 4 * sizeof(void*) + && offsetof(ZSTD_rust_sequenceCompressionState, blockSizeMax) == 5 * sizeof(void*) + && offsetof(ZSTD_rust_sequenceCompressionState, bmi2) == 6 * sizeof(void*) + && offsetof(ZSTD_rust_sequenceCompressionState, minMatch) + == 6 * sizeof(void*) + 6 * sizeof(int) + && offsetof(ZSTD_rust_sequenceCompressionState, useSequenceProducer) + == 6 * sizeof(void*) + 3 * sizeof(U32) + 6 * sizeof(int) + && offsetof(ZSTD_rust_sequenceCompressionState, isFirstBlock) + == 6 * sizeof(void*) + 3 * sizeof(U32) + 7 * sizeof(int) + && sizeof(ZSTD_rust_sequenceCompressionState) + == offsetof(ZSTD_rust_sequenceCompressionState, isFirstBlock) + sizeof(void*)) + ? 1 : -1]; + typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1]; typedef char ZSTD_rust_stats_block_summary_layout[ (sizeof(BlockSummary) == 3 * sizeof(size_t)) ? 1 : -1]; @@ -5110,12 +5143,9 @@ static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 return ZSTD_rust_finalizeOffBase(rawOffset, rep, ll0); } -/* This function scans through an array of ZSTD_Sequence, - * storing the sequences it reads, until it reaches a block delimiter. - * Note that the block delimiter includes the last literals of the block. - * @blockSize must be == sum(sequence_lengths). - * @returns @blockSize on success, and a ZSTD_error otherwise. - */ +/* The explicit-delimiter adapter is also used by the external sequence + * producer path. Keep that C-context-facing call site separate from the + * Rust-owned compressSequences block loop. */ static size_t ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx, ZSTD_SequencePosition* seqPos, @@ -5143,78 +5173,6 @@ ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx, ZSTD_hasExtSeqProd(&cctx->appliedParams)); } -/* - * This function attempts to scan through @blockSize bytes in @src - * represented by the sequences in @inSeqs, - * storing any (partial) sequences. - * - * Occasionally, we may want to reduce the actual number of bytes consumed from @src - * to avoid splitting a match, notably if it would produce a match smaller than MINMATCH. - * - * @returns the number of bytes consumed from @src, necessarily <= @blockSize. - * Otherwise, it may return a ZSTD error if something went wrong. - */ -static size_t -ZSTD_transferSequences_noDelim(ZSTD_CCtx* cctx, - ZSTD_SequencePosition* seqPos, - const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, - const void* src, size_t blockSize, - ZSTD_ParamSwitch_e externalRepSearch) -{ - size_t dictSize; - - if (cctx->cdict) { - dictSize = cctx->cdict->dictContentSize; - } else if (cctx->prefixDict.dict) { - dictSize = cctx->prefixDict.dictSize; - } else { - dictSize = 0; - } - (void)externalRepSearch; - return ZSTD_rust_transferSequencesNoDelim( - &cctx->seqStore, seqPos, inSeqs, inSeqsSize, - (const BYTE*)src, blockSize, - cctx->blockState.prevCBlock->rep, - cctx->blockState.nextCBlock->rep, (U32)dictSize, - cctx->appliedParams.validateSequences, - cctx->appliedParams.cParams.minMatch, - cctx->appliedParams.cParams.windowLog, - ZSTD_hasExtSeqProd(&cctx->appliedParams)); -} - -/* @seqPos represents a position within @inSeqs, - * it is read and updated by this function, - * once the goal to produce a block of size @blockSize is reached. - * @return: nb of bytes consumed from @src, necessarily <= @blockSize. - */ -typedef size_t (*ZSTD_SequenceCopier_f)(ZSTD_CCtx* cctx, - ZSTD_SequencePosition* seqPos, - const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, - const void* src, size_t blockSize, - ZSTD_ParamSwitch_e externalRepSearch); - -static ZSTD_SequenceCopier_f ZSTD_selectSequenceCopier(ZSTD_SequenceFormat_e mode) -{ - switch (ZSTD_rust_selectSequenceCopier((int)mode)) { - case ZSTD_sf_explicitBlockDelimiters: - return ZSTD_transferSequences_wBlockDelim; - case ZSTD_sf_noBlockDelimiters: - default: - return ZSTD_transferSequences_noDelim; - } -} - -static size_t determine_blockSize(ZSTD_SequenceFormat_e mode, - size_t blockSize, size_t remaining, - const ZSTD_Sequence* inSeqs, size_t inSeqsSize, - ZSTD_SequencePosition seqPos) -{ - DEBUGLOG(6, "determine_blockSize : remainingSize = %zu", remaining); - assert(mode == ZSTD_sf_noBlockDelimiters || mode == ZSTD_sf_explicitBlockDelimiters); - return ZSTD_rust_determineBlockSize((int)mode, blockSize, remaining, - inSeqs, inSeqsSize, seqPos.idx); -} - /* Compress all provided sequences, block-by-block. * * Returns the cumulative size of all compressed blocks (including their headers), @@ -5226,115 +5184,38 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, const ZSTD_Sequence* inSeqs, size_t inSeqsSize, const void* src, size_t srcSize) { - size_t cSize = 0; - size_t remaining = srcSize; - ZSTD_SequencePosition seqPos = {0, 0, 0}; + U32 dictSize; + ZSTD_rust_sequenceCompressionState state; - const BYTE* ip = (BYTE const*)src; - BYTE* op = (BYTE*)dst; - ZSTD_SequenceCopier_f const sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters); + if (cctx->cdict) { + dictSize = (U32)cctx->cdict->dictContentSize; + } else if (cctx->prefixDict.dict) { + dictSize = (U32)cctx->prefixDict.dictSize; + } else { + dictSize = 0; + } + + state.seqStore = &cctx->seqStore; + state.prevCBlock = &cctx->blockState.prevCBlock; + state.nextCBlock = &cctx->blockState.nextCBlock; + state.tmpWorkspace = cctx->tmpWorkspace; + state.tmpWkspSize = cctx->tmpWkspSize; + state.blockSizeMax = cctx->blockSizeMax; + state.bmi2 = cctx->bmi2; + state.blockDelimiters = (int)cctx->appliedParams.blockDelimiters; + state.strategy = (int)cctx->appliedParams.cParams.strategy; + state.disableLiteralCompression = ZSTD_literalsCompressionIsDisabled(&cctx->appliedParams); + state.searchForExternalRepcodes = (int)cctx->appliedParams.searchForExternalRepcodes; + state.validateSequences = cctx->appliedParams.validateSequences; + state.minMatch = cctx->appliedParams.cParams.minMatch; + state.windowLog = cctx->appliedParams.cParams.windowLog; + state.dictSize = dictSize; + state.useSequenceProducer = ZSTD_hasExtSeqProd(&cctx->appliedParams); + state.isFirstBlock = &cctx->isFirstBlock; DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize); - /* Special case: empty frame */ - if (remaining == 0) { - U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1); - RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "No room for empty frame block header"); - MEM_writeLE32(op, cBlockHeader24); - op += ZSTD_blockHeaderSize; - dstCapacity -= ZSTD_blockHeaderSize; - cSize += ZSTD_blockHeaderSize; - } - - while (remaining) { - size_t compressedSeqsSize; - size_t cBlockSize; - size_t blockSize = determine_blockSize(cctx->appliedParams.blockDelimiters, - cctx->blockSizeMax, remaining, - inSeqs, inSeqsSize, seqPos); - U32 const lastBlock = (blockSize == remaining); - FORWARD_IF_ERROR(blockSize, "Error while trying to determine block size"); - assert(blockSize <= remaining); - ZSTD_resetSeqStore(&cctx->seqStore); - - blockSize = sequenceCopier(cctx, - &seqPos, inSeqs, inSeqsSize, - ip, blockSize, - cctx->appliedParams.searchForExternalRepcodes); - FORWARD_IF_ERROR(blockSize, "Bad sequence copy"); - - /* If blocks are too small, emit as a nocompress block */ - /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding - * additional 1. We need to revisit and change this logic to be more consistent */ - if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { - cBlockSize = ZSTD_rust_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); - FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); - DEBUGLOG(5, "Block too small (%zu): data remains uncompressed: cSize=%zu", blockSize, cBlockSize); - cSize += cBlockSize; - ip += blockSize; - op += cBlockSize; - remaining -= blockSize; - dstCapacity -= cBlockSize; - continue; - } - - RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "not enough dstCapacity to write a new compressed block"); - compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore, - &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, - &cctx->appliedParams, - op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, - blockSize, - cctx->tmpWorkspace, cctx->tmpWkspSize /* statically allocated in resetCCtx */, - cctx->bmi2); - FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); - DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize); - - if (!cctx->isFirstBlock && - ZSTD_maybeRLE(&cctx->seqStore) && - ZSTD_isRLE(ip, blockSize)) { - /* Note: don't emit the first block as RLE even if it qualifies because - * doing so will cause the decoder (cli <= v1.4.3 only) to throw an (invalid) error - * "should consume all input error." - */ - compressedSeqsSize = 1; - } - - if (compressedSeqsSize == 0) { - /* ZSTD_noCompressBlock writes the block header as well */ - cBlockSize = ZSTD_rust_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); - FORWARD_IF_ERROR(cBlockSize, "ZSTD_noCompressBlock failed"); - DEBUGLOG(5, "Writing out nocompress block, size: %zu", cBlockSize); - } else if (compressedSeqsSize == 1) { - cBlockSize = ZSTD_rust_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock); - FORWARD_IF_ERROR(cBlockSize, "ZSTD_rleCompressBlock failed"); - DEBUGLOG(5, "Writing out RLE block, size: %zu", cBlockSize); - } else { - /* Error checking and repcodes update */ - ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState); - if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) - cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; - - /* Write block header into beginning of block*/ - ZSTD_rust_writeBlockHeader(op, compressedSeqsSize, blockSize, lastBlock); - cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; - DEBUGLOG(5, "Writing out compressed block, size: %zu", cBlockSize); - } - - cSize += cBlockSize; - - if (lastBlock) { - break; - } else { - ip += blockSize; - op += cBlockSize; - remaining -= blockSize; - dstCapacity -= cBlockSize; - cctx->isFirstBlock = 0; - } - DEBUGLOG(5, "cSize running total: %zu (remaining dstCapacity=%zu)", cSize, dstCapacity); - } - - DEBUGLOG(4, "cSize final total: %zu", cSize); - return cSize; + return ZSTD_rust_compressSequencesInternal(&state, dst, dstCapacity, + inSeqs, inSeqsSize, src, srcSize); } size_t ZSTD_compressSequences(ZSTD_CCtx* cctx, diff --git a/rust/src/zstd_compress.rs b/rust/src/zstd_compress.rs index 8e49f6162..bae204bb8 100644 --- a/rust/src/zstd_compress.rs +++ b/rust/src/zstd_compress.rs @@ -17,8 +17,8 @@ use crate::errors::{ERR_isError, ZstdErrorCode, ERROR}; #[cfg(not(test))] use crate::zstd_compress_api::ZSTD_compressBound; use crate::zstd_compress_frame::{ - write_raw_block, ZSTD_rust_writeBlockHeader, ZSTD_rust_writeFrameHeader, - ZSTD_writeLastEmptyBlock, + write_raw_block, ZSTD_rust_noCompressBlock, ZSTD_rust_rleCompressBlock, + ZSTD_rust_writeBlockHeader, ZSTD_rust_writeFrameHeader, ZSTD_writeLastEmptyBlock, }; use crate::zstd_compress_literals::min_gain; use crate::zstd_compress_params::{ @@ -27,10 +27,14 @@ use crate::zstd_compress_params::{ }; use crate::zstd_compress_sequences::SeqDef; use crate::zstd_compress_stats::{ - SeqStore_t, ZSTD_compressedBlockState_t, ZSTD_rust_entropyCompressSeqStore, ZSTD_rust_isRLE, + SeqStore_t, ZSTD_Sequence, ZSTD_SequencePosition, ZSTD_compressedBlockState_t, + ZSTD_rust_confirmRepcodesAndEntropyTables, ZSTD_rust_determineBlockSize, + ZSTD_rust_entropyCompressSeqStore, ZSTD_rust_isRLE, ZSTD_rust_maybeRLE, + ZSTD_rust_resetSeqStore, ZSTD_rust_transferSequencesNoDelim, + ZSTD_rust_transferSequencesWBlockDelim, }; use std::ffi::c_void; -use std::mem::{size_of, MaybeUninit}; +use std::mem::{offset_of, size_of, MaybeUninit}; use std::os::raw::{c_int, c_uint}; use std::ptr; @@ -101,6 +105,64 @@ const ZSTD_CHUNKSIZE_MAX: usize = u32::MAX as usize - ZSTD_CURRENT_MAX; #[cfg(not(test))] const ZSTD_E_END: c_int = 2; +/// Explicit projection of the state used by `ZSTD_compressSequences_internal`. +/// +/// The C context and its function-pointer-bearing parameter structure remain +/// private to C. Only the sequence store, compressed-block state slots, and +/// scalar policy/workspace fields read by the per-block loop cross the ABI. +#[repr(C)] +pub struct ZSTD_rust_sequenceCompressionState { + seq_store: *mut SeqStore_t, + prev_c_block: *mut *mut ZSTD_compressedBlockState_t, + next_c_block: *mut *mut ZSTD_compressedBlockState_t, + tmp_workspace: *mut c_void, + tmp_wksp_size: usize, + block_size_max: usize, + bmi2: c_int, + block_delimiters: c_int, + strategy: c_int, + disable_literal_compression: c_int, + search_for_external_repcodes: c_int, + validate_sequences: c_int, + min_match: c_uint, + window_log: c_uint, + dict_size: c_uint, + use_sequence_producer: c_int, + is_first_block: *mut c_int, +} + +const _: () = { + assert!(offset_of!(ZSTD_rust_sequenceCompressionState, seq_store) == 0); + assert!(offset_of!(ZSTD_rust_sequenceCompressionState, prev_c_block) == size_of::()); + assert!(offset_of!(ZSTD_rust_sequenceCompressionState, next_c_block) == 2 * size_of::()); + assert!( + offset_of!(ZSTD_rust_sequenceCompressionState, tmp_workspace) == 3 * size_of::() + ); + assert!( + offset_of!(ZSTD_rust_sequenceCompressionState, tmp_wksp_size) == 4 * size_of::() + ); + assert!( + offset_of!(ZSTD_rust_sequenceCompressionState, block_size_max) == 5 * size_of::() + ); + assert!(offset_of!(ZSTD_rust_sequenceCompressionState, bmi2) == 6 * size_of::()); + assert!( + offset_of!(ZSTD_rust_sequenceCompressionState, min_match) + == 6 * size_of::() + 6 * size_of::() + ); + assert!( + offset_of!(ZSTD_rust_sequenceCompressionState, use_sequence_producer) + == 6 * size_of::() + 6 * size_of::() + 3 * size_of::() + ); + assert!( + offset_of!(ZSTD_rust_sequenceCompressionState, is_first_block) + == 6 * size_of::() + 7 * size_of::() + 3 * size_of::() + ); + assert!( + size_of::() + == if size_of::() == 8 { 96 } else { 68 } + ); +}; + #[repr(i32)] #[derive(Debug, Copy, Clone, PartialEq, Eq)] enum TargetCBlockAction { @@ -110,6 +172,30 @@ enum TargetCBlockAction { Error = 3, } +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum SequenceBlockAction { + Raw, + Rle, + Compressed, +} + +#[inline] +fn sequence_block_action( + is_first_block: c_int, + maybe_rle: c_int, + is_rle: c_int, + compressed_size: usize, +) -> SequenceBlockAction { + if is_first_block == 0 && maybe_rle != 0 && is_rle != 0 { + return SequenceBlockAction::Rle; + } + match compressed_size { + 0 => SequenceBlockAction::Raw, + 1 => SequenceBlockAction::Rle, + _ => SequenceBlockAction::Compressed, + } +} + #[inline] fn target_c_block_size_action( bss: c_int, @@ -952,6 +1038,241 @@ fn ceil_log2(size: usize) -> u32 { } } +#[inline] +unsafe fn write_empty_sequence_block(dst: *mut u8, dst_capacity: usize) -> usize { + /* Keep the original C helper's four-byte write and capacity check. */ + if dst_capacity < 4 { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + let header = 1u32.to_le_bytes(); + unsafe { ptr::copy_nonoverlapping(header.as_ptr(), dst, header.len()) }; + ZSTD_BLOCK_HEADER_SIZE +} + +/// Rust implementation of the per-block loop from +/// `ZSTD_compressSequences_internal()`. +/// +/// Context initialization, frame-header/checksum handling, and public API +/// validation remain in C. The projected state keeps the ABI explicit while +/// allowing the loop to reuse the existing Rust sequence-transfer, entropy, +/// and block-serialization leaves. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_compressSequencesInternal( + state: *const ZSTD_rust_sequenceCompressionState, + dst: *mut c_void, + dst_capacity: usize, + in_seqs: *const ZSTD_Sequence, + in_seqs_size: usize, + src: *const c_void, + src_size: usize, +) -> usize { + if state.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + let state = unsafe { &*state }; + if state.seq_store.is_null() + || state.prev_c_block.is_null() + || state.next_c_block.is_null() + || state.is_first_block.is_null() + { + return ERROR(ZstdErrorCode::Generic); + } + if unsafe { (*state.prev_c_block).is_null() || (*state.next_c_block).is_null() } { + return ERROR(ZstdErrorCode::Generic); + } + + let mut c_size = 0usize; + let mut remaining = src_size; + let mut seq_pos = ZSTD_SequencePosition::default(); + let mut ip = src.cast::(); + let mut op = dst.cast::(); + let mut dst_capacity = dst_capacity; + let explicit_delimiters = + ZSTD_rust_selectSequenceCopier(state.block_delimiters) == ZSTD_SF_EXPLICIT_BLOCK_DELIMITERS; + + /* Special case: empty frame. */ + if remaining == 0 { + return unsafe { write_empty_sequence_block(op, dst_capacity) }; + } + + while remaining != 0 { + let mut block_size = unsafe { + ZSTD_rust_determineBlockSize( + state.block_delimiters, + state.block_size_max, + remaining, + in_seqs, + in_seqs_size, + seq_pos.idx, + ) + }; + let last_block = u32::from(block_size == remaining); + if ERR_isError(block_size) { + return block_size; + } + debug_assert!(block_size <= remaining); + + unsafe { ZSTD_rust_resetSeqStore(state.seq_store) }; + + let prev_block = unsafe { *state.prev_c_block }; + let next_block = unsafe { *state.next_c_block }; + block_size = if explicit_delimiters { + unsafe { + ZSTD_rust_transferSequencesWBlockDelim( + state.seq_store, + &mut seq_pos, + in_seqs, + in_seqs_size, + ip, + block_size, + state.search_for_external_repcodes, + (*prev_block).rep.as_ptr(), + (*next_block).rep.as_mut_ptr(), + state.dict_size, + state.validate_sequences, + state.min_match, + state.window_log, + state.use_sequence_producer, + ) + } + } else { + unsafe { + ZSTD_rust_transferSequencesNoDelim( + state.seq_store, + &mut seq_pos, + in_seqs, + in_seqs_size, + ip, + block_size, + (*prev_block).rep.as_ptr(), + (*next_block).rep.as_mut_ptr(), + state.dict_size, + state.validate_sequences, + state.min_match, + state.window_log, + state.use_sequence_producer, + ) + } + }; + if ERR_isError(block_size) { + return block_size; + } + + /* If blocks are too small, emit as a nocompress block. */ + if block_size < MIN_COMPRESSIBLE_BLOCK_SIZE { + let c_block_size = unsafe { + ZSTD_rust_noCompressBlock( + op.cast(), + dst_capacity, + ip.cast(), + block_size, + last_block, + ) + }; + if ERR_isError(c_block_size) { + return c_block_size; + } + c_size = c_size.wrapping_add(c_block_size); + unsafe { + ip = ip.add(block_size); + op = op.add(c_block_size); + } + remaining -= block_size; + dst_capacity -= c_block_size; + continue; + } + + if dst_capacity < ZSTD_BLOCK_HEADER_SIZE { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + + let compressed_size = unsafe { + ZSTD_rust_entropyCompressSeqStore( + state.seq_store, + ptr::addr_of!((*prev_block).entropy), + ptr::addr_of_mut!((*next_block).entropy), + state.strategy, + state.disable_literal_compression, + op.add(ZSTD_BLOCK_HEADER_SIZE).cast(), + dst_capacity - ZSTD_BLOCK_HEADER_SIZE, + block_size, + state.tmp_workspace, + state.tmp_wksp_size, + state.bmi2, + ) + }; + if ERR_isError(compressed_size) { + return compressed_size; + } + + let is_first_block = unsafe { *state.is_first_block }; + let (maybe_rle, is_rle) = if is_first_block == 0 { + let maybe_rle = unsafe { ZSTD_rust_maybeRLE(state.seq_store) }; + let is_rle = if maybe_rle != 0 { + unsafe { ZSTD_rust_isRLE(ip, block_size) } + } else { + 0 + }; + (maybe_rle, is_rle) + } else { + (0, 0) + }; + let action = sequence_block_action(is_first_block, maybe_rle, is_rle, compressed_size); + + let c_block_size = match action { + SequenceBlockAction::Raw => unsafe { + /* ZSTD_noCompressBlock writes the block header as well. */ + ZSTD_rust_noCompressBlock( + op.cast(), + dst_capacity, + ip.cast(), + block_size, + last_block, + ) + }, + SequenceBlockAction::Rle => unsafe { + ZSTD_rust_rleCompressBlock(op.cast(), dst_capacity, *ip, block_size, last_block) + }, + SequenceBlockAction::Compressed => { + /* Error checking and repcodes update. */ + unsafe { + ZSTD_rust_confirmRepcodesAndEntropyTables( + state.prev_c_block, + state.next_c_block, + ); + } + let prev_block = unsafe { *state.prev_c_block }; + if unsafe { (*prev_block).entropy.fse.offcode_repeatMode } == 2 { + unsafe { (*prev_block).entropy.fse.offcode_repeatMode = 1 }; + } + + unsafe { + ZSTD_rust_writeBlockHeader(op.cast(), compressed_size, block_size, last_block) + }; + ZSTD_BLOCK_HEADER_SIZE + compressed_size + } + }; + if ERR_isError(c_block_size) { + return c_block_size; + } + + c_size = c_size.wrapping_add(c_block_size); + if last_block != 0 { + break; + } + + unsafe { + ip = ip.add(block_size); + op = op.add(c_block_size); + *state.is_first_block = 0; + } + remaining -= block_size; + dst_capacity -= c_block_size; + } + + c_size +} + /// Compress one frame using the already migrated block leaves. /// /// This path owns the match tables and carries their history across the @@ -2242,6 +2563,42 @@ mod tests { ); } + #[test] + fn sequence_block_action_keeps_first_block_from_using_rle() { + assert_eq!( + sequence_block_action(1, 1, 1, 2), + SequenceBlockAction::Compressed + ); + assert_eq!(sequence_block_action(0, 1, 1, 2), SequenceBlockAction::Rle); + } + + #[test] + fn sequence_block_action_maps_entropy_fallbacks() { + assert_eq!(sequence_block_action(0, 0, 0, 0), SequenceBlockAction::Raw); + assert_eq!(sequence_block_action(0, 0, 0, 1), SequenceBlockAction::Rle); + assert_eq!( + sequence_block_action(0, 0, 0, 2), + SequenceBlockAction::Compressed + ); + } + + #[test] + fn empty_sequence_block_preserves_header_and_capacity_contract() { + let mut output = [0xa5; 4]; + assert_eq!( + unsafe { write_empty_sequence_block(output.as_mut_ptr(), output.len()) }, + ZSTD_BLOCK_HEADER_SIZE + ); + assert_eq!(output, [1, 0, 0, 0]); + + let mut short_output = [0xa5; 3]; + assert_eq!( + unsafe { write_empty_sequence_block(short_output.as_mut_ptr(), short_output.len()) }, + ERROR(ZstdErrorCode::DstSizeTooSmall) + ); + assert_eq!(short_output, [0xa5; 3]); + } + #[test] fn invalidate_rep_codes_clears_all_entries() { let mut rep = [11u32, 22, 33];