diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index d712d2c4d..6e8329440 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -664,6 +664,54 @@ typedef char ZSTD_rust_sequence_state_layout[ == offsetof(ZSTD_rust_sequenceCompressionState, isFirstBlock) + sizeof(void*)) ? 1 : -1]; +/* The external-sequence/literals block loop keeps sequence conversion as a + * C-private callback because it still reads the opaque CCtx. Everything + * else it mutates is passed through this explicit projection. */ +typedef size_t (*ZSTD_rust_sequenceLiteralsConvert_f)( + void* context, const ZSTD_Sequence* inSeqs, size_t nbSequences, + int repcodeResolution); +typedef struct { + SeqStore_t* seqStore; + ZSTD_compressedBlockState_t** prevCBlock; + ZSTD_compressedBlockState_t** nextCBlock; + void* tmpWorkspace; + size_t tmpWkspSize; + size_t blockSizeMax; + int bmi2; + int strategy; + int disableLiteralCompression; + int repcodeResolution; + int* isFirstBlock; + void* callbackContext; + ZSTD_rust_sequenceLiteralsConvert_f convertBlockSequences; +} ZSTD_rust_sequenceLiteralsState; + +size_t ZSTD_rust_compressSequencesAndLiteralsInternal( + const ZSTD_rust_sequenceLiteralsState* state, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t nbSequences, + const void* literals, size_t litSize, size_t srcSize); +typedef char ZSTD_rust_sequence_literals_state_layout[ + (offsetof(ZSTD_rust_sequenceLiteralsState, seqStore) == 0 + && offsetof(ZSTD_rust_sequenceLiteralsState, prevCBlock) == sizeof(void*) + && offsetof(ZSTD_rust_sequenceLiteralsState, nextCBlock) == 2 * sizeof(void*) + && offsetof(ZSTD_rust_sequenceLiteralsState, tmpWorkspace) == 3 * sizeof(void*) + && offsetof(ZSTD_rust_sequenceLiteralsState, tmpWkspSize) == 4 * sizeof(void*) + && offsetof(ZSTD_rust_sequenceLiteralsState, blockSizeMax) == 5 * sizeof(void*) + && offsetof(ZSTD_rust_sequenceLiteralsState, bmi2) == 6 * sizeof(void*) + && offsetof(ZSTD_rust_sequenceLiteralsState, strategy) == 6 * sizeof(void*) + sizeof(int) + && offsetof(ZSTD_rust_sequenceLiteralsState, disableLiteralCompression) == 6 * sizeof(void*) + 2 * sizeof(int) + && offsetof(ZSTD_rust_sequenceLiteralsState, repcodeResolution) == 6 * sizeof(void*) + 3 * sizeof(int) + && offsetof(ZSTD_rust_sequenceLiteralsState, isFirstBlock) + == 6 * sizeof(void*) + 4 * sizeof(int) + && offsetof(ZSTD_rust_sequenceLiteralsState, callbackContext) + == 6 * sizeof(void*) + 4 * sizeof(int) + sizeof(void*) + && offsetof(ZSTD_rust_sequenceLiteralsState, convertBlockSequences) + == 6 * sizeof(void*) + 4 * sizeof(int) + 2 * sizeof(void*) + && sizeof(ZSTD_rust_sequenceLiteralsState) + == 6 * sizeof(void*) + 4 * sizeof(int) + 3 * sizeof(void*)) + ? 1 : -1]; + typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1]; typedef char ZSTD_rust_stats_block_summary_layout[ (sizeof(BlockSummary) == 3 * sizeof(size_t)) ? 1 : -1]; @@ -2804,13 +2852,6 @@ size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, /* ZSTD_mergeBlockDelimiters() lives in rust/src/zstd_compress_api.rs. */ -static void -ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs) -{ - ZSTD_rust_confirmRepcodesAndEntropyTables( - &bs->prevCBlock, &bs->nextCBlock); -} - /** ZSTD_buildBlockEntropyStats() : * Builds entropy for the block. * Requires workspace size ENTROPY_WORKSPACE_SIZE @@ -5198,6 +5239,14 @@ size_t ZSTD_convertBlockSequences(ZSTD_CCtx* cctx, return 0; } +static size_t ZSTD_convertBlockSequencesForRust( + void* context, const ZSTD_Sequence* inSeqs, size_t nbSequences, + int repcodeResolution) +{ + return ZSTD_convertBlockSequences((ZSTD_CCtx*)context, inSeqs, nbSequences, + repcodeResolution); +} + BlockSummary ZSTD_get1BlockSummary(const ZSTD_Sequence* seqs, size_t nbSeqs) { return ZSTD_rust_get1BlockSummary(seqs, nbSeqs); @@ -5210,103 +5259,29 @@ ZSTD_compressSequencesAndLiterals_internal(ZSTD_CCtx* cctx, const ZSTD_Sequence* inSeqs, size_t nbSequences, const void* literals, size_t litSize, size_t srcSize) { - size_t remaining = srcSize; - size_t cSize = 0; - BYTE* op = (BYTE*)dst; - int const repcodeResolution = (cctx->appliedParams.searchForExternalRepcodes == ZSTD_ps_enable); + ZSTD_rust_sequenceLiteralsState state; + int const repcodeResolution = + (cctx->appliedParams.searchForExternalRepcodes == ZSTD_ps_enable); + assert(cctx->appliedParams.searchForExternalRepcodes != ZSTD_ps_auto); + state.seqStore = &cctx->seqStore; + state.prevCBlock = &cctx->blockState.prevCBlock; + state.nextCBlock = &cctx->blockState.nextCBlock; + state.tmpWorkspace = cctx->tmpWorkspace; + state.tmpWkspSize = cctx->tmpWkspSize; + state.blockSizeMax = cctx->blockSizeMax; + state.bmi2 = cctx->bmi2; + state.strategy = (int)cctx->appliedParams.cParams.strategy; + state.disableLiteralCompression = + ZSTD_literalsCompressionIsDisabled(&cctx->appliedParams); + state.repcodeResolution = repcodeResolution; + state.isFirstBlock = &cctx->isFirstBlock; + state.callbackContext = cctx; + state.convertBlockSequences = ZSTD_convertBlockSequencesForRust; - DEBUGLOG(4, "ZSTD_compressSequencesAndLiterals_internal: nbSeqs=%zu, litSize=%zu", nbSequences, litSize); - RETURN_ERROR_IF(nbSequences == 0, externalSequences_invalid, "Requires at least 1 end-of-block"); - - /* Special case: empty frame */ - if ((nbSequences == 1) && (inSeqs[0].litLength == 0)) { - U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1); - RETURN_ERROR_IF(dstCapacity<3, dstSize_tooSmall, "No room for empty frame block header"); - MEM_writeLE24(op, cBlockHeader24); - op += ZSTD_blockHeaderSize; - dstCapacity -= ZSTD_blockHeaderSize; - cSize += ZSTD_blockHeaderSize; - } - - while (nbSequences) { - size_t compressedSeqsSize, cBlockSize, conversionStatus; - BlockSummary const block = ZSTD_get1BlockSummary(inSeqs, nbSequences); - U32 const lastBlock = (block.nbSequences == nbSequences); - FORWARD_IF_ERROR(block.nbSequences, "Error while trying to determine nb of sequences for a block"); - assert(block.nbSequences <= nbSequences); - RETURN_ERROR_IF(block.litSize > litSize, externalSequences_invalid, "discrepancy: Sequences require more literals than present in buffer"); - ZSTD_resetSeqStore(&cctx->seqStore); - - conversionStatus = ZSTD_convertBlockSequences(cctx, - inSeqs, block.nbSequences, - repcodeResolution); - FORWARD_IF_ERROR(conversionStatus, "Bad sequence conversion"); - inSeqs += block.nbSequences; - nbSequences -= block.nbSequences; - remaining -= block.blockSize; - - /* Note: when blockSize is very small, other variant send it uncompressed. - * Here, we still send the sequences, because we don't have the original source to send it uncompressed. - * One could imagine in theory reproducing the source from the sequences, - * but that's complex and costly memory intensive, and goes against the objectives of this variant. */ - - RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "not enough dstCapacity to write a new compressed block"); - - compressedSeqsSize = ZSTD_entropyCompressSeqStore_internal( - op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, - literals, block.litSize, - &cctx->seqStore, - &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, - &cctx->appliedParams, - cctx->tmpWorkspace, cctx->tmpWkspSize /* statically allocated in resetCCtx */, - cctx->bmi2); - FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); - /* note: the spec forbids for any compressed block to be larger than maximum block size */ - if (compressedSeqsSize > cctx->blockSizeMax) compressedSeqsSize = 0; - DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize); - litSize -= block.litSize; - literals = (const char*)literals + block.litSize; - - /* Note: difficult to check source for RLE block when only Literals are provided, - * but it could be considered from analyzing the sequence directly */ - - if (compressedSeqsSize == 0) { - /* Sending uncompressed blocks is out of reach, because the source is not provided. - * In theory, one could use the sequences to regenerate the source, like a decompressor, - * but it's complex, and memory hungry, killing the purpose of this variant. - * Current outcome: generate an error code. - */ - RETURN_ERROR(cannotProduce_uncompressedBlock, "ZSTD_compressSequencesAndLiterals cannot generate an uncompressed block"); - } else { - assert(compressedSeqsSize > 1); /* no RLE */ - /* Error checking and repcodes update */ - ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState); - if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) - cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; - - /* Write block header into beginning of block*/ - ZSTD_rust_writeBlockHeader(op, compressedSeqsSize, block.blockSize, lastBlock); - cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; - DEBUGLOG(5, "Writing out compressed block, size: %zu", cBlockSize); - } - - cSize += cBlockSize; - op += cBlockSize; - dstCapacity -= cBlockSize; - cctx->isFirstBlock = 0; - DEBUGLOG(5, "cSize running total: %zu (remaining dstCapacity=%zu)", cSize, dstCapacity); - - if (lastBlock) { - assert(nbSequences == 0); - break; - } - } - - RETURN_ERROR_IF(litSize != 0, externalSequences_invalid, "literals must be entirely and exactly consumed"); - RETURN_ERROR_IF(remaining != 0, externalSequences_invalid, "Sequences must represent a total of exactly srcSize=%zu", srcSize); - DEBUGLOG(4, "cSize final total: %zu", cSize); - return cSize; + return ZSTD_rust_compressSequencesAndLiteralsInternal( + &state, dst, dstCapacity, inSeqs, nbSequences, literals, litSize, + srcSize); } size_t diff --git a/rust/src/zstd_compress.rs b/rust/src/zstd_compress.rs index 1222c8fd9..052f2d88f 100644 --- a/rust/src/zstd_compress.rs +++ b/rust/src/zstd_compress.rs @@ -31,9 +31,9 @@ use crate::zstd_compress_stats::{ ZSTD_entropyCTables_t, ZSTD_rust_confirmRepcodesAndEntropyTables, ZSTD_rust_copyBlockSequences, ZSTD_rust_countSeqStoreLiteralsBytes, ZSTD_rust_countSeqStoreMatchBytes, ZSTD_rust_deriveSeqStoreChunk, ZSTD_rust_determineBlockSize, ZSTD_rust_entropyCompressSeqStore, - ZSTD_rust_isRLE, ZSTD_rust_maybeRLE, ZSTD_rust_resetSeqStore, - ZSTD_rust_seqStore_resolveOffCodes, ZSTD_rust_transferSequencesNoDelim, - ZSTD_rust_transferSequencesWBlockDelim, + ZSTD_rust_entropyCompressSeqStore_internal, ZSTD_rust_get1BlockSummary, ZSTD_rust_isRLE, + ZSTD_rust_maybeRLE, ZSTD_rust_resetSeqStore, ZSTD_rust_seqStore_resolveOffCodes, + ZSTD_rust_transferSequencesNoDelim, ZSTD_rust_transferSequencesWBlockDelim, }; use crate::zstd_compress_superblock::ZSTD_rust_compressSuperBlock; use std::ffi::c_void; @@ -577,6 +577,71 @@ const _: () = { ); }; +/// C callback signature for the CCtx-dependent sequence conversion leaf. +type SequenceLiteralsConvertFn = + unsafe extern "C" fn(*mut c_void, *const ZSTD_Sequence, usize, c_int) -> usize; + +/// Explicit projection of the state used by +/// ZSTD_compressSequencesAndLiterals_internal. +/// +/// CCtx initialization, parameter validation, and the sequence conversion +/// callback remain in C. Rust owns the block loop, external literal cursor, +/// entropy invocation, block framing, and per-frame completion checks. +#[repr(C)] +pub struct ZSTD_rust_sequenceLiteralsState { + seq_store: *mut SeqStore_t, + prev_c_block: *mut *mut ZSTD_compressedBlockState_t, + next_c_block: *mut *mut ZSTD_compressedBlockState_t, + tmp_workspace: *mut c_void, + tmp_wksp_size: usize, + block_size_max: usize, + bmi2: c_int, + strategy: c_int, + disable_literal_compression: c_int, + repcode_resolution: c_int, + is_first_block: *mut c_int, + callback_context: *mut c_void, + convert_block_sequences: SequenceLiteralsConvertFn, +} + +const _: () = { + assert!(offset_of!(ZSTD_rust_sequenceLiteralsState, seq_store) == 0); + assert!(offset_of!(ZSTD_rust_sequenceLiteralsState, prev_c_block) == size_of::()); + assert!(offset_of!(ZSTD_rust_sequenceLiteralsState, next_c_block) == 2 * size_of::()); + assert!(offset_of!(ZSTD_rust_sequenceLiteralsState, tmp_workspace) == 3 * size_of::()); + assert!(offset_of!(ZSTD_rust_sequenceLiteralsState, tmp_wksp_size) == 4 * size_of::()); + assert!(offset_of!(ZSTD_rust_sequenceLiteralsState, block_size_max) == 5 * size_of::()); + assert!(offset_of!(ZSTD_rust_sequenceLiteralsState, bmi2) == 6 * size_of::()); + assert!( + offset_of!(ZSTD_rust_sequenceLiteralsState, strategy) + == 6 * size_of::() + size_of::() + ); + assert!( + offset_of!(ZSTD_rust_sequenceLiteralsState, disable_literal_compression) + == 6 * size_of::() + 2 * size_of::() + ); + assert!( + offset_of!(ZSTD_rust_sequenceLiteralsState, repcode_resolution) + == 6 * size_of::() + 3 * size_of::() + ); + assert!( + offset_of!(ZSTD_rust_sequenceLiteralsState, is_first_block) + == 6 * size_of::() + 4 * size_of::() + ); + assert!( + offset_of!(ZSTD_rust_sequenceLiteralsState, callback_context) + == 6 * size_of::() + 4 * size_of::() + size_of::() + ); + assert!( + offset_of!(ZSTD_rust_sequenceLiteralsState, convert_block_sequences) + == 6 * size_of::() + 4 * size_of::() + 2 * size_of::() + ); + assert!( + size_of::() + == 6 * size_of::() + 4 * size_of::() + 3 * size_of::() + ); +}; + /// Explicit projection of the state used by `ZSTD_compressSeqStore_singleBlock`. /// /// Sequence-store construction and split discovery remain in C. Only the @@ -2580,6 +2645,181 @@ pub unsafe extern "C" fn ZSTD_rust_compressSequencesInternal( /// This path owns the match tables and carries their history across the /// 128 KiB block boundaries, while keeping the private C /// `ZSTD_MatchState_t` window out of the Rust ABI. +/// Writes the three-byte raw header used by the external +/// sequences-and-literals empty-frame special case. +#[inline] +unsafe fn write_empty_sequence_literals_block(dst: *mut u8, dst_capacity: usize) -> usize { + if dst_capacity < ZSTD_BLOCK_HEADER_SIZE { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + let header = [1u8, 0, 0]; + unsafe { ptr::copy_nonoverlapping(header.as_ptr(), dst, header.len()) }; + ZSTD_BLOCK_HEADER_SIZE +} + +/// Rust implementation of the block loop from +/// ZSTD_compressSequencesAndLiterals_internal. +/// +/// The C wrapper retains public-API initialization and the CCtx-dependent +/// sequence conversion callback. Rust owns the external literal cursor, +/// per-block entropy pass, compressed-block framing, and completion checks. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_compressSequencesAndLiteralsInternal( + state: *const ZSTD_rust_sequenceLiteralsState, + dst: *mut c_void, + dst_capacity: usize, + in_seqs: *const ZSTD_Sequence, + nb_sequences: usize, + literals: *const c_void, + lit_size: usize, + src_size: usize, +) -> usize { + if state.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + let state = unsafe { &*state }; + if state.seq_store.is_null() + || state.prev_c_block.is_null() + || state.next_c_block.is_null() + || state.is_first_block.is_null() + || unsafe { (*state.prev_c_block).is_null() || (*state.next_c_block).is_null() } + { + return ERROR(ZstdErrorCode::Generic); + } + if nb_sequences == 0 { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + + let mut remaining = src_size; + let mut nb_sequences = nb_sequences; + let mut c_size = 0usize; + let mut op = dst.cast::(); + let mut dst_capacity = dst_capacity; + let mut in_seqs = in_seqs; + let mut literals = literals; + let mut lit_size = lit_size; + + /* Special case: empty frame. Keep the C ordering: this header is emitted + * before the ordinary sequence loop. */ + if nb_sequences == 1 && unsafe { (*in_seqs).litLength } == 0 { + let written = unsafe { write_empty_sequence_literals_block(op, dst_capacity) }; + if ERR_isError(written) { + return written; + } + unsafe { + op = op.add(written); + } + dst_capacity -= written; + c_size += written; + } + + while nb_sequences != 0 { + let block = unsafe { ZSTD_rust_get1BlockSummary(in_seqs, nb_sequences) }; + let last_block = c_uint::from(block.nbSequences == nb_sequences); + if ERR_isError(block.nbSequences) { + return block.nbSequences; + } + debug_assert!(block.nbSequences <= nb_sequences); + if block.litSize > lit_size { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + + unsafe { ZSTD_rust_resetSeqStore(state.seq_store) }; + let conversion_status = unsafe { + (state.convert_block_sequences)( + state.callback_context, + in_seqs, + block.nbSequences, + state.repcode_resolution, + ) + }; + if ERR_isError(conversion_status) { + return conversion_status; + } + + unsafe { + in_seqs = in_seqs.add(block.nbSequences); + } + nb_sequences -= block.nbSequences; + remaining = remaining.wrapping_sub(block.blockSize); + + if dst_capacity < ZSTD_BLOCK_HEADER_SIZE { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + + let prev_block = unsafe { *state.prev_c_block }; + let next_block = unsafe { *state.next_c_block }; + if prev_block.is_null() || next_block.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + let compressed_size = unsafe { + ZSTD_rust_entropyCompressSeqStore_internal( + op.add(ZSTD_BLOCK_HEADER_SIZE).cast(), + dst_capacity - ZSTD_BLOCK_HEADER_SIZE, + literals, + block.litSize, + state.seq_store, + ptr::addr_of!((*prev_block).entropy), + ptr::addr_of_mut!((*next_block).entropy), + state.strategy, + state.disable_literal_compression, + state.tmp_workspace, + state.tmp_wksp_size, + state.bmi2, + ) + }; + if ERR_isError(compressed_size) { + return compressed_size; + } + + let compressed_size = if compressed_size > state.block_size_max { + 0 + } else { + compressed_size + }; + unsafe { + literals = literals.cast::().add(block.litSize).cast(); + } + lit_size = lit_size.wrapping_sub(block.litSize); + + if compressed_size == 0 { + return ERROR(ZstdErrorCode::CannotProduceUncompressedBlock); + } + + debug_assert!(compressed_size > 1); + unsafe { + ZSTD_rust_confirmRepcodesAndEntropyTables(state.prev_c_block, state.next_c_block); + } + let prev_block = unsafe { *state.prev_c_block }; + if prev_block.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + if unsafe { (*prev_block).entropy.fse.offcode_repeatMode } == 2 { + unsafe { (*prev_block).entropy.fse.offcode_repeatMode = 1 }; + } + + unsafe { + ZSTD_rust_writeBlockHeader(op.cast(), compressed_size, block.blockSize, last_block); + } + let c_block_size = ZSTD_BLOCK_HEADER_SIZE + compressed_size; + c_size = c_size.wrapping_add(c_block_size); + unsafe { + op = op.add(c_block_size); + *state.is_first_block = 0; + } + dst_capacity = dst_capacity.wrapping_sub(c_block_size); + + if last_block != 0 { + break; + } + } + + if lit_size != 0 || remaining != 0 { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + c_size +} + unsafe fn compress_frame( dst: *mut c_void, dst_capacity: usize, @@ -5278,6 +5518,25 @@ mod tests { assert_eq!(short_output, [0xa5; 3]); } + #[test] + fn empty_sequence_literals_block_writes_only_the_three_byte_header() { + let mut output = [0xa5; 4]; + assert_eq!( + unsafe { write_empty_sequence_literals_block(output.as_mut_ptr(), output.len()) }, + ZSTD_BLOCK_HEADER_SIZE + ); + assert_eq!(output, [1, 0, 0, 0xa5]); + + let mut short_output = [0xa5; 2]; + assert_eq!( + unsafe { + write_empty_sequence_literals_block(short_output.as_mut_ptr(), short_output.len()) + }, + ERROR(ZstdErrorCode::DstSizeTooSmall) + ); + assert_eq!(short_output, [0xa5; 2]); + } + #[test] fn invalidate_rep_codes_clears_all_entries() { let mut rep = [11u32, 22, 33];