From 7ee445c43145bcc78a0125439aa0718b837acdb1 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Sat, 18 Jul 2026 07:49:11 +0200 Subject: [PATCH] feat(compress): move no-delimiter sequence transfer to Rust Port the no-delimiter external-sequence block scanner, match splitting, repcode updates, validation, and sequence/literal storage behind a narrow C shim. Preserve trailing bytes as literals when the sequence buffer ends before the requested block, and remove the obsolete C validation wrapper. Test Plan: - cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression (242 passed) - cargo clippy --manifest-path rust/Cargo.toml --no-default-features --features compression [--benches|--tests] (clean) - cargo +nightly fmt --manifest-path rust/Cargo.toml (clean) - make -B -C lib -j2 lib (passed) - make -C tests -j2 test-zstream (84 deterministic; 5960 and 8307 randomized passed) - make -C tests -j2 test-fuzzer (existing test 56 CCtx-reuse mismatch; also fails at 37eb75758) - make -C tests -j2 test-zstd (existing Rust CLI --patch-from gap) --- lib/compress/zstd_compress.c | 142 ++---------- rust/src/zstd_compress_stats.rs | 367 ++++++++++++++++++++++++++++++++ 2 files changed, 385 insertions(+), 124 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 80e4b31cc..4c8fbebc4 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -300,9 +300,6 @@ size_t ZSTD_rust_blockSizeExplicitDelimiter(const ZSTD_Sequence* inSeqs, size_t ZSTD_rust_determineBlockSize(int mode, size_t blockSize, size_t remaining, const ZSTD_Sequence* inSeqs, size_t inSeqsSize, U32 seqIdx); -size_t ZSTD_rust_validateSequence(U32 offBase, U32 matchLength, U32 minMatch, - size_t posInSrc, U32 windowLog, size_t dictSize, - int useSequenceProducer); size_t ZSTD_rust_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, const void* dict, size_t dictSize); size_t ZSTD_rust_transferSequencesWBlockDelim( @@ -313,6 +310,14 @@ size_t ZSTD_rust_transferSequencesWBlockDelim( U32 nextRepcodes[ZSTD_REP_NUM], U32 dictSize, int validateSequences, U32 minMatch, U32 windowLog, int useSequenceProducer); +size_t ZSTD_rust_transferSequencesNoDelim( + SeqStore_t* seqStore, ZSTD_SequencePosition* seqPos, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const BYTE* src, size_t blockSize, + const U32 prevRepcodes[ZSTD_REP_NUM], + U32 nextRepcodes[ZSTD_REP_NUM], U32 dictSize, + int validateSequences, U32 minMatch, U32 windowLog, + int useSequenceProducer); size_t ZSTD_rust_optimalBlockSize(const void* src, size_t srcSize, size_t blockSizeMax, int splitLevel, int strategy, S64 savings, @@ -5124,19 +5129,6 @@ size_t ZSTD_compress2_c(ZSTD_CCtx* cctx, } } -/* ZSTD_validateSequence() : - * @offBase : must use the format required by ZSTD_storeSeq() - * @returns a ZSTD error code if sequence is not valid - */ -static size_t -ZSTD_validateSequence(U32 offBase, U32 matchLength, U32 minMatch, - size_t posInSrc, U32 windowLog, size_t dictSize, int useSequenceProducer) -{ - return ZSTD_rust_validateSequence(offBase, matchLength, minMatch, - posInSrc, windowLog, dictSize, - useSequenceProducer); -} - /* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */ static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) { @@ -5194,19 +5186,7 @@ ZSTD_transferSequences_noDelim(ZSTD_CCtx* cctx, const void* src, size_t blockSize, ZSTD_ParamSwitch_e externalRepSearch) { - U32 idx = seqPos->idx; - U32 startPosInSequence = seqPos->posInSequence; - U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize; size_t dictSize; - const BYTE* const istart = (const BYTE*)(src); - const BYTE* ip = istart; - const BYTE* iend = istart + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */ - Repcodes_t updatedRepcodes; - U32 bytesAdjustment = 0; - U32 finalMatchSplit = 0; - - /* TODO(embg) support fast parsing mode in noBlockDelim mode */ - (void)externalRepSearch; if (cctx->cdict) { dictSize = cctx->cdict->dictContentSize; @@ -5215,102 +5195,16 @@ ZSTD_transferSequences_noDelim(ZSTD_CCtx* cctx, } else { dictSize = 0; } - DEBUGLOG(5, "ZSTD_transferSequences_noDelim: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize); - DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); - ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t)); - while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) { - const ZSTD_Sequence currSeq = inSeqs[idx]; - U32 litLength = currSeq.litLength; - U32 matchLength = currSeq.matchLength; - U32 const rawOffset = currSeq.offset; - U32 offBase; - - /* Modify the sequence depending on where endPosInSequence lies */ - if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) { - if (startPosInSequence >= litLength) { - startPosInSequence -= litLength; - litLength = 0; - matchLength -= startPosInSequence; - } else { - litLength -= startPosInSequence; - } - /* Move to the next sequence */ - endPosInSequence -= currSeq.litLength + currSeq.matchLength; - startPosInSequence = 0; - } else { - /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence - does not reach the end of the match. So, we have to split the sequence */ - DEBUGLOG(6, "Require a split: diff: %u, idx: %u PIS: %u", - currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence); - if (endPosInSequence > litLength) { - U32 firstHalfMatchLength; - litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence; - firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength; - if (matchLength > blockSize && firstHalfMatchLength >= cctx->appliedParams.cParams.minMatch) { - /* Only ever split the match if it is larger than the block size */ - U32 secondHalfMatchLength = currSeq.matchLength + currSeq.litLength - endPosInSequence; - if (secondHalfMatchLength < cctx->appliedParams.cParams.minMatch) { - /* Move the endPosInSequence backward so that it creates match of minMatch length */ - endPosInSequence -= cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; - bytesAdjustment = cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; - firstHalfMatchLength -= bytesAdjustment; - } - matchLength = firstHalfMatchLength; - /* Flag that we split the last match - after storing the sequence, exit the loop, - but keep the value of endPosInSequence */ - finalMatchSplit = 1; - } else { - /* Move the position in sequence backwards so that we don't split match, and break to store - * the last literals. We use the original currSeq.litLength as a marker for where endPosInSequence - * should go. We prefer to do this whenever it is not necessary to split the match, or if doing so - * would cause the first half of the match to be too small - */ - bytesAdjustment = endPosInSequence - currSeq.litLength; - endPosInSequence = currSeq.litLength; - break; - } - } else { - /* This sequence ends inside the literals, break to store the last literals */ - break; - } - } - /* Check if this offset can be represented with a repcode */ - { U32 const ll0 = (litLength == 0); - offBase = ZSTD_finalizeOffBase(rawOffset, updatedRepcodes.rep, ll0); - ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); - } - - if (cctx->appliedParams.validateSequences) { - seqPos->posInSrc += litLength + matchLength; - FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc, - cctx->appliedParams.cParams.windowLog, dictSize, ZSTD_hasExtSeqProd(&cctx->appliedParams)), - "Sequence validation failed"); - } - DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); - RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid, - "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); - ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength); - ip += matchLength + litLength; - if (!finalMatchSplit) - idx++; /* Next Sequence */ - } - DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); - assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength); - seqPos->idx = idx; - seqPos->posInSequence = endPosInSequence; - ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t)); - - iend -= bytesAdjustment; - if (ip != iend) { - /* Store any last literals */ - U32 const lastLLSize = (U32)(iend - ip); - assert(ip <= iend); - DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize); - ZSTD_storeLastLiterals(&cctx->seqStore, ip, lastLLSize); - seqPos->posInSrc += lastLLSize; - } - - return (size_t)(iend-istart); + (void)externalRepSearch; + return ZSTD_rust_transferSequencesNoDelim( + &cctx->seqStore, seqPos, inSeqs, inSeqsSize, + (const BYTE*)src, blockSize, + cctx->blockState.prevCBlock->rep, + cctx->blockState.nextCBlock->rep, (U32)dictSize, + cctx->appliedParams.validateSequences, + cctx->appliedParams.cParams.minMatch, + cctx->appliedParams.cParams.windowLog, + ZSTD_hasExtSeqProd(&cctx->appliedParams)); } /* @seqPos represents a position within @inSeqs, diff --git a/rust/src/zstd_compress_stats.rs b/rust/src/zstd_compress_stats.rs index d05543aa2..160466f16 100644 --- a/rust/src/zstd_compress_stats.rs +++ b/rust/src/zstd_compress_stats.rs @@ -784,6 +784,183 @@ pub unsafe extern "C" fn ZSTD_rust_transferSequencesWBlockDelim( block_size } +/// Transfers externally produced sequences through a block-size boundary when +/// the producer did not provide explicit delimiters. A block may end in the +/// middle of a match; in that case the sequence position is retained for the +/// next call. This is the Rust leaf for +/// `ZSTD_transferSequences_noDelim()`. +#[allow(clippy::too_many_arguments)] +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_transferSequencesNoDelim( + seq_store: *mut SeqStore_t, + seq_pos: *mut ZSTD_SequencePosition, + in_seqs: *const ZSTD_Sequence, + in_seqs_size: usize, + src: *const u8, + block_size: usize, + prev_repcodes: *const u32, + next_repcodes: *mut u32, + dict_size: u32, + validate_sequences: c_int, + min_match: u32, + window_log: u32, + use_sequence_producer: c_int, +) -> usize { + if seq_store.is_null() + || seq_pos.is_null() + || in_seqs.is_null() + || src.is_null() + || prev_repcodes.is_null() + || next_repcodes.is_null() + || block_size > u32::MAX as usize + { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + + let seq_store = unsafe { &mut *seq_store }; + let seq_pos = unsafe { &mut *seq_pos }; + let start_idx = seq_pos.idx as usize; + if start_idx > in_seqs_size { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + + let block_size_u32 = block_size as u32; + let mut start_pos_in_sequence = seq_pos.posInSequence; + let mut end_pos_in_sequence = start_pos_in_sequence.wrapping_add(block_size_u32); + let mut source_offset = 0usize; + let mut bytes_adjustment = 0u32; + let mut final_match_split = false; + let mut updated_repcodes = [0u32; ZSTD_REP_NUM]; + unsafe { + ptr::copy_nonoverlapping(prev_repcodes, updated_repcodes.as_mut_ptr(), ZSTD_REP_NUM); + } + + let mut idx = start_idx; + while end_pos_in_sequence != 0 && idx < in_seqs_size && !final_match_split { + let current_sequence = unsafe { *in_seqs.add(idx) }; + let mut lit_length = current_sequence.litLength; + let mut match_length = current_sequence.matchLength; + let raw_offset = current_sequence.offset; + let current_size = current_sequence + .litLength + .wrapping_add(current_sequence.matchLength); + + if end_pos_in_sequence >= current_size { + if start_pos_in_sequence >= lit_length { + start_pos_in_sequence = start_pos_in_sequence.wrapping_sub(lit_length); + lit_length = 0; + match_length = match_length.wrapping_sub(start_pos_in_sequence); + } else { + lit_length = lit_length.wrapping_sub(start_pos_in_sequence); + } + end_pos_in_sequence = end_pos_in_sequence.wrapping_sub(current_size); + start_pos_in_sequence = 0; + } else { + /* The block ends inside this sequence. */ + if end_pos_in_sequence > lit_length { + lit_length = if start_pos_in_sequence >= lit_length { + 0 + } else { + lit_length.wrapping_sub(start_pos_in_sequence) + }; + let first_half_match_length = end_pos_in_sequence + .wrapping_sub(start_pos_in_sequence) + .wrapping_sub(lit_length); + if match_length > block_size_u32 && first_half_match_length >= min_match { + let second_half_match_length = current_size.wrapping_sub(end_pos_in_sequence); + let mut first_half_match_length = first_half_match_length; + if second_half_match_length < min_match { + let adjustment = min_match.wrapping_sub(second_half_match_length); + end_pos_in_sequence = end_pos_in_sequence.wrapping_sub(adjustment); + bytes_adjustment = adjustment; + first_half_match_length = first_half_match_length.wrapping_sub(adjustment); + } + match_length = first_half_match_length; + final_match_split = true; + } else { + bytes_adjustment = end_pos_in_sequence.wrapping_sub(current_sequence.litLength); + end_pos_in_sequence = current_sequence.litLength; + break; + } + } else { + /* The block ends inside the literals; leave them as literals. */ + break; + } + } + + let ll0 = lit_length == 0; + let off_base = unsafe { + ZSTD_rust_finalizeOffBase(raw_offset, updated_repcodes.as_ptr(), u32::from(ll0)) + }; + update_rep(&mut updated_repcodes, off_base, ll0); + + if validate_sequences != 0 { + let sequence_size = lit_length.wrapping_add(match_length) as usize; + seq_pos.posInSrc = seq_pos.posInSrc.wrapping_add(sequence_size); + let validation = ZSTD_rust_validateSequence( + off_base, + match_length, + min_match, + seq_pos.posInSrc, + window_log, + dict_size as usize, + use_sequence_producer, + ); + if ERR_isError(validation) { + return validation; + } + } + + if idx - start_idx >= seq_store.maxNbSeq { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + let stored = unsafe { + store_external_sequence( + seq_store, + src, + source_offset, + block_size, + lit_length as usize, + off_base, + match_length as usize, + ) + }; + if !stored { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + let sequence_size = lit_length.wrapping_add(match_length) as usize; + source_offset = match source_offset.checked_add(sequence_size) { + Some(offset) => offset, + None => return ERROR(ZstdErrorCode::ExternalSequencesInvalid), + }; + if !final_match_split { + idx += 1; + } + } + + seq_pos.idx = idx as u32; + seq_pos.posInSequence = end_pos_in_sequence; + unsafe { + ptr::copy_nonoverlapping(updated_repcodes.as_ptr(), next_repcodes, ZSTD_REP_NUM); + } + + let consumed_size = match block_size.checked_sub(bytes_adjustment as usize) { + Some(size) => size, + None => return ERROR(ZstdErrorCode::ExternalSequencesInvalid), + }; + if source_offset > consumed_size { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + if source_offset != consumed_size { + let last_literal_size = consumed_size - source_offset; + unsafe { + ZSTD_rust_storeLastLiterals(seq_store, src.add(source_offset), last_literal_size); + } + seq_pos.posInSrc = seq_pos.posInSrc.wrapping_add(last_literal_size); + } + consumed_size +} + /// Finds the next explicit block delimiter and returns the represented size. /// /// The scan is half-open at `inSeqsSize`: a delimiter at the final element is @@ -2318,6 +2495,22 @@ mod tests { } } + fn empty_external_seq_store(sequences: &mut [SeqDef], literals: &mut [u8]) -> SeqStore_t { + SeqStore_t { + sequencesStart: sequences.as_mut_ptr(), + sequences: sequences.as_mut_ptr(), + litStart: literals.as_mut_ptr(), + lit: literals.as_mut_ptr(), + llCode: std::ptr::null_mut(), + mlCode: std::ptr::null_mut(), + ofCode: std::ptr::null_mut(), + maxNbSeq: sequences.len(), + maxNbLit: literals.len(), + longLengthType: 0, + longLengthPos: 0, + } + } + #[test] fn c_leaf_layouts_match_supported_abis() { assert_eq!(size_of::(), 8); @@ -2605,6 +2798,180 @@ mod tests { assert_eq!(literals, [9]); } + #[test] + fn transfer_sequences_no_delim_splits_a_match_at_the_block_boundary() { + let input = [ZSTD_Sequence { + offset: 50, + litLength: 0, + matchLength: 10, + rep: 0, + }]; + let source = [1u8, 2, 3, 4, 5, 6]; + let mut output = [SeqDef::default(); 1]; + let mut literals = [0u8; 1]; + let mut seq_store = empty_external_seq_store(&mut output, &mut literals); + let mut position = ZSTD_SequencePosition::default(); + let previous_repcodes = [7u32, 20, 30]; + let mut next_repcodes = [0u32; ZSTD_REP_NUM]; + + let result = unsafe { + ZSTD_rust_transferSequencesNoDelim( + &mut seq_store, + &mut position, + input.as_ptr(), + input.len(), + source.as_ptr(), + source.len(), + previous_repcodes.as_ptr(), + next_repcodes.as_mut_ptr(), + 0, + 0, + 3, + 10, + 0, + ) + }; + + assert_eq!(result, source.len()); + assert_eq!(position.idx, 0); + assert_eq!(position.posInSequence, source.len() as u32); + assert_eq!(position.posInSrc, 0); + assert_eq!(output[0].offBase, 50 + ZSTD_REP_NUM as u32); + assert_eq!(output[0].litLength, 0); + assert_eq!(output[0].mlBase, source.len() as u16 - MINMATCH as u16); + assert_eq!(next_repcodes, [50, 7, 20]); + } + + #[test] + fn transfer_sequences_no_delim_shortens_a_match_to_keep_the_tail_valid() { + let input = [ZSTD_Sequence { + offset: 50, + litLength: 0, + matchLength: 10, + rep: 0, + }]; + let source = [0u8; 8]; + let mut output = [SeqDef::default(); 1]; + let mut literals = [0u8; 1]; + let mut seq_store = empty_external_seq_store(&mut output, &mut literals); + let mut position = ZSTD_SequencePosition::default(); + let previous_repcodes = [7u32, 20, 30]; + let mut next_repcodes = [0u32; ZSTD_REP_NUM]; + + let result = unsafe { + ZSTD_rust_transferSequencesNoDelim( + &mut seq_store, + &mut position, + input.as_ptr(), + input.len(), + source.as_ptr(), + source.len(), + previous_repcodes.as_ptr(), + next_repcodes.as_mut_ptr(), + 0, + 0, + 3, + 10, + 0, + ) + }; + + assert_eq!(result, 7); + assert_eq!(position.idx, 0); + assert_eq!(position.posInSequence, 7); + assert_eq!(output[0].offBase, 50 + ZSTD_REP_NUM as u32); + assert_eq!(output[0].mlBase, 7 - MINMATCH as u16); + assert_eq!(next_repcodes, [50, 7, 20]); + } + + #[test] + fn transfer_sequences_no_delim_leaves_a_literal_prefix_unsequenced() { + let input = [ZSTD_Sequence { + offset: 50, + litLength: 4, + matchLength: 6, + rep: 0, + }]; + let source = [1u8, 2, 3, 4, 5, 6]; + let mut output = [SeqDef::default(); 1]; + let mut literals = [0u8; 2]; + let mut seq_store = empty_external_seq_store(&mut output, &mut literals); + let mut position = ZSTD_SequencePosition::default(); + let previous_repcodes = [7u32, 20, 30]; + let mut next_repcodes = [0u32; ZSTD_REP_NUM]; + + let result = unsafe { + ZSTD_rust_transferSequencesNoDelim( + &mut seq_store, + &mut position, + input.as_ptr(), + input.len(), + source.as_ptr(), + 2, + previous_repcodes.as_ptr(), + next_repcodes.as_mut_ptr(), + 0, + 0, + 3, + 10, + 0, + ) + }; + + assert_eq!(result, 2); + assert_eq!(position.idx, 0); + assert_eq!(position.posInSequence, 2); + assert_eq!(position.posInSrc, 2); + assert_eq!(seq_store.sequences, seq_store.sequencesStart); + assert_eq!(literals, [1, 2]); + assert_eq!(next_repcodes, previous_repcodes); + } + + #[test] + fn transfer_sequences_no_delim_treats_bytes_after_the_last_sequence_as_literals() { + let input = [ZSTD_Sequence { + offset: 50, + litLength: 2, + matchLength: 3, + rep: 0, + }]; + let source = [10u8, 11, 12, 13, 14, 15, 16]; + let mut output = [SeqDef::default(); 1]; + let mut literals = [0u8; 4]; + let mut seq_store = empty_external_seq_store(&mut output, &mut literals); + let mut position = ZSTD_SequencePosition::default(); + let previous_repcodes = [7u32, 20, 30]; + let mut next_repcodes = [0u32; ZSTD_REP_NUM]; + + let result = unsafe { + ZSTD_rust_transferSequencesNoDelim( + &mut seq_store, + &mut position, + input.as_ptr(), + input.len(), + source.as_ptr(), + source.len(), + previous_repcodes.as_ptr(), + next_repcodes.as_mut_ptr(), + 0, + 0, + 3, + 10, + 0, + ) + }; + + assert_eq!(result, source.len()); + assert_eq!(position.idx, 1); + assert_eq!(position.posInSequence, 2); + assert_eq!(position.posInSrc, 2); + assert_eq!(output[0].offBase, 50 + ZSTD_REP_NUM as u32); + assert_eq!(output[0].litLength, 2); + assert_eq!(output[0].mlBase, 0); + assert_eq!(literals, [10, 11, 15, 16]); + assert_eq!(next_repcodes, [50, 7, 20]); + } + #[test] fn transfer_sequences_rejects_invalid_match_lengths_and_block_mismatches() { let invalid_match = [