From d5ae2a450bb1292a14cbe064e4785f19bdb00741 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Sat, 18 Jul 2026 04:31:22 +0200 Subject: [PATCH] feat(compress): move external sequence sizing leaves to Rust Port repcode offBase finalization and explicit-delimiter block sizing through the Rust compression ABI. Preserve wrapped sequence arithmetic, external-sequence error codes, and the no-delimiter target-size policy while keeping the C control flow intact. Test Plan: cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression (176 passed); cargo clippy root, benches, and tests; make -B -C lib -j2 lib; make -B -C tests -j2 test-zstream (84 named tests plus 6,230 and 8,598 fuzz cases passed). --- lib/compress/zstd_compress.c | 62 ++------- rust/src/zstd_compress_stats.rs | 214 ++++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+), 52 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index ecd8f1aea..2d4e82722 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -251,6 +251,12 @@ void ZSTD_rust_deriveSeqStoreChunk(SeqStore_t* resultSeqStore, size_t startIdx, size_t endIdx); U32 ZSTD_rust_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], U32 offBase, U32 ll0); +U32 ZSTD_rust_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0); +size_t ZSTD_rust_blockSizeExplicitDelimiter(const ZSTD_Sequence* inSeqs, + size_t inSeqsSize, U32 seqIdx); +size_t ZSTD_rust_determineBlockSize(int mode, size_t blockSize, size_t remaining, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + U32 seqIdx); typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1]; typedef char ZSTD_rust_stats_seqstore_long_length_pos[ @@ -5440,18 +5446,7 @@ ZSTD_validateSequence(U32 offBase, U32 matchLength, U32 minMatch, /* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */ static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) { - U32 offBase = OFFSET_TO_OFFBASE(rawOffset); - - if (!ll0 && rawOffset == rep[0]) { - offBase = REPCODE1_TO_OFFBASE; - } else if (rawOffset == rep[1]) { - offBase = REPCODE_TO_OFFBASE(2 - ll0); - } else if (rawOffset == rep[2]) { - offBase = REPCODE_TO_OFFBASE(3 - ll0); - } else if (ll0 && rawOffset == rep[0] - 1) { - offBase = REPCODE3_TO_OFFBASE; - } - return offBase; + return ZSTD_rust_finalizeOffBase(rawOffset, rep, ll0); } /* This function scans through an array of ZSTD_Sequence, @@ -5707,52 +5702,15 @@ static ZSTD_SequenceCopier_f ZSTD_selectSequenceCopier(ZSTD_SequenceFormat_e mod return ZSTD_transferSequences_noDelim; } -/* Discover the size of next block by searching for the delimiter. - * Note that a block delimiter **must** exist in this mode, - * otherwise it's an input error. - * The block size retrieved will be later compared to ensure it remains within bounds */ -static size_t -blockSize_explicitDelimiter(const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_SequencePosition seqPos) -{ - int end = 0; - size_t blockSize = 0; - size_t spos = seqPos.idx; - DEBUGLOG(6, "blockSize_explicitDelimiter : seq %zu / %zu", spos, inSeqsSize); - assert(spos <= inSeqsSize); - while (spos < inSeqsSize) { - end = (inSeqs[spos].offset == 0); - blockSize += inSeqs[spos].litLength + inSeqs[spos].matchLength; - if (end) { - if (inSeqs[spos].matchLength != 0) - RETURN_ERROR(externalSequences_invalid, "delimiter format error : both matchlength and offset must be == 0"); - break; - } - spos++; - } - if (!end) - RETURN_ERROR(externalSequences_invalid, "Reached end of sequences without finding a block delimiter"); - return blockSize; -} - static size_t determine_blockSize(ZSTD_SequenceFormat_e mode, size_t blockSize, size_t remaining, const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_SequencePosition seqPos) { DEBUGLOG(6, "determine_blockSize : remainingSize = %zu", remaining); - if (mode == ZSTD_sf_noBlockDelimiters) { - /* Note: more a "target" block size */ - return MIN(remaining, blockSize); - } - assert(mode == ZSTD_sf_explicitBlockDelimiters); - { size_t const explicitBlockSize = blockSize_explicitDelimiter(inSeqs, inSeqsSize, seqPos); - FORWARD_IF_ERROR(explicitBlockSize, "Error while determining block size with explicit delimiters"); - if (explicitBlockSize > blockSize) - RETURN_ERROR(externalSequences_invalid, "sequences incorrectly define a too large block"); - if (explicitBlockSize > remaining) - RETURN_ERROR(externalSequences_invalid, "sequences define a frame longer than source"); - return explicitBlockSize; - } + assert(mode == ZSTD_sf_noBlockDelimiters || mode == ZSTD_sf_explicitBlockDelimiters); + return ZSTD_rust_determineBlockSize((int)mode, blockSize, remaining, + inSeqs, inSeqsSize, seqPos.idx); } /* Compress all provided sequences, block-by-block. diff --git a/rust/src/zstd_compress_stats.rs b/rust/src/zstd_compress_stats.rs index c604b055d..1ba8ef6bd 100644 --- a/rust/src/zstd_compress_stats.rs +++ b/rust/src/zstd_compress_stats.rs @@ -158,6 +158,99 @@ pub struct ZSTD_Sequence { pub rep: u32, } +/// Converts a raw sequence offset to the stored offBase representation. +/// +/// This is the Rust leaf for C's `ZSTD_finalizeOffBase()`. The repcode +/// numbering and the special `rep[0] - 1` form are part of the public sequence +/// ABI used by the external-sequence path. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_finalizeOffBase( + raw_offset: u32, + rep: *const u32, + ll0: u32, +) -> u32 { + let rep = unsafe { std::slice::from_raw_parts(rep, ZSTD_REP_NUM) }; + let mut off_base = raw_offset.wrapping_add(ZSTD_REP_NUM as u32); + + if ll0 == 0 && raw_offset == rep[0] { + off_base = 1; + } else if raw_offset == rep[1] { + off_base = 2u32.wrapping_sub(ll0); + } else if raw_offset == rep[2] { + off_base = 3u32.wrapping_sub(ll0); + } else if ll0 != 0 && raw_offset == rep[0].wrapping_sub(1) { + off_base = 3; + } + off_base +} + +/// Finds the next explicit block delimiter and returns the represented size. +/// +/// The scan is half-open at `inSeqsSize`: a delimiter at the final element is +/// included, while a missing delimiter returns the C external-sequences +/// error. The delimiter's literal length is part of the block size, but its +/// match length must be zero. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_blockSizeExplicitDelimiter( + in_seqs: *const ZSTD_Sequence, + in_seqs_size: usize, + seq_idx: u32, +) -> usize { + let mut end = false; + let mut block_size = 0usize; + let mut sequence_index = seq_idx as usize; + debug_assert!(sequence_index <= in_seqs_size); + + while sequence_index < in_seqs_size { + let sequence = unsafe { *in_seqs.add(sequence_index) }; + end = sequence.offset == 0; + let sequence_size = sequence.litLength.wrapping_add(sequence.matchLength) as usize; + block_size = block_size.wrapping_add(sequence_size); + if end { + if sequence.matchLength != 0 { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + break; + } + sequence_index += 1; + } + + if !end { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + block_size +} + +/// Determines the next external-sequence block size. +/// +/// Mode `0` is the no-delimiter mode and returns the target size capped at the +/// remaining source. Mode `1` scans an explicit delimiter and rejects blocks +/// larger than either configured block size or the remaining frame. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_determineBlockSize( + mode: c_int, + block_size: usize, + remaining: usize, + in_seqs: *const ZSTD_Sequence, + in_seqs_size: usize, + seq_idx: u32, +) -> usize { + if mode == 0 { + return remaining.min(block_size); + } + debug_assert_eq!(mode, 1); + + let explicit_block_size = + unsafe { ZSTD_rust_blockSizeExplicitDelimiter(in_seqs, in_seqs_size, seq_idx) }; + if ERR_isError(explicit_block_size) { + return explicit_block_size; + } + if explicit_block_size > block_size || explicit_block_size > remaining { + return ERROR(ZstdErrorCode::ExternalSequencesInvalid); + } + explicit_block_size +} + /// Validates and post-processes sequences returned by an external sequence /// producer. This is the Rust leaf for C's /// `ZSTD_postProcessSequenceProducerResult()`. @@ -2300,4 +2393,125 @@ mod tests { unsafe { ZSTD_rust_copyBlockSequences(&mut collector, &seq_store, reps.as_ptr()) }; assert_eq!(result, ERROR(ZstdErrorCode::DstSizeTooSmall)); } + + #[test] + fn finalize_off_base_preserves_repcode_boundaries() { + let reps = [10u32, 20, 30]; + let finalize = + |raw_offset, ll0| unsafe { ZSTD_rust_finalizeOffBase(raw_offset, reps.as_ptr(), ll0) }; + + assert_eq!(finalize(10, 0), 1); // repcode 1 with literals + assert_eq!(finalize(10, 1), 13); // raw offset, not repcode 1 + assert_eq!(finalize(20, 0), 2); // repcode 2 with literals + assert_eq!(finalize(20, 1), 1); // repcode 1 without literals + assert_eq!(finalize(30, 0), 3); // repcode 3 with literals + assert_eq!(finalize(30, 1), 2); // repcode 2 without literals + assert_eq!(finalize(9, 1), 3); // repcode 3's rep[0] - 1 form + assert_eq!(finalize(31, 0), 34); // ordinary raw offset + } + + #[test] + fn determine_block_size_without_delimiters_returns_minimum() { + let result = unsafe { ZSTD_rust_determineBlockSize(0, 128, 50, ptr::null(), 0, 0) }; + assert_eq!(result, 50); + + let result = unsafe { ZSTD_rust_determineBlockSize(0, 32, 50, ptr::null(), 0, 0) }; + assert_eq!(result, 32); + } + + #[test] + fn explicit_delimiter_size_includes_delimiter_literals() { + let sequences = [ + ZSTD_Sequence { + offset: 7, + litLength: 3, + matchLength: 4, + rep: 0, + }, + ZSTD_Sequence { + offset: 0, + litLength: 5, + matchLength: 0, + rep: 0, + }, + ]; + let result = + unsafe { ZSTD_rust_blockSizeExplicitDelimiter(sequences.as_ptr(), sequences.len(), 0) }; + assert_eq!(result, 12); + + let result = unsafe { + ZSTD_rust_determineBlockSize(1, 12, 12, sequences.as_ptr(), sequences.len(), 0) + }; + assert_eq!(result, 12); + } + + #[test] + fn explicit_delimiter_size_rejects_missing_delimiter() { + let sequences = [ZSTD_Sequence { + offset: 7, + litLength: 3, + matchLength: 4, + rep: 0, + }]; + let result = + unsafe { ZSTD_rust_blockSizeExplicitDelimiter(sequences.as_ptr(), sequences.len(), 0) }; + assert_eq!(result, ERROR(ZstdErrorCode::ExternalSequencesInvalid)); + } + + #[test] + fn explicit_delimiter_size_rejects_match_length() { + let sequences = [ZSTD_Sequence { + offset: 0, + litLength: 3, + matchLength: 4, + rep: 0, + }]; + let result = + unsafe { ZSTD_rust_blockSizeExplicitDelimiter(sequences.as_ptr(), sequences.len(), 0) }; + assert_eq!(result, ERROR(ZstdErrorCode::ExternalSequencesInvalid)); + } + + #[test] + fn determine_block_size_rejects_too_large_block() { + let sequences = [ + ZSTD_Sequence { + offset: 7, + litLength: 3, + matchLength: 4, + rep: 0, + }, + ZSTD_Sequence { + offset: 0, + litLength: 1, + matchLength: 0, + rep: 0, + }, + ]; + let result = unsafe { + ZSTD_rust_determineBlockSize(1, 7, 8, sequences.as_ptr(), sequences.len(), 0) + }; + assert_eq!(result, ERROR(ZstdErrorCode::ExternalSequencesInvalid)); + } + + #[test] + fn determine_block_size_rejects_too_long_frame() { + let sequences = [ + ZSTD_Sequence { + offset: 7, + litLength: 3, + matchLength: 4, + rep: 0, + }, + ZSTD_Sequence { + offset: 0, + litLength: 1, + matchLength: 0, + rep: 0, + }, + ]; + let result = unsafe { + ZSTD_rust_determineBlockSize(1, 8, 7, sequences.as_ptr(), sequences.len(), 0) + }; + assert_eq!(result, ERROR(ZstdErrorCode::ExternalSequencesInvalid)); + } }