feat(compress): move no-delimiter sequence transfer to Rust

Port the no-delimiter external-sequence block scanner, match splitting, repcode updates, validation, and sequence/literal storage behind a narrow C shim. Preserve trailing bytes as literals when the sequence buffer ends before the requested block, and remove the obsolete C validation wrapper.

Test Plan:

- cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression (242 passed)

- cargo clippy --manifest-path rust/Cargo.toml --no-default-features --features compression [--benches|--tests] (clean)

- cargo +nightly fmt --manifest-path rust/Cargo.toml (clean)

- make -B -C lib -j2 lib (passed)

- make -C tests -j2 test-zstream (84 deterministic; 5960 and 8307 randomized passed)

- make -C tests -j2 test-fuzzer (existing test 56 CCtx-reuse mismatch; also fails at 37eb75758)

- make -C tests -j2 test-zstd (existing Rust CLI --patch-from gap)
This commit is contained in:
2026-07-18 07:49:11 +02:00
parent 37eb757585
commit 7ee445c431
2 changed files with 385 additions and 124 deletions
+18 -124
View File
@@ -300,9 +300,6 @@ size_t ZSTD_rust_blockSizeExplicitDelimiter(const ZSTD_Sequence* inSeqs,
size_t ZSTD_rust_determineBlockSize(int mode, size_t blockSize, size_t remaining,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
U32 seqIdx);
size_t ZSTD_rust_validateSequence(U32 offBase, U32 matchLength, U32 minMatch,
size_t posInSrc, U32 windowLog, size_t dictSize,
int useSequenceProducer);
size_t ZSTD_rust_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
const void* dict, size_t dictSize);
size_t ZSTD_rust_transferSequencesWBlockDelim(
@@ -313,6 +310,14 @@ size_t ZSTD_rust_transferSequencesWBlockDelim(
U32 nextRepcodes[ZSTD_REP_NUM], U32 dictSize,
int validateSequences, U32 minMatch, U32 windowLog,
int useSequenceProducer);
size_t ZSTD_rust_transferSequencesNoDelim(
SeqStore_t* seqStore, ZSTD_SequencePosition* seqPos,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
const BYTE* src, size_t blockSize,
const U32 prevRepcodes[ZSTD_REP_NUM],
U32 nextRepcodes[ZSTD_REP_NUM], U32 dictSize,
int validateSequences, U32 minMatch, U32 windowLog,
int useSequenceProducer);
size_t ZSTD_rust_optimalBlockSize(const void* src, size_t srcSize,
size_t blockSizeMax, int splitLevel,
int strategy, S64 savings,
@@ -5124,19 +5129,6 @@ size_t ZSTD_compress2_c(ZSTD_CCtx* cctx,
}
}
/* ZSTD_validateSequence() :
* @offBase : must use the format required by ZSTD_storeSeq()
* @returns a ZSTD error code if sequence is not valid
*/
static size_t
ZSTD_validateSequence(U32 offBase, U32 matchLength, U32 minMatch,
size_t posInSrc, U32 windowLog, size_t dictSize, int useSequenceProducer)
{
return ZSTD_rust_validateSequence(offBase, matchLength, minMatch,
posInSrc, windowLog, dictSize,
useSequenceProducer);
}
/* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */
static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0)
{
@@ -5194,19 +5186,7 @@ ZSTD_transferSequences_noDelim(ZSTD_CCtx* cctx,
const void* src, size_t blockSize,
ZSTD_ParamSwitch_e externalRepSearch)
{
U32 idx = seqPos->idx;
U32 startPosInSequence = seqPos->posInSequence;
U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize;
size_t dictSize;
const BYTE* const istart = (const BYTE*)(src);
const BYTE* ip = istart;
const BYTE* iend = istart + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */
Repcodes_t updatedRepcodes;
U32 bytesAdjustment = 0;
U32 finalMatchSplit = 0;
/* TODO(embg) support fast parsing mode in noBlockDelim mode */
(void)externalRepSearch;
if (cctx->cdict) {
dictSize = cctx->cdict->dictContentSize;
@@ -5215,102 +5195,16 @@ ZSTD_transferSequences_noDelim(ZSTD_CCtx* cctx,
} else {
dictSize = 0;
}
DEBUGLOG(5, "ZSTD_transferSequences_noDelim: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize);
DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength);
ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t));
while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) {
const ZSTD_Sequence currSeq = inSeqs[idx];
U32 litLength = currSeq.litLength;
U32 matchLength = currSeq.matchLength;
U32 const rawOffset = currSeq.offset;
U32 offBase;
/* Modify the sequence depending on where endPosInSequence lies */
if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) {
if (startPosInSequence >= litLength) {
startPosInSequence -= litLength;
litLength = 0;
matchLength -= startPosInSequence;
} else {
litLength -= startPosInSequence;
}
/* Move to the next sequence */
endPosInSequence -= currSeq.litLength + currSeq.matchLength;
startPosInSequence = 0;
} else {
/* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence
does not reach the end of the match. So, we have to split the sequence */
DEBUGLOG(6, "Require a split: diff: %u, idx: %u PIS: %u",
currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence);
if (endPosInSequence > litLength) {
U32 firstHalfMatchLength;
litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence;
firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength;
if (matchLength > blockSize && firstHalfMatchLength >= cctx->appliedParams.cParams.minMatch) {
/* Only ever split the match if it is larger than the block size */
U32 secondHalfMatchLength = currSeq.matchLength + currSeq.litLength - endPosInSequence;
if (secondHalfMatchLength < cctx->appliedParams.cParams.minMatch) {
/* Move the endPosInSequence backward so that it creates match of minMatch length */
endPosInSequence -= cctx->appliedParams.cParams.minMatch - secondHalfMatchLength;
bytesAdjustment = cctx->appliedParams.cParams.minMatch - secondHalfMatchLength;
firstHalfMatchLength -= bytesAdjustment;
}
matchLength = firstHalfMatchLength;
/* Flag that we split the last match - after storing the sequence, exit the loop,
but keep the value of endPosInSequence */
finalMatchSplit = 1;
} else {
/* Move the position in sequence backwards so that we don't split match, and break to store
* the last literals. We use the original currSeq.litLength as a marker for where endPosInSequence
* should go. We prefer to do this whenever it is not necessary to split the match, or if doing so
* would cause the first half of the match to be too small
*/
bytesAdjustment = endPosInSequence - currSeq.litLength;
endPosInSequence = currSeq.litLength;
break;
}
} else {
/* This sequence ends inside the literals, break to store the last literals */
break;
}
}
/* Check if this offset can be represented with a repcode */
{ U32 const ll0 = (litLength == 0);
offBase = ZSTD_finalizeOffBase(rawOffset, updatedRepcodes.rep, ll0);
ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0);
}
if (cctx->appliedParams.validateSequences) {
seqPos->posInSrc += litLength + matchLength;
FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc,
cctx->appliedParams.cParams.windowLog, dictSize, ZSTD_hasExtSeqProd(&cctx->appliedParams)),
"Sequence validation failed");
}
DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
"Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
ip += matchLength + litLength;
if (!finalMatchSplit)
idx++; /* Next Sequence */
}
DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength);
assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength);
seqPos->idx = idx;
seqPos->posInSequence = endPosInSequence;
ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t));
iend -= bytesAdjustment;
if (ip != iend) {
/* Store any last literals */
U32 const lastLLSize = (U32)(iend - ip);
assert(ip <= iend);
DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize);
ZSTD_storeLastLiterals(&cctx->seqStore, ip, lastLLSize);
seqPos->posInSrc += lastLLSize;
}
return (size_t)(iend-istart);
(void)externalRepSearch;
return ZSTD_rust_transferSequencesNoDelim(
&cctx->seqStore, seqPos, inSeqs, inSeqsSize,
(const BYTE*)src, blockSize,
cctx->blockState.prevCBlock->rep,
cctx->blockState.nextCBlock->rep, (U32)dictSize,
cctx->appliedParams.validateSequences,
cctx->appliedParams.cParams.minMatch,
cctx->appliedParams.cParams.windowLog,
ZSTD_hasExtSeqProd(&cctx->appliedParams));
}
/* @seqPos represents a position within @inSeqs,
+367
View File
@@ -784,6 +784,183 @@ pub unsafe extern "C" fn ZSTD_rust_transferSequencesWBlockDelim(
block_size
}
/// Transfers externally produced sequences through a block-size boundary when
/// the producer did not provide explicit delimiters. A block may end in the
/// middle of a match; in that case the sequence position is retained for the
/// next call. This is the Rust leaf for
/// `ZSTD_transferSequences_noDelim()`.
#[allow(clippy::too_many_arguments)]
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_transferSequencesNoDelim(
seq_store: *mut SeqStore_t,
seq_pos: *mut ZSTD_SequencePosition,
in_seqs: *const ZSTD_Sequence,
in_seqs_size: usize,
src: *const u8,
block_size: usize,
prev_repcodes: *const u32,
next_repcodes: *mut u32,
dict_size: u32,
validate_sequences: c_int,
min_match: u32,
window_log: u32,
use_sequence_producer: c_int,
) -> usize {
if seq_store.is_null()
|| seq_pos.is_null()
|| in_seqs.is_null()
|| src.is_null()
|| prev_repcodes.is_null()
|| next_repcodes.is_null()
|| block_size > u32::MAX as usize
{
return ERROR(ZstdErrorCode::ExternalSequencesInvalid);
}
let seq_store = unsafe { &mut *seq_store };
let seq_pos = unsafe { &mut *seq_pos };
let start_idx = seq_pos.idx as usize;
if start_idx > in_seqs_size {
return ERROR(ZstdErrorCode::ExternalSequencesInvalid);
}
let block_size_u32 = block_size as u32;
let mut start_pos_in_sequence = seq_pos.posInSequence;
let mut end_pos_in_sequence = start_pos_in_sequence.wrapping_add(block_size_u32);
let mut source_offset = 0usize;
let mut bytes_adjustment = 0u32;
let mut final_match_split = false;
let mut updated_repcodes = [0u32; ZSTD_REP_NUM];
unsafe {
ptr::copy_nonoverlapping(prev_repcodes, updated_repcodes.as_mut_ptr(), ZSTD_REP_NUM);
}
let mut idx = start_idx;
while end_pos_in_sequence != 0 && idx < in_seqs_size && !final_match_split {
let current_sequence = unsafe { *in_seqs.add(idx) };
let mut lit_length = current_sequence.litLength;
let mut match_length = current_sequence.matchLength;
let raw_offset = current_sequence.offset;
let current_size = current_sequence
.litLength
.wrapping_add(current_sequence.matchLength);
if end_pos_in_sequence >= current_size {
if start_pos_in_sequence >= lit_length {
start_pos_in_sequence = start_pos_in_sequence.wrapping_sub(lit_length);
lit_length = 0;
match_length = match_length.wrapping_sub(start_pos_in_sequence);
} else {
lit_length = lit_length.wrapping_sub(start_pos_in_sequence);
}
end_pos_in_sequence = end_pos_in_sequence.wrapping_sub(current_size);
start_pos_in_sequence = 0;
} else {
/* The block ends inside this sequence. */
if end_pos_in_sequence > lit_length {
lit_length = if start_pos_in_sequence >= lit_length {
0
} else {
lit_length.wrapping_sub(start_pos_in_sequence)
};
let first_half_match_length = end_pos_in_sequence
.wrapping_sub(start_pos_in_sequence)
.wrapping_sub(lit_length);
if match_length > block_size_u32 && first_half_match_length >= min_match {
let second_half_match_length = current_size.wrapping_sub(end_pos_in_sequence);
let mut first_half_match_length = first_half_match_length;
if second_half_match_length < min_match {
let adjustment = min_match.wrapping_sub(second_half_match_length);
end_pos_in_sequence = end_pos_in_sequence.wrapping_sub(adjustment);
bytes_adjustment = adjustment;
first_half_match_length = first_half_match_length.wrapping_sub(adjustment);
}
match_length = first_half_match_length;
final_match_split = true;
} else {
bytes_adjustment = end_pos_in_sequence.wrapping_sub(current_sequence.litLength);
end_pos_in_sequence = current_sequence.litLength;
break;
}
} else {
/* The block ends inside the literals; leave them as literals. */
break;
}
}
let ll0 = lit_length == 0;
let off_base = unsafe {
ZSTD_rust_finalizeOffBase(raw_offset, updated_repcodes.as_ptr(), u32::from(ll0))
};
update_rep(&mut updated_repcodes, off_base, ll0);
if validate_sequences != 0 {
let sequence_size = lit_length.wrapping_add(match_length) as usize;
seq_pos.posInSrc = seq_pos.posInSrc.wrapping_add(sequence_size);
let validation = ZSTD_rust_validateSequence(
off_base,
match_length,
min_match,
seq_pos.posInSrc,
window_log,
dict_size as usize,
use_sequence_producer,
);
if ERR_isError(validation) {
return validation;
}
}
if idx - start_idx >= seq_store.maxNbSeq {
return ERROR(ZstdErrorCode::ExternalSequencesInvalid);
}
let stored = unsafe {
store_external_sequence(
seq_store,
src,
source_offset,
block_size,
lit_length as usize,
off_base,
match_length as usize,
)
};
if !stored {
return ERROR(ZstdErrorCode::ExternalSequencesInvalid);
}
let sequence_size = lit_length.wrapping_add(match_length) as usize;
source_offset = match source_offset.checked_add(sequence_size) {
Some(offset) => offset,
None => return ERROR(ZstdErrorCode::ExternalSequencesInvalid),
};
if !final_match_split {
idx += 1;
}
}
seq_pos.idx = idx as u32;
seq_pos.posInSequence = end_pos_in_sequence;
unsafe {
ptr::copy_nonoverlapping(updated_repcodes.as_ptr(), next_repcodes, ZSTD_REP_NUM);
}
let consumed_size = match block_size.checked_sub(bytes_adjustment as usize) {
Some(size) => size,
None => return ERROR(ZstdErrorCode::ExternalSequencesInvalid),
};
if source_offset > consumed_size {
return ERROR(ZstdErrorCode::ExternalSequencesInvalid);
}
if source_offset != consumed_size {
let last_literal_size = consumed_size - source_offset;
unsafe {
ZSTD_rust_storeLastLiterals(seq_store, src.add(source_offset), last_literal_size);
}
seq_pos.posInSrc = seq_pos.posInSrc.wrapping_add(last_literal_size);
}
consumed_size
}
/// Finds the next explicit block delimiter and returns the represented size.
///
/// The scan is half-open at `inSeqsSize`: a delimiter at the final element is
@@ -2318,6 +2495,22 @@ mod tests {
}
}
fn empty_external_seq_store(sequences: &mut [SeqDef], literals: &mut [u8]) -> SeqStore_t {
SeqStore_t {
sequencesStart: sequences.as_mut_ptr(),
sequences: sequences.as_mut_ptr(),
litStart: literals.as_mut_ptr(),
lit: literals.as_mut_ptr(),
llCode: std::ptr::null_mut(),
mlCode: std::ptr::null_mut(),
ofCode: std::ptr::null_mut(),
maxNbSeq: sequences.len(),
maxNbLit: literals.len(),
longLengthType: 0,
longLengthPos: 0,
}
}
#[test]
fn c_leaf_layouts_match_supported_abis() {
assert_eq!(size_of::<SeqDef>(), 8);
@@ -2605,6 +2798,180 @@ mod tests {
assert_eq!(literals, [9]);
}
#[test]
fn transfer_sequences_no_delim_splits_a_match_at_the_block_boundary() {
let input = [ZSTD_Sequence {
offset: 50,
litLength: 0,
matchLength: 10,
rep: 0,
}];
let source = [1u8, 2, 3, 4, 5, 6];
let mut output = [SeqDef::default(); 1];
let mut literals = [0u8; 1];
let mut seq_store = empty_external_seq_store(&mut output, &mut literals);
let mut position = ZSTD_SequencePosition::default();
let previous_repcodes = [7u32, 20, 30];
let mut next_repcodes = [0u32; ZSTD_REP_NUM];
let result = unsafe {
ZSTD_rust_transferSequencesNoDelim(
&mut seq_store,
&mut position,
input.as_ptr(),
input.len(),
source.as_ptr(),
source.len(),
previous_repcodes.as_ptr(),
next_repcodes.as_mut_ptr(),
0,
0,
3,
10,
0,
)
};
assert_eq!(result, source.len());
assert_eq!(position.idx, 0);
assert_eq!(position.posInSequence, source.len() as u32);
assert_eq!(position.posInSrc, 0);
assert_eq!(output[0].offBase, 50 + ZSTD_REP_NUM as u32);
assert_eq!(output[0].litLength, 0);
assert_eq!(output[0].mlBase, source.len() as u16 - MINMATCH as u16);
assert_eq!(next_repcodes, [50, 7, 20]);
}
#[test]
fn transfer_sequences_no_delim_shortens_a_match_to_keep_the_tail_valid() {
let input = [ZSTD_Sequence {
offset: 50,
litLength: 0,
matchLength: 10,
rep: 0,
}];
let source = [0u8; 8];
let mut output = [SeqDef::default(); 1];
let mut literals = [0u8; 1];
let mut seq_store = empty_external_seq_store(&mut output, &mut literals);
let mut position = ZSTD_SequencePosition::default();
let previous_repcodes = [7u32, 20, 30];
let mut next_repcodes = [0u32; ZSTD_REP_NUM];
let result = unsafe {
ZSTD_rust_transferSequencesNoDelim(
&mut seq_store,
&mut position,
input.as_ptr(),
input.len(),
source.as_ptr(),
source.len(),
previous_repcodes.as_ptr(),
next_repcodes.as_mut_ptr(),
0,
0,
3,
10,
0,
)
};
assert_eq!(result, 7);
assert_eq!(position.idx, 0);
assert_eq!(position.posInSequence, 7);
assert_eq!(output[0].offBase, 50 + ZSTD_REP_NUM as u32);
assert_eq!(output[0].mlBase, 7 - MINMATCH as u16);
assert_eq!(next_repcodes, [50, 7, 20]);
}
#[test]
fn transfer_sequences_no_delim_leaves_a_literal_prefix_unsequenced() {
let input = [ZSTD_Sequence {
offset: 50,
litLength: 4,
matchLength: 6,
rep: 0,
}];
let source = [1u8, 2, 3, 4, 5, 6];
let mut output = [SeqDef::default(); 1];
let mut literals = [0u8; 2];
let mut seq_store = empty_external_seq_store(&mut output, &mut literals);
let mut position = ZSTD_SequencePosition::default();
let previous_repcodes = [7u32, 20, 30];
let mut next_repcodes = [0u32; ZSTD_REP_NUM];
let result = unsafe {
ZSTD_rust_transferSequencesNoDelim(
&mut seq_store,
&mut position,
input.as_ptr(),
input.len(),
source.as_ptr(),
2,
previous_repcodes.as_ptr(),
next_repcodes.as_mut_ptr(),
0,
0,
3,
10,
0,
)
};
assert_eq!(result, 2);
assert_eq!(position.idx, 0);
assert_eq!(position.posInSequence, 2);
assert_eq!(position.posInSrc, 2);
assert_eq!(seq_store.sequences, seq_store.sequencesStart);
assert_eq!(literals, [1, 2]);
assert_eq!(next_repcodes, previous_repcodes);
}
#[test]
fn transfer_sequences_no_delim_treats_bytes_after_the_last_sequence_as_literals() {
let input = [ZSTD_Sequence {
offset: 50,
litLength: 2,
matchLength: 3,
rep: 0,
}];
let source = [10u8, 11, 12, 13, 14, 15, 16];
let mut output = [SeqDef::default(); 1];
let mut literals = [0u8; 4];
let mut seq_store = empty_external_seq_store(&mut output, &mut literals);
let mut position = ZSTD_SequencePosition::default();
let previous_repcodes = [7u32, 20, 30];
let mut next_repcodes = [0u32; ZSTD_REP_NUM];
let result = unsafe {
ZSTD_rust_transferSequencesNoDelim(
&mut seq_store,
&mut position,
input.as_ptr(),
input.len(),
source.as_ptr(),
source.len(),
previous_repcodes.as_ptr(),
next_repcodes.as_mut_ptr(),
0,
0,
3,
10,
0,
)
};
assert_eq!(result, source.len());
assert_eq!(position.idx, 1);
assert_eq!(position.posInSequence, 2);
assert_eq!(position.posInSrc, 2);
assert_eq!(output[0].offBase, 50 + ZSTD_REP_NUM as u32);
assert_eq!(output[0].litLength, 2);
assert_eq!(output[0].mlBase, 0);
assert_eq!(literals, [10, 11, 15, 16]);
assert_eq!(next_repcodes, [50, 7, 20]);
}
#[test]
fn transfer_sequences_rejects_invalid_match_lengths_and_block_mismatches() {
let invalid_match = [