feat(compress): move no-delimiter sequence transfer to Rust
Port the no-delimiter external-sequence block scanner, match splitting, repcode updates, validation, and sequence/literal storage behind a narrow C shim. Preserve trailing bytes as literals when the sequence buffer ends before the requested block, and remove the obsolete C validation wrapper.
Test Plan:
- cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression (242 passed)
- cargo clippy --manifest-path rust/Cargo.toml --no-default-features --features compression [--benches|--tests] (clean)
- cargo +nightly fmt --manifest-path rust/Cargo.toml (clean)
- make -B -C lib -j2 lib (passed)
- make -C tests -j2 test-zstream (84 deterministic; 5960 and 8307 randomized passed)
- make -C tests -j2 test-fuzzer (existing test 56 CCtx-reuse mismatch; also fails at 37eb75758)
- make -C tests -j2 test-zstd (existing Rust CLI --patch-from gap)
This commit is contained in:
+18
-124
@@ -300,9 +300,6 @@ size_t ZSTD_rust_blockSizeExplicitDelimiter(const ZSTD_Sequence* inSeqs,
|
||||
size_t ZSTD_rust_determineBlockSize(int mode, size_t blockSize, size_t remaining,
|
||||
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
|
||||
U32 seqIdx);
|
||||
size_t ZSTD_rust_validateSequence(U32 offBase, U32 matchLength, U32 minMatch,
|
||||
size_t posInSrc, U32 windowLog, size_t dictSize,
|
||||
int useSequenceProducer);
|
||||
size_t ZSTD_rust_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
|
||||
const void* dict, size_t dictSize);
|
||||
size_t ZSTD_rust_transferSequencesWBlockDelim(
|
||||
@@ -313,6 +310,14 @@ size_t ZSTD_rust_transferSequencesWBlockDelim(
|
||||
U32 nextRepcodes[ZSTD_REP_NUM], U32 dictSize,
|
||||
int validateSequences, U32 minMatch, U32 windowLog,
|
||||
int useSequenceProducer);
|
||||
size_t ZSTD_rust_transferSequencesNoDelim(
|
||||
SeqStore_t* seqStore, ZSTD_SequencePosition* seqPos,
|
||||
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
|
||||
const BYTE* src, size_t blockSize,
|
||||
const U32 prevRepcodes[ZSTD_REP_NUM],
|
||||
U32 nextRepcodes[ZSTD_REP_NUM], U32 dictSize,
|
||||
int validateSequences, U32 minMatch, U32 windowLog,
|
||||
int useSequenceProducer);
|
||||
size_t ZSTD_rust_optimalBlockSize(const void* src, size_t srcSize,
|
||||
size_t blockSizeMax, int splitLevel,
|
||||
int strategy, S64 savings,
|
||||
@@ -5124,19 +5129,6 @@ size_t ZSTD_compress2_c(ZSTD_CCtx* cctx,
|
||||
}
|
||||
}
|
||||
|
||||
/* ZSTD_validateSequence() :
|
||||
* @offBase : must use the format required by ZSTD_storeSeq()
|
||||
* @returns a ZSTD error code if sequence is not valid
|
||||
*/
|
||||
static size_t
|
||||
ZSTD_validateSequence(U32 offBase, U32 matchLength, U32 minMatch,
|
||||
size_t posInSrc, U32 windowLog, size_t dictSize, int useSequenceProducer)
|
||||
{
|
||||
return ZSTD_rust_validateSequence(offBase, matchLength, minMatch,
|
||||
posInSrc, windowLog, dictSize,
|
||||
useSequenceProducer);
|
||||
}
|
||||
|
||||
/* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */
|
||||
static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0)
|
||||
{
|
||||
@@ -5194,19 +5186,7 @@ ZSTD_transferSequences_noDelim(ZSTD_CCtx* cctx,
|
||||
const void* src, size_t blockSize,
|
||||
ZSTD_ParamSwitch_e externalRepSearch)
|
||||
{
|
||||
U32 idx = seqPos->idx;
|
||||
U32 startPosInSequence = seqPos->posInSequence;
|
||||
U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize;
|
||||
size_t dictSize;
|
||||
const BYTE* const istart = (const BYTE*)(src);
|
||||
const BYTE* ip = istart;
|
||||
const BYTE* iend = istart + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */
|
||||
Repcodes_t updatedRepcodes;
|
||||
U32 bytesAdjustment = 0;
|
||||
U32 finalMatchSplit = 0;
|
||||
|
||||
/* TODO(embg) support fast parsing mode in noBlockDelim mode */
|
||||
(void)externalRepSearch;
|
||||
|
||||
if (cctx->cdict) {
|
||||
dictSize = cctx->cdict->dictContentSize;
|
||||
@@ -5215,102 +5195,16 @@ ZSTD_transferSequences_noDelim(ZSTD_CCtx* cctx,
|
||||
} else {
|
||||
dictSize = 0;
|
||||
}
|
||||
DEBUGLOG(5, "ZSTD_transferSequences_noDelim: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize);
|
||||
DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength);
|
||||
ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t));
|
||||
while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) {
|
||||
const ZSTD_Sequence currSeq = inSeqs[idx];
|
||||
U32 litLength = currSeq.litLength;
|
||||
U32 matchLength = currSeq.matchLength;
|
||||
U32 const rawOffset = currSeq.offset;
|
||||
U32 offBase;
|
||||
|
||||
/* Modify the sequence depending on where endPosInSequence lies */
|
||||
if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) {
|
||||
if (startPosInSequence >= litLength) {
|
||||
startPosInSequence -= litLength;
|
||||
litLength = 0;
|
||||
matchLength -= startPosInSequence;
|
||||
} else {
|
||||
litLength -= startPosInSequence;
|
||||
}
|
||||
/* Move to the next sequence */
|
||||
endPosInSequence -= currSeq.litLength + currSeq.matchLength;
|
||||
startPosInSequence = 0;
|
||||
} else {
|
||||
/* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence
|
||||
does not reach the end of the match. So, we have to split the sequence */
|
||||
DEBUGLOG(6, "Require a split: diff: %u, idx: %u PIS: %u",
|
||||
currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence);
|
||||
if (endPosInSequence > litLength) {
|
||||
U32 firstHalfMatchLength;
|
||||
litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence;
|
||||
firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength;
|
||||
if (matchLength > blockSize && firstHalfMatchLength >= cctx->appliedParams.cParams.minMatch) {
|
||||
/* Only ever split the match if it is larger than the block size */
|
||||
U32 secondHalfMatchLength = currSeq.matchLength + currSeq.litLength - endPosInSequence;
|
||||
if (secondHalfMatchLength < cctx->appliedParams.cParams.minMatch) {
|
||||
/* Move the endPosInSequence backward so that it creates match of minMatch length */
|
||||
endPosInSequence -= cctx->appliedParams.cParams.minMatch - secondHalfMatchLength;
|
||||
bytesAdjustment = cctx->appliedParams.cParams.minMatch - secondHalfMatchLength;
|
||||
firstHalfMatchLength -= bytesAdjustment;
|
||||
}
|
||||
matchLength = firstHalfMatchLength;
|
||||
/* Flag that we split the last match - after storing the sequence, exit the loop,
|
||||
but keep the value of endPosInSequence */
|
||||
finalMatchSplit = 1;
|
||||
} else {
|
||||
/* Move the position in sequence backwards so that we don't split match, and break to store
|
||||
* the last literals. We use the original currSeq.litLength as a marker for where endPosInSequence
|
||||
* should go. We prefer to do this whenever it is not necessary to split the match, or if doing so
|
||||
* would cause the first half of the match to be too small
|
||||
*/
|
||||
bytesAdjustment = endPosInSequence - currSeq.litLength;
|
||||
endPosInSequence = currSeq.litLength;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
/* This sequence ends inside the literals, break to store the last literals */
|
||||
break;
|
||||
}
|
||||
}
|
||||
/* Check if this offset can be represented with a repcode */
|
||||
{ U32 const ll0 = (litLength == 0);
|
||||
offBase = ZSTD_finalizeOffBase(rawOffset, updatedRepcodes.rep, ll0);
|
||||
ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0);
|
||||
}
|
||||
|
||||
if (cctx->appliedParams.validateSequences) {
|
||||
seqPos->posInSrc += litLength + matchLength;
|
||||
FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc,
|
||||
cctx->appliedParams.cParams.windowLog, dictSize, ZSTD_hasExtSeqProd(&cctx->appliedParams)),
|
||||
"Sequence validation failed");
|
||||
}
|
||||
DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
|
||||
RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
|
||||
"Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
|
||||
ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
|
||||
ip += matchLength + litLength;
|
||||
if (!finalMatchSplit)
|
||||
idx++; /* Next Sequence */
|
||||
}
|
||||
DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength);
|
||||
assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength);
|
||||
seqPos->idx = idx;
|
||||
seqPos->posInSequence = endPosInSequence;
|
||||
ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t));
|
||||
|
||||
iend -= bytesAdjustment;
|
||||
if (ip != iend) {
|
||||
/* Store any last literals */
|
||||
U32 const lastLLSize = (U32)(iend - ip);
|
||||
assert(ip <= iend);
|
||||
DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize);
|
||||
ZSTD_storeLastLiterals(&cctx->seqStore, ip, lastLLSize);
|
||||
seqPos->posInSrc += lastLLSize;
|
||||
}
|
||||
|
||||
return (size_t)(iend-istart);
|
||||
(void)externalRepSearch;
|
||||
return ZSTD_rust_transferSequencesNoDelim(
|
||||
&cctx->seqStore, seqPos, inSeqs, inSeqsSize,
|
||||
(const BYTE*)src, blockSize,
|
||||
cctx->blockState.prevCBlock->rep,
|
||||
cctx->blockState.nextCBlock->rep, (U32)dictSize,
|
||||
cctx->appliedParams.validateSequences,
|
||||
cctx->appliedParams.cParams.minMatch,
|
||||
cctx->appliedParams.cParams.windowLog,
|
||||
ZSTD_hasExtSeqProd(&cctx->appliedParams));
|
||||
}
|
||||
|
||||
/* @seqPos represents a position within @inSeqs,
|
||||
|
||||
@@ -784,6 +784,183 @@ pub unsafe extern "C" fn ZSTD_rust_transferSequencesWBlockDelim(
|
||||
block_size
|
||||
}
|
||||
|
||||
/// Transfers externally produced sequences through a block-size boundary when
|
||||
/// the producer did not provide explicit delimiters. A block may end in the
|
||||
/// middle of a match; in that case the sequence position is retained for the
|
||||
/// next call. This is the Rust leaf for
|
||||
/// `ZSTD_transferSequences_noDelim()`.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_transferSequencesNoDelim(
|
||||
seq_store: *mut SeqStore_t,
|
||||
seq_pos: *mut ZSTD_SequencePosition,
|
||||
in_seqs: *const ZSTD_Sequence,
|
||||
in_seqs_size: usize,
|
||||
src: *const u8,
|
||||
block_size: usize,
|
||||
prev_repcodes: *const u32,
|
||||
next_repcodes: *mut u32,
|
||||
dict_size: u32,
|
||||
validate_sequences: c_int,
|
||||
min_match: u32,
|
||||
window_log: u32,
|
||||
use_sequence_producer: c_int,
|
||||
) -> usize {
|
||||
if seq_store.is_null()
|
||||
|| seq_pos.is_null()
|
||||
|| in_seqs.is_null()
|
||||
|| src.is_null()
|
||||
|| prev_repcodes.is_null()
|
||||
|| next_repcodes.is_null()
|
||||
|| block_size > u32::MAX as usize
|
||||
{
|
||||
return ERROR(ZstdErrorCode::ExternalSequencesInvalid);
|
||||
}
|
||||
|
||||
let seq_store = unsafe { &mut *seq_store };
|
||||
let seq_pos = unsafe { &mut *seq_pos };
|
||||
let start_idx = seq_pos.idx as usize;
|
||||
if start_idx > in_seqs_size {
|
||||
return ERROR(ZstdErrorCode::ExternalSequencesInvalid);
|
||||
}
|
||||
|
||||
let block_size_u32 = block_size as u32;
|
||||
let mut start_pos_in_sequence = seq_pos.posInSequence;
|
||||
let mut end_pos_in_sequence = start_pos_in_sequence.wrapping_add(block_size_u32);
|
||||
let mut source_offset = 0usize;
|
||||
let mut bytes_adjustment = 0u32;
|
||||
let mut final_match_split = false;
|
||||
let mut updated_repcodes = [0u32; ZSTD_REP_NUM];
|
||||
unsafe {
|
||||
ptr::copy_nonoverlapping(prev_repcodes, updated_repcodes.as_mut_ptr(), ZSTD_REP_NUM);
|
||||
}
|
||||
|
||||
let mut idx = start_idx;
|
||||
while end_pos_in_sequence != 0 && idx < in_seqs_size && !final_match_split {
|
||||
let current_sequence = unsafe { *in_seqs.add(idx) };
|
||||
let mut lit_length = current_sequence.litLength;
|
||||
let mut match_length = current_sequence.matchLength;
|
||||
let raw_offset = current_sequence.offset;
|
||||
let current_size = current_sequence
|
||||
.litLength
|
||||
.wrapping_add(current_sequence.matchLength);
|
||||
|
||||
if end_pos_in_sequence >= current_size {
|
||||
if start_pos_in_sequence >= lit_length {
|
||||
start_pos_in_sequence = start_pos_in_sequence.wrapping_sub(lit_length);
|
||||
lit_length = 0;
|
||||
match_length = match_length.wrapping_sub(start_pos_in_sequence);
|
||||
} else {
|
||||
lit_length = lit_length.wrapping_sub(start_pos_in_sequence);
|
||||
}
|
||||
end_pos_in_sequence = end_pos_in_sequence.wrapping_sub(current_size);
|
||||
start_pos_in_sequence = 0;
|
||||
} else {
|
||||
/* The block ends inside this sequence. */
|
||||
if end_pos_in_sequence > lit_length {
|
||||
lit_length = if start_pos_in_sequence >= lit_length {
|
||||
0
|
||||
} else {
|
||||
lit_length.wrapping_sub(start_pos_in_sequence)
|
||||
};
|
||||
let first_half_match_length = end_pos_in_sequence
|
||||
.wrapping_sub(start_pos_in_sequence)
|
||||
.wrapping_sub(lit_length);
|
||||
if match_length > block_size_u32 && first_half_match_length >= min_match {
|
||||
let second_half_match_length = current_size.wrapping_sub(end_pos_in_sequence);
|
||||
let mut first_half_match_length = first_half_match_length;
|
||||
if second_half_match_length < min_match {
|
||||
let adjustment = min_match.wrapping_sub(second_half_match_length);
|
||||
end_pos_in_sequence = end_pos_in_sequence.wrapping_sub(adjustment);
|
||||
bytes_adjustment = adjustment;
|
||||
first_half_match_length = first_half_match_length.wrapping_sub(adjustment);
|
||||
}
|
||||
match_length = first_half_match_length;
|
||||
final_match_split = true;
|
||||
} else {
|
||||
bytes_adjustment = end_pos_in_sequence.wrapping_sub(current_sequence.litLength);
|
||||
end_pos_in_sequence = current_sequence.litLength;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
/* The block ends inside the literals; leave them as literals. */
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let ll0 = lit_length == 0;
|
||||
let off_base = unsafe {
|
||||
ZSTD_rust_finalizeOffBase(raw_offset, updated_repcodes.as_ptr(), u32::from(ll0))
|
||||
};
|
||||
update_rep(&mut updated_repcodes, off_base, ll0);
|
||||
|
||||
if validate_sequences != 0 {
|
||||
let sequence_size = lit_length.wrapping_add(match_length) as usize;
|
||||
seq_pos.posInSrc = seq_pos.posInSrc.wrapping_add(sequence_size);
|
||||
let validation = ZSTD_rust_validateSequence(
|
||||
off_base,
|
||||
match_length,
|
||||
min_match,
|
||||
seq_pos.posInSrc,
|
||||
window_log,
|
||||
dict_size as usize,
|
||||
use_sequence_producer,
|
||||
);
|
||||
if ERR_isError(validation) {
|
||||
return validation;
|
||||
}
|
||||
}
|
||||
|
||||
if idx - start_idx >= seq_store.maxNbSeq {
|
||||
return ERROR(ZstdErrorCode::ExternalSequencesInvalid);
|
||||
}
|
||||
let stored = unsafe {
|
||||
store_external_sequence(
|
||||
seq_store,
|
||||
src,
|
||||
source_offset,
|
||||
block_size,
|
||||
lit_length as usize,
|
||||
off_base,
|
||||
match_length as usize,
|
||||
)
|
||||
};
|
||||
if !stored {
|
||||
return ERROR(ZstdErrorCode::ExternalSequencesInvalid);
|
||||
}
|
||||
let sequence_size = lit_length.wrapping_add(match_length) as usize;
|
||||
source_offset = match source_offset.checked_add(sequence_size) {
|
||||
Some(offset) => offset,
|
||||
None => return ERROR(ZstdErrorCode::ExternalSequencesInvalid),
|
||||
};
|
||||
if !final_match_split {
|
||||
idx += 1;
|
||||
}
|
||||
}
|
||||
|
||||
seq_pos.idx = idx as u32;
|
||||
seq_pos.posInSequence = end_pos_in_sequence;
|
||||
unsafe {
|
||||
ptr::copy_nonoverlapping(updated_repcodes.as_ptr(), next_repcodes, ZSTD_REP_NUM);
|
||||
}
|
||||
|
||||
let consumed_size = match block_size.checked_sub(bytes_adjustment as usize) {
|
||||
Some(size) => size,
|
||||
None => return ERROR(ZstdErrorCode::ExternalSequencesInvalid),
|
||||
};
|
||||
if source_offset > consumed_size {
|
||||
return ERROR(ZstdErrorCode::ExternalSequencesInvalid);
|
||||
}
|
||||
if source_offset != consumed_size {
|
||||
let last_literal_size = consumed_size - source_offset;
|
||||
unsafe {
|
||||
ZSTD_rust_storeLastLiterals(seq_store, src.add(source_offset), last_literal_size);
|
||||
}
|
||||
seq_pos.posInSrc = seq_pos.posInSrc.wrapping_add(last_literal_size);
|
||||
}
|
||||
consumed_size
|
||||
}
|
||||
|
||||
/// Finds the next explicit block delimiter and returns the represented size.
|
||||
///
|
||||
/// The scan is half-open at `inSeqsSize`: a delimiter at the final element is
|
||||
@@ -2318,6 +2495,22 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn empty_external_seq_store(sequences: &mut [SeqDef], literals: &mut [u8]) -> SeqStore_t {
|
||||
SeqStore_t {
|
||||
sequencesStart: sequences.as_mut_ptr(),
|
||||
sequences: sequences.as_mut_ptr(),
|
||||
litStart: literals.as_mut_ptr(),
|
||||
lit: literals.as_mut_ptr(),
|
||||
llCode: std::ptr::null_mut(),
|
||||
mlCode: std::ptr::null_mut(),
|
||||
ofCode: std::ptr::null_mut(),
|
||||
maxNbSeq: sequences.len(),
|
||||
maxNbLit: literals.len(),
|
||||
longLengthType: 0,
|
||||
longLengthPos: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn c_leaf_layouts_match_supported_abis() {
|
||||
assert_eq!(size_of::<SeqDef>(), 8);
|
||||
@@ -2605,6 +2798,180 @@ mod tests {
|
||||
assert_eq!(literals, [9]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_sequences_no_delim_splits_a_match_at_the_block_boundary() {
|
||||
let input = [ZSTD_Sequence {
|
||||
offset: 50,
|
||||
litLength: 0,
|
||||
matchLength: 10,
|
||||
rep: 0,
|
||||
}];
|
||||
let source = [1u8, 2, 3, 4, 5, 6];
|
||||
let mut output = [SeqDef::default(); 1];
|
||||
let mut literals = [0u8; 1];
|
||||
let mut seq_store = empty_external_seq_store(&mut output, &mut literals);
|
||||
let mut position = ZSTD_SequencePosition::default();
|
||||
let previous_repcodes = [7u32, 20, 30];
|
||||
let mut next_repcodes = [0u32; ZSTD_REP_NUM];
|
||||
|
||||
let result = unsafe {
|
||||
ZSTD_rust_transferSequencesNoDelim(
|
||||
&mut seq_store,
|
||||
&mut position,
|
||||
input.as_ptr(),
|
||||
input.len(),
|
||||
source.as_ptr(),
|
||||
source.len(),
|
||||
previous_repcodes.as_ptr(),
|
||||
next_repcodes.as_mut_ptr(),
|
||||
0,
|
||||
0,
|
||||
3,
|
||||
10,
|
||||
0,
|
||||
)
|
||||
};
|
||||
|
||||
assert_eq!(result, source.len());
|
||||
assert_eq!(position.idx, 0);
|
||||
assert_eq!(position.posInSequence, source.len() as u32);
|
||||
assert_eq!(position.posInSrc, 0);
|
||||
assert_eq!(output[0].offBase, 50 + ZSTD_REP_NUM as u32);
|
||||
assert_eq!(output[0].litLength, 0);
|
||||
assert_eq!(output[0].mlBase, source.len() as u16 - MINMATCH as u16);
|
||||
assert_eq!(next_repcodes, [50, 7, 20]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_sequences_no_delim_shortens_a_match_to_keep_the_tail_valid() {
|
||||
let input = [ZSTD_Sequence {
|
||||
offset: 50,
|
||||
litLength: 0,
|
||||
matchLength: 10,
|
||||
rep: 0,
|
||||
}];
|
||||
let source = [0u8; 8];
|
||||
let mut output = [SeqDef::default(); 1];
|
||||
let mut literals = [0u8; 1];
|
||||
let mut seq_store = empty_external_seq_store(&mut output, &mut literals);
|
||||
let mut position = ZSTD_SequencePosition::default();
|
||||
let previous_repcodes = [7u32, 20, 30];
|
||||
let mut next_repcodes = [0u32; ZSTD_REP_NUM];
|
||||
|
||||
let result = unsafe {
|
||||
ZSTD_rust_transferSequencesNoDelim(
|
||||
&mut seq_store,
|
||||
&mut position,
|
||||
input.as_ptr(),
|
||||
input.len(),
|
||||
source.as_ptr(),
|
||||
source.len(),
|
||||
previous_repcodes.as_ptr(),
|
||||
next_repcodes.as_mut_ptr(),
|
||||
0,
|
||||
0,
|
||||
3,
|
||||
10,
|
||||
0,
|
||||
)
|
||||
};
|
||||
|
||||
assert_eq!(result, 7);
|
||||
assert_eq!(position.idx, 0);
|
||||
assert_eq!(position.posInSequence, 7);
|
||||
assert_eq!(output[0].offBase, 50 + ZSTD_REP_NUM as u32);
|
||||
assert_eq!(output[0].mlBase, 7 - MINMATCH as u16);
|
||||
assert_eq!(next_repcodes, [50, 7, 20]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_sequences_no_delim_leaves_a_literal_prefix_unsequenced() {
|
||||
let input = [ZSTD_Sequence {
|
||||
offset: 50,
|
||||
litLength: 4,
|
||||
matchLength: 6,
|
||||
rep: 0,
|
||||
}];
|
||||
let source = [1u8, 2, 3, 4, 5, 6];
|
||||
let mut output = [SeqDef::default(); 1];
|
||||
let mut literals = [0u8; 2];
|
||||
let mut seq_store = empty_external_seq_store(&mut output, &mut literals);
|
||||
let mut position = ZSTD_SequencePosition::default();
|
||||
let previous_repcodes = [7u32, 20, 30];
|
||||
let mut next_repcodes = [0u32; ZSTD_REP_NUM];
|
||||
|
||||
let result = unsafe {
|
||||
ZSTD_rust_transferSequencesNoDelim(
|
||||
&mut seq_store,
|
||||
&mut position,
|
||||
input.as_ptr(),
|
||||
input.len(),
|
||||
source.as_ptr(),
|
||||
2,
|
||||
previous_repcodes.as_ptr(),
|
||||
next_repcodes.as_mut_ptr(),
|
||||
0,
|
||||
0,
|
||||
3,
|
||||
10,
|
||||
0,
|
||||
)
|
||||
};
|
||||
|
||||
assert_eq!(result, 2);
|
||||
assert_eq!(position.idx, 0);
|
||||
assert_eq!(position.posInSequence, 2);
|
||||
assert_eq!(position.posInSrc, 2);
|
||||
assert_eq!(seq_store.sequences, seq_store.sequencesStart);
|
||||
assert_eq!(literals, [1, 2]);
|
||||
assert_eq!(next_repcodes, previous_repcodes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_sequences_no_delim_treats_bytes_after_the_last_sequence_as_literals() {
|
||||
let input = [ZSTD_Sequence {
|
||||
offset: 50,
|
||||
litLength: 2,
|
||||
matchLength: 3,
|
||||
rep: 0,
|
||||
}];
|
||||
let source = [10u8, 11, 12, 13, 14, 15, 16];
|
||||
let mut output = [SeqDef::default(); 1];
|
||||
let mut literals = [0u8; 4];
|
||||
let mut seq_store = empty_external_seq_store(&mut output, &mut literals);
|
||||
let mut position = ZSTD_SequencePosition::default();
|
||||
let previous_repcodes = [7u32, 20, 30];
|
||||
let mut next_repcodes = [0u32; ZSTD_REP_NUM];
|
||||
|
||||
let result = unsafe {
|
||||
ZSTD_rust_transferSequencesNoDelim(
|
||||
&mut seq_store,
|
||||
&mut position,
|
||||
input.as_ptr(),
|
||||
input.len(),
|
||||
source.as_ptr(),
|
||||
source.len(),
|
||||
previous_repcodes.as_ptr(),
|
||||
next_repcodes.as_mut_ptr(),
|
||||
0,
|
||||
0,
|
||||
3,
|
||||
10,
|
||||
0,
|
||||
)
|
||||
};
|
||||
|
||||
assert_eq!(result, source.len());
|
||||
assert_eq!(position.idx, 1);
|
||||
assert_eq!(position.posInSequence, 2);
|
||||
assert_eq!(position.posInSrc, 2);
|
||||
assert_eq!(output[0].offBase, 50 + ZSTD_REP_NUM as u32);
|
||||
assert_eq!(output[0].litLength, 2);
|
||||
assert_eq!(output[0].mlBase, 0);
|
||||
assert_eq!(literals, [10, 11, 15, 16]);
|
||||
assert_eq!(next_repcodes, [50, 7, 20]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_sequences_rejects_invalid_match_lengths_and_block_mismatches() {
|
||||
let invalid_match = [
|
||||
|
||||
Reference in New Issue
Block a user