ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move explicit sequence transfer to Rust

Browse Source 
Move ZSTD_transferSequences_wBlockDelim's delimiter scan, literal copying, SeqDef storage, validation, and repcode bookkeeping into the Rust compression statistics module. Keep ZSTD_CCtx and block-state ownership in C through a narrow scalar/pointer shim, with ABI layout assertions for ZSTD_SequencePosition.

Test Plan:

- cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression

- cargo clippy --manifest-path rust/Cargo.toml --no-default-features --features compression --benches --tests

- make -B -C lib -j2 lib

- make -C tests -j2 test-zstream
This commit is contained in:
ddidderr
2026-07-18 07:24:18 +02:00
parent 511882e213
commit 77841fcfd2
2 changed files with 574 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+20 -71
View File
@@ -303,6 +303,14 @@ size_t ZSTD_rust_determineBlockSize(int mode, size_t blockSize, size_t remaining
size_t ZSTD_rust_validateSequence(U32 offBase, U32 matchLength, U32 minMatch,
size_t posInSrc, U32 windowLog, size_t dictSize,
int useSequenceProducer);
size_t ZSTD_rust_transferSequencesWBlockDelim(
SeqStore_t* seqStore, ZSTD_SequencePosition* seqPos,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
const BYTE* src, size_t blockSize, int externalRepSearch,
const U32 prevRepcodes[ZSTD_REP_NUM],
U32 nextRepcodes[ZSTD_REP_NUM], U32 dictSize,
int validateSequences, U32 minMatch, U32 windowLog,
int useSequenceProducer);
size_t ZSTD_rust_optimalBlockSize(const void* src, size_t srcSize,
size_t blockSizeMax, int splitLevel,
int strategy, S64 savings,
@@ -330,6 +338,9 @@ typedef char ZSTD_rust_stats_entropy_metadata_layout[
(offsetof(ZSTD_entropyCTablesMetadata_t, fseMetadata)
== sizeof(ZSTD_hufCTablesMetadata_t)) ? 1 : -1];
typedef char ZSTD_rust_stats_sequence_layout[(sizeof(ZSTD_Sequence) == 16) ? 1 : -1];
typedef char ZSTD_rust_stats_sequence_position_layout[
(sizeof(ZSTD_SequencePosition) == 2 * sizeof(U32) + sizeof(size_t)
&& offsetof(ZSTD_SequencePosition, posInSrc) == 2 * sizeof(U32)) ? 1 : -1];
typedef char ZSTD_rust_stats_seqcollector_layout[
(offsetof(SeqCollector, seqStart) == sizeof(size_t)
&& offsetof(SeqCollector, seqIndex) == 2 * sizeof(size_t)
@@ -5244,15 +5255,8 @@ ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx,
const void* src, size_t blockSize,
ZSTD_ParamSwitch_e externalRepSearch)
{
U32 idx = seqPos->idx;
U32 const startIdx = idx;
BYTE const* ip = (BYTE const*)(src);
const BYTE* const iend = ip + blockSize;
Repcodes_t updatedRepcodes;
U32 dictSize;
DEBUGLOG(5, "ZSTD_transferSequences_wBlockDelim (blockSize = %zu)", blockSize);
if (cctx->cdict) {
dictSize = (U32)cctx->cdict->dictContentSize;
} else if (cctx->prefixDict.dict) {
@@ -5260,70 +5264,15 @@ ZSTD_transferSequences_wBlockDelim(ZSTD_CCtx* cctx,
} else {
dictSize = 0;
}
ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(Repcodes_t));
for (; idx < inSeqsSize && (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0); ++idx) {
U32 const litLength = inSeqs[idx].litLength;
U32 const matchLength = inSeqs[idx].matchLength;
U32 offBase;
if (externalRepSearch == ZSTD_ps_disable) {
offBase = OFFSET_TO_OFFBASE(inSeqs[idx].offset);
} else {
U32 const ll0 = (litLength == 0);
offBase = ZSTD_finalizeOffBase(inSeqs[idx].offset, updatedRepcodes.rep, ll0);
ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0);
}
DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
if (cctx->appliedParams.validateSequences) {
seqPos->posInSrc += litLength + matchLength;
FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch,
seqPos->posInSrc,
cctx->appliedParams.cParams.windowLog, dictSize,
ZSTD_hasExtSeqProd(&cctx->appliedParams)),
"Sequence validation failed");
}
RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
"Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
ip += matchLength + litLength;
}
RETURN_ERROR_IF(idx == inSeqsSize, externalSequences_invalid, "Block delimiter not found.");
/* If we skipped repcode search while parsing, we need to update repcodes now */
assert(externalRepSearch != ZSTD_ps_auto);
assert(idx >= startIdx);
if (externalRepSearch == ZSTD_ps_disable && idx != startIdx) {
U32* const rep = updatedRepcodes.rep;
U32 lastSeqIdx = idx - 1; /* index of last non-block-delimiter sequence */
if (lastSeqIdx >= startIdx + 2) {
rep[2] = inSeqs[lastSeqIdx - 2].offset;
rep[1] = inSeqs[lastSeqIdx - 1].offset;
rep[0] = inSeqs[lastSeqIdx].offset;
} else if (lastSeqIdx == startIdx + 1) {
rep[2] = rep[0];
rep[1] = inSeqs[lastSeqIdx - 1].offset;
rep[0] = inSeqs[lastSeqIdx].offset;
} else {
assert(lastSeqIdx == startIdx);
rep[2] = rep[1];
rep[1] = rep[0];
rep[0] = inSeqs[lastSeqIdx].offset;
}
}
ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(Repcodes_t));
if (inSeqs[idx].litLength) {
DEBUGLOG(6, "Storing last literals of size: %u", inSeqs[idx].litLength);
ZSTD_storeLastLiterals(&cctx->seqStore, ip, inSeqs[idx].litLength);
ip += inSeqs[idx].litLength;
seqPos->posInSrc += inSeqs[idx].litLength;
}
RETURN_ERROR_IF(ip != iend, externalSequences_invalid, "Blocksize doesn't agree with block delimiter!");
seqPos->idx = idx+1;
return blockSize;
return ZSTD_rust_transferSequencesWBlockDelim(
&cctx->seqStore, seqPos, inSeqs, inSeqsSize,
(const BYTE*)src, blockSize, (int)externalRepSearch,
cctx->blockState.prevCBlock->rep,
cctx->blockState.nextCBlock->rep, dictSize,
cctx->appliedParams.validateSequences,
cctx->appliedParams.cParams.minMatch,
cctx->appliedParams.cParams.windowLog,
ZSTD_hasExtSeqProd(&cctx->appliedParams));
}
/*
rust/src/zstd_compress_stats.rs
+554
View File
@@ -44,6 +44,9 @@ const SET_RLE: c_int = 1;
const SET_COMPRESSED: c_int = 2;
const SET_REPEAT: c_int = 3;
const ZSTD_PS_AUTO: c_int = 0;
const ZSTD_PS_DISABLE: c_int = 2;
const FSE_REPEAT_NONE: c_int = 0;
const HUF_REPEAT_NONE: c_int = 0;
@@ -60,6 +63,7 @@ const HUF_SYMBOLVALUE_MAX: c_uint = 255;
const LIT_HUF_LOG: c_uint = 11;
const COMPRESS_LITERALS_SIZE_MIN: usize = 63;
const SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO: usize = 20;
const ZSTD_BLOCKSIZE_MAX: usize = 1 << 17;
const STREAM_ACCUMULATOR_MIN_32: u32 = 25;
const STREAM_ACCUMULATOR_MIN_64: u32 = 57;
@@ -159,6 +163,15 @@ pub struct ZSTD_Sequence {
pub rep: u32,
}
/// ABI-compatible `ZSTD_SequencePosition` from `zstd_compress_internal.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
pub struct ZSTD_SequencePosition {
pub idx: u32,
pub posInSequence: u32,
pub posInSrc: usize,
}
/// ABI-compatible `BlockSummary` from `zstd_compress_internal.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
@@ -539,6 +552,238 @@ pub extern "C" fn ZSTD_rust_validateSequence(
0
}
/// Stores one externally supplied sequence and copies its literal prefix into
/// the sequence store. This is the Rust equivalent of the C-only
/// `ZSTD_storeSeq()` helper used by the explicit-block-delimiter path.
unsafe fn store_external_sequence(
seq_store: &mut SeqStore_t,
src: *const u8,
source_offset: usize,
block_size: usize,
lit_length: usize,
off_base: u32,
match_length: usize,
) -> bool {
let sequence_index = unsafe { seq_store.sequences.offset_from(seq_store.sequencesStart) };
if sequence_index < 0 || sequence_index as usize >= seq_store.maxNbSeq {
return false;
}
let sequence_index = sequence_index as usize;
let sequence_size = match lit_length.checked_add(match_length) {
Some(size) => size,
None => return false,
};
let source_end = match source_offset.checked_add(sequence_size) {
Some(end) if end <= block_size => end,
_ => return false,
};
let literal_count = unsafe { seq_store.lit.offset_from(seq_store.litStart) };
if literal_count < 0
|| (literal_count as usize)
.checked_add(lit_length)
.is_none_or(|end| end > seq_store.maxNbLit)
{
return false;
}
let ml_base = match match_length.checked_sub(MINMATCH) {
Some(base) => base,
None => return false,
};
debug_assert!(source_end <= block_size);
debug_assert!(lit_length <= ZSTD_BLOCKSIZE_MAX);
debug_assert!(match_length <= ZSTD_BLOCKSIZE_MAX);
debug_assert!(seq_store.maxNbLit <= 128 << 10);
unsafe {
ptr::copy_nonoverlapping(src.add(source_offset), seq_store.lit, lit_length);
seq_store.lit = seq_store.lit.add(lit_length);
}
let sequence = unsafe { &mut *seq_store.sequences };
if lit_length > u16::MAX as usize {
debug_assert_eq!(seq_store.longLengthType, 0);
seq_store.longLengthType = ZSTD_LLT_LITERAL_LENGTH;
seq_store.longLengthPos = sequence_index as u32;
}
sequence.litLength = lit_length as u16;
sequence.offBase = off_base;
if ml_base > u16::MAX as usize {
debug_assert_eq!(seq_store.longLengthType, 0);
seq_store.longLengthType = ZSTD_LLT_MATCH_LENGTH;
seq_store.longLengthPos = sequence_index as u32;
}
sequence.mlBase = ml_base as u16;
unsafe {
seq_store.sequences = seq_store.sequences.add(1);
}
true
}
/// Transfers externally produced sequences through one explicit block
/// delimiter. The C wrapper supplies only the state extracted from
/// `ZSTD_CCtx`; the sequence-store mutation and delimiter bookkeeping live in
/// Rust. This is the Rust leaf for
/// `ZSTD_transferSequences_wBlockDelim()`.
#[allow(clippy::too_many_arguments)]
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_transferSequencesWBlockDelim(
seq_store: *mut SeqStore_t,
seq_pos: *mut ZSTD_SequencePosition,
in_seqs: *const ZSTD_Sequence,
in_seqs_size: usize,
src: *const u8,
block_size: usize,
external_rep_search: c_int,
prev_repcodes: *const u32,
next_repcodes: *mut u32,
dict_size: u32,
validate_sequences: c_int,
min_match: u32,
window_log: u32,
use_sequence_producer: c_int,
) -> usize {
if seq_store.is_null()
|| seq_pos.is_null()
|| in_seqs.is_null()
|| src.is_null()
|| prev_repcodes.is_null()
|| next_repcodes.is_null()
{
return ERROR(ZstdErrorCode::ExternalSequencesInvalid);
}
let seq_store = unsafe { &mut *seq_store };
let seq_pos = unsafe { &mut *seq_pos };
if (seq_pos.idx as usize) > in_seqs_size {
return ERROR(ZstdErrorCode::ExternalSequencesInvalid);
}
let mut updated_repcodes = [0u32; ZSTD_REP_NUM];
unsafe {
ptr::copy_nonoverlapping(prev_repcodes, updated_repcodes.as_mut_ptr(), ZSTD_REP_NUM);
}
let start_idx = seq_pos.idx as usize;
let mut idx = start_idx;
let mut source_offset = 0usize;
while idx < in_seqs_size {
let sequence = unsafe { *in_seqs.add(idx) };
if sequence.matchLength == 0 && sequence.offset == 0 {
break;
}
let lit_length = sequence.litLength as usize;
let match_length = sequence.matchLength as usize;
let off_base = if external_rep_search == ZSTD_PS_DISABLE {
if sequence.offset == 0 {
return ERROR(ZstdErrorCode::ExternalSequencesInvalid);
}
sequence.offset.wrapping_add(ZSTD_REP_NUM as u32)
} else {
let ll0 = lit_length == 0;
let off_base = unsafe {
ZSTD_rust_finalizeOffBase(
sequence.offset,
updated_repcodes.as_ptr(),
u32::from(ll0),
)
};
update_rep(&mut updated_repcodes, off_base, ll0);
off_base
};
if validate_sequences != 0 {
seq_pos.posInSrc = seq_pos
.posInSrc
.wrapping_add(lit_length.wrapping_add(match_length));
let validation = ZSTD_rust_validateSequence(
off_base,
sequence.matchLength,
min_match,
seq_pos.posInSrc,
window_log,
dict_size as usize,
use_sequence_producer,
);
if ERR_isError(validation) {
return validation;
}
}
if idx - start_idx >= seq_store.maxNbSeq {
return ERROR(ZstdErrorCode::ExternalSequencesInvalid);
}
if !unsafe {
store_external_sequence(
seq_store,
src,
source_offset,
block_size,
lit_length,
off_base,
match_length,
)
} {
return ERROR(ZstdErrorCode::ExternalSequencesInvalid);
}
source_offset = source_offset
.checked_add(lit_length.wrapping_add(match_length))
.expect("validated external sequence source offset overflow");
idx += 1;
}
if idx == in_seqs_size {
return ERROR(ZstdErrorCode::ExternalSequencesInvalid);
}
debug_assert_ne!(external_rep_search, ZSTD_PS_AUTO);
/* If repcode search was skipped while parsing, recover the raw-offset
* history once the delimiter identifies the complete block. */
if external_rep_search == ZSTD_PS_DISABLE && idx != start_idx {
let last_seq_idx = idx - 1;
if last_seq_idx >= start_idx + 2 {
updated_repcodes[2] = unsafe { (*in_seqs.add(last_seq_idx - 2)).offset };
updated_repcodes[1] = unsafe { (*in_seqs.add(last_seq_idx - 1)).offset };
updated_repcodes[0] = unsafe { (*in_seqs.add(last_seq_idx)).offset };
} else if last_seq_idx == start_idx + 1 {
updated_repcodes[2] = updated_repcodes[0];
updated_repcodes[1] = unsafe { (*in_seqs.add(last_seq_idx - 1)).offset };
updated_repcodes[0] = unsafe { (*in_seqs.add(last_seq_idx)).offset };
} else {
debug_assert_eq!(last_seq_idx, start_idx);
updated_repcodes[2] = updated_repcodes[1];
updated_repcodes[1] = updated_repcodes[0];
updated_repcodes[0] = unsafe { (*in_seqs.add(last_seq_idx)).offset };
}
}
unsafe {
ptr::copy_nonoverlapping(updated_repcodes.as_ptr(), next_repcodes, ZSTD_REP_NUM);
}
let delimiter = unsafe { *in_seqs.add(idx) };
let delimiter_literals = delimiter.litLength as usize;
if delimiter_literals != 0 {
let delimiter_end = match source_offset.checked_add(delimiter_literals) {
Some(end) if end <= block_size => end,
_ => return ERROR(ZstdErrorCode::ExternalSequencesInvalid),
};
unsafe {
ZSTD_rust_storeLastLiterals(seq_store, src.add(source_offset), delimiter_literals);
}
source_offset = delimiter_end;
seq_pos.posInSrc = seq_pos.posInSrc.wrapping_add(delimiter_literals);
}
if source_offset != block_size {
return ERROR(ZstdErrorCode::ExternalSequencesInvalid);
}
seq_pos.idx = (idx + 1) as u32;
block_size
}
/// Finds the next explicit block delimiter and returns the represented size.
///
/// The scan is half-open at `inSeqsSize`: a delimiter at the final element is
@@ -2093,6 +2338,19 @@ mod tests {
size_of::<ZSTD_entropyCTables_t>()
);
assert_eq!(size_of::<ZSTD_Sequence>(), 16);
assert_eq!(offset_of!(ZSTD_SequencePosition, idx), 0);
assert_eq!(
offset_of!(ZSTD_SequencePosition, posInSequence),
size_of::<u32>()
);
assert_eq!(
offset_of!(ZSTD_SequencePosition, posInSrc),
2 * size_of::<u32>()
);
assert_eq!(
size_of::<ZSTD_SequencePosition>(),
2 * size_of::<u32>() + size_of::<usize>()
);
assert_eq!(offset_of!(SeqCollector, seqStart), size_of::<usize>());
assert_eq!(offset_of!(SeqCollector, seqIndex), 2 * size_of::<usize>());
assert_eq!(size_of::<SeqCollector>(), 4 * size_of::<usize>());
@@ -2221,6 +2479,302 @@ mod tests {
);
}
#[test]
fn transfer_sequences_with_disabled_rep_search_copies_delimiter_literals() {
let input = [
ZSTD_Sequence {
offset: 5,
litLength: 2,
matchLength: 3,
rep: 0,
},
ZSTD_Sequence {
offset: 0,
litLength: 2,
matchLength: 0,
rep: 0,
},
];
let source = [1u8, 2, 3, 4, 5, 6, 7];
let mut output = [SeqDef::default(); 2];
let mut literals = [0u8; 4];
let mut seq_store = SeqStore_t {
sequencesStart: output.as_mut_ptr(),
sequences: output.as_mut_ptr(),
litStart: literals.as_mut_ptr(),
lit: literals.as_mut_ptr(),
llCode: std::ptr::null_mut(),
mlCode: std::ptr::null_mut(),
ofCode: std::ptr::null_mut(),
maxNbSeq: output.len(),
maxNbLit: literals.len(),
longLengthType: 0,
longLengthPos: 0,
};
let mut position = ZSTD_SequencePosition::default();
let previous_repcodes = [5u32, 6, 7];
let mut next_repcodes = [0u32; ZSTD_REP_NUM];
let result = unsafe {
ZSTD_rust_transferSequencesWBlockDelim(
&mut seq_store,
&mut position,
input.as_ptr(),
input.len(),
source.as_ptr(),
source.len(),
ZSTD_PS_DISABLE,
previous_repcodes.as_ptr(),
next_repcodes.as_mut_ptr(),
0,
0,
4,
10,
0,
)
};
assert_eq!(result, source.len());
assert_eq!(position.idx, 2);
assert_eq!(position.posInSrc, 2);
assert_eq!(output[0].offBase, 5 + ZSTD_REP_NUM as u32);
assert_eq!(output[0].litLength, 2);
assert_eq!(output[0].mlBase, 0);
assert_eq!(literals, [1, 2, 6, 7]);
assert_eq!(next_repcodes, [5, 5, 6]);
}
#[test]
fn transfer_sequences_with_enabled_rep_search_uses_repcode_offsets() {
let input = [
ZSTD_Sequence {
offset: 5,
litLength: 1,
matchLength: 3,
rep: 0,
},
ZSTD_Sequence {
offset: 0,
litLength: 0,
matchLength: 0,
rep: 0,
},
];
let source = [9u8, 8, 7, 6];
let mut output = [SeqDef::default(); 2];
let mut literals = [0u8; 1];
let mut seq_store = SeqStore_t {
sequencesStart: output.as_mut_ptr(),
sequences: output.as_mut_ptr(),
litStart: literals.as_mut_ptr(),
lit: literals.as_mut_ptr(),
llCode: std::ptr::null_mut(),
mlCode: std::ptr::null_mut(),
ofCode: std::ptr::null_mut(),
maxNbSeq: output.len(),
maxNbLit: literals.len(),
longLengthType: 0,
longLengthPos: 0,
};
let mut position = ZSTD_SequencePosition::default();
let previous_repcodes = [5u32, 6, 7];
let mut next_repcodes = [0u32; ZSTD_REP_NUM];
let result = unsafe {
ZSTD_rust_transferSequencesWBlockDelim(
&mut seq_store,
&mut position,
input.as_ptr(),
input.len(),
source.as_ptr(),
source.len(),
1,
previous_repcodes.as_ptr(),
next_repcodes.as_mut_ptr(),
0,
0,
4,
10,
0,
)
};
assert_eq!(result, source.len());
assert_eq!(output[0].offBase, 1);
assert_eq!(next_repcodes, previous_repcodes);
assert_eq!(literals, [9]);
}
#[test]
fn transfer_sequences_rejects_invalid_match_lengths_and_block_mismatches() {
let invalid_match = [
ZSTD_Sequence {
offset: 1,
litLength: 0,
matchLength: 2,
rep: 0,
},
ZSTD_Sequence {
offset: 0,
litLength: 0,
matchLength: 0,
rep: 0,
},
];
let source = [1u8, 2];
let mut output = [SeqDef::default(); 2];
let mut literals = [0u8; 1];
let mut seq_store = SeqStore_t {
sequencesStart: output.as_mut_ptr(),
sequences: output.as_mut_ptr(),
litStart: literals.as_mut_ptr(),
lit: literals.as_mut_ptr(),
llCode: std::ptr::null_mut(),
mlCode: std::ptr::null_mut(),
ofCode: std::ptr::null_mut(),
maxNbSeq: output.len(),
maxNbLit: literals.len(),
longLengthType: 0,
longLengthPos: 0,
};
let mut position = ZSTD_SequencePosition::default();
let previous_repcodes = [5u32, 6, 7];
let mut next_repcodes = [0u32; ZSTD_REP_NUM];
let result = unsafe {
ZSTD_rust_transferSequencesWBlockDelim(
&mut seq_store,
&mut position,
invalid_match.as_ptr(),
invalid_match.len(),
source.as_ptr(),
source.len(),
ZSTD_PS_DISABLE,
previous_repcodes.as_ptr(),
next_repcodes.as_mut_ptr(),
0,
1,
4,
10,
0,
)
};
assert_eq!(result, ERROR(ZstdErrorCode::ExternalSequencesInvalid));
assert_eq!(position.posInSrc, source.len());
assert_eq!(seq_store.sequences, seq_store.sequencesStart);
let mismatch = [
ZSTD_Sequence {
offset: 1,
litLength: 1,
matchLength: 3,
rep: 0,
},
ZSTD_Sequence {
offset: 0,
litLength: 1,
matchLength: 0,
rep: 0,
},
];
let mut mismatch_store = SeqStore_t {
sequencesStart: output.as_mut_ptr(),
sequences: output.as_mut_ptr(),
litStart: literals.as_mut_ptr(),
lit: literals.as_mut_ptr(),
llCode: std::ptr::null_mut(),
mlCode: std::ptr::null_mut(),
ofCode: std::ptr::null_mut(),
maxNbSeq: output.len(),
maxNbLit: literals.len(),
longLengthType: 0,
longLengthPos: 0,
};
let mut mismatch_position = ZSTD_SequencePosition::default();
let result = unsafe {
ZSTD_rust_transferSequencesWBlockDelim(
&mut mismatch_store,
&mut mismatch_position,
mismatch.as_ptr(),
mismatch.len(),
source.as_ptr(),
source.len(),
ZSTD_PS_DISABLE,
previous_repcodes.as_ptr(),
next_repcodes.as_mut_ptr(),
0,
0,
4,
10,
0,
)
};
assert_eq!(result, ERROR(ZstdErrorCode::ExternalSequencesInvalid));
}
#[test]
fn transfer_sequences_preserves_a_long_literal_side_band() {
let literal_length = 65536usize;
let input = [
ZSTD_Sequence {
offset: 1,
litLength: literal_length as u32,
matchLength: 3,
rep: 0,
},
ZSTD_Sequence {
offset: 0,
litLength: 0,
matchLength: 0,
rep: 0,
},
];
let source = vec![0xA5u8; literal_length + 3];
let mut output = [SeqDef::default(); 2];
let mut literals = vec![0u8; literal_length];
let mut seq_store = SeqStore_t {
sequencesStart: output.as_mut_ptr(),
sequences: output.as_mut_ptr(),
litStart: literals.as_mut_ptr(),
lit: literals.as_mut_ptr(),
llCode: std::ptr::null_mut(),
mlCode: std::ptr::null_mut(),
ofCode: std::ptr::null_mut(),
maxNbSeq: output.len(),
maxNbLit: literals.len(),
longLengthType: 0,
longLengthPos: 0,
};
let mut position = ZSTD_SequencePosition::default();
let repcodes = [5u32, 6, 7];
let mut next_repcodes = [0u32; ZSTD_REP_NUM];
let result = unsafe {
ZSTD_rust_transferSequencesWBlockDelim(
&mut seq_store,
&mut position,
input.as_ptr(),
input.len(),
source.as_ptr(),
source.len(),
ZSTD_PS_DISABLE,
repcodes.as_ptr(),
next_repcodes.as_mut_ptr(),
0,
0,
4,
10,
0,
)
};
assert_eq!(result, source.len());
assert_eq!(output[0].litLength, 0);
assert_eq!(output[0].mlBase, 0);
assert_eq!(seq_store.longLengthType, ZSTD_LLT_LITERAL_LENGTH);
assert_eq!(seq_store.longLengthPos, 0);
assert_eq!(literals, source[..literal_length]);
}
#[test]
fn estimate_block_size_counts_basic_literals_and_rle_sequences() {
let entropy = ZSTD_entropyCTables_t {