ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move seq-store leaves to Rust

Browse Source 
Move compressed-block state reset, final-literal storage, sequence-store reset, sequence-length summation, RLE detection, and RLE heuristics into the Rust compression statistics module. Preserve the C helper surface and add focused pointer/layout and boundary tests.

Test Plan: cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression; cargo clippy --manifest-path rust/Cargo.toml; cargo clippy --manifest-path rust/Cargo.toml --benches; cargo clippy --manifest-path rust/Cargo.toml --tests; make -B -C lib -j2 lib; make -B -C programs -j2 zstd zstd-small zstd-frugal; make -B -C tests -j2 test-cli-tests; make -B -C tests -j2 test-zstream
This commit is contained in:
ddidderr
2026-07-18 03:55:57 +02:00
parent 8ac9855309
commit 1edec2cc0e
2 changed files with 304 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+14 -43
View File
@@ -233,6 +233,14 @@ size_t ZSTD_rust_buildBlockEntropyStats(
size_t ZSTD_rust_copyBlockSequences(
SeqCollector* seqCollector, const SeqStore_t* seqStore,
const U32 prevRepcodes[ZSTD_REP_NUM]);
void ZSTD_rust_resetCompressedBlockState(ZSTD_compressedBlockState_t* bs);
void ZSTD_rust_storeLastLiterals(SeqStore_t* seqStorePtr,
const BYTE* anchor, size_t lastLLSize);
void ZSTD_rust_resetSeqStore(SeqStore_t* ssPtr);
size_t ZSTD_rust_fastSequenceLengthSum(const ZSTD_Sequence* seqBuf,
size_t seqBufSize);
int ZSTD_rust_isRLE(const BYTE* src, size_t length);
int ZSTD_rust_maybeRLE(const SeqStore_t* seqStore);
typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
typedef char ZSTD_rust_stats_seqstore_long_length_pos[
@@ -1373,13 +1381,7 @@ static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1,
void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs)
{
int i;
for (i = 0; i < ZSTD_REP_NUM; ++i)
bs->rep[i] = repStartValue[i];
bs->entropy.huf.repeatMode = HUF_repeat_none;
bs->entropy.fse.offcode_repeatMode = FSE_repeat_none;
bs->entropy.fse.matchlength_repeatMode = FSE_repeat_none;
bs->entropy.fse.litlength_repeatMode = FSE_repeat_none;
ZSTD_rust_resetCompressedBlockState(bs);
}
/*! ZSTD_invalidateMatchState()
@@ -2307,15 +2309,12 @@ ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_Para
static void ZSTD_storeLastLiterals(SeqStore_t* seqStorePtr,
const BYTE* anchor, size_t lastLLSize)
{
ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize);
seqStorePtr->lit += lastLLSize;
ZSTD_rust_storeLastLiterals(seqStorePtr, anchor, lastLLSize);
}
void ZSTD_resetSeqStore(SeqStore_t* ssPtr)
{
ssPtr->lit = ssPtr->litStart;
ssPtr->sequences = ssPtr->sequencesStart;
ssPtr->longLengthType = ZSTD_llt_none;
ZSTD_rust_resetSeqStore(ssPtr);
}
/* ZSTD_postProcessSequenceProducerResult() :
@@ -2374,14 +2373,7 @@ static size_t ZSTD_postProcessSequenceProducerResult(
* Removing the early exit allows the compiler to auto-vectorize (https://godbolt.org/z/cY1cajz9P).
* This function can be deleted and replaced by determine_blockSize after we resolve issue #3456. */
static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seqBufSize) {
size_t matchLenSum, litLenSum, i;
matchLenSum = 0;
litLenSum = 0;
for (i = 0; i < seqBufSize; i++) {
litLenSum += seqBuf[i].litLength;
matchLenSum += seqBuf[i].matchLength;
}
return litLenSum + matchLenSum;
return ZSTD_rust_fastSequenceLengthSum(seqBuf, seqBufSize);
}
/**
@@ -2626,25 +2618,7 @@ size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
/* Unrolled loop to read four size_ts of input at a time. Returns 1 if is RLE, 0 if not. */
static int ZSTD_isRLE(const BYTE* src, size_t length) {
const BYTE* ip = src;
const BYTE value = ip[0];
const size_t valueST = (size_t)((U64)value * 0x0101010101010101ULL);
const size_t unrollSize = sizeof(size_t) * 4;
const size_t unrollMask = unrollSize - 1;
const size_t prefixLength = length & unrollMask;
size_t i;
if (length == 1) return 1;
/* Check if prefix is RLE first before using unrolled loop */
if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) {
return 0;
}
for (i = prefixLength; i != length; i += unrollSize) {
size_t u;
for (u = 0; u < unrollSize; u += sizeof(size_t)) {
if (MEM_readST(ip + i + u) != valueST) {
return 0;
} } }
return 1;
return ZSTD_rust_isRLE(src, length);
}
/* Returns true if the given block may be RLE.
@@ -2653,10 +2627,7 @@ static int ZSTD_isRLE(const BYTE* src, size_t length) {
*/
static int ZSTD_maybeRLE(SeqStore_t const* seqStore)
{
size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart);
return nbSeqs < 4 && nbLits < 10;
return ZSTD_rust_maybeRLE(seqStore);
}
static void
rust/src/zstd_compress_stats.rs
+290 -3
View File
@@ -17,8 +17,8 @@ use crate::bits::ZSTD_highbit32;
use crate::common::{
DEFAULT_MAX_OFF, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML,
MAX_OFF, MAX_SEQ, MINMATCH, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, ML_FSE_LOG, OFF_FSE_LOG,
OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, ZSTD_MAX_FSE_HEADERS_SIZE, ZSTD_MAX_HUF_HEADER_SIZE,
ZSTD_REP_NUM,
OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, REP_START_VALUE, ZSTD_MAX_FSE_HEADERS_SIZE,
ZSTD_MAX_HUF_HEADER_SIZE, ZSTD_REP_NUM,
};
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
use crate::hist::{HIST_countFast_wksp, HIST_count_wksp};
@@ -26,7 +26,7 @@ use crate::huf_compress::{
HUF_buildCTable_wksp, HUF_estimateCompressedSize, HUF_optimalTableLog, HUF_validateCTable,
HUF_writeCTable_wksp,
};
use crate::mem::{MEM_32bits, MEM_writeLE16};
use crate::mem::{MEM_32bits, MEM_readST, MEM_writeLE16};
use crate::zstd_compress_literals::{
copy_huf_tables, min_gain, ZSTD_compressLiterals, ZSTD_hufCTables_t,
};
@@ -158,6 +158,120 @@ pub struct ZSTD_Sequence {
pub rep: u32,
}
/// Resets the state that is carried from one compressed block to the next.
///
/// This is the direct Rust leaf for C's
/// `ZSTD_reset_compressedBlockState()`. The caller must provide a valid
/// mutable compressed block state.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_resetCompressedBlockState(
block_state: *mut ZSTD_compressedBlockState_t,
) {
let block_state = unsafe { &mut *block_state };
block_state.rep.copy_from_slice(&REP_START_VALUE);
block_state.entropy.huf.repeatMode = HUF_REPEAT_NONE;
block_state.entropy.fse.offcode_repeatMode = FSE_REPEAT_NONE;
block_state.entropy.fse.matchlength_repeatMode = FSE_REPEAT_NONE;
block_state.entropy.fse.litlength_repeatMode = FSE_REPEAT_NONE;
}
/// Appends the final literals of a block to its sequence store.
///
/// The source and destination ranges must be valid and non-overlapping, as
/// required by the original C `memcpy` call. `last_ll_size == 0` preserves
/// the C pointer-update behavior without changing any bytes.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_storeLastLiterals(
seq_store: *mut SeqStore_t,
anchor: *const u8,
last_ll_size: usize,
) {
let seq_store = unsafe { &mut *seq_store };
unsafe {
ptr::copy_nonoverlapping(anchor, seq_store.lit, last_ll_size);
seq_store.lit = seq_store.lit.add(last_ll_size);
}
}
/// Resets the sequence and literal cursors for a new block.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_resetSeqStore(seq_store: *mut SeqStore_t) {
let seq_store = unsafe { &mut *seq_store };
seq_store.lit = seq_store.litStart;
seq_store.sequences = seq_store.sequencesStart;
seq_store.longLengthType = 0;
}
/// Returns the sum of all literal and match lengths in an external sequence
/// buffer, without looking for a block delimiter.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_fastSequenceLengthSum(
seq_buf: *const ZSTD_Sequence,
seq_buf_size: usize,
) -> usize {
let mut match_len_sum = 0usize;
let mut lit_len_sum = 0usize;
for index in 0..seq_buf_size {
let sequence = unsafe { *seq_buf.add(index) };
lit_len_sum = lit_len_sum.wrapping_add(sequence.litLength as usize);
match_len_sum = match_len_sum.wrapping_add(sequence.matchLength as usize);
}
lit_len_sum.wrapping_add(match_len_sum)
}
/// Returns whether all bytes in the input have the same value.
///
/// This keeps the original C helper's precondition that `src` points to at
/// least one readable byte, even when `length == 0`: the first byte is read
/// before the length checks.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_isRLE(src: *const u8, length: usize) -> c_int {
let value = unsafe { *src };
let value_st = (u64::from(value) * 0x0101_0101_0101_0101u64) as usize;
let unroll_size = size_of::<usize>() * 4;
let prefix_length = length & (unroll_size - 1);
if length == 1 {
return 1;
}
let mut index = 1;
while index < prefix_length {
if unsafe { *src.add(index) } != value {
return 0;
}
index += 1;
}
let mut index = prefix_length;
while index != length {
let mut unroll_offset = 0;
while unroll_offset < unroll_size {
let word = unsafe { MEM_readST(src.add(index + unroll_offset).cast()) };
if word != value_st {
return 0;
}
unroll_offset += size_of::<usize>();
}
index += unroll_size;
}
1
}
/// Heuristic used to decide whether a sequence store may describe an RLE
/// block. It intentionally only inspects the two pointer distances.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_maybeRLE(seq_store: *const SeqStore_t) -> c_int {
let seq_store = unsafe { &*seq_store };
let nb_seqs = unsafe { seq_store.sequences.offset_from(seq_store.sequencesStart) } as usize;
let nb_lits = unsafe { seq_store.lit.offset_from(seq_store.litStart) } as usize;
if nb_seqs < 4 && nb_lits < 10 {
1
} else {
0
}
}
/// C's `SeqCollector` leaf from `zstd_compress_internal.h`.
#[repr(C)]
pub struct SeqCollector {
@@ -1311,6 +1425,179 @@ mod tests {
}
}
#[test]
fn compressed_block_state_reset_restores_repcodes_and_repeat_modes() {
let mut block_state =
unsafe { std::mem::MaybeUninit::<ZSTD_compressedBlockState_t>::zeroed().assume_init() };
block_state.rep = [99; ZSTD_REP_NUM];
block_state.entropy.huf.repeatMode = HUF_REPEAT_VALID;
block_state.entropy.fse.offcode_repeatMode = 2;
block_state.entropy.fse.matchlength_repeatMode = 1;
block_state.entropy.fse.litlength_repeatMode = 2;
unsafe { ZSTD_rust_resetCompressedBlockState(&mut block_state) };
assert_eq!(block_state.rep, [1, 4, 8]);
assert_eq!(block_state.entropy.huf.repeatMode, HUF_REPEAT_NONE);
assert_eq!(block_state.entropy.fse.offcode_repeatMode, FSE_REPEAT_NONE);
assert_eq!(
block_state.entropy.fse.matchlength_repeatMode,
FSE_REPEAT_NONE
);
assert_eq!(
block_state.entropy.fse.litlength_repeatMode,
FSE_REPEAT_NONE
);
}
#[test]
fn store_last_literals_copies_bytes_and_advances_only_the_literal_cursor() {
let mut sequences = [SeqDef::default(); 2];
let mut literals = [0xA5u8; 8];
let sequences_start = sequences.as_mut_ptr();
let literals_start = literals.as_mut_ptr();
let mut seq_store = SeqStore_t {
sequencesStart: sequences_start,
sequences: unsafe { sequences_start.add(1) },
litStart: literals_start,
lit: unsafe { literals_start.add(2) },
llCode: std::ptr::null_mut(),
mlCode: std::ptr::null_mut(),
ofCode: std::ptr::null_mut(),
maxNbSeq: sequences.len(),
maxNbLit: literals.len(),
longLengthType: ZSTD_LLT_MATCH_LENGTH,
longLengthPos: 7,
};
let source = [1u8, 2, 3, 4];
unsafe {
ZSTD_rust_storeLastLiterals(&mut seq_store, source.as_ptr(), 3);
}
assert_eq!(literals, [0xA5, 0xA5, 1, 2, 3, 0xA5, 0xA5, 0xA5]);
assert_eq!(seq_store.lit, unsafe { literals_start.add(5) });
assert_eq!(seq_store.sequences, unsafe { sequences_start.add(1) });
let cursor = seq_store.lit;
unsafe {
ZSTD_rust_storeLastLiterals(&mut seq_store, source.as_ptr(), 0);
}
assert_eq!(seq_store.lit, cursor);
assert_eq!(literals, [0xA5, 0xA5, 1, 2, 3, 0xA5, 0xA5, 0xA5]);
}
#[test]
fn reset_seq_store_resets_cursors_and_long_length_type_but_not_position() {
let mut sequences = [SeqDef::default(); 2];
let mut literals = [0u8; 5];
let sequences_start = sequences.as_mut_ptr();
let literals_start = literals.as_mut_ptr();
let mut seq_store = SeqStore_t {
sequencesStart: sequences_start,
sequences: unsafe { sequences_start.add(2) },
litStart: literals_start,
lit: unsafe { literals_start.add(4) },
llCode: std::ptr::null_mut(),
mlCode: std::ptr::null_mut(),
ofCode: std::ptr::null_mut(),
maxNbSeq: sequences.len(),
maxNbLit: literals.len(),
longLengthType: ZSTD_LLT_LITERAL_LENGTH,
longLengthPos: 11,
};
unsafe { ZSTD_rust_resetSeqStore(&mut seq_store) };
assert_eq!(seq_store.sequences, sequences_start);
assert_eq!(seq_store.lit, literals_start);
assert_eq!(seq_store.longLengthType, 0);
assert_eq!(seq_store.longLengthPos, 11);
}
#[test]
fn fast_sequence_length_sum_handles_empty_and_populated_buffers() {
assert_eq!(
unsafe { ZSTD_rust_fastSequenceLengthSum(std::ptr::null(), 0) },
0
);
let sequences = [
ZSTD_Sequence {
offset: 0,
litLength: 3,
matchLength: 7,
rep: 0,
},
ZSTD_Sequence {
offset: 10,
litLength: 11,
matchLength: 13,
rep: 1,
},
];
assert_eq!(
unsafe { ZSTD_rust_fastSequenceLengthSum(sequences.as_ptr(), sequences.len()) },
34
);
}
#[test]
fn is_rle_matches_empty_one_byte_and_unrolled_inputs() {
let one_byte = [0x5Au8];
assert_eq!(unsafe { ZSTD_rust_isRLE(one_byte.as_ptr(), 0) }, 1);
assert_eq!(unsafe { ZSTD_rust_isRLE(one_byte.as_ptr(), 1) }, 1);
let unrolled_size = size_of::<usize>() * 4;
let mut repeated = vec![0xA7u8; unrolled_size + 3];
assert_eq!(
unsafe { ZSTD_rust_isRLE(repeated.as_ptr(), repeated.len()) },
1
);
repeated[1] ^= 1;
assert_eq!(
unsafe { ZSTD_rust_isRLE(repeated.as_ptr(), repeated.len()) },
0
);
repeated[1] ^= 1;
repeated[3] ^= 1;
assert_eq!(
unsafe { ZSTD_rust_isRLE(repeated.as_ptr(), repeated.len()) },
0
);
}
#[test]
fn maybe_rle_uses_the_original_sequence_and_literal_thresholds() {
let mut sequences = [SeqDef::default(); 4];
let mut literals = [0u8; 10];
let sequences_start = sequences.as_mut_ptr();
let literals_start = literals.as_mut_ptr();
let mut seq_store = SeqStore_t {
sequencesStart: sequences_start,
sequences: sequences_start,
litStart: literals_start,
lit: literals_start,
llCode: std::ptr::null_mut(),
mlCode: std::ptr::null_mut(),
ofCode: std::ptr::null_mut(),
maxNbSeq: sequences.len(),
maxNbLit: literals.len(),
longLengthType: 0,
longLengthPos: 0,
};
assert_eq!(unsafe { ZSTD_rust_maybeRLE(&seq_store) }, 1);
seq_store.sequences = unsafe { sequences_start.add(3) };
seq_store.lit = unsafe { literals_start.add(9) };
assert_eq!(unsafe { ZSTD_rust_maybeRLE(&seq_store) }, 1);
seq_store.sequences = unsafe { sequences_start.add(4) };
assert_eq!(unsafe { ZSTD_rust_maybeRLE(&seq_store) }, 0);
seq_store.sequences = unsafe { sequences_start.add(3) };
seq_store.lit = unsafe { literals_start.add(10) };
assert_eq!(unsafe { ZSTD_rust_maybeRLE(&seq_store) }, 0);
}
#[test]
fn repcode_updates_match_the_c_sum_type_rules() {
let mut reps = [1, 4, 8];