diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 6107861f7..1c9e166e3 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -233,6 +233,14 @@ size_t ZSTD_rust_buildBlockEntropyStats( size_t ZSTD_rust_copyBlockSequences( SeqCollector* seqCollector, const SeqStore_t* seqStore, const U32 prevRepcodes[ZSTD_REP_NUM]); +void ZSTD_rust_resetCompressedBlockState(ZSTD_compressedBlockState_t* bs); +void ZSTD_rust_storeLastLiterals(SeqStore_t* seqStorePtr, + const BYTE* anchor, size_t lastLLSize); +void ZSTD_rust_resetSeqStore(SeqStore_t* ssPtr); +size_t ZSTD_rust_fastSequenceLengthSum(const ZSTD_Sequence* seqBuf, + size_t seqBufSize); +int ZSTD_rust_isRLE(const BYTE* src, size_t length); +int ZSTD_rust_maybeRLE(const SeqStore_t* seqStore); typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1]; typedef char ZSTD_rust_stats_seqstore_long_length_pos[ @@ -1373,13 +1381,7 @@ static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) { - int i; - for (i = 0; i < ZSTD_REP_NUM; ++i) - bs->rep[i] = repStartValue[i]; - bs->entropy.huf.repeatMode = HUF_repeat_none; - bs->entropy.fse.offcode_repeatMode = FSE_repeat_none; - bs->entropy.fse.matchlength_repeatMode = FSE_repeat_none; - bs->entropy.fse.litlength_repeatMode = FSE_repeat_none; + ZSTD_rust_resetCompressedBlockState(bs); } /*! ZSTD_invalidateMatchState() @@ -2307,15 +2309,12 @@ ZSTD_BlockCompressor_f ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_Para static void ZSTD_storeLastLiterals(SeqStore_t* seqStorePtr, const BYTE* anchor, size_t lastLLSize) { - ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize); - seqStorePtr->lit += lastLLSize; + ZSTD_rust_storeLastLiterals(seqStorePtr, anchor, lastLLSize); } void ZSTD_resetSeqStore(SeqStore_t* ssPtr) { - ssPtr->lit = ssPtr->litStart; - ssPtr->sequences = ssPtr->sequencesStart; - ssPtr->longLengthType = ZSTD_llt_none; + ZSTD_rust_resetSeqStore(ssPtr); } /* ZSTD_postProcessSequenceProducerResult() : @@ -2374,14 +2373,7 @@ static size_t ZSTD_postProcessSequenceProducerResult( * Removing the early exit allows the compiler to auto-vectorize (https://godbolt.org/z/cY1cajz9P). * This function can be deleted and replaced by determine_blockSize after we resolve issue #3456. */ static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seqBufSize) { - size_t matchLenSum, litLenSum, i; - matchLenSum = 0; - litLenSum = 0; - for (i = 0; i < seqBufSize; i++) { - litLenSum += seqBuf[i].litLength; - matchLenSum += seqBuf[i].matchLength; - } - return litLenSum + matchLenSum; + return ZSTD_rust_fastSequenceLengthSum(seqBuf, seqBufSize); } /** @@ -2626,25 +2618,7 @@ size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, /* Unrolled loop to read four size_ts of input at a time. Returns 1 if is RLE, 0 if not. */ static int ZSTD_isRLE(const BYTE* src, size_t length) { - const BYTE* ip = src; - const BYTE value = ip[0]; - const size_t valueST = (size_t)((U64)value * 0x0101010101010101ULL); - const size_t unrollSize = sizeof(size_t) * 4; - const size_t unrollMask = unrollSize - 1; - const size_t prefixLength = length & unrollMask; - size_t i; - if (length == 1) return 1; - /* Check if prefix is RLE first before using unrolled loop */ - if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) { - return 0; - } - for (i = prefixLength; i != length; i += unrollSize) { - size_t u; - for (u = 0; u < unrollSize; u += sizeof(size_t)) { - if (MEM_readST(ip + i + u) != valueST) { - return 0; - } } } - return 1; + return ZSTD_rust_isRLE(src, length); } /* Returns true if the given block may be RLE. @@ -2653,10 +2627,7 @@ static int ZSTD_isRLE(const BYTE* src, size_t length) { */ static int ZSTD_maybeRLE(SeqStore_t const* seqStore) { - size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); - size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart); - - return nbSeqs < 4 && nbLits < 10; + return ZSTD_rust_maybeRLE(seqStore); } static void diff --git a/rust/src/zstd_compress_stats.rs b/rust/src/zstd_compress_stats.rs index d6121a1ab..10d9a5afd 100644 --- a/rust/src/zstd_compress_stats.rs +++ b/rust/src/zstd_compress_stats.rs @@ -17,8 +17,8 @@ use crate::bits::ZSTD_highbit32; use crate::common::{ DEFAULT_MAX_OFF, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML, MAX_OFF, MAX_SEQ, MINMATCH, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, ML_FSE_LOG, OFF_FSE_LOG, - OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, ZSTD_MAX_FSE_HEADERS_SIZE, ZSTD_MAX_HUF_HEADER_SIZE, - ZSTD_REP_NUM, + OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, REP_START_VALUE, ZSTD_MAX_FSE_HEADERS_SIZE, + ZSTD_MAX_HUF_HEADER_SIZE, ZSTD_REP_NUM, }; use crate::errors::{ERR_isError, ZstdErrorCode, ERROR}; use crate::hist::{HIST_countFast_wksp, HIST_count_wksp}; @@ -26,7 +26,7 @@ use crate::huf_compress::{ HUF_buildCTable_wksp, HUF_estimateCompressedSize, HUF_optimalTableLog, HUF_validateCTable, HUF_writeCTable_wksp, }; -use crate::mem::{MEM_32bits, MEM_writeLE16}; +use crate::mem::{MEM_32bits, MEM_readST, MEM_writeLE16}; use crate::zstd_compress_literals::{ copy_huf_tables, min_gain, ZSTD_compressLiterals, ZSTD_hufCTables_t, }; @@ -158,6 +158,120 @@ pub struct ZSTD_Sequence { pub rep: u32, } +/// Resets the state that is carried from one compressed block to the next. +/// +/// This is the direct Rust leaf for C's +/// `ZSTD_reset_compressedBlockState()`. The caller must provide a valid +/// mutable compressed block state. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_resetCompressedBlockState( + block_state: *mut ZSTD_compressedBlockState_t, +) { + let block_state = unsafe { &mut *block_state }; + block_state.rep.copy_from_slice(&REP_START_VALUE); + block_state.entropy.huf.repeatMode = HUF_REPEAT_NONE; + block_state.entropy.fse.offcode_repeatMode = FSE_REPEAT_NONE; + block_state.entropy.fse.matchlength_repeatMode = FSE_REPEAT_NONE; + block_state.entropy.fse.litlength_repeatMode = FSE_REPEAT_NONE; +} + +/// Appends the final literals of a block to its sequence store. +/// +/// The source and destination ranges must be valid and non-overlapping, as +/// required by the original C `memcpy` call. `last_ll_size == 0` preserves +/// the C pointer-update behavior without changing any bytes. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_storeLastLiterals( + seq_store: *mut SeqStore_t, + anchor: *const u8, + last_ll_size: usize, +) { + let seq_store = unsafe { &mut *seq_store }; + unsafe { + ptr::copy_nonoverlapping(anchor, seq_store.lit, last_ll_size); + seq_store.lit = seq_store.lit.add(last_ll_size); + } +} + +/// Resets the sequence and literal cursors for a new block. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_resetSeqStore(seq_store: *mut SeqStore_t) { + let seq_store = unsafe { &mut *seq_store }; + seq_store.lit = seq_store.litStart; + seq_store.sequences = seq_store.sequencesStart; + seq_store.longLengthType = 0; +} + +/// Returns the sum of all literal and match lengths in an external sequence +/// buffer, without looking for a block delimiter. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_fastSequenceLengthSum( + seq_buf: *const ZSTD_Sequence, + seq_buf_size: usize, +) -> usize { + let mut match_len_sum = 0usize; + let mut lit_len_sum = 0usize; + for index in 0..seq_buf_size { + let sequence = unsafe { *seq_buf.add(index) }; + lit_len_sum = lit_len_sum.wrapping_add(sequence.litLength as usize); + match_len_sum = match_len_sum.wrapping_add(sequence.matchLength as usize); + } + lit_len_sum.wrapping_add(match_len_sum) +} + +/// Returns whether all bytes in the input have the same value. +/// +/// This keeps the original C helper's precondition that `src` points to at +/// least one readable byte, even when `length == 0`: the first byte is read +/// before the length checks. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_isRLE(src: *const u8, length: usize) -> c_int { + let value = unsafe { *src }; + let value_st = (u64::from(value) * 0x0101_0101_0101_0101u64) as usize; + let unroll_size = size_of::() * 4; + let prefix_length = length & (unroll_size - 1); + + if length == 1 { + return 1; + } + + let mut index = 1; + while index < prefix_length { + if unsafe { *src.add(index) } != value { + return 0; + } + index += 1; + } + + let mut index = prefix_length; + while index != length { + let mut unroll_offset = 0; + while unroll_offset < unroll_size { + let word = unsafe { MEM_readST(src.add(index + unroll_offset).cast()) }; + if word != value_st { + return 0; + } + unroll_offset += size_of::(); + } + index += unroll_size; + } + 1 +} + +/// Heuristic used to decide whether a sequence store may describe an RLE +/// block. It intentionally only inspects the two pointer distances. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_maybeRLE(seq_store: *const SeqStore_t) -> c_int { + let seq_store = unsafe { &*seq_store }; + let nb_seqs = unsafe { seq_store.sequences.offset_from(seq_store.sequencesStart) } as usize; + let nb_lits = unsafe { seq_store.lit.offset_from(seq_store.litStart) } as usize; + if nb_seqs < 4 && nb_lits < 10 { + 1 + } else { + 0 + } +} + /// C's `SeqCollector` leaf from `zstd_compress_internal.h`. #[repr(C)] pub struct SeqCollector { @@ -1311,6 +1425,179 @@ mod tests { } } + #[test] + fn compressed_block_state_reset_restores_repcodes_and_repeat_modes() { + let mut block_state = + unsafe { std::mem::MaybeUninit::::zeroed().assume_init() }; + block_state.rep = [99; ZSTD_REP_NUM]; + block_state.entropy.huf.repeatMode = HUF_REPEAT_VALID; + block_state.entropy.fse.offcode_repeatMode = 2; + block_state.entropy.fse.matchlength_repeatMode = 1; + block_state.entropy.fse.litlength_repeatMode = 2; + + unsafe { ZSTD_rust_resetCompressedBlockState(&mut block_state) }; + + assert_eq!(block_state.rep, [1, 4, 8]); + assert_eq!(block_state.entropy.huf.repeatMode, HUF_REPEAT_NONE); + assert_eq!(block_state.entropy.fse.offcode_repeatMode, FSE_REPEAT_NONE); + assert_eq!( + block_state.entropy.fse.matchlength_repeatMode, + FSE_REPEAT_NONE + ); + assert_eq!( + block_state.entropy.fse.litlength_repeatMode, + FSE_REPEAT_NONE + ); + } + + #[test] + fn store_last_literals_copies_bytes_and_advances_only_the_literal_cursor() { + let mut sequences = [SeqDef::default(); 2]; + let mut literals = [0xA5u8; 8]; + let sequences_start = sequences.as_mut_ptr(); + let literals_start = literals.as_mut_ptr(); + let mut seq_store = SeqStore_t { + sequencesStart: sequences_start, + sequences: unsafe { sequences_start.add(1) }, + litStart: literals_start, + lit: unsafe { literals_start.add(2) }, + llCode: std::ptr::null_mut(), + mlCode: std::ptr::null_mut(), + ofCode: std::ptr::null_mut(), + maxNbSeq: sequences.len(), + maxNbLit: literals.len(), + longLengthType: ZSTD_LLT_MATCH_LENGTH, + longLengthPos: 7, + }; + let source = [1u8, 2, 3, 4]; + + unsafe { + ZSTD_rust_storeLastLiterals(&mut seq_store, source.as_ptr(), 3); + } + assert_eq!(literals, [0xA5, 0xA5, 1, 2, 3, 0xA5, 0xA5, 0xA5]); + assert_eq!(seq_store.lit, unsafe { literals_start.add(5) }); + assert_eq!(seq_store.sequences, unsafe { sequences_start.add(1) }); + + let cursor = seq_store.lit; + unsafe { + ZSTD_rust_storeLastLiterals(&mut seq_store, source.as_ptr(), 0); + } + assert_eq!(seq_store.lit, cursor); + assert_eq!(literals, [0xA5, 0xA5, 1, 2, 3, 0xA5, 0xA5, 0xA5]); + } + + #[test] + fn reset_seq_store_resets_cursors_and_long_length_type_but_not_position() { + let mut sequences = [SeqDef::default(); 2]; + let mut literals = [0u8; 5]; + let sequences_start = sequences.as_mut_ptr(); + let literals_start = literals.as_mut_ptr(); + let mut seq_store = SeqStore_t { + sequencesStart: sequences_start, + sequences: unsafe { sequences_start.add(2) }, + litStart: literals_start, + lit: unsafe { literals_start.add(4) }, + llCode: std::ptr::null_mut(), + mlCode: std::ptr::null_mut(), + ofCode: std::ptr::null_mut(), + maxNbSeq: sequences.len(), + maxNbLit: literals.len(), + longLengthType: ZSTD_LLT_LITERAL_LENGTH, + longLengthPos: 11, + }; + + unsafe { ZSTD_rust_resetSeqStore(&mut seq_store) }; + + assert_eq!(seq_store.sequences, sequences_start); + assert_eq!(seq_store.lit, literals_start); + assert_eq!(seq_store.longLengthType, 0); + assert_eq!(seq_store.longLengthPos, 11); + } + + #[test] + fn fast_sequence_length_sum_handles_empty_and_populated_buffers() { + assert_eq!( + unsafe { ZSTD_rust_fastSequenceLengthSum(std::ptr::null(), 0) }, + 0 + ); + + let sequences = [ + ZSTD_Sequence { + offset: 0, + litLength: 3, + matchLength: 7, + rep: 0, + }, + ZSTD_Sequence { + offset: 10, + litLength: 11, + matchLength: 13, + rep: 1, + }, + ]; + assert_eq!( + unsafe { ZSTD_rust_fastSequenceLengthSum(sequences.as_ptr(), sequences.len()) }, + 34 + ); + } + + #[test] + fn is_rle_matches_empty_one_byte_and_unrolled_inputs() { + let one_byte = [0x5Au8]; + assert_eq!(unsafe { ZSTD_rust_isRLE(one_byte.as_ptr(), 0) }, 1); + assert_eq!(unsafe { ZSTD_rust_isRLE(one_byte.as_ptr(), 1) }, 1); + + let unrolled_size = size_of::() * 4; + let mut repeated = vec![0xA7u8; unrolled_size + 3]; + assert_eq!( + unsafe { ZSTD_rust_isRLE(repeated.as_ptr(), repeated.len()) }, + 1 + ); + + repeated[1] ^= 1; + assert_eq!( + unsafe { ZSTD_rust_isRLE(repeated.as_ptr(), repeated.len()) }, + 0 + ); + repeated[1] ^= 1; + repeated[3] ^= 1; + assert_eq!( + unsafe { ZSTD_rust_isRLE(repeated.as_ptr(), repeated.len()) }, + 0 + ); + } + + #[test] + fn maybe_rle_uses_the_original_sequence_and_literal_thresholds() { + let mut sequences = [SeqDef::default(); 4]; + let mut literals = [0u8; 10]; + let sequences_start = sequences.as_mut_ptr(); + let literals_start = literals.as_mut_ptr(); + let mut seq_store = SeqStore_t { + sequencesStart: sequences_start, + sequences: sequences_start, + litStart: literals_start, + lit: literals_start, + llCode: std::ptr::null_mut(), + mlCode: std::ptr::null_mut(), + ofCode: std::ptr::null_mut(), + maxNbSeq: sequences.len(), + maxNbLit: literals.len(), + longLengthType: 0, + longLengthPos: 0, + }; + + assert_eq!(unsafe { ZSTD_rust_maybeRLE(&seq_store) }, 1); + seq_store.sequences = unsafe { sequences_start.add(3) }; + seq_store.lit = unsafe { literals_start.add(9) }; + assert_eq!(unsafe { ZSTD_rust_maybeRLE(&seq_store) }, 1); + seq_store.sequences = unsafe { sequences_start.add(4) }; + assert_eq!(unsafe { ZSTD_rust_maybeRLE(&seq_store) }, 0); + seq_store.sequences = unsafe { sequences_start.add(3) }; + seq_store.lit = unsafe { literals_start.add(10) }; + assert_eq!(unsafe { ZSTD_rust_maybeRLE(&seq_store) }, 0); + } + #[test] fn repcode_updates_match_the_c_sum_type_rules() { let mut reps = [1, 4, 8];