feat(compress): move match table reduction to Rust
Port the 16-cell match-table reduction leaf while preserving the window-index threshold, wrapping subtraction, and btlazy2 unsorted marker. Keep ZSTD_reduceIndex and its strategy-specific table selection in C behind the narrow Rust ABI. Test Plan: cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression (180 passed); root clippy, bench clippy, and test clippy; make -B -C lib -j2 lib; make -B -C tests -j2 test-zstream (84 named tests plus 6,845 and 9,628 fuzz cases passed).
This commit is contained in:
@@ -52,6 +52,8 @@ size_t ZSTD_compressStream2_c(ZSTD_CCtx* cctx,
|
||||
ZSTD_EndDirective endOp);
|
||||
int ZSTD_rust_simpleCompress2Level(const void* cctx);
|
||||
int ZSTD_rust_simpleCompressStream2Level(const void* cctx);
|
||||
void ZSTD_rust_reduceTable(U32* table, U32 size, U32 reducerValue,
|
||||
int preserveMark);
|
||||
|
||||
/* Context-free compression-parameter selection and sizing leaves live in
|
||||
* Rust (rust/src/zstd_compress_params.rs). This file retains
|
||||
@@ -2076,63 +2078,20 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
|
||||
}
|
||||
|
||||
|
||||
#define ZSTD_ROWSIZE 16
|
||||
/*! ZSTD_reduceTable() :
|
||||
* reduce table indexes by `reducerValue`, or squash to zero.
|
||||
* PreserveMark preserves "unsorted mark" for btlazy2 strategy.
|
||||
* It must be set to a clear 0/1 value, to remove branch during inlining.
|
||||
* Presume table size is a multiple of ZSTD_ROWSIZE
|
||||
* Presume table size is a multiple of 16 cells
|
||||
* to help auto-vectorization */
|
||||
FORCE_INLINE_TEMPLATE void
|
||||
ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerValue, int const preserveMark)
|
||||
{
|
||||
int const nbRows = (int)size / ZSTD_ROWSIZE;
|
||||
int cellNb = 0;
|
||||
int rowNb;
|
||||
/* Protect special index values < ZSTD_WINDOW_START_INDEX. */
|
||||
U32 const reducerThreshold = reducerValue + ZSTD_WINDOW_START_INDEX;
|
||||
assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */
|
||||
assert(size < (1U<<31)); /* can be cast to int */
|
||||
|
||||
#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE)
|
||||
/* To validate that the table reuse logic is sound, and that we don't
|
||||
* access table space that we haven't cleaned, we re-"poison" the table
|
||||
* space every time we mark it dirty.
|
||||
*
|
||||
* This function however is intended to operate on those dirty tables and
|
||||
* re-clean them. So when this function is used correctly, we can unpoison
|
||||
* the memory it operated on. This introduces a blind spot though, since
|
||||
* if we now try to operate on __actually__ poisoned memory, we will not
|
||||
* detect that. */
|
||||
__msan_unpoison(table, size * sizeof(U32));
|
||||
#endif
|
||||
|
||||
for (rowNb=0 ; rowNb < nbRows ; rowNb++) {
|
||||
int column;
|
||||
for (column=0; column<ZSTD_ROWSIZE; column++) {
|
||||
U32 newVal;
|
||||
if (preserveMark && table[cellNb] == ZSTD_DUBT_UNSORTED_MARK) {
|
||||
/* This write is pointless, but is required(?) for the compiler
|
||||
* to auto-vectorize the loop. */
|
||||
newVal = ZSTD_DUBT_UNSORTED_MARK;
|
||||
} else if (table[cellNb] < reducerThreshold) {
|
||||
newVal = 0;
|
||||
} else {
|
||||
newVal = table[cellNb] - reducerValue;
|
||||
}
|
||||
table[cellNb] = newVal;
|
||||
cellNb++;
|
||||
} }
|
||||
}
|
||||
|
||||
static void ZSTD_reduceTable(U32* const table, U32 const size, U32 const reducerValue)
|
||||
{
|
||||
ZSTD_reduceTable_internal(table, size, reducerValue, 0);
|
||||
ZSTD_rust_reduceTable(table, size, reducerValue, 0);
|
||||
}
|
||||
|
||||
static void ZSTD_reduceTable_btlazy2(U32* const table, U32 const size, U32 const reducerValue)
|
||||
{
|
||||
ZSTD_reduceTable_internal(table, size, reducerValue, 1);
|
||||
ZSTD_rust_reduceTable(table, size, reducerValue, 1);
|
||||
}
|
||||
|
||||
/*! ZSTD_reduceIndex() :
|
||||
|
||||
@@ -61,6 +61,9 @@ const ZSTD_FAST: c_int = 1;
|
||||
const ZSTD_DFAST: c_int = 2;
|
||||
const ZSTD_BLOCKSIZE_MAX: usize = 1 << 17;
|
||||
const ZSTD_CONTENTSIZE_UNKNOWN: u64 = u64::MAX;
|
||||
const ZSTD_ROWSIZE: usize = 16;
|
||||
const ZSTD_WINDOW_START_INDEX: u32 = 2;
|
||||
const ZSTD_DUBT_UNSORTED_MARK: u32 = 1;
|
||||
#[cfg(not(test))]
|
||||
const ZSTD_E_END: c_int = 2;
|
||||
|
||||
@@ -86,6 +89,52 @@ pub struct ZSTD_outBuffer {
|
||||
* supported pointer widths. */
|
||||
const TMP_WORKSPACE_SIZE: usize = 16 << 10;
|
||||
|
||||
#[inline]
|
||||
fn reduce_table_internal(table: &mut [u32], reducer_value: u32, preserve_mark: bool) {
|
||||
debug_assert_eq!(table.len() % ZSTD_ROWSIZE, 0);
|
||||
debug_assert!(table.len() < (1usize << 31));
|
||||
|
||||
/* Protect special index values < ZSTD_WINDOW_START_INDEX. */
|
||||
let reducer_threshold = reducer_value.wrapping_add(ZSTD_WINDOW_START_INDEX);
|
||||
let mut rows = table.chunks_exact_mut(ZSTD_ROWSIZE);
|
||||
for row in &mut rows {
|
||||
for cell in row {
|
||||
let value = *cell;
|
||||
*cell = if preserve_mark && value == ZSTD_DUBT_UNSORTED_MARK {
|
||||
/* Keep the btlazy2 unsorted marker across table reduction. */
|
||||
ZSTD_DUBT_UNSORTED_MARK
|
||||
} else if value < reducer_threshold {
|
||||
0
|
||||
} else {
|
||||
value.wrapping_sub(reducer_value)
|
||||
};
|
||||
}
|
||||
}
|
||||
debug_assert!(rows.into_remainder().is_empty());
|
||||
}
|
||||
|
||||
/// Rust implementation of the C match-table reduction leaf.
|
||||
///
|
||||
/// The C wrappers select the ordinary or btlazy2 policy by passing a clear
|
||||
/// zero/one `preserve_mark` value; `ZSTD_reduceIndex` remains C-owned.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_reduceTable(
|
||||
table: *mut u32,
|
||||
size: u32,
|
||||
reducer_value: u32,
|
||||
preserve_mark: c_int,
|
||||
) {
|
||||
debug_assert!(!table.is_null() || size == 0);
|
||||
debug_assert_eq!(size % ZSTD_ROWSIZE as u32, 0);
|
||||
debug_assert!(size < (1u32 << 31));
|
||||
if size == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let table = unsafe { std::slice::from_raw_parts_mut(table, size as usize) };
|
||||
reduce_table_internal(table, reducer_value, preserve_mark != 0);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn zeroed_state() -> ZSTD_compressedBlockState_t {
|
||||
/* The state contains only integer arrays and enum fields. */
|
||||
@@ -566,6 +615,48 @@ mod tests {
|
||||
output
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reduce_table_applies_threshold_and_wrapping_subtraction() {
|
||||
let mut table = [0, 1, 2, 3, 4, 5, 6, u32::MAX, 0, 0, 0, 0, 0, 0, 0, 0];
|
||||
reduce_table_internal(&mut table, 3, false);
|
||||
assert_eq!(&table[..8], &[0, 0, 0, 0, 0, 2, 3, u32::MAX - 3]);
|
||||
|
||||
let mut wrapped_threshold = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15];
|
||||
reduce_table_internal(&mut wrapped_threshold, u32::MAX, false);
|
||||
assert_eq!(wrapped_threshold[0], 0);
|
||||
assert_eq!(wrapped_threshold[1], 2);
|
||||
assert_eq!(wrapped_threshold[15], 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reduce_table_preserves_only_the_btlazy2_mark() {
|
||||
let mut ordinary = [0u32; ZSTD_ROWSIZE];
|
||||
ordinary[..3].copy_from_slice(&[ZSTD_DUBT_UNSORTED_MARK, 5, 6]);
|
||||
reduce_table_internal(&mut ordinary, 3, false);
|
||||
assert_eq!(&ordinary[..3], &[0, 2, 3]);
|
||||
|
||||
let mut btlazy2 = [0u32; ZSTD_ROWSIZE];
|
||||
btlazy2[..3].copy_from_slice(&[ZSTD_DUBT_UNSORTED_MARK, 5, 6]);
|
||||
reduce_table_internal(&mut btlazy2, 3, true);
|
||||
assert_eq!(&btlazy2[..3], &[ZSTD_DUBT_UNSORTED_MARK, 2, 3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reduce_table_processes_every_cell_in_multiple_rows() {
|
||||
let mut table = [0u32; ZSTD_ROWSIZE * 2];
|
||||
table[0] = 2;
|
||||
table[ZSTD_ROWSIZE - 1] = 8;
|
||||
table[ZSTD_ROWSIZE] = 1;
|
||||
table[ZSTD_ROWSIZE * 2 - 1] = u32::MAX;
|
||||
|
||||
reduce_table_internal(&mut table, 4, false);
|
||||
|
||||
assert_eq!(table[0], 0);
|
||||
assert_eq!(table[ZSTD_ROWSIZE - 1], 4);
|
||||
assert_eq!(table[ZSTD_ROWSIZE], 0);
|
||||
assert_eq!(table[ZSTD_ROWSIZE * 2 - 1], u32::MAX - 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_one_shot_abi_is_c_compatible() {
|
||||
let entry: unsafe extern "C" fn(*mut c_void, usize, *const c_void, usize, c_int) -> usize =
|
||||
|
||||
Reference in New Issue
Block a user