refactor(compress): move LDM parser threshold to Rust
Move the scalar policy that selects the optimal parser for long-distance matching into Rust. The C implementation continues to own match-state and compressor dispatch, but the strategy threshold is now defined and tested at the Rust seam. Test Plan: - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo +nightly fmt --manifest-path rust/Cargo.toml -- --check - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --all-targets (775 passed) - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/cli/Cargo.toml --all-targets (179 passed) - ulimit -v 41943040; make -j1 - ulimit -v 41943040; make -j1 -C tests test (all tests completed successfully)
This commit is contained in:
@@ -267,7 +267,7 @@ void ZSTD_rust_ldm_skipSequences(void* rawSeqStore, size_t srcSize, U32 minMatch
|
||||
void ZSTD_rust_ldm_skipRawSeqStoreBytes(void* rawSeqStore, size_t nbBytes);
|
||||
size_t ZSTD_rust_ldm_blockCompress(
|
||||
void* rawSeqStore, void* blockContext, void* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize, U32 minMatch, int useOptimalParser);
|
||||
const void* src, size_t srcSize, U32 minMatch, int strategy);
|
||||
U32 ZSTD_rust_ldm_limitTableUpdate(U32 curr, U32 nextToUpdate);
|
||||
|
||||
const U64* ZSTD_ldm_rust_gearTable(void);
|
||||
@@ -406,5 +406,5 @@ size_t ZSTD_ldm_blockCompress(RawSeqStore_t* rawSeqStore,
|
||||
assert(context.blockCompressor != NULL);
|
||||
return ZSTD_rust_ldm_blockCompress(
|
||||
rawSeqStore, &context, seqStore, rep, src, srcSize,
|
||||
ms->cParams.minMatch, ms->cParams.strategy >= ZSTD_btopt);
|
||||
ms->cParams.minMatch, (int)ms->cParams.strategy);
|
||||
}
|
||||
|
||||
+15
-2
@@ -23,6 +23,7 @@ const LDM_MIN_MATCH_LENGTH: u32 = 64;
|
||||
const HASH_READ_SIZE: usize = 8;
|
||||
const ZSTD_REP_NUM: usize = 3;
|
||||
const ZSTD_WINDOW_START_INDEX: u32 = 2;
|
||||
const ZSTD_BTOPT: c_int = 7;
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy)]
|
||||
@@ -89,6 +90,11 @@ const EMPTY_CANDIDATE: MatchCandidate = MatchCandidate {
|
||||
bucket: std::ptr::null_mut(),
|
||||
};
|
||||
|
||||
#[inline]
|
||||
fn use_optimal_parser(strategy: c_int) -> bool {
|
||||
strategy >= ZSTD_BTOPT
|
||||
}
|
||||
|
||||
unsafe extern "C" {
|
||||
fn ZSTD_ldm_rust_gearTable() -> *const u64;
|
||||
fn ZSTD_ldm_rust_prepareBlock(context: *mut c_void, anchor: *const c_void);
|
||||
@@ -940,12 +946,12 @@ pub unsafe extern "C" fn ZSTD_rust_ldm_blockCompress(
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
min_match: u32,
|
||||
use_optimal_parser: c_int,
|
||||
strategy: c_int,
|
||||
) -> usize {
|
||||
let raw_seq_store = raw_seq_store.cast::<RawSeqStore>();
|
||||
let input = src.cast::<u8>();
|
||||
let input_end = input.wrapping_add(src_size);
|
||||
if use_optimal_parser != 0 {
|
||||
if use_optimal_parser(strategy) {
|
||||
unsafe { ZSTD_ldm_rust_setLdmSeqStore(block_context, raw_seq_store.cast::<c_void>()) };
|
||||
let last_literals = unsafe {
|
||||
ZSTD_ldm_rust_compressLiterals(block_context, seq_store, reps, src, src_size)
|
||||
@@ -1062,6 +1068,13 @@ mod tests {
|
||||
assert_eq!(params.min_match_length, 64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn optimal_parser_starts_at_btopt_strategy() {
|
||||
assert!(!use_optimal_parser(ZSTD_BTOPT - 1));
|
||||
assert!(use_optimal_parser(ZSTD_BTOPT));
|
||||
assert!(use_optimal_parser(9));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_sequence_skipping_merges_short_tail_matches() {
|
||||
let mut sequences = [
|
||||
|
||||
Reference in New Issue
Block a user