ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move matchfinder policy to Rust

Browse Source 
Move row matchfinder, block-splitter, LDM, chain-table, external-repcode, and tagged-dictionary policy leaves into Rust while preserving the existing C helper names and private parameter flow. Add boundary coverage for each policy decision.

Test Plan: cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression; cargo clippy --manifest-path rust/Cargo.toml; cargo clippy --manifest-path rust/Cargo.toml --benches; cargo clippy --manifest-path rust/Cargo.toml --tests; make -B -C lib -j2 lib; make -B -C tests -j2 test-cli-tests; make -B -C tests -j2 test-zstream
This commit is contained in:
ddidderr
2026-07-18 03:55:30 +02:00
parent 309e227731
commit 8ac9855309
2 changed files with 298 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+24 -22
View File
@@ -79,6 +79,17 @@ ZSTD_parameters ZSTD_rust_params_makeParams(ZSTD_compressionParameters cParams);
size_t ZSTD_rust_params_maxNbSeq(size_t blockSize, U32 minMatch,
int useSequenceProducer);
size_t ZSTD_rust_params_resolveMaxBlockSize(size_t maxBlockSize);
int ZSTD_rust_params_rowMatchFinderSupported(int strategy);
int ZSTD_rust_params_rowMatchFinderUsed(int strategy, int mode);
int ZSTD_rust_params_resolveRowMatchFinderMode(
int mode, ZSTD_compressionParameters cParams);
int ZSTD_rust_params_resolveBlockSplitterMode(
int mode, ZSTD_compressionParameters cParams);
int ZSTD_rust_params_allocateChainTable(int strategy, int mode, int forDDSDict);
int ZSTD_rust_params_resolveEnableLdm(
int mode, ZSTD_compressionParameters cParams);
int ZSTD_rust_params_resolveExternalRepcodeSearch(int mode, int cLevel);
int ZSTD_rust_params_cdictIndicesAreTagged(ZSTD_compressionParameters cParams);
/* CCtx parameter state is mirrored by rust/src/zstd_compress_params_api.rs.
* Rust owns parameter bounds and clamping; C exposes only the
@@ -437,7 +448,7 @@ const SeqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStor
/* Returns true if the strategy supports using a row based matchfinder */
static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) {
return (strategy >= ZSTD_greedy && strategy <= ZSTD_lazy2);
return ZSTD_rust_params_rowMatchFinderSupported((int)strategy);
}
/* Returns true if the strategy and useRowMatchFinder mode indicate that we will use the row based matchfinder
@@ -445,24 +456,21 @@ static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) {
*/
static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_ParamSwitch_e mode) {
assert(mode != ZSTD_ps_auto);
return ZSTD_rowMatchFinderSupported(strategy) && (mode == ZSTD_ps_enable);
return ZSTD_rust_params_rowMatchFinderUsed((int)strategy, (int)mode);
}
/* Returns row matchfinder usage given an initial mode and cParams */
static ZSTD_ParamSwitch_e ZSTD_resolveRowMatchFinderMode(ZSTD_ParamSwitch_e mode,
const ZSTD_compressionParameters* const cParams) {
if (mode != ZSTD_ps_auto) return mode; /* if requested enabled, but no SIMD, we still will use row matchfinder */
mode = ZSTD_ps_disable;
if (!ZSTD_rowMatchFinderSupported(cParams->strategy)) return mode;
if (cParams->windowLog > 14) mode = ZSTD_ps_enable;
return mode;
return (ZSTD_ParamSwitch_e)ZSTD_rust_params_resolveRowMatchFinderMode(
(int)mode, *cParams);
}
/* Returns block splitter usage (generally speaking, when using slower/stronger compression modes) */
static ZSTD_ParamSwitch_e ZSTD_resolveBlockSplitterMode(ZSTD_ParamSwitch_e mode,
const ZSTD_compressionParameters* const cParams) {
if (mode != ZSTD_ps_auto) return mode;
return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17) ? ZSTD_ps_enable : ZSTD_ps_disable;
return (ZSTD_ParamSwitch_e)ZSTD_rust_params_resolveBlockSplitterMode(
(int)mode, *cParams);
}
/* Returns 1 if the arguments indicate that we should allocate a chainTable, 0 otherwise */
@@ -470,10 +478,8 @@ static int ZSTD_allocateChainTable(const ZSTD_strategy strategy,
const ZSTD_ParamSwitch_e useRowMatchFinder,
const U32 forDDSDict) {
assert(useRowMatchFinder != ZSTD_ps_auto);
/* We always should allocate a chaintable if we are allocating a matchstate for a DDS dictionary matchstate.
* We do not allocate a chaintable if we are using ZSTD_fast, or are using the row-based matchfinder.
*/
return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder));
return ZSTD_rust_params_allocateChainTable(
(int)strategy, (int)useRowMatchFinder, (int)forDDSDict);
}
/* Returns ZSTD_ps_enable if compression parameters are such that we should
@@ -482,8 +488,8 @@ static int ZSTD_allocateChainTable(const ZSTD_strategy strategy,
*/
static ZSTD_ParamSwitch_e ZSTD_resolveEnableLdm(ZSTD_ParamSwitch_e mode,
const ZSTD_compressionParameters* const cParams) {
if (mode != ZSTD_ps_auto) return mode;
return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_disable;
return (ZSTD_ParamSwitch_e)ZSTD_rust_params_resolveEnableLdm(
(int)mode, *cParams);
}
static int ZSTD_resolveExternalSequenceValidation(int mode) {
@@ -496,18 +502,14 @@ static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) {
}
static ZSTD_ParamSwitch_e ZSTD_resolveExternalRepcodeSearch(ZSTD_ParamSwitch_e value, int cLevel) {
if (value != ZSTD_ps_auto) return value;
if (cLevel < 10) {
return ZSTD_ps_disable;
} else {
return ZSTD_ps_enable;
}
return (ZSTD_ParamSwitch_e)ZSTD_rust_params_resolveExternalRepcodeSearch(
(int)value, cLevel);
}
/* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged.
* If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */
static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) {
return cParams->strategy == ZSTD_fast || cParams->strategy == ZSTD_dfast;
return ZSTD_rust_params_cdictIndicesAreTagged(*cParams);
}
static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
rust/src/zstd_compress_params.rs
+274
View File
@@ -319,6 +319,18 @@ fn row_match_finder_used(strategy: c_int, mode: c_int) -> bool {
strategy_supports_row_match_finder(strategy) && mode == ZSTD_RUST_PS_ENABLE
}
/// C ABI for `ZSTD_rowMatchFinderSupported()`.
#[no_mangle]
pub extern "C" fn ZSTD_rust_params_rowMatchFinderSupported(strategy: c_int) -> c_int {
c_int::from(strategy_supports_row_match_finder(strategy))
}
/// C ABI for `ZSTD_rowMatchFinderUsed()`.
#[no_mangle]
pub extern "C" fn ZSTD_rust_params_rowMatchFinderUsed(strategy: c_int, mode: c_int) -> c_int {
c_int::from(row_match_finder_used(strategy, mode))
}
#[inline]
fn resolve_row_match_finder(mode: c_int, cparams: ZSTD_compressionParameters) -> c_int {
if mode != ZSTD_RUST_PS_AUTO {
@@ -331,6 +343,91 @@ fn resolve_row_match_finder(mode: c_int, cparams: ZSTD_compressionParameters) ->
}
}
/// C ABI for `ZSTD_resolveRowMatchFinderMode()`.
#[no_mangle]
pub extern "C" fn ZSTD_rust_params_resolveRowMatchFinderMode(
mode: c_int,
cparams: ZSTD_compressionParameters,
) -> c_int {
resolve_row_match_finder(mode, cparams)
}
#[inline]
fn resolve_block_splitter(mode: c_int, cparams: ZSTD_compressionParameters) -> c_int {
if mode != ZSTD_RUST_PS_AUTO {
return mode;
}
if cparams.strategy >= ZSTD_BTOPT && cparams.windowLog >= 17 {
ZSTD_RUST_PS_ENABLE
} else {
ZSTD_RUST_PS_DISABLE
}
}
/// C ABI for `ZSTD_resolveBlockSplitterMode()`.
#[no_mangle]
pub extern "C" fn ZSTD_rust_params_resolveBlockSplitterMode(
mode: c_int,
cparams: ZSTD_compressionParameters,
) -> c_int {
resolve_block_splitter(mode, cparams)
}
#[inline]
fn resolve_enable_ldm(mode: c_int, cparams: ZSTD_compressionParameters) -> c_int {
if mode != ZSTD_RUST_PS_AUTO {
return mode;
}
if cparams.strategy >= ZSTD_BTOPT && cparams.windowLog >= 27 {
ZSTD_RUST_PS_ENABLE
} else {
ZSTD_RUST_PS_DISABLE
}
}
/// C ABI for `ZSTD_resolveEnableLdm()`.
#[no_mangle]
pub extern "C" fn ZSTD_rust_params_resolveEnableLdm(
mode: c_int,
cparams: ZSTD_compressionParameters,
) -> c_int {
resolve_enable_ldm(mode, cparams)
}
#[inline]
fn resolve_external_repcode_search(mode: c_int, compression_level: c_int) -> c_int {
if mode != ZSTD_RUST_PS_AUTO {
return mode;
}
if compression_level < 10 {
ZSTD_RUST_PS_DISABLE
} else {
ZSTD_RUST_PS_ENABLE
}
}
/// C ABI for `ZSTD_resolveExternalRepcodeSearch()`.
#[no_mangle]
pub extern "C" fn ZSTD_rust_params_resolveExternalRepcodeSearch(
mode: c_int,
compression_level: c_int,
) -> c_int {
resolve_external_repcode_search(mode, compression_level)
}
#[inline]
fn cdict_indices_are_tagged(cparams: ZSTD_compressionParameters) -> bool {
cparams.strategy == ZSTD_FAST || cparams.strategy == ZSTD_DFAST
}
/// C ABI for `ZSTD_CDictIndicesAreTagged()`.
#[no_mangle]
pub extern "C" fn ZSTD_rust_params_cdictIndicesAreTagged(
cparams: ZSTD_compressionParameters,
) -> c_int {
c_int::from(cdict_indices_are_tagged(cparams))
}
#[inline]
fn dict_and_window_log(window_log: u32, src_size: u64, dict_size: u64) -> u32 {
/* 1ULL << ZSTD_WINDOWLOG_MAX, which is smaller for 32-bit builds. */
@@ -661,6 +758,16 @@ fn allocate_chain_table(strategy: c_int, use_row_match_finder: c_int, for_dds_di
|| (strategy != ZSTD_FAST && !row_match_finder_used(strategy, use_row_match_finder))
}
/// C ABI for `ZSTD_allocateChainTable()`.
#[no_mangle]
pub extern "C" fn ZSTD_rust_params_allocateChainTable(
strategy: c_int,
mode: c_int,
for_dds_dict: c_int,
) -> c_int {
c_int::from(allocate_chain_table(strategy, mode, for_dds_dict != 0))
}
fn estimate_match_state_size(
cparams: ZSTD_compressionParameters,
use_row_match_finder: c_int,
@@ -890,6 +997,173 @@ mod tests {
);
}
fn policy_cparams(strategy: c_int, window_log: u32) -> ZSTD_compressionParameters {
ZSTD_compressionParameters {
windowLog: window_log,
strategy,
..ZSTD_compressionParameters::default()
}
}
#[test]
fn row_match_finder_policy_preserves_strategy_and_window_boundaries() {
assert_eq!(ZSTD_rust_params_rowMatchFinderSupported(ZSTD_FAST), 0);
assert_eq!(ZSTD_rust_params_rowMatchFinderSupported(ZSTD_GREEDY), 1);
assert_eq!(ZSTD_rust_params_rowMatchFinderSupported(ZSTD_LAZY2), 1);
assert_eq!(ZSTD_rust_params_rowMatchFinderSupported(ZSTD_BTLAZY2), 0);
assert_eq!(
ZSTD_rust_params_rowMatchFinderUsed(ZSTD_GREEDY, ZSTD_RUST_PS_ENABLE),
1
);
assert_eq!(
ZSTD_rust_params_rowMatchFinderUsed(ZSTD_GREEDY, ZSTD_RUST_PS_DISABLE),
0
);
assert_eq!(
ZSTD_rust_params_rowMatchFinderUsed(ZSTD_FAST, ZSTD_RUST_PS_ENABLE),
0
);
assert_eq!(
ZSTD_rust_params_resolveRowMatchFinderMode(
ZSTD_RUST_PS_AUTO,
policy_cparams(ZSTD_GREEDY, 14),
),
ZSTD_RUST_PS_DISABLE
);
assert_eq!(
ZSTD_rust_params_resolveRowMatchFinderMode(
ZSTD_RUST_PS_AUTO,
policy_cparams(ZSTD_GREEDY, 15),
),
ZSTD_RUST_PS_ENABLE
);
assert_eq!(
ZSTD_rust_params_resolveRowMatchFinderMode(
ZSTD_RUST_PS_AUTO,
policy_cparams(ZSTD_BTLAZY2, 31),
),
ZSTD_RUST_PS_DISABLE
);
assert_eq!(
ZSTD_rust_params_resolveRowMatchFinderMode(
ZSTD_RUST_PS_ENABLE,
policy_cparams(ZSTD_FAST, 14),
),
ZSTD_RUST_PS_ENABLE
);
}
#[test]
fn block_splitter_and_ldm_policy_match_window_boundaries() {
assert_eq!(
ZSTD_rust_params_resolveBlockSplitterMode(
ZSTD_RUST_PS_AUTO,
policy_cparams(ZSTD_BTOPT, 16),
),
ZSTD_RUST_PS_DISABLE
);
assert_eq!(
ZSTD_rust_params_resolveBlockSplitterMode(
ZSTD_RUST_PS_AUTO,
policy_cparams(ZSTD_BTOPT, 17),
),
ZSTD_RUST_PS_ENABLE
);
assert_eq!(
ZSTD_rust_params_resolveBlockSplitterMode(
ZSTD_RUST_PS_AUTO,
policy_cparams(ZSTD_BTLAZY2, 31),
),
ZSTD_RUST_PS_DISABLE
);
assert_eq!(
ZSTD_rust_params_resolveBlockSplitterMode(
ZSTD_RUST_PS_ENABLE,
policy_cparams(ZSTD_FAST, 1),
),
ZSTD_RUST_PS_ENABLE
);
assert_eq!(
ZSTD_rust_params_resolveEnableLdm(ZSTD_RUST_PS_AUTO, policy_cparams(ZSTD_BTOPT, 26),),
ZSTD_RUST_PS_DISABLE
);
assert_eq!(
ZSTD_rust_params_resolveEnableLdm(ZSTD_RUST_PS_AUTO, policy_cparams(ZSTD_BTOPT, 27),),
ZSTD_RUST_PS_ENABLE
);
assert_eq!(
ZSTD_rust_params_resolveEnableLdm(ZSTD_RUST_PS_AUTO, policy_cparams(ZSTD_BTLAZY2, 31),),
ZSTD_RUST_PS_DISABLE
);
assert_eq!(
ZSTD_rust_params_resolveEnableLdm(
ZSTD_RUST_PS_DISABLE,
policy_cparams(ZSTD_BTULTRA2, 31),
),
ZSTD_RUST_PS_DISABLE
);
}
#[test]
fn chain_table_policy_matches_dds_and_row_match_finder_modes() {
assert_eq!(
ZSTD_rust_params_allocateChainTable(ZSTD_FAST, ZSTD_RUST_PS_DISABLE, 0),
0
);
assert_eq!(
ZSTD_rust_params_allocateChainTable(ZSTD_DFAST, ZSTD_RUST_PS_DISABLE, 0),
1
);
assert_eq!(
ZSTD_rust_params_allocateChainTable(ZSTD_GREEDY, ZSTD_RUST_PS_ENABLE, 0),
0
);
assert_eq!(
ZSTD_rust_params_allocateChainTable(ZSTD_FAST, ZSTD_RUST_PS_DISABLE, 1),
1
);
}
#[test]
fn external_repcode_and_cdict_tagging_match_boundaries() {
assert_eq!(
ZSTD_rust_params_resolveExternalRepcodeSearch(ZSTD_RUST_PS_AUTO, 9),
ZSTD_RUST_PS_DISABLE
);
assert_eq!(
ZSTD_rust_params_resolveExternalRepcodeSearch(ZSTD_RUST_PS_AUTO, 10),
ZSTD_RUST_PS_ENABLE
);
assert_eq!(
ZSTD_rust_params_resolveExternalRepcodeSearch(ZSTD_RUST_PS_DISABLE, 100),
ZSTD_RUST_PS_DISABLE
);
assert_eq!(
ZSTD_rust_params_resolveExternalRepcodeSearch(ZSTD_RUST_PS_ENABLE, -100),
ZSTD_RUST_PS_ENABLE
);
assert_eq!(
ZSTD_rust_params_cdictIndicesAreTagged(policy_cparams(ZSTD_FAST, 1)),
1
);
assert_eq!(
ZSTD_rust_params_cdictIndicesAreTagged(policy_cparams(ZSTD_DFAST, 1)),
1
);
assert_eq!(
ZSTD_rust_params_cdictIndicesAreTagged(policy_cparams(ZSTD_GREEDY, 1)),
0
);
assert_eq!(
ZSTD_rust_params_cdictIndicesAreTagged(policy_cparams(ZSTD_BTULTRA2, 1)),
0
);
}
#[test]
fn level_tables_match_representative_clevels_entries() {
let large = select_cparams(3, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_RUST_CPM_UNKNOWN);