From 7a345bd2f7c580f5a1db911cc1b2e4d79f2c8ba3 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Sat, 18 Jul 2026 09:18:36 +0200 Subject: [PATCH] feat(compress): move dictionary attachment predicate to Rust Keep the private CDict and CCtx parameter layouts in C while passing only the five scalar inputs needed by the dictionary-attachment heuristic. Rust now owns the exact per-strategy cutoffs and preserves unknown-size handling, dedicated-search short-circuiting, force-attach and force-copy precedence, and the force-window prohibition. The existing C helper and reset/copy call paths remain unchanged. Test Plan: - `cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression dictionary_attachment` -- passed (2 tests) - `make -B -C lib -j2 lib` -- passed - `make -C tests -j2 fuzzer` -- passed - `./tests/fuzzer -s4560 -t47 -i48 -v` -- passed - Rust clippy for the library, benches, and tests before and after nightly formatting -- passed - `cargo +nightly fmt --manifest-path rust/Cargo.toml` -- passed - `git diff --cached --check` -- passed --- lib/compress/zstd_compress.c | 35 +++------ rust/src/zstd_compress_params.rs | 130 +++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 26 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 37c25fe8c..61d345bb2 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -124,6 +124,9 @@ int ZSTD_rust_params_cdictIndicesAreTagged(ZSTD_compressionParameters cParams); int ZSTD_rust_params_dedicatedDictSearchIsSupported(ZSTD_compressionParameters cParams); U32 ZSTD_rust_params_dedicatedDictSearch_getHashLog(U32 hashLog); U32 ZSTD_rust_params_dedicatedDictSearch_revertHashLog(U32 hashLog); +int ZSTD_rust_params_shouldAttachDict(int strategy, int dedicatedDictSearch, + U64 pledgedSrcSize, int attachDictPref, + int forceWindow); /* CCtx parameter state is mirrored by rust/src/zstd_compress_params_api.rs. * Rust owns parameter bounds and clamping; C exposes only the @@ -1786,36 +1789,16 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); } -/* These are the approximate sizes for each strategy past which copying the - * dictionary tables into the working context is faster than using them - * in-place. - */ -static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = { - 8 KB, /* unused */ - 8 KB, /* ZSTD_fast */ - 16 KB, /* ZSTD_dfast */ - 32 KB, /* ZSTD_greedy */ - 32 KB, /* ZSTD_lazy */ - 32 KB, /* ZSTD_lazy2 */ - 32 KB, /* ZSTD_btlazy2 */ - 32 KB, /* ZSTD_btopt */ - 8 KB, /* ZSTD_btultra */ - 8 KB /* ZSTD_btultra2 */ -}; - static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, const ZSTD_CCtx_params* params, U64 pledgedSrcSize) { - size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy]; - int const dedicatedDictSearch = cdict->matchState.dedicatedDictSearch; - return dedicatedDictSearch - || ( ( pledgedSrcSize <= cutoff - || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN - || params->attachDictPref == ZSTD_dictForceAttach ) - && params->attachDictPref != ZSTD_dictForceCopy - && !params->forceWindow ); /* dictMatchState isn't correctly - * handled in _enforceMaxDist */ + return ZSTD_rust_params_shouldAttachDict( + cdict->matchState.cParams.strategy, + cdict->matchState.dedicatedDictSearch, + pledgedSrcSize, + (int)params->attachDictPref, + params->forceWindow); } static size_t diff --git a/rust/src/zstd_compress_params.rs b/rust/src/zstd_compress_params.rs index 53c54cd42..591a9e96d 100644 --- a/rust/src/zstd_compress_params.rs +++ b/rust/src/zstd_compress_params.rs @@ -54,6 +54,9 @@ const ZSTD_BTOPT: c_int = 7; const ZSTD_BTULTRA: c_int = 8; const ZSTD_BTULTRA2: c_int = 9; +const ZSTD_DICT_FORCE_ATTACH: c_int = 1; +const ZSTD_DICT_FORCE_COPY: c_int = 2; + const ZSTD_C_COMPRESSION_LEVEL: c_int = 100; const ZSTD_C_WINDOW_LOG: c_int = 101; const ZSTD_C_HASH_LOG: c_int = 102; @@ -452,6 +455,36 @@ fn dedicated_dict_search_revert_hash_log(hash_log: u32) -> u32 { .max(ZSTD_HASHLOG_MIN as u32) } +const ATTACH_DICT_SIZE_CUTOFFS: [u64; 10] = [ + 8 * 1024, /* unused */ + 8 * 1024, /* ZSTD_fast */ + 16 * 1024, /* ZSTD_dfast */ + 32 * 1024, /* ZSTD_greedy */ + 32 * 1024, /* ZSTD_lazy */ + 32 * 1024, /* ZSTD_lazy2 */ + 32 * 1024, /* ZSTD_btlazy2 */ + 32 * 1024, /* ZSTD_btopt */ + 8 * 1024, /* ZSTD_btultra */ + 8 * 1024, /* ZSTD_btultra2 */ +]; + +#[inline] +fn should_attach_dict( + strategy: c_int, + dedicated_dict_search: c_int, + pledged_src_size: u64, + attach_dict_pref: c_int, + force_window: c_int, +) -> bool { + let cutoff = ATTACH_DICT_SIZE_CUTOFFS[strategy as usize]; + dedicated_dict_search != 0 + || ((pledged_src_size <= cutoff + || pledged_src_size == ZSTD_CONTENTSIZE_UNKNOWN + || attach_dict_pref == ZSTD_DICT_FORCE_ATTACH) + && attach_dict_pref != ZSTD_DICT_FORCE_COPY + && force_window == 0) +} + /// C ABI for `ZSTD_CDictIndicesAreTagged()`. #[no_mangle] pub extern "C" fn ZSTD_rust_params_cdictIndicesAreTagged( @@ -480,6 +513,24 @@ pub extern "C" fn ZSTD_rust_params_dedicatedDictSearch_revertHashLog(hash_log: u dedicated_dict_search_revert_hash_log(hash_log) } +/// C ABI for `ZSTD_shouldAttachDict()`. +#[no_mangle] +pub extern "C" fn ZSTD_rust_params_shouldAttachDict( + strategy: c_int, + dedicated_dict_search: c_int, + pledged_src_size: u64, + attach_dict_pref: c_int, + force_window: c_int, +) -> c_int { + c_int::from(should_attach_dict( + strategy, + dedicated_dict_search, + pledged_src_size, + attach_dict_pref, + force_window, + )) +} + #[inline] fn dict_and_window_log(window_log: u32, src_size: u64, dict_size: u64) -> u32 { /* 1ULL << ZSTD_WINDOWLOG_MAX, which is smaller for 32-bit builds. */ @@ -1338,6 +1389,85 @@ mod tests { ); } + #[test] + fn dictionary_attachment_matches_strategy_size_and_preference_boundaries() { + let strategy_cutoffs = [ + (ZSTD_FAST, 8 * 1024), + (ZSTD_DFAST, 16 * 1024), + (ZSTD_GREEDY, 32 * 1024), + (ZSTD_LAZY, 32 * 1024), + (ZSTD_LAZY2, 32 * 1024), + (ZSTD_BTLAZY2, 32 * 1024), + (ZSTD_BTOPT, 32 * 1024), + (ZSTD_BTULTRA, 8 * 1024), + (ZSTD_BTULTRA2, 8 * 1024), + ]; + + for (strategy, cutoff) in strategy_cutoffs { + assert_eq!( + ZSTD_rust_params_shouldAttachDict(strategy, 0, cutoff, 0, 0,), + 1, + "attach at cutoff for strategy {strategy}", + ); + assert_eq!( + ZSTD_rust_params_shouldAttachDict(strategy, 0, cutoff + 1, 0, 0,), + 0, + "copy above cutoff for strategy {strategy}", + ); + } + + assert_eq!( + ZSTD_rust_params_shouldAttachDict(ZSTD_FAST, 0, ZSTD_CONTENTSIZE_UNKNOWN, 0, 0,), + 1 + ); + assert_eq!( + ZSTD_rust_params_shouldAttachDict( + ZSTD_FAST, + 0, + 8 * 1024 + 1, + ZSTD_DICT_FORCE_ATTACH, + 0, + ), + 1 + ); + assert_eq!( + ZSTD_rust_params_shouldAttachDict( + ZSTD_FAST, + 0, + ZSTD_CONTENTSIZE_UNKNOWN, + ZSTD_DICT_FORCE_COPY, + 0, + ), + 0 + ); + assert_eq!( + ZSTD_rust_params_shouldAttachDict(ZSTD_FAST, 0, 1, ZSTD_DICT_FORCE_COPY, 0), + 0 + ); + } + + #[test] + fn dictionary_attachment_preserves_dedicated_search_and_force_window_precedence() { + assert_eq!( + ZSTD_rust_params_shouldAttachDict(ZSTD_BTULTRA2, 1, u64::MAX, ZSTD_DICT_FORCE_COPY, 1), + 1 + ); + assert_eq!( + ZSTD_rust_params_shouldAttachDict(ZSTD_BTULTRA2, 0, 1, 0, 1,), + 0 + ); + assert_eq!( + ZSTD_rust_params_shouldAttachDict( + ZSTD_BTULTRA2, + 0, + ZSTD_CONTENTSIZE_UNKNOWN, + ZSTD_DICT_FORCE_ATTACH, + 1, + ), + 0 + ); + } + #[test] fn external_sequence_validation_preserves_its_int_mode() { for mode in [c_int::MIN, -1, 0, 1, c_int::MAX] {