ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move dictionary attachment predicate to Rust

Browse Source 
Keep the private CDict and CCtx parameter layouts in C while passing only the
five scalar inputs needed by the dictionary-attachment heuristic. Rust now
owns the exact per-strategy cutoffs and preserves unknown-size handling,
dedicated-search short-circuiting, force-attach and force-copy precedence, and
the force-window prohibition. The existing C helper and reset/copy call paths
remain unchanged.

Test Plan:
- `cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression dictionary_attachment` -- passed (2 tests)
- `make -B -C lib -j2 lib` -- passed
- `make -C tests -j2 fuzzer` -- passed
- `./tests/fuzzer -s4560 -t47 -i48 -v` -- passed
- Rust clippy for the library, benches, and tests before and after nightly formatting -- passed
- `cargo +nightly fmt --manifest-path rust/Cargo.toml` -- passed
- `git diff --cached --check` -- passed
This commit is contained in:
ddidderr
2026-07-18 09:18:36 +02:00
parent ae9667052a
commit 7a345bd2f7
2 changed files with 139 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/zstd_compress_params.rs
+130
View File
@@ -54,6 +54,9 @@ const ZSTD_BTOPT: c_int = 7;
const ZSTD_BTULTRA: c_int = 8;
const ZSTD_BTULTRA2: c_int = 9;
const ZSTD_DICT_FORCE_ATTACH: c_int = 1;
const ZSTD_DICT_FORCE_COPY: c_int = 2;
const ZSTD_C_COMPRESSION_LEVEL: c_int = 100;
const ZSTD_C_WINDOW_LOG: c_int = 101;
const ZSTD_C_HASH_LOG: c_int = 102;
@@ -452,6 +455,36 @@ fn dedicated_dict_search_revert_hash_log(hash_log: u32) -> u32 {
.max(ZSTD_HASHLOG_MIN as u32)
}
const ATTACH_DICT_SIZE_CUTOFFS: [u64; 10] = [
8 * 1024, /* unused */
8 * 1024, /* ZSTD_fast */
16 * 1024, /* ZSTD_dfast */
32 * 1024, /* ZSTD_greedy */
32 * 1024, /* ZSTD_lazy */
32 * 1024, /* ZSTD_lazy2 */
32 * 1024, /* ZSTD_btlazy2 */
32 * 1024, /* ZSTD_btopt */
8 * 1024, /* ZSTD_btultra */
8 * 1024, /* ZSTD_btultra2 */
];
#[inline]
fn should_attach_dict(
strategy: c_int,
dedicated_dict_search: c_int,
pledged_src_size: u64,
attach_dict_pref: c_int,
force_window: c_int,
) -> bool {
let cutoff = ATTACH_DICT_SIZE_CUTOFFS[strategy as usize];
dedicated_dict_search != 0
|| ((pledged_src_size <= cutoff
|| pledged_src_size == ZSTD_CONTENTSIZE_UNKNOWN
|| attach_dict_pref == ZSTD_DICT_FORCE_ATTACH)
&& attach_dict_pref != ZSTD_DICT_FORCE_COPY
&& force_window == 0)
}
/// C ABI for `ZSTD_CDictIndicesAreTagged()`.
#[no_mangle]
pub extern "C" fn ZSTD_rust_params_cdictIndicesAreTagged(
@@ -480,6 +513,24 @@ pub extern "C" fn ZSTD_rust_params_dedicatedDictSearch_revertHashLog(hash_log: u
dedicated_dict_search_revert_hash_log(hash_log)
}
/// C ABI for `ZSTD_shouldAttachDict()`.
#[no_mangle]
pub extern "C" fn ZSTD_rust_params_shouldAttachDict(
strategy: c_int,
dedicated_dict_search: c_int,
pledged_src_size: u64,
attach_dict_pref: c_int,
force_window: c_int,
) -> c_int {
c_int::from(should_attach_dict(
strategy,
dedicated_dict_search,
pledged_src_size,
attach_dict_pref,
force_window,
))
}
#[inline]
fn dict_and_window_log(window_log: u32, src_size: u64, dict_size: u64) -> u32 {
/* 1ULL << ZSTD_WINDOWLOG_MAX, which is smaller for 32-bit builds. */
@@ -1338,6 +1389,85 @@ mod tests {
);
}
#[test]
fn dictionary_attachment_matches_strategy_size_and_preference_boundaries() {
let strategy_cutoffs = [
(ZSTD_FAST, 8 * 1024),
(ZSTD_DFAST, 16 * 1024),
(ZSTD_GREEDY, 32 * 1024),
(ZSTD_LAZY, 32 * 1024),
(ZSTD_LAZY2, 32 * 1024),
(ZSTD_BTLAZY2, 32 * 1024),
(ZSTD_BTOPT, 32 * 1024),
(ZSTD_BTULTRA, 8 * 1024),
(ZSTD_BTULTRA2, 8 * 1024),
];
for (strategy, cutoff) in strategy_cutoffs {
assert_eq!(
ZSTD_rust_params_shouldAttachDict(strategy, 0, cutoff, 0, 0,),
1,
"attach at cutoff for strategy {strategy}",
);
assert_eq!(
ZSTD_rust_params_shouldAttachDict(strategy, 0, cutoff + 1, 0, 0,),
0,
"copy above cutoff for strategy {strategy}",
);
}
assert_eq!(
ZSTD_rust_params_shouldAttachDict(ZSTD_FAST, 0, ZSTD_CONTENTSIZE_UNKNOWN, 0, 0,),
1
);
assert_eq!(
ZSTD_rust_params_shouldAttachDict(
ZSTD_FAST,
0,
8 * 1024 + 1,
ZSTD_DICT_FORCE_ATTACH,
0,
),
1
);
assert_eq!(
ZSTD_rust_params_shouldAttachDict(
ZSTD_FAST,
0,
ZSTD_CONTENTSIZE_UNKNOWN,
ZSTD_DICT_FORCE_COPY,
0,
),
0
);
assert_eq!(
ZSTD_rust_params_shouldAttachDict(ZSTD_FAST, 0, 1, ZSTD_DICT_FORCE_COPY, 0),
0
);
}
#[test]
fn dictionary_attachment_preserves_dedicated_search_and_force_window_precedence() {
assert_eq!(
ZSTD_rust_params_shouldAttachDict(ZSTD_BTULTRA2, 1, u64::MAX, ZSTD_DICT_FORCE_COPY, 1),
1
);
assert_eq!(
ZSTD_rust_params_shouldAttachDict(ZSTD_BTULTRA2, 0, 1, 0, 1,),
0
);
assert_eq!(
ZSTD_rust_params_shouldAttachDict(
ZSTD_BTULTRA2,
0,
ZSTD_CONTENTSIZE_UNKNOWN,
ZSTD_DICT_FORCE_ATTACH,
1,
),
0
);
}
#[test]
fn external_sequence_validation_preserves_its_int_mode() {
for mode in [c_int::MIN, -1, 0, 1, c_int::MAX] {