ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move dictionary attachment predicate to Rust

Browse Source 
Keep the private CDict and CCtx parameter layouts in C while passing only the
five scalar inputs needed by the dictionary-attachment heuristic. Rust now
owns the exact per-strategy cutoffs and preserves unknown-size handling,
dedicated-search short-circuiting, force-attach and force-copy precedence, and
the force-window prohibition. The existing C helper and reset/copy call paths
remain unchanged.

Test Plan:
- `cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression dictionary_attachment` -- passed (2 tests)
- `make -B -C lib -j2 lib` -- passed
- `make -C tests -j2 fuzzer` -- passed
- `./tests/fuzzer -s4560 -t47 -i48 -v` -- passed
- Rust clippy for the library, benches, and tests before and after nightly formatting -- passed
- `cargo +nightly fmt --manifest-path rust/Cargo.toml` -- passed
- `git diff --cached --check` -- passed
This commit is contained in:
ddidderr
2026-07-18 09:18:36 +02:00
parent ae9667052a
commit 7a345bd2f7
2 changed files with 139 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+9 -26
View File
@@ -124,6 +124,9 @@ int ZSTD_rust_params_cdictIndicesAreTagged(ZSTD_compressionParameters cParams);
int ZSTD_rust_params_dedicatedDictSearchIsSupported(ZSTD_compressionParameters cParams);
U32 ZSTD_rust_params_dedicatedDictSearch_getHashLog(U32 hashLog);
U32 ZSTD_rust_params_dedicatedDictSearch_revertHashLog(U32 hashLog);
int ZSTD_rust_params_shouldAttachDict(int strategy, int dedicatedDictSearch,
U64 pledgedSrcSize, int attachDictPref,
int forceWindow);
/* CCtx parameter state is mirrored by rust/src/zstd_compress_params_api.rs.
* Rust owns parameter bounds and clamping; C exposes only the
@@ -1786,36 +1789,16 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window));
}
/* These are the approximate sizes for each strategy past which copying the
* dictionary tables into the working context is faster than using them
* in-place.
*/
static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = {
8 KB, /* unused */
8 KB, /* ZSTD_fast */
16 KB, /* ZSTD_dfast */
32 KB, /* ZSTD_greedy */
32 KB, /* ZSTD_lazy */
32 KB, /* ZSTD_lazy2 */
32 KB, /* ZSTD_btlazy2 */
32 KB, /* ZSTD_btopt */
8 KB, /* ZSTD_btultra */
8 KB /* ZSTD_btultra2 */
};
static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict,
const ZSTD_CCtx_params* params,
U64 pledgedSrcSize)
{
size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy];
int const dedicatedDictSearch = cdict->matchState.dedicatedDictSearch;
return dedicatedDictSearch
|| ( ( pledgedSrcSize <= cutoff
|| pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN
|| params->attachDictPref == ZSTD_dictForceAttach )
&& params->attachDictPref != ZSTD_dictForceCopy
&& !params->forceWindow ); /* dictMatchState isn't correctly
* handled in _enforceMaxDist */
return ZSTD_rust_params_shouldAttachDict(
cdict->matchState.cParams.strategy,
cdict->matchState.dedicatedDictSearch,
pledgedSrcSize,
(int)params->attachDictPref,
params->forceWindow);
}
static size_t
rust/src/zstd_compress_params.rs
+130
View File
@@ -54,6 +54,9 @@ const ZSTD_BTOPT: c_int = 7;
const ZSTD_BTULTRA: c_int = 8;
const ZSTD_BTULTRA2: c_int = 9;
const ZSTD_DICT_FORCE_ATTACH: c_int = 1;
const ZSTD_DICT_FORCE_COPY: c_int = 2;
const ZSTD_C_COMPRESSION_LEVEL: c_int = 100;
const ZSTD_C_WINDOW_LOG: c_int = 101;
const ZSTD_C_HASH_LOG: c_int = 102;
@@ -452,6 +455,36 @@ fn dedicated_dict_search_revert_hash_log(hash_log: u32) -> u32 {
.max(ZSTD_HASHLOG_MIN as u32)
}
const ATTACH_DICT_SIZE_CUTOFFS: [u64; 10] = [
8 * 1024, /* unused */
8 * 1024, /* ZSTD_fast */
16 * 1024, /* ZSTD_dfast */
32 * 1024, /* ZSTD_greedy */
32 * 1024, /* ZSTD_lazy */
32 * 1024, /* ZSTD_lazy2 */
32 * 1024, /* ZSTD_btlazy2 */
32 * 1024, /* ZSTD_btopt */
8 * 1024, /* ZSTD_btultra */
8 * 1024, /* ZSTD_btultra2 */
];
#[inline]
fn should_attach_dict(
strategy: c_int,
dedicated_dict_search: c_int,
pledged_src_size: u64,
attach_dict_pref: c_int,
force_window: c_int,
) -> bool {
let cutoff = ATTACH_DICT_SIZE_CUTOFFS[strategy as usize];
dedicated_dict_search != 0
|| ((pledged_src_size <= cutoff
|| pledged_src_size == ZSTD_CONTENTSIZE_UNKNOWN
|| attach_dict_pref == ZSTD_DICT_FORCE_ATTACH)
&& attach_dict_pref != ZSTD_DICT_FORCE_COPY
&& force_window == 0)
}
/// C ABI for `ZSTD_CDictIndicesAreTagged()`.
#[no_mangle]
pub extern "C" fn ZSTD_rust_params_cdictIndicesAreTagged(
@@ -480,6 +513,24 @@ pub extern "C" fn ZSTD_rust_params_dedicatedDictSearch_revertHashLog(hash_log: u
dedicated_dict_search_revert_hash_log(hash_log)
}
/// C ABI for `ZSTD_shouldAttachDict()`.
#[no_mangle]
pub extern "C" fn ZSTD_rust_params_shouldAttachDict(
strategy: c_int,
dedicated_dict_search: c_int,
pledged_src_size: u64,
attach_dict_pref: c_int,
force_window: c_int,
) -> c_int {
c_int::from(should_attach_dict(
strategy,
dedicated_dict_search,
pledged_src_size,
attach_dict_pref,
force_window,
))
}
#[inline]
fn dict_and_window_log(window_log: u32, src_size: u64, dict_size: u64) -> u32 {
/* 1ULL << ZSTD_WINDOWLOG_MAX, which is smaller for 32-bit builds. */
@@ -1338,6 +1389,85 @@ mod tests {
);
}
#[test]
fn dictionary_attachment_matches_strategy_size_and_preference_boundaries() {
let strategy_cutoffs = [
(ZSTD_FAST, 8 * 1024),
(ZSTD_DFAST, 16 * 1024),
(ZSTD_GREEDY, 32 * 1024),
(ZSTD_LAZY, 32 * 1024),
(ZSTD_LAZY2, 32 * 1024),
(ZSTD_BTLAZY2, 32 * 1024),
(ZSTD_BTOPT, 32 * 1024),
(ZSTD_BTULTRA, 8 * 1024),
(ZSTD_BTULTRA2, 8 * 1024),
];
for (strategy, cutoff) in strategy_cutoffs {
assert_eq!(
ZSTD_rust_params_shouldAttachDict(strategy, 0, cutoff, 0, 0,),
1,
"attach at cutoff for strategy {strategy}",
);
assert_eq!(
ZSTD_rust_params_shouldAttachDict(strategy, 0, cutoff + 1, 0, 0,),
0,
"copy above cutoff for strategy {strategy}",
);
}
assert_eq!(
ZSTD_rust_params_shouldAttachDict(ZSTD_FAST, 0, ZSTD_CONTENTSIZE_UNKNOWN, 0, 0,),
1
);
assert_eq!(
ZSTD_rust_params_shouldAttachDict(
ZSTD_FAST,
0,
8 * 1024 + 1,
ZSTD_DICT_FORCE_ATTACH,
0,
),
1
);
assert_eq!(
ZSTD_rust_params_shouldAttachDict(
ZSTD_FAST,
0,
ZSTD_CONTENTSIZE_UNKNOWN,
ZSTD_DICT_FORCE_COPY,
0,
),
0
);
assert_eq!(
ZSTD_rust_params_shouldAttachDict(ZSTD_FAST, 0, 1, ZSTD_DICT_FORCE_COPY, 0),
0
);
}
#[test]
fn dictionary_attachment_preserves_dedicated_search_and_force_window_precedence() {
assert_eq!(
ZSTD_rust_params_shouldAttachDict(ZSTD_BTULTRA2, 1, u64::MAX, ZSTD_DICT_FORCE_COPY, 1),
1
);
assert_eq!(
ZSTD_rust_params_shouldAttachDict(ZSTD_BTULTRA2, 0, 1, 0, 1,),
0
);
assert_eq!(
ZSTD_rust_params_shouldAttachDict(
ZSTD_BTULTRA2,
0,
ZSTD_CONTENTSIZE_UNKNOWN,
ZSTD_DICT_FORCE_ATTACH,
1,
),
0
);
}
#[test]
fn external_sequence_validation_preserves_its_int_mode() {
for mode in [c_int::MIN, -1, 0, 1, c_int::MAX] {