fix(dict-builder): stabilize optimized dictionary scoring

Optimized dictionary scoring used the requested dictionary ID while building
its scoring CDict, so --dictID could change candidate quality and output
content. Parallel FastCover could also choose different equal-score candidates
based on completion order. Clear the ID only while constructing the scoring
CDict, restore it immediately afterward, and use deterministic (d, k)
tie-breakers in Cover and FastCover. The requested ID now remains output
metadata instead of influencing dictionary content or repeat-run stability.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml scoring_dictionary_id_is_restored_after_the_score --lib
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040; make -j1
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --all-targets
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/cli/Cargo.toml --all-targets
- ulimit -v 41943040; make -j1 -C tests test
This commit is contained in:
2026-07-20 05:22:25 +02:00
parent bbe3a5dd74
commit f3a2caa0e2
2 changed files with 52 additions and 7 deletions
+48 -6
View File
@@ -541,7 +541,10 @@ fn best_finish_slot(
.lock()
.unwrap_or_else(|poison| poison.into_inner());
data.live_jobs = data.live_jobs.wrapping_sub(1);
if selection.totalCompressedSize < data.compressed_size {
let improves_score = selection.totalCompressedSize < data.compressed_size;
let wins_equal_score = selection.totalCompressedSize == data.compressed_size
&& (parameters.d, parameters.k) < (data.parameters.d, data.parameters.k);
if improves_score || wins_equal_score {
if data.dict.is_null() || data.dict_capacity < selection.dictSize {
let replacement = unsafe { malloc_bytes(selection.dictSize) };
if replacement.is_null() && selection.dictSize != 0 {
@@ -629,6 +632,22 @@ unsafe extern "C" {
) -> usize;
}
unsafe fn with_dictionary_id_cleared<T>(
dict: *mut u8,
dict_buffer_capacity: usize,
action: impl FnOnce() -> T,
) -> T {
if dict_buffer_capacity < 8 {
return action();
}
let dict_id = unsafe { dict.add(4).cast::<u32>() };
let saved_id = unsafe { ptr::read_unaligned(dict_id) };
unsafe { ptr::write_unaligned(dict_id, 0) };
let result = action();
unsafe { ptr::write_unaligned(dict_id, saved_id) };
result
}
#[no_mangle]
pub unsafe extern "C" fn COVER_sum(samples_sizes: *const usize, nb_samples: c_uint) -> usize {
if nb_samples == 0 {
@@ -781,12 +800,19 @@ pub unsafe extern "C" fn COVER_checkTotalCompressedSize(
}
let dst = unsafe { malloc_bytes(dst_capacity) };
let cctx = unsafe { ZSTD_createCCtx() };
/* The dictionary ID is output metadata, not dictionary content. The
* fixed-frame `ZSTD_compress_usingCDict()` score includes that metadata,
* so temporarily clear it while constructing the scoring CDict. This
* keeps `--dictID` from changing which candidate wins optimization while
* leaving the requested ID in the finalized dictionary. */
let cdict = unsafe {
ZSTD_createCDict(
dict.cast(),
dict_buffer_capacity,
parameters.zParams.compressionLevel,
)
with_dictionary_id_cleared(dict, dict_buffer_capacity, || {
ZSTD_createCDict(
dict.cast(),
dict_buffer_capacity,
parameters.zParams.compressionLevel,
)
})
};
if dst.is_null() || cctx.is_null() || cdict.is_null() {
if !cctx.is_null() {
@@ -1287,6 +1313,22 @@ mod tests {
);
}
#[test]
fn scoring_dictionary_id_is_restored_after_the_score() {
let mut dictionary = [0u8; 16];
dictionary[4..8].copy_from_slice(&0x1234_5678u32.to_ne_bytes());
let observed_id = unsafe {
with_dictionary_id_cleared(dictionary.as_mut_ptr(), dictionary.len(), || {
ptr::read_unaligned(dictionary.as_ptr().add(4).cast::<u32>())
})
};
assert_eq!(observed_id, 0);
assert_eq!(
u32::from_ne_bytes(dictionary[4..8].try_into().unwrap()),
0x1234_5678
);
}
#[test]
fn context_groups_repeated_dmers_by_sample() {
let samples = [b'a'; 45];
+4 -1
View File
@@ -524,7 +524,10 @@ impl FastBest {
.lock()
.unwrap_or_else(|poison| poison.into_inner());
state.live_jobs = state.live_jobs.wrapping_sub(1);
if selection.totalCompressedSize < state.compressed_size {
let improves_score = selection.totalCompressedSize < state.compressed_size;
let wins_equal_score = selection.totalCompressedSize == state.compressed_size
&& (parameters.d, parameters.k) < (state.parameters.d, state.parameters.k);
if improves_score || wins_equal_score {
if state.dict.is_null() || state.dict_size < selection.dictSize {
unsafe { free_bytes(state.dict) };
let replacement = unsafe { malloc_bytes(selection.dictSize) };