ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move Double-Fast dictionary fill to Rust

Browse Source 
Replace the dictionary-content Double-Fast callback with a live-field
projection and call the existing Rust table-fill leaf directly. The
projection keeps hash tables, window base, match-state index, and parameters
live through the window/publication callbacks, preserving the timing behavior
of the former C adapter. Preserve the excluded-DFast assertion path and the
34-word dictionary bridge layout.

Test Plan:
- git diff --check
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --manifest-path rust/Cargo.toml --tests
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -A clippy::manual-bits -D warnings
- ulimit -v 41943040; make -j1
- capped strategy-2 compress/decompress round trip with cmp
- Full original suite deferred to the next serialized verification step
This commit is contained in:
ddidderr
2026-07-21 21:33:37 +02:00
parent de6c837d7f
commit f89ae75898
2 changed files with 195 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+59 -19
View File
@@ -2805,8 +2805,45 @@ typedef char ZSTD_rust_load_dictionary_fast_table_layout[
&& sizeof(ZSTD_rust_loadDictionaryContentFastTableState)
== 5 * sizeof(void*) + 2 * sizeof(int))
? 1 : -1];
typedef void (*ZSTD_rust_loadDictionaryContent_fillTable_f)(
void* context, const void* iend, int dtlm, int tfp);
/* The Double-Fast dictionary-table leaf is also implemented in Rust. Keep
* pointers to the live match-state fields because window/publish callbacks
* run before the Rust orchestrator selects this branch. */
typedef struct {
U32** hashLong;
U32** hashSmall;
const BYTE** base;
const U32* nextToUpdate;
const U32* hashLog;
const U32* chainLog;
const U32* minMatch;
int fullTableLoad;
int forCDict;
int available;
} ZSTD_rust_loadDictionaryContentDoubleFastTableState;
typedef char ZSTD_rust_load_dictionary_double_fast_table_layout[
(offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hashLong) == 0
&& offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hashSmall)
== sizeof(void*)
&& offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, base)
== 2 * sizeof(void*)
&& offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, nextToUpdate)
== 3 * sizeof(void*)
&& offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hashLog)
== 4 * sizeof(void*)
&& offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, chainLog)
== 5 * sizeof(void*)
&& offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, minMatch)
== 6 * sizeof(void*)
&& offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, fullTableLoad)
== 7 * sizeof(void*)
&& offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, forCDict)
== 7 * sizeof(void*) + sizeof(int)
&& offsetof(ZSTD_rust_loadDictionaryContentDoubleFastTableState, available)
== 7 * sizeof(void*) + 2 * sizeof(int)
&& sizeof(ZSTD_rust_loadDictionaryContentDoubleFastTableState)
== ((7 * sizeof(void*) + 3 * sizeof(int) + sizeof(void*) - 1)
/ sizeof(void*) * sizeof(void*)))
? 1 : -1];
typedef void (*ZSTD_rust_loadDictionaryContent_loadMatch_f)(
void* context, const void* ip);
typedef void (*ZSTD_rust_loadDictionaryContent_loadTree_f)(
@@ -2841,7 +2878,7 @@ typedef struct {
ZSTD_rust_loadDictionaryContent_fillLdm_f fillLdm;
ZSTD_rust_loadDictionaryContent_overflowCorrect_f overflowCorrect;
const ZSTD_rust_loadDictionaryContentFastTableState* fastTable;
ZSTD_rust_loadDictionaryContent_fillTable_f fillDoubleHashTable;
const ZSTD_rust_loadDictionaryContentDoubleFastTableState* doubleFastTable;
ZSTD_rust_loadDictionaryContent_loadMatch_f loadDedicated;
ZSTD_rust_loadDictionaryContent_loadMatch_f loadRow;
ZSTD_rust_loadDictionaryContent_loadMatch_f loadChain;
@@ -2865,6 +2902,9 @@ ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_callback_offset,
ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_fast_table_offset,
offsetof(ZSTD_rust_loadDictionaryContentState,
fastTable) == 25 * sizeof(void*));
ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_double_fast_table_offset,
offsetof(ZSTD_rust_loadDictionaryContentState,
doubleFastTable) == 26 * sizeof(void*));
ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_final_callback_offset,
offsetof(ZSTD_rust_loadDictionaryContentState,
publishFinalIndex) == 31 * sizeof(void*));
@@ -6891,21 +6931,6 @@ static void ZSTD_loadDictionaryContent_overflowCorrect(
context->params, ip, iend);
}
static void ZSTD_loadDictionaryContent_fillDoubleHashTable(
void* opaque, const void* iend, int dtlm, int tfp)
{
ZSTD_loadDictionaryContent_context const* const context =
(const ZSTD_loadDictionaryContent_context*)opaque;
#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR
ZSTD_fillDoubleHashTable(context->matchState, (const BYTE*)iend,
(ZSTD_dictTableLoadMethod_e)dtlm,
(ZSTD_tableFillPurpose_e)tfp);
#else
(void)context; (void)iend; (void)dtlm; (void)tfp;
assert(0); /* shouldn't be called: cparams should've been adjusted. */
#endif
}
static void ZSTD_loadDictionaryContent_loadDedicated(void* opaque, const void* ip)
{
ZSTD_loadDictionaryContent_context const* const context =
@@ -7002,6 +7027,7 @@ static size_t ZSTD_loadDictionaryContent_callback(
{
ZSTD_loadDictionaryContent_context context;
ZSTD_rust_loadDictionaryContentFastTableState fastTable;
ZSTD_rust_loadDictionaryContentDoubleFastTableState doubleFastTable;
ZSTD_rust_loadDictionaryContentState state;
ZSTD_MatchState_t* const ms = (ZSTD_MatchState_t*)matchState;
ldmState_t* const ls = (ldmState_t*)ldmState;
@@ -7025,6 +7051,20 @@ static size_t ZSTD_loadDictionaryContent_callback(
fastTable.minMatch = &ms->cParams.minMatch;
fastTable.fullTableLoad = dtlm == ZSTD_dtlm_full;
fastTable.forCDict = tfp == ZSTD_tfp_forCDict;
doubleFastTable.hashLong = &ms->hashTable;
doubleFastTable.hashSmall = &ms->chainTable;
doubleFastTable.base = &ms->window.base;
doubleFastTable.nextToUpdate = &ms->nextToUpdate;
doubleFastTable.hashLog = &ms->cParams.hashLog;
doubleFastTable.chainLog = &ms->cParams.chainLog;
doubleFastTable.minMatch = &ms->cParams.minMatch;
doubleFastTable.fullTableLoad = dtlm == ZSTD_dtlm_full;
doubleFastTable.forCDict = tfp == ZSTD_tfp_forCDict;
#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR
doubleFastTable.available = 1;
#else
doubleFastTable.available = 0;
#endif
state.callbackContext = &context;
state.currentMax = ZSTD_CURRENT_MAX;
@@ -7052,7 +7092,7 @@ static size_t ZSTD_loadDictionaryContent_callback(
state.fillLdm = ZSTD_loadDictionaryContent_fillLdm;
state.overflowCorrect = ZSTD_loadDictionaryContent_overflowCorrect;
state.fastTable = &fastTable;
state.fillDoubleHashTable = ZSTD_loadDictionaryContent_fillDoubleHashTable;
state.doubleFastTable = &doubleFastTable;
state.loadDedicated = ZSTD_loadDictionaryContent_loadDedicated;
state.loadRow = ZSTD_loadDictionaryContent_loadRow;
state.loadChain = ZSTD_loadDictionaryContent_loadChain;
rust/src/zstd_compress_dictionary.rs
+136 -10
View File
@@ -28,6 +28,7 @@ use crate::zstd_compress_params_api::ZSTD_CCtx_params;
use crate::zstd_compress_stats::{
ZSTD_compressedBlockState_t, ZSTD_rust_resetCompressedBlockState,
};
use crate::zstd_double_fast::ZSTD_rust_fillDoubleHashTable;
use crate::zstd_fast::ZSTD_rust_fillHashTable;
use std::ffi::c_void;
use std::mem::{offset_of, size_of};
@@ -156,8 +157,21 @@ struct ZSTD_rust_loadDictionaryContentFastTableState {
for_cdict: c_int,
}
type LoadDictionaryContentFillTableFn =
unsafe extern "C" fn(context: *mut c_void, iend: *const c_void, dtlm: c_int, tfp: c_int);
/// Direct projection for the Double-Fast dictionary-table leaf.
#[repr(C)]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct ZSTD_rust_loadDictionaryContentDoubleFastTableState {
hash_long: *const *mut c_uint,
hash_small: *const *mut c_uint,
base: *const *const u8,
next_to_update: *const c_uint,
hash_log: *const c_uint,
chain_log: *const c_uint,
min_match: *const c_uint,
full_table_load: c_int,
for_cdict: c_int,
available: c_int,
}
type LoadDictionaryContentLoadMatchFn =
unsafe extern "C" fn(context: *mut c_void, ip: *const c_void);
type LoadDictionaryContentLoadTreeFn =
@@ -200,7 +214,7 @@ pub struct ZSTD_rust_loadDictionaryContentState {
fill_ldm: LoadDictionaryContentFillLdmFn,
overflow_correct: LoadDictionaryContentOverflowCorrectFn,
fast_table: *const ZSTD_rust_loadDictionaryContentFastTableState,
fill_double_hash_table: LoadDictionaryContentFillTableFn,
double_fast_table: *const ZSTD_rust_loadDictionaryContentDoubleFastTableState,
load_dedicated: LoadDictionaryContentLoadMatchFn,
load_row: LoadDictionaryContentLoadMatchFn,
load_chain: LoadDictionaryContentLoadMatchFn,
@@ -218,7 +232,6 @@ const _: () = {
assert!(size_of::<LoadDictionaryContentPublishMatchStateFn>() == size_of::<usize>());
assert!(size_of::<LoadDictionaryContentFillLdmFn>() == size_of::<usize>());
assert!(size_of::<LoadDictionaryContentOverflowCorrectFn>() == size_of::<usize>());
assert!(size_of::<LoadDictionaryContentFillTableFn>() == size_of::<usize>());
assert!(offset_of!(ZSTD_rust_loadDictionaryContentFastTableState, hash_table) == 0);
assert!(offset_of!(ZSTD_rust_loadDictionaryContentFastTableState, base) == size_of::<usize>());
assert!(
@@ -249,6 +262,49 @@ const _: () = {
size_of::<ZSTD_rust_loadDictionaryContentFastTableState>()
== 5 * size_of::<usize>() + 2 * size_of::<c_int>()
);
assert!(offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hash_long) == 0);
assert!(
offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hash_small)
== size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, base)
== 2 * size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, next_to_update)
== 3 * size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, hash_log)
== 4 * size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, chain_log)
== 5 * size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, min_match)
== 6 * size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, full_table_load)
== 7 * size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, for_cdict)
== 7 * size_of::<usize>() + size_of::<c_int>()
);
assert!(
offset_of!(ZSTD_rust_loadDictionaryContentDoubleFastTableState, available)
== 7 * size_of::<usize>() + 2 * size_of::<c_int>()
);
assert!(
size_of::<ZSTD_rust_loadDictionaryContentDoubleFastTableState>()
== (7 * size_of::<usize>() + 3 * size_of::<c_int>())
.div_ceil(size_of::<usize>())
* size_of::<usize>()
);
assert!(size_of::<LoadDictionaryContentLoadMatchFn>() == size_of::<usize>());
assert!(size_of::<LoadDictionaryContentLoadTreeFn>() == size_of::<usize>());
assert!(size_of::<LoadDictionaryContentPublishFinalIndexFn>() == size_of::<usize>());
@@ -261,6 +317,10 @@ const _: () = {
assert!(
offset_of!(ZSTD_rust_loadDictionaryContentState, fast_table) == 25 * size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_loadDictionaryContentState, double_fast_table)
== 26 * size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_loadDictionaryContentState, publish_final_index)
== 31 * size_of::<usize>()
@@ -385,6 +445,31 @@ unsafe fn fill_fast_dictionary_table(
}
}
#[inline]
unsafe fn fill_double_fast_dictionary_table(
double_fast_table: &ZSTD_rust_loadDictionaryContentDoubleFastTableState,
end: *const c_void,
) -> bool {
if double_fast_table.available == 0 {
return false;
}
unsafe {
ZSTD_rust_fillDoubleHashTable(
*double_fast_table.hash_long,
*double_fast_table.hash_small,
*double_fast_table.base,
*double_fast_table.next_to_update,
end,
*double_fast_table.hash_log,
*double_fast_table.chain_log,
*double_fast_table.min_match,
double_fast_table.full_table_load,
double_fast_table.for_cdict,
)
}
true
}
unsafe fn load_dictionary_content(
state: &ZSTD_rust_loadDictionaryContentState,
src: *const c_void,
@@ -465,12 +550,10 @@ unsafe fn load_dictionary_content(
fill_fast_dictionary_table(fast_table, iend.cast())
},
DictionaryTablePolicy::DoubleFast => unsafe {
(state.fill_double_hash_table)(
state.callback_context,
iend.cast(),
state.dtlm as c_int,
state.tfp as c_int,
)
let double_fast_table = &*state.double_fast_table;
if !fill_double_fast_dictionary_table(double_fast_table, iend.cast()) {
(state.assert_invalid_strategy)(state.callback_context);
}
},
DictionaryTablePolicy::Dedicated => unsafe {
(state.load_dedicated)(state.callback_context, hash_ip.cast())
@@ -6753,4 +6836,47 @@ mod tests {
assert_eq!(projection.full_table_load, 1);
assert_eq!(projection.for_cdict, 1);
}
#[test]
fn double_fast_dictionary_table_projection_dispatches_to_rust_leaf() {
let hash_log = 4u32;
let chain_log = 4u32;
let mut hash_long = vec![0u32; 1usize << (hash_log + 8)];
let mut hash_small = vec![0u32; 1usize << (chain_log + 8)];
let mut input = [0u8; 64];
for (index, byte) in input.iter_mut().enumerate() {
*byte = (index as u8).wrapping_mul(29).wrapping_add(7);
}
let hash_long_ptr = hash_long.as_mut_ptr();
let hash_small_ptr = hash_small.as_mut_ptr();
let base_ptr = input.as_ptr();
let next_to_update = 17u32;
let min_match = 4u32;
let projection = ZSTD_rust_loadDictionaryContentDoubleFastTableState {
hash_long: &hash_long_ptr,
hash_small: &hash_small_ptr,
base: &base_ptr,
next_to_update: &next_to_update,
hash_log: &hash_log,
chain_log: &chain_log,
min_match: &min_match,
full_table_load: 1,
for_cdict: 1,
available: 1,
};
assert!(unsafe {
fill_double_fast_dictionary_table(
&projection,
input.as_ptr().wrapping_add(input.len()).cast(),
)
});
assert!(hash_long.iter().any(|&entry| entry != 0));
assert!(hash_small.iter().any(|&entry| entry != 0));
assert_eq!(unsafe { *projection.next_to_update }, 17);
assert_eq!(projection.full_table_load, 1);
assert_eq!(projection.for_cdict, 1);
assert_eq!(projection.available, 1);
}
}