ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move dictionary state publication to Rust

Browse Source 
Move LDM loadedDictEnd and final nextToUpdate publication out of the C
dictionary-content callbacks. Rust now writes through compact projections with
live window bases, while C retains the configuration-sensitive window,
matchfinder, and LDM operations.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --tests
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --tests -- -A clippy::manual-bits -D warnings
- ulimit -v 41943040; make -j1
- ulimit -v 41943040; make -j1 -C tests test
This commit is contained in:
ddidderr
2026-07-21 23:01:49 +02:00
parent 87b616c49b
commit cf6a316693
2 changed files with 144 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+25 -27
View File
@@ -2767,12 +2767,22 @@ typedef void (*ZSTD_rust_loadDictionaryContent_assertWindowEmpty_f)(
void* context, int ldm);
typedef void (*ZSTD_rust_loadDictionaryContent_windowUpdate_f)(
void* context, int ldm, const void* src, size_t srcSize);
typedef void (*ZSTD_rust_loadDictionaryContent_setLdmLoadedDictEnd_f)(
void* context, const void* iend, int forceWindow);
typedef void (*ZSTD_rust_loadDictionaryContent_fillLdm_f)(
void* context, const void* ip, const void* iend);
typedef void (*ZSTD_rust_loadDictionaryContent_overflowCorrect_f)(
void* context, const void* ip, const void* iend);
typedef struct {
const BYTE** base;
U32* loadedDictEnd;
} ZSTD_rust_loadDictionaryContentLdmPublicationState;
typedef char ZSTD_rust_load_dictionary_ldm_publication_layout[
(offsetof(ZSTD_rust_loadDictionaryContentLdmPublicationState, base) == 0
&& offsetof(ZSTD_rust_loadDictionaryContentLdmPublicationState,
loadedDictEnd)
== sizeof(void*)
&& sizeof(ZSTD_rust_loadDictionaryContentLdmPublicationState)
== 2 * sizeof(void*))
? 1 : -1];
typedef struct {
const BYTE** base;
U32* nextToUpdate;
@@ -2866,8 +2876,6 @@ typedef void (*ZSTD_rust_loadDictionaryContent_loadMatch_f)(
void* context, const void* ip);
typedef void (*ZSTD_rust_loadDictionaryContent_loadTree_f)(
void* context, const void* ip, const void* iend);
typedef void (*ZSTD_rust_loadDictionaryContent_publishFinalIndex_f)(
void* context, const void* iend);
typedef struct {
void* callbackContext;
@@ -2891,7 +2899,7 @@ typedef struct {
ZSTD_rust_loadDictionaryContent_assert_f assertCParams;
ZSTD_rust_loadDictionaryContent_assertWindowEmpty_f assertWindowEmpty;
ZSTD_rust_loadDictionaryContent_windowUpdate_f windowUpdate;
ZSTD_rust_loadDictionaryContent_setLdmLoadedDictEnd_f setLdmLoadedDictEnd;
const ZSTD_rust_loadDictionaryContentLdmPublicationState* setLdmLoadedDictEnd;
const ZSTD_rust_loadDictionaryContentMatchStatePublicationState* publishMatchState;
ZSTD_rust_loadDictionaryContent_fillLdm_f fillLdm;
ZSTD_rust_loadDictionaryContent_overflowCorrect_f overflowCorrect;
@@ -2901,7 +2909,7 @@ typedef struct {
ZSTD_rust_loadDictionaryContent_loadMatch_f loadRow;
ZSTD_rust_loadDictionaryContent_loadMatch_f loadChain;
ZSTD_rust_loadDictionaryContent_loadTree_f loadTree;
ZSTD_rust_loadDictionaryContent_publishFinalIndex_f publishFinalIndex;
const ZSTD_rust_loadDictionaryContentMatchStatePublicationState* publishFinalIndex;
ZSTD_rust_loadDictionaryContent_assert_f assertLazyConfiguration;
ZSTD_rust_loadDictionaryContent_assert_f assertInvalidStrategy;
} ZSTD_rust_loadDictionaryContentState;
@@ -2917,6 +2925,12 @@ ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_scalar_offset,
ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_callback_offset,
offsetof(ZSTD_rust_loadDictionaryContentState,
assertCParams) == 18 * sizeof(void*));
ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_ldm_publication_offset,
offsetof(ZSTD_rust_loadDictionaryContentState,
setLdmLoadedDictEnd) == 21 * sizeof(void*));
ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_match_publication_offset,
offsetof(ZSTD_rust_loadDictionaryContentState,
publishMatchState) == 22 * sizeof(void*));
ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_fast_table_offset,
offsetof(ZSTD_rust_loadDictionaryContentState,
fastTable) == 25 * sizeof(void*));
@@ -6904,17 +6918,6 @@ static void ZSTD_loadDictionaryContent_windowUpdate(
}
}
static void ZSTD_loadDictionaryContent_setLdmLoadedDictEnd(
void* opaque, const void* iend, int forceWindow)
{
ZSTD_loadDictionaryContent_context const* const context =
(const ZSTD_loadDictionaryContent_context*)opaque;
assert(context->ldmState != NULL);
context->ldmState->loadedDictEnd = forceWindow
? 0
: (U32)((const BYTE*)iend - context->ldmState->window.base);
}
static void ZSTD_loadDictionaryContent_fillLdm(
void* opaque, const void* ip, const void* iend)
{
@@ -7000,14 +7003,6 @@ static void ZSTD_loadDictionaryContent_loadTree(
#endif
}
static void ZSTD_loadDictionaryContent_publishFinalIndex(void* opaque, const void* iend)
{
ZSTD_loadDictionaryContent_context const* const context =
(const ZSTD_loadDictionaryContent_context*)opaque;
context->matchState->nextToUpdate =
(U32)((const BYTE*)iend - context->matchState->window.base);
}
static void ZSTD_loadDictionaryContent_assertLazyConfiguration(void* opaque)
{
ZSTD_loadDictionaryContent_context const* const context =
@@ -7030,6 +7025,7 @@ static size_t ZSTD_loadDictionaryContent_callback(
int dtlm, int tfp)
{
ZSTD_loadDictionaryContent_context context;
ZSTD_rust_loadDictionaryContentLdmPublicationState ldmPublication;
ZSTD_rust_loadDictionaryContentMatchStatePublicationState matchStatePublication;
ZSTD_rust_loadDictionaryContentFastTableState fastTable;
ZSTD_rust_loadDictionaryContentDoubleFastTableState doubleFastTable;
@@ -7048,6 +7044,8 @@ static size_t ZSTD_loadDictionaryContent_callback(
matchStatePublication.nextToUpdate = &ms->nextToUpdate;
matchStatePublication.loadedDictEnd = &ms->loadedDictEnd;
matchStatePublication.forceNonContiguous = &ms->forceNonContiguous;
ldmPublication.base = ls == NULL ? NULL : &ls->window.base;
ldmPublication.loadedDictEnd = ls == NULL ? NULL : &ls->loadedDictEnd;
assert((tfp == ZSTD_tfp_forCDict && dtlm == ZSTD_dtlm_full)
|| (tfp != ZSTD_tfp_forCDict && dtlm == ZSTD_dtlm_fast));
@@ -7097,7 +7095,7 @@ static size_t ZSTD_loadDictionaryContent_callback(
state.assertCParams = ZSTD_loadDictionaryContent_assertCParams;
state.assertWindowEmpty = ZSTD_loadDictionaryContent_assertWindowEmpty;
state.windowUpdate = ZSTD_loadDictionaryContent_windowUpdate;
state.setLdmLoadedDictEnd = ZSTD_loadDictionaryContent_setLdmLoadedDictEnd;
state.setLdmLoadedDictEnd = &ldmPublication;
state.publishMatchState = &matchStatePublication;
state.fillLdm = ZSTD_loadDictionaryContent_fillLdm;
state.overflowCorrect = ZSTD_loadDictionaryContent_overflowCorrect;
@@ -7107,7 +7105,7 @@ static size_t ZSTD_loadDictionaryContent_callback(
state.loadRow = ZSTD_loadDictionaryContent_loadRow;
state.loadChain = ZSTD_loadDictionaryContent_loadChain;
state.loadTree = ZSTD_loadDictionaryContent_loadTree;
state.publishFinalIndex = ZSTD_loadDictionaryContent_publishFinalIndex;
state.publishFinalIndex = &matchStatePublication;
state.assertLazyConfiguration = ZSTD_loadDictionaryContent_assertLazyConfiguration;
state.assertInvalidStrategy = ZSTD_loadDictionaryContent_assertInvalidStrategy;
return ZSTD_rust_loadDictionaryContent(&state, src, srcSize);
rust/src/zstd_compress_dictionary.rs
+119 -14
View File
@@ -127,13 +127,32 @@ type LoadDictionaryContentAssertWindowEmptyFn =
unsafe extern "C" fn(context: *mut c_void, ldm: c_int);
type LoadDictionaryContentWindowUpdateFn =
unsafe extern "C" fn(context: *mut c_void, ldm: c_int, src: *const c_void, src_size: usize);
type LoadDictionaryContentSetLdmLoadedDictEndFn =
unsafe extern "C" fn(context: *mut c_void, iend: *const c_void, force_window: c_int);
type LoadDictionaryContentFillLdmFn =
unsafe extern "C" fn(context: *mut c_void, ip: *const c_void, iend: *const c_void);
type LoadDictionaryContentOverflowCorrectFn =
unsafe extern "C" fn(context: *mut c_void, ip: *const c_void, iend: *const c_void);
#[repr(C)]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct ZSTD_rust_loadDictionaryContentLdmPublicationState {
base: *const *const u8,
loaded_dict_end: *mut c_uint,
}
const _: () = {
assert!(offset_of!(ZSTD_rust_loadDictionaryContentLdmPublicationState, base) == 0);
assert!(
offset_of!(
ZSTD_rust_loadDictionaryContentLdmPublicationState,
loaded_dict_end
) == size_of::<usize>()
);
assert!(
size_of::<ZSTD_rust_loadDictionaryContentLdmPublicationState>()
== 2 * size_of::<usize>()
);
};
/// Live match-state fields used when the dictionary loader publishes the
/// current prefix. The base is indirect because window updates may replace it
/// before this publication point.
@@ -209,8 +228,6 @@ type LoadDictionaryContentLoadMatchFn =
unsafe extern "C" fn(context: *mut c_void, ip: *const c_void);
type LoadDictionaryContentLoadTreeFn =
unsafe extern "C" fn(context: *mut c_void, ip: *const c_void, iend: *const c_void);
type LoadDictionaryContentPublishFinalIndexFn =
unsafe extern "C" fn(context: *mut c_void, iend: *const c_void);
/// Rust-owned policy projection for `ZSTD_loadDictionaryContent()`.
///
@@ -242,7 +259,7 @@ pub struct ZSTD_rust_loadDictionaryContentState {
assert_c_params: LoadDictionaryContentAssertFn,
assert_window_empty: LoadDictionaryContentAssertWindowEmptyFn,
window_update: LoadDictionaryContentWindowUpdateFn,
set_ldm_loaded_dict_end: LoadDictionaryContentSetLdmLoadedDictEndFn,
set_ldm_loaded_dict_end: *const ZSTD_rust_loadDictionaryContentLdmPublicationState,
publish_match_state: *const ZSTD_rust_loadDictionaryContentMatchStatePublicationState,
fill_ldm: LoadDictionaryContentFillLdmFn,
overflow_correct: LoadDictionaryContentOverflowCorrectFn,
@@ -252,7 +269,7 @@ pub struct ZSTD_rust_loadDictionaryContentState {
load_row: LoadDictionaryContentLoadMatchFn,
load_chain: LoadDictionaryContentLoadMatchFn,
load_tree: LoadDictionaryContentLoadTreeFn,
publish_final_index: LoadDictionaryContentPublishFinalIndexFn,
publish_final_index: *const ZSTD_rust_loadDictionaryContentMatchStatePublicationState,
assert_lazy_configuration: LoadDictionaryContentAssertFn,
assert_invalid_strategy: LoadDictionaryContentAssertFn,
}
@@ -261,7 +278,6 @@ const _: () = {
assert!(size_of::<LoadDictionaryContentAssertFn>() == size_of::<usize>());
assert!(size_of::<LoadDictionaryContentAssertWindowEmptyFn>() == size_of::<usize>());
assert!(size_of::<LoadDictionaryContentWindowUpdateFn>() == size_of::<usize>());
assert!(size_of::<LoadDictionaryContentSetLdmLoadedDictEndFn>() == size_of::<usize>());
assert!(size_of::<LoadDictionaryContentFillLdmFn>() == size_of::<usize>());
assert!(size_of::<LoadDictionaryContentOverflowCorrectFn>() == size_of::<usize>());
assert!(offset_of!(ZSTD_rust_loadDictionaryContentFastTableState, hash_table) == 0);
@@ -339,13 +355,16 @@ const _: () = {
);
assert!(size_of::<LoadDictionaryContentLoadMatchFn>() == size_of::<usize>());
assert!(size_of::<LoadDictionaryContentLoadTreeFn>() == size_of::<usize>());
assert!(size_of::<LoadDictionaryContentPublishFinalIndexFn>() == size_of::<usize>());
assert!(offset_of!(ZSTD_rust_loadDictionaryContentState, callback_context) == 0);
assert!(offset_of!(ZSTD_rust_loadDictionaryContentState, current_max) == size_of::<usize>());
assert!(
offset_of!(ZSTD_rust_loadDictionaryContentState, assert_c_params)
== 18 * size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_loadDictionaryContentState, set_ldm_loaded_dict_end)
== 21 * size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_loadDictionaryContentState, publish_match_state)
== 22 * size_of::<usize>()
@@ -526,12 +545,39 @@ unsafe fn publish_dictionary_match_state(
}
}
#[inline]
unsafe fn publish_dictionary_ldm_loaded_dict_end(
publication: &ZSTD_rust_loadDictionaryContentLdmPublicationState,
iend: *const u8,
force_window: usize,
) {
let base = unsafe { *publication.base };
unsafe {
*publication.loaded_dict_end = if force_window != 0 {
0
} else {
iend.offset_from(base) as c_uint
};
}
}
#[inline]
unsafe fn publish_dictionary_final_index(
publication: &ZSTD_rust_loadDictionaryContentMatchStatePublicationState,
iend: *const u8,
) {
let base = unsafe { *publication.base };
unsafe {
*publication.next_to_update = iend.offset_from(base) as c_uint;
}
}
unsafe fn load_dictionary_content(
state: &ZSTD_rust_loadDictionaryContentState,
src: *const c_void,
src_size: usize,
) -> usize {
if state.publish_match_state.is_null() {
if state.publish_match_state.is_null() || state.publish_final_index.is_null() {
return ERROR(ZstdErrorCode::Generic);
}
let match_state_publication = unsafe { &*state.publish_match_state };
@@ -563,12 +609,16 @@ unsafe fn load_dictionary_content(
unsafe { (state.window_update)(state.callback_context, 0, ip.cast(), loaded_src_size) };
if load_ldm_dict {
if state.set_ldm_loaded_dict_end.is_null() {
return ERROR(ZstdErrorCode::Generic);
}
let ldm_publication = unsafe { &*state.set_ldm_loaded_dict_end };
unsafe {
(state.window_update)(state.callback_context, 1, ip.cast(), loaded_src_size);
(state.set_ldm_loaded_dict_end)(
state.callback_context,
iend.cast(),
(state.force_window != 0) as c_int,
publish_dictionary_ldm_loaded_dict_end(
ldm_publication,
iend,
state.force_window,
);
(state.fill_ldm)(state.callback_context, ip.cast(), iend.cast());
}
@@ -634,7 +684,8 @@ unsafe fn load_dictionary_content(
},
}
unsafe { (state.publish_final_index)(state.callback_context, iend.cast()) };
let final_index_publication = unsafe { &*state.publish_final_index };
unsafe { publish_dictionary_final_index(final_index_publication, iend) };
0
}
@@ -6910,6 +6961,60 @@ mod tests {
assert_eq!(force_non_contiguous, 0);
}
#[test]
fn dictionary_ldm_and_final_index_projections_use_live_bases() {
let input = [0u8; 64];
let mut ldm_base_ptr = input.as_ptr();
let mut ldm_loaded_dict_end = 0u32;
let ldm_projection = ZSTD_rust_loadDictionaryContentLdmPublicationState {
base: &ldm_base_ptr,
loaded_dict_end: &mut ldm_loaded_dict_end,
};
unsafe {
publish_dictionary_ldm_loaded_dict_end(
&ldm_projection,
input.as_ptr().add(32),
0,
);
assert_eq!(ldm_loaded_dict_end, 32);
ptr::write(&mut ldm_base_ptr, input.as_ptr().add(4));
assert_eq!(*ldm_projection.base, input.as_ptr().add(4));
publish_dictionary_ldm_loaded_dict_end(
&ldm_projection,
input.as_ptr().add(32),
0,
);
publish_dictionary_ldm_loaded_dict_end(
&ldm_projection,
input.as_ptr().add(32),
1,
);
}
assert_eq!(ldm_loaded_dict_end, 0);
let mut match_base_ptr = input.as_ptr();
let mut next_to_update = 0u32;
let mut loaded_dict_end = 0u32;
let mut force_non_contiguous = 0i32;
let match_projection = ZSTD_rust_loadDictionaryContentMatchStatePublicationState {
base: &match_base_ptr,
next_to_update: &mut next_to_update,
loaded_dict_end: &mut loaded_dict_end,
force_non_contiguous: &mut force_non_contiguous,
};
unsafe {
ptr::write(&mut match_base_ptr, input.as_ptr().add(6));
assert_eq!(*match_projection.base, input.as_ptr().add(6));
publish_dictionary_final_index(
&match_projection,
input.as_ptr().add(38),
);
}
assert_eq!(next_to_update, 32);
}
#[test]
fn fast_dictionary_table_projection_dispatches_to_rust_leaf() {
let hash_log = 4u32;