ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move dictionary state publication to Rust

Browse Source 
Move LDM loadedDictEnd and final nextToUpdate publication out of the C
dictionary-content callbacks. Rust now writes through compact projections with
live window bases, while C retains the configuration-sensitive window,
matchfinder, and LDM operations.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --tests
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --tests -- -A clippy::manual-bits -D warnings
- ulimit -v 41943040; make -j1
- ulimit -v 41943040; make -j1 -C tests test
This commit is contained in:
ddidderr
2026-07-21 23:01:49 +02:00
parent 87b616c49b
commit cf6a316693
2 changed files with 144 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+25 -27
View File
@@ -2767,12 +2767,22 @@ typedef void (*ZSTD_rust_loadDictionaryContent_assertWindowEmpty_f)(
void* context, int ldm);
typedef void (*ZSTD_rust_loadDictionaryContent_windowUpdate_f)(
void* context, int ldm, const void* src, size_t srcSize);
typedef void (*ZSTD_rust_loadDictionaryContent_setLdmLoadedDictEnd_f)(
void* context, const void* iend, int forceWindow);
typedef void (*ZSTD_rust_loadDictionaryContent_fillLdm_f)(
void* context, const void* ip, const void* iend);
typedef void (*ZSTD_rust_loadDictionaryContent_overflowCorrect_f)(
void* context, const void* ip, const void* iend);
typedef struct {
const BYTE** base;
U32* loadedDictEnd;
} ZSTD_rust_loadDictionaryContentLdmPublicationState;
typedef char ZSTD_rust_load_dictionary_ldm_publication_layout[
(offsetof(ZSTD_rust_loadDictionaryContentLdmPublicationState, base) == 0
&& offsetof(ZSTD_rust_loadDictionaryContentLdmPublicationState,
loadedDictEnd)
== sizeof(void*)
&& sizeof(ZSTD_rust_loadDictionaryContentLdmPublicationState)
== 2 * sizeof(void*))
? 1 : -1];
typedef struct {
const BYTE** base;
U32* nextToUpdate;
@@ -2866,8 +2876,6 @@ typedef void (*ZSTD_rust_loadDictionaryContent_loadMatch_f)(
void* context, const void* ip);
typedef void (*ZSTD_rust_loadDictionaryContent_loadTree_f)(
void* context, const void* ip, const void* iend);
typedef void (*ZSTD_rust_loadDictionaryContent_publishFinalIndex_f)(
void* context, const void* iend);
typedef struct {
void* callbackContext;
@@ -2891,7 +2899,7 @@ typedef struct {
ZSTD_rust_loadDictionaryContent_assert_f assertCParams;
ZSTD_rust_loadDictionaryContent_assertWindowEmpty_f assertWindowEmpty;
ZSTD_rust_loadDictionaryContent_windowUpdate_f windowUpdate;
ZSTD_rust_loadDictionaryContent_setLdmLoadedDictEnd_f setLdmLoadedDictEnd;
const ZSTD_rust_loadDictionaryContentLdmPublicationState* setLdmLoadedDictEnd;
const ZSTD_rust_loadDictionaryContentMatchStatePublicationState* publishMatchState;
ZSTD_rust_loadDictionaryContent_fillLdm_f fillLdm;
ZSTD_rust_loadDictionaryContent_overflowCorrect_f overflowCorrect;
@@ -2901,7 +2909,7 @@ typedef struct {
ZSTD_rust_loadDictionaryContent_loadMatch_f loadRow;
ZSTD_rust_loadDictionaryContent_loadMatch_f loadChain;
ZSTD_rust_loadDictionaryContent_loadTree_f loadTree;
ZSTD_rust_loadDictionaryContent_publishFinalIndex_f publishFinalIndex;
const ZSTD_rust_loadDictionaryContentMatchStatePublicationState* publishFinalIndex;
ZSTD_rust_loadDictionaryContent_assert_f assertLazyConfiguration;
ZSTD_rust_loadDictionaryContent_assert_f assertInvalidStrategy;
} ZSTD_rust_loadDictionaryContentState;
@@ -2917,6 +2925,12 @@ ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_scalar_offset,
ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_callback_offset,
offsetof(ZSTD_rust_loadDictionaryContentState,
assertCParams) == 18 * sizeof(void*));
ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_ldm_publication_offset,
offsetof(ZSTD_rust_loadDictionaryContentState,
setLdmLoadedDictEnd) == 21 * sizeof(void*));
ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_match_publication_offset,
offsetof(ZSTD_rust_loadDictionaryContentState,
publishMatchState) == 22 * sizeof(void*));
ZSTD_RUST_LOAD_DICT_ASSERT(ZSTD_rust_load_dict_fast_table_offset,
offsetof(ZSTD_rust_loadDictionaryContentState,
fastTable) == 25 * sizeof(void*));
@@ -6904,17 +6918,6 @@ static void ZSTD_loadDictionaryContent_windowUpdate(
}
}
static void ZSTD_loadDictionaryContent_setLdmLoadedDictEnd(
void* opaque, const void* iend, int forceWindow)
{
ZSTD_loadDictionaryContent_context const* const context =
(const ZSTD_loadDictionaryContent_context*)opaque;
assert(context->ldmState != NULL);
context->ldmState->loadedDictEnd = forceWindow
? 0
: (U32)((const BYTE*)iend - context->ldmState->window.base);
}
static void ZSTD_loadDictionaryContent_fillLdm(
void* opaque, const void* ip, const void* iend)
{
@@ -7000,14 +7003,6 @@ static void ZSTD_loadDictionaryContent_loadTree(
#endif
}
static void ZSTD_loadDictionaryContent_publishFinalIndex(void* opaque, const void* iend)
{
ZSTD_loadDictionaryContent_context const* const context =
(const ZSTD_loadDictionaryContent_context*)opaque;
context->matchState->nextToUpdate =
(U32)((const BYTE*)iend - context->matchState->window.base);
}
static void ZSTD_loadDictionaryContent_assertLazyConfiguration(void* opaque)
{
ZSTD_loadDictionaryContent_context const* const context =
@@ -7030,6 +7025,7 @@ static size_t ZSTD_loadDictionaryContent_callback(
int dtlm, int tfp)
{
ZSTD_loadDictionaryContent_context context;
ZSTD_rust_loadDictionaryContentLdmPublicationState ldmPublication;
ZSTD_rust_loadDictionaryContentMatchStatePublicationState matchStatePublication;
ZSTD_rust_loadDictionaryContentFastTableState fastTable;
ZSTD_rust_loadDictionaryContentDoubleFastTableState doubleFastTable;
@@ -7048,6 +7044,8 @@ static size_t ZSTD_loadDictionaryContent_callback(
matchStatePublication.nextToUpdate = &ms->nextToUpdate;
matchStatePublication.loadedDictEnd = &ms->loadedDictEnd;
matchStatePublication.forceNonContiguous = &ms->forceNonContiguous;
ldmPublication.base = ls == NULL ? NULL : &ls->window.base;
ldmPublication.loadedDictEnd = ls == NULL ? NULL : &ls->loadedDictEnd;
assert((tfp == ZSTD_tfp_forCDict && dtlm == ZSTD_dtlm_full)
|| (tfp != ZSTD_tfp_forCDict && dtlm == ZSTD_dtlm_fast));
@@ -7097,7 +7095,7 @@ static size_t ZSTD_loadDictionaryContent_callback(
state.assertCParams = ZSTD_loadDictionaryContent_assertCParams;
state.assertWindowEmpty = ZSTD_loadDictionaryContent_assertWindowEmpty;
state.windowUpdate = ZSTD_loadDictionaryContent_windowUpdate;
state.setLdmLoadedDictEnd = ZSTD_loadDictionaryContent_setLdmLoadedDictEnd;
state.setLdmLoadedDictEnd = &ldmPublication;
state.publishMatchState = &matchStatePublication;
state.fillLdm = ZSTD_loadDictionaryContent_fillLdm;
state.overflowCorrect = ZSTD_loadDictionaryContent_overflowCorrect;
@@ -7107,7 +7105,7 @@ static size_t ZSTD_loadDictionaryContent_callback(
state.loadRow = ZSTD_loadDictionaryContent_loadRow;
state.loadChain = ZSTD_loadDictionaryContent_loadChain;
state.loadTree = ZSTD_loadDictionaryContent_loadTree;
state.publishFinalIndex = ZSTD_loadDictionaryContent_publishFinalIndex;
state.publishFinalIndex = &matchStatePublication;
state.assertLazyConfiguration = ZSTD_loadDictionaryContent_assertLazyConfiguration;
state.assertInvalidStrategy = ZSTD_loadDictionaryContent_assertInvalidStrategy;
return ZSTD_rust_loadDictionaryContent(&state, src, srcSize);