ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move MT serial dictionary policy to Rust

Browse Source 
Move the serial LDM dictionary preload policy into Rust. The C MT adapter now
exposes only the private window and hash-table operations through checked
callbacks, while Rust owns empty/raw-content eligibility, loaded-dictionary
reset ordering, and the force-window publication rule. The explicit projection
keeps the dictionary inputs and callback ABI auditable without exposing the
private SerialState layout.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --manifest-path rust/Cargo.toml --tests
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1
- ulimit -v 41943040; make -j1 -C tests test
This commit is contained in:
ddidderr
2026-07-21 18:28:03 +02:00
parent 67c6056203
commit 62eea1dd83
2 changed files with 210 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstdmt_compress.c
+57 -14
View File
@@ -837,7 +837,32 @@ typedef int (*ZSTDMT_serialResetResizeFn)(
unsigned bucketLog, unsigned previousBucketLog);
typedef void (*ZSTDMT_serialResetZeroTablesFn)(
void* opaque, size_t hashSize, size_t numBuckets);
typedef size_t (*ZSTDMT_serialResetLoadRawDictionaryFn)(
void* opaque, const void* dict, size_t dictSize);
typedef void (*ZSTDMT_serialResetSetLoadedDictEndFn)(
void* opaque, U32 loadedDictEnd);
typedef void (*ZSTDMT_serialResetPublishFn)(void* opaque, size_t jobSize);
typedef struct {
const void* dict;
size_t dictSize;
int dictContentType;
int forceWindow;
} ZSTDMT_RustSerialDictionaryProjection;
typedef char ZSTDMT_rust_serial_dictionary_projection_layout[
(offsetof(ZSTDMT_RustSerialDictionaryProjection, dict) == 0
&& offsetof(ZSTDMT_RustSerialDictionaryProjection, dictSize)
== sizeof(void*)
&& offsetof(ZSTDMT_RustSerialDictionaryProjection, dictContentType)
== sizeof(void*) + sizeof(size_t)
&& offsetof(ZSTDMT_RustSerialDictionaryProjection, forceWindow)
== sizeof(void*) + sizeof(size_t) + sizeof(int)
&& sizeof(ZSTDMT_RustSerialDictionaryProjection)
== sizeof(void*) + sizeof(size_t) + 2 * sizeof(int))
? 1 : -1];
void ZSTDMT_rust_serialStateLoadDictionary(
const ZSTDMT_RustSerialDictionaryProjection* projection,
void* opaque, ZSTDMT_serialResetLoadRawDictionaryFn loadRawDictionary,
ZSTDMT_serialResetSetLoadedDictEndFn setLoadedDictEnd);
int ZSTDMT_rust_serialStateReset(
const ZSTDMT_RustSerialResetProjection* projection, void* opaque,
ZSTDMT_serialResetSetLdmParamsFn setLdmParams,
@@ -1487,24 +1512,42 @@ static void ZSTDMT_serialResetZeroTables(
ZSTD_memset(serialState->ldmState.bucketOffsets, 0, numBuckets);
}
static void ZSTDMT_serialResetLoadDictionary(void* opaque)
static size_t ZSTDMT_serialResetLoadRawDictionary(
void* opaque, const void* dict, size_t dictSize)
{
ZSTDMT_serialResetContext* const context = (ZSTDMT_serialResetContext*)opaque;
SerialState* const serialState = context->serialState;
BYTE const* const dictEnd = (const BYTE*)dict + dictSize;
serialState->ldmState.loadedDictEnd = 0;
if (context->dictSize > 0 &&
context->dictContentType == ZSTD_dct_rawContent) {
BYTE const* const dictEnd = (const BYTE*)context->dict + context->dictSize;
ZSTD_window_update(&serialState->ldmState.window,
context->dict, context->dictSize,
/* forceNonContiguous */ 0);
ZSTD_ldm_fillHashTable(&serialState->ldmState,
(const BYTE*)context->dict, dictEnd,
&context->params->ldmParams);
serialState->ldmState.loadedDictEnd = context->params->forceWindow
? 0 : (U32)(dictEnd - serialState->ldmState.window.base);
}
ZSTD_window_update(&serialState->ldmState.window,
dict, dictSize,
/* forceNonContiguous */ 0);
ZSTD_ldm_fillHashTable(&serialState->ldmState,
(const BYTE*)dict, dictEnd,
&context->params->ldmParams);
return (size_t)(dictEnd - serialState->ldmState.window.base);
}
static void ZSTDMT_serialResetSetLoadedDictEnd(
void* opaque, U32 loadedDictEnd)
{
ZSTDMT_serialResetContext* const context = (ZSTDMT_serialResetContext*)opaque;
context->serialState->ldmState.loadedDictEnd = loadedDictEnd;
}
static void ZSTDMT_serialResetLoadDictionary(void* opaque)
{
ZSTDMT_serialResetContext* const context = (ZSTDMT_serialResetContext*)opaque;
ZSTDMT_RustSerialDictionaryProjection const projection = {
context->dict,
context->dictSize,
(int)context->dictContentType,
context->params->forceWindow
};
ZSTDMT_rust_serialStateLoadDictionary(
&projection, context,
ZSTDMT_serialResetLoadRawDictionary,
ZSTDMT_serialResetSetLoadedDictEnd);
}
static void ZSTDMT_serialResetCopyWindow(void* opaque)