ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move local dictionary init policy into Rust

Browse Source 
The transparent stream initializer previously called a C helper that both
inspected local-dictionary state and chose the no-dictionary, already-created,
and create/publish branches.  Project that private state through a checked C/Rust
ABI so Rust owns the branch and callback order while C retains the requested
parameters, custom allocator, and CDict implementation details.  Keep creation
and publication split so allocation failure cannot publish a partial dictionary.

Test Plan:
- cargo fmt --manifest-path rust/Cargo.toml -- --check
- CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml compress_stream_init_
- CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- make -j1
- make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s
- make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests
- CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml
This commit is contained in:
ddidderr
2026-07-19 21:16:32 +02:00
parent 2d2214eb1d
commit 9d8c03d87d
2 changed files with 269 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+78 -42
View File
@@ -1033,7 +1033,22 @@ typedef struct {
int cdictCompressionLevel;
size_t cdictDictContentSize;
} ZSTD_rust_compressStreamInitDictionaryState;
typedef size_t (*ZSTD_rust_compressStreamInitLocalDict_f)(void* context);
typedef struct {
const void* dict;
void* dictBuffer;
size_t dictSize;
int dictContentType;
void* localCDict;
const void* cdict;
const void* prefixDict;
void* createdCDict;
} ZSTD_rust_compressStreamInitLocalDictState;
typedef void (*ZSTD_rust_compressStreamInitGetLocalDict_f)(
void* context, ZSTD_rust_compressStreamInitLocalDictState* state);
typedef size_t (*ZSTD_rust_compressStreamInitCreateLocalDict_f)(
void* context, ZSTD_rust_compressStreamInitLocalDictState* state);
typedef void (*ZSTD_rust_compressStreamInitPublishLocalDict_f)(
void* context, const ZSTD_rust_compressStreamInitLocalDictState* state);
typedef void (*ZSTD_rust_compressStreamInitRefreshCDict_f)(
void* context, ZSTD_rust_compressStreamInitDictionaryState* state);
typedef void (*ZSTD_rust_compressStreamInitClearPrefix_f)(void* context);
@@ -1083,7 +1098,9 @@ typedef struct {
size_t inSize;
int multithreaded;
size_t mtJobSizeMin;
ZSTD_rust_compressStreamInitLocalDict_f initLocalDict;
ZSTD_rust_compressStreamInitGetLocalDict_f getLocalDict;
ZSTD_rust_compressStreamInitCreateLocalDict_f createLocalDict;
ZSTD_rust_compressStreamInitPublishLocalDict_f publishLocalDict;
ZSTD_rust_compressStreamInitRefreshCDict_f refreshCDict;
ZSTD_rust_compressStreamInitClearPrefix_f clearPrefix;
ZSTD_rust_compressStreamInitAssertDictionaries_f assertDictionaries;
@@ -1109,6 +1126,25 @@ typedef struct {
ZSTD_rust_compressStreamInitGetBlockSize_f getBlockSize;
ZSTD_rust_compressStreamInitCommitOrdinary_f commitOrdinary;
} ZSTD_rust_compressStreamInitState;
typedef char ZSTD_rust_compress_stream_init_local_dict_layout[
(offsetof(ZSTD_rust_compressStreamInitLocalDictState, dict) == 0
&& offsetof(ZSTD_rust_compressStreamInitLocalDictState, dictBuffer)
== sizeof(void*)
&& offsetof(ZSTD_rust_compressStreamInitLocalDictState, dictSize)
== 2 * sizeof(void*)
&& offsetof(ZSTD_rust_compressStreamInitLocalDictState, dictContentType)
== 3 * sizeof(void*)
&& offsetof(ZSTD_rust_compressStreamInitLocalDictState, localCDict)
== 4 * sizeof(void*)
&& offsetof(ZSTD_rust_compressStreamInitLocalDictState, cdict)
== 5 * sizeof(void*)
&& offsetof(ZSTD_rust_compressStreamInitLocalDictState, prefixDict)
== 6 * sizeof(void*)
&& offsetof(ZSTD_rust_compressStreamInitLocalDictState, createdCDict)
== 7 * sizeof(void*)
&& sizeof(ZSTD_rust_compressStreamInitLocalDictState)
== 8 * sizeof(void*))
? 1 : -1];
typedef char ZSTD_rust_compress_stream_init_dictionary_layout[
(offsetof(ZSTD_rust_compressStreamInitDictionaryState, prefixDict) == 0
&& offsetof(ZSTD_rust_compressStreamInitDictionaryState, prefixDictSize)
@@ -1140,7 +1176,7 @@ typedef char ZSTD_rust_compress_stream_init_state_layout[
&& offsetof(ZSTD_rust_compressStreamInitState, mtJobSizeMin)
== 5 * sizeof(void*) + sizeof(size_t)
&& sizeof(ZSTD_rust_compressStreamInitState)
== (sizeof(void*) == 8 ? 256 : 128))
== (sizeof(void*) == 8 ? 272 : 144))
? 1 : -1];
size_t ZSTD_rust_compressStreamInit(
const ZSTD_rust_compressStreamInitState* state);
@@ -3263,42 +3299,6 @@ size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSr
static void ZSTD_dedicatedDictSearch_revertCParams(
ZSTD_compressionParameters* cParams);
/**
* Initializes the local dictionary using requested parameters.
* NOTE: Initialization does not employ the pledged src size,
* because the dictionary may be used for multiple compressions.
*/
static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx)
{
ZSTD_localDict* const dl = &cctx->localDict;
if (dl->dict == NULL) {
/* No local dictionary. */
assert(dl->dictBuffer == NULL);
assert(dl->cdict == NULL);
assert(dl->dictSize == 0);
return 0;
}
if (dl->cdict != NULL) {
/* Local dictionary already initialized. */
assert(cctx->cdict == dl->cdict);
return 0;
}
assert(dl->dictSize > 0);
assert(cctx->cdict == NULL);
assert(cctx->prefixDict.dict == NULL);
dl->cdict = ZSTD_createCDict_advanced2(
dl->dict,
dl->dictSize,
ZSTD_dlm_byRef,
dl->dictContentType,
&cctx->requestedParams,
cctx->customMem);
RETURN_ERROR_IF(!dl->cdict, memory_allocation, "ZSTD_createCDict_advanced failed");
cctx->cdict = dl->cdict;
return 0;
}
size_t ZSTD_CCtx_loadDictionary_advanced(
ZSTD_CCtx* cctx,
const void* dict, size_t dictSize,
@@ -7353,9 +7353,43 @@ enum {
ZSTD_RUST_INIT_RESOLVE_EXTERNAL_REPCODE_SEARCH = 5
};
static size_t ZSTD_rust_compressStreamInit_localDict(void* context)
static void ZSTD_rust_compressStreamInit_getLocalDict(
void* context, ZSTD_rust_compressStreamInitLocalDictState* state)
{
return ZSTD_initLocalDict((ZSTD_CCtx*)context);
ZSTD_CCtx const* const cctx = (ZSTD_CCtx const*)context;
ZSTD_localDict const* const localDict = &cctx->localDict;
state->dict = localDict->dict;
state->dictBuffer = localDict->dictBuffer;
state->dictSize = localDict->dictSize;
state->dictContentType = (int)localDict->dictContentType;
state->localCDict = (void*)localDict->cdict;
state->cdict = cctx->cdict;
state->prefixDict = cctx->prefixDict.dict;
state->createdCDict = NULL;
}
static size_t ZSTD_rust_compressStreamInit_createLocalDict(
void* context, ZSTD_rust_compressStreamInitLocalDictState* state)
{
ZSTD_CCtx const* const cctx = (ZSTD_CCtx const*)context;
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced2(
state->dict,
state->dictSize,
ZSTD_dlm_byRef,
(ZSTD_dictContentType_e)state->dictContentType,
&cctx->requestedParams,
cctx->customMem);
RETURN_ERROR_IF(!cdict, memory_allocation, "ZSTD_createCDict_advanced failed");
state->createdCDict = cdict;
return 0;
}
static void ZSTD_rust_compressStreamInit_publishLocalDict(
void* context, const ZSTD_rust_compressStreamInitLocalDictState* state)
{
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context;
cctx->localDict.cdict = (ZSTD_CDict*)state->createdCDict;
cctx->cdict = cctx->localDict.cdict;
}
static void ZSTD_rust_compressStreamInit_refreshCDict(
@@ -7628,7 +7662,9 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
0,
0,
#endif
ZSTD_rust_compressStreamInit_localDict,
ZSTD_rust_compressStreamInit_getLocalDict,
ZSTD_rust_compressStreamInit_createLocalDict,
ZSTD_rust_compressStreamInit_publishLocalDict,
ZSTD_rust_compressStreamInit_refreshCDict,
ZSTD_rust_compressStreamInit_clearPrefix,
ZSTD_rust_compressStreamInit_assertDictionaries,