ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move CCtx dictionary attachment policy into Rust

Browse Source 
Move stage validation, dictionary clearing, and local/CDict/prefix assignment
selection for the three CCtx dictionary APIs into Rust. Keep the private CCtx
layouts, allocator and by-copy lifetime behavior, CDict ownership, reset
policy, and dictionary-content loader behind explicit C callbacks so the
language boundary carries policy rather than private state.

Test Plan:
- Rust library all-target tests: 517 passed.
- Rust legacy feature matrix: 572 passed.
- Rust and CLI clippy, nightly fmt, native CLI tests (41), and library smoke.
- Native test-zstd, bounded fuzzer (319), zstream (152 + 297), and decode
  corpus (1,647) all passed.
- All heavy checks ran serially with CARGO_BUILD_JOBS=1 or make -j1 and
  ulimit -v 41943040 (40 GiB virtual memory).
- Commit is intentionally unsigned because configured GPG pinentry was
  unavailable and hung during the signing attempt.
This commit is contained in:
ddidderr
2026-07-19 09:14:26 +02:00
parent 0492e7f7ec
commit 47b0da7ad7
2 changed files with 357 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+85 -37
View File
@@ -701,6 +701,33 @@ size_t ZSTD_rust_compressInsertDictionary(
int dictContentType, int dtlm, int tfp,
void* workspace, int noDictIDFlag,
ZSTD_rust_loadDictionaryContent_f loadDictionaryContent);
/* CCtx dictionary attachment policy lives in Rust. These callbacks keep
* the private CCtx, local/prefix dictionary layouts, allocator, and CDict
* lifetime operations in C. */
typedef void (*ZSTD_rust_CCtxDictionaryClear_f)(void* context);
typedef size_t (*ZSTD_rust_CCtxAssignLocalDict_f)(
void* context, const void* dict, size_t dictSize,
int dictLoadMethod, int dictContentType);
typedef void (*ZSTD_rust_CCtxAssignCDict_f)(
void* context, const void* cdict);
typedef void (*ZSTD_rust_CCtxAssignPrefixDict_f)(
void* context, const void* prefix, size_t prefixSize,
int dictContentType);
size_t ZSTD_rust_CCtx_loadDictionaryAdvanced(
void* context, int streamStage,
const void* dict, size_t dictSize,
int dictLoadMethod, int dictContentType,
ZSTD_rust_CCtxDictionaryClear_f clearDictionaries,
ZSTD_rust_CCtxAssignLocalDict_f assignLocalDict);
size_t ZSTD_rust_CCtx_refCDict(
void* context, int streamStage, const void* cdict,
ZSTD_rust_CCtxDictionaryClear_f clearDictionaries,
ZSTD_rust_CCtxAssignCDict_f assignCDict);
size_t ZSTD_rust_CCtx_refPrefixAdvanced(
void* context, int streamStage,
const void* prefix, size_t prefixSize, int dictContentType,
ZSTD_rust_CCtxDictionaryClear_f clearDictionaries,
ZSTD_rust_CCtxAssignPrefixDict_f assignPrefixDict);
size_t ZSTD_rust_transferSequencesWBlockDelim(
SeqStore_t* seqStore, ZSTD_SequencePosition* seqPos,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
@@ -1025,6 +1052,53 @@ static void ZSTD_clearAllDicts(ZSTD_CCtx* cctx)
cctx->cdict = NULL;
}
static void ZSTD_clearAllDicts_callback(void* context)
{
ZSTD_clearAllDicts((ZSTD_CCtx*)context);
}
static size_t ZSTD_assignLocalDict_callback(
void* context, const void* dict, size_t dictSize,
int dictLoadMethod, int dictContentType)
{
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context;
ZSTD_localDict* const localDict = &cctx->localDict;
if (dictLoadMethod == ZSTD_dlm_byRef) {
localDict->dict = dict;
} else {
void* dictBuffer;
RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
"static CCtx can't allocate for an internal copy of dictionary");
dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem);
RETURN_ERROR_IF(dictBuffer == NULL, memory_allocation,
"allocation failed for dictionary content");
ZSTD_memcpy(dictBuffer, dict, dictSize);
localDict->dictBuffer = dictBuffer; /* owned ptr to free */
localDict->dict = dictBuffer; /* read-only reference */
}
localDict->dictSize = dictSize;
localDict->dictContentType = (ZSTD_dictContentType_e)dictContentType;
return 0;
}
static void ZSTD_assignCDict_callback(void* context, const void* cdict)
{
((ZSTD_CCtx*)context)->cdict = (const ZSTD_CDict*)cdict;
}
static void ZSTD_assignPrefixDict_callback(
void* context, const void* prefix, size_t prefixSize,
int dictContentType)
{
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context;
if (prefix != NULL && prefixSize > 0) {
cctx->prefixDict.dict = prefix;
cctx->prefixDict.dictSize = prefixSize;
cctx->prefixDict.dictContentType = (ZSTD_dictContentType_e)dictContentType;
}
}
static size_t ZSTD_sizeof_localDict(ZSTD_localDict dict)
{
size_t const cdictSize = ZSTD_sizeof_CDict(dict.cdict);
@@ -1482,28 +1556,10 @@ size_t ZSTD_CCtx_loadDictionary_advanced(
ZSTD_dictContentType_e dictContentType)
{
DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize);
RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
"Can't load a dictionary when cctx is not in init stage.");
ZSTD_clearAllDicts(cctx); /* erase any previously set dictionary */
if (dict == NULL || dictSize == 0) /* no dictionary */
return 0;
if (dictLoadMethod == ZSTD_dlm_byRef) {
cctx->localDict.dict = dict;
} else {
/* copy dictionary content inside CCtx to own its lifetime */
void* dictBuffer;
RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
"static CCtx can't allocate for an internal copy of dictionary");
dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem);
RETURN_ERROR_IF(dictBuffer==NULL, memory_allocation,
"allocation failed for dictionary content");
ZSTD_memcpy(dictBuffer, dict, dictSize);
cctx->localDict.dictBuffer = dictBuffer; /* owned ptr to free */
cctx->localDict.dict = dictBuffer; /* read-only reference */
}
cctx->localDict.dictSize = dictSize;
cctx->localDict.dictContentType = dictContentType;
return 0;
return ZSTD_rust_CCtx_loadDictionaryAdvanced(
cctx, (int)cctx->streamStage,
dict, dictSize, (int)dictLoadMethod, (int)dictContentType,
ZSTD_clearAllDicts_callback, ZSTD_assignLocalDict_callback);
}
size_t ZSTD_CCtx_loadDictionary_byReference(
@@ -1522,12 +1578,9 @@ size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSi
size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
{
RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
"Can't ref a dict when ctx not in init stage.");
/* Free the existing local cdict (if any) to save memory. */
ZSTD_clearAllDicts(cctx);
cctx->cdict = cdict;
return 0;
return ZSTD_rust_CCtx_refCDict(
cctx, (int)cctx->streamStage, cdict,
ZSTD_clearAllDicts_callback, ZSTD_assignCDict_callback);
}
size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool)
@@ -1546,15 +1599,10 @@ size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSiz
size_t ZSTD_CCtx_refPrefix_advanced(
ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
{
RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
"Can't ref a prefix when ctx not in init stage.");
ZSTD_clearAllDicts(cctx);
if (prefix != NULL && prefixSize > 0) {
cctx->prefixDict.dict = prefix;
cctx->prefixDict.dictSize = prefixSize;
cctx->prefixDict.dictContentType = dictContentType;
}
return 0;
return ZSTD_rust_CCtx_refPrefixAdvanced(
cctx, (int)cctx->streamStage,
prefix, prefixSize, (int)dictContentType,
ZSTD_clearAllDicts_callback, ZSTD_assignPrefixDict_callback);
}
/*! ZSTD_CCtx_reset() :