ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move CCtx dictionary attachment policy into Rust

Browse Source 
Move stage validation, dictionary clearing, and local/CDict/prefix assignment
selection for the three CCtx dictionary APIs into Rust. Keep the private CCtx
layouts, allocator and by-copy lifetime behavior, CDict ownership, reset
policy, and dictionary-content loader behind explicit C callbacks so the
language boundary carries policy rather than private state.

Test Plan:
- Rust library all-target tests: 517 passed.
- Rust legacy feature matrix: 572 passed.
- Rust and CLI clippy, nightly fmt, native CLI tests (41), and library smoke.
- Native test-zstd, bounded fuzzer (319), zstream (152 + 297), and decode
  corpus (1,647) all passed.
- All heavy checks ran serially with CARGO_BUILD_JOBS=1 or make -j1 and
  ulimit -v 41943040 (40 GiB virtual memory).
- Commit is intentionally unsigned because configured GPG pinentry was
  unavailable and hung during the signing attempt.
This commit is contained in:
ddidderr
2026-07-19 09:14:26 +02:00
parent 0492e7f7ec
commit 47b0da7ad7
2 changed files with 357 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+85 -37
View File
@@ -701,6 +701,33 @@ size_t ZSTD_rust_compressInsertDictionary(
int dictContentType, int dtlm, int tfp,
void* workspace, int noDictIDFlag,
ZSTD_rust_loadDictionaryContent_f loadDictionaryContent);
/* CCtx dictionary attachment policy lives in Rust. These callbacks keep
* the private CCtx, local/prefix dictionary layouts, allocator, and CDict
* lifetime operations in C. */
typedef void (*ZSTD_rust_CCtxDictionaryClear_f)(void* context);
typedef size_t (*ZSTD_rust_CCtxAssignLocalDict_f)(
void* context, const void* dict, size_t dictSize,
int dictLoadMethod, int dictContentType);
typedef void (*ZSTD_rust_CCtxAssignCDict_f)(
void* context, const void* cdict);
typedef void (*ZSTD_rust_CCtxAssignPrefixDict_f)(
void* context, const void* prefix, size_t prefixSize,
int dictContentType);
size_t ZSTD_rust_CCtx_loadDictionaryAdvanced(
void* context, int streamStage,
const void* dict, size_t dictSize,
int dictLoadMethod, int dictContentType,
ZSTD_rust_CCtxDictionaryClear_f clearDictionaries,
ZSTD_rust_CCtxAssignLocalDict_f assignLocalDict);
size_t ZSTD_rust_CCtx_refCDict(
void* context, int streamStage, const void* cdict,
ZSTD_rust_CCtxDictionaryClear_f clearDictionaries,
ZSTD_rust_CCtxAssignCDict_f assignCDict);
size_t ZSTD_rust_CCtx_refPrefixAdvanced(
void* context, int streamStage,
const void* prefix, size_t prefixSize, int dictContentType,
ZSTD_rust_CCtxDictionaryClear_f clearDictionaries,
ZSTD_rust_CCtxAssignPrefixDict_f assignPrefixDict);
size_t ZSTD_rust_transferSequencesWBlockDelim(
SeqStore_t* seqStore, ZSTD_SequencePosition* seqPos,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
@@ -1025,6 +1052,53 @@ static void ZSTD_clearAllDicts(ZSTD_CCtx* cctx)
cctx->cdict = NULL;
}
static void ZSTD_clearAllDicts_callback(void* context)
{
ZSTD_clearAllDicts((ZSTD_CCtx*)context);
}
static size_t ZSTD_assignLocalDict_callback(
void* context, const void* dict, size_t dictSize,
int dictLoadMethod, int dictContentType)
{
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context;
ZSTD_localDict* const localDict = &cctx->localDict;
if (dictLoadMethod == ZSTD_dlm_byRef) {
localDict->dict = dict;
} else {
void* dictBuffer;
RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
"static CCtx can't allocate for an internal copy of dictionary");
dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem);
RETURN_ERROR_IF(dictBuffer == NULL, memory_allocation,
"allocation failed for dictionary content");
ZSTD_memcpy(dictBuffer, dict, dictSize);
localDict->dictBuffer = dictBuffer; /* owned ptr to free */
localDict->dict = dictBuffer; /* read-only reference */
}
localDict->dictSize = dictSize;
localDict->dictContentType = (ZSTD_dictContentType_e)dictContentType;
return 0;
}
static void ZSTD_assignCDict_callback(void* context, const void* cdict)
{
((ZSTD_CCtx*)context)->cdict = (const ZSTD_CDict*)cdict;
}
static void ZSTD_assignPrefixDict_callback(
void* context, const void* prefix, size_t prefixSize,
int dictContentType)
{
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context;
if (prefix != NULL && prefixSize > 0) {
cctx->prefixDict.dict = prefix;
cctx->prefixDict.dictSize = prefixSize;
cctx->prefixDict.dictContentType = (ZSTD_dictContentType_e)dictContentType;
}
}
static size_t ZSTD_sizeof_localDict(ZSTD_localDict dict)
{
size_t const cdictSize = ZSTD_sizeof_CDict(dict.cdict);
@@ -1482,28 +1556,10 @@ size_t ZSTD_CCtx_loadDictionary_advanced(
ZSTD_dictContentType_e dictContentType)
{
DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize);
RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
"Can't load a dictionary when cctx is not in init stage.");
ZSTD_clearAllDicts(cctx); /* erase any previously set dictionary */
if (dict == NULL || dictSize == 0) /* no dictionary */
return 0;
if (dictLoadMethod == ZSTD_dlm_byRef) {
cctx->localDict.dict = dict;
} else {
/* copy dictionary content inside CCtx to own its lifetime */
void* dictBuffer;
RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
"static CCtx can't allocate for an internal copy of dictionary");
dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem);
RETURN_ERROR_IF(dictBuffer==NULL, memory_allocation,
"allocation failed for dictionary content");
ZSTD_memcpy(dictBuffer, dict, dictSize);
cctx->localDict.dictBuffer = dictBuffer; /* owned ptr to free */
cctx->localDict.dict = dictBuffer; /* read-only reference */
}
cctx->localDict.dictSize = dictSize;
cctx->localDict.dictContentType = dictContentType;
return 0;
return ZSTD_rust_CCtx_loadDictionaryAdvanced(
cctx, (int)cctx->streamStage,
dict, dictSize, (int)dictLoadMethod, (int)dictContentType,
ZSTD_clearAllDicts_callback, ZSTD_assignLocalDict_callback);
}
size_t ZSTD_CCtx_loadDictionary_byReference(
@@ -1522,12 +1578,9 @@ size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSi
size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
{
RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
"Can't ref a dict when ctx not in init stage.");
/* Free the existing local cdict (if any) to save memory. */
ZSTD_clearAllDicts(cctx);
cctx->cdict = cdict;
return 0;
return ZSTD_rust_CCtx_refCDict(
cctx, (int)cctx->streamStage, cdict,
ZSTD_clearAllDicts_callback, ZSTD_assignCDict_callback);
}
size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool)
@@ -1546,15 +1599,10 @@ size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSiz
size_t ZSTD_CCtx_refPrefix_advanced(
ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType)
{
RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
"Can't ref a prefix when ctx not in init stage.");
ZSTD_clearAllDicts(cctx);
if (prefix != NULL && prefixSize > 0) {
cctx->prefixDict.dict = prefix;
cctx->prefixDict.dictSize = prefixSize;
cctx->prefixDict.dictContentType = dictContentType;
}
return 0;
return ZSTD_rust_CCtx_refPrefixAdvanced(
cctx, (int)cctx->streamStage,
prefix, prefixSize, (int)dictContentType,
ZSTD_clearAllDicts_callback, ZSTD_assignPrefixDict_callback);
}
/*! ZSTD_CCtx_reset() :
rust/src/zstd_compress_dictionary.rs
+272
View File
@@ -50,11 +50,114 @@ pub type LoadDictionaryContentFn = unsafe extern "C" fn(
tfp: c_int,
) -> usize;
/// These callbacks are deliberately opaque: C retains the private CCtx,
/// local/prefix dictionary layouts, allocation, and CDict lifetime rules.
/// Rust owns the stage checks and the order in which clear/assign operations
/// are selected.
pub type CctxDictionaryClearFn = unsafe extern "C" fn(context: *mut c_void);
pub type CctxAssignLocalDictFn = unsafe extern "C" fn(
context: *mut c_void,
dict: *const c_void,
dict_size: usize,
dict_load_method: c_int,
dict_content_type: c_int,
) -> usize;
pub type CctxAssignCDictFn = unsafe extern "C" fn(context: *mut c_void, cdict: *const c_void);
pub type CctxAssignPrefixDictFn = unsafe extern "C" fn(
context: *mut c_void,
prefix: *const c_void,
prefix_size: usize,
dict_content_type: c_int,
);
#[inline]
fn dictionary_corrupted() -> usize {
ERROR(ZstdErrorCode::DictionaryCorrupted)
}
const ZSTD_CCTX_INIT_STAGE: c_int = 0;
#[cfg(test)]
const ZSTD_DLM_BY_REF: c_int = 1;
#[inline]
fn stage_wrong() -> usize {
ERROR(ZstdErrorCode::StageWrong)
}
/// Rust-owned policy for `ZSTD_CCtx_loadDictionary_advanced()`.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_CCtx_loadDictionaryAdvanced(
context: *mut c_void,
stream_stage: c_int,
dict: *const c_void,
dict_size: usize,
dict_load_method: c_int,
dict_content_type: c_int,
clear_dictionaries: CctxDictionaryClearFn,
assign_local_dict: CctxAssignLocalDictFn,
) -> usize {
if stream_stage != ZSTD_CCTX_INIT_STAGE {
return stage_wrong();
}
unsafe { clear_dictionaries(context) };
if dict.is_null() || dict_size == 0 {
return 0;
}
unsafe {
assign_local_dict(
context,
dict,
dict_size,
dict_load_method,
dict_content_type,
)
}
}
/// Rust-owned policy for `ZSTD_CCtx_refCDict()`.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_CCtx_refCDict(
context: *mut c_void,
stream_stage: c_int,
cdict: *const c_void,
clear_dictionaries: CctxDictionaryClearFn,
assign_cdict: CctxAssignCDictFn,
) -> usize {
if stream_stage != ZSTD_CCTX_INIT_STAGE {
return stage_wrong();
}
unsafe { clear_dictionaries(context) };
unsafe { assign_cdict(context, cdict) };
0
}
/// Rust-owned policy for `ZSTD_CCtx_refPrefix_advanced()`.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_CCtx_refPrefixAdvanced(
context: *mut c_void,
stream_stage: c_int,
prefix: *const c_void,
prefix_size: usize,
dict_content_type: c_int,
clear_dictionaries: CctxDictionaryClearFn,
assign_prefix_dict: CctxAssignPrefixDictFn,
) -> usize {
if stream_stage != ZSTD_CCTX_INIT_STAGE {
return stage_wrong();
}
unsafe { clear_dictionaries(context) };
if prefix.is_null() || prefix_size == 0 {
return 0;
}
unsafe { assign_prefix_dict(context, prefix, prefix_size, dict_content_type) };
0
}
/// Matches C's `ZSTD_dictNCountRepeat()`.
#[inline]
fn dict_n_count_repeat(
@@ -422,6 +525,79 @@ mod tests {
}
}
struct CctxPolicyProbe {
events: Vec<u8>,
local_dict: *const c_void,
local_dict_size: usize,
local_dict_load_method: c_int,
local_dict_content_type: c_int,
local_result: usize,
cdict: *const c_void,
prefix: *const c_void,
prefix_size: usize,
prefix_content_type: c_int,
}
impl Default for CctxPolicyProbe {
fn default() -> Self {
Self {
events: Vec::new(),
local_dict: ptr::null(),
local_dict_size: 0,
local_dict_load_method: 0,
local_dict_content_type: 0,
local_result: 0,
cdict: ptr::null(),
prefix: ptr::null(),
prefix_size: 0,
prefix_content_type: 0,
}
}
}
unsafe fn cctx_policy_probe(context: *mut c_void) -> &'static mut CctxPolicyProbe {
unsafe { &mut *context.cast::<CctxPolicyProbe>() }
}
unsafe extern "C" fn policy_clear(context: *mut c_void) {
unsafe { cctx_policy_probe(context) }.events.push(3);
}
unsafe extern "C" fn policy_assign_local(
context: *mut c_void,
dict: *const c_void,
dict_size: usize,
dict_load_method: c_int,
dict_content_type: c_int,
) -> usize {
let probe = unsafe { cctx_policy_probe(context) };
probe.events.push(4);
probe.local_dict = dict;
probe.local_dict_size = dict_size;
probe.local_dict_load_method = dict_load_method;
probe.local_dict_content_type = dict_content_type;
probe.local_result
}
unsafe extern "C" fn policy_assign_cdict(context: *mut c_void, cdict: *const c_void) {
let probe = unsafe { cctx_policy_probe(context) };
probe.events.push(5);
probe.cdict = cdict;
}
unsafe extern "C" fn policy_assign_prefix(
context: *mut c_void,
prefix: *const c_void,
prefix_size: usize,
dict_content_type: c_int,
) {
let probe = unsafe { cctx_policy_probe(context) };
probe.events.push(6);
probe.prefix = prefix;
probe.prefix_size = prefix_size;
probe.prefix_content_type = dict_content_type;
}
unsafe extern "C" fn record_dictionary_load(
match_state: *mut c_void,
ldm_state: *mut c_void,
@@ -471,6 +647,102 @@ mod tests {
}
}
#[test]
fn cctx_dictionary_policy_checks_stage_and_orders_callbacks() {
let mut probe = CctxPolicyProbe::default();
let dictionary = [1u8, 2, 3];
let context = (&mut probe as *mut CctxPolicyProbe).cast::<c_void>();
let result = unsafe {
ZSTD_rust_CCtx_loadDictionaryAdvanced(
context,
1,
dictionary.as_ptr().cast(),
dictionary.len(),
ZSTD_DLM_BY_REF,
ZSTD_DCT_RAW_CONTENT,
policy_clear,
policy_assign_local,
)
};
assert_eq!(result, ERROR(ZstdErrorCode::StageWrong));
assert!(probe.events.is_empty());
probe.local_result = ERROR(ZstdErrorCode::MemoryAllocation);
let result = unsafe {
ZSTD_rust_CCtx_loadDictionaryAdvanced(
context,
ZSTD_CCTX_INIT_STAGE,
dictionary.as_ptr().cast(),
dictionary.len(),
ZSTD_DLM_BY_REF,
ZSTD_DCT_RAW_CONTENT,
policy_clear,
policy_assign_local,
)
};
assert_eq!(result, ERROR(ZstdErrorCode::MemoryAllocation));
assert_eq!(probe.events, [3, 4]);
assert_eq!(probe.local_dict, dictionary.as_ptr().cast());
assert_eq!(probe.local_dict_size, dictionary.len());
assert_eq!(probe.local_dict_load_method, ZSTD_DLM_BY_REF);
assert_eq!(probe.local_dict_content_type, ZSTD_DCT_RAW_CONTENT);
}
#[test]
fn cctx_dictionary_policy_clears_before_cdict_and_prefix_assignment() {
let mut probe = CctxPolicyProbe::default();
let cdict_marker = 17u8;
let prefix = [4u8, 5, 6];
let context = (&mut probe as *mut CctxPolicyProbe).cast::<c_void>();
let result = unsafe {
ZSTD_rust_CCtx_refCDict(
context,
ZSTD_CCTX_INIT_STAGE,
(&cdict_marker as *const u8).cast(),
policy_clear,
policy_assign_cdict,
)
};
assert_eq!(result, 0);
assert_eq!(probe.events, [3, 5]);
assert_eq!(probe.cdict, (&cdict_marker as *const u8).cast());
probe.events.clear();
let result = unsafe {
ZSTD_rust_CCtx_refPrefixAdvanced(
context,
ZSTD_CCTX_INIT_STAGE,
prefix.as_ptr().cast(),
prefix.len(),
ZSTD_DCT_FULL_DICT,
policy_clear,
policy_assign_prefix,
)
};
assert_eq!(result, 0);
assert_eq!(probe.events, [3, 6]);
assert_eq!(probe.prefix, prefix.as_ptr().cast());
assert_eq!(probe.prefix_size, prefix.len());
assert_eq!(probe.prefix_content_type, ZSTD_DCT_FULL_DICT);
probe.events.clear();
let result = unsafe {
ZSTD_rust_CCtx_refPrefixAdvanced(
context,
ZSTD_CCTX_INIT_STAGE,
ptr::null(),
0,
ZSTD_DCT_AUTO,
policy_clear,
policy_assign_prefix,
)
};
assert_eq!(result, 0);
assert_eq!(probe.events, [3]);
}
fn assert_dictionary_corrupted(result: usize) {
assert!(ERR_isError(result));
assert_eq!(