ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move local dictionary assignment to Rust

Browse Source 
The local-dictionary callback still contained the policy that selected
by-reference versus by-copy loading, rejected internal copies for static
CCtx instances, allocated and copied dictionary bytes, and published the
resulting local-dictionary state. Move that policy into the Rust compression
rewrite so the C callback is only an adapter around the private CCtx and
localDict layouts.

The adapter retains the C custom allocator and output slots, preserving the
existing ownership and free path while keeping the ABI boundary explicit.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --manifest-path rust/Cargo.toml --tests
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1 -C tests invalidDictionaries
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -B -j1 -C tests fuzzer
- (cd tests && ulimit -v 41943040; ./fuzzer -v -i1 -s6572)
- ulimit -v 41943040; make -j1 -C tests test
This commit is contained in:
ddidderr
2026-07-21 17:28:44 +02:00
parent b0fba9c87b
commit 49a448602a
2 changed files with 128 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+40 -17
View File
@@ -2839,6 +2839,30 @@ typedef void (*ZSTD_rust_CCtxDictionaryClear_f)(void* context);
typedef size_t (*ZSTD_rust_CCtxAssignLocalDict_f)(
void* context, const void* dict, size_t dictSize,
int dictLoadMethod, int dictContentType);
typedef void* (*ZSTD_rust_assignLocalDictAllocate_f)(
void* context, size_t size);
typedef struct {
void* callbackContext;
size_t staticSize;
const void** dict;
void** dictBuffer;
size_t* dictSize;
int* dictContentType;
ZSTD_rust_assignLocalDictAllocate_f allocate;
} ZSTD_rust_assignLocalDictState;
typedef char ZSTD_rust_assign_local_dict_state_layout[
(offsetof(ZSTD_rust_assignLocalDictState, callbackContext) == 0
&& offsetof(ZSTD_rust_assignLocalDictState, staticSize) == sizeof(void*)
&& offsetof(ZSTD_rust_assignLocalDictState, dict) == 2 * sizeof(void*)
&& offsetof(ZSTD_rust_assignLocalDictState, dictBuffer) == 3 * sizeof(void*)
&& offsetof(ZSTD_rust_assignLocalDictState, dictSize) == 4 * sizeof(void*)
&& offsetof(ZSTD_rust_assignLocalDictState, dictContentType) == 5 * sizeof(void*)
&& offsetof(ZSTD_rust_assignLocalDictState, allocate) == 6 * sizeof(void*)
&& sizeof(ZSTD_rust_assignLocalDictState) == 7 * sizeof(void*)) ? 1 : -1];
size_t ZSTD_rust_assignLocalDict(
const ZSTD_rust_assignLocalDictState* state,
const void* dict, size_t dictSize,
int dictLoadMethod, int dictContentType);
typedef void (*ZSTD_rust_CCtxAssignCDict_f)(
void* context, const void* cdict);
typedef void (*ZSTD_rust_CCtxAssignPrefixDict_f)(
@@ -4106,29 +4130,28 @@ static void ZSTD_clearAllDicts_callback(void* context)
ZSTD_clearAllDicts((ZSTD_CCtx*)context);
}
static void* ZSTD_rust_assignLocalDict_allocate(void* context, size_t size)
{
ZSTD_CCtx const* const cctx = (const ZSTD_CCtx*)context;
return ZSTD_customMalloc(size, cctx->customMem);
}
static size_t ZSTD_assignLocalDict_callback(
void* context, const void* dict, size_t dictSize,
int dictLoadMethod, int dictContentType)
{
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context;
ZSTD_localDict* const localDict = &cctx->localDict;
if (dictLoadMethod == ZSTD_dlm_byRef) {
localDict->dict = dict;
} else {
void* dictBuffer;
RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
"static CCtx can't allocate for an internal copy of dictionary");
dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem);
RETURN_ERROR_IF(dictBuffer == NULL, memory_allocation,
"allocation failed for dictionary content");
ZSTD_memcpy(dictBuffer, dict, dictSize);
localDict->dictBuffer = dictBuffer; /* owned ptr to free */
localDict->dict = dictBuffer; /* read-only reference */
}
localDict->dictSize = dictSize;
localDict->dictContentType = (ZSTD_dictContentType_e)dictContentType;
return 0;
ZSTD_rust_assignLocalDictState state;
state.callbackContext = cctx;
state.staticSize = cctx->staticSize;
state.dict = &localDict->dict;
state.dictBuffer = &localDict->dictBuffer;
state.dictSize = &localDict->dictSize;
state.dictContentType = (int*)&localDict->dictContentType;
state.allocate = ZSTD_rust_assignLocalDict_allocate;
return ZSTD_rust_assignLocalDict(
&state, dict, dictSize, dictLoadMethod, dictContentType);
}
static void ZSTD_assignCDict_callback(void* context, const void* cdict)
rust/src/zstd_compress_dictionary.rs
+88
View File
@@ -462,6 +462,94 @@ pub type CctxAssignPrefixDictFn = unsafe extern "C" fn(
dict_content_type: c_int,
);
type CctxAssignLocalDictAllocateFn = unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void;
/// C-owned outputs and allocator for assigning a local dictionary to a CCtx.
///
/// Rust owns the by-reference/by-copy decision, static-context rejection,
/// allocation/copy ordering, and publication ordering. C retains the private
/// `ZSTD_CCtx`/`ZSTD_localDict` layouts and supplies only output slots plus its
/// custom allocator adapter.
#[repr(C)]
pub struct ZSTD_rust_assignLocalDictState {
callback_context: *mut c_void,
static_size: usize,
dict: *mut *const c_void,
dict_buffer: *mut *mut c_void,
dict_size: *mut usize,
dict_content_type: *mut c_int,
allocate: CctxAssignLocalDictAllocateFn,
}
const _: () = {
assert!(offset_of!(ZSTD_rust_assignLocalDictState, callback_context) == 0);
assert!(offset_of!(ZSTD_rust_assignLocalDictState, static_size) == size_of::<usize>());
assert!(offset_of!(ZSTD_rust_assignLocalDictState, dict) == 2 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_assignLocalDictState, dict_buffer) == 3 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_assignLocalDictState, dict_size) == 4 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_assignLocalDictState, dict_content_type) == 5 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_assignLocalDictState, allocate) == 6 * size_of::<usize>());
assert!(size_of::<ZSTD_rust_assignLocalDictState>() == size_of::<[usize; 7]>());
};
/// Assign a local dictionary while keeping the CCtx and local-dictionary
/// layouts in C.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_assignLocalDict(
state: *const ZSTD_rust_assignLocalDictState,
dict: *const c_void,
dict_size: usize,
dict_load_method: c_int,
dict_content_type: c_int,
) -> usize {
if state.is_null() {
return ERROR(ZstdErrorCode::Generic);
}
let state = unsafe { &*state };
if state.callback_context.is_null()
|| state.dict.is_null()
|| state.dict_buffer.is_null()
|| state.dict_size.is_null()
|| state.dict_content_type.is_null()
{
return ERROR(ZstdErrorCode::Generic);
}
if dict_load_method == ZSTD_DLM_BY_REF {
unsafe { *state.dict = dict };
} else {
if state.static_size != 0 {
return ERROR(ZstdErrorCode::MemoryAllocation);
}
if dict.is_null() && dict_size != 0 {
return ERROR(ZstdErrorCode::Generic);
}
let dict_buffer = unsafe { (state.allocate)(state.callback_context, dict_size) };
if dict_buffer.is_null() {
return ERROR(ZstdErrorCode::MemoryAllocation);
}
if dict_size != 0 {
unsafe {
ptr::copy_nonoverlapping(
dict.cast::<u8>(),
dict_buffer.cast::<u8>(),
dict_size,
);
}
}
unsafe {
*state.dict_buffer = dict_buffer;
*state.dict = dict_buffer.cast_const();
}
}
unsafe {
*state.dict_size = dict_size;
*state.dict_content_type = dict_content_type;
}
0
}
#[inline]
fn dictionary_corrupted() -> usize {
ERROR(ZstdErrorCode::DictionaryCorrupted)