From 49a448602a2fe31442a8a3854a3a7820f293f2b9 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Tue, 21 Jul 2026 17:28:44 +0200 Subject: [PATCH] feat(compress): move local dictionary assignment to Rust The local-dictionary callback still contained the policy that selected by-reference versus by-copy loading, rejected internal copies for static CCtx instances, allocated and copied dictionary bytes, and published the resulting local-dictionary state. Move that policy into the Rust compression rewrite so the C callback is only an adapter around the private CCtx and localDict layouts. The adapter retains the C custom allocator and output slots, preserving the existing ownership and free path while keeping the ABI boundary explicit. Test Plan: - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --manifest-path rust/Cargo.toml --tests - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1 - ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1 -C tests invalidDictionaries - ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -B -j1 -C tests fuzzer - (cd tests && ulimit -v 41943040; ./fuzzer -v -i1 -s6572) - ulimit -v 41943040; make -j1 -C tests test --- lib/compress/zstd_compress.c | 57 ++++++++++++------ rust/src/zstd_compress_dictionary.rs | 88 ++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 17 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 4c2931374..e74676f21 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2839,6 +2839,30 @@ typedef void (*ZSTD_rust_CCtxDictionaryClear_f)(void* context); typedef size_t (*ZSTD_rust_CCtxAssignLocalDict_f)( void* context, const void* dict, size_t dictSize, int dictLoadMethod, int dictContentType); +typedef void* (*ZSTD_rust_assignLocalDictAllocate_f)( + void* context, size_t size); +typedef struct { + void* callbackContext; + size_t staticSize; + const void** dict; + void** dictBuffer; + size_t* dictSize; + int* dictContentType; + ZSTD_rust_assignLocalDictAllocate_f allocate; +} ZSTD_rust_assignLocalDictState; +typedef char ZSTD_rust_assign_local_dict_state_layout[ + (offsetof(ZSTD_rust_assignLocalDictState, callbackContext) == 0 + && offsetof(ZSTD_rust_assignLocalDictState, staticSize) == sizeof(void*) + && offsetof(ZSTD_rust_assignLocalDictState, dict) == 2 * sizeof(void*) + && offsetof(ZSTD_rust_assignLocalDictState, dictBuffer) == 3 * sizeof(void*) + && offsetof(ZSTD_rust_assignLocalDictState, dictSize) == 4 * sizeof(void*) + && offsetof(ZSTD_rust_assignLocalDictState, dictContentType) == 5 * sizeof(void*) + && offsetof(ZSTD_rust_assignLocalDictState, allocate) == 6 * sizeof(void*) + && sizeof(ZSTD_rust_assignLocalDictState) == 7 * sizeof(void*)) ? 1 : -1]; +size_t ZSTD_rust_assignLocalDict( + const ZSTD_rust_assignLocalDictState* state, + const void* dict, size_t dictSize, + int dictLoadMethod, int dictContentType); typedef void (*ZSTD_rust_CCtxAssignCDict_f)( void* context, const void* cdict); typedef void (*ZSTD_rust_CCtxAssignPrefixDict_f)( @@ -4106,29 +4130,28 @@ static void ZSTD_clearAllDicts_callback(void* context) ZSTD_clearAllDicts((ZSTD_CCtx*)context); } +static void* ZSTD_rust_assignLocalDict_allocate(void* context, size_t size) +{ + ZSTD_CCtx const* const cctx = (const ZSTD_CCtx*)context; + return ZSTD_customMalloc(size, cctx->customMem); +} + static size_t ZSTD_assignLocalDict_callback( void* context, const void* dict, size_t dictSize, int dictLoadMethod, int dictContentType) { ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context; ZSTD_localDict* const localDict = &cctx->localDict; - - if (dictLoadMethod == ZSTD_dlm_byRef) { - localDict->dict = dict; - } else { - void* dictBuffer; - RETURN_ERROR_IF(cctx->staticSize, memory_allocation, - "static CCtx can't allocate for an internal copy of dictionary"); - dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem); - RETURN_ERROR_IF(dictBuffer == NULL, memory_allocation, - "allocation failed for dictionary content"); - ZSTD_memcpy(dictBuffer, dict, dictSize); - localDict->dictBuffer = dictBuffer; /* owned ptr to free */ - localDict->dict = dictBuffer; /* read-only reference */ - } - localDict->dictSize = dictSize; - localDict->dictContentType = (ZSTD_dictContentType_e)dictContentType; - return 0; + ZSTD_rust_assignLocalDictState state; + state.callbackContext = cctx; + state.staticSize = cctx->staticSize; + state.dict = &localDict->dict; + state.dictBuffer = &localDict->dictBuffer; + state.dictSize = &localDict->dictSize; + state.dictContentType = (int*)&localDict->dictContentType; + state.allocate = ZSTD_rust_assignLocalDict_allocate; + return ZSTD_rust_assignLocalDict( + &state, dict, dictSize, dictLoadMethod, dictContentType); } static void ZSTD_assignCDict_callback(void* context, const void* cdict) diff --git a/rust/src/zstd_compress_dictionary.rs b/rust/src/zstd_compress_dictionary.rs index 0fe06bc8d..f24566e3f 100644 --- a/rust/src/zstd_compress_dictionary.rs +++ b/rust/src/zstd_compress_dictionary.rs @@ -462,6 +462,94 @@ pub type CctxAssignPrefixDictFn = unsafe extern "C" fn( dict_content_type: c_int, ); +type CctxAssignLocalDictAllocateFn = unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void; + +/// C-owned outputs and allocator for assigning a local dictionary to a CCtx. +/// +/// Rust owns the by-reference/by-copy decision, static-context rejection, +/// allocation/copy ordering, and publication ordering. C retains the private +/// `ZSTD_CCtx`/`ZSTD_localDict` layouts and supplies only output slots plus its +/// custom allocator adapter. +#[repr(C)] +pub struct ZSTD_rust_assignLocalDictState { + callback_context: *mut c_void, + static_size: usize, + dict: *mut *const c_void, + dict_buffer: *mut *mut c_void, + dict_size: *mut usize, + dict_content_type: *mut c_int, + allocate: CctxAssignLocalDictAllocateFn, +} + +const _: () = { + assert!(offset_of!(ZSTD_rust_assignLocalDictState, callback_context) == 0); + assert!(offset_of!(ZSTD_rust_assignLocalDictState, static_size) == size_of::()); + assert!(offset_of!(ZSTD_rust_assignLocalDictState, dict) == 2 * size_of::()); + assert!(offset_of!(ZSTD_rust_assignLocalDictState, dict_buffer) == 3 * size_of::()); + assert!(offset_of!(ZSTD_rust_assignLocalDictState, dict_size) == 4 * size_of::()); + assert!(offset_of!(ZSTD_rust_assignLocalDictState, dict_content_type) == 5 * size_of::()); + assert!(offset_of!(ZSTD_rust_assignLocalDictState, allocate) == 6 * size_of::()); + assert!(size_of::() == size_of::<[usize; 7]>()); +}; + +/// Assign a local dictionary while keeping the CCtx and local-dictionary +/// layouts in C. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_assignLocalDict( + state: *const ZSTD_rust_assignLocalDictState, + dict: *const c_void, + dict_size: usize, + dict_load_method: c_int, + dict_content_type: c_int, +) -> usize { + if state.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + let state = unsafe { &*state }; + if state.callback_context.is_null() + || state.dict.is_null() + || state.dict_buffer.is_null() + || state.dict_size.is_null() + || state.dict_content_type.is_null() + { + return ERROR(ZstdErrorCode::Generic); + } + + if dict_load_method == ZSTD_DLM_BY_REF { + unsafe { *state.dict = dict }; + } else { + if state.static_size != 0 { + return ERROR(ZstdErrorCode::MemoryAllocation); + } + if dict.is_null() && dict_size != 0 { + return ERROR(ZstdErrorCode::Generic); + } + let dict_buffer = unsafe { (state.allocate)(state.callback_context, dict_size) }; + if dict_buffer.is_null() { + return ERROR(ZstdErrorCode::MemoryAllocation); + } + if dict_size != 0 { + unsafe { + ptr::copy_nonoverlapping( + dict.cast::(), + dict_buffer.cast::(), + dict_size, + ); + } + } + unsafe { + *state.dict_buffer = dict_buffer; + *state.dict = dict_buffer.cast_const(); + } + } + + unsafe { + *state.dict_size = dict_size; + *state.dict_content_type = dict_content_type; + } + 0 +} + #[inline] fn dictionary_corrupted() -> usize { ERROR(ZstdErrorCode::DictionaryCorrupted)