From 1422d64300906e3bd1b2cd625457efc4289b982d Mon Sep 17 00:00:00 2001 From: ddidderr Date: Sat, 18 Jul 2026 20:17:16 +0200 Subject: [PATCH] feat(decompress): move decoder allocation bridges into Rust Decoder orchestration already crossed into Rust, but the C translation unit still owned the allocation bridge used by every decoder context and dictionary set. That split kept custom malloc/calloc/free behavior, allocator-pair validation, and the decoder-context storage lifecycle in C without focused Rust coverage. Move those bridges into Rust while keeping the C-defined `ZSTD_DCtx` layout and configuration-dependent platform, legacy, and trace leaves in C. Rust now validates custom allocator pairs, allocates and frees the opaque decoder storage, dispatches default allocations to libc, and preserves calloc's zero-fill contract for custom allocators. Focused tests cover default memory, custom callbacks, zeroing, null-safe free, and invalid callback pairs. Test Plan: - `cargo test --manifest-path rust/Cargo.toml --lib -- --test-threads=1` -- 447 passed - `cargo test --manifest-path rust/Cargo.toml --lib zstd_decompress -- --test-threads=1` -- 6 passed - `cargo clippy --manifest-path rust/Cargo.toml --lib -- -D warnings` -- passed - `cargo +nightly fmt --manifest-path rust/Cargo.toml -- --check` -- passed - `make -B -C lib -j2 lib` -- passed - `make -B -C tests -j2 test-cli-tests` -- passed - `ZSTREAM_TESTTIME=-T2s make -B -C tests -j2 test-zstream` -- passed - `FUZZERTEST=-T5s make -B -C tests -j2 test-fuzzer` -- passed - `make -B -C tests/fuzz -j2 all` and `sequence_compression_api` -- passed - `make -C tests -j2 test-zstd` -- passed, including large-data cases --- lib/decompress/zstd_decompress.c | 37 +------ rust/src/zstd_decompress.rs | 159 ++++++++++++++++++++++++++++++- 2 files changed, 156 insertions(+), 40 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index c741ce1e1..9c8823f98 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -4,14 +4,12 @@ * `ZSTD_DCtx_s` is intentionally still allocated and laid out by C: its * optional members vary with the build configuration and the block decoder * shares that object. This translation unit is therefore a deliberately - * narrow ABI adapter. It projects field addresses to Rust, keeps allocation - * ownership in the C allocator domain, and retains the configuration-bound - * legacy and trace leaves. + * narrow ABI adapter. It projects field addresses to Rust while retaining + * the private layout and configuration-bound legacy and trace leaves. */ #define ZSTD_STATIC_LINKING_ONLY #include "../common/zstd_deps.h" -#include "../common/allocations.h" #include "../common/error_private.h" #include "../common/mem.h" #include "../common/zstd_internal.h" @@ -113,17 +111,12 @@ typedef struct { void ZSTD_rust_dctx_view(ZSTD_DCtx* dctx, ZSTD_rustDctxView* out); size_t ZSTD_rust_dctx_sizeof(void); -ZSTD_DCtx* ZSTD_rust_dctx_alloc(ZSTD_customMem customMem); -void ZSTD_rust_dctx_free_storage(ZSTD_DCtx* dctx, ZSTD_customMem customMem); void ZSTD_rust_dctx_init_platform(ZSTD_DCtx* dctx); size_t ZSTD_rust_dctx_default_max_window_size(void); int ZSTD_rust_no_forward_progress_max(void); int ZSTD_rust_heapmode(void); size_t ZSTD_rust_decompress_stack(void* dst, size_t dstCapacity, const void* src, size_t srcSize); -void* ZSTD_rust_custom_malloc(size_t size, ZSTD_customMem customMem); -void* ZSTD_rust_custom_calloc(size_t size, ZSTD_customMem customMem); -void ZSTD_rust_custom_free(void* allocation, ZSTD_customMem customMem); ZSTD_DDict* ZSTD_rust_create_ddict(const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, @@ -229,17 +222,6 @@ size_t ZSTD_rust_dctx_sizeof(void) return sizeof(ZSTD_DCtx); } -ZSTD_DCtx* ZSTD_rust_dctx_alloc(ZSTD_customMem customMem) -{ - if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; - return (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(ZSTD_DCtx), customMem); -} - -void ZSTD_rust_dctx_free_storage(ZSTD_DCtx* dctx, ZSTD_customMem customMem) -{ - ZSTD_customFree(dctx, customMem); -} - void ZSTD_rust_dctx_init_platform(ZSTD_DCtx* dctx) { #if DYNAMIC_BMI2 @@ -285,21 +267,6 @@ size_t ZSTD_rust_decompress_stack(void* dst, size_t dstCapacity, #endif } -void* ZSTD_rust_custom_malloc(size_t size, ZSTD_customMem customMem) -{ - return ZSTD_customMalloc(size, customMem); -} - -void* ZSTD_rust_custom_calloc(size_t size, ZSTD_customMem customMem) -{ - return ZSTD_customCalloc(size, customMem); -} - -void ZSTD_rust_custom_free(void* allocation, ZSTD_customMem customMem) -{ - ZSTD_customFree(allocation, customMem); -} - ZSTD_DDict* ZSTD_rust_create_ddict(const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType, diff --git a/rust/src/zstd_decompress.rs b/rust/src/zstd_decompress.rs index a2602089f..f0e4146f6 100644 --- a/rust/src/zstd_decompress.rs +++ b/rust/src/zstd_decompress.rs @@ -301,8 +301,6 @@ struct ZSTD_rustDctxView { unsafe extern "C" { fn ZSTD_rust_dctx_view(dctx: *mut ZSTD_DCtx, out: *mut ZSTD_rustDctxView); fn ZSTD_rust_dctx_sizeof() -> usize; - fn ZSTD_rust_dctx_alloc(custom_mem: ZSTD_customMem) -> *mut ZSTD_DCtx; - fn ZSTD_rust_dctx_free_storage(dctx: *mut ZSTD_DCtx, custom_mem: ZSTD_customMem); fn ZSTD_rust_dctx_init_platform(dctx: *mut ZSTD_DCtx); fn ZSTD_rust_dctx_default_max_window_size() -> usize; fn ZSTD_rust_no_forward_progress_max() -> c_int; @@ -313,9 +311,6 @@ unsafe extern "C" { src: *const c_void, src_size: usize, ) -> usize; - fn ZSTD_rust_custom_malloc(size: usize, custom_mem: ZSTD_customMem) -> *mut c_void; - fn ZSTD_rust_custom_calloc(size: usize, custom_mem: ZSTD_customMem) -> *mut c_void; - fn ZSTD_rust_custom_free(allocation: *mut c_void, custom_mem: ZSTD_customMem); fn ZSTD_rust_create_ddict( dict: *const c_void, dict_size: usize, @@ -871,6 +866,160 @@ fn custom_mem_valid(custom_mem: ZSTD_customMem) -> bool { custom_mem.custom_alloc.is_some() == custom_mem.custom_free.is_some() } +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_dctx_alloc(custom_mem: ZSTD_customMem) -> *mut ZSTD_DCtx { + if !custom_mem_valid(custom_mem) { + return ptr::null_mut(); + } + let dctx_size = unsafe { ZSTD_rust_dctx_sizeof() }; + unsafe { ZSTD_rust_custom_malloc(dctx_size, custom_mem) }.cast() +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_dctx_free_storage( + dctx: *mut ZSTD_DCtx, + custom_mem: ZSTD_customMem, +) { + unsafe { ZSTD_rust_custom_free(dctx.cast(), custom_mem) }; +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_custom_malloc( + size: usize, + custom_mem: ZSTD_customMem, +) -> *mut c_void { + match custom_mem.custom_alloc { + Some(alloc) => unsafe { alloc(custom_mem.opaque, size) }, + None => unsafe { libc::malloc(size) }, + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_custom_calloc( + size: usize, + custom_mem: ZSTD_customMem, +) -> *mut c_void { + match custom_mem.custom_alloc { + Some(alloc) => { + let allocation = unsafe { alloc(custom_mem.opaque, size) }; + if !allocation.is_null() { + unsafe { ptr::write_bytes(allocation.cast::(), 0, size) }; + } + allocation + } + None => unsafe { libc::calloc(1, size) }, + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_custom_free( + allocation: *mut c_void, + custom_mem: ZSTD_customMem, +) { + if allocation.is_null() { + return; + } + match custom_mem.custom_free { + Some(free) => unsafe { free(custom_mem.opaque, allocation) }, + None => unsafe { libc::free(allocation) }, + } +} + +#[cfg(test)] +mod allocation_tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct AllocationStats { + allocations: AtomicUsize, + frees: AtomicUsize, + } + + unsafe extern "C" fn tracking_alloc(opaque: *mut c_void, size: usize) -> *mut c_void { + let stats = unsafe { &*opaque.cast::() }; + stats.allocations.fetch_add(1, Ordering::Relaxed); + let allocation = unsafe { libc::malloc(size) }; + if !allocation.is_null() { + unsafe { ptr::write_bytes(allocation.cast::(), 0xA5, size) }; + } + allocation + } + + unsafe extern "C" fn tracking_free(opaque: *mut c_void, allocation: *mut c_void) { + let stats = unsafe { &*opaque.cast::() }; + stats.frees.fetch_add(1, Ordering::Relaxed); + unsafe { libc::free(allocation) }; + } + + fn custom_mem(stats: &AllocationStats) -> ZSTD_customMem { + ZSTD_customMem { + custom_alloc: Some(tracking_alloc), + custom_free: Some(tracking_free), + opaque: (stats as *const AllocationStats).cast_mut().cast(), + } + } + + #[test] + fn default_allocation_uses_libc_and_zeroes_calloc() { + let default_mem = default_custom_mem(); + unsafe { + let allocation = ZSTD_rust_custom_malloc(32, default_mem); + assert!(!allocation.is_null()); + ZSTD_rust_custom_free(allocation, default_mem); + + let zeroed = ZSTD_rust_custom_calloc(32, default_mem); + assert!(!zeroed.is_null()); + let bytes = std::slice::from_raw_parts(zeroed.cast::(), 32); + assert!(bytes.iter().all(|&byte| byte == 0)); + ZSTD_rust_custom_free(zeroed, default_mem); + ZSTD_rust_custom_free(ptr::null_mut(), default_mem); + } + } + + #[test] + fn custom_allocation_calls_callbacks_and_zeroes_calloc() { + let stats = AllocationStats { + allocations: AtomicUsize::new(0), + frees: AtomicUsize::new(0), + }; + let custom_mem = custom_mem(&stats); + + unsafe { + let allocation = ZSTD_rust_custom_malloc(32, custom_mem); + assert!(!allocation.is_null()); + assert_eq!(stats.allocations.load(Ordering::Relaxed), 1); + + let zeroed = ZSTD_rust_custom_calloc(32, custom_mem); + assert!(!zeroed.is_null()); + assert_eq!(stats.allocations.load(Ordering::Relaxed), 2); + let bytes = std::slice::from_raw_parts(zeroed.cast::(), 32); + assert!(bytes.iter().all(|&byte| byte == 0)); + + ZSTD_rust_custom_free(allocation, custom_mem); + ZSTD_rust_custom_free(zeroed, custom_mem); + ZSTD_rust_custom_free(ptr::null_mut(), custom_mem); + } + assert_eq!(stats.frees.load(Ordering::Relaxed), 2); + } + + #[test] + fn dctx_allocation_rejects_mismatched_callback_pairs() { + let invalid_alloc_only = ZSTD_customMem { + custom_alloc: Some(tracking_alloc), + custom_free: None, + opaque: ptr::null_mut(), + }; + let invalid_free_only = ZSTD_customMem { + custom_alloc: None, + custom_free: Some(tracking_free), + opaque: ptr::null_mut(), + }; + + assert!(!custom_mem_valid(invalid_alloc_only)); + assert!(!custom_mem_valid(invalid_free_only)); + } +} + #[inline] unsafe fn get_frame_header_ptr(view: &ZSTD_rustDctxView) -> *mut ZSTD_FrameHeader { view.f_params.cast()