From 865943e07297a2013fc509ee82dc9871b43d3f67 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Tue, 21 Jul 2026 21:09:55 +0200 Subject: [PATCH] test(compress): cover Rust local dictionary assignment The local-dictionary callback is now a C adapter around Rust-owned assignment policy. Add a small allocator probe and focused tests for by-reference loading, by-copy data publication, static-CCtx rejection, and allocation failure. The probe observes the output slots during allocation so the test also protects the original allocation-before-publication ordering. Strengthen the C and Rust repr(C) checks with an explicit custom-allocator function-pointer size check. The capped C-integrated build and original test suite remain deferred to the main worker, which must run them serially under the repository's 40 GiB virtual memory limit. Test Plan: - `git diff --cached --check` -- passed. - `rustfmt --check rust/src/zstd_compress_dictionary.rs` -- passed. - Capped cargo, make, native, fuzzer, and full-suite verification -- not run by design; deferred to the main worker. --- lib/compress/zstd_compress.c | 1 + rust/src/zstd_compress_dictionary.rs | 269 ++++++++++++++++++++++++++- 2 files changed, 264 insertions(+), 6 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index b33b985c3..d794b309b 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2914,6 +2914,7 @@ typedef char ZSTD_rust_assign_local_dict_state_layout[ && offsetof(ZSTD_rust_assignLocalDictState, dictBuffer) == 3 * sizeof(void*) && offsetof(ZSTD_rust_assignLocalDictState, dictSize) == 4 * sizeof(void*) && offsetof(ZSTD_rust_assignLocalDictState, dictContentType) == 5 * sizeof(void*) + && sizeof(ZSTD_rust_assignLocalDictAllocate_f) == sizeof(void*) && offsetof(ZSTD_rust_assignLocalDictState, allocate) == 6 * sizeof(void*) && sizeof(ZSTD_rust_assignLocalDictState) == 7 * sizeof(void*)) ? 1 : -1]; size_t ZSTD_rust_assignLocalDict( diff --git a/rust/src/zstd_compress_dictionary.rs b/rust/src/zstd_compress_dictionary.rs index 5e9dd3105..0671d834a 100644 --- a/rust/src/zstd_compress_dictionary.rs +++ b/rust/src/zstd_compress_dictionary.rs @@ -549,12 +549,15 @@ pub struct ZSTD_rust_assignLocalDictState { } const _: () = { + assert!(size_of::() == size_of::()); assert!(offset_of!(ZSTD_rust_assignLocalDictState, callback_context) == 0); assert!(offset_of!(ZSTD_rust_assignLocalDictState, static_size) == size_of::()); assert!(offset_of!(ZSTD_rust_assignLocalDictState, dict) == 2 * size_of::()); assert!(offset_of!(ZSTD_rust_assignLocalDictState, dict_buffer) == 3 * size_of::()); assert!(offset_of!(ZSTD_rust_assignLocalDictState, dict_size) == 4 * size_of::()); - assert!(offset_of!(ZSTD_rust_assignLocalDictState, dict_content_type) == 5 * size_of::()); + assert!( + offset_of!(ZSTD_rust_assignLocalDictState, dict_content_type) == 5 * size_of::() + ); assert!(offset_of!(ZSTD_rust_assignLocalDictState, allocate) == 6 * size_of::()); assert!(size_of::() == size_of::<[usize; 7]>()); }; @@ -597,11 +600,7 @@ pub unsafe extern "C" fn ZSTD_rust_assignLocalDict( } if dict_size != 0 { unsafe { - ptr::copy_nonoverlapping( - dict.cast::(), - dict_buffer.cast::(), - dict_size, - ); + ptr::copy_nonoverlapping(dict.cast::(), dict_buffer.cast::(), dict_size); } } unsafe { @@ -2833,6 +2832,264 @@ mod tests { use crate::mem::MEM_writeLE32; use std::mem::{size_of, MaybeUninit}; + struct LocalDictAssignmentProbe { + allocate_calls: usize, + requested_size: usize, + fail_allocation: bool, + allocation: *mut u8, + allocation_capacity: usize, + dict_slot: *mut *const c_void, + dict_buffer_slot: *mut *mut c_void, + dict_size_slot: *mut usize, + dict_content_type_slot: *mut c_int, + observed_dict: *const c_void, + observed_dict_buffer: *mut c_void, + observed_dict_size: usize, + observed_dict_content_type: c_int, + } + + impl LocalDictAssignmentProbe { + fn new(fail_allocation: bool) -> Self { + Self { + allocate_calls: 0, + requested_size: 0, + fail_allocation, + allocation: ptr::null_mut(), + allocation_capacity: 0, + dict_slot: ptr::null_mut(), + dict_buffer_slot: ptr::null_mut(), + dict_size_slot: ptr::null_mut(), + dict_content_type_slot: ptr::null_mut(), + observed_dict: ptr::null(), + observed_dict_buffer: ptr::null_mut(), + observed_dict_size: 0, + observed_dict_content_type: 0, + } + } + } + + unsafe extern "C" fn assign_local_dict_allocate_test_buffer( + context: *mut c_void, + size: usize, + ) -> *mut c_void { + let probe = unsafe { &mut *context.cast::() }; + probe.allocate_calls += 1; + probe.requested_size = size; + probe.observed_dict = unsafe { *probe.dict_slot }; + probe.observed_dict_buffer = unsafe { *probe.dict_buffer_slot }; + probe.observed_dict_size = unsafe { *probe.dict_size_slot }; + probe.observed_dict_content_type = unsafe { *probe.dict_content_type_slot }; + if probe.fail_allocation { + return ptr::null_mut(); + } + + let mut allocation = Vec::::with_capacity(size); + allocation.resize(size, 0); + let buffer = allocation.as_mut_ptr(); + probe.allocation = buffer; + probe.allocation_capacity = allocation.capacity(); + std::mem::forget(allocation); + buffer.cast() + } + + fn assign_local_dict_test_state( + probe: &mut LocalDictAssignmentProbe, + static_size: usize, + dict: &mut *const c_void, + dict_buffer: &mut *mut c_void, + dict_size: &mut usize, + dict_content_type: &mut c_int, + ) -> ZSTD_rust_assignLocalDictState { + probe.dict_slot = dict; + probe.dict_buffer_slot = dict_buffer; + probe.dict_size_slot = dict_size; + probe.dict_content_type_slot = dict_content_type; + ZSTD_rust_assignLocalDictState { + callback_context: (probe as *mut LocalDictAssignmentProbe).cast(), + static_size, + dict: dict as *mut *const c_void, + dict_buffer: dict_buffer as *mut *mut c_void, + dict_size: dict_size as *mut usize, + dict_content_type: dict_content_type as *mut c_int, + allocate: assign_local_dict_allocate_test_buffer, + } + } + + #[test] + fn assign_local_dict_by_reference_publishes_without_allocation() { + let dictionary = [1u8, 2, 3, 4]; + let sentinel_dict = 0x1010usize as *const c_void; + let sentinel_buffer = 0x2020usize as *mut c_void; + let mut local_dict = sentinel_dict; + let mut local_dict_buffer = sentinel_buffer; + let mut local_dict_size = usize::MAX; + let mut local_dict_content_type = -1; + let mut probe = LocalDictAssignmentProbe::new(false); + let state = assign_local_dict_test_state( + &mut probe, + 0, + &mut local_dict, + &mut local_dict_buffer, + &mut local_dict_size, + &mut local_dict_content_type, + ); + + let result = unsafe { + ZSTD_rust_assignLocalDict( + &state, + dictionary.as_ptr().cast(), + dictionary.len(), + ZSTD_DLM_BY_REF, + ZSTD_DCT_RAW_CONTENT, + ) + }; + + assert_eq!(result, 0); + assert_eq!(probe.allocate_calls, 0); + assert_eq!(local_dict, dictionary.as_ptr().cast()); + assert_eq!(local_dict_buffer, sentinel_buffer); + assert_eq!(local_dict_size, dictionary.len()); + assert_eq!(local_dict_content_type, ZSTD_DCT_RAW_CONTENT); + } + + #[test] + fn assign_local_dict_by_copy_copies_before_publishing_outputs() { + let dictionary = [5u8, 6, 7, 8, 9]; + let sentinel_dict = 0x3030usize as *const c_void; + let sentinel_buffer = 0x4040usize as *mut c_void; + let sentinel_size = 0x5050usize; + let sentinel_content_type = -2; + let mut local_dict = sentinel_dict; + let mut local_dict_buffer = sentinel_buffer; + let mut local_dict_size = sentinel_size; + let mut local_dict_content_type = sentinel_content_type; + let mut probe = LocalDictAssignmentProbe::new(false); + let state = assign_local_dict_test_state( + &mut probe, + 0, + &mut local_dict, + &mut local_dict_buffer, + &mut local_dict_size, + &mut local_dict_content_type, + ); + + let result = unsafe { + ZSTD_rust_assignLocalDict( + &state, + dictionary.as_ptr().cast(), + dictionary.len(), + ZSTD_DLM_BY_COPY, + ZSTD_DCT_RAW_CONTENT, + ) + }; + + assert_eq!(result, 0); + assert_eq!(probe.allocate_calls, 1); + assert_eq!(probe.requested_size, dictionary.len()); + assert_eq!(probe.observed_dict, sentinel_dict); + assert_eq!(probe.observed_dict_buffer, sentinel_buffer); + assert_eq!(probe.observed_dict_size, sentinel_size); + assert_eq!(probe.observed_dict_content_type, sentinel_content_type); + assert_eq!(local_dict, local_dict_buffer.cast_const()); + assert_eq!(local_dict_buffer, probe.allocation.cast()); + assert_eq!(local_dict_size, dictionary.len()); + assert_eq!(local_dict_content_type, ZSTD_DCT_RAW_CONTENT); + let copied = + unsafe { std::slice::from_raw_parts(local_dict as *const u8, dictionary.len()) }; + assert_eq!(copied, dictionary); + + unsafe { + drop(Vec::from_raw_parts( + probe.allocation, + dictionary.len(), + probe.allocation_capacity, + )); + } + } + + #[test] + fn assign_local_dict_rejects_static_copy_before_allocation() { + let dictionary = [10u8, 11, 12]; + let sentinel_dict = 0x6060usize as *const c_void; + let sentinel_buffer = 0x7070usize as *mut c_void; + let sentinel_size = 0x8080usize; + let sentinel_content_type = -3; + let mut local_dict = sentinel_dict; + let mut local_dict_buffer = sentinel_buffer; + let mut local_dict_size = sentinel_size; + let mut local_dict_content_type = sentinel_content_type; + let mut probe = LocalDictAssignmentProbe::new(false); + let state = assign_local_dict_test_state( + &mut probe, + 1, + &mut local_dict, + &mut local_dict_buffer, + &mut local_dict_size, + &mut local_dict_content_type, + ); + + let result = unsafe { + ZSTD_rust_assignLocalDict( + &state, + dictionary.as_ptr().cast(), + dictionary.len(), + ZSTD_DLM_BY_COPY, + ZSTD_DCT_RAW_CONTENT, + ) + }; + + assert_eq!(result, ERROR(ZstdErrorCode::MemoryAllocation)); + assert_eq!(probe.allocate_calls, 0); + assert_eq!(local_dict, sentinel_dict); + assert_eq!(local_dict_buffer, sentinel_buffer); + assert_eq!(local_dict_size, sentinel_size); + assert_eq!(local_dict_content_type, sentinel_content_type); + } + + #[test] + fn assign_local_dict_preserves_outputs_on_allocation_failure() { + let dictionary = [13u8, 14, 15]; + let sentinel_dict = 0x9090usize as *const c_void; + let sentinel_buffer = 0xa0a0usize as *mut c_void; + let sentinel_size = 0xb0b0usize; + let sentinel_content_type = -4; + let mut local_dict = sentinel_dict; + let mut local_dict_buffer = sentinel_buffer; + let mut local_dict_size = sentinel_size; + let mut local_dict_content_type = sentinel_content_type; + let mut probe = LocalDictAssignmentProbe::new(true); + let state = assign_local_dict_test_state( + &mut probe, + 0, + &mut local_dict, + &mut local_dict_buffer, + &mut local_dict_size, + &mut local_dict_content_type, + ); + + let result = unsafe { + ZSTD_rust_assignLocalDict( + &state, + dictionary.as_ptr().cast(), + dictionary.len(), + ZSTD_DLM_BY_COPY, + ZSTD_DCT_RAW_CONTENT, + ) + }; + + assert_eq!(result, ERROR(ZstdErrorCode::MemoryAllocation)); + assert_eq!(probe.allocate_calls, 1); + assert_eq!(probe.requested_size, dictionary.len()); + assert_eq!(probe.observed_dict, sentinel_dict); + assert_eq!(probe.observed_dict_buffer, sentinel_buffer); + assert_eq!(probe.observed_dict_size, sentinel_size); + assert_eq!(probe.observed_dict_content_type, sentinel_content_type); + assert_eq!(local_dict, sentinel_dict); + assert_eq!(local_dict_buffer, sentinel_buffer); + assert_eq!(local_dict_size, sentinel_size); + assert_eq!(local_dict_content_type, sentinel_content_type); + } + #[derive(Default)] struct EstimateCDictSizeProbe { events: Vec<&'static str>,