From 556cdc53bf9c7a80e3d55c15b6f0aa2427d80aa1 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Sun, 19 Jul 2026 15:42:07 +0200 Subject: [PATCH] feat(compress): move usingDict policy into Rust Move ZSTD_compress_usingDict parameter selection, dictionary-presence handling, and default compression-level normalization behind a Rust-owned orchestration boundary. Keep C responsible for the private ZSTD_CCtx_params initialization and advanced compression operation through explicit callbacks, including the build-specific excluded-strategy mask. Test Plan: - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --release - ulimit -v 41943040; make -B -C programs -j1 zstd - ulimit -v 41943040; make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s --- lib/compress/zstd_compress.c | 47 ++++++-- rust/src/zstd_compress.rs | 223 ++++++++++++++++++++++++++++++++++- 2 files changed, 260 insertions(+), 10 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f38918b07..b99ef37a6 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -399,6 +399,29 @@ typedef char ZSTD_rust_compress_advanced_state_layout[ == 2 * sizeof(void*) && sizeof(ZSTD_rust_compressAdvancedState) == 3 * sizeof(void*)) ? 1 : -1]; +typedef void (*ZSTD_rust_compressUsingDictInitParams_f)( + void* context, const ZSTD_parameters* params, int compressionLevel); +typedef struct { + void* callbackContext; + const U32* exclusionMask; + ZSTD_rust_compressUsingDictInitParams_f initParams; + ZSTD_rust_compressAdvancedInternal_f compressInternal; +} ZSTD_rust_compressUsingDictState; +size_t ZSTD_rust_compressUsingDict( + const ZSTD_rust_compressUsingDictState* state, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, int compressionLevel); +typedef char ZSTD_rust_compress_using_dict_state_layout[ + (offsetof(ZSTD_rust_compressUsingDictState, callbackContext) == 0 + && offsetof(ZSTD_rust_compressUsingDictState, exclusionMask) + == sizeof(void*) + && offsetof(ZSTD_rust_compressUsingDictState, initParams) + == 2 * sizeof(void*) + && offsetof(ZSTD_rust_compressUsingDictState, compressInternal) + == 3 * sizeof(void*) + && sizeof(ZSTD_rust_compressUsingDictState) == 4 * sizeof(void*)) + ? 1 : -1]; ZSTD_frameProgression ZSTD_rust_frameProgression(U64 consumedSrcSize, size_t buffered, U64 producedCSize); @@ -4949,19 +4972,29 @@ static size_t ZSTD_rust_compressAdvanced_compressInternal( &cctx->simpleApiParams); } +static void ZSTD_rust_compressUsingDict_initParams( + void* context, const ZSTD_parameters* params, int compressionLevel) +{ + ZSTD_CCtxParams_init_internal( + &((ZSTD_CCtx*)context)->simpleApiParams, params, compressionLevel); +} + size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, - const void* dict, size_t dictSize, + const void* dict, size_t dictSize, int compressionLevel) { - { - ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict); - assert(params.fParams.contentSizeFlag == 1); - ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel); - } + ZSTD_rust_compressUsingDictState state; + U32 const exclusionMask = ZSTD_getCParamsExclusionMask(); DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize); - return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams); + state.callbackContext = cctx; + state.exclusionMask = &exclusionMask; + state.initParams = ZSTD_rust_compressUsingDict_initParams; + state.compressInternal = ZSTD_rust_compressAdvanced_compressInternal; + return ZSTD_rust_compressUsingDict( + &state, dst, dstCapacity, src, srcSize, + dict, dictSize, compressionLevel); } /* ZSTD_compressCCtx() is implemented by rust/src/zstd_compress.rs. */ diff --git a/rust/src/zstd_compress.rs b/rust/src/zstd_compress.rs index 6d277b33f..714ea6bb3 100644 --- a/rust/src/zstd_compress.rs +++ b/rust/src/zstd_compress.rs @@ -24,9 +24,9 @@ use crate::zstd_compress_frame::{ use crate::zstd_compress_literals::min_gain; use crate::zstd_compress_params::{ ZSTD_compressionParameters, ZSTD_frameParameters, ZSTD_parameters, - ZSTD_rust_params_adjustCParams, ZSTD_rust_params_checkCParams, ZSTD_rust_params_maxNbSeq, - ZSTD_rust_params_selectCParams, ZSTD_RUST_CPM_NO_ATTACH_DICT, ZSTD_RUST_PS_AUTO, - ZSTD_RUST_PS_DISABLE, + ZSTD_rust_params_adjustCParams, ZSTD_rust_params_checkCParams, ZSTD_rust_params_defaultCLevel, + ZSTD_rust_params_getParamsInternal, ZSTD_rust_params_maxNbSeq, ZSTD_rust_params_selectCParams, + ZSTD_RUST_CPM_NO_ATTACH_DICT, ZSTD_RUST_PS_AUTO, ZSTD_RUST_PS_DISABLE, }; use crate::zstd_compress_params_api::{ ZSTD_CCtxParams_setParameter, ZSTD_CCtx_params, ZSTD_rust_isUpdateAuthorized, @@ -1547,6 +1547,87 @@ pub unsafe extern "C" fn ZSTD_rust_compressAdvanced( } } +type CompressUsingDictInitParamsFn = + unsafe extern "C" fn(*mut c_void, *const ZSTD_parameters, c_int); + +/// Explicit projection for the public `ZSTD_compress_usingDict` wrapper. +/// +/// Rust owns compression-parameter selection, dictionary-presence handling, +/// and the default-level normalization. C retains the private +/// `ZSTD_CCtx_params` initialization and advanced compression operation. +#[repr(C)] +pub struct ZSTD_rust_compressUsingDictState { + callback_context: *mut c_void, + exclusion_mask: *const c_uint, + init_params: Option, + compress_internal: Option, +} + +const _: () = { + assert!(size_of::() == size_of::()); + assert!(size_of::() == size_of::()); + assert!(offset_of!(ZSTD_rust_compressUsingDictState, callback_context) == 0); + assert!(offset_of!(ZSTD_rust_compressUsingDictState, exclusion_mask) == size_of::()); + assert!(offset_of!(ZSTD_rust_compressUsingDictState, init_params) == 2 * size_of::()); + assert!( + offset_of!(ZSTD_rust_compressUsingDictState, compress_internal) == 3 * size_of::() + ); + assert!(size_of::() == size_of::<[usize; 4]>()); +}; + +/// Select parameters and invoke the C-owned dictionary compression operation. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_compressUsingDict( + state: *const ZSTD_rust_compressUsingDictState, + dst: *mut c_void, + dst_capacity: usize, + src: *const c_void, + src_size: usize, + dict: *const c_void, + dict_size: usize, + compression_level: c_int, +) -> usize { + if state.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + let state = unsafe { &*state }; + let Some(init_params) = state.init_params else { + return ERROR(ZstdErrorCode::Generic); + }; + let Some(compress_internal) = state.compress_internal else { + return ERROR(ZstdErrorCode::Generic); + }; + if state.callback_context.is_null() || state.exclusion_mask.is_null() { + return ERROR(ZstdErrorCode::Generic); + } + + let dict_size_for_params = if dict.is_null() { 0 } else { dict_size }; + let params = ZSTD_rust_params_getParamsInternal( + compression_level, + src_size as u64, + dict_size_for_params, + ZSTD_RUST_CPM_NO_ATTACH_DICT, + unsafe { *state.exclusion_mask }, + ); + let init_level = if compression_level == 0 { + ZSTD_rust_params_defaultCLevel() + } else { + compression_level + }; + unsafe { init_params(state.callback_context, ¶ms, init_level) }; + unsafe { + compress_internal( + state.callback_context, + dst, + dst_capacity, + src, + src_size, + dict, + dict_size, + ) + } +} + type ResetCStreamResetFn = unsafe extern "C" fn(*mut c_void) -> usize; type ResetCStreamSetPledgedSrcSizeFn = unsafe extern "C" fn(*mut c_void, u64) -> usize; @@ -10576,6 +10657,142 @@ mod tests { assert!(context.events.is_empty()); } + #[derive(Default)] + struct CompressUsingDictTestContext { + events: Vec<&'static str>, + init_params: ZSTD_parameters, + init_level: c_int, + dst: *mut c_void, + dst_capacity: usize, + src: *const c_void, + src_size: usize, + dict: *const c_void, + dict_size: usize, + result: usize, + } + + unsafe extern "C" fn compress_using_dict_test_init( + context: *mut c_void, + params: *const ZSTD_parameters, + compression_level: c_int, + ) { + let context = unsafe { &mut *context.cast::() }; + context.events.push("init"); + context.init_params = unsafe { *params }; + context.init_level = compression_level; + } + + unsafe extern "C" fn compress_using_dict_test_internal( + context: *mut c_void, + dst: *mut c_void, + dst_capacity: usize, + src: *const c_void, + src_size: usize, + dict: *const c_void, + dict_size: usize, + ) -> usize { + let context = unsafe { &mut *context.cast::() }; + context.events.push("compress"); + context.dst = dst; + context.dst_capacity = dst_capacity; + context.src = src; + context.src_size = src_size; + context.dict = dict; + context.dict_size = dict_size; + context.result + } + + fn compress_using_dict_test_state( + context: &mut CompressUsingDictTestContext, + exclusion_mask: &u32, + ) -> ZSTD_rust_compressUsingDictState { + ZSTD_rust_compressUsingDictState { + callback_context: (context as *mut CompressUsingDictTestContext).cast(), + exclusion_mask, + init_params: Some(compress_using_dict_test_init), + compress_internal: Some(compress_using_dict_test_internal), + } + } + + #[test] + fn compress_using_dict_selects_params_and_normalizes_default_level() { + let mut context = CompressUsingDictTestContext { + result: 23, + ..CompressUsingDictTestContext::default() + }; + let exclusion_mask = 0; + let state = compress_using_dict_test_state(&mut context, &exclusion_mask); + let mut dst = [0u8; 8]; + let src = [1u8, 2, 3]; + let dict = [4u8, 5]; + + let result = unsafe { + ZSTD_rust_compressUsingDict( + &state, + dst.as_mut_ptr().cast(), + dst.len(), + src.as_ptr().cast(), + src.len(), + dict.as_ptr().cast(), + dict.len(), + 0, + ) + }; + + assert_eq!(result, context.result); + assert_eq!(context.events, ["init", "compress"]); + assert_eq!( + context.init_params, + ZSTD_rust_params_getParamsInternal( + 0, + src.len() as u64, + dict.len(), + ZSTD_RUST_CPM_NO_ATTACH_DICT, + exclusion_mask, + ) + ); + assert_eq!(context.init_level, ZSTD_rust_params_defaultCLevel()); + assert_eq!(context.dst, dst.as_mut_ptr().cast()); + assert_eq!(context.dst_capacity, dst.len()); + assert_eq!(context.src, src.as_ptr().cast()); + assert_eq!(context.src_size, src.len()); + assert_eq!(context.dict, dict.as_ptr().cast()); + assert_eq!(context.dict_size, dict.len()); + } + + #[test] + fn compress_using_dict_ignores_size_without_a_dictionary_buffer_for_params() { + let mut context = CompressUsingDictTestContext::default(); + let exclusion_mask = 0; + let state = compress_using_dict_test_state(&mut context, &exclusion_mask); + let expected = ZSTD_rust_params_getParamsInternal( + 3, + 0, + 0, + ZSTD_RUST_CPM_NO_ATTACH_DICT, + exclusion_mask, + ); + + let result = unsafe { + ZSTD_rust_compressUsingDict( + &state, + ptr::null_mut(), + 0, + ptr::null(), + 0, + ptr::null(), + 17, + 3, + ) + }; + + assert_eq!(result, 0); + assert_eq!(context.events, ["init", "compress"]); + assert_eq!(context.init_params, expected); + assert_eq!(context.dict, ptr::null()); + assert_eq!(context.dict_size, 17); + } + #[derive(Default)] struct ResetCStreamTestContext { events: Vec<&'static str>,