feat(compress): move usingDict policy into Rust
Move ZSTD_compress_usingDict parameter selection, dictionary-presence handling, and default compression-level normalization behind a Rust-owned orchestration boundary. Keep C responsible for the private ZSTD_CCtx_params initialization and advanced compression operation through explicit callbacks, including the build-specific excluded-strategy mask. Test Plan: - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --release - ulimit -v 41943040; make -B -C programs -j1 zstd - ulimit -v 41943040; make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s
This commit is contained in:
@@ -399,6 +399,29 @@ typedef char ZSTD_rust_compress_advanced_state_layout[
|
||||
== 2 * sizeof(void*)
|
||||
&& sizeof(ZSTD_rust_compressAdvancedState) == 3 * sizeof(void*))
|
||||
? 1 : -1];
|
||||
typedef void (*ZSTD_rust_compressUsingDictInitParams_f)(
|
||||
void* context, const ZSTD_parameters* params, int compressionLevel);
|
||||
typedef struct {
|
||||
void* callbackContext;
|
||||
const U32* exclusionMask;
|
||||
ZSTD_rust_compressUsingDictInitParams_f initParams;
|
||||
ZSTD_rust_compressAdvancedInternal_f compressInternal;
|
||||
} ZSTD_rust_compressUsingDictState;
|
||||
size_t ZSTD_rust_compressUsingDict(
|
||||
const ZSTD_rust_compressUsingDictState* state,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict, size_t dictSize, int compressionLevel);
|
||||
typedef char ZSTD_rust_compress_using_dict_state_layout[
|
||||
(offsetof(ZSTD_rust_compressUsingDictState, callbackContext) == 0
|
||||
&& offsetof(ZSTD_rust_compressUsingDictState, exclusionMask)
|
||||
== sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_compressUsingDictState, initParams)
|
||||
== 2 * sizeof(void*)
|
||||
&& offsetof(ZSTD_rust_compressUsingDictState, compressInternal)
|
||||
== 3 * sizeof(void*)
|
||||
&& sizeof(ZSTD_rust_compressUsingDictState) == 4 * sizeof(void*))
|
||||
? 1 : -1];
|
||||
ZSTD_frameProgression ZSTD_rust_frameProgression(U64 consumedSrcSize,
|
||||
size_t buffered,
|
||||
U64 producedCSize);
|
||||
@@ -4949,19 +4972,29 @@ static size_t ZSTD_rust_compressAdvanced_compressInternal(
|
||||
&cctx->simpleApiParams);
|
||||
}
|
||||
|
||||
static void ZSTD_rust_compressUsingDict_initParams(
|
||||
void* context, const ZSTD_parameters* params, int compressionLevel)
|
||||
{
|
||||
ZSTD_CCtxParams_init_internal(
|
||||
&((ZSTD_CCtx*)context)->simpleApiParams, params, compressionLevel);
|
||||
}
|
||||
|
||||
size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
const void* src, size_t srcSize,
|
||||
const void* dict, size_t dictSize,
|
||||
const void* dict, size_t dictSize,
|
||||
int compressionLevel)
|
||||
{
|
||||
{
|
||||
ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict);
|
||||
assert(params.fParams.contentSizeFlag == 1);
|
||||
ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel);
|
||||
}
|
||||
ZSTD_rust_compressUsingDictState state;
|
||||
U32 const exclusionMask = ZSTD_getCParamsExclusionMask();
|
||||
DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize);
|
||||
return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams);
|
||||
state.callbackContext = cctx;
|
||||
state.exclusionMask = &exclusionMask;
|
||||
state.initParams = ZSTD_rust_compressUsingDict_initParams;
|
||||
state.compressInternal = ZSTD_rust_compressAdvanced_compressInternal;
|
||||
return ZSTD_rust_compressUsingDict(
|
||||
&state, dst, dstCapacity, src, srcSize,
|
||||
dict, dictSize, compressionLevel);
|
||||
}
|
||||
|
||||
/* ZSTD_compressCCtx() is implemented by rust/src/zstd_compress.rs. */
|
||||
|
||||
+220
-3
@@ -24,9 +24,9 @@ use crate::zstd_compress_frame::{
|
||||
use crate::zstd_compress_literals::min_gain;
|
||||
use crate::zstd_compress_params::{
|
||||
ZSTD_compressionParameters, ZSTD_frameParameters, ZSTD_parameters,
|
||||
ZSTD_rust_params_adjustCParams, ZSTD_rust_params_checkCParams, ZSTD_rust_params_maxNbSeq,
|
||||
ZSTD_rust_params_selectCParams, ZSTD_RUST_CPM_NO_ATTACH_DICT, ZSTD_RUST_PS_AUTO,
|
||||
ZSTD_RUST_PS_DISABLE,
|
||||
ZSTD_rust_params_adjustCParams, ZSTD_rust_params_checkCParams, ZSTD_rust_params_defaultCLevel,
|
||||
ZSTD_rust_params_getParamsInternal, ZSTD_rust_params_maxNbSeq, ZSTD_rust_params_selectCParams,
|
||||
ZSTD_RUST_CPM_NO_ATTACH_DICT, ZSTD_RUST_PS_AUTO, ZSTD_RUST_PS_DISABLE,
|
||||
};
|
||||
use crate::zstd_compress_params_api::{
|
||||
ZSTD_CCtxParams_setParameter, ZSTD_CCtx_params, ZSTD_rust_isUpdateAuthorized,
|
||||
@@ -1547,6 +1547,87 @@ pub unsafe extern "C" fn ZSTD_rust_compressAdvanced(
|
||||
}
|
||||
}
|
||||
|
||||
type CompressUsingDictInitParamsFn =
|
||||
unsafe extern "C" fn(*mut c_void, *const ZSTD_parameters, c_int);
|
||||
|
||||
/// Explicit projection for the public `ZSTD_compress_usingDict` wrapper.
|
||||
///
|
||||
/// Rust owns compression-parameter selection, dictionary-presence handling,
|
||||
/// and the default-level normalization. C retains the private
|
||||
/// `ZSTD_CCtx_params` initialization and advanced compression operation.
|
||||
#[repr(C)]
|
||||
pub struct ZSTD_rust_compressUsingDictState {
|
||||
callback_context: *mut c_void,
|
||||
exclusion_mask: *const c_uint,
|
||||
init_params: Option<CompressUsingDictInitParamsFn>,
|
||||
compress_internal: Option<CompressAdvancedInternalFn>,
|
||||
}
|
||||
|
||||
const _: () = {
|
||||
assert!(size_of::<CompressUsingDictInitParamsFn>() == size_of::<usize>());
|
||||
assert!(size_of::<CompressAdvancedInternalFn>() == size_of::<usize>());
|
||||
assert!(offset_of!(ZSTD_rust_compressUsingDictState, callback_context) == 0);
|
||||
assert!(offset_of!(ZSTD_rust_compressUsingDictState, exclusion_mask) == size_of::<usize>());
|
||||
assert!(offset_of!(ZSTD_rust_compressUsingDictState, init_params) == 2 * size_of::<usize>());
|
||||
assert!(
|
||||
offset_of!(ZSTD_rust_compressUsingDictState, compress_internal) == 3 * size_of::<usize>()
|
||||
);
|
||||
assert!(size_of::<ZSTD_rust_compressUsingDictState>() == size_of::<[usize; 4]>());
|
||||
};
|
||||
|
||||
/// Select parameters and invoke the C-owned dictionary compression operation.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_compressUsingDict(
|
||||
state: *const ZSTD_rust_compressUsingDictState,
|
||||
dst: *mut c_void,
|
||||
dst_capacity: usize,
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
dict: *const c_void,
|
||||
dict_size: usize,
|
||||
compression_level: c_int,
|
||||
) -> usize {
|
||||
if state.is_null() {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
}
|
||||
let state = unsafe { &*state };
|
||||
let Some(init_params) = state.init_params else {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
};
|
||||
let Some(compress_internal) = state.compress_internal else {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
};
|
||||
if state.callback_context.is_null() || state.exclusion_mask.is_null() {
|
||||
return ERROR(ZstdErrorCode::Generic);
|
||||
}
|
||||
|
||||
let dict_size_for_params = if dict.is_null() { 0 } else { dict_size };
|
||||
let params = ZSTD_rust_params_getParamsInternal(
|
||||
compression_level,
|
||||
src_size as u64,
|
||||
dict_size_for_params,
|
||||
ZSTD_RUST_CPM_NO_ATTACH_DICT,
|
||||
unsafe { *state.exclusion_mask },
|
||||
);
|
||||
let init_level = if compression_level == 0 {
|
||||
ZSTD_rust_params_defaultCLevel()
|
||||
} else {
|
||||
compression_level
|
||||
};
|
||||
unsafe { init_params(state.callback_context, ¶ms, init_level) };
|
||||
unsafe {
|
||||
compress_internal(
|
||||
state.callback_context,
|
||||
dst,
|
||||
dst_capacity,
|
||||
src,
|
||||
src_size,
|
||||
dict,
|
||||
dict_size,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
type ResetCStreamResetFn = unsafe extern "C" fn(*mut c_void) -> usize;
|
||||
type ResetCStreamSetPledgedSrcSizeFn = unsafe extern "C" fn(*mut c_void, u64) -> usize;
|
||||
|
||||
@@ -10576,6 +10657,142 @@ mod tests {
|
||||
assert!(context.events.is_empty());
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CompressUsingDictTestContext {
|
||||
events: Vec<&'static str>,
|
||||
init_params: ZSTD_parameters,
|
||||
init_level: c_int,
|
||||
dst: *mut c_void,
|
||||
dst_capacity: usize,
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
dict: *const c_void,
|
||||
dict_size: usize,
|
||||
result: usize,
|
||||
}
|
||||
|
||||
unsafe extern "C" fn compress_using_dict_test_init(
|
||||
context: *mut c_void,
|
||||
params: *const ZSTD_parameters,
|
||||
compression_level: c_int,
|
||||
) {
|
||||
let context = unsafe { &mut *context.cast::<CompressUsingDictTestContext>() };
|
||||
context.events.push("init");
|
||||
context.init_params = unsafe { *params };
|
||||
context.init_level = compression_level;
|
||||
}
|
||||
|
||||
unsafe extern "C" fn compress_using_dict_test_internal(
|
||||
context: *mut c_void,
|
||||
dst: *mut c_void,
|
||||
dst_capacity: usize,
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
dict: *const c_void,
|
||||
dict_size: usize,
|
||||
) -> usize {
|
||||
let context = unsafe { &mut *context.cast::<CompressUsingDictTestContext>() };
|
||||
context.events.push("compress");
|
||||
context.dst = dst;
|
||||
context.dst_capacity = dst_capacity;
|
||||
context.src = src;
|
||||
context.src_size = src_size;
|
||||
context.dict = dict;
|
||||
context.dict_size = dict_size;
|
||||
context.result
|
||||
}
|
||||
|
||||
fn compress_using_dict_test_state(
|
||||
context: &mut CompressUsingDictTestContext,
|
||||
exclusion_mask: &u32,
|
||||
) -> ZSTD_rust_compressUsingDictState {
|
||||
ZSTD_rust_compressUsingDictState {
|
||||
callback_context: (context as *mut CompressUsingDictTestContext).cast(),
|
||||
exclusion_mask,
|
||||
init_params: Some(compress_using_dict_test_init),
|
||||
compress_internal: Some(compress_using_dict_test_internal),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compress_using_dict_selects_params_and_normalizes_default_level() {
|
||||
let mut context = CompressUsingDictTestContext {
|
||||
result: 23,
|
||||
..CompressUsingDictTestContext::default()
|
||||
};
|
||||
let exclusion_mask = 0;
|
||||
let state = compress_using_dict_test_state(&mut context, &exclusion_mask);
|
||||
let mut dst = [0u8; 8];
|
||||
let src = [1u8, 2, 3];
|
||||
let dict = [4u8, 5];
|
||||
|
||||
let result = unsafe {
|
||||
ZSTD_rust_compressUsingDict(
|
||||
&state,
|
||||
dst.as_mut_ptr().cast(),
|
||||
dst.len(),
|
||||
src.as_ptr().cast(),
|
||||
src.len(),
|
||||
dict.as_ptr().cast(),
|
||||
dict.len(),
|
||||
0,
|
||||
)
|
||||
};
|
||||
|
||||
assert_eq!(result, context.result);
|
||||
assert_eq!(context.events, ["init", "compress"]);
|
||||
assert_eq!(
|
||||
context.init_params,
|
||||
ZSTD_rust_params_getParamsInternal(
|
||||
0,
|
||||
src.len() as u64,
|
||||
dict.len(),
|
||||
ZSTD_RUST_CPM_NO_ATTACH_DICT,
|
||||
exclusion_mask,
|
||||
)
|
||||
);
|
||||
assert_eq!(context.init_level, ZSTD_rust_params_defaultCLevel());
|
||||
assert_eq!(context.dst, dst.as_mut_ptr().cast());
|
||||
assert_eq!(context.dst_capacity, dst.len());
|
||||
assert_eq!(context.src, src.as_ptr().cast());
|
||||
assert_eq!(context.src_size, src.len());
|
||||
assert_eq!(context.dict, dict.as_ptr().cast());
|
||||
assert_eq!(context.dict_size, dict.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compress_using_dict_ignores_size_without_a_dictionary_buffer_for_params() {
|
||||
let mut context = CompressUsingDictTestContext::default();
|
||||
let exclusion_mask = 0;
|
||||
let state = compress_using_dict_test_state(&mut context, &exclusion_mask);
|
||||
let expected = ZSTD_rust_params_getParamsInternal(
|
||||
3,
|
||||
0,
|
||||
0,
|
||||
ZSTD_RUST_CPM_NO_ATTACH_DICT,
|
||||
exclusion_mask,
|
||||
);
|
||||
|
||||
let result = unsafe {
|
||||
ZSTD_rust_compressUsingDict(
|
||||
&state,
|
||||
ptr::null_mut(),
|
||||
0,
|
||||
ptr::null(),
|
||||
0,
|
||||
ptr::null(),
|
||||
17,
|
||||
3,
|
||||
)
|
||||
};
|
||||
|
||||
assert_eq!(result, 0);
|
||||
assert_eq!(context.events, ["init", "compress"]);
|
||||
assert_eq!(context.init_params, expected);
|
||||
assert_eq!(context.dict, ptr::null());
|
||||
assert_eq!(context.dict_size, 17);
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ResetCStreamTestContext {
|
||||
events: Vec<&'static str>,
|
||||
|
||||
Reference in New Issue
Block a user