feat(compress): move CDict constructor policy into Rust
Move regular and by-reference CDict parameter selection and default-level normalization behind a Rust-owned boundary. Keep advanced allocation, private workspace setup, and CDict scalar publication in C callbacks. Test Plan: - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --release create_cdict -- --nocapture - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --release - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --release --all-targets -- -D warnings - ulimit -v 41943040; make -B -C programs -j1 zstd - ulimit -v 41943040; make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s
This commit is contained in:
@@ -344,6 +344,30 @@ typedef char ZSTD_rust_create_cctx_state_layout[
|
|||||||
== 4 * sizeof(void*)
|
== 4 * sizeof(void*)
|
||||||
&& sizeof(ZSTD_rust_createCCtxState) == 5 * sizeof(void*))
|
&& sizeof(ZSTD_rust_createCCtxState) == 5 * sizeof(void*))
|
||||||
? 1 : -1];
|
? 1 : -1];
|
||||||
|
typedef void* (*ZSTD_rust_createCDictCreate_f)(
|
||||||
|
void* context, const void* dict, size_t dictSize,
|
||||||
|
int dictLoadMethod, int dictContentType,
|
||||||
|
const ZSTD_compressionParameters* cParams);
|
||||||
|
typedef void (*ZSTD_rust_createCDictSetCompressionLevel_f)(
|
||||||
|
void* context, void* cdict, int compressionLevel);
|
||||||
|
typedef struct {
|
||||||
|
void* callbackContext;
|
||||||
|
const U32* exclusionMask;
|
||||||
|
ZSTD_rust_createCDictCreate_f create;
|
||||||
|
ZSTD_rust_createCDictSetCompressionLevel_f setCompressionLevel;
|
||||||
|
} ZSTD_rust_createCDictState;
|
||||||
|
void* ZSTD_rust_createCDict(
|
||||||
|
const ZSTD_rust_createCDictState* state,
|
||||||
|
const void* dict, size_t dictSize,
|
||||||
|
int compressionLevel, int dictLoadMethod);
|
||||||
|
typedef char ZSTD_rust_create_cdict_state_layout[
|
||||||
|
(offsetof(ZSTD_rust_createCDictState, callbackContext) == 0
|
||||||
|
&& offsetof(ZSTD_rust_createCDictState, exclusionMask) == sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_createCDictState, create) == 2 * sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_createCDictState, setCompressionLevel)
|
||||||
|
== 3 * sizeof(void*)
|
||||||
|
&& sizeof(ZSTD_rust_createCDictState) == 4 * sizeof(void*))
|
||||||
|
? 1 : -1];
|
||||||
typedef void (*ZSTD_rust_resetCCtxClearAllDicts_f)(void* context);
|
typedef void (*ZSTD_rust_resetCCtxClearAllDicts_f)(void* context);
|
||||||
typedef size_t (*ZSTD_rust_resetCCtxResetParams_f)(void* context);
|
typedef size_t (*ZSTD_rust_resetCCtxResetParams_f)(void* context);
|
||||||
typedef struct {
|
typedef struct {
|
||||||
@@ -5479,26 +5503,48 @@ ZSTD_CDict* ZSTD_createCDict_advanced2(
|
|||||||
return cdict;
|
return cdict;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void* ZSTD_rust_createCDict_create(
|
||||||
|
void* context, const void* dict, size_t dictSize,
|
||||||
|
int dictLoadMethod, int dictContentType,
|
||||||
|
const ZSTD_compressionParameters* cParams)
|
||||||
|
{
|
||||||
|
(void)context;
|
||||||
|
return ZSTD_createCDict_advanced(
|
||||||
|
dict, dictSize,
|
||||||
|
(ZSTD_dictLoadMethod_e)dictLoadMethod,
|
||||||
|
(ZSTD_dictContentType_e)dictContentType,
|
||||||
|
*cParams, ZSTD_defaultCMem);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void ZSTD_rust_createCDict_setCompressionLevel(
|
||||||
|
void* context, void* cdict, int compressionLevel)
|
||||||
|
{
|
||||||
|
(void)context;
|
||||||
|
((ZSTD_CDict*)cdict)->compressionLevel = compressionLevel;
|
||||||
|
}
|
||||||
|
|
||||||
ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel)
|
ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel)
|
||||||
{
|
{
|
||||||
ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
|
ZSTD_rust_createCDictState state;
|
||||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize,
|
U32 const exclusionMask = ZSTD_getCParamsExclusionMask();
|
||||||
ZSTD_dlm_byCopy, ZSTD_dct_auto,
|
state.callbackContext = NULL;
|
||||||
cParams, ZSTD_defaultCMem);
|
state.exclusionMask = &exclusionMask;
|
||||||
if (cdict)
|
state.create = ZSTD_rust_createCDict_create;
|
||||||
cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel;
|
state.setCompressionLevel = ZSTD_rust_createCDict_setCompressionLevel;
|
||||||
return cdict;
|
return (ZSTD_CDict*)ZSTD_rust_createCDict(
|
||||||
|
&state, dict, dictSize, compressionLevel, ZSTD_dlm_byCopy);
|
||||||
}
|
}
|
||||||
|
|
||||||
ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel)
|
ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel)
|
||||||
{
|
{
|
||||||
ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
|
ZSTD_rust_createCDictState state;
|
||||||
ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize,
|
U32 const exclusionMask = ZSTD_getCParamsExclusionMask();
|
||||||
ZSTD_dlm_byRef, ZSTD_dct_auto,
|
state.callbackContext = NULL;
|
||||||
cParams, ZSTD_defaultCMem);
|
state.exclusionMask = &exclusionMask;
|
||||||
if (cdict)
|
state.create = ZSTD_rust_createCDict_create;
|
||||||
cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel;
|
state.setCompressionLevel = ZSTD_rust_createCDict_setCompressionLevel;
|
||||||
return cdict;
|
return (ZSTD_CDict*)ZSTD_rust_createCDict(
|
||||||
|
&state, dict, dictSize, compressionLevel, ZSTD_dlm_byRef);
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
|
size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ use crate::zstd_compress_params::{
|
|||||||
ZSTD_compressionParameters, ZSTD_frameParameters, ZSTD_parameters,
|
ZSTD_compressionParameters, ZSTD_frameParameters, ZSTD_parameters,
|
||||||
ZSTD_rust_params_defaultCLevel, ZSTD_rust_params_getCParams,
|
ZSTD_rust_params_defaultCLevel, ZSTD_rust_params_getCParams,
|
||||||
ZSTD_rust_params_getParamsInternal, ZSTD_rust_params_shouldAttachDict,
|
ZSTD_rust_params_getParamsInternal, ZSTD_rust_params_shouldAttachDict,
|
||||||
ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_RUST_CPM_NO_ATTACH_DICT,
|
ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_RUST_CPM_CREATE_CDICT, ZSTD_RUST_CPM_NO_ATTACH_DICT,
|
||||||
};
|
};
|
||||||
use crate::zstd_compress_params_api::ZSTD_CCtx_params;
|
use crate::zstd_compress_params_api::ZSTD_CCtx_params;
|
||||||
use crate::zstd_compress_stats::{
|
use crate::zstd_compress_stats::{
|
||||||
@@ -358,6 +358,85 @@ pub unsafe extern "C" fn ZSTD_rust_compressBeginUsingCDictPublic(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type CreateCDictFn = unsafe extern "C" fn(
|
||||||
|
*mut c_void,
|
||||||
|
*const c_void,
|
||||||
|
usize,
|
||||||
|
c_int,
|
||||||
|
c_int,
|
||||||
|
*const ZSTD_compressionParameters,
|
||||||
|
) -> *mut c_void;
|
||||||
|
type CreateCDictSetCompressionLevelFn = unsafe extern "C" fn(*mut c_void, *mut c_void, c_int);
|
||||||
|
|
||||||
|
/// Explicit projection for the public `ZSTD_createCDict` constructors.
|
||||||
|
///
|
||||||
|
/// Rust owns create-CDict parameter selection and default-level normalization.
|
||||||
|
/// C retains advanced allocation, private workspace setup, and CDict scalar
|
||||||
|
/// publication behind callbacks.
|
||||||
|
#[repr(C)]
|
||||||
|
pub struct ZSTD_rust_createCDictState {
|
||||||
|
callback_context: *mut c_void,
|
||||||
|
exclusion_mask: *const c_uint,
|
||||||
|
create: CreateCDictFn,
|
||||||
|
set_compression_level: CreateCDictSetCompressionLevelFn,
|
||||||
|
}
|
||||||
|
|
||||||
|
const _: () = {
|
||||||
|
assert!(offset_of!(ZSTD_rust_createCDictState, callback_context) == 0);
|
||||||
|
assert!(offset_of!(ZSTD_rust_createCDictState, exclusion_mask) == size_of::<usize>());
|
||||||
|
assert!(offset_of!(ZSTD_rust_createCDictState, create) == 2 * size_of::<usize>());
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_createCDictState, set_compression_level) == 3 * size_of::<usize>()
|
||||||
|
);
|
||||||
|
assert!(size_of::<ZSTD_rust_createCDictState>() == size_of::<[usize; 4]>());
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Select parameters and construct a regular or by-reference CDict.
|
||||||
|
#[no_mangle]
|
||||||
|
pub unsafe extern "C" fn ZSTD_rust_createCDict(
|
||||||
|
state: *const ZSTD_rust_createCDictState,
|
||||||
|
dict: *const c_void,
|
||||||
|
dict_size: usize,
|
||||||
|
compression_level: c_int,
|
||||||
|
dict_load_method: c_int,
|
||||||
|
) -> *mut c_void {
|
||||||
|
if state.is_null() {
|
||||||
|
return ptr::null_mut();
|
||||||
|
}
|
||||||
|
let state = unsafe { &*state };
|
||||||
|
if state.exclusion_mask.is_null() {
|
||||||
|
return ptr::null_mut();
|
||||||
|
}
|
||||||
|
let cparams = ZSTD_rust_params_getParamsInternal(
|
||||||
|
compression_level,
|
||||||
|
ZSTD_CONTENTSIZE_UNKNOWN,
|
||||||
|
dict_size,
|
||||||
|
ZSTD_RUST_CPM_CREATE_CDICT,
|
||||||
|
unsafe { *state.exclusion_mask },
|
||||||
|
)
|
||||||
|
.cParams;
|
||||||
|
let cdict = unsafe {
|
||||||
|
(state.create)(
|
||||||
|
state.callback_context,
|
||||||
|
dict,
|
||||||
|
dict_size,
|
||||||
|
dict_load_method,
|
||||||
|
ZSTD_DCT_AUTO,
|
||||||
|
&cparams,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
if cdict.is_null() {
|
||||||
|
return ptr::null_mut();
|
||||||
|
}
|
||||||
|
let normalized_level = if compression_level == 0 {
|
||||||
|
ZSTD_rust_params_defaultCLevel()
|
||||||
|
} else {
|
||||||
|
compression_level
|
||||||
|
};
|
||||||
|
unsafe { (state.set_compression_level)(state.callback_context, cdict, normalized_level) };
|
||||||
|
cdict
|
||||||
|
}
|
||||||
|
|
||||||
type CompressBeginUsingDictBeginFn =
|
type CompressBeginUsingDictBeginFn =
|
||||||
unsafe extern "C" fn(*mut c_void, *const c_void, usize, *const c_void, u64) -> usize;
|
unsafe extern "C" fn(*mut c_void, *const c_void, usize, *const c_void, u64) -> usize;
|
||||||
|
|
||||||
@@ -2655,6 +2734,106 @@ mod tests {
|
|||||||
assert!(probe.events.is_empty());
|
assert!(probe.events.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct CreateCDictProbe {
|
||||||
|
events: Vec<&'static str>,
|
||||||
|
dict: *const c_void,
|
||||||
|
dict_size: usize,
|
||||||
|
dict_load_method: c_int,
|
||||||
|
dict_content_type: c_int,
|
||||||
|
cparams: ZSTD_compressionParameters,
|
||||||
|
cdict_result: *mut c_void,
|
||||||
|
published_cdict: *mut c_void,
|
||||||
|
compression_level: c_int,
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn create_cdict_test_create(
|
||||||
|
context: *mut c_void,
|
||||||
|
dict: *const c_void,
|
||||||
|
dict_size: usize,
|
||||||
|
dict_load_method: c_int,
|
||||||
|
dict_content_type: c_int,
|
||||||
|
cparams: *const ZSTD_compressionParameters,
|
||||||
|
) -> *mut c_void {
|
||||||
|
let probe = unsafe { &mut *context.cast::<CreateCDictProbe>() };
|
||||||
|
probe.events.push("create");
|
||||||
|
probe.dict = dict;
|
||||||
|
probe.dict_size = dict_size;
|
||||||
|
probe.dict_load_method = dict_load_method;
|
||||||
|
probe.dict_content_type = dict_content_type;
|
||||||
|
probe.cparams = unsafe { *cparams };
|
||||||
|
probe.cdict_result
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn create_cdict_test_set_compression_level(
|
||||||
|
context: *mut c_void,
|
||||||
|
cdict: *mut c_void,
|
||||||
|
compression_level: c_int,
|
||||||
|
) {
|
||||||
|
let probe = unsafe { &mut *context.cast::<CreateCDictProbe>() };
|
||||||
|
probe.events.push("set");
|
||||||
|
probe.published_cdict = cdict;
|
||||||
|
probe.compression_level = compression_level;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_cdict_test_state(
|
||||||
|
probe: &mut CreateCDictProbe,
|
||||||
|
exclusion_mask: &c_uint,
|
||||||
|
) -> ZSTD_rust_createCDictState {
|
||||||
|
ZSTD_rust_createCDictState {
|
||||||
|
callback_context: (probe as *mut CreateCDictProbe).cast(),
|
||||||
|
exclusion_mask,
|
||||||
|
create: create_cdict_test_create,
|
||||||
|
set_compression_level: create_cdict_test_set_compression_level,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn create_cdict_selects_params_and_normalizes_default_level() {
|
||||||
|
let mut probe = CreateCDictProbe {
|
||||||
|
cdict_result: ptr::dangling_mut(),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let dict = [1u8, 2, 3, 4];
|
||||||
|
let exclusion_mask = 0;
|
||||||
|
let state = create_cdict_test_state(&mut probe, &exclusion_mask);
|
||||||
|
|
||||||
|
let result = unsafe {
|
||||||
|
ZSTD_rust_createCDict(&state, dict.as_ptr().cast(), dict.len(), 0, ZSTD_DLM_BY_REF)
|
||||||
|
};
|
||||||
|
|
||||||
|
let expected_cparams = ZSTD_rust_params_getParamsInternal(
|
||||||
|
0,
|
||||||
|
ZSTD_CONTENTSIZE_UNKNOWN,
|
||||||
|
dict.len(),
|
||||||
|
ZSTD_RUST_CPM_CREATE_CDICT,
|
||||||
|
exclusion_mask,
|
||||||
|
)
|
||||||
|
.cParams;
|
||||||
|
assert_eq!(result, probe.cdict_result);
|
||||||
|
assert_eq!(probe.events, ["create", "set"]);
|
||||||
|
assert_eq!(probe.dict, dict.as_ptr().cast());
|
||||||
|
assert_eq!(probe.dict_size, dict.len());
|
||||||
|
assert_eq!(probe.dict_load_method, ZSTD_DLM_BY_REF);
|
||||||
|
assert_eq!(probe.dict_content_type, ZSTD_DCT_AUTO);
|
||||||
|
assert_eq!(probe.cparams, expected_cparams);
|
||||||
|
assert_eq!(probe.published_cdict, probe.cdict_result);
|
||||||
|
assert_eq!(probe.compression_level, ZSTD_rust_params_defaultCLevel());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn create_cdict_stops_before_publication_after_creation_failure() {
|
||||||
|
let mut probe = CreateCDictProbe::default();
|
||||||
|
let exclusion_mask = 0;
|
||||||
|
let state = create_cdict_test_state(&mut probe, &exclusion_mask);
|
||||||
|
|
||||||
|
let result = unsafe { ZSTD_rust_createCDict(&state, ptr::null(), 0, 3, ZSTD_DLM_BY_REF) };
|
||||||
|
|
||||||
|
assert!(result.is_null());
|
||||||
|
assert_eq!(probe.events, ["create"]);
|
||||||
|
assert!(probe.published_cdict.is_null());
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
struct CompressBeginUsingDictProbe {
|
struct CompressBeginUsingDictProbe {
|
||||||
events: Vec<&'static str>,
|
events: Vec<&'static str>,
|
||||||
|
|||||||
Reference in New Issue
Block a user