ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move dictionary begin policy into Rust

Browse Source 
Move the compressBegin_usingDict family parameter selection and default-level
normalization into Rust, covering the public ZSTD_compressBegin entry point as
well. C retains private ZSTD_CCtx_params initialization and
ZSTD_compressBegin_internal through callbacks, with the unknown pledged-size
contract preserved.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --release compress_begin_using_dict -- --nocapture
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --release
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --release --all-targets -- -D warnings
- ulimit -v 41943040; make -B -C programs -j1 zstd
- ulimit -v 41943040; make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s
This commit is contained in:
ddidderr
2026-07-19 15:51:36 +02:00
parent da3d4ed232
commit 9c0b9bd250
2 changed files with 228 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/zstd_compress_dictionary.rs
+174 -2
View File
@@ -16,8 +16,10 @@ use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
use crate::fse_compress::FSE_buildCTable_wksp;
use crate::huf_compress::HUF_readCTable;
use crate::zstd_compress_params::{
ZSTD_compressionParameters, ZSTD_frameParameters, ZSTD_parameters, ZSTD_rust_params_getCParams,
ZSTD_rust_params_shouldAttachDict, ZSTD_CONTENTSIZE_UNKNOWN,
ZSTD_compressionParameters, ZSTD_frameParameters, ZSTD_parameters,
ZSTD_rust_params_defaultCLevel, ZSTD_rust_params_getCParams,
ZSTD_rust_params_getParamsInternal, ZSTD_rust_params_shouldAttachDict,
ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_RUST_CPM_NO_ATTACH_DICT,
};
use crate::zstd_compress_params_api::ZSTD_CCtx_params;
use crate::zstd_compress_stats::{
@@ -297,6 +299,76 @@ pub unsafe extern "C" fn ZSTD_rust_compressUsingCDict(
unsafe { (state.end)(state.callback_context, dst, dst_capacity, src, src_size) }
}
type CompressBeginUsingDictBeginFn =
unsafe extern "C" fn(*mut c_void, *const c_void, usize, *const c_void, u64) -> usize;
/// Explicit projection for the public `ZSTD_compressBegin_usingDict` family.
///
/// Rust owns unknown-source parameter selection and default-level
/// normalization. C retains private `ZSTD_CCtx_params` initialization and the
/// final `ZSTD_compressBegin_internal` call behind callbacks.
#[repr(C)]
pub struct ZSTD_rust_compressBeginUsingDictState {
cctx: *mut c_void,
cctx_params: *mut ZSTD_CCtx_params,
exclusion_mask: *const c_uint,
init_params: CompressBeginUsingCDictInitParamsFn,
begin: CompressBeginUsingDictBeginFn,
}
const _: () = {
assert!(offset_of!(ZSTD_rust_compressBeginUsingDictState, cctx) == 0);
assert!(offset_of!(ZSTD_rust_compressBeginUsingDictState, cctx_params) == size_of::<usize>());
assert!(
offset_of!(ZSTD_rust_compressBeginUsingDictState, exclusion_mask) == 2 * size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_compressBeginUsingDictState, init_params) == 3 * size_of::<usize>()
);
assert!(offset_of!(ZSTD_rust_compressBeginUsingDictState, begin) == 4 * size_of::<usize>());
assert!(size_of::<ZSTD_rust_compressBeginUsingDictState>() == size_of::<[usize; 5]>());
};
/// Select parameters and start a dictionary-backed frame.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBeginUsingDict(
state: *const ZSTD_rust_compressBeginUsingDictState,
dict: *const c_void,
dict_size: usize,
compression_level: c_int,
) -> usize {
if state.is_null() {
return ERROR(ZstdErrorCode::Generic);
}
let state = unsafe { &*state };
if state.cctx.is_null() || state.cctx_params.is_null() || state.exclusion_mask.is_null() {
return ERROR(ZstdErrorCode::Generic);
}
let params = ZSTD_rust_params_getParamsInternal(
compression_level,
ZSTD_CONTENTSIZE_UNKNOWN,
dict_size,
ZSTD_RUST_CPM_NO_ATTACH_DICT,
unsafe { *state.exclusion_mask },
);
let init_level = if compression_level == 0 {
ZSTD_rust_params_defaultCLevel()
} else {
compression_level
};
unsafe {
(state.init_params)(state.cctx_params.cast(), &params, init_level);
(state.begin)(
state.cctx,
dict,
dict_size,
state.cctx_params.cast_const().cast(),
ZSTD_CONTENTSIZE_UNKNOWN,
)
}
}
type InitCDictAssignContentFn =
unsafe extern "C" fn(*mut c_void, *const c_void, usize, c_int) -> usize;
type InitCDictReserveEntropyFn = unsafe extern "C" fn(*mut c_void) -> *mut c_void;
@@ -2482,6 +2554,106 @@ mod tests {
assert_eq!(probe.events, ["begin"]);
}
#[derive(Default)]
struct CompressBeginUsingDictProbe {
events: Vec<&'static str>,
params: ZSTD_parameters,
compression_level: c_int,
dict: *const c_void,
dict_size: usize,
cctx_params: *const c_void,
pledged_src_size: u64,
begin_result: usize,
}
unsafe extern "C" fn compress_begin_using_dict_test_init(
context: *mut c_void,
params: *const ZSTD_parameters,
compression_level: c_int,
) {
let probe = unsafe { &mut *context.cast::<CompressBeginUsingDictProbe>() };
probe.events.push("init");
probe.params = unsafe { *params };
probe.compression_level = compression_level;
}
unsafe extern "C" fn compress_begin_using_dict_test_begin(
context: *mut c_void,
dict: *const c_void,
dict_size: usize,
cctx_params: *const c_void,
pledged_src_size: u64,
) -> usize {
let probe = unsafe { &mut *context.cast::<CompressBeginUsingDictProbe>() };
probe.events.push("begin");
probe.dict = dict;
probe.dict_size = dict_size;
probe.cctx_params = cctx_params;
probe.pledged_src_size = pledged_src_size;
probe.begin_result
}
fn compress_begin_using_dict_test_state(
probe: &mut CompressBeginUsingDictProbe,
exclusion_mask: &c_uint,
) -> ZSTD_rust_compressBeginUsingDictState {
ZSTD_rust_compressBeginUsingDictState {
cctx: (probe as *mut CompressBeginUsingDictProbe).cast(),
cctx_params: (probe as *mut CompressBeginUsingDictProbe).cast(),
exclusion_mask,
init_params: compress_begin_using_dict_test_init,
begin: compress_begin_using_dict_test_begin,
}
}
#[test]
fn compress_begin_using_dict_selects_unknown_source_params_and_default_level() {
let mut probe = CompressBeginUsingDictProbe::default();
let exclusion_mask = 0;
let state = compress_begin_using_dict_test_state(&mut probe, &exclusion_mask);
let dict = [1u8, 2, 3];
let result = unsafe {
ZSTD_rust_compressBeginUsingDict(&state, dict.as_ptr().cast(), dict.len(), 0)
};
assert_eq!(result, 0);
assert_eq!(probe.events, ["init", "begin"]);
assert_eq!(
probe.params,
ZSTD_rust_params_getParamsInternal(
0,
ZSTD_CONTENTSIZE_UNKNOWN,
dict.len(),
ZSTD_RUST_CPM_NO_ATTACH_DICT,
exclusion_mask,
)
);
assert_eq!(probe.compression_level, ZSTD_rust_params_defaultCLevel());
assert_eq!(probe.dict, dict.as_ptr().cast());
assert_eq!(probe.dict_size, dict.len());
assert_eq!(
probe.cctx_params,
(&probe as *const CompressBeginUsingDictProbe).cast()
);
assert_eq!(probe.pledged_src_size, ZSTD_CONTENTSIZE_UNKNOWN);
}
#[test]
fn compress_begin_using_dict_propagates_begin_errors_after_initialization() {
let mut probe = CompressBeginUsingDictProbe {
begin_result: ERROR(ZstdErrorCode::StageWrong),
..Default::default()
};
let exclusion_mask = 0;
let state = compress_begin_using_dict_test_state(&mut probe, &exclusion_mask);
let result = unsafe { ZSTD_rust_compressBeginUsingDict(&state, ptr::null(), 0, 3) };
assert_eq!(result, probe.begin_result);
assert_eq!(probe.events, ["init", "begin"]);
}
fn assert_dictionary_corrupted(result: usize) {
assert!(ERR_isError(result));
assert_eq!(