ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(compress): move dictionary begin policy into Rust

Browse Source 
Move the compressBegin_usingDict family parameter selection and default-level
normalization into Rust, covering the public ZSTD_compressBegin entry point as
well. C retains private ZSTD_CCtx_params initialization and
ZSTD_compressBegin_internal through callbacks, with the unknown pledged-size
contract preserved.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --release compress_begin_using_dict -- --nocapture
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --release
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --release --all-targets -- -D warnings
- ulimit -v 41943040; make -B -C programs -j1 zstd
- ulimit -v 41943040; make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s
This commit is contained in:
ddidderr
2026-07-19 15:51:36 +02:00
parent da3d4ed232
commit 9c0b9bd250
2 changed files with 228 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+54 -5
View File
@@ -1367,6 +1367,33 @@ typedef char ZSTD_rust_compress_using_cdict_state_layout[
== 3 * sizeof(void*)
&& sizeof(ZSTD_rust_compressUsingCDictState) == 4 * sizeof(void*))
? 1 : -1];
typedef void (*ZSTD_rust_compressBeginUsingDictInitParams_f)(
void* cctxParams, const ZSTD_parameters* params, int compressionLevel);
typedef size_t (*ZSTD_rust_compressBeginUsingDictBegin_f)(
void* context, const void* dict, size_t dictSize,
const void* cctxParams, U64 pledgedSrcSize);
typedef struct {
void* cctx;
void* cctxParams;
const U32* exclusionMask;
ZSTD_rust_compressBeginUsingDictInitParams_f initParams;
ZSTD_rust_compressBeginUsingDictBegin_f begin;
} ZSTD_rust_compressBeginUsingDictState;
size_t ZSTD_rust_compressBeginUsingDict(
const ZSTD_rust_compressBeginUsingDictState* state,
const void* dict, size_t dictSize, int compressionLevel);
typedef char ZSTD_rust_compress_begin_using_dict_state_layout[
(offsetof(ZSTD_rust_compressBeginUsingDictState, cctx) == 0
&& offsetof(ZSTD_rust_compressBeginUsingDictState, cctxParams)
== sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginUsingDictState, exclusionMask)
== 2 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginUsingDictState, initParams)
== 3 * sizeof(void*)
&& offsetof(ZSTD_rust_compressBeginUsingDictState, begin)
== 4 * sizeof(void*)
&& sizeof(ZSTD_rust_compressBeginUsingDictState) == 5 * sizeof(void*))
? 1 : -1];
typedef struct {
const ZSTD_compressionParameters* cParams;
const unsigned* dictID;
@@ -4796,6 +4823,24 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
return ZSTD_rust_compressBegin(&state);
}
static void ZSTD_rust_compressBeginUsingDict_initParams(
void* cctxParams, const ZSTD_parameters* params, int compressionLevel)
{
ZSTD_CCtxParams_init_internal(
(ZSTD_CCtx_params*)cctxParams, params, compressionLevel);
}
static size_t ZSTD_rust_compressBeginUsingDict_begin(
void* context, const void* dict, size_t dictSize,
const void* cctxParams, U64 pledgedSrcSize)
{
return ZSTD_compressBegin_internal(
(ZSTD_CCtx*)context, dict, dictSize,
ZSTD_dct_auto, ZSTD_dtlm_fast, NULL,
(const ZSTD_CCtx_params*)cctxParams,
pledgedSrcSize, ZSTDb_not_buffered);
}
size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx,
const void* dict, size_t dictSize,
ZSTD_dictContentType_e dictContentType,
@@ -4831,13 +4876,17 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
static size_t
ZSTD_compressBegin_usingDict_deprecated(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_rust_compressBeginUsingDictState state;
ZSTD_CCtx_params cctxParams;
{ ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict);
ZSTD_CCtxParams_init_internal(&cctxParams, &params, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel);
}
U32 const exclusionMask = ZSTD_getCParamsExclusionMask();
DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize);
return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL,
&cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered);
state.cctx = cctx;
state.cctxParams = &cctxParams;
state.exclusionMask = &exclusionMask;
state.initParams = ZSTD_rust_compressBeginUsingDict_initParams;
state.begin = ZSTD_rust_compressBeginUsingDict_begin;
return ZSTD_rust_compressBeginUsingDict(
&state, dict, dictSize, compressionLevel);
}
size_t
rust/src/zstd_compress_dictionary.rs
+174 -2
View File
@@ -16,8 +16,10 @@ use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
use crate::fse_compress::FSE_buildCTable_wksp;
use crate::huf_compress::HUF_readCTable;
use crate::zstd_compress_params::{
ZSTD_compressionParameters, ZSTD_frameParameters, ZSTD_parameters, ZSTD_rust_params_getCParams,
ZSTD_rust_params_shouldAttachDict, ZSTD_CONTENTSIZE_UNKNOWN,
ZSTD_compressionParameters, ZSTD_frameParameters, ZSTD_parameters,
ZSTD_rust_params_defaultCLevel, ZSTD_rust_params_getCParams,
ZSTD_rust_params_getParamsInternal, ZSTD_rust_params_shouldAttachDict,
ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_RUST_CPM_NO_ATTACH_DICT,
};
use crate::zstd_compress_params_api::ZSTD_CCtx_params;
use crate::zstd_compress_stats::{
@@ -297,6 +299,76 @@ pub unsafe extern "C" fn ZSTD_rust_compressUsingCDict(
unsafe { (state.end)(state.callback_context, dst, dst_capacity, src, src_size) }
}
type CompressBeginUsingDictBeginFn =
unsafe extern "C" fn(*mut c_void, *const c_void, usize, *const c_void, u64) -> usize;
/// Explicit projection for the public `ZSTD_compressBegin_usingDict` family.
///
/// Rust owns unknown-source parameter selection and default-level
/// normalization. C retains private `ZSTD_CCtx_params` initialization and the
/// final `ZSTD_compressBegin_internal` call behind callbacks.
#[repr(C)]
pub struct ZSTD_rust_compressBeginUsingDictState {
cctx: *mut c_void,
cctx_params: *mut ZSTD_CCtx_params,
exclusion_mask: *const c_uint,
init_params: CompressBeginUsingCDictInitParamsFn,
begin: CompressBeginUsingDictBeginFn,
}
const _: () = {
assert!(offset_of!(ZSTD_rust_compressBeginUsingDictState, cctx) == 0);
assert!(offset_of!(ZSTD_rust_compressBeginUsingDictState, cctx_params) == size_of::<usize>());
assert!(
offset_of!(ZSTD_rust_compressBeginUsingDictState, exclusion_mask) == 2 * size_of::<usize>()
);
assert!(
offset_of!(ZSTD_rust_compressBeginUsingDictState, init_params) == 3 * size_of::<usize>()
);
assert!(offset_of!(ZSTD_rust_compressBeginUsingDictState, begin) == 4 * size_of::<usize>());
assert!(size_of::<ZSTD_rust_compressBeginUsingDictState>() == size_of::<[usize; 5]>());
};
/// Select parameters and start a dictionary-backed frame.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBeginUsingDict(
state: *const ZSTD_rust_compressBeginUsingDictState,
dict: *const c_void,
dict_size: usize,
compression_level: c_int,
) -> usize {
if state.is_null() {
return ERROR(ZstdErrorCode::Generic);
}
let state = unsafe { &*state };
if state.cctx.is_null() || state.cctx_params.is_null() || state.exclusion_mask.is_null() {
return ERROR(ZstdErrorCode::Generic);
}
let params = ZSTD_rust_params_getParamsInternal(
compression_level,
ZSTD_CONTENTSIZE_UNKNOWN,
dict_size,
ZSTD_RUST_CPM_NO_ATTACH_DICT,
unsafe { *state.exclusion_mask },
);
let init_level = if compression_level == 0 {
ZSTD_rust_params_defaultCLevel()
} else {
compression_level
};
unsafe {
(state.init_params)(state.cctx_params.cast(), &params, init_level);
(state.begin)(
state.cctx,
dict,
dict_size,
state.cctx_params.cast_const().cast(),
ZSTD_CONTENTSIZE_UNKNOWN,
)
}
}
type InitCDictAssignContentFn =
unsafe extern "C" fn(*mut c_void, *const c_void, usize, c_int) -> usize;
type InitCDictReserveEntropyFn = unsafe extern "C" fn(*mut c_void) -> *mut c_void;
@@ -2482,6 +2554,106 @@ mod tests {
assert_eq!(probe.events, ["begin"]);
}
#[derive(Default)]
struct CompressBeginUsingDictProbe {
events: Vec<&'static str>,
params: ZSTD_parameters,
compression_level: c_int,
dict: *const c_void,
dict_size: usize,
cctx_params: *const c_void,
pledged_src_size: u64,
begin_result: usize,
}
unsafe extern "C" fn compress_begin_using_dict_test_init(
context: *mut c_void,
params: *const ZSTD_parameters,
compression_level: c_int,
) {
let probe = unsafe { &mut *context.cast::<CompressBeginUsingDictProbe>() };
probe.events.push("init");
probe.params = unsafe { *params };
probe.compression_level = compression_level;
}
unsafe extern "C" fn compress_begin_using_dict_test_begin(
context: *mut c_void,
dict: *const c_void,
dict_size: usize,
cctx_params: *const c_void,
pledged_src_size: u64,
) -> usize {
let probe = unsafe { &mut *context.cast::<CompressBeginUsingDictProbe>() };
probe.events.push("begin");
probe.dict = dict;
probe.dict_size = dict_size;
probe.cctx_params = cctx_params;
probe.pledged_src_size = pledged_src_size;
probe.begin_result
}
fn compress_begin_using_dict_test_state(
probe: &mut CompressBeginUsingDictProbe,
exclusion_mask: &c_uint,
) -> ZSTD_rust_compressBeginUsingDictState {
ZSTD_rust_compressBeginUsingDictState {
cctx: (probe as *mut CompressBeginUsingDictProbe).cast(),
cctx_params: (probe as *mut CompressBeginUsingDictProbe).cast(),
exclusion_mask,
init_params: compress_begin_using_dict_test_init,
begin: compress_begin_using_dict_test_begin,
}
}
#[test]
fn compress_begin_using_dict_selects_unknown_source_params_and_default_level() {
let mut probe = CompressBeginUsingDictProbe::default();
let exclusion_mask = 0;
let state = compress_begin_using_dict_test_state(&mut probe, &exclusion_mask);
let dict = [1u8, 2, 3];
let result = unsafe {
ZSTD_rust_compressBeginUsingDict(&state, dict.as_ptr().cast(), dict.len(), 0)
};
assert_eq!(result, 0);
assert_eq!(probe.events, ["init", "begin"]);
assert_eq!(
probe.params,
ZSTD_rust_params_getParamsInternal(
0,
ZSTD_CONTENTSIZE_UNKNOWN,
dict.len(),
ZSTD_RUST_CPM_NO_ATTACH_DICT,
exclusion_mask,
)
);
assert_eq!(probe.compression_level, ZSTD_rust_params_defaultCLevel());
assert_eq!(probe.dict, dict.as_ptr().cast());
assert_eq!(probe.dict_size, dict.len());
assert_eq!(
probe.cctx_params,
(&probe as *const CompressBeginUsingDictProbe).cast()
);
assert_eq!(probe.pledged_src_size, ZSTD_CONTENTSIZE_UNKNOWN);
}
#[test]
fn compress_begin_using_dict_propagates_begin_errors_after_initialization() {
let mut probe = CompressBeginUsingDictProbe {
begin_result: ERROR(ZstdErrorCode::StageWrong),
..Default::default()
};
let exclusion_mask = 0;
let state = compress_begin_using_dict_test_state(&mut probe, &exclusion_mask);
let result = unsafe { ZSTD_rust_compressBeginUsingDict(&state, ptr::null(), 0, 3) };
assert_eq!(result, probe.begin_result);
assert_eq!(probe.events, ["init", "begin"]);
}
fn assert_dictionary_corrupted(result: usize) {
assert!(ERR_isError(result));
assert_eq!(