ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(compress): move CDict source selection to Rust

Browse Source 
ZSTD_compressBegin_internal already routes attach-versus-reload policy through
Rust, but its C insertion callback still inspected a non-null ZSTD_CDict and
selected dictContent, dictContentSize, and dictContentType. That left source
selection coupled to the private CDict layout and kept the Rust bridge from
receiving the same selected source for direct and prepared dictionaries.

Project the three CDict content fields into the repr(C) begin state. Rust now
selects either those fields or the direct dictionary arguments before invoking
the C callback. The callback keeps the private CCtx insertion path and no
longer branches on CDict. ABI offsets and sizes are asserted on both sides,
with focused by-reference and forced-CDict tests covering pointer, size, type,
reset order, and dictionary-result publication.

Test Plan:
- `cargo check --manifest-path rust/Cargo.toml --lib --tests` -- passed under
  `ulimit -v 41943040; CARGO_BUILD_JOBS=1`.
- Serial GCC/Clang syntax-only checks for `lib/compress/zstd_compress.c` --
  passed under the same cap; only existing warnings were reported.
- `cargo clippy` for lib, benches, and tests with `-D warnings`, plus nightly
  format check -- passed serially under the same cap.
- `cargo test --manifest-path rust/Cargo.toml --lib zstd_compress_dictionary
  -- --test-threads=1` -- compiled but standalone linking failed on the three
  pre-existing C bridge symbols `ZSTD_rust_dctx_trace_view`,
  `ZSTD_rust_dctx_view`, and `ZSTD_rust_block_context_init`.
- Full native, upstream, and fuzzer tests were not run per scope.
This commit is contained in:
ddidderr
2026-07-21 10:13:54 +02:00
parent 639865d138
commit af91fb23ab
2 changed files with 147 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/zstd_compress_dictionary.rs
+114 -30
View File
@@ -1537,18 +1537,22 @@ type CompressBeginResetInternalFn =
type CompressBeginResetUsingCDictFn =
unsafe extern "C" fn(*mut c_void, *const c_void, *const c_void, u64, c_int) -> usize;
type CompressBeginInsertDictionaryFn =
unsafe extern "C" fn(*mut c_void, *const c_void, *const c_void, usize, c_int, c_int) -> usize;
unsafe extern "C" fn(*mut c_void, *const c_void, usize, c_int, c_int) -> usize;
/// Projection for the dictionary-selection portion of
/// `ZSTD_compressBegin_internal`.
///
/// Rust owns the CDict attach decision and dictionary-result publication. C
/// retains the context reset, CDict attach, and private insertion callbacks.
/// Rust owns the CDict attach decision, dictionary-source selection, and
/// dictionary-result publication. C retains the context reset, CDict attach,
/// and private insertion callbacks.
#[repr(C)]
pub struct ZSTD_rust_compressBeginState {
callback_context: *mut c_void,
params: *const c_void,
cdict: *const c_void,
cdict_dict_content: *const c_void,
cdict_dict_content_size: usize,
cdict_dict_content_type: c_int,
cdict_content_size: *const usize,
cdict_compression_level: *const c_int,
attach_dict_pref: *const c_int,
@@ -1573,26 +1577,32 @@ const _: () = {
assert!(offset_of!(ZSTD_rust_compressBeginState, callback_context) == 0);
assert!(offset_of!(ZSTD_rust_compressBeginState, params) == size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, cdict) == 2 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, cdict_content_size) == 3 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, cdict_dict_content) == 3 * size_of::<usize>());
assert!(
offset_of!(ZSTD_rust_compressBeginState, cdict_compression_level) == 4 * size_of::<usize>()
offset_of!(ZSTD_rust_compressBeginState, cdict_dict_content_size) == 4 * size_of::<usize>()
);
assert!(offset_of!(ZSTD_rust_compressBeginState, attach_dict_pref) == 5 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, dict) == 6 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, dict_size) == 7 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, dict_content_type) == size_of::<[usize; 8]>());
assert!(offset_of!(ZSTD_rust_compressBeginState, dtlm) == 9 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, pledged_src_size) == 10 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, zbuff) == 11 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, force_load) == size_of::<[usize; 12]>());
assert!(offset_of!(ZSTD_rust_compressBeginState, dict_id) == 13 * size_of::<usize>());
assert!(
offset_of!(ZSTD_rust_compressBeginState, dict_content_size) == size_of::<[usize; 14]>()
offset_of!(ZSTD_rust_compressBeginState, cdict_dict_content_type)
== 4 * size_of::<usize>() + size_of::<usize>()
);
assert!(offset_of!(ZSTD_rust_compressBeginState, reset_internal) == 15 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, reset_using_cdict) == 16 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, insert_dictionary) == 17 * size_of::<usize>());
assert!(size_of::<ZSTD_rust_compressBeginState>() == size_of::<[usize; 18]>());
assert!(offset_of!(ZSTD_rust_compressBeginState, cdict_content_size) == 6 * size_of::<usize>());
assert!(
offset_of!(ZSTD_rust_compressBeginState, cdict_compression_level) == 7 * size_of::<usize>()
);
assert!(offset_of!(ZSTD_rust_compressBeginState, attach_dict_pref) == size_of::<[usize; 8]>());
assert!(offset_of!(ZSTD_rust_compressBeginState, dict) == 9 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, dict_size) == 10 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, dict_content_type) == 11 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, dtlm) == 12 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, pledged_src_size) == 13 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, zbuff) == 14 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, force_load) == 15 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, dict_id) == 16 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, dict_content_size) == 17 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, reset_internal) == 18 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, reset_using_cdict) == 19 * size_of::<usize>());
assert!(offset_of!(ZSTD_rust_compressBeginState, insert_dictionary) == 20 * size_of::<usize>());
assert!(size_of::<ZSTD_rust_compressBeginState>() == size_of::<[usize; 21]>());
};
/// Select and begin a dictionary-backed compression context.
@@ -1676,13 +1686,23 @@ pub unsafe extern "C" fn ZSTD_rust_compressBegin(
return reset_result;
}
let (dict, dict_content_size, dict_content_type) = if cdict_present {
(
state.cdict_dict_content,
state.cdict_dict_content_size,
state.cdict_dict_content_type,
)
} else {
(state.dict, unsafe { *state.dict_size }, unsafe {
*state.dict_content_type
})
};
let dict_id = unsafe {
insert_dictionary(
state.callback_context,
state.cdict,
state.dict,
*state.dict_size,
*state.dict_content_type,
dict,
dict_content_size,
dict_content_type,
*state.dtlm,
)
};
@@ -3104,7 +3124,6 @@ mod tests {
attach_pledged_src_size: u64,
attach_zbuff: c_int,
attach_result: usize,
insert_cdict: *const c_void,
insert_dict: *const c_void,
insert_size: usize,
insert_content_type: c_int,
@@ -3126,7 +3145,6 @@ mod tests {
attach_pledged_src_size: 0,
attach_zbuff: 0,
attach_result: 0,
insert_cdict: ptr::null(),
insert_dict: ptr::null(),
insert_size: 0,
insert_content_type: 0,
@@ -3174,7 +3192,6 @@ mod tests {
unsafe extern "C" fn compress_begin_insert_dictionary(
context: *mut c_void,
cdict: *const c_void,
dict: *const c_void,
dict_size: usize,
dict_content_type: c_int,
@@ -3182,7 +3199,6 @@ mod tests {
) -> usize {
let probe = unsafe { compress_begin_probe(context) };
probe.events.push("insert");
probe.insert_cdict = cdict;
probe.insert_dict = dict;
probe.insert_size = dict_size;
probe.insert_content_type = dict_content_type;
@@ -3195,6 +3211,9 @@ mod tests {
probe: &mut CompressBeginProbe,
params: *const c_void,
cdict: *const c_void,
cdict_dict_content: *const c_void,
cdict_dict_content_size: usize,
cdict_dict_content_type: c_int,
cdict_content_size: *const usize,
cdict_compression_level: *const c_int,
attach_dict_pref: &c_int,
@@ -3212,6 +3231,9 @@ mod tests {
callback_context: (probe as *mut CompressBeginProbe).cast(),
params,
cdict,
cdict_dict_content,
cdict_dict_content_size,
cdict_dict_content_type,
cdict_content_size,
cdict_compression_level,
attach_dict_pref,
@@ -3231,7 +3253,7 @@ mod tests {
}
#[test]
fn compress_begin_resets_then_inserts_for_loaded_dictionary() {
fn compress_begin_resets_then_inserts_by_reference_dictionary() {
let dictionary = [1u8, 2, 3, 4];
let mut probe = CompressBeginProbe {
insert_result: 17,
@@ -3241,7 +3263,7 @@ mod tests {
let attach_dict_pref = 0;
let dict_size = dictionary.len();
let dict_content_type = ZSTD_DCT_RAW_CONTENT;
let dtlm = 7;
let dtlm = ZSTD_DLM_BY_REF;
let pledged_src_size = 1u64 << 20;
let zbuff = 9;
let force_load = 1;
@@ -3252,6 +3274,9 @@ mod tests {
params,
ptr::null(),
ptr::null(),
0,
0,
ptr::null(),
ptr::null(),
&attach_dict_pref,
dictionary.as_ptr().cast(),
@@ -3273,7 +3298,6 @@ mod tests {
assert_eq!(probe.reset_loaded_dict_size, dictionary.len());
assert_eq!(probe.reset_pledged_src_size, pledged_src_size);
assert_eq!(probe.reset_zbuff, zbuff);
assert!(probe.insert_cdict.is_null());
assert_eq!(probe.insert_dict, dictionary.as_ptr().cast());
assert_eq!(probe.insert_size, dictionary.len());
assert_eq!(probe.insert_content_type, dict_content_type);
@@ -3302,6 +3326,9 @@ mod tests {
&mut probe,
params,
cdict,
ptr::null(),
0,
0,
&cdict_content_size,
&cdict_compression_level,
&attach_dict_pref,
@@ -3326,6 +3353,63 @@ mod tests {
assert_eq!(probe.attach_zbuff, zbuff);
}
#[test]
fn compress_begin_inserts_projected_cdict_content_when_forced() {
let dictionary = [5u8, 6, 7, 8];
let mut probe = CompressBeginProbe {
insert_result: 23,
..Default::default()
};
let params = 0x3000usize as *const c_void;
let cdict = 0x4000usize as *const c_void;
let cdict_dict_content = dictionary.as_ptr().cast();
let cdict_dict_content_size = dictionary.len();
let cdict_dict_content_type = ZSTD_DCT_RAW_CONTENT;
let cdict_content_size = dictionary.len();
let cdict_compression_level = 3;
let attach_dict_pref = 0;
let dict_size = 0;
let dict_content_type = ZSTD_DCT_AUTO;
let dtlm = ZSTD_DLM_BY_REF;
let pledged_src_size = 1u64 << 20;
let zbuff = 9;
let force_load = 1;
let mut dict_id = 0;
let mut dict_content_size = 0;
let state = compress_begin_test_state(
&mut probe,
params,
cdict,
cdict_dict_content,
cdict_dict_content_size,
cdict_dict_content_type,
&cdict_content_size,
&cdict_compression_level,
&attach_dict_pref,
ptr::null(),
&dict_size,
&dict_content_type,
&dtlm,
&pledged_src_size,
&zbuff,
&force_load,
&mut dict_id,
&mut dict_content_size,
);
let result = unsafe { ZSTD_rust_compressBegin(&state) };
assert_eq!(result, 0);
assert_eq!(probe.events, ["reset", "insert"]);
assert_eq!(probe.reset_loaded_dict_size, dictionary.len());
assert_eq!(probe.insert_dict, cdict_dict_content);
assert_eq!(probe.insert_size, cdict_dict_content_size);
assert_eq!(probe.insert_content_type, cdict_dict_content_type);
assert_eq!(probe.insert_dtlm, dtlm);
assert_eq!(dict_id, probe.insert_result as c_uint);
assert_eq!(dict_content_size, cdict_dict_content_size);
}
struct ResetUsingCDictProbe {
events: Vec<&'static str>,
attach_cdict: *const c_void,