ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
0caec24fd43eca6af5bb12152387e242c0fedc82
zstd-rs/rust/src/zstd_ddict.rs
T
ddidderr ca70ca8061 feat(rust): port decode dictionaries 
Move ZSTD_DDict allocation, ownership, construction, and public ABI exports
into Rust. The C decoder keeps ZSTD_loadDEntropy until its table loader moves.

The C shim now exposes C-computed DCtx field offsets. This keeps dictionary
state correct across conditional C layouts, including fuzz fields and 32-bit
static-BMI2 builds, without duplicating decoder-context layout in Rust.

Test Plan:
- cargo fmt, cargo clippy, cargo clippy --benches, cargo clippy --tests
- cargo test --all-targets and cargo build --release
- cargo test/build --target i686-unknown-linux-gnu
- C fuzzer, zstreamtest, invalidDictionaries, and fuzzer32 smoke runs
- fuzz-enabled and i686 -mbmi2 C/Rust dictionary roundtrip harnesses

Refs: rust/README.md
2026-07-10 21:20:50 +02:00

897 lines
29 KiB
Rust
Raw Blame History

#![allow(non_snake_case)]
//! Digested decompression dictionaries (`ZSTD_DDict`).
//!
//! The decoder proper is still implemented in C during the migration. In
//! particular, its `ZSTD_loadDEntropy()` helper owns the format-specific
//! construction of the decoder tables. This module owns the DDict object,
//! its allocation and lifetime, and passes an ABI-identical table layout to
//! that helper.
use crate::common::{LL_FSE_LOG, ML_FSE_LOG, OFF_FSE_LOG, ZSTD_FRAMEIDSIZE, ZSTD_REP_NUM};
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
use crate::mem::MEM_readLE32;
use std::ffi::c_void;
use std::mem::size_of;
use std::os::raw::{c_int, c_uint};
use std::ptr;
const ZSTD_MAGIC_DICTIONARY: u32 = 0xEC30_A437;
const ZSTD_HUFFDTABLE_CAPACITY_LOG: u32 = 12;
const HUF_DTABLE_SIZE: usize = 1 + (1 << ZSTD_HUFFDTABLE_CAPACITY_LOG);
const ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32: usize = 157;
const ZSTD_DLM_BY_COPY: c_int = 0;
const ZSTD_DLM_BY_REF: c_int = 1;
const ZSTD_DCT_RAW_CONTENT: c_int = 1;
const ZSTD_DCT_FULL_DICT: c_int = 2;
const ZSTD_DDICT_NO_FUZZING_OFFSET: usize = usize::MAX;
type ZstdAllocFunction = unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void;
type ZstdFreeFunction = unsafe extern "C" fn(*mut c_void, *mut c_void);
/// ABI-compatible representation of `ZSTD_customMem` from `zstd.h`.
#[repr(C)]
#[derive(Clone, Copy)]
pub struct ZSTD_customMem {
custom_alloc: Option<ZstdAllocFunction>,
custom_free: Option<ZstdFreeFunction>,
opaque: *mut c_void,
}
const ZSTD_DEFAULT_CMEM: ZSTD_customMem = ZSTD_customMem {
custom_alloc: None,
custom_free: None,
opaque: ptr::null_mut(),
};
#[repr(C)]
#[derive(Clone, Copy)]
struct ZSTD_seqSymbol {
next_state: u16,
nb_additional_bits: u8,
nb_bits: u8,
base_value: u32,
}
#[repr(C)]
struct ZSTD_entropyDTables {
ll_table: [ZSTD_seqSymbol; 1 + (1 << LL_FSE_LOG)],
of_table: [ZSTD_seqSymbol; 1 + (1 << OFF_FSE_LOG)],
ml_table: [ZSTD_seqSymbol; 1 + (1 << ML_FSE_LOG)],
huf_table: [u32; HUF_DTABLE_SIZE],
rep: [u32; ZSTD_REP_NUM],
workspace: [u32; ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32],
}
/// The opaque public `ZSTD_DDict` object, with its C-private representation.
#[repr(C)]
pub struct ZSTD_DDict {
dict_buffer: *mut c_void,
dict_content: *const c_void,
dict_size: usize,
entropy: ZSTD_entropyDTables,
dict_id: u32,
entropy_present: u32,
c_mem: ZSTD_customMem,
}
/// Opaque C decoder context. Callers continue to create and own contexts in
/// the C API.
#[repr(C)]
pub struct ZSTD_DCtx {
_private: [u8; 0],
}
#[derive(Clone, Copy)]
struct ZstdDctxOffsets {
llt_ptr: usize,
mlt_ptr: usize,
oft_ptr: usize,
huf_ptr: usize,
entropy_rep: usize,
previous_dst_end: usize,
prefix_start: usize,
virtual_start: usize,
dict_end: usize,
lit_entropy: usize,
fse_entropy: usize,
dict_id: usize,
fuzz_begin: usize,
fuzz_end: usize,
}
#[cfg(not(test))]
unsafe extern "C" {
fn ZSTD_loadDEntropy(
entropy: *mut ZSTD_entropyDTables,
dict: *const c_void,
dict_size: usize,
) -> usize;
/*
* These are emitted by zstd_ddict.c with the exact C preprocessor state
* used for struct ZSTD_DCtx_s. Keeping the opaque context as C-provided
* offsets avoids duplicating optional fields such as DYNAMIC_BMI2 in Rust.
*/
static ZSTD_rust_ddict_llt_ptr_offset: usize;
static ZSTD_rust_ddict_mlt_ptr_offset: usize;
static ZSTD_rust_ddict_oft_ptr_offset: usize;
static ZSTD_rust_ddict_huf_ptr_offset: usize;
static ZSTD_rust_ddict_entropy_rep_offset: usize;
static ZSTD_rust_ddict_previous_dst_end_offset: usize;
static ZSTD_rust_ddict_prefix_start_offset: usize;
static ZSTD_rust_ddict_virtual_start_offset: usize;
static ZSTD_rust_ddict_dict_end_offset: usize;
static ZSTD_rust_ddict_lit_entropy_offset: usize;
static ZSTD_rust_ddict_fse_entropy_offset: usize;
static ZSTD_rust_ddict_dict_id_offset: usize;
static ZSTD_rust_ddict_fuzz_begin_offset: usize;
static ZSTD_rust_ddict_fuzz_end_offset: usize;
}
#[inline]
unsafe fn ddict_dctx_offsets() -> ZstdDctxOffsets {
#[cfg(not(test))]
{
unsafe {
ZstdDctxOffsets {
llt_ptr: ZSTD_rust_ddict_llt_ptr_offset,
mlt_ptr: ZSTD_rust_ddict_mlt_ptr_offset,
oft_ptr: ZSTD_rust_ddict_oft_ptr_offset,
huf_ptr: ZSTD_rust_ddict_huf_ptr_offset,
entropy_rep: ZSTD_rust_ddict_entropy_rep_offset,
previous_dst_end: ZSTD_rust_ddict_previous_dst_end_offset,
prefix_start: ZSTD_rust_ddict_prefix_start_offset,
virtual_start: ZSTD_rust_ddict_virtual_start_offset,
dict_end: ZSTD_rust_ddict_dict_end_offset,
lit_entropy: ZSTD_rust_ddict_lit_entropy_offset,
fse_entropy: ZSTD_rust_ddict_fse_entropy_offset,
dict_id: ZSTD_rust_ddict_dict_id_offset,
fuzz_begin: ZSTD_rust_ddict_fuzz_begin_offset,
fuzz_end: ZSTD_rust_ddict_fuzz_end_offset,
}
}
}
#[cfg(test)]
{
ZstdDctxOffsets {
llt_ptr: 0,
mlt_ptr: size_of::<*const c_void>(),
oft_ptr: 2 * size_of::<*const c_void>(),
huf_ptr: 3 * size_of::<*const c_void>(),
entropy_rep: 4 * size_of::<*const c_void>(),
previous_dst_end: 4 * size_of::<*const c_void>() + 3 * size_of::<u32>(),
prefix_start: 5 * size_of::<*const c_void>() + 3 * size_of::<u32>(),
virtual_start: 6 * size_of::<*const c_void>() + 3 * size_of::<u32>(),
dict_end: 7 * size_of::<*const c_void>() + 3 * size_of::<u32>(),
lit_entropy: 8 * size_of::<*const c_void>() + 3 * size_of::<u32>(),
fse_entropy: 8 * size_of::<*const c_void>() + 4 * size_of::<u32>(),
dict_id: 8 * size_of::<*const c_void>() + 5 * size_of::<u32>(),
fuzz_begin: ZSTD_DDICT_NO_FUZZING_OFFSET,
fuzz_end: ZSTD_DDICT_NO_FUZZING_OFFSET,
}
}
}
unsafe fn write_dctx_field<T>(dctx: *mut ZSTD_DCtx, offset: usize, value: T) {
unsafe {
dctx.cast::<u8>()
.add(offset)
.cast::<T>()
.write_unaligned(value);
}
}
unsafe fn copy_fuzzing_bounds_at_offsets(
dctx: *mut ZSTD_DCtx,
begin: *const c_void,
end: *const c_void,
begin_offset: usize,
end_offset: usize,
) {
if begin_offset == ZSTD_DDICT_NO_FUZZING_OFFSET {
debug_assert_eq!(end_offset, ZSTD_DDICT_NO_FUZZING_OFFSET);
return;
}
debug_assert_ne!(end_offset, ZSTD_DDICT_NO_FUZZING_OFFSET);
unsafe {
let dctx = dctx.cast::<u8>();
dctx.add(begin_offset)
.cast::<*const c_void>()
.write_unaligned(begin);
dctx.add(end_offset)
.cast::<*const c_void>()
.write_unaligned(end);
}
}
unsafe fn copy_fuzzing_bounds(dctx: *mut ZSTD_DCtx, begin: *const c_void, end: *const c_void) {
let offsets = unsafe { ddict_dctx_offsets() };
unsafe {
copy_fuzzing_bounds_at_offsets(dctx, begin, end, offsets.fuzz_begin, offsets.fuzz_end)
};
}
#[inline]
unsafe fn load_d_entropy(
entropy: *mut ZSTD_entropyDTables,
dict: *const c_void,
dict_size: usize,
) -> usize {
#[cfg(not(test))]
{
// The existing C decoder owns this helper until its translation unit is
// moved. Its argument is the ABI-compatible table storage above.
unsafe { ZSTD_loadDEntropy(entropy, dict, dict_size) }
}
#[cfg(test)]
{
let _ = (entropy, dict, dict_size);
// Unit tests exercise the DDict ownership and raw-content paths. A
// full dictionary is covered by the C compatibility executables, where
// the decoder's real ZSTD_loadDEntropy() is linked in.
ERROR(ZstdErrorCode::DictionaryCorrupted)
}
}
#[inline]
unsafe fn custom_malloc(size: usize, c_mem: ZSTD_customMem) -> *mut c_void {
match c_mem.custom_alloc {
Some(alloc) => unsafe { alloc(c_mem.opaque, size) },
None => unsafe { libc::malloc(size) },
}
}
#[inline]
unsafe fn custom_free(allocation: *mut c_void, c_mem: ZSTD_customMem) {
if allocation.is_null() {
return;
}
match c_mem.custom_free {
Some(free) => unsafe { free(c_mem.opaque, allocation) },
None => unsafe { libc::free(allocation) },
}
}
#[inline]
fn valid_custom_mem(c_mem: ZSTD_customMem) -> bool {
c_mem.custom_alloc.is_some() == c_mem.custom_free.is_some()
}
unsafe fn load_entropy_into_ddict(ddict: *mut ZSTD_DDict, dict_content_type: c_int) -> usize {
unsafe {
(*ddict).dict_id = 0;
(*ddict).entropy_present = 0;
if dict_content_type == ZSTD_DCT_RAW_CONTENT {
return 0;
}
if (*ddict).dict_size < 8 {
return if dict_content_type == ZSTD_DCT_FULL_DICT {
ERROR(ZstdErrorCode::DictionaryCorrupted)
} else {
0
};
}
if MEM_readLE32((*ddict).dict_content) != ZSTD_MAGIC_DICTIONARY {
return if dict_content_type == ZSTD_DCT_FULL_DICT {
ERROR(ZstdErrorCode::DictionaryCorrupted)
} else {
0
};
}
(*ddict).dict_id = MEM_readLE32(
(*ddict)
.dict_content
.cast::<u8>()
.add(ZSTD_FRAMEIDSIZE)
.cast(),
);
let result = load_d_entropy(
ptr::addr_of_mut!((*ddict).entropy),
(*ddict).dict_content,
(*ddict).dict_size,
);
if ERR_isError(result) {
return ERROR(ZstdErrorCode::DictionaryCorrupted);
}
(*ddict).entropy_present = 1;
0
}
}
unsafe fn init_ddict_internal(
ddict: *mut ZSTD_DDict,
dict: *const c_void,
mut dict_size: usize,
dict_load_method: c_int,
dict_content_type: c_int,
) -> usize {
unsafe {
if dict_load_method == ZSTD_DLM_BY_REF || dict.is_null() || dict_size == 0 {
(*ddict).dict_buffer = ptr::null_mut();
(*ddict).dict_content = dict;
if dict.is_null() {
dict_size = 0;
}
} else {
let internal_buffer = custom_malloc(dict_size, (*ddict).c_mem);
(*ddict).dict_buffer = internal_buffer;
(*ddict).dict_content = internal_buffer.cast_const();
if internal_buffer.is_null() {
return ERROR(ZstdErrorCode::MemoryAllocation);
}
ptr::copy_nonoverlapping(dict.cast::<u8>(), internal_buffer.cast::<u8>(), dict_size);
}
(*ddict).dict_size = dict_size;
(*ddict).entropy.huf_table[0] = ZSTD_HUFFDTABLE_CAPACITY_LOG * 0x0100_0001;
load_entropy_into_ddict(ddict, dict_content_type)
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_DDict_dictContent(ddict: *const ZSTD_DDict) -> *const c_void {
debug_assert!(!ddict.is_null());
unsafe { (*ddict).dict_content }
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_DDict_dictSize(ddict: *const ZSTD_DDict) -> usize {
debug_assert!(!ddict.is_null());
unsafe { (*ddict).dict_size }
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_copyDDictParameters(dctx: *mut ZSTD_DCtx, ddict: *const ZSTD_DDict) {
debug_assert!(!dctx.is_null());
debug_assert!(!ddict.is_null());
unsafe {
let offsets = ddict_dctx_offsets();
let dict_content = (*ddict).dict_content;
let dict_end = if dict_content.is_null() {
ptr::null()
} else {
dict_content.cast::<u8>().add((*ddict).dict_size).cast()
};
write_dctx_field(dctx, offsets.dict_id, (*ddict).dict_id);
write_dctx_field(dctx, offsets.prefix_start, dict_content);
write_dctx_field(dctx, offsets.virtual_start, dict_content);
write_dctx_field(dctx, offsets.dict_end, dict_end);
write_dctx_field(dctx, offsets.previous_dst_end, dict_end);
copy_fuzzing_bounds(dctx, dict_content, dict_end);
if (*ddict).entropy_present != 0 {
write_dctx_field(dctx, offsets.lit_entropy, 1u32);
write_dctx_field(dctx, offsets.fse_entropy, 1u32);
write_dctx_field(
dctx,
offsets.llt_ptr,
(*ddict).entropy.ll_table.as_ptr().cast::<c_void>(),
);
write_dctx_field(
dctx,
offsets.mlt_ptr,
(*ddict).entropy.ml_table.as_ptr().cast::<c_void>(),
);
write_dctx_field(
dctx,
offsets.oft_ptr,
(*ddict).entropy.of_table.as_ptr().cast::<c_void>(),
);
write_dctx_field(
dctx,
offsets.huf_ptr,
(*ddict).entropy.huf_table.as_ptr().cast::<c_void>(),
);
for (index, value) in (*ddict).entropy.rep.iter().copied().enumerate() {
write_dctx_field(dctx, offsets.entropy_rep + index * size_of::<u32>(), value);
}
} else {
write_dctx_field(dctx, offsets.lit_entropy, 0u32);
write_dctx_field(dctx, offsets.fse_entropy, 0u32);
}
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_createDDict_advanced(
dict: *const c_void,
dict_size: usize,
dict_load_method: c_int,
dict_content_type: c_int,
custom_mem: ZSTD_customMem,
) -> *mut ZSTD_DDict {
if !valid_custom_mem(custom_mem) {
return ptr::null_mut();
}
let ddict = unsafe { custom_malloc(size_of::<ZSTD_DDict>(), custom_mem) }.cast::<ZSTD_DDict>();
if ddict.is_null() {
return ptr::null_mut();
}
unsafe {
(*ddict).c_mem = custom_mem;
if ERR_isError(init_ddict_internal(
ddict,
dict,
dict_size,
dict_load_method,
dict_content_type,
)) {
ZSTD_freeDDict(ddict);
return ptr::null_mut();
}
}
ddict
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_createDDict(
dict: *const c_void,
dict_size: usize,
) -> *mut ZSTD_DDict {
unsafe { ZSTD_createDDict_advanced(dict, dict_size, ZSTD_DLM_BY_COPY, 0, ZSTD_DEFAULT_CMEM) }
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_createDDict_byReference(
dict: *const c_void,
dict_size: usize,
) -> *mut ZSTD_DDict {
unsafe { ZSTD_createDDict_advanced(dict, dict_size, ZSTD_DLM_BY_REF, 0, ZSTD_DEFAULT_CMEM) }
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_initStaticDDict(
s_buffer: *mut c_void,
s_buffer_size: usize,
mut dict: *const c_void,
dict_size: usize,
dict_load_method: c_int,
dict_content_type: c_int,
) -> *const ZSTD_DDict {
debug_assert!(!s_buffer.is_null());
debug_assert!(!dict.is_null());
let copy_size = if dict_load_method == ZSTD_DLM_BY_REF {
0
} else {
dict_size
};
let Some(needed_space) = size_of::<ZSTD_DDict>().checked_add(copy_size) else {
return ptr::null();
};
if s_buffer.is_null()
|| dict.is_null()
|| (s_buffer as usize & 7) != 0
|| s_buffer_size < needed_space
{
return ptr::null();
}
let ddict = s_buffer.cast::<ZSTD_DDict>();
unsafe {
if dict_load_method == ZSTD_DLM_BY_COPY {
let local_copy = s_buffer.cast::<u8>().add(size_of::<ZSTD_DDict>());
ptr::copy_nonoverlapping(dict.cast::<u8>(), local_copy, dict_size);
dict = local_copy.cast();
}
if ERR_isError(init_ddict_internal(
ddict,
dict,
dict_size,
ZSTD_DLM_BY_REF,
dict_content_type,
)) {
return ptr::null();
}
}
ddict
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_freeDDict(ddict: *mut ZSTD_DDict) -> usize {
if ddict.is_null() {
return 0;
}
unsafe {
let c_mem = (*ddict).c_mem;
custom_free((*ddict).dict_buffer, c_mem);
custom_free(ddict.cast(), c_mem);
}
0
}
#[no_mangle]
pub extern "C" fn ZSTD_estimateDDictSize(dict_size: usize, dict_load_method: c_int) -> usize {
size_of::<ZSTD_DDict>().wrapping_add(if dict_load_method == ZSTD_DLM_BY_REF {
0
} else {
dict_size
})
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_sizeof_DDict(ddict: *const ZSTD_DDict) -> usize {
if ddict.is_null() {
return 0;
}
unsafe {
size_of::<ZSTD_DDict>().wrapping_add(if (*ddict).dict_buffer.is_null() {
0
} else {
(*ddict).dict_size
})
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_getDictID_fromDDict(ddict: *const ZSTD_DDict) -> c_uint {
if ddict.is_null() {
return 0;
}
unsafe { (*ddict).dict_id }
}
#[cfg(test)]
mod tests {
use super::*;
use std::mem::{align_of, offset_of};
use std::sync::atomic::{AtomicUsize, Ordering};
static ALLOCATIONS: AtomicUsize = AtomicUsize::new(0);
static FREES: AtomicUsize = AtomicUsize::new(0);
unsafe extern "C" fn counting_alloc(_opaque: *mut c_void, size: usize) -> *mut c_void {
ALLOCATIONS.fetch_add(1, Ordering::SeqCst);
unsafe { libc::malloc(size) }
}
unsafe extern "C" fn counting_free(_opaque: *mut c_void, allocation: *mut c_void) {
FREES.fetch_add(1, Ordering::SeqCst);
unsafe { libc::free(allocation) };
}
fn custom_mem() -> ZSTD_customMem {
ZSTD_customMem {
custom_alloc: Some(counting_alloc),
custom_free: Some(counting_free),
opaque: ptr::null_mut(),
}
}
unsafe fn read_dctx_field<T: Copy>(dctx: &[u8], offset: usize) -> T {
unsafe { dctx.as_ptr().add(offset).cast::<T>().read_unaligned() }
}
#[test]
fn c_layout_matches_ddict_and_decoder_headers() {
assert_eq!(size_of::<ZSTD_seqSymbol>(), 8);
assert_eq!(align_of::<ZSTD_seqSymbol>(), 4);
assert_eq!(size_of::<ZSTD_entropyDTables>(), 27_292);
assert_eq!(align_of::<ZSTD_entropyDTables>(), 4);
assert_eq!(offset_of!(ZSTD_entropyDTables, of_table), 4_104);
assert_eq!(offset_of!(ZSTD_entropyDTables, ml_table), 6_160);
assert_eq!(offset_of!(ZSTD_entropyDTables, huf_table), 10_264);
assert_eq!(offset_of!(ZSTD_entropyDTables, rep), 26_652);
assert_eq!(offset_of!(ZSTD_entropyDTables, workspace), 26_664);
if size_of::<usize>() == 8 {
assert_eq!(size_of::<ZSTD_DDict>(), 27_352);
assert_eq!(align_of::<ZSTD_DDict>(), 8);
assert_eq!(offset_of!(ZSTD_DDict, entropy), 24);
assert_eq!(offset_of!(ZSTD_DDict, dict_id), 27_316);
assert_eq!(offset_of!(ZSTD_DDict, c_mem), 27_328);
}
}
#[test]
fn copied_and_referenced_raw_dictionaries_preserve_lifetime_contracts() {
let source = b"raw dictionary contents";
unsafe {
let copied = ZSTD_createDDict(source.as_ptr().cast(), source.len());
assert!(!copied.is_null());
assert_ne!(ZSTD_DDict_dictContent(copied), source.as_ptr().cast());
assert_eq!(ZSTD_DDict_dictSize(copied), source.len());
assert_eq!(ZSTD_getDictID_fromDDict(copied), 0);
assert_eq!(
ZSTD_sizeof_DDict(copied),
size_of::<ZSTD_DDict>() + source.len()
);
assert_eq!(ZSTD_freeDDict(copied), 0);
let referenced = ZSTD_createDDict_byReference(source.as_ptr().cast(), source.len());
assert!(!referenced.is_null());
assert_eq!(ZSTD_DDict_dictContent(referenced), source.as_ptr().cast());
assert_eq!(ZSTD_DDict_dictSize(referenced), source.len());
assert_eq!(ZSTD_sizeof_DDict(referenced), size_of::<ZSTD_DDict>());
assert_eq!(ZSTD_freeDDict(referenced), 0);
}
}
#[test]
fn custom_allocator_pairs_object_and_copied_dictionary_frees() {
ALLOCATIONS.store(0, Ordering::SeqCst);
FREES.store(0, Ordering::SeqCst);
let source = b"custom allocator dictionary";
unsafe {
let ddict = ZSTD_createDDict_advanced(
source.as_ptr().cast(),
source.len(),
ZSTD_DLM_BY_COPY,
ZSTD_DCT_RAW_CONTENT,
custom_mem(),
);
assert!(!ddict.is_null());
assert_eq!(ALLOCATIONS.load(Ordering::SeqCst), 2);
assert_eq!(ZSTD_freeDDict(ddict), 0);
}
assert_eq!(FREES.load(Ordering::SeqCst), 2);
}
#[test]
fn invalid_custom_allocator_pair_and_forced_full_mode_fail() {
let source = b"not a formatted dictionary";
let invalid_mem = ZSTD_customMem {
custom_alloc: Some(counting_alloc),
custom_free: None,
opaque: ptr::null_mut(),
};
unsafe {
assert!(ZSTD_createDDict_advanced(
source.as_ptr().cast(),
source.len(),
ZSTD_DLM_BY_COPY,
0,
invalid_mem,
)
.is_null());
assert!(ZSTD_createDDict_advanced(
source.as_ptr().cast(),
source.len(),
ZSTD_DLM_BY_COPY,
ZSTD_DCT_FULL_DICT,
ZSTD_DEFAULT_CMEM,
)
.is_null());
}
}
#[test]
fn static_ddicts_copy_or_reference_exactly_as_requested() {
let source = b"static raw dictionary";
let needed = ZSTD_estimateDDictSize(source.len(), ZSTD_DLM_BY_COPY);
let mut storage = vec![0usize; needed.div_ceil(size_of::<usize>())];
let storage_ptr = storage.as_mut_ptr().cast::<c_void>();
unsafe {
let copied = ZSTD_initStaticDDict(
storage_ptr,
storage.len() * size_of::<usize>(),
source.as_ptr().cast(),
source.len(),
ZSTD_DLM_BY_COPY,
ZSTD_DCT_RAW_CONTENT,
);
assert_eq!(copied.cast_mut().cast::<c_void>(), storage_ptr);
let copied_contents = ZSTD_DDict_dictContent(copied);
assert_eq!(
copied_contents,
storage
.as_mut_ptr()
.cast::<u8>()
.add(size_of::<ZSTD_DDict>())
.cast()
);
assert_eq!(
std::slice::from_raw_parts(copied_contents.cast::<u8>(), source.len()),
source
);
assert_eq!(ZSTD_sizeof_DDict(copied), size_of::<ZSTD_DDict>());
assert!(ZSTD_initStaticDDict(
storage_ptr.cast::<u8>().add(1).cast(),
storage.len() * size_of::<usize>() - 1,
source.as_ptr().cast(),
source.len(),
ZSTD_DLM_BY_COPY,
ZSTD_DCT_RAW_CONTENT,
)
.is_null());
assert!(ZSTD_initStaticDDict(
storage_ptr,
needed - 1,
source.as_ptr().cast(),
source.len(),
ZSTD_DLM_BY_COPY,
ZSTD_DCT_RAW_CONTENT,
)
.is_null());
let referenced = ZSTD_initStaticDDict(
storage_ptr,
size_of::<ZSTD_DDict>(),
source.as_ptr().cast(),
source.len(),
ZSTD_DLM_BY_REF,
ZSTD_DCT_RAW_CONTENT,
);
assert_eq!(ZSTD_DDict_dictContent(referenced), source.as_ptr().cast());
assert_eq!(ZSTD_sizeof_DDict(referenced), size_of::<ZSTD_DDict>());
}
}
#[test]
fn empty_dictionary_and_size_estimate_follow_c_api_rules() {
unsafe {
let empty = ZSTD_createDDict(ptr::null(), 123);
assert!(!empty.is_null());
assert!(ZSTD_DDict_dictContent(empty).is_null());
assert_eq!(ZSTD_DDict_dictSize(empty), 0);
assert_eq!(ZSTD_freeDDict(empty), 0);
assert_eq!(ZSTD_freeDDict(ptr::null_mut()), 0);
}
assert_eq!(
ZSTD_estimateDDictSize(19, ZSTD_DLM_BY_REF),
size_of::<ZSTD_DDict>()
);
assert_eq!(
ZSTD_estimateDDictSize(19, ZSTD_DLM_BY_COPY),
size_of::<ZSTD_DDict>() + 19
);
}
#[test]
fn copying_ddict_parameters_sets_content_window_and_entropy_views() {
let content = b"dictionary window";
let mut ddict = unsafe { std::mem::MaybeUninit::<ZSTD_DDict>::zeroed().assume_init() };
ddict.dict_content = content.as_ptr().cast();
ddict.dict_size = content.len();
ddict.dict_id = 0xA1B2_C3D4;
ddict.entropy_present = 1;
ddict.entropy.rep = [1, 4, 8];
let offsets = unsafe { ddict_dctx_offsets() };
let mut dctx = [0u8; 128];
unsafe {
ZSTD_copyDDictParameters(dctx.as_mut_ptr().cast(), ptr::from_ref(&ddict));
}
assert_eq!(
unsafe { read_dctx_field::<u32>(&dctx, offsets.dict_id) },
ddict.dict_id
);
assert_eq!(
unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.prefix_start) },
ddict.dict_content
);
assert_eq!(
unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.virtual_start) },
ddict.dict_content
);
assert_eq!(
unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.dict_end) },
unsafe { ddict.dict_content.cast::<u8>().add(content.len()).cast() }
);
assert_eq!(
unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.previous_dst_end) },
unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.dict_end) }
);
assert_eq!(
unsafe { read_dctx_field::<u32>(&dctx, offsets.lit_entropy) },
1
);
assert_eq!(
unsafe { read_dctx_field::<u32>(&dctx, offsets.fse_entropy) },
1
);
assert_eq!(
unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.llt_ptr) },
ddict.entropy.ll_table.as_ptr().cast()
);
assert_eq!(
unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.mlt_ptr) },
ddict.entropy.ml_table.as_ptr().cast()
);
assert_eq!(
unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.oft_ptr) },
ddict.entropy.of_table.as_ptr().cast()
);
assert_eq!(
unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.huf_ptr) },
ddict.entropy.huf_table.as_ptr().cast()
);
assert_eq!(
[
unsafe { read_dctx_field::<u32>(&dctx, offsets.entropy_rep) },
unsafe { read_dctx_field::<u32>(&dctx, offsets.entropy_rep + 4) },
unsafe { read_dctx_field::<u32>(&dctx, offsets.entropy_rep + 8) },
],
[1, 4, 8]
);
let empty = unsafe { ZSTD_createDDict(ptr::null(), 0) };
assert!(!empty.is_null());
unsafe {
ZSTD_copyDDictParameters(dctx.as_mut_ptr().cast(), empty);
assert_eq!(ZSTD_freeDDict(empty), 0);
}
assert!(unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.prefix_start) }.is_null());
assert!(
unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.virtual_start) }.is_null()
);
assert!(unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.dict_end) }.is_null());
assert!(
unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.previous_dst_end) }.is_null()
);
assert_eq!(
unsafe { read_dctx_field::<u32>(&dctx, offsets.lit_entropy) },
0
);
assert_eq!(
unsafe { read_dctx_field::<u32>(&dctx, offsets.fse_entropy) },
0
);
}
#[test]
fn fuzzing_bounds_writer_uses_c_provided_offsets() {
let mut dctx = [0u8; 64];
let dictionary = b"fuzzing dictionary";
let begin = dictionary.as_ptr().cast::<c_void>();
let end = unsafe { begin.cast::<u8>().add(dictionary.len()).cast::<c_void>() };
unsafe {
copy_fuzzing_bounds_at_offsets(dctx.as_mut_ptr().cast(), begin, end, 16, 32);
assert_eq!(
dctx.as_ptr()
.add(16)
.cast::<*const c_void>()
.read_unaligned(),
begin
);
assert_eq!(
dctx.as_ptr()
.add(32)
.cast::<*const c_void>()
.read_unaligned(),
end
);
copy_fuzzing_bounds_at_offsets(
dctx.as_mut_ptr().cast(),
end,
begin,
ZSTD_DDICT_NO_FUZZING_OFFSET,
ZSTD_DDICT_NO_FUZZING_OFFSET,
);
assert_eq!(
dctx.as_ptr()
.add(16)
.cast::<*const c_void>()
.read_unaligned(),
begin
);
assert_eq!(
dctx.as_ptr()
.add(32)
.cast::<*const c_void>()
.read_unaligned(),
end
);
}
}
}
Reference in New Issue View Git Blame Copy Permalink