From ca70ca806181992635e129e1244c054ee79bcedc Mon Sep 17 00:00:00 2001 From: ddidderr Date: Fri, 10 Jul 2026 21:20:50 +0200 Subject: [PATCH] feat(rust): port decode dictionaries Move ZSTD_DDict allocation, ownership, construction, and public ABI exports into Rust. The C decoder keeps ZSTD_loadDEntropy until its table loader moves. The C shim now exposes C-computed DCtx field offsets. This keeps dictionary state correct across conditional C layouts, including fuzz fields and 32-bit static-BMI2 builds, without duplicating decoder-context layout in Rust. Test Plan: - cargo fmt, cargo clippy, cargo clippy --benches, cargo clippy --tests - cargo test --all-targets and cargo build --release - cargo test/build --target i686-unknown-linux-gnu - C fuzzer, zstreamtest, invalidDictionaries, and fuzzer32 smoke runs - fuzz-enabled and i686 -mbmi2 C/Rust dictionary roundtrip harnesses Refs: rust/README.md --- lib/decompress/zstd_ddict.c | 259 +---------- rust/README.md | 10 +- rust/src/lib.rs | 1 + rust/src/zstd_ddict.rs | 896 ++++++++++++++++++++++++++++++++++++ 4 files changed, 927 insertions(+), 239 deletions(-) create mode 100644 rust/src/zstd_ddict.rs diff --git a/lib/decompress/zstd_ddict.c b/lib/decompress/zstd_ddict.c index 309ec0d03..96cc1f2e9 100644 --- a/lib/decompress/zstd_ddict.c +++ b/lib/decompress/zstd_ddict.c @@ -1,244 +1,33 @@ /* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * All rights reserved. + * ZSTD_DDict implementation moved to rust/src/zstd_ddict.rs. * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. + * Keep this translation unit in the original source list so the C build keeps + * the same header configuration while the Rust static library provides the + * exported symbols. The metadata below is deliberately compiled with the C + * decoder's exact conditional layout: Rust uses it to fill the opaque DCtx + * fields without duplicating optional C layout choices such as DYNAMIC_BMI2. */ - -/* zstd_ddict.c : - * concentrates all logic that needs to know the internals of ZSTD_DDict object */ - -/*-******************************************************* -* Dependencies -*********************************************************/ -#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */ -#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ -#include "../common/cpu.h" /* bmi2 */ -#include "../common/mem.h" /* low level memory routines */ -#define FSE_STATIC_LINKING_ONLY -#include "../common/fse.h" -#include "../common/huf.h" +#define ZSTD_STATIC_LINKING_ONLY #include "zstd_decompress_internal.h" #include "zstd_ddict.h" -#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) -# include "../legacy/zstd_legacy.h" -#endif +const size_t ZSTD_rust_ddict_llt_ptr_offset = offsetof(struct ZSTD_DCtx_s, LLTptr); +const size_t ZSTD_rust_ddict_mlt_ptr_offset = offsetof(struct ZSTD_DCtx_s, MLTptr); +const size_t ZSTD_rust_ddict_oft_ptr_offset = offsetof(struct ZSTD_DCtx_s, OFTptr); +const size_t ZSTD_rust_ddict_huf_ptr_offset = offsetof(struct ZSTD_DCtx_s, HUFptr); +const size_t ZSTD_rust_ddict_entropy_rep_offset = offsetof(struct ZSTD_DCtx_s, entropy.rep); +const size_t ZSTD_rust_ddict_previous_dst_end_offset = offsetof(struct ZSTD_DCtx_s, previousDstEnd); +const size_t ZSTD_rust_ddict_prefix_start_offset = offsetof(struct ZSTD_DCtx_s, prefixStart); +const size_t ZSTD_rust_ddict_virtual_start_offset = offsetof(struct ZSTD_DCtx_s, virtualStart); +const size_t ZSTD_rust_ddict_dict_end_offset = offsetof(struct ZSTD_DCtx_s, dictEnd); +const size_t ZSTD_rust_ddict_lit_entropy_offset = offsetof(struct ZSTD_DCtx_s, litEntropy); +const size_t ZSTD_rust_ddict_fse_entropy_offset = offsetof(struct ZSTD_DCtx_s, fseEntropy); +const size_t ZSTD_rust_ddict_dict_id_offset = offsetof(struct ZSTD_DCtx_s, dictID); - - -/*-******************************************************* -* Types -*********************************************************/ -struct ZSTD_DDict_s { - void* dictBuffer; - const void* dictContent; - size_t dictSize; - ZSTD_entropyDTables_t entropy; - U32 dictID; - U32 entropyPresent; - ZSTD_customMem cMem; -}; /* typedef'd to ZSTD_DDict within "zstd.h" */ - -const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict) -{ - assert(ddict != NULL); - return ddict->dictContent; -} - -size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict) -{ - assert(ddict != NULL); - return ddict->dictSize; -} - -void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) -{ - DEBUGLOG(4, "ZSTD_copyDDictParameters"); - assert(dctx != NULL); - assert(ddict != NULL); - dctx->dictID = ddict->dictID; - dctx->prefixStart = ddict->dictContent; - dctx->virtualStart = ddict->dictContent; - dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; - dctx->previousDstEnd = dctx->dictEnd; #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - dctx->dictContentBeginForFuzzing = dctx->prefixStart; - dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +const size_t ZSTD_rust_ddict_fuzz_begin_offset = offsetof(struct ZSTD_DCtx_s, dictContentBeginForFuzzing); +const size_t ZSTD_rust_ddict_fuzz_end_offset = offsetof(struct ZSTD_DCtx_s, dictContentEndForFuzzing); +#else +const size_t ZSTD_rust_ddict_fuzz_begin_offset = (size_t)-1; +const size_t ZSTD_rust_ddict_fuzz_end_offset = (size_t)-1; #endif - if (ddict->entropyPresent) { - dctx->litEntropy = 1; - dctx->fseEntropy = 1; - dctx->LLTptr = ddict->entropy.LLTable; - dctx->MLTptr = ddict->entropy.MLTable; - dctx->OFTptr = ddict->entropy.OFTable; - dctx->HUFptr = ddict->entropy.hufTable; - dctx->entropy.rep[0] = ddict->entropy.rep[0]; - dctx->entropy.rep[1] = ddict->entropy.rep[1]; - dctx->entropy.rep[2] = ddict->entropy.rep[2]; - } else { - dctx->litEntropy = 0; - dctx->fseEntropy = 0; - } -} - - -static size_t -ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict, - ZSTD_dictContentType_e dictContentType) -{ - ddict->dictID = 0; - ddict->entropyPresent = 0; - if (dictContentType == ZSTD_dct_rawContent) return 0; - - if (ddict->dictSize < 8) { - if (dictContentType == ZSTD_dct_fullDict) - return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ - return 0; /* pure content mode */ - } - { U32 const magic = MEM_readLE32(ddict->dictContent); - if (magic != ZSTD_MAGIC_DICTIONARY) { - if (dictContentType == ZSTD_dct_fullDict) - return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ - return 0; /* pure content mode */ - } - } - ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); - - /* load entropy tables */ - RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy( - &ddict->entropy, ddict->dictContent, ddict->dictSize)), - dictionary_corrupted, ""); - ddict->entropyPresent = 1; - return 0; -} - - -static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType) -{ - if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { - ddict->dictBuffer = NULL; - ddict->dictContent = dict; - if (!dict) dictSize = 0; - } else { - void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem); - ddict->dictBuffer = internalBuffer; - ddict->dictContent = internalBuffer; - if (!internalBuffer) return ERROR(memory_allocation); - ZSTD_memcpy(internalBuffer, dict, dictSize); - } - ddict->dictSize = dictSize; - ddict->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ - - /* parse dictionary content */ - FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); - - return 0; -} - -ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - ZSTD_customMem customMem) -{ - if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; - - { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem); - if (ddict == NULL) return NULL; - ddict->cMem = customMem; - { size_t const initResult = ZSTD_initDDict_internal(ddict, - dict, dictSize, - dictLoadMethod, dictContentType); - if (ZSTD_isError(initResult)) { - ZSTD_freeDDict(ddict); - return NULL; - } } - return ddict; - } -} - -/*! ZSTD_createDDict() : -* Create a digested dictionary, to start decompression without startup delay. -* `dict` content is copied inside DDict. -* Consequently, `dict` can be released after `ZSTD_DDict` creation */ -ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) -{ - ZSTD_customMem const allocator = { NULL, NULL, NULL }; - return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator); -} - -/*! ZSTD_createDDict_byReference() : - * Create a digested dictionary, to start decompression without startup delay. - * Dictionary content is simply referenced, it will be accessed during decompression. - * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ -ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) -{ - ZSTD_customMem const allocator = { NULL, NULL, NULL }; - return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator); -} - - -const ZSTD_DDict* ZSTD_initStaticDDict( - void* sBuffer, size_t sBufferSize, - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType) -{ - size_t const neededSpace = sizeof(ZSTD_DDict) - + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); - ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer; - assert(sBuffer != NULL); - assert(dict != NULL); - if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */ - if (sBufferSize < neededSpace) return NULL; - if (dictLoadMethod == ZSTD_dlm_byCopy) { - ZSTD_memcpy(ddict+1, dict, dictSize); /* local copy */ - dict = ddict+1; - } - if (ZSTD_isError( ZSTD_initDDict_internal(ddict, - dict, dictSize, - ZSTD_dlm_byRef, dictContentType) )) - return NULL; - return ddict; -} - - -size_t ZSTD_freeDDict(ZSTD_DDict* ddict) -{ - if (ddict==NULL) return 0; /* support free on NULL */ - { ZSTD_customMem const cMem = ddict->cMem; - ZSTD_customFree(ddict->dictBuffer, cMem); - ZSTD_customFree(ddict, cMem); - return 0; - } -} - -/*! ZSTD_estimateDDictSize() : - * Estimate amount of memory that will be needed to create a dictionary for decompression. - * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ -size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) -{ - return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); -} - -size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) -{ - if (ddict==NULL) return 0; /* support sizeof on NULL */ - return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ; -} - -/*! ZSTD_getDictID_fromDDict() : - * Provides the dictID of the dictionary loaded into `ddict`. - * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. - * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) -{ - if (ddict==NULL) return 0; - return ddict->dictID; -} diff --git a/rust/README.md b/rust/README.md index 3e9be4d08..083353e3b 100644 --- a/rust/README.md +++ b/rust/README.md @@ -29,11 +29,13 @@ zstd ABI: - Runtime support - `threading` provides platform pthread wrappers required by zstd headers. - `pool` implements the bounded worker pool used by multithreaded compression. +- Dictionary support + - `zstd_ddict` owns, loads, copies, and references decode dictionaries. -The remaining block compression, general decompression, dictionary, legacy, -and CLI translation units are still C. They must move before the rewrite is -complete. Keeping that boundary explicit prevents a passing hybrid build from -being mistaken for the final all-Rust result. +The remaining block compression, general decompression, dictionary-building, +legacy, and CLI translation units are still C. They must move before the +rewrite is complete. Keeping that boundary explicit prevents a passing hybrid +build from being mistaken for the final all-Rust result. ## Compatibility boundary diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 68f013b86..288cbc342 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -16,3 +16,4 @@ pub mod pool; pub mod threading; pub mod xxhash; pub mod zstd_common; +pub mod zstd_ddict; diff --git a/rust/src/zstd_ddict.rs b/rust/src/zstd_ddict.rs new file mode 100644 index 000000000..021fa635d --- /dev/null +++ b/rust/src/zstd_ddict.rs @@ -0,0 +1,896 @@ +#![allow(non_snake_case)] + +//! Digested decompression dictionaries (`ZSTD_DDict`). +//! +//! The decoder proper is still implemented in C during the migration. In +//! particular, its `ZSTD_loadDEntropy()` helper owns the format-specific +//! construction of the decoder tables. This module owns the DDict object, +//! its allocation and lifetime, and passes an ABI-identical table layout to +//! that helper. + +use crate::common::{LL_FSE_LOG, ML_FSE_LOG, OFF_FSE_LOG, ZSTD_FRAMEIDSIZE, ZSTD_REP_NUM}; +use crate::errors::{ERR_isError, ZstdErrorCode, ERROR}; +use crate::mem::MEM_readLE32; +use std::ffi::c_void; +use std::mem::size_of; +use std::os::raw::{c_int, c_uint}; +use std::ptr; + +const ZSTD_MAGIC_DICTIONARY: u32 = 0xEC30_A437; +const ZSTD_HUFFDTABLE_CAPACITY_LOG: u32 = 12; +const HUF_DTABLE_SIZE: usize = 1 + (1 << ZSTD_HUFFDTABLE_CAPACITY_LOG); +const ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32: usize = 157; + +const ZSTD_DLM_BY_COPY: c_int = 0; +const ZSTD_DLM_BY_REF: c_int = 1; +const ZSTD_DCT_RAW_CONTENT: c_int = 1; +const ZSTD_DCT_FULL_DICT: c_int = 2; +const ZSTD_DDICT_NO_FUZZING_OFFSET: usize = usize::MAX; + +type ZstdAllocFunction = unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void; +type ZstdFreeFunction = unsafe extern "C" fn(*mut c_void, *mut c_void); + +/// ABI-compatible representation of `ZSTD_customMem` from `zstd.h`. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ZSTD_customMem { + custom_alloc: Option, + custom_free: Option, + opaque: *mut c_void, +} + +const ZSTD_DEFAULT_CMEM: ZSTD_customMem = ZSTD_customMem { + custom_alloc: None, + custom_free: None, + opaque: ptr::null_mut(), +}; + +#[repr(C)] +#[derive(Clone, Copy)] +struct ZSTD_seqSymbol { + next_state: u16, + nb_additional_bits: u8, + nb_bits: u8, + base_value: u32, +} + +#[repr(C)] +struct ZSTD_entropyDTables { + ll_table: [ZSTD_seqSymbol; 1 + (1 << LL_FSE_LOG)], + of_table: [ZSTD_seqSymbol; 1 + (1 << OFF_FSE_LOG)], + ml_table: [ZSTD_seqSymbol; 1 + (1 << ML_FSE_LOG)], + huf_table: [u32; HUF_DTABLE_SIZE], + rep: [u32; ZSTD_REP_NUM], + workspace: [u32; ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32], +} + +/// The opaque public `ZSTD_DDict` object, with its C-private representation. +#[repr(C)] +pub struct ZSTD_DDict { + dict_buffer: *mut c_void, + dict_content: *const c_void, + dict_size: usize, + entropy: ZSTD_entropyDTables, + dict_id: u32, + entropy_present: u32, + c_mem: ZSTD_customMem, +} + +/// Opaque C decoder context. Callers continue to create and own contexts in +/// the C API. +#[repr(C)] +pub struct ZSTD_DCtx { + _private: [u8; 0], +} + +#[derive(Clone, Copy)] +struct ZstdDctxOffsets { + llt_ptr: usize, + mlt_ptr: usize, + oft_ptr: usize, + huf_ptr: usize, + entropy_rep: usize, + previous_dst_end: usize, + prefix_start: usize, + virtual_start: usize, + dict_end: usize, + lit_entropy: usize, + fse_entropy: usize, + dict_id: usize, + fuzz_begin: usize, + fuzz_end: usize, +} + +#[cfg(not(test))] +unsafe extern "C" { + fn ZSTD_loadDEntropy( + entropy: *mut ZSTD_entropyDTables, + dict: *const c_void, + dict_size: usize, + ) -> usize; + + /* + * These are emitted by zstd_ddict.c with the exact C preprocessor state + * used for struct ZSTD_DCtx_s. Keeping the opaque context as C-provided + * offsets avoids duplicating optional fields such as DYNAMIC_BMI2 in Rust. + */ + static ZSTD_rust_ddict_llt_ptr_offset: usize; + static ZSTD_rust_ddict_mlt_ptr_offset: usize; + static ZSTD_rust_ddict_oft_ptr_offset: usize; + static ZSTD_rust_ddict_huf_ptr_offset: usize; + static ZSTD_rust_ddict_entropy_rep_offset: usize; + static ZSTD_rust_ddict_previous_dst_end_offset: usize; + static ZSTD_rust_ddict_prefix_start_offset: usize; + static ZSTD_rust_ddict_virtual_start_offset: usize; + static ZSTD_rust_ddict_dict_end_offset: usize; + static ZSTD_rust_ddict_lit_entropy_offset: usize; + static ZSTD_rust_ddict_fse_entropy_offset: usize; + static ZSTD_rust_ddict_dict_id_offset: usize; + static ZSTD_rust_ddict_fuzz_begin_offset: usize; + static ZSTD_rust_ddict_fuzz_end_offset: usize; +} + +#[inline] +unsafe fn ddict_dctx_offsets() -> ZstdDctxOffsets { + #[cfg(not(test))] + { + unsafe { + ZstdDctxOffsets { + llt_ptr: ZSTD_rust_ddict_llt_ptr_offset, + mlt_ptr: ZSTD_rust_ddict_mlt_ptr_offset, + oft_ptr: ZSTD_rust_ddict_oft_ptr_offset, + huf_ptr: ZSTD_rust_ddict_huf_ptr_offset, + entropy_rep: ZSTD_rust_ddict_entropy_rep_offset, + previous_dst_end: ZSTD_rust_ddict_previous_dst_end_offset, + prefix_start: ZSTD_rust_ddict_prefix_start_offset, + virtual_start: ZSTD_rust_ddict_virtual_start_offset, + dict_end: ZSTD_rust_ddict_dict_end_offset, + lit_entropy: ZSTD_rust_ddict_lit_entropy_offset, + fse_entropy: ZSTD_rust_ddict_fse_entropy_offset, + dict_id: ZSTD_rust_ddict_dict_id_offset, + fuzz_begin: ZSTD_rust_ddict_fuzz_begin_offset, + fuzz_end: ZSTD_rust_ddict_fuzz_end_offset, + } + } + } + + #[cfg(test)] + { + ZstdDctxOffsets { + llt_ptr: 0, + mlt_ptr: size_of::<*const c_void>(), + oft_ptr: 2 * size_of::<*const c_void>(), + huf_ptr: 3 * size_of::<*const c_void>(), + entropy_rep: 4 * size_of::<*const c_void>(), + previous_dst_end: 4 * size_of::<*const c_void>() + 3 * size_of::(), + prefix_start: 5 * size_of::<*const c_void>() + 3 * size_of::(), + virtual_start: 6 * size_of::<*const c_void>() + 3 * size_of::(), + dict_end: 7 * size_of::<*const c_void>() + 3 * size_of::(), + lit_entropy: 8 * size_of::<*const c_void>() + 3 * size_of::(), + fse_entropy: 8 * size_of::<*const c_void>() + 4 * size_of::(), + dict_id: 8 * size_of::<*const c_void>() + 5 * size_of::(), + fuzz_begin: ZSTD_DDICT_NO_FUZZING_OFFSET, + fuzz_end: ZSTD_DDICT_NO_FUZZING_OFFSET, + } + } +} + +unsafe fn write_dctx_field(dctx: *mut ZSTD_DCtx, offset: usize, value: T) { + unsafe { + dctx.cast::() + .add(offset) + .cast::() + .write_unaligned(value); + } +} + +unsafe fn copy_fuzzing_bounds_at_offsets( + dctx: *mut ZSTD_DCtx, + begin: *const c_void, + end: *const c_void, + begin_offset: usize, + end_offset: usize, +) { + if begin_offset == ZSTD_DDICT_NO_FUZZING_OFFSET { + debug_assert_eq!(end_offset, ZSTD_DDICT_NO_FUZZING_OFFSET); + return; + } + debug_assert_ne!(end_offset, ZSTD_DDICT_NO_FUZZING_OFFSET); + + unsafe { + let dctx = dctx.cast::(); + dctx.add(begin_offset) + .cast::<*const c_void>() + .write_unaligned(begin); + dctx.add(end_offset) + .cast::<*const c_void>() + .write_unaligned(end); + } +} + +unsafe fn copy_fuzzing_bounds(dctx: *mut ZSTD_DCtx, begin: *const c_void, end: *const c_void) { + let offsets = unsafe { ddict_dctx_offsets() }; + unsafe { + copy_fuzzing_bounds_at_offsets(dctx, begin, end, offsets.fuzz_begin, offsets.fuzz_end) + }; +} + +#[inline] +unsafe fn load_d_entropy( + entropy: *mut ZSTD_entropyDTables, + dict: *const c_void, + dict_size: usize, +) -> usize { + #[cfg(not(test))] + { + // The existing C decoder owns this helper until its translation unit is + // moved. Its argument is the ABI-compatible table storage above. + unsafe { ZSTD_loadDEntropy(entropy, dict, dict_size) } + } + + #[cfg(test)] + { + let _ = (entropy, dict, dict_size); + // Unit tests exercise the DDict ownership and raw-content paths. A + // full dictionary is covered by the C compatibility executables, where + // the decoder's real ZSTD_loadDEntropy() is linked in. + ERROR(ZstdErrorCode::DictionaryCorrupted) + } +} + +#[inline] +unsafe fn custom_malloc(size: usize, c_mem: ZSTD_customMem) -> *mut c_void { + match c_mem.custom_alloc { + Some(alloc) => unsafe { alloc(c_mem.opaque, size) }, + None => unsafe { libc::malloc(size) }, + } +} + +#[inline] +unsafe fn custom_free(allocation: *mut c_void, c_mem: ZSTD_customMem) { + if allocation.is_null() { + return; + } + match c_mem.custom_free { + Some(free) => unsafe { free(c_mem.opaque, allocation) }, + None => unsafe { libc::free(allocation) }, + } +} + +#[inline] +fn valid_custom_mem(c_mem: ZSTD_customMem) -> bool { + c_mem.custom_alloc.is_some() == c_mem.custom_free.is_some() +} + +unsafe fn load_entropy_into_ddict(ddict: *mut ZSTD_DDict, dict_content_type: c_int) -> usize { + unsafe { + (*ddict).dict_id = 0; + (*ddict).entropy_present = 0; + if dict_content_type == ZSTD_DCT_RAW_CONTENT { + return 0; + } + + if (*ddict).dict_size < 8 { + return if dict_content_type == ZSTD_DCT_FULL_DICT { + ERROR(ZstdErrorCode::DictionaryCorrupted) + } else { + 0 + }; + } + + if MEM_readLE32((*ddict).dict_content) != ZSTD_MAGIC_DICTIONARY { + return if dict_content_type == ZSTD_DCT_FULL_DICT { + ERROR(ZstdErrorCode::DictionaryCorrupted) + } else { + 0 + }; + } + + (*ddict).dict_id = MEM_readLE32( + (*ddict) + .dict_content + .cast::() + .add(ZSTD_FRAMEIDSIZE) + .cast(), + ); + let result = load_d_entropy( + ptr::addr_of_mut!((*ddict).entropy), + (*ddict).dict_content, + (*ddict).dict_size, + ); + if ERR_isError(result) { + return ERROR(ZstdErrorCode::DictionaryCorrupted); + } + (*ddict).entropy_present = 1; + 0 + } +} + +unsafe fn init_ddict_internal( + ddict: *mut ZSTD_DDict, + dict: *const c_void, + mut dict_size: usize, + dict_load_method: c_int, + dict_content_type: c_int, +) -> usize { + unsafe { + if dict_load_method == ZSTD_DLM_BY_REF || dict.is_null() || dict_size == 0 { + (*ddict).dict_buffer = ptr::null_mut(); + (*ddict).dict_content = dict; + if dict.is_null() { + dict_size = 0; + } + } else { + let internal_buffer = custom_malloc(dict_size, (*ddict).c_mem); + (*ddict).dict_buffer = internal_buffer; + (*ddict).dict_content = internal_buffer.cast_const(); + if internal_buffer.is_null() { + return ERROR(ZstdErrorCode::MemoryAllocation); + } + ptr::copy_nonoverlapping(dict.cast::(), internal_buffer.cast::(), dict_size); + } + + (*ddict).dict_size = dict_size; + (*ddict).entropy.huf_table[0] = ZSTD_HUFFDTABLE_CAPACITY_LOG * 0x0100_0001; + load_entropy_into_ddict(ddict, dict_content_type) + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_DDict_dictContent(ddict: *const ZSTD_DDict) -> *const c_void { + debug_assert!(!ddict.is_null()); + unsafe { (*ddict).dict_content } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_DDict_dictSize(ddict: *const ZSTD_DDict) -> usize { + debug_assert!(!ddict.is_null()); + unsafe { (*ddict).dict_size } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_copyDDictParameters(dctx: *mut ZSTD_DCtx, ddict: *const ZSTD_DDict) { + debug_assert!(!dctx.is_null()); + debug_assert!(!ddict.is_null()); + + unsafe { + let offsets = ddict_dctx_offsets(); + let dict_content = (*ddict).dict_content; + let dict_end = if dict_content.is_null() { + ptr::null() + } else { + dict_content.cast::().add((*ddict).dict_size).cast() + }; + + write_dctx_field(dctx, offsets.dict_id, (*ddict).dict_id); + write_dctx_field(dctx, offsets.prefix_start, dict_content); + write_dctx_field(dctx, offsets.virtual_start, dict_content); + write_dctx_field(dctx, offsets.dict_end, dict_end); + write_dctx_field(dctx, offsets.previous_dst_end, dict_end); + copy_fuzzing_bounds(dctx, dict_content, dict_end); + + if (*ddict).entropy_present != 0 { + write_dctx_field(dctx, offsets.lit_entropy, 1u32); + write_dctx_field(dctx, offsets.fse_entropy, 1u32); + write_dctx_field( + dctx, + offsets.llt_ptr, + (*ddict).entropy.ll_table.as_ptr().cast::(), + ); + write_dctx_field( + dctx, + offsets.mlt_ptr, + (*ddict).entropy.ml_table.as_ptr().cast::(), + ); + write_dctx_field( + dctx, + offsets.oft_ptr, + (*ddict).entropy.of_table.as_ptr().cast::(), + ); + write_dctx_field( + dctx, + offsets.huf_ptr, + (*ddict).entropy.huf_table.as_ptr().cast::(), + ); + for (index, value) in (*ddict).entropy.rep.iter().copied().enumerate() { + write_dctx_field(dctx, offsets.entropy_rep + index * size_of::(), value); + } + } else { + write_dctx_field(dctx, offsets.lit_entropy, 0u32); + write_dctx_field(dctx, offsets.fse_entropy, 0u32); + } + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_createDDict_advanced( + dict: *const c_void, + dict_size: usize, + dict_load_method: c_int, + dict_content_type: c_int, + custom_mem: ZSTD_customMem, +) -> *mut ZSTD_DDict { + if !valid_custom_mem(custom_mem) { + return ptr::null_mut(); + } + + let ddict = unsafe { custom_malloc(size_of::(), custom_mem) }.cast::(); + if ddict.is_null() { + return ptr::null_mut(); + } + + unsafe { + (*ddict).c_mem = custom_mem; + if ERR_isError(init_ddict_internal( + ddict, + dict, + dict_size, + dict_load_method, + dict_content_type, + )) { + ZSTD_freeDDict(ddict); + return ptr::null_mut(); + } + } + ddict +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_createDDict( + dict: *const c_void, + dict_size: usize, +) -> *mut ZSTD_DDict { + unsafe { ZSTD_createDDict_advanced(dict, dict_size, ZSTD_DLM_BY_COPY, 0, ZSTD_DEFAULT_CMEM) } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_createDDict_byReference( + dict: *const c_void, + dict_size: usize, +) -> *mut ZSTD_DDict { + unsafe { ZSTD_createDDict_advanced(dict, dict_size, ZSTD_DLM_BY_REF, 0, ZSTD_DEFAULT_CMEM) } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_initStaticDDict( + s_buffer: *mut c_void, + s_buffer_size: usize, + mut dict: *const c_void, + dict_size: usize, + dict_load_method: c_int, + dict_content_type: c_int, +) -> *const ZSTD_DDict { + debug_assert!(!s_buffer.is_null()); + debug_assert!(!dict.is_null()); + + let copy_size = if dict_load_method == ZSTD_DLM_BY_REF { + 0 + } else { + dict_size + }; + let Some(needed_space) = size_of::().checked_add(copy_size) else { + return ptr::null(); + }; + + if s_buffer.is_null() + || dict.is_null() + || (s_buffer as usize & 7) != 0 + || s_buffer_size < needed_space + { + return ptr::null(); + } + + let ddict = s_buffer.cast::(); + unsafe { + if dict_load_method == ZSTD_DLM_BY_COPY { + let local_copy = s_buffer.cast::().add(size_of::()); + ptr::copy_nonoverlapping(dict.cast::(), local_copy, dict_size); + dict = local_copy.cast(); + } + if ERR_isError(init_ddict_internal( + ddict, + dict, + dict_size, + ZSTD_DLM_BY_REF, + dict_content_type, + )) { + return ptr::null(); + } + } + ddict +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_freeDDict(ddict: *mut ZSTD_DDict) -> usize { + if ddict.is_null() { + return 0; + } + + unsafe { + let c_mem = (*ddict).c_mem; + custom_free((*ddict).dict_buffer, c_mem); + custom_free(ddict.cast(), c_mem); + } + 0 +} + +#[no_mangle] +pub extern "C" fn ZSTD_estimateDDictSize(dict_size: usize, dict_load_method: c_int) -> usize { + size_of::().wrapping_add(if dict_load_method == ZSTD_DLM_BY_REF { + 0 + } else { + dict_size + }) +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_sizeof_DDict(ddict: *const ZSTD_DDict) -> usize { + if ddict.is_null() { + return 0; + } + unsafe { + size_of::().wrapping_add(if (*ddict).dict_buffer.is_null() { + 0 + } else { + (*ddict).dict_size + }) + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_getDictID_fromDDict(ddict: *const ZSTD_DDict) -> c_uint { + if ddict.is_null() { + return 0; + } + unsafe { (*ddict).dict_id } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::mem::{align_of, offset_of}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + static ALLOCATIONS: AtomicUsize = AtomicUsize::new(0); + static FREES: AtomicUsize = AtomicUsize::new(0); + + unsafe extern "C" fn counting_alloc(_opaque: *mut c_void, size: usize) -> *mut c_void { + ALLOCATIONS.fetch_add(1, Ordering::SeqCst); + unsafe { libc::malloc(size) } + } + + unsafe extern "C" fn counting_free(_opaque: *mut c_void, allocation: *mut c_void) { + FREES.fetch_add(1, Ordering::SeqCst); + unsafe { libc::free(allocation) }; + } + + fn custom_mem() -> ZSTD_customMem { + ZSTD_customMem { + custom_alloc: Some(counting_alloc), + custom_free: Some(counting_free), + opaque: ptr::null_mut(), + } + } + + unsafe fn read_dctx_field(dctx: &[u8], offset: usize) -> T { + unsafe { dctx.as_ptr().add(offset).cast::().read_unaligned() } + } + + #[test] + fn c_layout_matches_ddict_and_decoder_headers() { + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), 4); + assert_eq!(size_of::(), 27_292); + assert_eq!(align_of::(), 4); + assert_eq!(offset_of!(ZSTD_entropyDTables, of_table), 4_104); + assert_eq!(offset_of!(ZSTD_entropyDTables, ml_table), 6_160); + assert_eq!(offset_of!(ZSTD_entropyDTables, huf_table), 10_264); + assert_eq!(offset_of!(ZSTD_entropyDTables, rep), 26_652); + assert_eq!(offset_of!(ZSTD_entropyDTables, workspace), 26_664); + + if size_of::() == 8 { + assert_eq!(size_of::(), 27_352); + assert_eq!(align_of::(), 8); + assert_eq!(offset_of!(ZSTD_DDict, entropy), 24); + assert_eq!(offset_of!(ZSTD_DDict, dict_id), 27_316); + assert_eq!(offset_of!(ZSTD_DDict, c_mem), 27_328); + } + } + + #[test] + fn copied_and_referenced_raw_dictionaries_preserve_lifetime_contracts() { + let source = b"raw dictionary contents"; + unsafe { + let copied = ZSTD_createDDict(source.as_ptr().cast(), source.len()); + assert!(!copied.is_null()); + assert_ne!(ZSTD_DDict_dictContent(copied), source.as_ptr().cast()); + assert_eq!(ZSTD_DDict_dictSize(copied), source.len()); + assert_eq!(ZSTD_getDictID_fromDDict(copied), 0); + assert_eq!( + ZSTD_sizeof_DDict(copied), + size_of::() + source.len() + ); + assert_eq!(ZSTD_freeDDict(copied), 0); + + let referenced = ZSTD_createDDict_byReference(source.as_ptr().cast(), source.len()); + assert!(!referenced.is_null()); + assert_eq!(ZSTD_DDict_dictContent(referenced), source.as_ptr().cast()); + assert_eq!(ZSTD_DDict_dictSize(referenced), source.len()); + assert_eq!(ZSTD_sizeof_DDict(referenced), size_of::()); + assert_eq!(ZSTD_freeDDict(referenced), 0); + } + } + + #[test] + fn custom_allocator_pairs_object_and_copied_dictionary_frees() { + ALLOCATIONS.store(0, Ordering::SeqCst); + FREES.store(0, Ordering::SeqCst); + let source = b"custom allocator dictionary"; + unsafe { + let ddict = ZSTD_createDDict_advanced( + source.as_ptr().cast(), + source.len(), + ZSTD_DLM_BY_COPY, + ZSTD_DCT_RAW_CONTENT, + custom_mem(), + ); + assert!(!ddict.is_null()); + assert_eq!(ALLOCATIONS.load(Ordering::SeqCst), 2); + assert_eq!(ZSTD_freeDDict(ddict), 0); + } + assert_eq!(FREES.load(Ordering::SeqCst), 2); + } + + #[test] + fn invalid_custom_allocator_pair_and_forced_full_mode_fail() { + let source = b"not a formatted dictionary"; + let invalid_mem = ZSTD_customMem { + custom_alloc: Some(counting_alloc), + custom_free: None, + opaque: ptr::null_mut(), + }; + unsafe { + assert!(ZSTD_createDDict_advanced( + source.as_ptr().cast(), + source.len(), + ZSTD_DLM_BY_COPY, + 0, + invalid_mem, + ) + .is_null()); + assert!(ZSTD_createDDict_advanced( + source.as_ptr().cast(), + source.len(), + ZSTD_DLM_BY_COPY, + ZSTD_DCT_FULL_DICT, + ZSTD_DEFAULT_CMEM, + ) + .is_null()); + } + } + + #[test] + fn static_ddicts_copy_or_reference_exactly_as_requested() { + let source = b"static raw dictionary"; + let needed = ZSTD_estimateDDictSize(source.len(), ZSTD_DLM_BY_COPY); + let mut storage = vec![0usize; needed.div_ceil(size_of::())]; + let storage_ptr = storage.as_mut_ptr().cast::(); + + unsafe { + let copied = ZSTD_initStaticDDict( + storage_ptr, + storage.len() * size_of::(), + source.as_ptr().cast(), + source.len(), + ZSTD_DLM_BY_COPY, + ZSTD_DCT_RAW_CONTENT, + ); + assert_eq!(copied.cast_mut().cast::(), storage_ptr); + let copied_contents = ZSTD_DDict_dictContent(copied); + assert_eq!( + copied_contents, + storage + .as_mut_ptr() + .cast::() + .add(size_of::()) + .cast() + ); + assert_eq!( + std::slice::from_raw_parts(copied_contents.cast::(), source.len()), + source + ); + assert_eq!(ZSTD_sizeof_DDict(copied), size_of::()); + + assert!(ZSTD_initStaticDDict( + storage_ptr.cast::().add(1).cast(), + storage.len() * size_of::() - 1, + source.as_ptr().cast(), + source.len(), + ZSTD_DLM_BY_COPY, + ZSTD_DCT_RAW_CONTENT, + ) + .is_null()); + assert!(ZSTD_initStaticDDict( + storage_ptr, + needed - 1, + source.as_ptr().cast(), + source.len(), + ZSTD_DLM_BY_COPY, + ZSTD_DCT_RAW_CONTENT, + ) + .is_null()); + + let referenced = ZSTD_initStaticDDict( + storage_ptr, + size_of::(), + source.as_ptr().cast(), + source.len(), + ZSTD_DLM_BY_REF, + ZSTD_DCT_RAW_CONTENT, + ); + assert_eq!(ZSTD_DDict_dictContent(referenced), source.as_ptr().cast()); + assert_eq!(ZSTD_sizeof_DDict(referenced), size_of::()); + } + } + + #[test] + fn empty_dictionary_and_size_estimate_follow_c_api_rules() { + unsafe { + let empty = ZSTD_createDDict(ptr::null(), 123); + assert!(!empty.is_null()); + assert!(ZSTD_DDict_dictContent(empty).is_null()); + assert_eq!(ZSTD_DDict_dictSize(empty), 0); + assert_eq!(ZSTD_freeDDict(empty), 0); + assert_eq!(ZSTD_freeDDict(ptr::null_mut()), 0); + } + assert_eq!( + ZSTD_estimateDDictSize(19, ZSTD_DLM_BY_REF), + size_of::() + ); + assert_eq!( + ZSTD_estimateDDictSize(19, ZSTD_DLM_BY_COPY), + size_of::() + 19 + ); + } + + #[test] + fn copying_ddict_parameters_sets_content_window_and_entropy_views() { + let content = b"dictionary window"; + let mut ddict = unsafe { std::mem::MaybeUninit::::zeroed().assume_init() }; + ddict.dict_content = content.as_ptr().cast(); + ddict.dict_size = content.len(); + ddict.dict_id = 0xA1B2_C3D4; + ddict.entropy_present = 1; + ddict.entropy.rep = [1, 4, 8]; + let offsets = unsafe { ddict_dctx_offsets() }; + let mut dctx = [0u8; 128]; + + unsafe { + ZSTD_copyDDictParameters(dctx.as_mut_ptr().cast(), ptr::from_ref(&ddict)); + } + + assert_eq!( + unsafe { read_dctx_field::(&dctx, offsets.dict_id) }, + ddict.dict_id + ); + assert_eq!( + unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.prefix_start) }, + ddict.dict_content + ); + assert_eq!( + unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.virtual_start) }, + ddict.dict_content + ); + assert_eq!( + unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.dict_end) }, + unsafe { ddict.dict_content.cast::().add(content.len()).cast() } + ); + assert_eq!( + unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.previous_dst_end) }, + unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.dict_end) } + ); + assert_eq!( + unsafe { read_dctx_field::(&dctx, offsets.lit_entropy) }, + 1 + ); + assert_eq!( + unsafe { read_dctx_field::(&dctx, offsets.fse_entropy) }, + 1 + ); + assert_eq!( + unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.llt_ptr) }, + ddict.entropy.ll_table.as_ptr().cast() + ); + assert_eq!( + unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.mlt_ptr) }, + ddict.entropy.ml_table.as_ptr().cast() + ); + assert_eq!( + unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.oft_ptr) }, + ddict.entropy.of_table.as_ptr().cast() + ); + assert_eq!( + unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.huf_ptr) }, + ddict.entropy.huf_table.as_ptr().cast() + ); + assert_eq!( + [ + unsafe { read_dctx_field::(&dctx, offsets.entropy_rep) }, + unsafe { read_dctx_field::(&dctx, offsets.entropy_rep + 4) }, + unsafe { read_dctx_field::(&dctx, offsets.entropy_rep + 8) }, + ], + [1, 4, 8] + ); + + let empty = unsafe { ZSTD_createDDict(ptr::null(), 0) }; + assert!(!empty.is_null()); + unsafe { + ZSTD_copyDDictParameters(dctx.as_mut_ptr().cast(), empty); + assert_eq!(ZSTD_freeDDict(empty), 0); + } + assert!(unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.prefix_start) }.is_null()); + assert!( + unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.virtual_start) }.is_null() + ); + assert!(unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.dict_end) }.is_null()); + assert!( + unsafe { read_dctx_field::<*const c_void>(&dctx, offsets.previous_dst_end) }.is_null() + ); + assert_eq!( + unsafe { read_dctx_field::(&dctx, offsets.lit_entropy) }, + 0 + ); + assert_eq!( + unsafe { read_dctx_field::(&dctx, offsets.fse_entropy) }, + 0 + ); + } + + #[test] + fn fuzzing_bounds_writer_uses_c_provided_offsets() { + let mut dctx = [0u8; 64]; + let dictionary = b"fuzzing dictionary"; + let begin = dictionary.as_ptr().cast::(); + let end = unsafe { begin.cast::().add(dictionary.len()).cast::() }; + + unsafe { + copy_fuzzing_bounds_at_offsets(dctx.as_mut_ptr().cast(), begin, end, 16, 32); + assert_eq!( + dctx.as_ptr() + .add(16) + .cast::<*const c_void>() + .read_unaligned(), + begin + ); + assert_eq!( + dctx.as_ptr() + .add(32) + .cast::<*const c_void>() + .read_unaligned(), + end + ); + + copy_fuzzing_bounds_at_offsets( + dctx.as_mut_ptr().cast(), + end, + begin, + ZSTD_DDICT_NO_FUZZING_OFFSET, + ZSTD_DDICT_NO_FUZZING_OFFSET, + ); + assert_eq!( + dctx.as_ptr() + .add(16) + .cast::<*const c_void>() + .read_unaligned(), + begin + ); + assert_eq!( + dctx.as_ptr() + .add(32) + .cast::<*const c_void>() + .read_unaligned(), + end + ); + } + } +}