From 0dfe757d95de1b41833bf0f442a85bfc51dead12 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Sat, 18 Jul 2026 14:53:23 +0200 Subject: [PATCH] feat(dict-builder): remove internal Rust-to-C leaf calls `dict_builder_zdict.rs` still declared the compressed-block reset, dictionary entropy loader, and FastCOVER optimizer as C externs even though their Rust implementations were already linked into the archive. That left unnecessary Rust-to-C edges and kept a duplicate private FastCOVER `repr(C)` definition. Call the Rust reset and entropy leaves directly, import the Rust FastCOVER entry point, and use its canonical ABI parameter type. The eight opaque compression-context APIs used by entropy analysis remain C-owned for a later context port, so this change preserves the existing ownership boundary and public layouts. Test Plan: - `cargo check --manifest-path rust/Cargo.toml --no-default-features --features compression,dict-builder` -- passed. - Required `cargo clippy` runs for the library, benches, and tests, before and after nightly formatting -- passed. - `cargo +nightly fmt --manifest-path rust/Cargo.toml -- --check` and `git diff --cached --check` -- passed. - Rebuilt `libzstd_rs.a`; the old `ZSTD_loadCEntropy` and `ZSTD_reset_compressedBlockState` names are absent from its undefined symbols, while the Rust reset, entropy, and FastCOVER symbols are defined. - `make -C tests -B -j2 zstreamtest` -- linked successfully; only the pre-existing `tests/zstreamtest.c:1899` warning remains. - `make -C tests -j2 test-zstream ZSTREAM_TESTTIME=-T5s` -- passed 84 deterministic, 220 legacy-API randomized, and 404 new-API randomized cases. - `cargo test --manifest-path rust/Cargo.toml --no-default-features --features compression,dict-builder --lib` -- cannot link until the eight intentionally deferred opaque C compression-context symbols are ported. - `./tests/fuzzer -v -T5s` stopped at existing test 56 from unrelated concurrent compression edits before reaching dict-builder tests. --- rust/src/dict_builder_zdict.rs | 50 ++++++++-------------------------- 1 file changed, 11 insertions(+), 39 deletions(-) diff --git a/rust/src/dict_builder_zdict.rs b/rust/src/dict_builder_zdict.rs index 56f3cd67a..5488b2fed 100644 --- a/rust/src/dict_builder_zdict.rs +++ b/rust/src/dict_builder_zdict.rs @@ -14,15 +14,21 @@ use crate::bits::ZSTD_highbit32; use crate::common::{LL_FSE_LOG, MAX_LL, MAX_ML, ML_FSE_LOG, OFF_FSE_LOG, ZSTD_REP_NUM}; +use crate::dict_builder_fastcover::{ + ZDICT_fastCover_params_t, ZDICT_optimizeTrainFromBuffer_fastCover, +}; use crate::divsufsort::divsufsort; use crate::errors::{ERR_getErrorName, ERR_isError, ZstdErrorCode, ERROR}; use crate::fse_compress::{FSE_normalizeCount, FSE_writeNCount}; use crate::huf_compress::{HUF_buildCTable_wksp, HUF_writeCTable_wksp}; use crate::mem::{MEM_readLE32, MEM_writeLE32}; use crate::xxhash::XXH64; +use crate::zstd_compress_dictionary::ZSTD_rust_loadCEntropy; use crate::zstd_compress_params::{ZSTD_compressionParameters, ZSTD_parameters}; use crate::zstd_compress_sequences::SeqDef; -use crate::zstd_compress_stats::{SeqStore_t, ZSTD_compressedBlockState_t, ZSTD_seqToCodes}; +use crate::zstd_compress_stats::{ + SeqStore_t, ZSTD_compressedBlockState_t, ZSTD_rust_resetCompressedBlockState, ZSTD_seqToCodes, +}; use std::ffi::{c_char, c_void}; use std::mem::{size_of, MaybeUninit}; use std::os::raw::{c_int, c_uint}; @@ -68,25 +74,6 @@ pub struct ZDICT_legacy_params_t { pub zParams: ZDICT_params_t, } -/// ABI-compatible fastCOVER parameters used by `ZDICT_trainFromBuffer()`. -/// -/// The type is local to this module because the public fastCOVER declaration -/// is still provided by the C header and implementation. -#[repr(C)] -#[derive(Clone, Copy, Debug, Default)] -struct ZDICT_fastCover_params_t { - k: c_uint, - d: c_uint, - f: c_uint, - steps: c_uint, - nbThreads: c_uint, - splitPoint: f64, - accel: c_uint, - shrinkDict: c_uint, - shrinkDictMaxRegression: c_uint, - zParams: ZDICT_params_t, -} - type ZSTD_CCtx = c_void; type ZSTD_CDict = c_void; @@ -105,13 +92,6 @@ const ZSTD_DEFAULT_CMEM: ZSTD_customMem = ZSTD_customMem { }; unsafe extern "C" { - fn ZSTD_loadCEntropy( - bs: *mut ZSTD_compressedBlockState_t, - workspace: *mut c_void, - dict: *const c_void, - dict_size: usize, - ) -> usize; - fn ZSTD_reset_compressedBlockState(bs: *mut ZSTD_compressedBlockState_t); fn ZSTD_getParams( compression_level: c_int, estimated_src_size: u64, @@ -140,14 +120,6 @@ unsafe extern "C" { src_size: usize, ) -> usize; fn ZSTD_getSeqStore(cctx: *const ZSTD_CCtx) -> *const SeqStore_t; - fn ZDICT_optimizeTrainFromBuffer_fastCover( - dict_buffer: *mut c_void, - dict_buffer_capacity: usize, - samples_buffer: *const c_void, - samples_sizes: *const usize, - nb_samples: c_uint, - parameters: *mut ZDICT_fastCover_params_t, - ) -> usize; } #[inline] @@ -959,8 +931,8 @@ pub unsafe extern "C" fn ZDICT_getDictHeaderSize( let mut state = unsafe { MaybeUninit::::zeroed().assume_init() }; let mut workspace = vec![0u32; HUF_WORKSPACE_SIZE / size_of::()]; unsafe { - ZSTD_reset_compressedBlockState(&mut state); - ZSTD_loadCEntropy( + ZSTD_rust_resetCompressedBlockState(&mut state); + ZSTD_rust_loadCEntropy( &mut state, workspace.as_mut_ptr().cast(), dict_buffer, @@ -1234,9 +1206,9 @@ pub unsafe extern "C" fn ZDICT_trainFromBuffer( let mut params = ZDICT_fastCover_params_t { d: 8, steps: 4, - zParams: ZDICT_params_t { + zParams: crate::dict_builder_cover::ZDICT_params_t { compressionLevel: ZSTD_CLEVEL_DEFAULT, - ..ZDICT_params_t::default() + ..crate::dict_builder_cover::ZDICT_params_t::default() }, ..ZDICT_fastCover_params_t::default() };