From 3599d223f981c7d294d251e57a563e218dab1ebd Mon Sep 17 00:00:00 2001 From: ddidderr Date: Sat, 18 Jul 2026 06:02:22 +0200 Subject: [PATCH] feat(cli): move dictionary loading backend to Rust Move malloc-backed dictionary file reads into the Rust CLI backend while keeping C responsible for stat metadata, patch-mode size policy, diagnostics, buffer-type selection, and eventual free(). The Rust leaf returns explicit open, size, allocation, and read statuses and publishes only fully read libc allocations. Test Plan: - cargo test --manifest-path rust/cli/Cargo.toml (126 tests) - cargo clippy --manifest-path rust/cli/Cargo.toml - make -B -C programs -j2 zstd - make -C tests -j2 test-cli-tests (41 tests) --- programs/fileio.c | 58 ++++++++---- rust/src/fileio_backend.rs | 189 ++++++++++++++++++++++++++++++++++++- 2 files changed, 223 insertions(+), 24 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 22c0f0e58..22fc28025 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -320,6 +320,11 @@ const char* FIO_rust_determineDstName(const char* srcFileName, const char* outDi int FIO_rust_adjustMemLimitForPatchFromMode(FIO_prefs_t* prefs, unsigned long long dictSize, unsigned long long maxSrcFileSize); +int FIO_rust_setDictBufferMalloc(const char* fileName, + unsigned long long expectedFileSize, + size_t maxSize, + void** buffer, + size_t* loadedSize); int FIO_rust_removeFile(const char* path); #ifdef ZSTD_LZ4COMPRESS int FIO_rust_LZ4_GetBlockSize_FromBlockId(int id); @@ -527,9 +532,18 @@ static void FIO_getDictFileStat(const char* fileName, stat_t* dictFileStat) { */ static size_t FIO_setDictBufferMalloc(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat) { - FILE* fileHandle; U64 fileSize; + size_t loadedSize = 0; + size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX; void** bufferPtr = &dict->dictBuffer; + enum { + FIO_DICT_LOAD_SUCCESS = 0, + FIO_DICT_LOAD_OPEN_FAILED = 1, + FIO_DICT_LOAD_TOO_LARGE = 2, + FIO_DICT_LOAD_ALLOCATION_FAILED = 3, + FIO_DICT_LOAD_READ_FAILED = 4, + }; + int status; assert(bufferPtr != NULL); assert(dictFileStat != NULL); @@ -538,30 +552,34 @@ static size_t FIO_setDictBufferMalloc(FIO_Dict_t* dict, const char* fileName, FI DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName); - fileHandle = fopen(fileName, "rb"); + fileSize = UTIL_getFileSizeStat(dictFileStat); + if (fileSize > dictSizeMax) { + EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)", + fileName, (unsigned)dictSizeMax); /* avoid extreme cases */ + } - if (fileHandle == NULL) { + status = FIO_rust_setDictBufferMalloc(fileName, + (unsigned long long)fileSize, + dictSizeMax, + bufferPtr, + &loadedSize); + if (status == FIO_DICT_LOAD_SUCCESS) return loadedSize; + if (status == FIO_DICT_LOAD_OPEN_FAILED) { EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno)); } - - fileSize = UTIL_getFileSizeStat(dictFileStat); - { - size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX; - if (fileSize > dictSizeMax) { - EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)", - fileName, (unsigned)dictSizeMax); /* avoid extreme cases */ - } + if (status == FIO_DICT_LOAD_TOO_LARGE) { + EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)", + fileName, (unsigned)dictSizeMax); /* avoid extreme cases */ } - *bufferPtr = malloc((size_t)fileSize); - if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno)); - { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle); - if (readSize != fileSize) { - EXM_THROW(35, "Error reading dictionary file %s : %s", - fileName, strerror(errno)); - } + if (status == FIO_DICT_LOAD_ALLOCATION_FAILED) { + EXM_THROW(34, "%s", strerror(errno)); } - fclose(fileHandle); - return (size_t)fileSize; + if (status == FIO_DICT_LOAD_READ_FAILED) { + EXM_THROW(35, "Error reading dictionary file %s : %s", + fileName, strerror(errno)); + } + assert(0); /* unexpected Rust status */ + return 0; } #if (PLATFORM_POSIX_VERSION > 0) diff --git a/rust/src/fileio_backend.rs b/rust/src/fileio_backend.rs index 2d8f33b51..b28e06a38 100644 --- a/rust/src/fileio_backend.rs +++ b/rust/src/fileio_backend.rs @@ -4,19 +4,28 @@ //! Rust-owned filesystem leaves for the command-line backend. //! //! The surrounding CLI still owns policy, diagnostics, and stream orchestration -//! in C. This module only implements the filesystem operation behind -//! `FIO_removeFile()`, returning a small status code so the C wrapper can retain -//! its existing messages and return convention. +//! in C. This module implements the filesystem operations behind +//! `FIO_removeFile()` and dictionary loading, returning small status codes so +//! the C wrappers can retain their existing messages and conventions. -use std::ffi::{c_char, CStr}; +use std::ffi::{c_char, c_void, CStr}; +use std::fs::File; +use std::io::Read; use std::os::raw::c_int; use std::path::{Path, PathBuf}; +use std::ptr; const FIO_REMOVE_SUCCESS: c_int = 0; const FIO_REMOVE_STAT_FAILED: c_int = 1; const FIO_REMOVE_NON_REGULAR: c_int = 2; const FIO_REMOVE_FAILED: c_int = 3; +const FIO_DICT_LOAD_SUCCESS: c_int = 0; +const FIO_DICT_LOAD_OPEN_FAILED: c_int = 1; +const FIO_DICT_LOAD_TOO_LARGE: c_int = 2; +const FIO_DICT_LOAD_ALLOCATION_FAILED: c_int = 3; +const FIO_DICT_LOAD_READ_FAILED: c_int = 4; + fn path_from_c(path: *const c_char) -> Option { if path.is_null() { return None; @@ -68,6 +77,63 @@ pub unsafe extern "C" fn FIO_rust_removeFile(path: *const c_char) -> c_int { } } +/// Loads a dictionary into a libc allocation owned by the C caller. +/// +/// The expected size and limit come from C's existing stat and policy paths. +/// The file is read exactly to the expected size; the allocation is published +/// only after that read succeeds, so C can continue to release it with `free`. +#[no_mangle] +pub unsafe extern "C" fn FIO_rust_setDictBufferMalloc( + file_name: *const c_char, + expected_file_size: u64, + max_size: usize, + buffer: *mut *mut c_void, + loaded_size: *mut usize, +) -> c_int { + unsafe { + *buffer = ptr::null_mut(); + *loaded_size = 0; + } + + let Some(path) = path_from_c(file_name) else { + return FIO_DICT_LOAD_SUCCESS; + }; + + let mut file = match File::open(path) { + Ok(file) => file, + Err(_) => return FIO_DICT_LOAD_OPEN_FAILED, + }; + let expected_size = match usize::try_from(expected_file_size) { + Ok(size) => size, + Err(_) => return FIO_DICT_LOAD_TOO_LARGE, + }; + if expected_size > max_size { + return FIO_DICT_LOAD_TOO_LARGE; + } + + // Keep C's non-null dictionary-buffer invariant for an empty file while + // still reporting that no file bytes were loaded. + let allocation = unsafe { libc::malloc(expected_size.max(1)) }; + if allocation.is_null() { + return FIO_DICT_LOAD_ALLOCATION_FAILED; + } + + if expected_size != 0 { + let destination = + unsafe { std::slice::from_raw_parts_mut(allocation.cast::(), expected_size) }; + if file.read_exact(destination).is_err() { + unsafe { libc::free(allocation) }; + return FIO_DICT_LOAD_READ_FAILED; + } + } + + unsafe { + *buffer = allocation; + *loaded_size = expected_size; + } + FIO_DICT_LOAD_SUCCESS +} + #[cfg(test)] mod tests { use super::*; @@ -94,6 +160,121 @@ mod tests { CString::new(path.to_string_lossy().as_bytes()).expect("temporary path contains NUL") } + fn load_dict(path: &Path, expected_size: u64, max_size: usize) -> (c_int, *mut c_void, usize) { + let path = c_path(path); + let mut buffer = ptr::null_mut(); + let mut loaded_size = usize::MAX; + let status = unsafe { + FIO_rust_setDictBufferMalloc( + path.as_ptr(), + expected_size, + max_size, + &mut buffer, + &mut loaded_size, + ) + }; + (status, buffer, loaded_size) + } + + fn free_buffer(buffer: *mut c_void) { + if !buffer.is_null() { + unsafe { libc::free(buffer) }; + } + } + + #[test] + fn null_filename_leaves_empty_outputs() { + let mut buffer = ptr::null_mut(); + let mut loaded_size = usize::MAX; + let status = unsafe { + FIO_rust_setDictBufferMalloc(ptr::null(), 123, 0, &mut buffer, &mut loaded_size) + }; + assert_eq!(status, FIO_DICT_LOAD_SUCCESS); + assert!(buffer.is_null()); + assert_eq!(loaded_size, 0); + } + + #[test] + fn loads_dictionary_into_libc_buffer() { + let path = temp_path("dictionary"); + let payload = b"dictionary payload"; + fs::write(&path, payload).expect("create temporary file"); + + let (status, buffer, loaded_size) = load_dict(&path, payload.len() as u64, payload.len()); + assert_eq!(status, FIO_DICT_LOAD_SUCCESS); + assert_eq!(loaded_size, payload.len()); + let loaded = unsafe { std::slice::from_raw_parts(buffer.cast::(), loaded_size) }; + assert_eq!(loaded, payload); + + free_buffer(buffer); + fs::remove_file(&path).expect("remove temporary file"); + } + + #[test] + fn loads_empty_dictionary_with_zero_size() { + let path = temp_path("empty-dictionary"); + fs::write(&path, b"").expect("create empty temporary file"); + + let (status, buffer, loaded_size) = load_dict(&path, 0, 0); + assert_eq!(status, FIO_DICT_LOAD_SUCCESS); + assert!(!buffer.is_null()); + assert_eq!(loaded_size, 0); + + free_buffer(buffer); + fs::remove_file(&path).expect("remove temporary file"); + } + + #[test] + fn rejects_missing_dictionary() { + let path = temp_path("missing-dictionary"); + let (status, buffer, loaded_size) = load_dict(&path, 1, 1); + assert_eq!(status, FIO_DICT_LOAD_OPEN_FAILED); + assert!(buffer.is_null()); + assert_eq!(loaded_size, 0); + } + + #[test] + fn rejects_truncated_dictionary() { + let path = temp_path("truncated-dictionary"); + fs::write(&path, b"short").expect("create temporary file"); + + let (status, buffer, loaded_size) = load_dict(&path, 10, 10); + assert_eq!(status, FIO_DICT_LOAD_READ_FAILED); + assert!(buffer.is_null()); + assert_eq!(loaded_size, 0); + + fs::remove_file(&path).expect("remove temporary file"); + } + + #[cfg(unix)] + #[test] + fn reports_non_eof_read_failure() { + let path = temp_path("read-failure"); + fs::create_dir(&path).expect("create temporary directory"); + + let (status, buffer, loaded_size) = load_dict(&path, 1, 1); + assert_eq!(status, FIO_DICT_LOAD_READ_FAILED); + assert!(buffer.is_null()); + assert_eq!(loaded_size, 0); + + fs::remove_dir(&path).expect("remove temporary directory"); + } + + #[test] + fn rejects_dictionary_above_limit() { + let path = temp_path("oversized-dictionary"); + let payload = b"too large"; + fs::write(&path, payload).expect("create temporary file"); + + let (status, buffer, loaded_size) = + load_dict(&path, payload.len() as u64, payload.len() - 1); + assert_eq!(status, FIO_DICT_LOAD_TOO_LARGE); + assert!(buffer.is_null()); + assert_eq!(loaded_size, 0); + + fs::remove_file(&path).expect("remove temporary file"); + } + #[test] fn removes_regular_file() { let path = temp_path("regular");