feat(cli): move dictionary loading backend to Rust
Move malloc-backed dictionary file reads into the Rust CLI backend while keeping C responsible for stat metadata, patch-mode size policy, diagnostics, buffer-type selection, and eventual free(). The Rust leaf returns explicit open, size, allocation, and read statuses and publishes only fully read libc allocations. Test Plan: - cargo test --manifest-path rust/cli/Cargo.toml (126 tests) - cargo clippy --manifest-path rust/cli/Cargo.toml - make -B -C programs -j2 zstd - make -C tests -j2 test-cli-tests (41 tests)
This commit is contained in:
+38
-20
@@ -320,6 +320,11 @@ const char* FIO_rust_determineDstName(const char* srcFileName, const char* outDi
|
||||
int FIO_rust_adjustMemLimitForPatchFromMode(FIO_prefs_t* prefs,
|
||||
unsigned long long dictSize,
|
||||
unsigned long long maxSrcFileSize);
|
||||
int FIO_rust_setDictBufferMalloc(const char* fileName,
|
||||
unsigned long long expectedFileSize,
|
||||
size_t maxSize,
|
||||
void** buffer,
|
||||
size_t* loadedSize);
|
||||
int FIO_rust_removeFile(const char* path);
|
||||
#ifdef ZSTD_LZ4COMPRESS
|
||||
int FIO_rust_LZ4_GetBlockSize_FromBlockId(int id);
|
||||
@@ -527,9 +532,18 @@ static void FIO_getDictFileStat(const char* fileName, stat_t* dictFileStat) {
|
||||
*/
|
||||
static size_t FIO_setDictBufferMalloc(FIO_Dict_t* dict, const char* fileName, FIO_prefs_t* const prefs, stat_t* dictFileStat)
|
||||
{
|
||||
FILE* fileHandle;
|
||||
U64 fileSize;
|
||||
size_t loadedSize = 0;
|
||||
size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
|
||||
void** bufferPtr = &dict->dictBuffer;
|
||||
enum {
|
||||
FIO_DICT_LOAD_SUCCESS = 0,
|
||||
FIO_DICT_LOAD_OPEN_FAILED = 1,
|
||||
FIO_DICT_LOAD_TOO_LARGE = 2,
|
||||
FIO_DICT_LOAD_ALLOCATION_FAILED = 3,
|
||||
FIO_DICT_LOAD_READ_FAILED = 4,
|
||||
};
|
||||
int status;
|
||||
|
||||
assert(bufferPtr != NULL);
|
||||
assert(dictFileStat != NULL);
|
||||
@@ -538,30 +552,34 @@ static size_t FIO_setDictBufferMalloc(FIO_Dict_t* dict, const char* fileName, FI
|
||||
|
||||
DISPLAYLEVEL(4,"Loading %s as dictionary \n", fileName);
|
||||
|
||||
fileHandle = fopen(fileName, "rb");
|
||||
fileSize = UTIL_getFileSizeStat(dictFileStat);
|
||||
if (fileSize > dictSizeMax) {
|
||||
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
|
||||
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
|
||||
}
|
||||
|
||||
if (fileHandle == NULL) {
|
||||
status = FIO_rust_setDictBufferMalloc(fileName,
|
||||
(unsigned long long)fileSize,
|
||||
dictSizeMax,
|
||||
bufferPtr,
|
||||
&loadedSize);
|
||||
if (status == FIO_DICT_LOAD_SUCCESS) return loadedSize;
|
||||
if (status == FIO_DICT_LOAD_OPEN_FAILED) {
|
||||
EXM_THROW(33, "Couldn't open dictionary %s: %s", fileName, strerror(errno));
|
||||
}
|
||||
|
||||
fileSize = UTIL_getFileSizeStat(dictFileStat);
|
||||
{
|
||||
size_t const dictSizeMax = prefs->patchFromMode ? prefs->memLimit : DICTSIZE_MAX;
|
||||
if (fileSize > dictSizeMax) {
|
||||
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
|
||||
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
|
||||
}
|
||||
if (status == FIO_DICT_LOAD_TOO_LARGE) {
|
||||
EXM_THROW(34, "Dictionary file %s is too large (> %u bytes)",
|
||||
fileName, (unsigned)dictSizeMax); /* avoid extreme cases */
|
||||
}
|
||||
*bufferPtr = malloc((size_t)fileSize);
|
||||
if (*bufferPtr==NULL) EXM_THROW(34, "%s", strerror(errno));
|
||||
{ size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
|
||||
if (readSize != fileSize) {
|
||||
EXM_THROW(35, "Error reading dictionary file %s : %s",
|
||||
fileName, strerror(errno));
|
||||
}
|
||||
if (status == FIO_DICT_LOAD_ALLOCATION_FAILED) {
|
||||
EXM_THROW(34, "%s", strerror(errno));
|
||||
}
|
||||
fclose(fileHandle);
|
||||
return (size_t)fileSize;
|
||||
if (status == FIO_DICT_LOAD_READ_FAILED) {
|
||||
EXM_THROW(35, "Error reading dictionary file %s : %s",
|
||||
fileName, strerror(errno));
|
||||
}
|
||||
assert(0); /* unexpected Rust status */
|
||||
return 0;
|
||||
}
|
||||
|
||||
#if (PLATFORM_POSIX_VERSION > 0)
|
||||
|
||||
+185
-4
@@ -4,19 +4,28 @@
|
||||
//! Rust-owned filesystem leaves for the command-line backend.
|
||||
//!
|
||||
//! The surrounding CLI still owns policy, diagnostics, and stream orchestration
|
||||
//! in C. This module only implements the filesystem operation behind
|
||||
//! `FIO_removeFile()`, returning a small status code so the C wrapper can retain
|
||||
//! its existing messages and return convention.
|
||||
//! in C. This module implements the filesystem operations behind
|
||||
//! `FIO_removeFile()` and dictionary loading, returning small status codes so
|
||||
//! the C wrappers can retain their existing messages and conventions.
|
||||
|
||||
use std::ffi::{c_char, CStr};
|
||||
use std::ffi::{c_char, c_void, CStr};
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
use std::os::raw::c_int;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::ptr;
|
||||
|
||||
const FIO_REMOVE_SUCCESS: c_int = 0;
|
||||
const FIO_REMOVE_STAT_FAILED: c_int = 1;
|
||||
const FIO_REMOVE_NON_REGULAR: c_int = 2;
|
||||
const FIO_REMOVE_FAILED: c_int = 3;
|
||||
|
||||
const FIO_DICT_LOAD_SUCCESS: c_int = 0;
|
||||
const FIO_DICT_LOAD_OPEN_FAILED: c_int = 1;
|
||||
const FIO_DICT_LOAD_TOO_LARGE: c_int = 2;
|
||||
const FIO_DICT_LOAD_ALLOCATION_FAILED: c_int = 3;
|
||||
const FIO_DICT_LOAD_READ_FAILED: c_int = 4;
|
||||
|
||||
fn path_from_c(path: *const c_char) -> Option<PathBuf> {
|
||||
if path.is_null() {
|
||||
return None;
|
||||
@@ -68,6 +77,63 @@ pub unsafe extern "C" fn FIO_rust_removeFile(path: *const c_char) -> c_int {
|
||||
}
|
||||
}
|
||||
|
||||
/// Loads a dictionary into a libc allocation owned by the C caller.
|
||||
///
|
||||
/// The expected size and limit come from C's existing stat and policy paths.
|
||||
/// The file is read exactly to the expected size; the allocation is published
|
||||
/// only after that read succeeds, so C can continue to release it with `free`.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn FIO_rust_setDictBufferMalloc(
|
||||
file_name: *const c_char,
|
||||
expected_file_size: u64,
|
||||
max_size: usize,
|
||||
buffer: *mut *mut c_void,
|
||||
loaded_size: *mut usize,
|
||||
) -> c_int {
|
||||
unsafe {
|
||||
*buffer = ptr::null_mut();
|
||||
*loaded_size = 0;
|
||||
}
|
||||
|
||||
let Some(path) = path_from_c(file_name) else {
|
||||
return FIO_DICT_LOAD_SUCCESS;
|
||||
};
|
||||
|
||||
let mut file = match File::open(path) {
|
||||
Ok(file) => file,
|
||||
Err(_) => return FIO_DICT_LOAD_OPEN_FAILED,
|
||||
};
|
||||
let expected_size = match usize::try_from(expected_file_size) {
|
||||
Ok(size) => size,
|
||||
Err(_) => return FIO_DICT_LOAD_TOO_LARGE,
|
||||
};
|
||||
if expected_size > max_size {
|
||||
return FIO_DICT_LOAD_TOO_LARGE;
|
||||
}
|
||||
|
||||
// Keep C's non-null dictionary-buffer invariant for an empty file while
|
||||
// still reporting that no file bytes were loaded.
|
||||
let allocation = unsafe { libc::malloc(expected_size.max(1)) };
|
||||
if allocation.is_null() {
|
||||
return FIO_DICT_LOAD_ALLOCATION_FAILED;
|
||||
}
|
||||
|
||||
if expected_size != 0 {
|
||||
let destination =
|
||||
unsafe { std::slice::from_raw_parts_mut(allocation.cast::<u8>(), expected_size) };
|
||||
if file.read_exact(destination).is_err() {
|
||||
unsafe { libc::free(allocation) };
|
||||
return FIO_DICT_LOAD_READ_FAILED;
|
||||
}
|
||||
}
|
||||
|
||||
unsafe {
|
||||
*buffer = allocation;
|
||||
*loaded_size = expected_size;
|
||||
}
|
||||
FIO_DICT_LOAD_SUCCESS
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -94,6 +160,121 @@ mod tests {
|
||||
CString::new(path.to_string_lossy().as_bytes()).expect("temporary path contains NUL")
|
||||
}
|
||||
|
||||
fn load_dict(path: &Path, expected_size: u64, max_size: usize) -> (c_int, *mut c_void, usize) {
|
||||
let path = c_path(path);
|
||||
let mut buffer = ptr::null_mut();
|
||||
let mut loaded_size = usize::MAX;
|
||||
let status = unsafe {
|
||||
FIO_rust_setDictBufferMalloc(
|
||||
path.as_ptr(),
|
||||
expected_size,
|
||||
max_size,
|
||||
&mut buffer,
|
||||
&mut loaded_size,
|
||||
)
|
||||
};
|
||||
(status, buffer, loaded_size)
|
||||
}
|
||||
|
||||
fn free_buffer(buffer: *mut c_void) {
|
||||
if !buffer.is_null() {
|
||||
unsafe { libc::free(buffer) };
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn null_filename_leaves_empty_outputs() {
|
||||
let mut buffer = ptr::null_mut();
|
||||
let mut loaded_size = usize::MAX;
|
||||
let status = unsafe {
|
||||
FIO_rust_setDictBufferMalloc(ptr::null(), 123, 0, &mut buffer, &mut loaded_size)
|
||||
};
|
||||
assert_eq!(status, FIO_DICT_LOAD_SUCCESS);
|
||||
assert!(buffer.is_null());
|
||||
assert_eq!(loaded_size, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn loads_dictionary_into_libc_buffer() {
|
||||
let path = temp_path("dictionary");
|
||||
let payload = b"dictionary payload";
|
||||
fs::write(&path, payload).expect("create temporary file");
|
||||
|
||||
let (status, buffer, loaded_size) = load_dict(&path, payload.len() as u64, payload.len());
|
||||
assert_eq!(status, FIO_DICT_LOAD_SUCCESS);
|
||||
assert_eq!(loaded_size, payload.len());
|
||||
let loaded = unsafe { std::slice::from_raw_parts(buffer.cast::<u8>(), loaded_size) };
|
||||
assert_eq!(loaded, payload);
|
||||
|
||||
free_buffer(buffer);
|
||||
fs::remove_file(&path).expect("remove temporary file");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn loads_empty_dictionary_with_zero_size() {
|
||||
let path = temp_path("empty-dictionary");
|
||||
fs::write(&path, b"").expect("create empty temporary file");
|
||||
|
||||
let (status, buffer, loaded_size) = load_dict(&path, 0, 0);
|
||||
assert_eq!(status, FIO_DICT_LOAD_SUCCESS);
|
||||
assert!(!buffer.is_null());
|
||||
assert_eq!(loaded_size, 0);
|
||||
|
||||
free_buffer(buffer);
|
||||
fs::remove_file(&path).expect("remove temporary file");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_missing_dictionary() {
|
||||
let path = temp_path("missing-dictionary");
|
||||
let (status, buffer, loaded_size) = load_dict(&path, 1, 1);
|
||||
assert_eq!(status, FIO_DICT_LOAD_OPEN_FAILED);
|
||||
assert!(buffer.is_null());
|
||||
assert_eq!(loaded_size, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_truncated_dictionary() {
|
||||
let path = temp_path("truncated-dictionary");
|
||||
fs::write(&path, b"short").expect("create temporary file");
|
||||
|
||||
let (status, buffer, loaded_size) = load_dict(&path, 10, 10);
|
||||
assert_eq!(status, FIO_DICT_LOAD_READ_FAILED);
|
||||
assert!(buffer.is_null());
|
||||
assert_eq!(loaded_size, 0);
|
||||
|
||||
fs::remove_file(&path).expect("remove temporary file");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn reports_non_eof_read_failure() {
|
||||
let path = temp_path("read-failure");
|
||||
fs::create_dir(&path).expect("create temporary directory");
|
||||
|
||||
let (status, buffer, loaded_size) = load_dict(&path, 1, 1);
|
||||
assert_eq!(status, FIO_DICT_LOAD_READ_FAILED);
|
||||
assert!(buffer.is_null());
|
||||
assert_eq!(loaded_size, 0);
|
||||
|
||||
fs::remove_dir(&path).expect("remove temporary directory");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_dictionary_above_limit() {
|
||||
let path = temp_path("oversized-dictionary");
|
||||
let payload = b"too large";
|
||||
fs::write(&path, payload).expect("create temporary file");
|
||||
|
||||
let (status, buffer, loaded_size) =
|
||||
load_dict(&path, payload.len() as u64, payload.len() - 1);
|
||||
assert_eq!(status, FIO_DICT_LOAD_TOO_LARGE);
|
||||
assert!(buffer.is_null());
|
||||
assert_eq!(loaded_size, 0);
|
||||
|
||||
fs::remove_file(&path).expect("remove temporary file");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn removes_regular_file() {
|
||||
let path = temp_path("regular");
|
||||
|
||||
Reference in New Issue
Block a user