ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(fileio): move compression parameter policy to Rust

Browse Source 
Move the non-threaded compression parameter sequence and adaptive window policy
out of FIO_createCResources.  C continues to own context and resource
allocation, dictionary and pool setup, and the multithread-specific settings;
Rust drives the ordered parameter callbacks and stops on the first error.
The projection mirrors the C ABI and has focused order and short-circuit tests.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/cli/Cargo.toml create_c_resources --lib
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040; make -j1
This commit is contained in:
ddidderr
2026-07-20 04:49:35 +02:00
parent 907c3af89e
commit bbe3a5dd74
2 changed files with 325 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/fileio_prefs.rs
+271
View File
@@ -31,10 +31,32 @@ const FIO_MULTI_FILES_ACTION_CONFIRM: c_int = 5;
const FIO_LIST_INFO_SUCCESS: c_int = 0;
const FIO_LIST_INFO_FRAME_ERROR: c_int = 1;
const UTIL_FILESIZE_UNKNOWN: u64 = u64::MAX;
const FIO_ADAPT_WINDOWLOG_DEFAULT: c_uint = 23;
const ZSTD_WINDOWLOG_MIN: u32 = 10;
const ZSTD_WINDOWLOG_MAX: u32 = if size_of::<usize>() == 4 { 30 } else { 31 };
const ZSTD_BTLAZY2: c_int = 6;
const ZSTD_BTOPT: c_int = 7;
const FIO_ZSTD_C_WINDOW_LOG: c_int = 101;
const FIO_ZSTD_C_HASH_LOG: c_int = 102;
const FIO_ZSTD_C_CHAIN_LOG: c_int = 103;
const FIO_ZSTD_C_SEARCH_LOG: c_int = 104;
const FIO_ZSTD_C_MIN_MATCH: c_int = 105;
const FIO_ZSTD_C_TARGET_LENGTH: c_int = 106;
const FIO_ZSTD_C_STRATEGY: c_int = 107;
const FIO_ZSTD_C_COMPRESSION_LEVEL: c_int = 100;
const FIO_ZSTD_C_TARGET_C_BLOCK_SIZE: c_int = 130;
const FIO_ZSTD_C_ENABLE_LDM: c_int = 160;
const FIO_ZSTD_C_LDM_HASH_LOG: c_int = 161;
const FIO_ZSTD_C_LDM_MIN_MATCH: c_int = 162;
const FIO_ZSTD_C_LDM_BUCKET_SIZE_LOG: c_int = 163;
const FIO_ZSTD_C_LDM_HASH_RATE_LOG: c_int = 164;
const FIO_ZSTD_C_CONTENT_SIZE_FLAG: c_int = 200;
const FIO_ZSTD_C_CHECKSUM_FLAG: c_int = 201;
const FIO_ZSTD_C_DICT_ID_FLAG: c_int = 202;
const FIO_ZSTD_C_LITERAL_COMPRESSION_MODE: c_int = 1002;
const FIO_ZSTD_C_SRC_SIZE_HINT: c_int = 1004;
const FIO_ZSTD_C_ENABLE_DEDICATED_DICT_SEARCH: c_int = 1005;
const FIO_ZSTD_C_USE_ROW_MATCH_FINDER: c_int = 1011;
static STDOUT_MARK: &[u8] = b"/*stdout*\\\0";
static mut COMPRESSED_NAME_CAPACITY: usize = 0;
@@ -147,6 +169,137 @@ pub unsafe extern "C" fn FIO_rust_freeCResources(
}
}
pub type FIO_createCResourcesSetParameterFn =
unsafe extern "C" fn(*mut c_void, c_int, c_int) -> usize;
pub type FIO_createCResourcesIsErrorFn = unsafe extern "C" fn(usize) -> c_int;
/// C-owned compression-resource parameter policy projection.
///
/// Rust owns the ordered, short-circuiting parameter policy. C keeps the
/// `ZSTD_CCtx` layout private and exposes only the parameter callback; resource
/// allocation, dictionary/pool setup, and dictionary loading remain in C.
#[repr(C)]
pub struct FIO_rust_createCResourcesState {
callback_context: *mut c_void,
prefs: *const FIO_prefs_t,
compression_params: FIO_compressionParameters,
compression_level: c_int,
set_parameter: FIO_createCResourcesSetParameterFn,
is_error: FIO_createCResourcesIsErrorFn,
}
const _: () = {
let callback_offset = (2 * size_of::<usize>()
+ size_of::<FIO_compressionParameters>()
+ size_of::<c_int>())
.div_ceil(size_of::<usize>())
* size_of::<usize>();
assert!(offset_of!(FIO_rust_createCResourcesState, callback_context) == 0);
assert!(offset_of!(FIO_rust_createCResourcesState, prefs) == size_of::<usize>());
assert!(
offset_of!(FIO_rust_createCResourcesState, compression_params)
== 2 * size_of::<usize>()
);
assert!(
offset_of!(FIO_rust_createCResourcesState, compression_level)
== 2 * size_of::<usize>() + size_of::<FIO_compressionParameters>()
);
assert!(offset_of!(FIO_rust_createCResourcesState, set_parameter) == callback_offset);
assert!(offset_of!(FIO_rust_createCResourcesState, is_error) == callback_offset + size_of::<usize>());
assert!(
size_of::<FIO_rust_createCResourcesState>() == callback_offset + 2 * size_of::<usize>()
);
};
#[no_mangle]
pub unsafe extern "C" fn FIO_rust_createCResources(
state: *const FIO_rust_createCResourcesState,
) -> usize {
if state.is_null() {
return usize::MAX;
}
let state = unsafe { &*state };
if state.prefs.is_null() {
return usize::MAX;
}
let prefs = unsafe { &*state.prefs };
let mut compression_params = state.compression_params;
if prefs.adaptiveMode != 0 && prefs.ldmFlag == 0 && compression_params.windowLog == 0 {
compression_params.windowLog = FIO_ADAPT_WINDOWLOG_DEFAULT;
}
let parameters = [
(FIO_ZSTD_C_CONTENT_SIZE_FLAG, prefs.contentSize),
(FIO_ZSTD_C_DICT_ID_FLAG, prefs.dictIDFlag),
(FIO_ZSTD_C_CHECKSUM_FLAG, prefs.checksumFlag),
(FIO_ZSTD_C_COMPRESSION_LEVEL, state.compression_level),
(
FIO_ZSTD_C_TARGET_C_BLOCK_SIZE,
prefs.targetCBlockSize as c_int,
),
(FIO_ZSTD_C_SRC_SIZE_HINT, prefs.srcSizeHint),
(FIO_ZSTD_C_ENABLE_LDM, prefs.ldmFlag),
(FIO_ZSTD_C_LDM_HASH_LOG, prefs.ldmHashLog),
(FIO_ZSTD_C_LDM_MIN_MATCH, prefs.ldmMinMatch),
];
for (parameter, value) in parameters {
let result = unsafe { (state.set_parameter)(state.callback_context, parameter, value) };
if unsafe { (state.is_error)(result) } != 0 {
return result;
}
}
if prefs.ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET {
let result = unsafe {
(state.set_parameter)(
state.callback_context,
FIO_ZSTD_C_LDM_BUCKET_SIZE_LOG,
prefs.ldmBucketSizeLog,
)
};
if unsafe { (state.is_error)(result) } != 0 {
return result;
}
}
if prefs.ldmHashRateLog != FIO_LDM_PARAM_NOTSET {
let result = unsafe {
(state.set_parameter)(
state.callback_context,
FIO_ZSTD_C_LDM_HASH_RATE_LOG,
prefs.ldmHashRateLog,
)
};
if unsafe { (state.is_error)(result) } != 0 {
return result;
}
}
let parameters = [
(FIO_ZSTD_C_USE_ROW_MATCH_FINDER, prefs.useRowMatchFinder),
(FIO_ZSTD_C_WINDOW_LOG, compression_params.windowLog as c_int),
(FIO_ZSTD_C_CHAIN_LOG, compression_params.chainLog as c_int),
(FIO_ZSTD_C_HASH_LOG, compression_params.hashLog as c_int),
(FIO_ZSTD_C_SEARCH_LOG, compression_params.searchLog as c_int),
(FIO_ZSTD_C_MIN_MATCH, compression_params.minMatch as c_int),
(
FIO_ZSTD_C_TARGET_LENGTH,
compression_params.targetLength as c_int,
),
(FIO_ZSTD_C_STRATEGY, compression_params.strategy),
(
FIO_ZSTD_C_LITERAL_COMPRESSION_MODE,
prefs.literalCompressionMode,
),
(FIO_ZSTD_C_ENABLE_DEDICATED_DICT_SEARCH, 1),
];
for (parameter, value) in parameters {
let result = unsafe { (state.set_parameter)(state.callback_context, parameter, value) };
if unsafe { (state.is_error)(result) } != 0 {
return result;
}
}
0
}
pub type FIO_listMultipleFilesIsStdinFn = unsafe extern "C" fn(*mut c_void, *const c_char) -> c_int;
pub type FIO_listMultipleFilesDisplayFn = unsafe extern "C" fn(*mut c_void);
pub type FIO_listMultipleFilesListFileFn = unsafe extern "C" fn(
@@ -1957,6 +2110,124 @@ mod tests {
assert_eq!(context.events, ["dict", "write", "read", "cctx"]);
}
#[derive(Default)]
struct CreateCResourcesTestState {
events: Vec<(c_int, c_int)>,
fail_parameter: Option<c_int>,
}
unsafe extern "C" fn create_c_resources_set_parameter_test(
context: *mut c_void,
parameter: c_int,
value: c_int,
) -> usize {
let state = unsafe { &mut *context.cast::<CreateCResourcesTestState>() };
state.events.push((parameter, value));
if state.fail_parameter == Some(parameter) {
usize::MAX
} else {
0
}
}
unsafe extern "C" fn create_c_resources_is_error_test(result: usize) -> c_int {
c_int::from(result == usize::MAX)
}
fn create_c_resources_test_prefs() -> FIO_prefs_t {
let mut prefs: FIO_prefs_t = unsafe { std::mem::zeroed() };
prefs.contentSize = 1;
prefs.dictIDFlag = 1;
prefs.checksumFlag = 1;
prefs.adaptiveMode = 1;
prefs.ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET;
prefs.ldmHashRateLog = FIO_LDM_PARAM_NOTSET;
prefs.targetCBlockSize = 17;
prefs.srcSizeHint = 19;
prefs
}
fn create_c_resources_test_state(
prefs: &FIO_prefs_t,
context: &mut CreateCResourcesTestState,
) -> FIO_rust_createCResourcesState {
FIO_rust_createCResourcesState {
callback_context: context as *mut CreateCResourcesTestState as *mut c_void,
prefs,
compression_params: FIO_compressionParameters {
windowLog: 0,
chainLog: 1,
hashLog: 2,
searchLog: 3,
minMatch: 4,
targetLength: 5,
strategy: 6,
},
compression_level: 7,
set_parameter: create_c_resources_set_parameter_test,
is_error: create_c_resources_is_error_test,
}
}
#[test]
fn create_c_resources_preserves_parameter_order_and_adaptive_policy() {
let prefs = create_c_resources_test_prefs();
let mut context = CreateCResourcesTestState::default();
let state = create_c_resources_test_state(&prefs, &mut context);
let result = unsafe { FIO_rust_createCResources(&state) };
assert_eq!(result, 0);
assert_eq!(
context.events,
vec![
(FIO_ZSTD_C_CONTENT_SIZE_FLAG, 1),
(FIO_ZSTD_C_DICT_ID_FLAG, 1),
(FIO_ZSTD_C_CHECKSUM_FLAG, 1),
(FIO_ZSTD_C_COMPRESSION_LEVEL, 7),
(FIO_ZSTD_C_TARGET_C_BLOCK_SIZE, 17),
(FIO_ZSTD_C_SRC_SIZE_HINT, 19),
(FIO_ZSTD_C_ENABLE_LDM, 0),
(FIO_ZSTD_C_LDM_HASH_LOG, 0),
(FIO_ZSTD_C_LDM_MIN_MATCH, 0),
(FIO_ZSTD_C_USE_ROW_MATCH_FINDER, 0),
(FIO_ZSTD_C_WINDOW_LOG, FIO_ADAPT_WINDOWLOG_DEFAULT as c_int),
(FIO_ZSTD_C_CHAIN_LOG, 1),
(FIO_ZSTD_C_HASH_LOG, 2),
(FIO_ZSTD_C_SEARCH_LOG, 3),
(FIO_ZSTD_C_MIN_MATCH, 4),
(FIO_ZSTD_C_TARGET_LENGTH, 5),
(FIO_ZSTD_C_STRATEGY, 6),
(FIO_ZSTD_C_LITERAL_COMPRESSION_MODE, 0),
(FIO_ZSTD_C_ENABLE_DEDICATED_DICT_SEARCH, 1),
]
);
}
#[test]
fn create_c_resources_short_circuits_on_parameter_error() {
let prefs = create_c_resources_test_prefs();
let mut context = CreateCResourcesTestState {
fail_parameter: Some(FIO_ZSTD_C_TARGET_C_BLOCK_SIZE),
..CreateCResourcesTestState::default()
};
let state = create_c_resources_test_state(&prefs, &mut context);
let result = unsafe { FIO_rust_createCResources(&state) };
assert_eq!(result, usize::MAX);
assert_eq!(
context.events,
vec![
(FIO_ZSTD_C_CONTENT_SIZE_FLAG, 1),
(FIO_ZSTD_C_DICT_ID_FLAG, 1),
(FIO_ZSTD_C_CHECKSUM_FLAG, 1),
(FIO_ZSTD_C_COMPRESSION_LEVEL, 7),
(FIO_ZSTD_C_TARGET_C_BLOCK_SIZE, 17),
]
);
}
fn temporary_file_path(name: &str) -> std::path::PathBuf {
std::env::temp_dir().join(format!("zstd-fileio-prefs-{}-{name}", std::process::id()))
}