refactor(fileio): move zstd window diagnostic policy to Rust
Project the zstd window-too-large diagnostic inputs into Rust so Rust owns error filtering and the concrete-versus-unsupported guidance choice. C keeps frame-header parsing, window-log extraction, and the exact display text behind a callback, preserving the existing diagnostics and fallback behavior. All heavy verification was run serially with a 40 GiB virtual-memory cap and one build job. Test Plan: - git diff --cached --check - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --all-targets (790 passed) - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/cli/Cargo.toml --all-targets (184 passed) - ulimit -v 41943040; cargo +nightly fmt --manifest-path rust/Cargo.toml --all -- --check - ulimit -v 41943040; make -j1 (clean after C90 declaration cleanup) - ulimit -v 41943040; make -j1 -C tests test (passed; the later cleanup only moved a declaration before statements)
This commit is contained in:
+103
-20
@@ -3755,6 +3755,83 @@ static int FIO_passThrough(dRess_t *ress)
|
||||
return FIO_rust_passThrough(ress->readCtx, ress->writeCtx);
|
||||
}
|
||||
|
||||
enum {
|
||||
FIO_RUST_ZSTD_ERROR_HELP_WINDOW_SIZE = 0,
|
||||
FIO_RUST_ZSTD_ERROR_HELP_WINDOW_GUIDANCE = 1,
|
||||
FIO_RUST_ZSTD_ERROR_HELP_UNSUPPORTED_WINDOW_LOG = 2
|
||||
};
|
||||
typedef void (*FIO_rust_zstdErrorHelpDisplayFn)(
|
||||
void* callbackContext,
|
||||
int diagnostic,
|
||||
const char* srcFileName,
|
||||
unsigned long long windowSize,
|
||||
unsigned windowLog,
|
||||
unsigned windowMB,
|
||||
unsigned memLimit);
|
||||
typedef struct {
|
||||
void* callbackContext;
|
||||
const char* srcFileName;
|
||||
unsigned long long windowSize;
|
||||
size_t headerError;
|
||||
unsigned windowLog;
|
||||
unsigned memLimit;
|
||||
int errorCode;
|
||||
FIO_rust_zstdErrorHelpDisplayFn display;
|
||||
} FIO_rust_zstdErrorHelpState;
|
||||
typedef char FIO_rust_zstd_error_help_state_layout[
|
||||
(offsetof(FIO_rust_zstdErrorHelpState, callbackContext) == 0
|
||||
&& offsetof(FIO_rust_zstdErrorHelpState, srcFileName)
|
||||
== sizeof(void*)
|
||||
&& offsetof(FIO_rust_zstdErrorHelpState, windowSize)
|
||||
== 2 * sizeof(void*)
|
||||
&& offsetof(FIO_rust_zstdErrorHelpState, headerError)
|
||||
== offsetof(FIO_rust_zstdErrorHelpState, windowSize)
|
||||
+ sizeof(unsigned long long)
|
||||
&& offsetof(FIO_rust_zstdErrorHelpState, windowLog)
|
||||
== offsetof(FIO_rust_zstdErrorHelpState, headerError)
|
||||
+ sizeof(size_t)
|
||||
&& offsetof(FIO_rust_zstdErrorHelpState, memLimit)
|
||||
== offsetof(FIO_rust_zstdErrorHelpState, windowLog)
|
||||
+ sizeof(unsigned)
|
||||
&& offsetof(FIO_rust_zstdErrorHelpState, errorCode)
|
||||
== offsetof(FIO_rust_zstdErrorHelpState, memLimit)
|
||||
+ sizeof(unsigned)
|
||||
&& offsetof(FIO_rust_zstdErrorHelpState, display)
|
||||
== ((offsetof(FIO_rust_zstdErrorHelpState, errorCode)
|
||||
+ sizeof(int) + sizeof(void*) - 1)
|
||||
/ sizeof(void*)) * sizeof(void*)
|
||||
&& sizeof(FIO_rust_zstdErrorHelpState)
|
||||
== offsetof(FIO_rust_zstdErrorHelpState, display)
|
||||
+ sizeof(void*)
|
||||
&& sizeof(FIO_rust_zstdErrorHelpDisplayFn) == sizeof(void*))
|
||||
? 1 : -1];
|
||||
void FIO_rust_zstdErrorHelp(const FIO_rust_zstdErrorHelpState* state);
|
||||
|
||||
static void FIO_rust_zstdErrorHelp_display(
|
||||
void* callbackContext,
|
||||
int diagnostic,
|
||||
const char* srcFileName,
|
||||
unsigned long long windowSize,
|
||||
unsigned windowLog,
|
||||
unsigned windowMB,
|
||||
unsigned memLimit)
|
||||
{
|
||||
(void)callbackContext;
|
||||
if (diagnostic == FIO_RUST_ZSTD_ERROR_HELP_WINDOW_SIZE) {
|
||||
DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u \n",
|
||||
srcFileName, windowSize, memLimit);
|
||||
return;
|
||||
}
|
||||
if (diagnostic == FIO_RUST_ZSTD_ERROR_HELP_WINDOW_GUIDANCE) {
|
||||
DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB \n",
|
||||
srcFileName, windowLog, windowMB);
|
||||
return;
|
||||
}
|
||||
assert(diagnostic == FIO_RUST_ZSTD_ERROR_HELP_UNSUPPORTED_WINDOW_LOG);
|
||||
DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u; not supported \n",
|
||||
srcFileName, ZSTD_WINDOWLOG_MAX);
|
||||
}
|
||||
|
||||
/* FIO_zstdErrorHelp() :
|
||||
* detailed error message when requested window size is too large */
|
||||
static void
|
||||
@@ -3764,28 +3841,34 @@ FIO_zstdErrorHelp(const FIO_prefs_t* const prefs,
|
||||
const char* srcFileName)
|
||||
{
|
||||
ZSTD_FrameHeader header;
|
||||
int const errorCode = (int)ZSTD_getErrorCode(err);
|
||||
size_t headerError = 1;
|
||||
unsigned long long windowSize = 0;
|
||||
unsigned windowLog = 0;
|
||||
unsigned memLimit = 0;
|
||||
FIO_rust_zstdErrorHelpState state = { 0 };
|
||||
|
||||
/* Help message only for one specific error */
|
||||
if (ZSTD_getErrorCode(err) != ZSTD_error_frameParameter_windowTooLarge)
|
||||
return;
|
||||
if (errorCode == ZSTD_error_frameParameter_windowTooLarge) {
|
||||
/* Keep codec/header parsing and the private read-pool layout in C. */
|
||||
headerError = ZSTD_getFrameHeader(
|
||||
&header, ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded);
|
||||
if (headerError == 0) {
|
||||
windowSize = header.windowSize;
|
||||
windowLog = FIO_highbit64(windowSize)
|
||||
+ ((windowSize & (windowSize - 1)) != 0);
|
||||
memLimit = prefs->memLimit;
|
||||
}
|
||||
}
|
||||
|
||||
/* Try to decode the frame header */
|
||||
err = ZSTD_getFrameHeader(&header, ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded);
|
||||
if (err == 0) {
|
||||
unsigned long long const windowSize = header.windowSize;
|
||||
unsigned const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0);
|
||||
assert(prefs->memLimit > 0);
|
||||
DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u \n",
|
||||
srcFileName, windowSize, prefs->memLimit);
|
||||
if (windowLog <= ZSTD_WINDOWLOG_MAX) {
|
||||
unsigned const windowMB = (unsigned)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0));
|
||||
assert(windowSize < (U64)(1ULL << 52)); /* ensure now overflow for windowMB */
|
||||
DISPLAYLEVEL(1, "%s : Use --long=%u or --memory=%uMB \n",
|
||||
srcFileName, windowLog, windowMB);
|
||||
return;
|
||||
} }
|
||||
DISPLAYLEVEL(1, "%s : Window log larger than ZSTD_WINDOWLOG_MAX=%u; not supported \n",
|
||||
srcFileName, ZSTD_WINDOWLOG_MAX);
|
||||
state.callbackContext = NULL;
|
||||
state.srcFileName = srcFileName;
|
||||
state.windowSize = windowSize;
|
||||
state.headerError = headerError;
|
||||
state.windowLog = windowLog;
|
||||
state.memLimit = memLimit;
|
||||
state.errorCode = errorCode;
|
||||
state.display = FIO_rust_zstdErrorHelp_display;
|
||||
FIO_rust_zstdErrorHelp(&state);
|
||||
}
|
||||
|
||||
/** FIO_decompressFrame() :
|
||||
|
||||
@@ -223,6 +223,127 @@ pub unsafe extern "C" fn FIO_rust_freeDResources(state: *const FIO_rust_freeDRes
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
// `ZSTD_error_frameParameter_windowTooLarge` from zstd_errors.h.
|
||||
const ZSTD_ERROR_FRAME_PARAMETER_WINDOW_TOO_LARGE: c_int = 16;
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_RUST_ZSTD_ERROR_HELP_WINDOW_SIZE: c_int = 0;
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_RUST_ZSTD_ERROR_HELP_WINDOW_GUIDANCE: c_int = 1;
|
||||
#[cfg(feature = "decompression")]
|
||||
const FIO_RUST_ZSTD_ERROR_HELP_UNSUPPORTED_WINDOW_LOG: c_int = 2;
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
type FIO_rust_zstdErrorHelpDisplayFn =
|
||||
unsafe extern "C" fn(*mut c_void, c_int, *const c_char, u64, c_uint, c_uint, c_uint);
|
||||
|
||||
/// Scalar projection for the zstd window-size error diagnostic.
|
||||
///
|
||||
/// C keeps frame-header decoding, window-log extraction, and output formatting
|
||||
/// in C. Rust owns the error filter and selects the concrete or unsupported
|
||||
/// diagnostic through the display callback.
|
||||
#[cfg(feature = "decompression")]
|
||||
#[repr(C)]
|
||||
pub struct FIO_rust_zstdErrorHelpState {
|
||||
callback_context: *mut c_void,
|
||||
src_file_name: *const c_char,
|
||||
window_size: u64,
|
||||
header_error: usize,
|
||||
window_log: c_uint,
|
||||
mem_limit: c_uint,
|
||||
error_code: c_int,
|
||||
display: Option<FIO_rust_zstdErrorHelpDisplayFn>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
const _: () = {
|
||||
let callback_offset = (offset_of!(FIO_rust_zstdErrorHelpState, error_code)
|
||||
+ size_of::<c_int>())
|
||||
.div_ceil(size_of::<usize>())
|
||||
* size_of::<usize>();
|
||||
assert!(offset_of!(FIO_rust_zstdErrorHelpState, callback_context) == 0);
|
||||
assert!(offset_of!(FIO_rust_zstdErrorHelpState, src_file_name) == size_of::<usize>());
|
||||
assert!(offset_of!(FIO_rust_zstdErrorHelpState, window_size) == 2 * size_of::<usize>());
|
||||
assert!(
|
||||
offset_of!(FIO_rust_zstdErrorHelpState, header_error)
|
||||
== offset_of!(FIO_rust_zstdErrorHelpState, window_size) + size_of::<u64>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(FIO_rust_zstdErrorHelpState, window_log)
|
||||
== offset_of!(FIO_rust_zstdErrorHelpState, header_error) + size_of::<usize>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(FIO_rust_zstdErrorHelpState, mem_limit)
|
||||
== offset_of!(FIO_rust_zstdErrorHelpState, window_log) + size_of::<c_uint>()
|
||||
);
|
||||
assert!(
|
||||
offset_of!(FIO_rust_zstdErrorHelpState, error_code)
|
||||
== offset_of!(FIO_rust_zstdErrorHelpState, mem_limit) + size_of::<c_uint>()
|
||||
);
|
||||
assert!(size_of::<Option<FIO_rust_zstdErrorHelpDisplayFn>>() == size_of::<usize>());
|
||||
assert!(offset_of!(FIO_rust_zstdErrorHelpState, display) == callback_offset);
|
||||
assert!(size_of::<FIO_rust_zstdErrorHelpState>() == callback_offset + size_of::<usize>());
|
||||
};
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn FIO_rust_zstdErrorHelp(state: *const FIO_rust_zstdErrorHelpState) {
|
||||
let Some(state) = (unsafe { state.as_ref() }) else {
|
||||
return;
|
||||
};
|
||||
if state.error_code != ZSTD_ERROR_FRAME_PARAMETER_WINDOW_TOO_LARGE {
|
||||
return;
|
||||
}
|
||||
let Some(display) = state.display else {
|
||||
return;
|
||||
};
|
||||
|
||||
if state.header_error == 0 {
|
||||
assert!(state.mem_limit > 0);
|
||||
unsafe {
|
||||
display(
|
||||
state.callback_context,
|
||||
FIO_RUST_ZSTD_ERROR_HELP_WINDOW_SIZE,
|
||||
state.src_file_name,
|
||||
state.window_size,
|
||||
state.window_log,
|
||||
0,
|
||||
state.mem_limit,
|
||||
);
|
||||
}
|
||||
if state.window_log <= ZSTD_WINDOWLOG_MAX {
|
||||
let window_mb = ((state.window_size >> 20)
|
||||
+ u64::from((state.window_size & ((1u64 << 20) - 1)) != 0))
|
||||
as c_uint;
|
||||
assert!(state.window_size < (1u64 << 52));
|
||||
unsafe {
|
||||
display(
|
||||
state.callback_context,
|
||||
FIO_RUST_ZSTD_ERROR_HELP_WINDOW_GUIDANCE,
|
||||
state.src_file_name,
|
||||
state.window_size,
|
||||
state.window_log,
|
||||
window_mb,
|
||||
state.mem_limit,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
unsafe {
|
||||
display(
|
||||
state.callback_context,
|
||||
FIO_RUST_ZSTD_ERROR_HELP_UNSUPPORTED_WINDOW_LOG,
|
||||
state.src_file_name,
|
||||
state.window_size,
|
||||
state.window_log,
|
||||
0,
|
||||
state.mem_limit,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub type FIO_createCResourcesSetParameterFn =
|
||||
unsafe extern "C" fn(*mut c_void, c_int, c_int) -> usize;
|
||||
pub type FIO_createCResourcesIsErrorFn = unsafe extern "C" fn(usize) -> c_int;
|
||||
@@ -2280,6 +2401,127 @@ mod tests {
|
||||
assert!(context.events.is_empty());
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[derive(Default)]
|
||||
struct ZstdErrorHelpTestState {
|
||||
events: Vec<(c_int, u64, c_uint, c_uint, c_uint)>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
unsafe extern "C" fn zstd_error_help_display_test(
|
||||
context: *mut c_void,
|
||||
diagnostic: c_int,
|
||||
_src_file_name: *const c_char,
|
||||
window_size: u64,
|
||||
window_log: c_uint,
|
||||
window_mb: c_uint,
|
||||
mem_limit: c_uint,
|
||||
) {
|
||||
let state = unsafe { &mut *context.cast::<ZstdErrorHelpTestState>() };
|
||||
state
|
||||
.events
|
||||
.push((diagnostic, window_size, window_log, window_mb, mem_limit));
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
fn zstd_error_help_test_state(
|
||||
context: &mut ZstdErrorHelpTestState,
|
||||
error_code: c_int,
|
||||
header_error: usize,
|
||||
window_size: u64,
|
||||
window_log: c_uint,
|
||||
mem_limit: c_uint,
|
||||
) -> FIO_rust_zstdErrorHelpState {
|
||||
FIO_rust_zstdErrorHelpState {
|
||||
callback_context: context as *mut ZstdErrorHelpTestState as *mut c_void,
|
||||
src_file_name: ptr::null(),
|
||||
window_size,
|
||||
header_error,
|
||||
window_log,
|
||||
mem_limit,
|
||||
error_code,
|
||||
display: Some(zstd_error_help_display_test),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[test]
|
||||
fn zstd_error_help_ignores_irrelevant_errors() {
|
||||
let mut context = ZstdErrorHelpTestState::default();
|
||||
let state = zstd_error_help_test_state(&mut context, 0, 0, 8u64 << 20, 23, 64);
|
||||
|
||||
unsafe { FIO_rust_zstdErrorHelp(&state) };
|
||||
|
||||
assert!(context.events.is_empty());
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[test]
|
||||
fn zstd_error_help_selects_concrete_window_guidance() {
|
||||
let mut context = ZstdErrorHelpTestState::default();
|
||||
let state = zstd_error_help_test_state(
|
||||
&mut context,
|
||||
ZSTD_ERROR_FRAME_PARAMETER_WINDOW_TOO_LARGE,
|
||||
0,
|
||||
9u64 << 20,
|
||||
24,
|
||||
64,
|
||||
);
|
||||
|
||||
unsafe { FIO_rust_zstdErrorHelp(&state) };
|
||||
|
||||
assert_eq!(
|
||||
context.events,
|
||||
vec![
|
||||
(FIO_RUST_ZSTD_ERROR_HELP_WINDOW_SIZE, 9u64 << 20, 24, 0, 64,),
|
||||
(
|
||||
FIO_RUST_ZSTD_ERROR_HELP_WINDOW_GUIDANCE,
|
||||
9u64 << 20,
|
||||
24,
|
||||
9,
|
||||
64,
|
||||
),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "decompression")]
|
||||
#[test]
|
||||
fn zstd_error_help_selects_oversized_window_log_guidance() {
|
||||
let mut context = ZstdErrorHelpTestState::default();
|
||||
let window_log = ZSTD_WINDOWLOG_MAX + 1;
|
||||
let state = zstd_error_help_test_state(
|
||||
&mut context,
|
||||
ZSTD_ERROR_FRAME_PARAMETER_WINDOW_TOO_LARGE,
|
||||
0,
|
||||
1u64 << window_log,
|
||||
window_log,
|
||||
64,
|
||||
);
|
||||
|
||||
unsafe { FIO_rust_zstdErrorHelp(&state) };
|
||||
|
||||
assert_eq!(
|
||||
context.events,
|
||||
vec![
|
||||
(
|
||||
FIO_RUST_ZSTD_ERROR_HELP_WINDOW_SIZE,
|
||||
1u64 << window_log,
|
||||
window_log,
|
||||
0,
|
||||
64,
|
||||
),
|
||||
(
|
||||
FIO_RUST_ZSTD_ERROR_HELP_UNSUPPORTED_WINDOW_LOG,
|
||||
1u64 << window_log,
|
||||
window_log,
|
||||
0,
|
||||
64,
|
||||
),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CreateCResourcesTestState {
|
||||
events: Vec<(c_int, c_int)>,
|
||||
|
||||
Reference in New Issue
Block a user