feat(cli): move xz/lzma and lz4 codec loops into Rust

Move the xz/lzma and LZ4 file-compression orchestration into Rust projection
functions. C retains liblzma and LZ4F contexts, asynchronous pool ownership,
format-specific callbacks, sparse-output handling, and user-facing diagnostics.
The Rust leaves preserve input accounting, output enqueue order, progress
updates, finish/end handling, cleanup, and the original error-code mapping.
Focused projection tests cover success ordering and codec initialization,
header, update, finish, and end failures.

Test Plan:
- cargo test --manifest-path rust/Cargo.toml --all-targets -- --test-threads=1
- cargo test --manifest-path rust/cli/Cargo.toml --all-targets -- --test-threads=1
- cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings
- make -B -C programs -j2 zstd
- make -B -C tests -j2 test-cli-tests test-zstd
- FUZZERTEST=-T5s make -B -C tests -j2 test-fuzzer
This commit is contained in:
2026-07-18 23:28:53 +02:00
parent 45d54a71ad
commit fce8d34ffc
2 changed files with 1656 additions and 146 deletions
+1209
View File
@@ -180,6 +180,118 @@ pub struct FIO_rust_gzip_compress_projection_t {
pub progress: Option<FIO_rust_gzip_progress_fn>,
}
pub const FIO_RUST_LZMA_OK: c_int = 0;
pub const FIO_RUST_LZMA_INIT_PRESET_ERROR: c_int = 1;
pub const FIO_RUST_LZMA_INIT_ALONE_ERROR: c_int = 2;
pub const FIO_RUST_LZMA_INIT_XZ_ERROR: c_int = 3;
pub const FIO_RUST_LZMA_CODE_ERROR: c_int = 4;
pub const FIO_RUST_LZMA_INVALID_PROJECTION: c_int = 5;
const FIO_RUST_LZMA_OK_CODE: c_int = 0;
const FIO_RUST_LZMA_STREAM_END: c_int = 1;
const FIO_RUST_LZMA_RUN: c_int = 0;
const FIO_RUST_LZMA_FINISH: c_int = 3;
pub type FIO_rust_lzma_read_fill_fn =
unsafe extern "C" fn(*mut c_void, usize, *mut *const u8, *mut usize);
pub type FIO_rust_lzma_read_consume_fn = unsafe extern "C" fn(*mut c_void, usize);
pub type FIO_rust_lzma_write_acquire_fn =
unsafe extern "C" fn(*mut c_void, *mut *mut c_void, *mut *mut u8, *mut usize);
pub type FIO_rust_lzma_write_enqueue_fn =
unsafe extern "C" fn(*mut c_void, *mut *mut c_void, usize, *mut *mut u8, *mut usize);
pub type FIO_rust_lzma_write_release_fn = unsafe extern "C" fn(*mut c_void, *mut c_void);
pub type FIO_rust_lzma_sparse_write_end_fn = unsafe extern "C" fn(*mut c_void);
pub type FIO_rust_lzma_init_fn =
unsafe extern "C" fn(*mut c_void, c_int, c_int, *mut c_int) -> c_int;
pub type FIO_rust_lzma_code_fn = unsafe extern "C" fn(
*mut c_void,
*const u8,
usize,
*mut u8,
usize,
c_int,
*mut usize,
*mut usize,
) -> c_int;
pub type FIO_rust_lzma_end_fn = unsafe extern "C" fn(*mut c_void);
pub type FIO_rust_lzma_progress_fn = unsafe extern "C" fn(*mut c_void, u64, u64, u64);
/// Rust owns the LZMA/xz read, code, finish, and accounting loop. C keeps
/// the liblzma stream, asynchronous pools, sparse-output behavior, and
/// diagnostics behind opaque callbacks.
#[repr(C)]
pub struct FIO_rust_lzma_compress_projection_t {
pub read_opaque: *mut c_void,
pub write_opaque: *mut c_void,
pub lzma_opaque: *mut c_void,
pub progress_opaque: *mut c_void,
pub read_buffer_size: usize,
pub read_fill: Option<FIO_rust_lzma_read_fill_fn>,
pub read_consume: Option<FIO_rust_lzma_read_consume_fn>,
pub write_acquire: Option<FIO_rust_lzma_write_acquire_fn>,
pub write_enqueue: Option<FIO_rust_lzma_write_enqueue_fn>,
pub write_release: Option<FIO_rust_lzma_write_release_fn>,
pub sparse_write_end: Option<FIO_rust_lzma_sparse_write_end_fn>,
pub lzma_init: Option<FIO_rust_lzma_init_fn>,
pub lzma_code: Option<FIO_rust_lzma_code_fn>,
pub lzma_end: Option<FIO_rust_lzma_end_fn>,
pub progress: Option<FIO_rust_lzma_progress_fn>,
}
pub const FIO_RUST_LZ4_OK: c_int = 0;
pub const FIO_RUST_LZ4_CREATE_ERROR: c_int = 1;
pub const FIO_RUST_LZ4_HEADER_ERROR: c_int = 2;
pub const FIO_RUST_LZ4_UPDATE_ERROR: c_int = 3;
pub const FIO_RUST_LZ4_END_ERROR: c_int = 4;
pub const FIO_RUST_LZ4_INVALID_PROJECTION: c_int = 5;
pub type FIO_rust_lz4_create_fn = unsafe extern "C" fn(*mut c_void, c_uint, *mut usize) -> c_int;
pub type FIO_rust_lz4_prepare_fn =
unsafe extern "C" fn(*mut c_void, u64, c_int, c_int, usize, usize);
pub type FIO_rust_lz4_begin_fn =
unsafe extern "C" fn(*mut c_void, *mut u8, usize, *mut usize) -> c_int;
pub type FIO_rust_lz4_update_fn =
unsafe extern "C" fn(*mut c_void, *mut u8, usize, *const u8, usize, *mut usize) -> c_int;
pub type FIO_rust_lz4_end_fn =
unsafe extern "C" fn(*mut c_void, *mut u8, usize, *mut usize) -> c_int;
pub type FIO_rust_lz4_free_fn = unsafe extern "C" fn(*mut c_void);
pub type FIO_rust_lz4_read_fill_fn =
unsafe extern "C" fn(*mut c_void, usize, *mut *const u8, *mut usize) -> usize;
pub type FIO_rust_lz4_read_consume_fn = unsafe extern "C" fn(*mut c_void, usize);
pub type FIO_rust_lz4_write_acquire_fn =
unsafe extern "C" fn(*mut c_void, *mut *mut c_void, *mut *mut u8, *mut usize);
pub type FIO_rust_lz4_write_enqueue_fn =
unsafe extern "C" fn(*mut c_void, *mut *mut c_void, usize, *mut *mut u8, *mut usize);
pub type FIO_rust_lz4_write_release_fn = unsafe extern "C" fn(*mut c_void, *mut c_void);
pub type FIO_rust_lz4_sparse_write_end_fn = unsafe extern "C" fn(*mut c_void);
pub type FIO_rust_lz4_progress_fn = unsafe extern "C" fn(*mut c_void, u64, u64, u64);
/// Rust owns the LZ4 frame loop. C keeps the LZ4F context/preferences and
/// asynchronous pool objects private, exposing only the operations required
/// to preserve the original header, block, progress, and cleanup ordering.
#[repr(C)]
pub struct FIO_rust_lz4_compress_projection_t {
pub read_opaque: *mut c_void,
pub write_opaque: *mut c_void,
pub codec_opaque: *mut c_void,
pub progress_opaque: *mut c_void,
pub version: c_uint,
pub block_size: usize,
pub create: Option<FIO_rust_lz4_create_fn>,
pub prepare: Option<FIO_rust_lz4_prepare_fn>,
pub begin: Option<FIO_rust_lz4_begin_fn>,
pub update: Option<FIO_rust_lz4_update_fn>,
pub end: Option<FIO_rust_lz4_end_fn>,
pub free_context: Option<FIO_rust_lz4_free_fn>,
pub read_fill: Option<FIO_rust_lz4_read_fill_fn>,
pub read_consume: Option<FIO_rust_lz4_read_consume_fn>,
pub write_acquire: Option<FIO_rust_lz4_write_acquire_fn>,
pub write_enqueue: Option<FIO_rust_lz4_write_enqueue_fn>,
pub write_release: Option<FIO_rust_lz4_write_release_fn>,
pub sparse_write_end: Option<FIO_rust_lz4_sparse_write_end_fn>,
pub progress: Option<FIO_rust_lz4_progress_fn>,
}
/// C's `FIO_prefs_t` from `programs/fileio_types.h`.
///
/// `fileio_prefs.rs` contains the same C layout for the preferences API. It
@@ -1932,6 +2044,390 @@ pub unsafe extern "C" fn FIO_rust_compressGzipFrame(
FIO_RUST_GZIP_OK
}
/// Compresses one LZ4 frame through C-owned LZ4F and asynchronous resource
/// callbacks. The callback projection deliberately reports bytes added by a
/// read-pool fill separately from the currently loaded byte count: the former
/// is the accounting used by the original C leaf, while the latter drives the
/// current block update.
#[no_mangle]
pub unsafe extern "C" fn FIO_rust_compressLz4Frame(
projection: *const FIO_rust_lz4_compress_projection_t,
_src_file_name: *const c_char,
src_file_size: u64,
compression_level: c_int,
checksum_flag: c_int,
read_size: *mut u64,
compressed_size: *mut u64,
lz4_result: *mut usize,
) -> c_int {
assert!(!projection.is_null());
assert!(!read_size.is_null());
assert!(!compressed_size.is_null());
assert!(!lz4_result.is_null());
unsafe {
*read_size = 0;
*compressed_size = 0;
*lz4_result = 0;
}
let projection = unsafe { &*projection };
if projection.block_size == 0 {
return FIO_RUST_LZ4_INVALID_PROJECTION;
}
let Some(create) = projection.create else {
return FIO_RUST_LZ4_INVALID_PROJECTION;
};
let Some(prepare) = projection.prepare else {
return FIO_RUST_LZ4_INVALID_PROJECTION;
};
let Some(begin) = projection.begin else {
return FIO_RUST_LZ4_INVALID_PROJECTION;
};
let Some(update) = projection.update else {
return FIO_RUST_LZ4_INVALID_PROJECTION;
};
let Some(end) = projection.end else {
return FIO_RUST_LZ4_INVALID_PROJECTION;
};
let Some(free_context) = projection.free_context else {
return FIO_RUST_LZ4_INVALID_PROJECTION;
};
let Some(read_fill) = projection.read_fill else {
return FIO_RUST_LZ4_INVALID_PROJECTION;
};
let Some(read_consume) = projection.read_consume else {
return FIO_RUST_LZ4_INVALID_PROJECTION;
};
let Some(write_acquire) = projection.write_acquire else {
return FIO_RUST_LZ4_INVALID_PROJECTION;
};
let Some(write_enqueue) = projection.write_enqueue else {
return FIO_RUST_LZ4_INVALID_PROJECTION;
};
let Some(write_release) = projection.write_release else {
return FIO_RUST_LZ4_INVALID_PROJECTION;
};
let Some(sparse_write_end) = projection.sparse_write_end else {
return FIO_RUST_LZ4_INVALID_PROJECTION;
};
let Some(progress) = projection.progress else {
return FIO_RUST_LZ4_INVALID_PROJECTION;
};
let mut job = ptr::null_mut::<c_void>();
let mut output = ptr::null_mut::<u8>();
let mut output_size = 0_usize;
unsafe {
write_acquire(
projection.write_opaque,
&mut job,
&mut output,
&mut output_size,
);
}
if job.is_null() || (output.is_null() && output_size != 0) {
return FIO_RUST_LZ4_INVALID_PROJECTION;
}
let mut result = 0_usize;
let create_status = unsafe { create(projection.codec_opaque, projection.version, &mut result) };
if create_status != 0 {
unsafe { *lz4_result = result };
return FIO_RUST_LZ4_CREATE_ERROR;
}
unsafe {
prepare(
projection.codec_opaque,
src_file_size,
compression_level,
checksum_flag,
projection.block_size,
output_size,
);
}
result = 0;
let begin_status = unsafe { begin(projection.codec_opaque, output, output_size, &mut result) };
if begin_status != 0 {
unsafe { *lz4_result = result };
return FIO_RUST_LZ4_HEADER_ERROR;
}
let header_size = result;
unsafe {
write_enqueue(
projection.write_opaque,
&mut job,
header_size,
&mut output,
&mut output_size,
);
}
let mut in_file_size = 0_u64;
let mut out_file_size = header_size as u64;
let mut input = ptr::null::<u8>();
let mut loaded = 0_usize;
in_file_size = in_file_size.wrapping_add(unsafe {
read_fill(
projection.read_opaque,
projection.block_size,
&mut input,
&mut loaded,
) as u64
});
while loaded != 0 {
if input.is_null() {
return FIO_RUST_LZ4_INVALID_PROJECTION;
}
let input_size = projection.block_size.min(loaded);
result = 0;
let update_status = unsafe {
update(
projection.codec_opaque,
output,
output_size,
input,
input_size,
&mut result,
)
};
if update_status != 0 {
unsafe { *lz4_result = result };
return FIO_RUST_LZ4_UPDATE_ERROR;
}
out_file_size = out_file_size.wrapping_add(result as u64);
unsafe {
progress(
projection.progress_opaque,
src_file_size,
in_file_size,
out_file_size,
);
write_enqueue(
projection.write_opaque,
&mut job,
result,
&mut output,
&mut output_size,
);
read_consume(projection.read_opaque, input_size);
}
in_file_size = in_file_size.wrapping_add(unsafe {
read_fill(
projection.read_opaque,
projection.block_size,
&mut input,
&mut loaded,
) as u64
});
}
result = 0;
let end_status = unsafe { end(projection.codec_opaque, output, output_size, &mut result) };
if end_status != 0 {
unsafe { *lz4_result = result };
return FIO_RUST_LZ4_END_ERROR;
}
unsafe {
write_enqueue(
projection.write_opaque,
&mut job,
result,
&mut output,
&mut output_size,
);
}
out_file_size = out_file_size.wrapping_add(result as u64);
unsafe {
*read_size = in_file_size;
*compressed_size = out_file_size;
free_context(projection.codec_opaque);
write_release(projection.write_opaque, job);
sparse_write_end(projection.write_opaque);
}
FIO_RUST_LZ4_OK
}
/// Compresses one LZMA or xz member through the C-owned liblzma stream and
/// asynchronous resource callbacks. The action transition and accounting
/// intentionally mirror the original C loop: input is counted when a pool
/// buffer is filled, consumed bytes are returned immediately after each
/// `lzma_code()` call, non-empty output is enqueued before progress is shown,
/// and `LZMA_FINISH` is retried until `LZMA_STREAM_END`.
#[no_mangle]
pub unsafe extern "C" fn FIO_rust_compressLzmaFrame(
projection: *const FIO_rust_lzma_compress_projection_t,
_src_file_name: *const c_char,
src_file_size: u64,
compression_level: c_int,
plain_lzma: c_int,
read_size: *mut u64,
compressed_size: *mut u64,
lzma_result: *mut c_int,
) -> c_int {
assert!(!projection.is_null());
assert!(!read_size.is_null());
assert!(!compressed_size.is_null());
assert!(!lzma_result.is_null());
unsafe {
*read_size = 0;
*compressed_size = 0;
*lzma_result = FIO_RUST_LZMA_OK_CODE;
}
let projection = unsafe { &*projection };
let Some(read_fill) = projection.read_fill else {
return FIO_RUST_LZMA_INVALID_PROJECTION;
};
let Some(read_consume) = projection.read_consume else {
return FIO_RUST_LZMA_INVALID_PROJECTION;
};
let Some(write_acquire) = projection.write_acquire else {
return FIO_RUST_LZMA_INVALID_PROJECTION;
};
let Some(write_enqueue) = projection.write_enqueue else {
return FIO_RUST_LZMA_INVALID_PROJECTION;
};
let Some(write_release) = projection.write_release else {
return FIO_RUST_LZMA_INVALID_PROJECTION;
};
let Some(sparse_write_end) = projection.sparse_write_end else {
return FIO_RUST_LZMA_INVALID_PROJECTION;
};
let Some(lzma_init) = projection.lzma_init else {
return FIO_RUST_LZMA_INVALID_PROJECTION;
};
let Some(lzma_code) = projection.lzma_code else {
return FIO_RUST_LZMA_INVALID_PROJECTION;
};
let Some(lzma_end) = projection.lzma_end else {
return FIO_RUST_LZMA_INVALID_PROJECTION;
};
let Some(progress) = projection.progress else {
return FIO_RUST_LZMA_INVALID_PROJECTION;
};
let compression_level = compression_level.clamp(0, 9);
let init_result = unsafe {
lzma_init(
projection.lzma_opaque,
compression_level,
plain_lzma,
lzma_result,
)
};
if init_result != FIO_RUST_LZMA_OK {
return init_result;
}
let mut job = ptr::null_mut::<c_void>();
let mut output = ptr::null_mut::<u8>();
let mut output_size = 0_usize;
unsafe {
write_acquire(
projection.write_opaque,
&mut job,
&mut output,
&mut output_size,
);
}
assert!(!job.is_null());
assert!(!output.is_null() || output_size == 0);
let mut action = FIO_RUST_LZMA_RUN;
let mut input = ptr::null();
let mut input_size = 0_usize;
let mut in_file_size = 0_u64;
let mut out_file_size = 0_u64;
loop {
if input_size == 0 {
let mut loaded = 0_usize;
unsafe {
read_fill(
projection.read_opaque,
projection.read_buffer_size,
&mut input,
&mut loaded,
);
}
if loaded == 0 {
action = FIO_RUST_LZMA_FINISH;
} else {
assert!(!input.is_null());
input_size = loaded;
in_file_size = in_file_size.wrapping_add(loaded as u64);
}
}
let mut consumed = 0_usize;
let mut produced = 0_usize;
let result = unsafe {
lzma_code(
projection.lzma_opaque,
input,
input_size,
output,
output_size,
action,
&mut consumed,
&mut produced,
)
};
assert!(consumed <= input_size);
assert!(produced <= output_size);
unsafe { read_consume(projection.read_opaque, consumed) };
if consumed != 0 {
input = unsafe { input.add(consumed) };
}
input_size -= consumed;
if result != FIO_RUST_LZMA_OK_CODE && result != FIO_RUST_LZMA_STREAM_END {
unsafe { *lzma_result = result };
return FIO_RUST_LZMA_CODE_ERROR;
}
if produced != 0 {
unsafe {
write_enqueue(
projection.write_opaque,
&mut job,
produced,
&mut output,
&mut output_size,
);
}
out_file_size = out_file_size.wrapping_add(produced as u64);
}
unsafe {
progress(
projection.progress_opaque,
src_file_size,
in_file_size,
out_file_size,
)
};
if result == FIO_RUST_LZMA_STREAM_END {
break;
}
}
unsafe { lzma_end(projection.lzma_opaque) };
unsafe {
*read_size = in_file_size;
*compressed_size = out_file_size;
write_release(projection.write_opaque, job);
sparse_write_end(projection.write_opaque);
}
FIO_RUST_LZMA_OK
}
/// Decompresses exactly one zstd frame using the existing asynchronous pools.
///
/// C retains the frame dispatcher and all user-facing diagnostics. This ABI
@@ -2927,6 +3423,719 @@ mod tests {
assert_eq!(state.sparse_end_calls, 0);
}
struct LzmaProjectionState {
input: [u8; 5],
input_pos: usize,
output_buffer: [u8; 4],
read_fill_calls: usize,
consumed: Vec<usize>,
enqueue_sizes: Vec<usize>,
output_chunks: Vec<Vec<u8>>,
acquire_calls: usize,
release_calls: usize,
sparse_end_calls: usize,
init_args: Vec<(c_int, c_int)>,
init_status: c_int,
init_result: c_int,
actions: Vec<c_int>,
finish_calls: usize,
code_result: Option<c_int>,
end_calls: usize,
progress: Vec<(u64, u64, u64)>,
}
impl Default for LzmaProjectionState {
fn default() -> Self {
Self {
input: *b"abcde",
input_pos: 0,
output_buffer: [0; 4],
read_fill_calls: 0,
consumed: Vec::new(),
enqueue_sizes: Vec::new(),
output_chunks: Vec::new(),
acquire_calls: 0,
release_calls: 0,
sparse_end_calls: 0,
init_args: Vec::new(),
init_status: FIO_RUST_LZMA_OK,
init_result: FIO_RUST_LZMA_OK_CODE,
actions: Vec::new(),
finish_calls: 0,
code_result: None,
end_calls: 0,
progress: Vec::new(),
}
}
}
unsafe extern "C" fn lzma_test_read_fill(
opaque: *mut c_void,
requested: usize,
buffer: *mut *const u8,
loaded: *mut usize,
) {
let state = unsafe { &mut *opaque.cast::<LzmaProjectionState>() };
state.read_fill_calls += 1;
let available = state.input.len() - state.input_pos;
let amount = available.min(requested);
unsafe {
*buffer = state.input.as_ptr().add(state.input_pos);
*loaded = amount;
}
}
unsafe extern "C" fn lzma_test_read_consume(opaque: *mut c_void, amount: usize) {
let state = unsafe { &mut *opaque.cast::<LzmaProjectionState>() };
assert!(amount <= state.input.len() - state.input_pos);
state.input_pos += amount;
state.consumed.push(amount);
}
unsafe extern "C" fn lzma_test_write_acquire(
opaque: *mut c_void,
job: *mut *mut c_void,
buffer: *mut *mut u8,
buffer_size: *mut usize,
) {
let state = unsafe { &mut *opaque.cast::<LzmaProjectionState>() };
state.acquire_calls += 1;
unsafe {
*job = opaque;
*buffer = state.output_buffer.as_mut_ptr();
*buffer_size = state.output_buffer.len();
}
}
unsafe extern "C" fn lzma_test_write_enqueue(
opaque: *mut c_void,
job: *mut *mut c_void,
used: usize,
buffer: *mut *mut u8,
buffer_size: *mut usize,
) {
let state = unsafe { &mut *opaque.cast::<LzmaProjectionState>() };
assert_eq!(unsafe { *job }, opaque);
assert!(used <= state.output_buffer.len());
state.enqueue_sizes.push(used);
state
.output_chunks
.push(state.output_buffer[..used].to_vec());
unsafe {
*buffer = state.output_buffer.as_mut_ptr();
*buffer_size = state.output_buffer.len();
}
}
unsafe extern "C" fn lzma_test_write_release(opaque: *mut c_void, job: *mut c_void) {
let state = unsafe { &mut *opaque.cast::<LzmaProjectionState>() };
assert_eq!(job, opaque);
state.release_calls += 1;
}
unsafe extern "C" fn lzma_test_sparse_end(opaque: *mut c_void) {
let state = unsafe { &mut *opaque.cast::<LzmaProjectionState>() };
state.sparse_end_calls += 1;
}
unsafe extern "C" fn lzma_test_init(
opaque: *mut c_void,
level: c_int,
plain_lzma: c_int,
result: *mut c_int,
) -> c_int {
let state = unsafe { &mut *opaque.cast::<LzmaProjectionState>() };
state.init_args.push((level, plain_lzma));
unsafe { *result = state.init_result };
state.init_status
}
unsafe extern "C" fn lzma_test_code(
opaque: *mut c_void,
_input: *const u8,
input_size: usize,
output: *mut u8,
output_size: usize,
action: c_int,
consumed: *mut usize,
produced: *mut usize,
) -> c_int {
let state = unsafe { &mut *opaque.cast::<LzmaProjectionState>() };
state.actions.push(action);
unsafe {
*consumed = 0;
*produced = 0;
}
if action == FIO_RUST_LZMA_RUN {
assert!(input_size != 0);
assert!(output_size != 0);
unsafe {
*consumed = input_size.min(2);
*produced = 1;
*output = b'r';
}
return state.code_result.unwrap_or(FIO_RUST_LZMA_OK_CODE);
}
assert_eq!(action, FIO_RUST_LZMA_FINISH);
assert_eq!(input_size, 0);
assert!(output_size != 0);
state.finish_calls += 1;
if state.finish_calls == 1 {
unsafe {
*produced = 1;
*output = b'f';
}
return FIO_RUST_LZMA_OK_CODE;
}
assert!(output_size >= 2);
unsafe {
*produced = 2;
*output.add(0) = b'g';
*output.add(1) = b'h';
}
FIO_RUST_LZMA_STREAM_END
}
unsafe extern "C" fn lzma_test_end(opaque: *mut c_void) {
let state = unsafe { &mut *opaque.cast::<LzmaProjectionState>() };
state.end_calls += 1;
}
unsafe extern "C" fn lzma_test_progress(
opaque: *mut c_void,
src_file_size: u64,
in_file_size: u64,
out_file_size: u64,
) {
let state = unsafe { &mut *opaque.cast::<LzmaProjectionState>() };
state
.progress
.push((src_file_size, in_file_size, out_file_size));
}
fn lzma_test_projection(
state: &mut LzmaProjectionState,
) -> FIO_rust_lzma_compress_projection_t {
let opaque = (state as *mut LzmaProjectionState).cast::<c_void>();
FIO_rust_lzma_compress_projection_t {
read_opaque: opaque,
write_opaque: opaque,
lzma_opaque: opaque,
progress_opaque: opaque,
read_buffer_size: 3,
read_fill: Some(lzma_test_read_fill),
read_consume: Some(lzma_test_read_consume),
write_acquire: Some(lzma_test_write_acquire),
write_enqueue: Some(lzma_test_write_enqueue),
write_release: Some(lzma_test_write_release),
sparse_write_end: Some(lzma_test_sparse_end),
lzma_init: Some(lzma_test_init),
lzma_code: Some(lzma_test_code),
lzma_end: Some(lzma_test_end),
progress: Some(lzma_test_progress),
}
}
#[test]
fn lzma_projection_preserves_run_finish_order_and_accounting() {
let mut state = LzmaProjectionState::default();
let projection = lzma_test_projection(&mut state);
let mut read_size = 0;
let mut compressed_size = 0;
let mut lzma_result = 0;
assert_eq!(
unsafe {
FIO_rust_compressLzmaFrame(
&projection,
c"lzma-test".as_ptr(),
5,
99,
1,
&mut read_size,
&mut compressed_size,
&mut lzma_result,
)
},
FIO_RUST_LZMA_OK
);
assert_eq!(read_size, 5);
assert_eq!(compressed_size, 6);
assert_eq!(lzma_result, FIO_RUST_LZMA_OK_CODE);
assert_eq!(state.init_args, vec![(9, 1)]);
assert_eq!(state.read_fill_calls, 4);
assert_eq!(state.consumed, vec![2, 1, 2, 0, 0]);
assert_eq!(
state.actions,
vec![
FIO_RUST_LZMA_RUN,
FIO_RUST_LZMA_RUN,
FIO_RUST_LZMA_RUN,
FIO_RUST_LZMA_FINISH,
FIO_RUST_LZMA_FINISH,
]
);
assert_eq!(state.enqueue_sizes, vec![1, 1, 1, 1, 2]);
assert_eq!(
state.output_chunks,
vec![
vec![b'r'],
vec![b'r'],
vec![b'r'],
vec![b'f'],
vec![b'g', b'h']
]
);
assert_eq!(state.acquire_calls, 1);
assert_eq!(state.release_calls, 1);
assert_eq!(state.sparse_end_calls, 1);
assert_eq!(state.finish_calls, 2);
assert_eq!(state.end_calls, 1);
assert_eq!(
state.progress,
vec![(5, 3, 1), (5, 3, 2), (5, 5, 3), (5, 5, 4), (5, 5, 6)]
);
}
#[test]
fn lzma_projection_clamps_level_and_stops_before_pool_on_init_error() {
let mut state = LzmaProjectionState {
init_status: FIO_RUST_LZMA_INIT_XZ_ERROR,
init_result: 11,
..LzmaProjectionState::default()
};
let projection = lzma_test_projection(&mut state);
let mut read_size = 41;
let mut compressed_size = 43;
let mut lzma_result = 0;
assert_eq!(
unsafe {
FIO_rust_compressLzmaFrame(
&projection,
c"lzma-test".as_ptr(),
5,
-7,
0,
&mut read_size,
&mut compressed_size,
&mut lzma_result,
)
},
FIO_RUST_LZMA_INIT_XZ_ERROR
);
assert_eq!(lzma_result, 11);
assert_eq!(read_size, 0);
assert_eq!(compressed_size, 0);
assert_eq!(state.init_args, vec![(0, 0)]);
assert_eq!(state.acquire_calls, 0);
assert_eq!(state.end_calls, 0);
assert_eq!(state.sparse_end_calls, 0);
}
#[test]
fn lzma_projection_propagates_code_error_after_consuming_input() {
let mut state = LzmaProjectionState {
code_result: Some(17),
..LzmaProjectionState::default()
};
let projection = lzma_test_projection(&mut state);
let mut read_size = 0;
let mut compressed_size = 0;
let mut lzma_result = 0;
assert_eq!(
unsafe {
FIO_rust_compressLzmaFrame(
&projection,
c"lzma-test".as_ptr(),
5,
3,
1,
&mut read_size,
&mut compressed_size,
&mut lzma_result,
)
},
FIO_RUST_LZMA_CODE_ERROR
);
assert_eq!(lzma_result, 17);
assert_eq!(read_size, 0);
assert_eq!(compressed_size, 0);
assert_eq!(state.consumed, vec![2]);
assert_eq!(state.enqueue_sizes, Vec::<usize>::new());
assert_eq!(state.release_calls, 0);
assert_eq!(state.end_calls, 0);
assert_eq!(state.sparse_end_calls, 0);
}
struct Lz4ProjectionState {
input: [u8; 7],
input_len: usize,
input_pos: usize,
output_buffer: [u8; 8],
events: Vec<&'static str>,
consumed: Vec<usize>,
enqueue_sizes: Vec<usize>,
progress: Vec<(u64, u64, u64)>,
prepare_args: Vec<(u64, c_int, c_int, usize, usize)>,
acquire_calls: usize,
release_calls: usize,
sparse_end_calls: usize,
create_status: c_int,
create_result: usize,
begin_status: c_int,
begin_result: usize,
update_status: c_int,
update_result: usize,
end_status: c_int,
end_result: usize,
}
impl Default for Lz4ProjectionState {
fn default() -> Self {
Self {
input: *b"abcdefg",
input_len: 7,
input_pos: 0,
output_buffer: [0; 8],
events: Vec::new(),
consumed: Vec::new(),
enqueue_sizes: Vec::new(),
progress: Vec::new(),
prepare_args: Vec::new(),
acquire_calls: 0,
release_calls: 0,
sparse_end_calls: 0,
create_status: 0,
create_result: 0,
begin_status: 0,
begin_result: 1,
update_status: 0,
update_result: 2,
end_status: 0,
end_result: 3,
}
}
}
unsafe extern "C" fn lz4_test_create(
opaque: *mut c_void,
_version: c_uint,
result: *mut usize,
) -> c_int {
let state = unsafe { &mut *opaque.cast::<Lz4ProjectionState>() };
state.events.push("create");
unsafe { *result = state.create_result };
state.create_status
}
unsafe extern "C" fn lz4_test_prepare(
opaque: *mut c_void,
src_file_size: u64,
compression_level: c_int,
checksum_flag: c_int,
block_size: usize,
buffer_size: usize,
) {
let state = unsafe { &mut *opaque.cast::<Lz4ProjectionState>() };
state.events.push("prepare");
state.prepare_args.push((
src_file_size,
compression_level,
checksum_flag,
block_size,
buffer_size,
));
}
unsafe extern "C" fn lz4_test_begin(
opaque: *mut c_void,
_output: *mut u8,
_output_size: usize,
result: *mut usize,
) -> c_int {
let state = unsafe { &mut *opaque.cast::<Lz4ProjectionState>() };
state.events.push("begin");
unsafe { *result = state.begin_result };
state.begin_status
}
unsafe extern "C" fn lz4_test_update(
opaque: *mut c_void,
output: *mut u8,
output_size: usize,
_input: *const u8,
input_size: usize,
result: *mut usize,
) -> c_int {
let state = unsafe { &mut *opaque.cast::<Lz4ProjectionState>() };
state.events.push("update");
assert!(input_size != 0);
if state.update_status == 0 {
assert!(output_size >= state.update_result);
unsafe {
*output = b'u';
*result = state.update_result;
}
} else {
unsafe { *result = state.update_result };
}
state.update_status
}
unsafe extern "C" fn lz4_test_end(
opaque: *mut c_void,
_output: *mut u8,
_output_size: usize,
result: *mut usize,
) -> c_int {
let state = unsafe { &mut *opaque.cast::<Lz4ProjectionState>() };
state.events.push("end");
unsafe { *result = state.end_result };
state.end_status
}
unsafe extern "C" fn lz4_test_free(opaque: *mut c_void) {
let state = unsafe { &mut *opaque.cast::<Lz4ProjectionState>() };
state.events.push("free");
}
unsafe extern "C" fn lz4_test_read_fill(
opaque: *mut c_void,
requested: usize,
buffer: *mut *const u8,
loaded: *mut usize,
) -> usize {
let state = unsafe { &mut *opaque.cast::<Lz4ProjectionState>() };
state.events.push("fill");
let available = state.input_len - state.input_pos;
let amount = available.min(requested);
unsafe {
*buffer = state.input.as_ptr().add(state.input_pos);
*loaded = amount;
}
amount
}
unsafe extern "C" fn lz4_test_read_consume(opaque: *mut c_void, amount: usize) {
let state = unsafe { &mut *opaque.cast::<Lz4ProjectionState>() };
state.events.push("consume");
assert!(amount <= state.input_len - state.input_pos);
state.input_pos += amount;
state.consumed.push(amount);
}
unsafe extern "C" fn lz4_test_write_acquire(
opaque: *mut c_void,
job: *mut *mut c_void,
buffer: *mut *mut u8,
buffer_size: *mut usize,
) {
let state = unsafe { &mut *opaque.cast::<Lz4ProjectionState>() };
state.events.push("acquire");
state.acquire_calls += 1;
unsafe {
*job = opaque;
*buffer = state.output_buffer.as_mut_ptr();
*buffer_size = state.output_buffer.len();
}
}
unsafe extern "C" fn lz4_test_write_enqueue(
opaque: *mut c_void,
job: *mut *mut c_void,
used: usize,
buffer: *mut *mut u8,
buffer_size: *mut usize,
) {
let state = unsafe { &mut *opaque.cast::<Lz4ProjectionState>() };
state.events.push("enqueue");
assert_eq!(unsafe { *job }, opaque);
assert!(used <= state.output_buffer.len());
state.enqueue_sizes.push(used);
unsafe {
*buffer = state.output_buffer.as_mut_ptr();
*buffer_size = state.output_buffer.len();
}
}
unsafe extern "C" fn lz4_test_write_release(opaque: *mut c_void, job: *mut c_void) {
let state = unsafe { &mut *opaque.cast::<Lz4ProjectionState>() };
state.events.push("release");
assert_eq!(job, opaque);
state.release_calls += 1;
}
unsafe extern "C" fn lz4_test_sparse_end(opaque: *mut c_void) {
let state = unsafe { &mut *opaque.cast::<Lz4ProjectionState>() };
state.events.push("sparse");
state.sparse_end_calls += 1;
}
unsafe extern "C" fn lz4_test_progress(
opaque: *mut c_void,
src_file_size: u64,
in_file_size: u64,
out_file_size: u64,
) {
let state = unsafe { &mut *opaque.cast::<Lz4ProjectionState>() };
state.events.push("progress");
state
.progress
.push((src_file_size, in_file_size, out_file_size));
}
fn lz4_test_projection(state: &mut Lz4ProjectionState) -> FIO_rust_lz4_compress_projection_t {
let opaque = (state as *mut Lz4ProjectionState).cast::<c_void>();
FIO_rust_lz4_compress_projection_t {
read_opaque: opaque,
write_opaque: opaque,
codec_opaque: opaque,
progress_opaque: opaque,
version: 100,
block_size: 3,
create: Some(lz4_test_create),
prepare: Some(lz4_test_prepare),
begin: Some(lz4_test_begin),
update: Some(lz4_test_update),
end: Some(lz4_test_end),
free_context: Some(lz4_test_free),
read_fill: Some(lz4_test_read_fill),
read_consume: Some(lz4_test_read_consume),
write_acquire: Some(lz4_test_write_acquire),
write_enqueue: Some(lz4_test_write_enqueue),
write_release: Some(lz4_test_write_release),
sparse_write_end: Some(lz4_test_sparse_end),
progress: Some(lz4_test_progress),
}
}
#[test]
fn lz4_projection_preserves_header_update_end_order_and_accounting() {
let mut state = Lz4ProjectionState::default();
let projection = lz4_test_projection(&mut state);
let mut read_size = 0;
let mut compressed_size = 0;
let mut lz4_result = 0;
assert_eq!(
unsafe {
FIO_rust_compressLz4Frame(
&projection,
c"lz4-test".as_ptr(),
7,
12,
1,
&mut read_size,
&mut compressed_size,
&mut lz4_result,
)
},
FIO_RUST_LZ4_OK
);
assert_eq!(read_size, 7);
assert_eq!(compressed_size, 10);
assert_eq!(lz4_result, 0);
assert_eq!(state.prepare_args, vec![(7, 12, 1, 3, 8)]);
assert_eq!(state.consumed, vec![3, 3, 1]);
assert_eq!(state.enqueue_sizes, vec![1, 2, 2, 2, 3]);
assert_eq!(state.progress, vec![(7, 3, 3), (7, 6, 5), (7, 7, 7)]);
assert_eq!(state.acquire_calls, 1);
assert_eq!(state.release_calls, 1);
assert_eq!(state.sparse_end_calls, 1);
assert_eq!(
state.events,
vec![
"acquire", "create", "prepare", "begin", "enqueue", "fill", "update", "progress",
"enqueue", "consume", "fill", "update", "progress", "enqueue", "consume", "fill",
"update", "progress", "enqueue", "consume", "fill", "end", "enqueue", "free",
"release", "sparse",
]
);
}
#[test]
fn lz4_projection_reports_creation_header_update_and_end_errors() {
let cases = [
(
Lz4ProjectionState {
create_status: 1,
create_result: 17,
..Lz4ProjectionState::default()
},
FIO_RUST_LZ4_CREATE_ERROR,
17,
vec!["acquire", "create"],
),
(
Lz4ProjectionState {
begin_status: 1,
begin_result: 23,
..Lz4ProjectionState::default()
},
FIO_RUST_LZ4_HEADER_ERROR,
23,
vec!["acquire", "create", "prepare", "begin"],
),
(
Lz4ProjectionState {
update_status: 1,
update_result: 29,
..Lz4ProjectionState::default()
},
FIO_RUST_LZ4_UPDATE_ERROR,
29,
vec![
"acquire", "create", "prepare", "begin", "enqueue", "fill", "update",
],
),
(
Lz4ProjectionState {
input_len: 0,
end_status: 1,
end_result: 31,
..Lz4ProjectionState::default()
},
FIO_RUST_LZ4_END_ERROR,
31,
vec![
"acquire", "create", "prepare", "begin", "enqueue", "fill", "end",
],
),
];
for (mut state, expected_status, expected_result, expected_events) in cases {
let projection = lz4_test_projection(&mut state);
let mut read_size = 41;
let mut compressed_size = 43;
let mut lz4_result = 0;
assert_eq!(
unsafe {
FIO_rust_compressLz4Frame(
&projection,
c"lz4-test".as_ptr(),
7,
1,
0,
&mut read_size,
&mut compressed_size,
&mut lz4_result,
)
},
expected_status
);
assert_eq!(lz4_result, expected_result);
assert_eq!(read_size, 0);
assert_eq!(compressed_size, 0);
assert_eq!(state.events, expected_events);
assert_eq!(state.release_calls, 0);
assert_eq!(state.sparse_end_calls, 0);
}
}
#[test]
fn classifies_all_cli_decompression_headers() {
let is_mock_zstd = |buffer: &[u8]| buffer.starts_with(&[0x28, 0xB5, 0x2F, 0xFD]);