ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
3bbfacdfd0112023abb60d3c62cf38362cf605d3
zstd-rs/rust/src/fileio_prefs.rs
T
ddidderr ce992fb0be feat(cli): move cycle log policy to Rust 
Move the CLI's strategy-dependent cycle-log calculation behind a scalar Rust ABI shim while retaining the C assertion and call-site behavior.

Test Plan:

- cargo test --manifest-path rust/cli/Cargo.toml --no-default-features --features cli,compression,decompression,benchmark (116 tests)

- cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets with the full CLI feature set

- make -B -C programs -j2 zstd zstd-small zstd-frugal

- make -C tests -j2 test-cli-tests (41 scenarios)
2026-07-18 05:06:48 +02:00

1668 lines
56 KiB
Rust
Raw Blame History

#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
//! Rust implementation of the small preference/context part of `fileio.c`.
//!
//! The file-I/O operations themselves remain in C. These layouts are passed
//! directly to that code, so allocation deliberately uses the C allocator and
//! preference creation initializes exactly the fields initialized by the C
//! implementation. In particular, the four legacy fields which C leaves
//! unspecified are not zero-filled here either.
use std::ffi::{c_void, CStr};
use std::io::{self, Write};
use std::mem::size_of;
use std::os::raw::{c_char, c_int, c_uint};
use std::ptr;
const FIO_ZSTD_COMPRESSION: c_int = 0;
const FIO_OVERLAP_LOG_NOTSET: c_int = 9999;
const FIO_LDM_PARAM_NOTSET: c_int = 9999;
const FIO_PATCH_MEM_LIMIT_SUCCESS: c_int = 0;
const FIO_PATCH_MEM_LIMIT_UNKNOWN_SIZE: c_int = 1;
const FIO_PATCH_MEM_LIMIT_TOO_LARGE: c_int = 2;
const UTIL_FILESIZE_UNKNOWN: u64 = u64::MAX;
const ZSTD_WINDOWLOG_MAX: u32 = if size_of::<usize>() == 4 { 30 } else { 31 };
const ZSTD_BTLAZY2: c_int = 6;
static STDOUT_MARK: &[u8] = b"/*stdout*\\\0";
static mut COMPRESSED_NAME_CAPACITY: usize = 0;
static mut COMPRESSED_NAME_BUFFER: *mut c_char = ptr::null_mut();
static mut DESTINATION_NAME_CAPACITY: usize = 0;
static mut DESTINATION_NAME_BUFFER: *mut c_char = ptr::null_mut();
#[repr(C)]
pub struct FIO_inBuffer {
src: *const c_void,
size: usize,
pos: usize,
}
#[repr(C)]
pub struct FIO_outBuffer {
dst: *mut c_void,
size: usize,
pos: usize,
}
/// C's private `fileInfo_t` from `programs/fileio.c`.
#[repr(C)]
pub struct FIO_fileInfo_t {
decompressedSize: u64,
compressedSize: u64,
windowSize: u64,
numActualFrames: c_int,
numSkippableFrames: c_int,
decompUnavailable: c_int,
usesCheck: c_int,
checksum: [u8; 4],
nbFiles: u32,
dictID: c_uint,
}
#[cfg(target_vendor = "apple")]
const ZSTD_SPARSE_DEFAULT: c_int = 0;
#[cfg(not(target_vendor = "apple"))]
const ZSTD_SPARSE_DEFAULT: c_int = 1;
#[repr(C)]
struct FIO_display_prefs_t {
displayLevel: c_int,
progressSetting: c_int,
}
/// C's `FIO_prefs_t` from `programs/fileio_types.h`.
#[repr(C)]
pub struct FIO_prefs_t {
compressionType: c_int,
sparseFileSupport: c_int,
dictIDFlag: c_int,
checksumFlag: c_int,
blockSize: c_int,
overlapLog: c_int,
adaptiveMode: c_int,
useRowMatchFinder: c_int,
rsyncable: c_int,
minAdaptLevel: c_int,
maxAdaptLevel: c_int,
ldmFlag: c_int,
ldmHashLog: c_int,
ldmMinMatch: c_int,
ldmBucketSizeLog: c_int,
ldmHashRateLog: c_int,
streamSrcSize: usize,
targetCBlockSize: usize,
srcSizeHint: c_int,
testMode: c_int,
literalCompressionMode: c_int,
removeSrcFile: c_int,
overwrite: c_int,
asyncIO: c_int,
memLimit: c_uint,
nbWorkers: c_int,
excludeCompressedFiles: c_int,
patchFromMode: c_int,
contentSize: c_int,
allowBlockDevices: c_int,
passThrough: c_int,
mmapDict: c_int,
}
/// C's private `FIO_ctx_s` from `programs/fileio.c`.
#[repr(C)]
pub struct FIO_ctx_t {
nbFilesTotal: c_int,
hasStdinInput: c_int,
hasStdoutOutput: c_int,
currFileIdx: c_int,
nbFilesProcessed: c_int,
totalBytesInput: usize,
totalBytesOutput: usize,
}
/// Mirror of the `FileNamesTable` used by `FIO_determineHasStdinInput`.
#[repr(C)]
pub struct FileNamesTable {
fileNames: *mut *const c_char,
buf: *mut c_char,
tableSize: usize,
tableCapacity: usize,
}
#[cfg(not(test))]
unsafe extern "C" {
static mut g_display_prefs: FIO_display_prefs_t;
fn AIO_supported() -> c_int;
#[cfg(feature = "compression")]
fn ZSTD_minCLevel() -> c_int;
}
#[cfg(test)]
static mut TEST_DISPLAY_PREFS: FIO_display_prefs_t = FIO_display_prefs_t {
displayLevel: 2,
progressSetting: 0,
};
#[inline]
unsafe fn display_prefs() -> *mut FIO_display_prefs_t {
#[cfg(test)]
{
ptr::addr_of_mut!(TEST_DISPLAY_PREFS)
}
#[cfg(not(test))]
{
ptr::addr_of_mut!(g_display_prefs)
}
}
#[inline]
unsafe fn aio_supported() -> bool {
#[cfg(test)]
{
false
}
#[cfg(not(test))]
{
unsafe { AIO_supported() != 0 }
}
}
fn display(level: c_int, message: &str) {
let enabled = unsafe { (*display_prefs()).displayLevel >= level };
if enabled {
eprint!("{message}");
}
}
fn throw(error: c_int, message: &str) -> ! {
let enabled = unsafe { (*display_prefs()).displayLevel >= 1 };
if enabled {
eprintln!("zstd: error {error} : {message} ");
}
std::process::exit(error);
}
unsafe fn allocate<T>() -> *mut T {
let allocation = unsafe { libc::malloc(size_of::<T>()) }.cast::<T>();
if allocation.is_null() {
throw(21, "Allocation error : not enough memory");
}
allocation
}
#[no_mangle]
pub unsafe extern "C" fn FIO_createPreferences() -> *mut FIO_prefs_t {
let ret = unsafe { allocate::<FIO_prefs_t>() };
/* Keep this list in lock-step with the original C initialization. The
* useRowMatchFinder, patchFromMode, contentSize, and mmapDict fields are
* intentionally left unspecified until their CLI setters are called. */
unsafe {
ptr::addr_of_mut!((*ret).compressionType).write(FIO_ZSTD_COMPRESSION);
ptr::addr_of_mut!((*ret).overwrite).write(0);
ptr::addr_of_mut!((*ret).sparseFileSupport).write(ZSTD_SPARSE_DEFAULT);
ptr::addr_of_mut!((*ret).dictIDFlag).write(1);
ptr::addr_of_mut!((*ret).checksumFlag).write(1);
ptr::addr_of_mut!((*ret).removeSrcFile).write(0);
ptr::addr_of_mut!((*ret).memLimit).write(0);
ptr::addr_of_mut!((*ret).nbWorkers).write(1);
ptr::addr_of_mut!((*ret).blockSize).write(0);
ptr::addr_of_mut!((*ret).overlapLog).write(FIO_OVERLAP_LOG_NOTSET);
ptr::addr_of_mut!((*ret).adaptiveMode).write(0);
ptr::addr_of_mut!((*ret).rsyncable).write(0);
ptr::addr_of_mut!((*ret).minAdaptLevel).write(-50);
ptr::addr_of_mut!((*ret).maxAdaptLevel).write(22);
ptr::addr_of_mut!((*ret).ldmFlag).write(0);
ptr::addr_of_mut!((*ret).ldmHashLog).write(0);
ptr::addr_of_mut!((*ret).ldmMinMatch).write(0);
ptr::addr_of_mut!((*ret).ldmBucketSizeLog).write(FIO_LDM_PARAM_NOTSET);
ptr::addr_of_mut!((*ret).ldmHashRateLog).write(FIO_LDM_PARAM_NOTSET);
ptr::addr_of_mut!((*ret).streamSrcSize).write(0);
ptr::addr_of_mut!((*ret).targetCBlockSize).write(0);
ptr::addr_of_mut!((*ret).srcSizeHint).write(0);
ptr::addr_of_mut!((*ret).testMode).write(0);
ptr::addr_of_mut!((*ret).literalCompressionMode).write(0);
ptr::addr_of_mut!((*ret).excludeCompressedFiles).write(0);
ptr::addr_of_mut!((*ret).allowBlockDevices).write(0);
ptr::addr_of_mut!((*ret).asyncIO).write(c_int::from(aio_supported()));
ptr::addr_of_mut!((*ret).passThrough).write(-1);
}
ret
}
#[no_mangle]
pub unsafe extern "C" fn FIO_createContext() -> *mut FIO_ctx_t {
let ret = unsafe { allocate::<FIO_ctx_t>() };
unsafe {
ptr::addr_of_mut!((*ret).currFileIdx).write(0);
ptr::addr_of_mut!((*ret).hasStdinInput).write(0);
ptr::addr_of_mut!((*ret).hasStdoutOutput).write(0);
ptr::addr_of_mut!((*ret).nbFilesTotal).write(1);
ptr::addr_of_mut!((*ret).nbFilesProcessed).write(0);
ptr::addr_of_mut!((*ret).totalBytesInput).write(0);
ptr::addr_of_mut!((*ret).totalBytesOutput).write(0);
}
ret
}
#[no_mangle]
pub unsafe extern "C" fn FIO_freePreferences(prefs: *mut FIO_prefs_t) {
unsafe { libc::free(prefs.cast::<c_void>()) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_freeContext(ctx: *mut FIO_ctx_t) {
unsafe { libc::free(ctx.cast::<c_void>()) };
}
#[no_mangle]
pub extern "C" fn FIO_setNotificationLevel(level: c_int) {
unsafe { (*display_prefs()).displayLevel = level };
}
#[no_mangle]
pub extern "C" fn FIO_setProgressSetting(setting: c_int) {
unsafe { (*display_prefs()).progressSetting = setting };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setCompressionType(prefs: *mut FIO_prefs_t, compression_type: c_int) {
unsafe { ptr::addr_of_mut!((*prefs).compressionType).write(compression_type) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_overwriteMode(prefs: *mut FIO_prefs_t) {
unsafe { ptr::addr_of_mut!((*prefs).overwrite).write(1) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setSparseWrite(prefs: *mut FIO_prefs_t, sparse: c_int) {
unsafe { ptr::addr_of_mut!((*prefs).sparseFileSupport).write(sparse) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setDictIDFlag(prefs: *mut FIO_prefs_t, dict_id_flag: c_int) {
unsafe { ptr::addr_of_mut!((*prefs).dictIDFlag).write(dict_id_flag) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setChecksumFlag(prefs: *mut FIO_prefs_t, checksum_flag: c_int) {
unsafe { ptr::addr_of_mut!((*prefs).checksumFlag).write(checksum_flag) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setRemoveSrcFile(prefs: *mut FIO_prefs_t, flag: c_int) {
unsafe { ptr::addr_of_mut!((*prefs).removeSrcFile).write(c_int::from(flag != 0)) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setMemLimit(prefs: *mut FIO_prefs_t, mem_limit: c_uint) {
unsafe { ptr::addr_of_mut!((*prefs).memLimit).write(mem_limit) };
}
fn adjusted_patch_mem_limit(
current_mem_limit: c_uint,
dict_size: u64,
max_src_file_size: u64,
) -> Result<c_uint, c_int> {
let max_size = u64::from(current_mem_limit)
.max(dict_size)
.max(max_src_file_size);
if max_size == UTIL_FILESIZE_UNKNOWN {
return Err(FIO_PATCH_MEM_LIMIT_UNKNOWN_SIZE);
}
if max_size > (1u64 << ZSTD_WINDOWLOG_MAX) {
return Err(FIO_PATCH_MEM_LIMIT_TOO_LARGE);
}
Ok(max_size as c_uint)
}
/// Adjust the patch-from memory limit without changing it on a rejected size.
///
/// The status values are mirrored by the C diagnostic wrapper immediately
/// below its ABI declarations.
#[no_mangle]
pub unsafe extern "C" fn FIO_rust_adjustMemLimitForPatchFromMode(
prefs: *mut FIO_prefs_t,
dict_size: u64,
max_src_file_size: u64,
) -> c_int {
let current_mem_limit = unsafe { (*prefs).memLimit };
match adjusted_patch_mem_limit(current_mem_limit, dict_size, max_src_file_size) {
Ok(mem_limit) => {
unsafe { ptr::addr_of_mut!((*prefs).memLimit).write(mem_limit) };
FIO_PATCH_MEM_LIMIT_SUCCESS
}
Err(status) => status,
}
}
#[cfg(feature = "compression")]
#[no_mangle]
pub unsafe extern "C" fn FIO_setNbWorkers(prefs: *mut FIO_prefs_t, nb_workers: c_int) {
if unsafe { !aio_supported() } && nb_workers > 0 {
display(2, "Note : multi-threading is disabled \n");
}
unsafe { ptr::addr_of_mut!((*prefs).nbWorkers).write(nb_workers) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setExcludeCompressedFile(
prefs: *mut FIO_prefs_t,
exclude_compressed_files: c_int,
) {
unsafe { ptr::addr_of_mut!((*prefs).excludeCompressedFiles).write(exclude_compressed_files) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setAllowBlockDevices(
prefs: *mut FIO_prefs_t,
allow_block_devices: c_int,
) {
unsafe { ptr::addr_of_mut!((*prefs).allowBlockDevices).write(allow_block_devices) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setBlockSize(prefs: *mut FIO_prefs_t, block_size: c_int) {
if block_size != 0 && unsafe { (*prefs).nbWorkers == 0 } {
display(2, "Setting block size is useless in single-thread mode \n");
}
unsafe { ptr::addr_of_mut!((*prefs).blockSize).write(block_size) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setOverlapLog(prefs: *mut FIO_prefs_t, overlap_log: c_int) {
if overlap_log != 0 && unsafe { (*prefs).nbWorkers == 0 } {
display(2, "Setting overlapLog is useless in single-thread mode \n");
}
unsafe { ptr::addr_of_mut!((*prefs).overlapLog).write(overlap_log) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setAdaptiveMode(prefs: *mut FIO_prefs_t, adapt: c_int) {
if adapt > 0 && unsafe { (*prefs).nbWorkers == 0 } {
throw(
1,
"Adaptive mode is not compatible with single thread mode \n",
);
}
unsafe { ptr::addr_of_mut!((*prefs).adaptiveMode).write(adapt) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setUseRowMatchFinder(
prefs: *mut FIO_prefs_t,
use_row_match_finder: c_int,
) {
unsafe { ptr::addr_of_mut!((*prefs).useRowMatchFinder).write(use_row_match_finder) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setRsyncable(prefs: *mut FIO_prefs_t, rsyncable: c_int) {
if rsyncable > 0 && unsafe { (*prefs).nbWorkers == 0 } {
throw(
1,
"Rsyncable mode is not compatible with single thread mode \n",
);
}
unsafe { ptr::addr_of_mut!((*prefs).rsyncable).write(rsyncable) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setStreamSrcSize(prefs: *mut FIO_prefs_t, stream_src_size: usize) {
unsafe { ptr::addr_of_mut!((*prefs).streamSrcSize).write(stream_src_size) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setTargetCBlockSize(
prefs: *mut FIO_prefs_t,
target_c_block_size: usize,
) {
unsafe { ptr::addr_of_mut!((*prefs).targetCBlockSize).write(target_c_block_size) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setSrcSizeHint(prefs: *mut FIO_prefs_t, src_size_hint: usize) {
let capped = src_size_hint.min(c_int::MAX as usize) as c_int;
unsafe { ptr::addr_of_mut!((*prefs).srcSizeHint).write(capped) };
}
#[cfg(feature = "decompression")]
#[no_mangle]
pub unsafe extern "C" fn FIO_setTestMode(prefs: *mut FIO_prefs_t, test_mode: c_int) {
unsafe { ptr::addr_of_mut!((*prefs).testMode).write(c_int::from(test_mode != 0)) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setLiteralCompressionMode(prefs: *mut FIO_prefs_t, mode: c_int) {
unsafe { ptr::addr_of_mut!((*prefs).literalCompressionMode).write(mode) };
}
#[cfg(feature = "compression")]
#[inline]
fn min_compression_level() -> c_int {
#[cfg(test)]
{
-50
}
#[cfg(not(test))]
{
unsafe { ZSTD_minCLevel() }
}
}
#[cfg(feature = "compression")]
#[no_mangle]
pub unsafe extern "C" fn FIO_setAdaptMin(prefs: *mut FIO_prefs_t, min_c_level: c_int) {
assert!(min_c_level >= min_compression_level());
unsafe { ptr::addr_of_mut!((*prefs).minAdaptLevel).write(min_c_level) };
}
#[cfg(feature = "compression")]
#[no_mangle]
pub unsafe extern "C" fn FIO_setAdaptMax(prefs: *mut FIO_prefs_t, max_c_level: c_int) {
unsafe { ptr::addr_of_mut!((*prefs).maxAdaptLevel).write(max_c_level) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setLdmFlag(prefs: *mut FIO_prefs_t, ldm_flag: c_uint) {
unsafe { ptr::addr_of_mut!((*prefs).ldmFlag).write(c_int::from(ldm_flag > 0)) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setLdmHashLog(prefs: *mut FIO_prefs_t, ldm_hash_log: c_int) {
unsafe { ptr::addr_of_mut!((*prefs).ldmHashLog).write(ldm_hash_log) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setLdmMinMatch(prefs: *mut FIO_prefs_t, ldm_min_match: c_int) {
unsafe { ptr::addr_of_mut!((*prefs).ldmMinMatch).write(ldm_min_match) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setLdmBucketSizeLog(
prefs: *mut FIO_prefs_t,
ldm_bucket_size_log: c_int,
) {
unsafe { ptr::addr_of_mut!((*prefs).ldmBucketSizeLog).write(ldm_bucket_size_log) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setLdmHashRateLog(prefs: *mut FIO_prefs_t, ldm_hash_rate_log: c_int) {
unsafe { ptr::addr_of_mut!((*prefs).ldmHashRateLog).write(ldm_hash_rate_log) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setPatchFromMode(prefs: *mut FIO_prefs_t, value: c_int) {
unsafe { ptr::addr_of_mut!((*prefs).patchFromMode).write(c_int::from(value != 0)) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setContentSize(prefs: *mut FIO_prefs_t, value: c_int) {
unsafe { ptr::addr_of_mut!((*prefs).contentSize).write(c_int::from(value != 0)) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setAsyncIOFlag(prefs: *mut FIO_prefs_t, value: c_int) {
if unsafe { aio_supported() } {
unsafe { ptr::addr_of_mut!((*prefs).asyncIO).write(value) };
} else {
display(
2,
"Note : asyncio is disabled (lack of multithreading support) \n",
);
}
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setPassThroughFlag(prefs: *mut FIO_prefs_t, value: c_int) {
unsafe { ptr::addr_of_mut!((*prefs).passThrough).write(c_int::from(value != 0)) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setMMapDict(prefs: *mut FIO_prefs_t, value: c_int) {
unsafe { ptr::addr_of_mut!((*prefs).mmapDict).write(value) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setHasStdoutOutput(ctx: *mut FIO_ctx_t, value: c_int) {
unsafe { ptr::addr_of_mut!((*ctx).hasStdoutOutput).write(value) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setNbFilesTotal(ctx: *mut FIO_ctx_t, value: c_int) {
unsafe { ptr::addr_of_mut!((*ctx).nbFilesTotal).write(value) };
}
#[no_mangle]
pub unsafe extern "C" fn FIO_setHasStdinInput(ctx: *mut FIO_ctx_t, value: c_int) {
unsafe { ptr::addr_of_mut!((*ctx).hasStdinInput).write(c_int::from(value != 0)) };
}
#[inline]
fn should_display_file_summary(ctx: &FIO_ctx_t) -> bool {
ctx.nbFilesTotal <= 1 || unsafe { (*display_prefs()).displayLevel >= 3 }
}
/// Return whether the single-file summary should be displayed.
#[no_mangle]
pub unsafe extern "C" fn FIO_rust_shouldDisplayFileSummary(ctx: *const FIO_ctx_t) -> c_int {
let ctx = unsafe { &*ctx };
c_int::from(should_display_file_summary(ctx))
}
/// Return whether the multiple-file summary should be displayed.
///
/// The assertion mirrors the original C helper's invariant exactly.
#[no_mangle]
pub unsafe extern "C" fn FIO_rust_shouldDisplayMultipleFileSummary(ctx: *const FIO_ctx_t) -> c_int {
let ctx = unsafe { &*ctx };
let should_display = ctx.nbFilesProcessed >= 1 && ctx.nbFilesTotal > 1;
assert!(should_display || should_display_file_summary(ctx) || ctx.nbFilesProcessed == 0);
c_int::from(should_display)
}
#[no_mangle]
pub unsafe extern "C" fn FIO_determineHasStdinInput(
ctx: *mut FIO_ctx_t,
filenames: *const FileNamesTable,
) {
for index in 0..unsafe { (*filenames).tableSize } {
let file_name = unsafe { *(*filenames).fileNames.add(index) };
if unsafe { libc::strcmp(c"/*stdin*\\".as_ptr(), file_name) == 0 } {
unsafe { ptr::addr_of_mut!((*ctx).hasStdinInput).write(1) };
return;
}
}
}
/// Return the position of the highest set bit in a non-zero 64-bit value.
///
/// This mirrors `FIO_highbit64()` and deliberately keeps its zero-input
/// assertion semantics. In builds where assertions are disabled, zero
/// still produces the same result as the original shift loop.
#[no_mangle]
pub extern "C" fn FIO_rust_highbit64(value: u64) -> c_uint {
debug_assert!(value != 0);
if value == 0 {
return 0;
}
(u64::BITS - 1 - value.leading_zeros()) as c_uint
}
#[inline]
fn cycle_log(hash_log: c_uint, strategy: c_int) -> c_uint {
hash_log.wrapping_sub(c_uint::from(strategy >= ZSTD_BTLAZY2))
}
/// Rust scalar ABI for the `ZSTD_cycleLog()` policy in `programs/fileio.c`.
#[no_mangle]
pub extern "C" fn FIO_rust_cycleLog(hash_log: c_uint, strategy: c_int) -> c_uint {
cycle_log(hash_log, strategy)
}
fn largest_file_size<I>(sizes: I) -> u64
where
I: IntoIterator<Item = u64>,
{
let mut largest = 0;
for size in sizes {
largest = size.max(largest);
}
largest
}
/// Rust implementation of the size scan used by `FIO_getLargestFileSize`.
///
/// `UTIL_getFileSize()` remains responsible for the actual stat operation.
/// In particular, an unknown size is `UTIL_FILESIZE_UNKNOWN` and therefore
/// wins the max scan exactly as it does in the original C implementation.
#[no_mangle]
pub unsafe extern "C" fn FIO_rust_getLargestFileSize(
file_names: *const *const c_char,
nb_files: c_uint,
) -> u64 {
largest_file_size(
(0..nb_files as usize)
.map(|index| unsafe { crate::util::UTIL_getFileSize(*file_names.add(index)) }),
)
}
/// Fill a C-compatible input buffer through an output pointer.
///
/// The pointer form avoids relying on a cross-language struct return ABI while
/// retaining the public `ZSTD_inBuffer` field order and values.
#[no_mangle]
pub unsafe extern "C" fn FIO_rust_setInBuffer(
output: *mut FIO_inBuffer,
buf: *const c_void,
size: usize,
pos: usize,
) {
unsafe {
output.write(FIO_inBuffer {
src: buf,
size,
pos,
});
}
}
/// Fill a C-compatible output buffer through an output pointer.
#[no_mangle]
pub unsafe extern "C" fn FIO_rust_setOutBuffer(
output: *mut FIO_outBuffer,
buf: *mut c_void,
size: usize,
pos: usize,
) {
unsafe {
output.write(FIO_outBuffer {
dst: buf,
size,
pos,
});
}
}
/// Aggregate two file-info records through an output pointer to avoid a
/// cross-language struct return ABI.
#[no_mangle]
pub unsafe extern "C" fn FIO_rust_addFInfo(
output: *mut FIO_fileInfo_t,
fi1: *const FIO_fileInfo_t,
fi2: *const FIO_fileInfo_t,
) {
let fi1 = unsafe { &*fi1 };
let fi2 = unsafe { &*fi2 };
let mut total = FIO_fileInfo_t {
decompressedSize: 0,
compressedSize: 0,
windowSize: 0,
numActualFrames: 0,
numSkippableFrames: 0,
decompUnavailable: 0,
usesCheck: 0,
checksum: [0; 4],
nbFiles: 0,
dictID: 0,
};
total.numActualFrames = fi1.numActualFrames.wrapping_add(fi2.numActualFrames);
total.numSkippableFrames = fi1.numSkippableFrames.wrapping_add(fi2.numSkippableFrames);
total.compressedSize = fi1.compressedSize.wrapping_add(fi2.compressedSize);
total.decompressedSize = fi1.decompressedSize.wrapping_add(fi2.decompressedSize);
total.decompUnavailable = fi1.decompUnavailable | fi2.decompUnavailable;
total.usesCheck = fi1.usesCheck & fi2.usesCheck;
total.nbFiles = fi1.nbFiles.wrapping_add(fi2.nbFiles);
unsafe { output.write(total) };
}
#[inline]
fn lz4_block_size_from_block_id(id: c_int) -> c_int {
1 << (8 + 2 * id)
}
#[no_mangle]
pub extern "C" fn FIO_rust_LZ4_GetBlockSize_FromBlockId(id: c_int) -> c_int {
lz4_block_size_from_block_id(id)
}
/// Build the compressed destination name used by the multi-file CLI path.
///
/// This deliberately keeps the original static allocation contract: callers
/// must consume the returned pointer before the next call, and the helper is
/// not thread-safe. The output-directory basename construction is shared
/// with `UTIL_createFilenameFromOutDir`, while suffix assembly remains here.
#[no_mangle]
pub unsafe extern "C" fn FIO_rust_determineCompressedName(
src_file_name: *const c_char,
out_dir_name: *const c_char,
suffix: *const c_char,
) -> *const c_char {
let source = unsafe { CStr::from_ptr(src_file_name).to_bytes() };
let suffix_bytes = unsafe { CStr::from_ptr(suffix).to_bytes() };
if source == b"/*stdin*\\" {
return STDOUT_MARK.as_ptr().cast();
}
let mut out_dir_filename = ptr::null_mut();
let source_bytes = if out_dir_name.is_null() {
source
} else {
out_dir_filename = unsafe {
crate::util::UTIL_createFilenameFromOutDir(
src_file_name,
out_dir_name,
suffix_bytes.len(),
)
};
assert!(!out_dir_filename.is_null());
unsafe { CStr::from_ptr(out_dir_filename).to_bytes() }
};
let source_len = source_bytes.len();
let suffix_len = suffix_bytes.len();
let required = source_len.wrapping_add(suffix_len).wrapping_add(1);
unsafe {
let capacity = ptr::addr_of_mut!(COMPRESSED_NAME_CAPACITY);
let buffer = ptr::addr_of_mut!(COMPRESSED_NAME_BUFFER);
if capacity.read() <= required {
libc::free(buffer.read().cast::<c_void>());
let new_capacity = source_len.wrapping_add(suffix_len).wrapping_add(30);
let new_buffer = libc::malloc(new_capacity).cast::<c_char>();
if new_buffer.is_null() {
throw(30, "Allocation error : not enough memory");
}
capacity.write(new_capacity);
buffer.write(new_buffer);
}
let destination = buffer.read();
assert!(!destination.is_null());
ptr::copy_nonoverlapping(source_bytes.as_ptr(), destination.cast(), source_len);
ptr::copy_nonoverlapping(
suffix_bytes.as_ptr(),
destination.add(source_len).cast(),
suffix_len + 1,
);
if !out_dir_filename.is_null() {
libc::free(out_dir_filename.cast::<c_void>());
}
destination.cast()
}
}
fn display_unknown_suffix(source: &[u8], suffix_list: &[u8]) {
let enabled = unsafe { (*display_prefs()).displayLevel >= 1 };
if !enabled {
return;
}
let mut stderr = io::stderr().lock();
let _ = stderr.write_all(b"zstd: ");
let _ = stderr.write_all(source);
let _ = stderr.write_all(b": unknown suffix (");
let _ = stderr.write_all(suffix_list);
let _ = stderr.write_all(
b" expected). Can't derive the output file name. Specify it with -o dstFileName. Ignoring.\n",
);
}
/// Build the decompressed destination name used by the multi-file CLI path.
///
/// The suffix table and its display string come from C because both are
/// selected by the C build configuration. As with the original helper, the
/// returned pointer refers to a static buffer and is overwritten by the next
/// successful call.
#[no_mangle]
pub unsafe extern "C" fn FIO_rust_determineDstName(
src_file_name: *const c_char,
out_dir_name: *const c_char,
suffix_list: *const *const c_char,
suffix_list_str: *const c_char,
) -> *const c_char {
let source = unsafe { CStr::from_ptr(src_file_name).to_bytes() };
let suffix_display = unsafe { CStr::from_ptr(suffix_list_str).to_bytes() };
if source == b"/*stdin*\\" {
return STDOUT_MARK.as_ptr().cast();
}
let Some(src_suffix_start) = source.iter().rposition(|&byte| byte == b'.') else {
display_unknown_suffix(source, suffix_display);
return ptr::null();
};
let src_suffix = &source[src_suffix_start..];
let src_suffix_len = src_suffix.len();
let mut matched_suffix = ptr::null();
let mut suffix_ptr = suffix_list;
unsafe {
while !suffix_ptr.is_null() {
let candidate = *suffix_ptr;
if candidate.is_null() {
break;
}
if CStr::from_ptr(candidate).to_bytes() == src_suffix {
matched_suffix = candidate;
break;
}
suffix_ptr = suffix_ptr.add(1);
}
}
if source.len() <= src_suffix_len || matched_suffix.is_null() {
display_unknown_suffix(source, suffix_display);
return ptr::null();
}
let matched_suffix_bytes = unsafe { CStr::from_ptr(matched_suffix).to_bytes() };
let dst_suffix: &[u8] = if matched_suffix_bytes.get(1) == Some(&b't') {
b".tar\0"
} else {
b"\0"
};
let dst_suffix_len = dst_suffix.len() - 1;
let mut out_dir_filename = ptr::null_mut();
let source_for_destination = if out_dir_name.is_null() {
source
} else {
out_dir_filename =
unsafe { crate::util::UTIL_createFilenameFromOutDir(src_file_name, out_dir_name, 0) };
assert!(!out_dir_filename.is_null());
unsafe { CStr::from_ptr(out_dir_filename).to_bytes() }
};
let source_len = source_for_destination.len();
unsafe {
let capacity = ptr::addr_of_mut!(DESTINATION_NAME_CAPACITY);
let buffer = ptr::addr_of_mut!(DESTINATION_NAME_BUFFER);
if capacity.read().wrapping_add(src_suffix_len)
<= source_len.wrapping_add(1).wrapping_add(dst_suffix_len)
{
libc::free(buffer.read().cast::<c_void>());
let new_capacity = source_len.wrapping_add(20);
let new_buffer = libc::malloc(new_capacity).cast::<c_char>();
if new_buffer.is_null() {
throw(
74,
&format!(
"{} : not enough memory for dstFileName",
std::io::Error::last_os_error()
),
);
}
capacity.write(new_capacity);
buffer.write(new_buffer);
}
let destination = buffer.read();
assert!(!destination.is_null());
let destination_end = source_len - src_suffix_len;
ptr::copy_nonoverlapping(
source_for_destination.as_ptr(),
destination.cast::<u8>(),
destination_end,
);
ptr::copy_nonoverlapping(
dst_suffix.as_ptr(),
destination.add(destination_end).cast::<u8>(),
dst_suffix.len(),
);
if !out_dir_filename.is_null() {
libc::free(out_dir_filename.cast::<c_void>());
}
destination.cast()
}
}
#[inline]
fn filename_path_separator() -> u8 {
if cfg!(windows) {
b'\\'
} else {
b'/'
}
}
/// Return the byte-level basename used for collision checks.
///
/// A null entry has no filename and is ignored by the exported checker. It
/// still maps to an empty key here so the key operation is safe to exercise
/// independently and never passes a null pointer to `CStr::from_ptr`.
unsafe fn filename_collision_key<'a>(filename: *const c_char) -> &'a [u8] {
if filename.is_null() {
return &[];
}
let bytes = unsafe { CStr::from_ptr(filename).to_bytes() };
match bytes
.iter()
.rposition(|&byte| byte == filename_path_separator())
{
Some(index) => &bytes[index + 1..],
None => bytes,
}
}
unsafe fn same_filename_collision_key(left: *const c_char, right: *const c_char) -> bool {
unsafe { filename_collision_key(left) == filename_collision_key(right) }
}
fn display_filename_collision_warning(filename: *const c_char) {
let enabled = unsafe { (*display_prefs()).displayLevel >= 2 };
if !enabled {
return;
}
let basename = unsafe { filename_collision_key(filename) };
let mut stderr = io::stderr().lock();
let _ = stderr.write_all(b"WARNING: Two files have same filename: ");
let _ = stderr.write_all(basename);
let _ = stderr.write_all(b"\n");
}
/// Rust implementation of the filename collision checker used by
/// `FIO_checkFilenameCollisions` in `programs/fileio.c`.
#[no_mangle]
pub unsafe extern "C" fn FIO_rust_checkFilenameCollisions(
filename_table: *const *const c_char,
nb_files: c_uint,
) -> c_int {
if filename_table.is_null() || nb_files == 0 {
return 0;
}
let mut filenames = Vec::<*const c_char>::new();
if filenames.try_reserve_exact(nb_files as usize).is_err() {
display(1, "Allocation error during filename collision checking \n");
return 1;
}
for index in 0..nb_files as usize {
let filename = unsafe { *filename_table.add(index) };
if !filename.is_null() {
filenames.push(filename);
}
}
filenames.sort_unstable_by(|left, right| unsafe {
filename_collision_key(*left).cmp(filename_collision_key(*right))
});
for pair in filenames.windows(2) {
if unsafe { same_filename_collision_key(pair[0], pair[1]) } {
display_filename_collision_warning(pair[0]);
}
}
0
}
#[cfg(test)]
mod tests {
use super::*;
use std::ffi::CString;
use std::fs;
use std::mem::{align_of, offset_of, size_of};
fn temporary_file_path(name: &str) -> std::path::PathBuf {
std::env::temp_dir().join(format!("zstd-fileio-prefs-{}-{name}", std::process::id()))
}
fn prefs_with_mem_limit(mem_limit: c_uint) -> FIO_prefs_t {
let mut prefs = unsafe { std::mem::zeroed::<FIO_prefs_t>() };
prefs.memLimit = mem_limit;
prefs
}
#[test]
fn patch_mem_limit_preserves_existing_limit() {
let mut prefs = prefs_with_mem_limit(4096);
let status = unsafe { FIO_rust_adjustMemLimitForPatchFromMode(&mut prefs, 1024, 2048) };
assert_eq!(status, FIO_PATCH_MEM_LIMIT_SUCCESS);
assert_eq!(prefs.memLimit, 4096);
}
#[test]
fn patch_mem_limit_uses_dictionary_and_source_maxima() {
let mut prefs = prefs_with_mem_limit(128);
let status = unsafe { FIO_rust_adjustMemLimitForPatchFromMode(&mut prefs, 4096, 1024) };
assert_eq!(status, FIO_PATCH_MEM_LIMIT_SUCCESS);
assert_eq!(prefs.memLimit, 4096);
let mut prefs = prefs_with_mem_limit(128);
let status = unsafe { FIO_rust_adjustMemLimitForPatchFromMode(&mut prefs, 1024, 8192) };
assert_eq!(status, FIO_PATCH_MEM_LIMIT_SUCCESS);
assert_eq!(prefs.memLimit, 8192);
}
#[test]
fn patch_mem_limit_rejects_unknown_sizes_without_update() {
for (dict_size, max_src_file_size) in
[(UTIL_FILESIZE_UNKNOWN, 0), (0, UTIL_FILESIZE_UNKNOWN)]
{
let mut prefs = prefs_with_mem_limit(4096);
let status = unsafe {
FIO_rust_adjustMemLimitForPatchFromMode(&mut prefs, dict_size, max_src_file_size)
};
assert_eq!(status, FIO_PATCH_MEM_LIMIT_UNKNOWN_SIZE);
assert_eq!(prefs.memLimit, 4096);
}
}
#[test]
fn patch_mem_limit_rejects_sizes_over_the_window_without_update() {
let mut prefs = prefs_with_mem_limit(4096);
let status = unsafe {
FIO_rust_adjustMemLimitForPatchFromMode(&mut prefs, (1u64 << ZSTD_WINDOWLOG_MAX) + 1, 0)
};
assert_eq!(status, FIO_PATCH_MEM_LIMIT_TOO_LARGE);
assert_eq!(prefs.memLimit, 4096);
}
#[test]
fn patch_mem_limit_updates_on_success() {
let mut prefs = prefs_with_mem_limit(128);
let status = unsafe { FIO_rust_adjustMemLimitForPatchFromMode(&mut prefs, 1024, 8192) };
assert_eq!(status, FIO_PATCH_MEM_LIMIT_SUCCESS);
assert_eq!(prefs.memLimit, 8192);
}
#[test]
fn largest_file_size_scan_handles_empty_input() {
assert_eq!(unsafe { FIO_rust_getLargestFileSize(ptr::null(), 0) }, 0);
}
#[test]
fn largest_file_size_scan_handles_ordinary_files() {
let smaller_path = temporary_file_path("smaller");
let larger_path = temporary_file_path("larger");
fs::write(&smaller_path, b"small").unwrap();
fs::write(&larger_path, b"larger file").unwrap();
let smaller = CString::new(smaller_path.to_string_lossy().as_bytes()).unwrap();
let larger = CString::new(larger_path.to_string_lossy().as_bytes()).unwrap();
let names = [smaller.as_ptr(), larger.as_ptr()];
assert_eq!(
unsafe { FIO_rust_getLargestFileSize(names.as_ptr(), names.len() as c_uint) },
11
);
fs::remove_file(smaller_path).unwrap();
fs::remove_file(larger_path).unwrap();
}
#[test]
fn largest_file_size_scan_preserves_large_values_and_errors() {
assert_eq!(largest_file_size([0, 1u64 << 63, 7]), 1u64 << 63);
assert_eq!(FIO_rust_highbit64(1u64 << 63), 63);
assert_eq!(FIO_rust_highbit64(u64::MAX), 63);
let missing_path = temporary_file_path("missing");
let missing = CString::new(missing_path.to_string_lossy().as_bytes()).unwrap();
let names = [missing.as_ptr()];
assert_eq!(
unsafe { FIO_rust_getLargestFileSize(names.as_ptr(), 1) },
UTIL_FILESIZE_UNKNOWN
);
}
#[test]
fn cycle_log_keeps_lower_strategies_unchanged() {
assert_eq!(FIO_rust_cycleLog(20, ZSTD_BTLAZY2 - 1), 20);
}
#[test]
fn cycle_log_subtracts_at_btlazy2_threshold() {
assert_eq!(FIO_rust_cycleLog(20, ZSTD_BTLAZY2), 19);
}
#[test]
fn cycle_log_subtracts_for_upper_strategies() {
assert_eq!(FIO_rust_cycleLog(20, ZSTD_BTLAZY2 + 3), 19);
}
#[test]
fn buffer_shims_preserve_fields_and_c_layout() {
let input_word = size_of::<*const c_void>();
let output_word = size_of::<*mut c_void>();
assert_eq!(offset_of!(FIO_inBuffer, src), 0);
assert_eq!(offset_of!(FIO_inBuffer, size), input_word);
assert_eq!(
offset_of!(FIO_inBuffer, pos),
input_word + size_of::<usize>()
);
assert_eq!(
size_of::<FIO_inBuffer>(),
input_word + 2 * size_of::<usize>()
);
assert_eq!(offset_of!(FIO_outBuffer, dst), 0);
assert_eq!(offset_of!(FIO_outBuffer, size), output_word);
assert_eq!(
offset_of!(FIO_outBuffer, pos),
output_word + size_of::<usize>()
);
assert_eq!(
size_of::<FIO_outBuffer>(),
output_word + 2 * size_of::<usize>()
);
let source = [1u8, 2, 3, 4];
let mut destination = [0u8; 8];
let mut input = FIO_inBuffer {
src: ptr::null(),
size: 0,
pos: 0,
};
let mut output = FIO_outBuffer {
dst: ptr::null_mut(),
size: 0,
pos: 0,
};
unsafe {
FIO_rust_setInBuffer(
&mut input,
source.as_ptr().cast::<c_void>(),
source.len(),
1,
);
FIO_rust_setOutBuffer(
&mut output,
destination.as_mut_ptr().cast::<c_void>(),
destination.len(),
2,
);
}
assert_eq!(input.src, source.as_ptr().cast::<c_void>());
assert_eq!(input.size, source.len());
assert_eq!(input.pos, 1);
assert_eq!(output.dst, destination.as_mut_ptr().cast::<c_void>());
assert_eq!(output.size, destination.len());
assert_eq!(output.pos, 2);
}
#[test]
fn file_info_aggregation_preserves_totals_flags_and_zero_fields() {
let first = FIO_fileInfo_t {
decompressedSize: 100,
compressedSize: 40,
windowSize: 1,
numActualFrames: 2,
numSkippableFrames: 3,
decompUnavailable: 0,
usesCheck: 1,
checksum: [1, 2, 3, 4],
nbFiles: 5,
dictID: 6,
};
let second = FIO_fileInfo_t {
decompressedSize: 200,
compressedSize: 60,
windowSize: 2,
numActualFrames: 7,
numSkippableFrames: 11,
decompUnavailable: 1,
usesCheck: 0,
checksum: [5, 6, 7, 8],
nbFiles: 13,
dictID: 14,
};
let mut total = FIO_fileInfo_t {
decompressedSize: u64::MAX,
compressedSize: u64::MAX,
windowSize: u64::MAX,
numActualFrames: -1,
numSkippableFrames: -1,
decompUnavailable: -1,
usesCheck: -1,
checksum: [u8::MAX; 4],
nbFiles: u32::MAX,
dictID: c_uint::MAX,
};
unsafe { FIO_rust_addFInfo(&mut total, &first, &second) };
assert_eq!(total.decompressedSize, 300);
assert_eq!(total.compressedSize, 100);
assert_eq!(total.windowSize, 0);
assert_eq!(total.numActualFrames, 9);
assert_eq!(total.numSkippableFrames, 14);
assert_eq!(total.decompUnavailable, 1);
assert_eq!(total.usesCheck, 0);
assert_eq!(total.checksum, [0; 4]);
assert_eq!(total.nbFiles, 18);
assert_eq!(total.dictID, 0);
}
#[test]
fn file_info_layout_matches_private_c_record() {
let word = size_of::<u64>();
let int = size_of::<c_int>();
assert_eq!(align_of::<FIO_fileInfo_t>(), align_of::<u64>());
assert_eq!(offset_of!(FIO_fileInfo_t, decompressedSize), 0);
assert_eq!(offset_of!(FIO_fileInfo_t, compressedSize), word);
assert_eq!(offset_of!(FIO_fileInfo_t, windowSize), 2 * word);
assert_eq!(offset_of!(FIO_fileInfo_t, numActualFrames), 3 * word);
assert_eq!(
offset_of!(FIO_fileInfo_t, numSkippableFrames),
3 * word + int
);
assert_eq!(
offset_of!(FIO_fileInfo_t, decompUnavailable),
3 * word + 2 * int
);
assert_eq!(offset_of!(FIO_fileInfo_t, usesCheck), 3 * word + 3 * int);
assert_eq!(offset_of!(FIO_fileInfo_t, checksum), 3 * word + 4 * int);
assert_eq!(offset_of!(FIO_fileInfo_t, nbFiles), 3 * word + 4 * int + 4);
assert_eq!(offset_of!(FIO_fileInfo_t, dictID), 3 * word + 4 * int + 8);
let expected_size = if size_of::<usize>() == 8 {
7 * word
} else {
13 * size_of::<u32>()
};
assert_eq!(size_of::<FIO_fileInfo_t>(), expected_size);
}
#[test]
fn lz4_block_size_shim_preserves_the_block_id_formula() {
assert_eq!(FIO_rust_LZ4_GetBlockSize_FromBlockId(0), 1 << 8);
assert_eq!(FIO_rust_LZ4_GetBlockSize_FromBlockId(1), 1 << 10);
assert_eq!(FIO_rust_LZ4_GetBlockSize_FromBlockId(3), 1 << 14);
assert_eq!(FIO_rust_LZ4_GetBlockSize_FromBlockId(4), 1 << 16);
}
#[test]
fn compressed_filename_shim_preserves_sentinels_suffixes_and_output_dirs() {
let source = CString::new("input/nested/file").unwrap();
let suffix = CString::new(".zst").unwrap();
let output_dir = CString::new("out").unwrap();
unsafe {
let name =
FIO_rust_determineCompressedName(source.as_ptr(), ptr::null(), suffix.as_ptr());
assert_eq!(CStr::from_ptr(name).to_bytes(), b"input/nested/file.zst");
let name = FIO_rust_determineCompressedName(
source.as_ptr(),
output_dir.as_ptr(),
suffix.as_ptr(),
);
assert_eq!(CStr::from_ptr(name).to_bytes(), b"out/file.zst");
let stdin = CString::new("/*stdin*\\").unwrap();
let name =
FIO_rust_determineCompressedName(stdin.as_ptr(), ptr::null(), suffix.as_ptr());
assert_eq!(CStr::from_ptr(name).to_bytes(), b"/*stdout*\\");
}
}
#[test]
fn destination_filename_shim_preserves_suffixes_tar_names_and_sentinels() {
let zst = CString::new(".zst").unwrap();
let tzst = CString::new(".tzst").unwrap();
let gz = CString::new(".gz").unwrap();
let tgz = CString::new(".tgz").unwrap();
let suffixes = [
zst.as_ptr(),
tzst.as_ptr(),
gz.as_ptr(),
tgz.as_ptr(),
ptr::null(),
];
let suffixes_display = CString::new(".zst/.tzst/.gz/.tgz").unwrap();
let source = CString::new("input/nested/file.zst").unwrap();
let tar_source = CString::new("input/nested/archive.tzst").unwrap();
let tar_gzip_source = CString::new("archive.tgz").unwrap();
let output_dir = CString::new("out").unwrap();
let stdin = CString::new("/*stdin*\\").unwrap();
unsafe {
let name = FIO_rust_determineDstName(
source.as_ptr(),
ptr::null(),
suffixes.as_ptr(),
suffixes_display.as_ptr(),
);
assert_eq!(CStr::from_ptr(name).to_bytes(), b"input/nested/file");
let name = FIO_rust_determineDstName(
tar_source.as_ptr(),
output_dir.as_ptr(),
suffixes.as_ptr(),
suffixes_display.as_ptr(),
);
assert_eq!(CStr::from_ptr(name).to_bytes(), b"out/archive.tar");
let name = FIO_rust_determineDstName(
tar_gzip_source.as_ptr(),
ptr::null(),
suffixes.as_ptr(),
suffixes_display.as_ptr(),
);
assert_eq!(CStr::from_ptr(name).to_bytes(), b"archive.tar");
let name = FIO_rust_determineDstName(
stdin.as_ptr(),
output_dir.as_ptr(),
suffixes.as_ptr(),
suffixes_display.as_ptr(),
);
assert_eq!(CStr::from_ptr(name).to_bytes(), b"/*stdout*\\");
}
}
#[test]
fn destination_filename_shim_rejects_unknown_suffix() {
let zst = CString::new(".zst").unwrap();
let suffixes = [zst.as_ptr(), ptr::null()];
let suffixes_display = CString::new(".zst").unwrap();
let source = CString::new("input/file.zip").unwrap();
unsafe {
assert!(FIO_rust_determineDstName(
source.as_ptr(),
ptr::null(),
suffixes.as_ptr(),
suffixes_display.as_ptr(),
)
.is_null());
}
}
#[test]
fn c_layouts_match_the_headers() {
let word = size_of::<usize>();
let int = size_of::<c_int>();
assert_eq!(align_of::<FIO_display_prefs_t>(), align_of::<c_int>());
assert_eq!(size_of::<FIO_display_prefs_t>(), 2 * int);
assert_eq!(align_of::<FIO_prefs_t>(), align_of::<usize>());
assert_eq!(offset_of!(FIO_prefs_t, compressionType), 0);
assert_eq!(offset_of!(FIO_prefs_t, sparseFileSupport), int);
assert_eq!(offset_of!(FIO_prefs_t, ldmHashRateLog), 15 * int);
assert_eq!(offset_of!(FIO_prefs_t, streamSrcSize), 16 * int);
assert_eq!(offset_of!(FIO_prefs_t, targetCBlockSize), 16 * int + word);
let tail = 16 * int + 2 * word;
assert_eq!(offset_of!(FIO_prefs_t, srcSizeHint), tail);
assert_eq!(offset_of!(FIO_prefs_t, testMode), tail + int);
assert_eq!(
offset_of!(FIO_prefs_t, literalCompressionMode),
tail + 2 * int
);
assert_eq!(offset_of!(FIO_prefs_t, removeSrcFile), tail + 3 * int);
assert_eq!(offset_of!(FIO_prefs_t, overwrite), tail + 4 * int);
assert_eq!(offset_of!(FIO_prefs_t, asyncIO), tail + 5 * int);
assert_eq!(offset_of!(FIO_prefs_t, memLimit), tail + 6 * int);
assert_eq!(offset_of!(FIO_prefs_t, mmapDict), tail + 13 * int);
assert_eq!(size_of::<FIO_prefs_t>(), tail + 14 * int);
assert_eq!(align_of::<FIO_ctx_t>(), align_of::<usize>());
assert_eq!(offset_of!(FIO_ctx_t, currFileIdx), 3 * int);
let total_bytes = (5 * int).next_multiple_of(word);
assert_eq!(offset_of!(FIO_ctx_t, totalBytesInput), total_bytes);
assert_eq!(offset_of!(FIO_ctx_t, totalBytesOutput), total_bytes + word);
assert_eq!(size_of::<FIO_ctx_t>(), total_bytes + 2 * word);
}
#[test]
fn filename_collision_checker_handles_null_and_empty_inputs() {
let empty = CString::new("").unwrap();
let names = [ptr::null(), empty.as_ptr()];
unsafe {
assert_eq!(FIO_rust_checkFilenameCollisions(ptr::null(), 0), 0);
assert_eq!(FIO_rust_checkFilenameCollisions(names.as_ptr(), 2), 0);
assert!(same_filename_collision_key(ptr::null(), empty.as_ptr()));
}
}
#[test]
fn filename_collision_keys_compare_basename_bytes_without_utf8_conversion() {
let first = CString::new(b"left/\xffname".to_vec()).unwrap();
let duplicate = CString::new(b"right/\xffname".to_vec()).unwrap();
let different = CString::new(b"right/\xfename".to_vec()).unwrap();
let names = [first.as_ptr(), duplicate.as_ptr()];
unsafe {
assert_eq!(filename_collision_key(first.as_ptr()), b"\xffname");
assert!(same_filename_collision_key(
first.as_ptr(),
duplicate.as_ptr()
));
assert!(!same_filename_collision_key(
first.as_ptr(),
different.as_ptr()
));
assert_eq!(FIO_rust_checkFilenameCollisions(names.as_ptr(), 2), 0);
}
}
#[test]
fn preference_defaults_match_fileio_c() {
let prefs = unsafe { FIO_createPreferences() };
assert!(!prefs.is_null());
let prefs = unsafe { &*prefs };
assert_eq!(prefs.compressionType, 0);
assert_eq!(prefs.overwrite, 0);
assert_eq!(prefs.sparseFileSupport, ZSTD_SPARSE_DEFAULT);
assert_eq!(prefs.dictIDFlag, 1);
assert_eq!(prefs.checksumFlag, 1);
assert_eq!(prefs.removeSrcFile, 0);
assert_eq!(prefs.memLimit, 0);
assert_eq!(prefs.nbWorkers, 1);
assert_eq!(prefs.blockSize, 0);
assert_eq!(prefs.overlapLog, FIO_OVERLAP_LOG_NOTSET);
assert_eq!(prefs.adaptiveMode, 0);
assert_eq!(prefs.rsyncable, 0);
assert_eq!(prefs.minAdaptLevel, -50);
assert_eq!(prefs.maxAdaptLevel, 22);
assert_eq!(prefs.ldmFlag, 0);
assert_eq!(prefs.ldmHashLog, 0);
assert_eq!(prefs.ldmMinMatch, 0);
assert_eq!(prefs.ldmBucketSizeLog, FIO_LDM_PARAM_NOTSET);
assert_eq!(prefs.ldmHashRateLog, FIO_LDM_PARAM_NOTSET);
assert_eq!(prefs.streamSrcSize, 0);
assert_eq!(prefs.targetCBlockSize, 0);
assert_eq!(prefs.srcSizeHint, 0);
assert_eq!(prefs.testMode, 0);
assert_eq!(prefs.literalCompressionMode, 0);
assert_eq!(prefs.excludeCompressedFiles, 0);
assert_eq!(prefs.allowBlockDevices, 0);
assert_eq!(prefs.asyncIO, 0);
assert_eq!(prefs.passThrough, -1);
unsafe { FIO_freePreferences(prefs as *const FIO_prefs_t as *mut FIO_prefs_t) };
}
#[test]
fn context_defaults_and_setters_match_fileio_c() {
let ctx = unsafe { FIO_createContext() };
assert!(!ctx.is_null());
unsafe {
assert_eq!((*ctx).currFileIdx, 0);
assert_eq!((*ctx).hasStdinInput, 0);
assert_eq!((*ctx).hasStdoutOutput, 0);
assert_eq!((*ctx).nbFilesTotal, 1);
assert_eq!((*ctx).nbFilesProcessed, 0);
assert_eq!((*ctx).totalBytesInput, 0);
assert_eq!((*ctx).totalBytesOutput, 0);
FIO_setNbFilesTotal(ctx, 7);
FIO_setHasStdinInput(ctx, -1);
FIO_setHasStdoutOutput(ctx, -2);
assert_eq!((*ctx).nbFilesTotal, 7);
assert_eq!((*ctx).hasStdinInput, 1);
assert_eq!((*ctx).hasStdoutOutput, -2);
}
let names = [c"input".as_ptr(), c"/*stdin*\\".as_ptr()];
let table = FileNamesTable {
fileNames: names.as_ptr().cast_mut(),
buf: ptr::null_mut(),
tableSize: names.len(),
tableCapacity: names.len(),
};
unsafe { FIO_determineHasStdinInput(ctx, &table) };
assert_eq!(unsafe { (*ctx).hasStdinInput }, 1);
unsafe { FIO_freeContext(ctx) };
}
fn summary_context(nb_files_total: c_int, nb_files_processed: c_int) -> FIO_ctx_t {
FIO_ctx_t {
nbFilesTotal: nb_files_total,
hasStdinInput: 0,
hasStdoutOutput: 0,
currFileIdx: 0,
nbFilesProcessed: nb_files_processed,
totalBytesInput: 0,
totalBytesOutput: 0,
}
}
#[test]
fn file_summary_display_matches_file_count_and_display_level() {
let mut ctx = summary_context(1, 0);
for display_level in [0, 2] {
FIO_setNotificationLevel(display_level);
assert_eq!(unsafe { FIO_rust_shouldDisplayFileSummary(&ctx) }, 1);
}
ctx.nbFilesTotal = 2;
for display_level in [0, 2] {
FIO_setNotificationLevel(display_level);
assert_eq!(unsafe { FIO_rust_shouldDisplayFileSummary(&ctx) }, 0);
}
FIO_setNotificationLevel(3);
assert_eq!(unsafe { FIO_rust_shouldDisplayFileSummary(&ctx) }, 1);
FIO_setNotificationLevel(2);
}
#[test]
fn multiple_file_summary_display_matches_processed_count_and_file_count() {
let mut ctx = summary_context(2, 0);
for display_level in [0, 2, 3] {
FIO_setNotificationLevel(display_level);
assert_eq!(
unsafe { FIO_rust_shouldDisplayMultipleFileSummary(&ctx) },
0
);
}
ctx.nbFilesProcessed = 1;
for display_level in [0, 2, 3] {
FIO_setNotificationLevel(display_level);
assert_eq!(
unsafe { FIO_rust_shouldDisplayMultipleFileSummary(&ctx) },
1
);
}
ctx.nbFilesProcessed = 2;
FIO_setNotificationLevel(0);
assert_eq!(
unsafe { FIO_rust_shouldDisplayMultipleFileSummary(&ctx) },
1
);
ctx.nbFilesTotal = 1;
assert_eq!(
unsafe { FIO_rust_shouldDisplayMultipleFileSummary(&ctx) },
0
);
FIO_setNotificationLevel(2);
}
#[test]
fn preference_setters_update_the_c_layout() {
let prefs = unsafe { FIO_createPreferences() };
unsafe {
FIO_setCompressionType(prefs, 3);
FIO_overwriteMode(prefs);
FIO_setSparseWrite(prefs, 2);
FIO_setDictIDFlag(prefs, 0);
FIO_setChecksumFlag(prefs, 2);
FIO_setRemoveSrcFile(prefs, -1);
FIO_setMemLimit(prefs, 123);
#[cfg(feature = "compression")]
FIO_setNbWorkers(prefs, 2);
FIO_setExcludeCompressedFile(prefs, 1);
FIO_setAllowBlockDevices(prefs, 1);
FIO_setUseRowMatchFinder(prefs, 2);
FIO_setBlockSize(prefs, 4096);
FIO_setOverlapLog(prefs, 5);
FIO_setAdaptiveMode(prefs, 1);
FIO_setRsyncable(prefs, 1);
FIO_setStreamSrcSize(prefs, 100);
FIO_setTargetCBlockSize(prefs, 200);
FIO_setSrcSizeHint(prefs, usize::MAX);
FIO_setLiteralCompressionMode(prefs, 2);
FIO_setLdmFlag(prefs, 1);
FIO_setLdmHashLog(prefs, 6);
FIO_setLdmMinMatch(prefs, 7);
FIO_setLdmBucketSizeLog(prefs, 8);
FIO_setLdmHashRateLog(prefs, 9);
FIO_setPatchFromMode(prefs, -1);
FIO_setContentSize(prefs, -1);
FIO_setPassThroughFlag(prefs, -1);
FIO_setMMapDict(prefs, 2);
FIO_setAsyncIOFlag(prefs, 1);
#[cfg(feature = "compression")]
{
FIO_setAdaptMin(prefs, -40);
FIO_setAdaptMax(prefs, 18);
}
#[cfg(feature = "decompression")]
FIO_setTestMode(prefs, -1);
assert_eq!((*prefs).compressionType, 3);
assert_eq!((*prefs).overwrite, 1);
assert_eq!((*prefs).sparseFileSupport, 2);
assert_eq!((*prefs).dictIDFlag, 0);
assert_eq!((*prefs).checksumFlag, 2);
assert_eq!((*prefs).removeSrcFile, 1);
assert_eq!((*prefs).memLimit, 123);
#[cfg(feature = "compression")]
assert_eq!((*prefs).nbWorkers, 2);
assert_eq!((*prefs).excludeCompressedFiles, 1);
assert_eq!((*prefs).allowBlockDevices, 1);
assert_eq!((*prefs).useRowMatchFinder, 2);
assert_eq!((*prefs).blockSize, 4096);
assert_eq!((*prefs).overlapLog, 5);
assert_eq!((*prefs).adaptiveMode, 1);
assert_eq!((*prefs).rsyncable, 1);
assert_eq!((*prefs).streamSrcSize, 100);
assert_eq!((*prefs).targetCBlockSize, 200);
assert_eq!((*prefs).srcSizeHint, c_int::MAX);
assert_eq!((*prefs).literalCompressionMode, 2);
assert_eq!((*prefs).ldmFlag, 1);
assert_eq!((*prefs).ldmHashLog, 6);
assert_eq!((*prefs).ldmMinMatch, 7);
assert_eq!((*prefs).ldmBucketSizeLog, 8);
assert_eq!((*prefs).ldmHashRateLog, 9);
assert_eq!((*prefs).patchFromMode, 1);
assert_eq!((*prefs).contentSize, 1);
assert_eq!((*prefs).passThrough, 1);
assert_eq!((*prefs).mmapDict, 2);
#[cfg(test)]
assert_eq!((*prefs).asyncIO, 0);
#[cfg(feature = "compression")]
{
assert_eq!((*prefs).minAdaptLevel, -40);
assert_eq!((*prefs).maxAdaptLevel, 18);
}
#[cfg(feature = "decompression")]
assert_eq!((*prefs).testMode, 1);
}
FIO_setNotificationLevel(0);
FIO_setProgressSetting(2);
unsafe {
assert_eq!((*display_prefs()).displayLevel, 0);
assert_eq!((*display_prefs()).progressSetting, 2);
FIO_freePreferences(prefs);
}
}
}
Reference in New Issue View Git Blame Copy Permalink