ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(cli): port utility and dictionary I/O helpers to Rust

Browse Source 
Move filename tables, file-list expansion, core-count helpers, statistics,
and dictionary sample loading behind Rust implementations while retaining
the narrow C ABI used by the CLI.

Test Plan:
- RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= cargo test --manifest-path rust/cli/Cargo.toml
- RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings
- make -B -C programs zstd V=1
- make -C tests check V=1
This commit is contained in:
ddidderr
2026-07-12 18:06:51 +02:00
parent 0d3dae8fd8
commit cd7ae43da7
6 changed files with 2580 additions and 2058 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/dibio.rs
+1022
View File
@@ -0,0 +1,1022 @@
#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::too_many_arguments)]
//! Dictionary training I/O for the command-line interface.
//!
//! The public entry point keeps the `programs/dibio.h` ABI. File discovery,
//! sample selection, buffering, and dictionary output are Rust-owned; the
//! actual legacy, COVER, and fastCOVER trainers are reached through their
//! existing C ABI symbols.
use std::cmp::min;
use std::ffi::{c_char, c_void, CStr};
use std::fmt;
use std::fs;
use std::io::{self, Write};
use std::mem::size_of;
use std::os::raw::{c_int, c_uint};
use std::path::PathBuf;
use std::time::{Duration, Instant};
#[cfg(unix)]
use std::ffi::OsStr;
#[cfg(unix)]
use std::os::unix::ffi::OsStrExt;
const KB: usize = 1 << 10;
const MB: usize = 1 << 20;
const GB: usize = 1 << 30;
const SAMPLESIZE_MAX: usize = 128 * KB;
const MEMMULT: u64 = 11;
const COVER_MEMMULT: u64 = 9;
const FASTCOVER_MEMMULT: u64 = 1;
const NOISELENGTH: usize = 32;
const MAX_SAMPLES_SIZE: usize = 2 * GB;
const REFRESH_RATE: Duration = Duration::from_micros(1_000_000 / 6);
// The C implementation initializes `dictSize` with the public enum value
// `ZSTD_error_GENERIC` (1), rather than an encoded size_t error.
const ERROR_GENERIC: usize = 1;
const MODE_READ: &[u8] = b"rb\0";
const MODE_WRITE: &[u8] = b"wb\0";
/// ABI-compatible `ZDICT_params_t` from `lib/zdict.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZDICT_params_t {
pub compressionLevel: c_int,
pub notificationLevel: c_uint,
pub dictID: c_uint,
}
/// ABI-compatible `ZDICT_legacy_params_t` from `lib/zdict.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ZDICT_legacy_params_t {
pub selectivityLevel: c_uint,
pub zParams: ZDICT_params_t,
}
/// ABI-compatible `ZDICT_cover_params_t` from `lib/zdict.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq)]
pub struct ZDICT_cover_params_t {
pub k: c_uint,
pub d: c_uint,
pub steps: c_uint,
pub nbThreads: c_uint,
pub splitPoint: f64,
pub shrinkDict: c_uint,
pub shrinkDictMaxRegression: c_uint,
pub zParams: ZDICT_params_t,
}
/// ABI-compatible `ZDICT_fastCover_params_t` from `lib/zdict.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq)]
pub struct ZDICT_fastCover_params_t {
pub k: c_uint,
pub d: c_uint,
pub f: c_uint,
pub steps: c_uint,
pub nbThreads: c_uint,
pub splitPoint: f64,
pub accel: c_uint,
pub shrinkDict: c_uint,
pub shrinkDictMaxRegression: c_uint,
pub zParams: ZDICT_params_t,
}
unsafe extern "C" {
fn malloc(size: usize) -> *mut c_void;
fn free(pointer: *mut c_void);
fn fopen(filename: *const c_char, mode: *const c_char) -> *mut c_void;
fn fread(pointer: *mut c_void, size: usize, count: usize, stream: *mut c_void) -> usize;
fn fwrite(pointer: *const c_void, size: usize, count: usize, stream: *mut c_void) -> usize;
fn fclose(stream: *mut c_void) -> c_int;
fn ZDICT_isError(error_code: usize) -> c_uint;
fn ZDICT_getErrorName(error_code: usize) -> *const c_char;
fn ZDICT_trainFromBuffer_legacy(
dict_buffer: *mut c_void,
dict_buffer_capacity: usize,
samples_buffer: *const c_void,
samples_sizes: *const usize,
nb_samples: c_uint,
parameters: ZDICT_legacy_params_t,
) -> usize;
fn ZDICT_trainFromBuffer_cover(
dict_buffer: *mut c_void,
dict_buffer_capacity: usize,
samples_buffer: *const c_void,
samples_sizes: *const usize,
nb_samples: c_uint,
parameters: ZDICT_cover_params_t,
) -> usize;
fn ZDICT_optimizeTrainFromBuffer_cover(
dict_buffer: *mut c_void,
dict_buffer_capacity: usize,
samples_buffer: *const c_void,
samples_sizes: *const usize,
nb_samples: c_uint,
parameters: *mut ZDICT_cover_params_t,
) -> usize;
fn ZDICT_trainFromBuffer_fastCover(
dict_buffer: *mut c_void,
dict_buffer_capacity: usize,
samples_buffer: *const c_void,
samples_sizes: *const usize,
nb_samples: c_uint,
parameters: ZDICT_fastCover_params_t,
) -> usize;
fn ZDICT_optimizeTrainFromBuffer_fastCover(
dict_buffer: *mut c_void,
dict_buffer_capacity: usize,
samples_buffer: *const c_void,
samples_sizes: *const usize,
nb_samples: c_uint,
parameters: *mut ZDICT_fastCover_params_t,
) -> usize;
}
#[cfg(test)]
mod test_zdict_symbols {
use super::*;
static ERROR_NAME: &[u8] = b"test dictionary trainer error\0";
unsafe fn write_test_dictionary(buffer: *mut c_void, capacity: usize) -> usize {
let size = capacity.min(64);
if !buffer.is_null() {
std::ptr::write_bytes(buffer.cast::<u8>(), 0xA5, size);
}
size
}
#[no_mangle]
pub unsafe extern "C" fn ZDICT_isError(_code: usize) -> c_uint {
0
}
#[no_mangle]
pub unsafe extern "C" fn ZDICT_getErrorName(_code: usize) -> *const c_char {
ERROR_NAME.as_ptr().cast()
}
#[no_mangle]
pub unsafe extern "C" fn ZDICT_trainFromBuffer_legacy(
buffer: *mut c_void,
capacity: usize,
_samples: *const c_void,
_sample_sizes: *const usize,
_nb_samples: c_uint,
_parameters: ZDICT_legacy_params_t,
) -> usize {
write_test_dictionary(buffer, capacity)
}
#[no_mangle]
pub unsafe extern "C" fn ZDICT_trainFromBuffer_cover(
buffer: *mut c_void,
capacity: usize,
_samples: *const c_void,
_sample_sizes: *const usize,
_nb_samples: c_uint,
_parameters: ZDICT_cover_params_t,
) -> usize {
write_test_dictionary(buffer, capacity)
}
#[no_mangle]
pub unsafe extern "C" fn ZDICT_optimizeTrainFromBuffer_cover(
buffer: *mut c_void,
capacity: usize,
_samples: *const c_void,
_sample_sizes: *const usize,
_nb_samples: c_uint,
_parameters: *mut ZDICT_cover_params_t,
) -> usize {
write_test_dictionary(buffer, capacity)
}
#[no_mangle]
pub unsafe extern "C" fn ZDICT_trainFromBuffer_fastCover(
buffer: *mut c_void,
capacity: usize,
_samples: *const c_void,
_sample_sizes: *const usize,
_nb_samples: c_uint,
_parameters: ZDICT_fastCover_params_t,
) -> usize {
write_test_dictionary(buffer, capacity)
}
#[no_mangle]
pub unsafe extern "C" fn ZDICT_optimizeTrainFromBuffer_fastCover(
buffer: *mut c_void,
capacity: usize,
_samples: *const c_void,
_sample_sizes: *const usize,
_nb_samples: c_uint,
_parameters: *mut ZDICT_fastCover_params_t,
) -> usize {
write_test_dictionary(buffer, capacity)
}
}
#[derive(Clone, Copy, Debug, Default)]
struct FileStats {
total_size_to_load: i64,
nb_samples: c_int,
one_sample_too_large: bool,
}
struct DisplayClock {
last_update: Instant,
}
impl DisplayClock {
fn new() -> Self {
Self {
// `g_displayClock` is zero-initialized in the C implementation,
// so its first span always permits a progress update.
last_update: Instant::now() - REFRESH_RATE,
}
}
fn update(&mut self, display_level: c_int, level: c_int, message: fmt::Arguments<'_>) {
if display_level < level {
return;
}
if self.last_update.elapsed() > REFRESH_RATE || display_level >= 4 {
self.last_update = Instant::now();
eprint!("{message}");
if display_level >= 4 {
let _ = io::stderr().flush();
}
}
}
}
fn display(display_level: c_int, level: c_int, message: fmt::Arguments<'_>) {
if display_level >= level {
eprint!("{message}");
}
}
fn fatal(code: c_int, message: impl fmt::Display) -> ! {
eprintln!("Error {code} : {message}");
std::process::exit(code);
}
fn c_string_display(pointer: *const c_char) -> String {
unsafe { CStr::from_ptr(pointer) }
.to_string_lossy()
.into_owned()
}
#[cfg(unix)]
fn path_from_c_string(value: &CStr) -> PathBuf {
PathBuf::from(OsStr::from_bytes(value.to_bytes()))
}
#[cfg(not(unix))]
fn path_from_c_string(value: &CStr) -> PathBuf {
PathBuf::from(value.to_string_lossy().into_owned())
}
/// Returns `-1` for a missing or non-regular file, matching
/// `DiB_getFileSize()`'s `UTIL_FILESIZE_UNKNOWN` conversion.
fn get_file_size(file_name: *const c_char) -> i64 {
let path = path_from_c_string(unsafe { CStr::from_ptr(file_name) });
match fs::metadata(path) {
Ok(metadata) if metadata.is_file() => i64::try_from(metadata.len()).unwrap_or(-1),
_ => -1,
}
}
/// `MIN(fileSize, SAMPLESIZE_MAX)` in the original C source has different
/// usual-arithmetic-conversion behavior on 32-bit and 64-bit targets. Keep
/// that detail here because unknown files are represented by `-1`.
fn c_min_file_size(file_size: i64) -> i64 {
if size_of::<usize>() == 8 {
if (file_size as u64) < SAMPLESIZE_MAX as u64 {
file_size
} else {
SAMPLESIZE_MAX as i64
}
} else {
min(file_size, SAMPLESIZE_MAX as i64)
}
}
unsafe fn shuffle(file_names: *const *const c_char, nb_files: c_int) {
if nb_files <= 1 {
return;
}
// The C API takes `const char**` but deliberately shuffles the caller's
// table in place. The CLI supplies a writable pointer array.
let table = unsafe { std::slice::from_raw_parts_mut(file_names.cast_mut(), nb_files as usize) };
let mut seed = 0xFD2F_B528u32;
for index in (1..table.len()).rev() {
seed = seed.wrapping_mul(2_654_435_761);
seed ^= 2_246_822_519;
seed = seed.rotate_left(13);
let random = (seed >> 5) as usize;
table.swap(index, random % (index + 1));
}
}
fn file_stats(
file_names: *const *const c_char,
nb_files: c_int,
chunk_size: usize,
display_level: c_int,
) -> FileStats {
debug_assert!(chunk_size <= SAMPLESIZE_MAX);
let mut stats = FileStats::default();
for index in 0..nb_files.max(0) as usize {
let file_name = unsafe { *file_names.add(index) };
let file_size = get_file_size(file_name);
if file_size == 0 {
display(
display_level,
3,
format_args!(
"Sample file '{}' has zero size, skipping...\n",
c_string_display(file_name)
),
);
continue;
}
if chunk_size > 0 {
// This is the C expression `(fileSize + chunkSize - 1) /
// chunkSize`, including its unsigned wrap for an unknown file on
// the 64-bit targets where size_t and S64 have equal width.
let chunks = (file_size as u64)
.wrapping_add(chunk_size as u64)
.wrapping_sub(1)
/ chunk_size as u64;
stats.nb_samples = stats.nb_samples.wrapping_add(chunks as c_int);
stats.total_size_to_load = stats.total_size_to_load.wrapping_add(file_size);
} else {
if file_size > SAMPLESIZE_MAX as i64 {
if file_size > (2 * SAMPLESIZE_MAX) as i64 {
stats.one_sample_too_large = true;
}
display(
display_level,
3,
format_args!(
"Sample file '{}' is too large, limiting to {} KB\n",
c_string_display(file_name),
SAMPLESIZE_MAX / KB
),
);
}
stats.nb_samples = stats.nb_samples.wrapping_add(1);
stats.total_size_to_load = stats
.total_size_to_load
.wrapping_add(c_min_file_size(file_size));
}
}
display(
display_level,
4,
format_args!(
"Found training data {} files, {} KB, {} samples\n",
nb_files,
(stats.total_size_to_load / KB as i64) as c_int,
stats.nb_samples
),
);
stats
}
fn maximum_memory() -> usize {
if size_of::<usize>() == 4 {
2 * GB - 64 * MB
} else {
(512 * MB) << size_of::<usize>()
}
}
unsafe fn find_max_memory(required_memory: u64) -> usize {
let step = (8 * MB) as u64;
let mut required = (((required_memory >> 23) + 1) << 23).wrapping_add(step);
required = min(required, maximum_memory() as u64);
loop {
let test_memory = unsafe { malloc(required as usize) };
if !test_memory.is_null() {
unsafe { free(test_memory) };
return required as usize;
}
required = required.wrapping_sub(step);
}
}
unsafe fn fill_noise(buffer: *mut u8, length: usize) {
let mut accumulator = 2_654_435_761u32;
for index in 0..length {
accumulator = accumulator.wrapping_mul(2_246_822_519);
unsafe { buffer.add(index).write((accumulator >> 21) as u8) };
}
}
unsafe fn read_chunk(
stream: *mut c_void,
destination: *mut u8,
length: usize,
file_name: *const c_char,
) {
let read = unsafe { fread(destination.cast(), 1, length, stream) };
if read != length {
fatal(
11,
format_args!("Pb reading {}", c_string_display(file_name)),
);
}
}
unsafe fn load_files(
buffer: *mut u8,
buffer_size: usize,
sample_sizes: *mut usize,
sample_count: usize,
file_names: *const *const c_char,
nb_files: c_int,
target_chunk_size: usize,
display_level: c_int,
display_clock: &mut DisplayClock,
) -> (usize, usize) {
debug_assert!(target_chunk_size <= SAMPLESIZE_MAX);
let mut total_data_loaded = 0usize;
let mut nb_samples_loaded = 0usize;
let mut file_index = 0usize;
let file_count = nb_files.max(0) as usize;
while nb_samples_loaded < sample_count && file_index < file_count {
let file_name = unsafe { *file_names.add(file_index) };
let file_size = get_file_size(file_name);
if file_size <= 0 {
file_index += 1;
continue;
}
let stream = unsafe { fopen(file_name, MODE_READ.as_ptr().cast()) };
if stream.is_null() {
let error = io::Error::last_os_error();
fatal(
10,
format_args!(
"zstd: dictBuilder: {} {} ",
c_string_display(file_name),
error
),
);
}
display_clock.update(
display_level,
2,
format_args!("Loading {}... \r", c_string_display(file_name)),
);
let first_chunk = if target_chunk_size > 0 {
min(file_size as usize, target_chunk_size)
} else {
min(file_size as usize, SAMPLESIZE_MAX)
};
if total_data_loaded.wrapping_add(first_chunk) > buffer_size {
unsafe { fclose(stream) };
break;
}
unsafe {
read_chunk(
stream,
buffer.add(total_data_loaded),
first_chunk,
file_name,
);
sample_sizes.add(nb_samples_loaded).write(first_chunk);
}
nb_samples_loaded += 1;
total_data_loaded += first_chunk;
if target_chunk_size > 0 {
let mut file_data_loaded = first_chunk;
while (file_data_loaded as i64) < file_size && nb_samples_loaded < sample_count {
let remaining = (file_size - file_data_loaded as i64) as usize;
let chunk_size = min(remaining, target_chunk_size);
if total_data_loaded.wrapping_add(chunk_size) > buffer_size {
break;
}
unsafe {
read_chunk(stream, buffer.add(total_data_loaded), chunk_size, file_name);
sample_sizes.add(nb_samples_loaded).write(chunk_size);
}
nb_samples_loaded += 1;
total_data_loaded += chunk_size;
file_data_loaded += chunk_size;
}
}
file_index += 1;
unsafe { fclose(stream) };
}
display(display_level, 2, format_args!("\r{:79}\r", ""));
display(
display_level,
4,
format_args!(
"Loaded {} KB total training data, {} nb samples \n",
total_data_loaded / KB,
nb_samples_loaded
),
);
(total_data_loaded, nb_samples_loaded)
}
fn loaded_size(total_size_to_load: i64, max_memory: usize) -> usize {
let first = min(max_memory as i64, total_size_to_load);
if size_of::<usize>() == 8 {
min(first as u64, MAX_SAMPLES_SIZE as u64) as usize
} else {
min(first, MAX_SAMPLES_SIZE as i64) as usize
}
}
unsafe fn save_dictionary(
dictionary_file_name: *const c_char,
dictionary: *const c_void,
dictionary_size: usize,
) {
let stream = unsafe { fopen(dictionary_file_name, MODE_WRITE.as_ptr().cast()) };
if stream.is_null() {
fatal(
3,
format_args!("cannot open {} ", c_string_display(dictionary_file_name)),
);
}
let written = unsafe { fwrite(dictionary, 1, dictionary_size, stream) };
if written != dictionary_size {
fatal(
4,
format_args!("{} : write error", c_string_display(dictionary_file_name)),
);
}
if unsafe { fclose(stream) } != 0 {
fatal(
5,
format_args!("{} : flush error", c_string_display(dictionary_file_name)),
);
}
}
unsafe fn error_name(error_code: usize) -> String {
let name = unsafe { ZDICT_getErrorName(error_code) };
if name.is_null() {
return "unknown error".to_owned();
}
unsafe { CStr::from_ptr(name) }
.to_string_lossy()
.into_owned()
}
unsafe fn free_buffers(
source_buffer: *mut c_void,
sample_sizes: *mut usize,
dictionary_buffer: *mut c_void,
) {
unsafe {
free(source_buffer);
free(sample_sizes.cast());
free(dictionary_buffer);
}
}
/// Train a dictionary from files and write it to `dictionary_file_name`.
///
/// This is the Rust implementation of the public `DiB_trainFromFiles()` ABI
/// declared in `programs/dibio.h`. Its exit-on-I/O-error behavior is kept for
/// compatibility with the command-line program; trainer errors return `1`.
#[no_mangle]
pub unsafe extern "C" fn DiB_trainFromFiles(
dictionary_file_name: *const c_char,
max_dictionary_size: usize,
file_names: *const *const c_char,
nb_files: c_int,
chunk_size: usize,
legacy_params: *mut ZDICT_legacy_params_t,
cover_params: *mut ZDICT_cover_params_t,
fast_cover_params: *mut ZDICT_fastCover_params_t,
optimize: c_int,
memory_limit: c_uint,
) -> c_int {
let display_level = if !legacy_params.is_null() {
unsafe { (*legacy_params).zParams.notificationLevel as c_int }
} else if !cover_params.is_null() {
unsafe { (*cover_params).zParams.notificationLevel as c_int }
} else if !fast_cover_params.is_null() {
unsafe { (*fast_cover_params).zParams.notificationLevel as c_int }
} else {
0
};
let mut display_clock = DisplayClock::new();
let dictionary_buffer = unsafe { malloc(max_dictionary_size) };
display(display_level, 3, format_args!("Shuffling input files\n"));
unsafe { shuffle(file_names, nb_files) };
let stats = file_stats(file_names, nb_files, chunk_size, display_level);
let memory_multiplier = if !legacy_params.is_null() {
MEMMULT
} else if !cover_params.is_null() {
COVER_MEMMULT
} else {
FASTCOVER_MEMMULT
};
let required_memory = (stats.total_size_to_load as u64).wrapping_mul(memory_multiplier);
let max_memory = unsafe { find_max_memory(required_memory) } / memory_multiplier as usize;
let mut loaded_size = loaded_size(stats.total_size_to_load, max_memory);
if memory_limit != 0 {
display(
display_level,
2,
format_args!(
"! Warning : setting manual memory limit for dictionary training data at {} MB \n",
(memory_limit as usize) / MB
),
);
loaded_size = min(loaded_size, memory_limit as usize);
}
let source_buffer_size = loaded_size.wrapping_add(NOISELENGTH);
let source_buffer = unsafe { malloc(source_buffer_size) };
let sample_count = stats.nb_samples.max(0) as usize;
let sample_sizes_bytes = sample_count.wrapping_mul(size_of::<usize>());
let sample_sizes = unsafe { malloc(sample_sizes_bytes) }.cast::<usize>();
if ((stats.nb_samples > 0) && sample_sizes.is_null())
|| source_buffer.is_null()
|| dictionary_buffer.is_null()
{
fatal(12, "not enough memory for DiB_trainFiles");
}
if stats.one_sample_too_large {
display(
display_level,
2,
format_args!("! Warning : some sample(s) are very large \n"),
);
display(
display_level,
2,
format_args!("! Note that dictionary is only useful for small samples. \n"),
);
display(
display_level,
2,
format_args!(
"! As a consequence, only the first {} bytes of each sample are loaded \n",
SAMPLESIZE_MAX
),
);
}
if stats.nb_samples < 5 {
display(
display_level,
2,
format_args!("! Warning : nb of samples too low for proper processing ! \n"),
);
display(
display_level,
2,
format_args!("! Please provide _one file per sample_. \n"),
);
display(
display_level,
2,
format_args!(
"! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n"
),
);
fatal(14, "nb of samples too low");
}
let dictionary_size_threshold = (max_dictionary_size as i64).wrapping_mul(8);
if stats.total_size_to_load < dictionary_size_threshold {
display(
display_level,
2,
format_args!(
"! Warning : data size of samples too small for target dictionary size \n"
),
);
display(
display_level,
2,
format_args!("! Samples should be about 100x larger than target dictionary size \n"),
);
}
if (loaded_size as i64) < stats.total_size_to_load {
display(
display_level,
1,
format_args!(
"Training samples set too large ({} MB); training on {} MB only...\n",
(stats.total_size_to_load / MB as i64) as c_uint,
loaded_size / MB
),
);
}
let (loaded_size, nb_samples_loaded) = unsafe {
load_files(
source_buffer.cast(),
loaded_size,
sample_sizes,
sample_count,
file_names,
nb_files,
chunk_size,
display_level,
&mut display_clock,
)
};
let mut dictionary_size = ERROR_GENERIC;
if !legacy_params.is_null() {
unsafe { fill_noise(source_buffer.cast::<u8>().add(loaded_size), NOISELENGTH) };
dictionary_size = unsafe {
ZDICT_trainFromBuffer_legacy(
dictionary_buffer,
max_dictionary_size,
source_buffer,
sample_sizes,
nb_samples_loaded as c_uint,
*legacy_params,
)
};
} else if !cover_params.is_null() {
if optimize != 0 {
dictionary_size = unsafe {
ZDICT_optimizeTrainFromBuffer_cover(
dictionary_buffer,
max_dictionary_size,
source_buffer,
sample_sizes,
nb_samples_loaded as c_uint,
cover_params,
)
};
if unsafe { ZDICT_isError(dictionary_size) } == 0 {
let parameters = unsafe { &*cover_params };
display(
display_level,
2,
format_args!(
"k={}\nd={}\nsteps={}\nsplit={}\n",
parameters.k,
parameters.d,
parameters.steps,
(parameters.splitPoint * 100.0) as c_uint
),
);
}
} else {
dictionary_size = unsafe {
ZDICT_trainFromBuffer_cover(
dictionary_buffer,
max_dictionary_size,
source_buffer,
sample_sizes,
nb_samples_loaded as c_uint,
*cover_params,
)
};
}
} else if !fast_cover_params.is_null() {
if optimize != 0 {
dictionary_size = unsafe {
ZDICT_optimizeTrainFromBuffer_fastCover(
dictionary_buffer,
max_dictionary_size,
source_buffer,
sample_sizes,
nb_samples_loaded as c_uint,
fast_cover_params,
)
};
if unsafe { ZDICT_isError(dictionary_size) } == 0 {
let parameters = unsafe { &*fast_cover_params };
display(
display_level,
2,
format_args!(
"k={}\nd={}\nf={}\nsteps={}\nsplit={}\naccel={}\n",
parameters.k,
parameters.d,
parameters.f,
parameters.steps,
(parameters.splitPoint * 100.0) as c_uint,
parameters.accel
),
);
}
} else {
dictionary_size = unsafe {
ZDICT_trainFromBuffer_fastCover(
dictionary_buffer,
max_dictionary_size,
source_buffer,
sample_sizes,
nb_samples_loaded as c_uint,
*fast_cover_params,
)
};
}
} else {
debug_assert!(false, "one dictionary trainer parameter set is required");
}
if unsafe { ZDICT_isError(dictionary_size) } != 0 {
display(
display_level,
1,
format_args!("dictionary training failed : {} \n", unsafe {
error_name(dictionary_size)
}),
);
unsafe { free_buffers(source_buffer, sample_sizes, dictionary_buffer) };
return 1;
}
display(
display_level,
2,
format_args!(
"Save dictionary of size {} into file {} \n",
dictionary_size as c_uint,
c_string_display(dictionary_file_name)
),
);
unsafe {
save_dictionary(dictionary_file_name, dictionary_buffer, dictionary_size);
free_buffers(source_buffer, sample_sizes, dictionary_buffer);
}
0
}
#[cfg(test)]
mod tests {
use super::*;
use std::ffi::CString;
use std::fs;
use std::mem::{offset_of, size_of};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
#[cfg(unix)]
fn path_to_c_string(path: &Path) -> CString {
CString::new(path.as_os_str().as_bytes()).expect("temporary path has no NUL")
}
#[cfg(not(unix))]
fn path_to_c_string(path: &Path) -> CString {
CString::new(path.to_string_lossy().as_bytes()).expect("temporary path has no NUL")
}
fn sample(index: usize) -> Vec<u8> {
let line = format!(
"record={index:02};common-key=common-value;payload=abcdefghijklmnopqrstuvwxyz\n"
);
let mut result = Vec::with_capacity(1024);
while result.len() < 1024 {
result.extend_from_slice(line.as_bytes());
}
result.truncate(1024);
result
}
fn temporary_paths() -> (Vec<PathBuf>, PathBuf) {
static NEXT: AtomicUsize = AtomicUsize::new(0);
let prefix = format!(
"zstd-dibio-{}-{}",
std::process::id(),
NEXT.fetch_add(1, Ordering::Relaxed)
);
let root = std::env::temp_dir();
let samples = (0..5)
.map(|index| root.join(format!("{prefix}-sample-{index}")))
.collect::<Vec<_>>();
let dictionary = root.join(format!("{prefix}-dictionary"));
(samples, dictionary)
}
#[test]
fn dictionary_parameter_layouts_match_zdict_h() {
assert_eq!(size_of::<ZDICT_params_t>(), 12);
assert_eq!(size_of::<ZDICT_legacy_params_t>(), 16);
assert_eq!(size_of::<ZDICT_cover_params_t>(), 48);
assert_eq!(size_of::<ZDICT_fastCover_params_t>(), 56);
assert_eq!(offset_of!(ZDICT_cover_params_t, splitPoint), 16);
assert_eq!(offset_of!(ZDICT_fastCover_params_t, splitPoint), 24);
}
#[test]
fn trains_from_chunked_temporary_sample_files() {
let (sample_paths, dictionary_path) = temporary_paths();
let _cleanup = TemporaryPathCleanup {
paths: sample_paths
.iter()
.cloned()
.chain(std::iter::once(dictionary_path.clone()))
.collect(),
};
for (index, path) in sample_paths.iter().enumerate() {
fs::write(path, sample(index)).expect("write temporary training sample");
}
let sample_names = sample_paths
.iter()
.map(|path| path_to_c_string(path))
.collect::<Vec<_>>();
let file_names = sample_names
.iter()
.map(|name| name.as_ptr())
.collect::<Vec<_>>();
let dictionary_name = path_to_c_string(&dictionary_path);
let mut parameters = ZDICT_legacy_params_t {
zParams: ZDICT_params_t {
compressionLevel: 3,
..ZDICT_params_t::default()
},
..ZDICT_legacy_params_t::default()
};
let result = unsafe {
DiB_trainFromFiles(
dictionary_name.as_ptr(),
2048,
file_names.as_ptr(),
file_names.len() as c_int,
256,
&mut parameters,
std::ptr::null_mut(),
std::ptr::null_mut(),
0,
0,
)
};
assert_eq!(result, 0);
let dictionary = fs::read(&dictionary_path).expect("read trained dictionary");
assert!(!dictionary.is_empty());
}
#[test]
fn legacy_trainer_abi_smoke() {
let samples = (0..5).map(sample).collect::<Vec<_>>();
let sample_sizes = samples.iter().map(Vec::len).collect::<Vec<_>>();
let flat_samples = samples.concat();
let mut dictionary = vec![0u8; 2048];
let parameters = ZDICT_legacy_params_t::default();
let result = unsafe {
ZDICT_trainFromBuffer_legacy(
dictionary.as_mut_ptr().cast(),
dictionary.len(),
flat_samples.as_ptr().cast(),
sample_sizes.as_ptr(),
sample_sizes.len() as c_uint,
parameters,
)
};
assert_eq!(unsafe { ZDICT_isError(result) }, 0);
assert!(result > 0);
}
impl Drop for TemporaryPathCleanup {
fn drop(&mut self) {
for path in &self.paths {
let _ = fs::remove_file(path);
}
}
}
struct TemporaryPathCleanup {
paths: Vec<PathBuf>,
}
}
rust/src/util.rs
+1501
View File
@@ -0,0 +1,1501 @@
#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
//! Rust implementation of the program utility layer from `programs/util.c`.
//!
//! The C header remains the public ABI. Functions in this module therefore
//! use the C allocator for objects which are returned to C, keep all exported
//! structures `repr(C)`, and use the target's `libc::stat` representation for
//! the `stat_t` pointer passed by `util.h`.
use std::ffi::{CStr, CString, OsString};
use std::fs;
use std::mem::{offset_of, size_of, MaybeUninit};
use std::os::raw::{c_char, c_int, c_uint, c_void};
use std::path::PathBuf;
use std::ptr;
use std::sync::atomic::{AtomicBool, AtomicI32, Ordering};
use std::sync::OnceLock;
#[cfg(unix)]
use std::os::unix::ffi::OsStringExt;
type Stat = libc::stat;
#[cfg(unix)]
type UtilMode = libc::mode_t;
#[cfg(windows)]
type UtilMode = c_int;
#[cfg(not(any(unix, windows)))]
type UtilMode = c_uint;
const LIST_SIZE_INCREASE: usize = 8 * 1024;
const MAX_FILE_OF_FILE_NAMES_SIZE: u64 = 50 * (1 << 20);
const UTIL_FILESIZE_UNKNOWN: u64 = u64::MAX;
const DIR_DEFAULT_MODE: u32 = 0o755;
/// `g_utilDisplayLevel` is a public C global, not a Rust-owned preference.
#[no_mangle]
pub static mut g_utilDisplayLevel: c_int = 0;
/// The two structs below mirror the declarations in `programs/util.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct UTIL_HumanReadableSize_t {
pub value: f64,
pub precision: c_int,
pub suffix: *const c_char,
}
#[repr(C)]
#[derive(Debug)]
pub struct FileNamesTable {
pub fileNames: *mut *const c_char,
pub buf: *mut c_char,
pub tableSize: usize,
pub tableCapacity: usize,
}
const _: () = assert!(offset_of!(UTIL_HumanReadableSize_t, value) == 0);
const _: () = assert!(offset_of!(UTIL_HumanReadableSize_t, precision) == size_of::<f64>());
const _: () = assert!(
offset_of!(UTIL_HumanReadableSize_t, suffix)
>= offset_of!(UTIL_HumanReadableSize_t, precision) + size_of::<c_int>()
);
const _: () = assert!(
size_of::<UTIL_HumanReadableSize_t>()
== offset_of!(UTIL_HumanReadableSize_t, suffix) + size_of::<*const c_char>()
);
const _: () = assert!(offset_of!(FileNamesTable, fileNames) == 0);
const _: () = assert!(offset_of!(FileNamesTable, buf) == size_of::<*const c_char>());
const _: () = assert!(
offset_of!(FileNamesTable, tableSize)
== offset_of!(FileNamesTable, buf) + size_of::<*mut c_char>()
);
const _: () = assert!(
offset_of!(FileNamesTable, tableCapacity)
== offset_of!(FileNamesTable, tableSize) + size_of::<usize>()
);
const _: () = assert!(
size_of::<FileNamesTable>() == offset_of!(FileNamesTable, tableCapacity) + size_of::<usize>()
);
static EMPTY_EXTENSION: [u8; 1] = [0];
static SUFFIX_B: &[u8] = b" B\0";
static SUFFIX_KIB: &[u8] = b" KiB\0";
static SUFFIX_MIB: &[u8] = b" MiB\0";
static SUFFIX_GIB: &[u8] = b" GiB\0";
static SUFFIX_TIB: &[u8] = b" TiB\0";
static SUFFIX_PIB: &[u8] = b" PiB\0";
static SUFFIX_EIB: &[u8] = b" EiB\0";
static FAKE_STDIN_IS_CONSOLE: AtomicBool = AtomicBool::new(false);
static FAKE_STDOUT_IS_CONSOLE: AtomicBool = AtomicBool::new(false);
static FAKE_STDERR_IS_CONSOLE: AtomicBool = AtomicBool::new(false);
static TRACE_FILE_STAT: AtomicBool = AtomicBool::new(false);
static TRACE_DEPTH: AtomicI32 = AtomicI32::new(0);
static CORE_COUNT: OnceLock<c_int> = OnceLock::new();
#[cfg(windows)]
unsafe extern "C" {
fn UTIL_rust_utime(filename: *const c_char, statbuf: *const Stat) -> c_int;
}
#[inline]
fn display_level() -> c_int {
// The C API intentionally exposes this mutable global. All reads mirror
// the unsynchronized reads in the original implementation.
unsafe { g_utilDisplayLevel }
}
fn display(message: &str) {
eprint!("{message}");
}
unsafe fn display_c_string(message: *const c_char) {
if !message.is_null() {
display(&String::from_utf8_lossy(CStr::from_ptr(message).to_bytes()));
}
}
fn trace_call(message: &str) {
if TRACE_FILE_STAT.load(Ordering::Relaxed) {
let depth = TRACE_DEPTH.fetch_add(1, Ordering::Relaxed).max(0) as usize;
eprintln!("Trace:FileStat: {:width$}> {message}", "", width = depth);
}
}
fn trace_return(ret: c_int) {
if TRACE_FILE_STAT.load(Ordering::Relaxed) {
let depth = TRACE_DEPTH.fetch_sub(1, Ordering::Relaxed) - 1;
eprintln!(
"Trace:FileStat: {:width$}< {ret}",
"",
width = depth.max(0) as usize
);
}
}
#[inline]
unsafe fn c_bytes<'a>(value: *const c_char) -> &'a [u8] {
CStr::from_ptr(value).to_bytes()
}
#[inline]
fn path_separator() -> u8 {
if cfg!(windows) {
b'\\'
} else {
b'/'
}
}
fn path_from_bytes(bytes: &[u8]) -> PathBuf {
#[cfg(unix)]
{
PathBuf::from(OsString::from_vec(bytes.to_vec()))
}
#[cfg(not(unix))]
{
PathBuf::from(String::from_utf8_lossy(bytes).into_owned())
}
}
fn os_string_bytes(value: OsString) -> Vec<u8> {
#[cfg(unix)]
{
value.into_vec()
}
#[cfg(not(unix))]
{
value.to_string_lossy().into_owned().into_bytes()
}
}
fn c_path(bytes: &[u8]) -> CString {
CString::new(bytes).unwrap_or_else(|_| std::process::abort())
}
unsafe fn malloc_bytes(size: usize) -> *mut u8 {
let ptr = libc::malloc(size).cast::<u8>();
if ptr.is_null() {
std::process::abort();
}
ptr
}
unsafe fn malloc_array<T>(count: usize) -> *mut T {
let Some(size) = count.checked_mul(size_of::<T>()) else {
return ptr::null_mut();
};
libc::malloc(size).cast::<T>()
}
unsafe fn allocate_struct<T>() -> *mut T {
let result = malloc_bytes(size_of::<T>()).cast::<T>();
// The allocation helper is used only for C functions whose original
// CONTROL() path terminates the program on allocation failure.
ptr::write_bytes(result.cast::<u8>(), 0, size_of::<T>());
result
}
unsafe fn free_ptr<T>(value: *mut T) {
libc::free(value.cast::<c_void>());
}
unsafe fn table_entries<'a>(table: *const FileNamesTable) -> &'a [*const c_char] {
if (*table).fileNames.is_null() || (*table).tableSize == 0 {
&[]
} else {
std::slice::from_raw_parts((*table).fileNames, (*table).tableSize)
}
}
unsafe fn table_names_size(table: *const FileNamesTable) -> usize {
let mut total = 0usize;
for &name in table_entries(table) {
if name.is_null() {
break;
}
total = total.saturating_add(c_bytes(name).len().saturating_add(1));
}
total
}
unsafe fn make_table(
file_names: *mut *const c_char,
table_size: usize,
table_capacity: usize,
buf: *mut c_char,
) -> *mut FileNamesTable {
let table = allocate_struct::<FileNamesTable>();
(*table).fileNames = file_names;
(*table).buf = buf;
(*table).tableSize = table_size;
(*table).tableCapacity = table_capacity;
table
}
unsafe fn make_table_from_buffer(
buf: *mut u8,
buffer_len: usize,
table_size: usize,
table_capacity: usize,
) -> *mut FileNamesTable {
let Some(pointer_count) = table_capacity.checked_mul(size_of::<*const c_char>()) else {
free_ptr(buf);
return ptr::null_mut();
};
let file_names = malloc_bytes(pointer_count.max(1)).cast::<*const c_char>();
let mut pos = 0usize;
for index in 0..table_size {
if pos > buffer_len {
free_ptr(file_names);
free_ptr(buf);
return ptr::null_mut();
}
*file_names.add(index) = buf.add(pos).cast::<c_char>();
let name_len = CStr::from_ptr(buf.add(pos).cast::<c_char>())
.to_bytes()
.len();
pos = pos.saturating_add(name_len + 1);
}
make_table(file_names, table_size, table_capacity, buf.cast::<c_char>())
}
#[inline]
unsafe fn stat_mode(statbuf: *const Stat) -> u64 {
(*statbuf).st_mode as u64
}
#[cfg(unix)]
#[inline]
unsafe fn mode_is(statbuf: *const Stat, kind: libc::mode_t) -> bool {
stat_mode(statbuf) & libc::S_IFMT as u64 == kind as u64
}
#[cfg(windows)]
#[inline]
unsafe fn mode_is(statbuf: *const Stat, kind: c_int) -> bool {
// util.c intentionally uses `(st_mode & S_IFREG) != 0` for the CRT.
stat_mode(statbuf) & kind as u64 != 0
}
#[cfg(not(any(unix, windows)))]
#[inline]
unsafe fn mode_is(_statbuf: *const Stat, _kind: c_uint) -> bool {
false
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_fstat(
fd: c_int,
filename: *const c_char,
statbuf: *mut Stat,
) -> c_int {
trace_call("UTIL_stat");
let result = if fd >= 0 {
libc::fstat(fd, statbuf)
} else {
libc::stat(filename, statbuf)
};
let ret = (result == 0) as c_int;
trace_return(ret);
ret
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_stat(filename: *const c_char, statbuf: *mut Stat) -> c_int {
UTIL_fstat(-1, filename, statbuf)
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isRegularFileStat(statbuf: *const Stat) -> c_int {
#[cfg(any(unix, windows))]
{
mode_is(statbuf, libc::S_IFREG) as c_int
}
#[cfg(not(any(unix, windows)))]
{
0
}
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isDirectoryStat(statbuf: *const Stat) -> c_int {
#[cfg(any(unix, windows))]
{
mode_is(statbuf, libc::S_IFDIR) as c_int
}
#[cfg(not(any(unix, windows)))]
{
0
}
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isFIFOStat(statbuf: *const Stat) -> c_int {
#[cfg(unix)]
{
mode_is(statbuf, libc::S_IFIFO) as c_int
}
#[cfg(not(unix))]
{
let _ = statbuf;
0
}
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isBlockDevStat(statbuf: *const Stat) -> c_int {
#[cfg(unix)]
{
mode_is(statbuf, libc::S_IFBLK) as c_int
}
#[cfg(not(unix))]
{
let _ = statbuf;
0
}
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isRegularFile(infilename: *const c_char) -> c_int {
let mut statbuf = MaybeUninit::<Stat>::uninit();
if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 {
return 0;
}
UTIL_isRegularFileStat(statbuf.as_ptr())
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isDirectory(infilename: *const c_char) -> c_int {
let mut statbuf = MaybeUninit::<Stat>::uninit();
if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 {
return 0;
}
UTIL_isDirectoryStat(statbuf.as_ptr())
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isFIFO(infilename: *const c_char) -> c_int {
#[cfg(unix)]
{
let mut statbuf = MaybeUninit::<Stat>::uninit();
if UTIL_stat(infilename, statbuf.as_mut_ptr()) != 0 {
return UTIL_isFIFOStat(statbuf.as_ptr());
}
}
let _ = infilename;
0
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isLink(infilename: *const c_char) -> c_int {
#[cfg(unix)]
{
let mut statbuf = MaybeUninit::<Stat>::uninit();
if libc::lstat(infilename, statbuf.as_mut_ptr()) == 0 {
return mode_is(statbuf.as_ptr(), libc::S_IFLNK) as c_int;
}
}
let _ = infilename;
0
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isSameFile(file1: *const c_char, file2: *const c_char) -> c_int {
assert!(!file1.is_null() && !file2.is_null());
#[cfg(windows)]
{
return (c_bytes(file1) == c_bytes(file2)) as c_int;
}
#[cfg(not(windows))]
{
let mut file1_stat = MaybeUninit::<Stat>::uninit();
let mut file2_stat = MaybeUninit::<Stat>::uninit();
if UTIL_stat(file1, file1_stat.as_mut_ptr()) == 0
|| UTIL_stat(file2, file2_stat.as_mut_ptr()) == 0
{
return 0;
}
UTIL_isSameFileStat(file1, file2, file1_stat.as_ptr(), file2_stat.as_ptr())
}
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isSameFileStat(
file1: *const c_char,
file2: *const c_char,
file1_stat: *const Stat,
file2_stat: *const Stat,
) -> c_int {
assert!(!file1.is_null() && !file2.is_null());
#[cfg(windows)]
{
let _ = (file1_stat, file2_stat);
return (c_bytes(file1) == c_bytes(file2)) as c_int;
}
#[cfg(not(windows))]
{
((*file1_stat).st_dev == (*file2_stat).st_dev
&& (*file1_stat).st_ino == (*file2_stat).st_ino) as c_int
}
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_getFileSizeStat(statbuf: *const Stat) -> u64 {
if UTIL_isRegularFileStat(statbuf) == 0 {
return UTIL_FILESIZE_UNKNOWN;
}
(*statbuf).st_size as u64
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_getFileSize(infilename: *const c_char) -> u64 {
let mut statbuf = MaybeUninit::<Stat>::uninit();
if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 {
return UTIL_FILESIZE_UNKNOWN;
}
UTIL_getFileSizeStat(statbuf.as_ptr())
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_getTotalFileSize(
file_names: *const *const c_char,
nb_files: c_uint,
) -> u64 {
let mut total = 0u64;
for index in 0..nb_files as usize {
let size = UTIL_getFileSize(*file_names.add(index));
if size == UTIL_FILESIZE_UNKNOWN {
return UTIL_FILESIZE_UNKNOWN;
}
total = total.wrapping_add(size);
}
total
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_chmod(
filename: *const c_char,
statbuf: *const Stat,
permissions: UtilMode,
) -> c_int {
UTIL_fchmod(-1, filename, statbuf, permissions)
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_fchmod(
fd: c_int,
filename: *const c_char,
statbuf: *const Stat,
permissions: UtilMode,
) -> c_int {
let mut local_stat = MaybeUninit::<Stat>::uninit();
let stat_ptr = if statbuf.is_null() {
if UTIL_fstat(fd, filename, local_stat.as_mut_ptr()) == 0 {
return 0;
}
local_stat.as_ptr()
} else {
statbuf
};
if UTIL_isRegularFileStat(stat_ptr) == 0 {
return 0;
}
#[cfg(unix)]
{
if fd >= 0 {
return libc::fchmod(fd, permissions);
}
libc::chmod(filename, permissions)
}
#[cfg(windows)]
{
let _ = fd;
libc::chmod(filename, permissions)
}
#[cfg(not(any(unix, windows)))]
{
let _ = (fd, filename, permissions);
0
}
}
#[cfg(any(target_os = "linux", target_os = "android"))]
unsafe fn set_mtime(filename: *const c_char, statbuf: *const Stat) -> c_int {
let now = libc::timespec {
tv_sec: 0,
tv_nsec: libc::UTIME_NOW,
};
let mtime = libc::timespec {
tv_sec: (*statbuf).st_mtime,
tv_nsec: (*statbuf).st_mtime_nsec as _,
};
let times = [now, mtime];
libc::utimensat(libc::AT_FDCWD, filename, times.as_ptr(), 0)
}
#[cfg(all(unix, not(any(target_os = "linux", target_os = "android"))))]
unsafe fn set_mtime(filename: *const c_char, statbuf: *const Stat) -> c_int {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |duration| duration.as_secs()) as libc::time_t;
let times = libc::utimbuf {
actime: now,
modtime: (*statbuf).st_mtime,
};
libc::utime(filename, &times)
}
#[cfg(windows)]
unsafe fn set_mtime(filename: *const c_char, statbuf: *const Stat) -> c_int {
UTIL_rust_utime(filename, statbuf)
}
#[cfg(not(any(unix, windows)))]
unsafe fn set_mtime(_filename: *const c_char, _statbuf: *const Stat) -> c_int {
-1
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_utime(filename: *const c_char, statbuf: *const Stat) -> c_int {
set_mtime(filename, statbuf)
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_setFileStat(filename: *const c_char, statbuf: *const Stat) -> c_int {
UTIL_setFDStat(-1, filename, statbuf)
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_setFDStat(
fd: c_int,
filename: *const c_char,
statbuf: *const Stat,
) -> c_int {
let mut current_stat = MaybeUninit::<Stat>::uninit();
if UTIL_fstat(fd, filename, current_stat.as_mut_ptr()) == 0
|| UTIL_isRegularFileStat(current_stat.as_ptr()) == 0
{
return -1;
}
let mut result = 0;
#[cfg(unix)]
{
let no_uid = !0 as libc::uid_t;
if fd >= 0 {
result += libc::fchown(fd, no_uid, (*statbuf).st_gid);
} else {
result += libc::chown(filename, no_uid, (*statbuf).st_gid);
}
}
let permissions = ((*statbuf).st_mode as u64 & 0o777) as UtilMode;
result += UTIL_fchmod(fd, filename, current_stat.as_ptr(), permissions);
#[cfg(unix)]
{
let no_gid = !0 as libc::gid_t;
if fd >= 0 {
result += libc::fchown(fd, (*statbuf).st_uid, no_gid);
} else {
result += libc::chown(filename, (*statbuf).st_uid, no_gid);
}
}
-result
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_requireUserConfirmation(
prompt: *const c_char,
abort_msg: *const c_char,
acceptable_letters: *const c_char,
has_stdin_input: c_int,
) -> c_int {
if has_stdin_input != 0 {
display("stdin is an input - not proceeding.\n");
return 1;
}
display_c_string(prompt);
let ch = libc::getchar();
let accepted = if acceptable_letters.is_null() {
false
} else {
c_bytes(acceptable_letters).contains(&(ch as u8))
};
let result = if accepted { 0 } else { 1 };
if result != 0 {
display_c_string(abort_msg);
display(" \n");
}
let mut next = ch;
while next != libc::EOF && next != b'\n' as c_int {
next = libc::getchar();
}
result
}
#[no_mangle]
pub extern "C" fn UTIL_traceFileStat() {
TRACE_FILE_STAT.store(true, Ordering::Relaxed);
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isConsole(file: *mut libc::FILE) -> c_int {
if file.is_null() {
return 0;
}
let fd = libc::fileno(file);
if fd == 0 && FAKE_STDIN_IS_CONSOLE.load(Ordering::Relaxed)
|| fd == 1 && FAKE_STDOUT_IS_CONSOLE.load(Ordering::Relaxed)
|| fd == 2 && FAKE_STDERR_IS_CONSOLE.load(Ordering::Relaxed)
{
return 1;
}
if fd < 0 {
return 0;
}
libc::isatty(fd)
}
#[no_mangle]
pub extern "C" fn UTIL_fakeStdinIsConsole() {
FAKE_STDIN_IS_CONSOLE.store(true, Ordering::Relaxed);
}
#[no_mangle]
pub extern "C" fn UTIL_fakeStdoutIsConsole() {
FAKE_STDOUT_IS_CONSOLE.store(true, Ordering::Relaxed);
}
#[no_mangle]
pub extern "C" fn UTIL_fakeStderrIsConsole() {
FAKE_STDERR_IS_CONSOLE.store(true, Ordering::Relaxed);
}
fn suffix_ptr(suffix: &'static [u8]) -> *const c_char {
suffix.as_ptr().cast::<c_char>()
}
#[no_mangle]
pub extern "C" fn UTIL_makeHumanReadableSize(size: u64) -> UTIL_HumanReadableSize_t {
let result;
if display_level() > 3 {
if size >= 1u64 << 53 {
result = UTIL_HumanReadableSize_t {
value: size as f64 / (1u64 << 20) as f64,
precision: 2,
suffix: suffix_ptr(SUFFIX_MIB),
};
} else {
result = UTIL_HumanReadableSize_t {
value: size as f64,
precision: 0,
suffix: suffix_ptr(SUFFIX_B),
};
}
} else {
let (value, suffix) = if size >= 1u64 << 60 {
(size as f64 / (1u64 << 60) as f64, SUFFIX_EIB)
} else if size >= 1u64 << 50 {
(size as f64 / (1u64 << 50) as f64, SUFFIX_PIB)
} else if size >= 1u64 << 40 {
(size as f64 / (1u64 << 40) as f64, SUFFIX_TIB)
} else if size >= 1u64 << 30 {
(size as f64 / (1u64 << 30) as f64, SUFFIX_GIB)
} else if size >= 1u64 << 20 {
(size as f64 / (1u64 << 20) as f64, SUFFIX_MIB)
} else if size >= 1u64 << 10 {
(size as f64 / (1u64 << 10) as f64, SUFFIX_KIB)
} else {
(size as f64, SUFFIX_B)
};
let precision = if value >= 100.0 || value as u64 == size {
0
} else if value >= 10.0 {
1
} else if value > 1.0 {
2
} else {
3
};
result = UTIL_HumanReadableSize_t {
value,
precision,
suffix: suffix_ptr(suffix),
};
}
result
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_compareStr(p1: *const c_void, p2: *const c_void) -> c_int {
let left = *(p1.cast::<*const c_char>());
let right = *(p2.cast::<*const c_char>());
let left = c_bytes(left);
let right = c_bytes(right);
for (&a, &b) in left.iter().zip(right.iter()) {
if a != b {
return a as c_int - b as c_int;
}
}
left.len() as c_int - right.len() as c_int
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_getFileExtension(infilename: *const c_char) -> *const c_char {
let bytes = c_bytes(infilename);
let Some(position) = bytes.iter().rposition(|&byte| byte == b'.') else {
return EMPTY_EXTENSION.as_ptr().cast::<c_char>();
};
if position == 0 {
EMPTY_EXTENSION.as_ptr().cast::<c_char>()
} else {
infilename.add(position)
}
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isCompressedFile(
input_name: *const c_char,
extension_list: *const *const c_char,
) -> c_int {
let extension = CStr::from_ptr(UTIL_getFileExtension(input_name)).to_bytes();
if extension_list.is_null() {
return 0;
}
let mut current = extension_list;
loop {
let candidate = *current;
if candidate.is_null() {
return 0;
}
if c_bytes(candidate) == extension {
return 1;
}
current = current.add(1);
}
}
unsafe fn pathname_has_two_dots(pathname: &[u8]) -> bool {
let separator = path_separator();
for (index, pair) in pathname.windows(2).enumerate() {
if pair != b".." {
continue;
}
let left_boundary = index == 0 || pathname[index - 1] == separator;
let right_index = index + 2;
let right_boundary = right_index == pathname.len() || pathname[right_index] == separator;
if left_boundary && right_boundary {
return true;
}
}
false
}
fn trim_path(pathname: &[u8]) -> &[u8] {
let separator = path_separator();
let mut path = pathname;
if path.first() == Some(&separator) {
path = &path[1..];
}
if path.len() >= 2 && path[0] == b'.' && path[1] == separator {
path = &path[2..];
}
path
}
fn join_two_dirs(dir1: &[u8], dir2: &[u8]) -> Vec<u8> {
let separator = path_separator();
let mut result = Vec::with_capacity(dir1.len() + dir2.len() + 1);
result.extend_from_slice(dir1);
if !dir1.is_empty() && dir1.last() != Some(&separator) {
result.push(separator);
}
result.extend_from_slice(dir2);
result
}
fn convert_pathname_to_dir_name(pathname: &mut Vec<u8>) {
let separator = path_separator();
if let Some(position) = pathname.iter().rposition(|&byte| byte == separator) {
pathname.truncate(position);
} else {
pathname.clear();
pathname.push(b'.');
}
}
unsafe fn get_dir_mode(dir_name: &[u8]) -> u32 {
let path = c_path(dir_name);
let mut statbuf = MaybeUninit::<Stat>::uninit();
if UTIL_stat(path.as_ptr(), statbuf.as_mut_ptr()) == 0 {
if display_level() >= 1 {
eprintln!(
"zstd: failed to get DIR stats {}",
String::from_utf8_lossy(dir_name)
);
}
return DIR_DEFAULT_MODE;
}
if UTIL_isDirectoryStat(statbuf.as_ptr()) == 0 {
if display_level() >= 1 {
eprintln!(
"zstd: expected directory: {}",
String::from_utf8_lossy(dir_name)
);
}
return DIR_DEFAULT_MODE;
}
stat_mode(statbuf.as_ptr()) as u32
}
unsafe fn make_dir(dir: &[u8], mode: u32) -> c_int {
let path = c_path(dir);
#[cfg(unix)]
let result = libc::mkdir(path.as_ptr(), mode as libc::mode_t);
#[cfg(windows)]
let result = {
let _ = mode;
libc::mkdir(path.as_ptr())
};
#[cfg(not(any(unix, windows)))]
let result = {
let _ = (path, mode);
-1
};
if result != 0 {
if std::io::Error::last_os_error().raw_os_error() == Some(libc::EEXIST) {
return 0;
}
if display_level() >= 1 {
eprintln!(
"zstd: failed to create DIR {}: {}",
String::from_utf8_lossy(dir),
std::io::Error::last_os_error()
);
}
}
result
}
unsafe fn mirror_src_dir(src_dir_name: &[u8], out_dir_name: &[u8]) -> c_int {
let new_dir = join_two_dirs(out_dir_name, trim_path(src_dir_name));
let mode = get_dir_mode(src_dir_name);
make_dir(&new_dir, mode)
}
fn mirror_src_dir_recursive(src_dir_name: &[u8], out_dir_name: &[u8]) {
let separator = path_separator();
let start =
if src_dir_name.len() >= 2 && src_dir_name[0] == b'.' && src_dir_name[1] == separator {
2
} else {
0
};
let mut previous = start;
for (position, &byte) in src_dir_name.iter().enumerate().skip(start) {
if byte != separator {
continue;
}
if position != previous {
unsafe {
let _ = mirror_src_dir(&src_dir_name[..position], out_dir_name);
}
}
previous = position + 1;
}
unsafe {
let _ = mirror_src_dir(src_dir_name, out_dir_name);
}
}
fn first_is_parent_or_same_dir(first: &[u8], second: &[u8]) -> bool {
let separator = path_separator();
first.len() <= second.len()
&& (second.get(first.len()) == Some(&separator) || second.get(first.len()).is_none())
&& second.starts_with(first)
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_createMirroredDestDirName(
src_file_name: *const c_char,
out_dir_root_name: *const c_char,
) -> *mut c_char {
let src = c_bytes(src_file_name);
if pathname_has_two_dots(src) {
return ptr::null_mut();
}
let mut pathname = join_two_dirs(c_bytes(out_dir_root_name), trim_path(src));
convert_pathname_to_dir_name(&mut pathname);
let result = malloc_bytes(pathname.len() + 1).cast::<c_char>();
ptr::copy_nonoverlapping(pathname.as_ptr().cast::<c_char>(), result, pathname.len());
*result.add(pathname.len()) = 0;
result
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_mirrorSourceFilesDirectories(
file_names: *const *const c_char,
nb_files: c_uint,
out_dir_name: *const c_char,
) {
let out_dir = c_bytes(out_dir_name).to_vec();
let mut source_dirs = Vec::new();
for index in 0..nb_files as usize {
let source = c_bytes(*file_names.add(index));
if !pathname_has_two_dots(source) {
let mut source_dir = source.to_vec();
convert_pathname_to_dir_name(&mut source_dir);
source_dirs.push(source_dir);
}
}
if source_dirs.is_empty() {
return;
}
let _ = make_dir(&out_dir, DIR_DEFAULT_MODE);
source_dirs.sort_by(|left, right| trim_path(left).cmp(trim_path(right)));
let mut unique_dirs: Vec<&[u8]> = Vec::new();
unique_dirs.push(source_dirs[0].as_slice());
for index in 1..source_dirs.len() {
let previous = trim_path(&source_dirs[index - 1]);
let current = trim_path(&source_dirs[index]);
if first_is_parent_or_same_dir(previous, current) {
*unique_dirs.last_mut().unwrap() = source_dirs[index].as_slice();
} else {
unique_dirs.push(source_dirs[index].as_slice());
}
}
for source_dir in unique_dirs {
mirror_src_dir_recursive(source_dir, &out_dir);
}
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_createFileNamesTable_fromFileName(
input_file_name: *const c_char,
) -> *mut FileNamesTable {
let input_name = c_bytes(input_file_name);
let path = path_from_bytes(input_name);
let c_input_name = c_path(input_name);
let mut statbuf = MaybeUninit::<Stat>::uninit();
if UTIL_stat(c_input_name.as_ptr(), statbuf.as_mut_ptr()) == 0
|| UTIL_isRegularFileStat(statbuf.as_ptr()) == 0
{
return ptr::null_mut();
}
let file_size = UTIL_getFileSizeStat(statbuf.as_ptr());
if file_size > MAX_FILE_OF_FILE_NAMES_SIZE {
return ptr::null_mut();
}
let data = match fs::read(path) {
Ok(data) => data,
Err(error) => {
if display_level() >= 1 {
eprintln!("zstd:util:readLinesFromFile: {error}");
}
return ptr::null_mut();
}
};
if data.is_empty() {
return ptr::null_mut();
}
let line_count = data.iter().filter(|&&byte| byte == b'\n').count()
+ usize::from(data.last() != Some(&b'\n'));
let buffer = malloc_bytes(data.len() + 1);
ptr::copy_nonoverlapping(data.as_ptr(), buffer, data.len());
for index in 0..data.len() {
if *buffer.add(index) == b'\n' {
*buffer.add(index) = 0;
}
}
*buffer.add(data.len()) = 0;
make_table_from_buffer(buffer, data.len() + 1, line_count, line_count)
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_assembleFileNamesTable(
filenames: *mut *const c_char,
table_size: usize,
buf: *mut c_char,
) -> *mut FileNamesTable {
make_table(filenames, table_size, table_size, buf)
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_freeFileNamesTable(table: *mut FileNamesTable) {
if table.is_null() {
return;
}
free_ptr((*table).fileNames);
free_ptr((*table).buf);
free_ptr(table);
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_allocateFileNamesTable(table_size: usize) -> *mut FileNamesTable {
let names = malloc_array::<*const c_char>(table_size);
if names.is_null() {
return ptr::null_mut();
}
make_table(names, 0, table_size, ptr::null_mut())
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_searchFileNamesTable(
table: *mut FileNamesTable,
name: *const c_char,
) -> c_int {
for (index, &candidate) in table_entries(table).iter().enumerate() {
if !candidate.is_null() && c_bytes(candidate) == c_bytes(name) {
return index as c_int;
}
}
-1
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_refFilename(table: *mut FileNamesTable, filename: *const c_char) {
if table.is_null() || (*table).tableSize >= (*table).tableCapacity {
std::process::abort();
}
*(*table).fileNames.add((*table).tableSize) = filename;
(*table).tableSize += 1;
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_mergeFileNamesTable(
table1: *mut FileNamesTable,
table2: *mut FileNamesTable,
) -> *mut FileNamesTable {
let total_size = table_names_size(table1).saturating_add(table_names_size(table2));
let new_size = (*table1).tableSize.saturating_add((*table2).tableSize);
let buffer = malloc_bytes(total_size.max(1));
ptr::write_bytes(buffer, 0, total_size.max(1));
let names = malloc_bytes(new_size.saturating_mul(size_of::<*const c_char>()).max(1))
.cast::<*const c_char>();
ptr::write_bytes(
names.cast::<u8>(),
0,
new_size.saturating_mul(size_of::<*const c_char>()),
);
let mut new_index = 0usize;
let mut position = 0usize;
for source in [table1, table2] {
for &name in table_entries(source) {
if name.is_null() || position >= total_size {
break;
}
let name_bytes = c_bytes(name);
ptr::copy_nonoverlapping(name_bytes.as_ptr(), buffer.add(position), name_bytes.len());
*names.add(new_index) = buffer.add(position).cast::<c_char>();
position += name_bytes.len() + 1;
new_index += 1;
}
}
let result = make_table(names, new_index, 0, buffer.cast::<c_char>());
UTIL_freeFileNamesTable(table1);
UTIL_freeFileNamesTable(table2);
result
}
fn append_name(buffer: &mut Vec<u8>, name: &[u8]) {
buffer.extend_from_slice(name);
buffer.push(0);
}
fn join_entry_path(dir_name: &[u8], entry_name: &[u8]) -> Vec<u8> {
let mut path = Vec::with_capacity(dir_name.len() + entry_name.len() + 1);
path.extend_from_slice(dir_name);
path.push(path_separator());
path.extend_from_slice(entry_name);
path
}
unsafe fn is_directory_bytes(name: &[u8]) -> bool {
let path = c_path(name);
let mut statbuf = MaybeUninit::<Stat>::uninit();
UTIL_stat(path.as_ptr(), statbuf.as_mut_ptr()) != 0
&& UTIL_isDirectoryStat(statbuf.as_ptr()) != 0
}
unsafe fn is_link_bytes(name: &[u8]) -> bool {
#[cfg(unix)]
{
let path = c_path(name);
let mut statbuf = MaybeUninit::<Stat>::uninit();
libc::lstat(path.as_ptr(), statbuf.as_mut_ptr()) == 0
&& mode_is(statbuf.as_ptr(), libc::S_IFLNK)
}
#[cfg(not(unix))]
{
let _ = name;
false
}
}
fn walk_directory(
dir_name: &[u8],
buffer: &mut Vec<u8>,
nb_files: &mut usize,
follow_links: bool,
) -> Result<(), ()> {
let entries = match fs::read_dir(path_from_bytes(dir_name)) {
Ok(entries) => entries,
Err(error) => {
if display_level() >= 1 {
eprintln!(
"Cannot open directory '{}': {error}",
String::from_utf8_lossy(dir_name)
);
}
return Ok(());
}
};
for entry in entries {
let entry = entry.map_err(|_| ())?;
let entry_name = os_string_bytes(entry.file_name());
if entry_name == b"." || entry_name == b".." {
continue;
}
let path = join_entry_path(dir_name, &entry_name);
if !follow_links && unsafe { is_link_bytes(&path) } {
if display_level() >= 2 {
eprintln!(
"Warning : {} is a symbolic link, ignoring",
String::from_utf8_lossy(&path)
);
}
continue;
}
if unsafe { is_directory_bytes(&path) } {
walk_directory(&path, buffer, nb_files, follow_links)?;
} else {
append_name(buffer, &path);
*nb_files += 1;
}
}
Ok(())
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_createExpandedFNT(
input_names: *const *const c_char,
nb_input_names: usize,
follow_links: c_int,
) -> *mut FileNamesTable {
let mut buffer = Vec::with_capacity(LIST_SIZE_INCREASE);
let mut nb_files = 0usize;
for index in 0..nb_input_names {
let input = *input_names.add(index);
let input_bytes = c_bytes(input);
if !is_directory_bytes(input_bytes) {
append_name(&mut buffer, input_bytes);
nb_files += 1;
} else if walk_directory(input_bytes, &mut buffer, &mut nb_files, follow_links != 0)
.is_err()
{
return ptr::null_mut();
}
}
let table_capacity = nb_files.saturating_add(1);
let allocated_len = buffer.len().max(LIST_SIZE_INCREASE);
let output_buffer = malloc_bytes(allocated_len);
if !buffer.is_empty() {
ptr::copy_nonoverlapping(buffer.as_ptr(), output_buffer, buffer.len());
}
make_table_from_buffer(output_buffer, buffer.len(), nb_files, table_capacity)
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_expandFNT(table: *mut *mut FileNamesTable, follow_links: c_int) {
let old_table = *table;
let new_table =
UTIL_createExpandedFNT((*old_table).fileNames, (*old_table).tableSize, follow_links);
if new_table.is_null() {
std::process::abort();
}
UTIL_freeFileNamesTable(old_table);
*table = new_table;
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_createFNT_fromROTable(
filenames: *const *const c_char,
nb_filenames: usize,
) -> *mut FileNamesTable {
let new_names = malloc_array::<*const c_char>(nb_filenames);
if new_names.is_null() {
return ptr::null_mut();
}
ptr::copy_nonoverlapping(filenames, new_names, nb_filenames);
UTIL_assembleFileNamesTable(new_names, nb_filenames, ptr::null_mut())
}
#[cfg(windows)]
#[no_mangle]
pub extern "system" fn CountSetBits(bit_mask: usize) -> u32 {
bit_mask.count_ones()
}
#[cfg(target_os = "linux")]
fn count_cores_platform(logical: c_int) -> c_int {
let online = unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) };
let mut count = if online < 0 { 1 } else { online as c_int };
if logical != 0 {
return count;
}
let Ok(cpuinfo) = fs::read_to_string("/proc/cpuinfo") else {
return count;
};
let mut siblings = 0;
let mut cpu_cores = 0;
for line in cpuinfo.lines() {
if line.starts_with("siblings") {
let Some((_, value)) = line.split_once(':') else {
return count;
};
let Ok(value) = value.trim().parse::<c_int>() else {
return count;
};
siblings = value;
}
if line.starts_with("cpu cores") {
let Some((_, value)) = line.split_once(':') else {
return count;
};
let Ok(value) = value.trim().parse::<c_int>() else {
return count;
};
cpu_cores = value;
}
}
if siblings > cpu_cores && cpu_cores > 0 {
let ratio = siblings / cpu_cores;
if ratio > 0 && count > ratio {
count /= ratio;
}
}
count
}
#[cfg(all(unix, not(target_os = "linux"), not(target_vendor = "apple")))]
fn count_cores_platform(_logical: c_int) -> c_int {
let count = unsafe { libc::sysconf(libc::_SC_NPROCESSORS_ONLN) };
if count < 0 {
1
} else {
count as c_int
}
}
#[cfg(target_vendor = "apple")]
fn count_cores_platform(logical: c_int) -> c_int {
let name = if logical != 0 {
CString::new("hw.logicalcpu").unwrap()
} else {
CString::new("hw.physicalcpu").unwrap()
};
let mut count = 0i32;
let mut size = size_of::<i32>();
let result = unsafe {
libc::sysctlbyname(
name.as_ptr(),
(&mut count as *mut i32).cast::<c_void>(),
&mut size,
ptr::null_mut(),
0,
)
};
if result != 0 {
if std::io::Error::last_os_error().raw_os_error() == Some(libc::ENOENT) {
1
} else {
std::process::abort()
}
} else {
count
}
}
#[cfg(windows)]
fn count_cores_platform(_logical: c_int) -> c_int {
std::thread::available_parallelism()
.map_or(1, |count| count.get().min(c_int::MAX as usize) as c_int)
}
#[cfg(not(any(unix, windows)))]
fn count_cores_platform(_logical: c_int) -> c_int {
1
}
#[no_mangle]
pub extern "C" fn UTIL_countCores(logical: c_int) -> c_int {
*CORE_COUNT.get_or_init(|| count_cores_platform(logical))
}
#[no_mangle]
pub extern "C" fn UTIL_countPhysicalCores() -> c_int {
UTIL_countCores(0)
}
#[no_mangle]
pub extern "C" fn UTIL_countLogicalCores() -> c_int {
UTIL_countCores(1)
}
#[cfg(test)]
mod tests {
use super::*;
use std::ffi::CString;
use std::fs::{self, File};
use std::io::Write;
#[test]
fn public_struct_layout_is_pointer_and_c_int_stable() {
assert_eq!(
std::mem::align_of::<UTIL_HumanReadableSize_t>(),
std::mem::align_of::<f64>()
);
assert_eq!(offset_of!(UTIL_HumanReadableSize_t, value), 0);
assert_eq!(
offset_of!(UTIL_HumanReadableSize_t, precision),
size_of::<f64>()
);
assert_eq!(offset_of!(FileNamesTable, fileNames), 0);
assert_eq!(offset_of!(FileNamesTable, buf), size_of::<*const c_char>());
assert_eq!(
offset_of!(FileNamesTable, tableSize),
2 * size_of::<*const c_char>()
);
assert_eq!(
size_of::<FileNamesTable>(),
2 * size_of::<*const c_char>() + 2 * size_of::<usize>()
);
}
#[test]
fn stat_extension_and_size_helpers_follow_c_contract() {
let root = std::env::temp_dir().join(format!("zstd-util-{}", std::process::id()));
let _ = fs::remove_dir_all(&root);
fs::create_dir(&root).unwrap();
let file_path = root.join("sample.txt");
fs::write(&file_path, b"hello").unwrap();
let path = CString::new(file_path.to_string_lossy().as_bytes()).unwrap();
let mut statbuf = MaybeUninit::<Stat>::uninit();
unsafe {
assert_eq!(UTIL_stat(path.as_ptr(), statbuf.as_mut_ptr()), 1);
assert_eq!(UTIL_isRegularFileStat(statbuf.as_ptr()), 1);
assert_eq!(UTIL_getFileSizeStat(statbuf.as_ptr()), 5);
assert_eq!(
CStr::from_ptr(UTIL_getFileExtension(path.as_ptr())).to_bytes(),
b".txt"
);
let extension = CString::new(".txt").unwrap();
let extensions = [extension.as_ptr(), ptr::null()];
assert_eq!(UTIL_isCompressedFile(path.as_ptr(), extensions.as_ptr()), 1);
}
fs::remove_dir_all(root).unwrap();
}
#[test]
fn filename_tables_own_c_allocations_and_merge_in_order() {
let first = CString::new("first").unwrap();
let second = CString::new("second").unwrap();
let names = [first.as_ptr(), second.as_ptr()];
unsafe {
let table = UTIL_createFNT_fromROTable(names.as_ptr(), names.len());
assert!(!table.is_null());
assert_eq!((*table).tableSize, 2);
assert_eq!(UTIL_searchFileNamesTable(table, second.as_ptr()), 1);
let empty = UTIL_allocateFileNamesTable(1);
assert!(!empty.is_null());
UTIL_refFilename(empty, first.as_ptr());
let merged = UTIL_mergeFileNamesTable(table, empty);
assert_eq!((*merged).tableSize, 3);
assert_eq!(
CStr::from_ptr(*(*merged).fileNames.add(2)).to_bytes(),
b"first"
);
UTIL_freeFileNamesTable(merged);
}
}
#[test]
fn file_list_loading_and_directory_expansion_return_nul_tables() {
let mut root = std::env::temp_dir();
root.push(format!("zstd-util-list-{}", std::process::id()));
let _ = fs::remove_dir_all(&root);
fs::create_dir(&root).unwrap();
let file_list = root.join("list");
let nested = root.join("nested");
fs::create_dir(&nested).unwrap();
File::create(nested.join("one")).unwrap();
let mut list = File::create(&file_list).unwrap();
writeln!(list, "alpha").unwrap();
writeln!(list, "beta").unwrap();
drop(list);
let list_name = CString::new(file_list.to_string_lossy().as_bytes()).unwrap();
let nested_name = CString::new(nested.to_string_lossy().as_bytes()).unwrap();
unsafe {
let table = UTIL_createFileNamesTable_fromFileName(list_name.as_ptr());
assert!(!table.is_null());
assert_eq!((*table).tableSize, 2);
assert_eq!(CStr::from_ptr(*(*table).fileNames).to_bytes(), b"alpha");
UTIL_freeFileNamesTable(table);
let input = [nested_name.as_ptr()];
let expanded = UTIL_createExpandedFNT(input.as_ptr(), 1, 1);
assert!(!expanded.is_null());
assert_eq!((*expanded).tableSize, 1);
assert!(CStr::from_ptr(*(*expanded).fileNames)
.to_bytes()
.ends_with(b"/one"));
UTIL_freeFileNamesTable(expanded);
}
fs::remove_dir_all(root).unwrap();
}
#[test]
fn mirrored_directory_name_rejects_parent_components() {
let source = CString::new("a/../b/file").unwrap();
let root = CString::new("out").unwrap();
unsafe {
assert!(UTIL_createMirroredDestDirName(source.as_ptr(), root.as_ptr()).is_null());
}
let source = CString::new("a/b/file.txt").unwrap();
unsafe {
let name = UTIL_createMirroredDestDirName(source.as_ptr(), root.as_ptr());
assert_eq!(CStr::from_ptr(name).to_bytes(), b"out/a/b");
free_ptr(name);
}
}
#[test]
fn human_readable_size_uses_the_c_scaling_policy() {
unsafe {
g_utilDisplayLevel = 0;
}
let size = UTIL_makeHumanReadableSize(1536);
assert_eq!(size.precision, 2);
unsafe {
assert_eq!(CStr::from_ptr(size.suffix).to_bytes(), b" KiB");
}
assert_eq!(size.value, 1.5);
}
#[test]
fn core_count_helpers_never_report_zero() {
assert!(UTIL_countPhysicalCores() > 0);
assert!(UTIL_countLogicalCores() > 0);
}
}