ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
b0488f285ad9883f489ebc915648488b7027dc66
zstd-rs/rust/src/zstd_cli.rs
T
ddidderr 428dfbc4f5 docs(rust): refresh migration boundary 
Update the Rust rewrite documentation to match the current implementation
boundary.  The legacy decoder set is complete, and Rust now owns several
CLI file-I/O policy and resource leaves while C retains format dispatch,
metadata, and remaining orchestration.  Keep the stated boundary explicit so
passing hybrid tests do not imply that the full rewrite is finished.

Test Plan:
- git diff --cached --check
2026-07-18 18:56:29 +02:00

3567 lines
116 KiB
Rust
Raw Blame History

#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
//! Rust command-line frontend for zstd.
//!
//! This is the Rust parser and dispatch layer plus selected file-I/O leaves,
//! not yet a fully independent file-I/O implementation. It shares the
//! mature C `fileio` layer through a narrow public-in-the-programs-tree ABI:
//! Rust owns preference policy, filename decisions, file opening, dictionary
//! loading, asynchronous pools, and pass-through copying, while C still owns
//! format-specific dispatch, metadata preservation, and remaining streaming
//! orchestration.
//!
//! Benchmark mode (`-b`) parses and dispatches through the Rust CLI archive.
//! The bridge preserves the C CLI's argument normalization and uses the
//! existing Rust `benchzstd` APIs. Builds without the `benchmark` feature
//! return the same unavailable-build result as `ZSTD_NOBENCH` builds without
//! referencing benchmark symbols.
//!
//! Recursive expansion (`-r`), `--filelist`, the output-directory modes, and
//! `--list` reuse the C `FileNamesTable` helpers from `programs/util.c` and
//! `FIO_listMultipleFiles`, so directory traversal and frame inspection stay
//! byte-identical with the C CLI.
//!
//! Remaining C-only CLI boundaries are called out in `unsupported()` below:
//! a few compatibility-only options. Trace logging uses the existing C trace
//! backend through a narrow lifecycle bridge, while dictionary training uses
//! the temporary `DiB_trainFromFiles` bridge and Rust-owned algorithms.
use std::env;
use std::ffi::{CStr, CString, OsStr, OsString};
use std::fs;
use std::io::{self, IsTerminal, Write};
use std::os::raw::{c_char, c_int, c_uint};
use std::path::Path;
use std::ptr;
#[cfg(unix)]
use std::os::unix::ffi::{OsStrExt, OsStringExt};
#[cfg(unix)]
use std::os::unix::fs::FileTypeExt;
const DEFAULT_CLEVEL: i32 = 3;
#[cfg(feature = "compression")]
const DEFAULT_MAX_CLEVEL: i32 = 19;
const DEFAULT_BENCH_NB_SECONDS: u32 = 3;
const DEFAULT_MEM_LIMIT: u32 = 1 << 27;
const DEFAULT_LONG_WINDOW_LOG: u32 = 27;
const DEFAULT_DICT_NAME: &str = "dictionary";
const DEFAULT_MAX_DICT_SIZE: usize = 110 << 10;
const DEFAULT_DICT_SELECTIVITY: u32 = 9;
const DEFAULT_SHRINK_DICT_REGRESSION: u32 = 1;
const DEFAULT_FASTCOVER_ACCEL: u32 = 1;
const MAX_FAST_ACCELERATION: i32 = 128 << 10;
#[cfg(target_pointer_width = "32")]
const MAX_WINDOW_LOG: u32 = 30;
#[cfg(not(target_pointer_width = "32"))]
const MAX_WINDOW_LOG: u32 = 31;
#[cfg(target_pointer_width = "32")]
const MAX_CHAIN_LOG: u32 = 29;
#[cfg(not(target_pointer_width = "32"))]
const MAX_CHAIN_LOG: u32 = 30;
const MAX_HASH_LOG: u32 = 30;
const MAX_SEARCH_LOG: u32 = MAX_WINDOW_LOG - 1;
const MAX_MIN_MATCH: u32 = 3;
const MAX_TARGET_LENGTH: u32 = 1 << 17;
const MAX_STRATEGY: c_int = 9;
const MAX_OVERLAP_LOG: i32 = 9;
const MAX_LDM_HASH_LOG: i32 = 30;
const MAX_LDM_MIN_MATCH: i32 = 16;
const MAX_LDM_BUCKET_SIZE_LOG: i32 = 8;
const STDIN_MARK: &str = "/*stdin*\\";
const STDOUT_MARK: &str = "/*stdout*\\";
#[cfg(windows)]
const NULL_MARK: &str = "NUL";
#[cfg(not(windows))]
const NULL_MARK: &str = "/dev/null";
const ZSTD_SUFFIX: &[u8] = b".zst\0";
const GZ_SUFFIX: &[u8] = b".gz\0";
const XZ_SUFFIX: &[u8] = b".xz\0";
const LZMA_SUFFIX: &[u8] = b".lzma\0";
const LZ4_SUFFIX: &[u8] = b".lz4\0";
const FIO_ZSTD_COMPRESSION: c_int = 0;
const FIO_GZIP_COMPRESSION: c_int = 1;
const FIO_XZ_COMPRESSION: c_int = 2;
const FIO_LZMA_COMPRESSION: c_int = 3;
const FIO_LZ4_COMPRESSION: c_int = 4;
const FIO_PS_AUTO: c_int = 0;
const FIO_PS_NEVER: c_int = 1;
const FIO_PS_ALWAYS: c_int = 2;
const ZSTD_PS_AUTO: c_int = 0;
const ZSTD_PS_ENABLE: c_int = 1;
const ZSTD_PS_DISABLE: c_int = 2;
#[repr(C)]
struct FIO_prefs_t {
_private: [u8; 0],
}
#[repr(C)]
struct FIO_ctx_t {
_private: [u8; 0],
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq)]
pub(crate) struct ZSTD_compressionParameters {
windowLog: u32,
chainLog: u32,
hashLog: u32,
searchLog: u32,
minMatch: u32,
targetLength: u32,
strategy: c_int,
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq)]
struct ZDICT_params_t {
compressionLevel: c_int,
notificationLevel: c_uint,
dictID: c_uint,
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq)]
struct ZDICT_cover_params_t {
k: c_uint,
d: c_uint,
steps: c_uint,
nbThreads: c_uint,
splitPoint: f64,
shrinkDict: c_uint,
shrinkDictMaxRegression: c_uint,
zParams: ZDICT_params_t,
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq)]
struct ZDICT_fastCover_params_t {
k: c_uint,
d: c_uint,
f: c_uint,
steps: c_uint,
nbThreads: c_uint,
splitPoint: f64,
accel: c_uint,
shrinkDict: c_uint,
shrinkDictMaxRegression: c_uint,
zParams: ZDICT_params_t,
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq)]
#[cfg(all(feature = "compression", feature = "dict-builder"))]
struct ZDICT_legacy_params_t {
selectivityLevel: c_uint,
zParams: ZDICT_params_t,
}
/// Mirror of the `FileNamesTable` in `programs/util.h`; the tables returned
/// by the `UTIL_*FNT` helpers are only read and released here, never resized.
#[repr(C)]
struct FileNamesTable {
fileNames: *mut *const c_char,
buf: *mut c_char,
tableSize: usize,
tableCapacity: usize,
}
unsafe extern "C" {
fn ZSTD_versionString() -> *const c_char;
fn ZSTD_rust_cli_expected_version() -> *const c_char;
static mut g_utilDisplayLevel: c_int;
#[cfg(feature = "compression")]
fn ZSTD_minCLevel() -> c_int;
#[cfg(feature = "compression")]
fn ZSTD_maxCLevel() -> c_int;
#[cfg(feature = "compression")]
fn UTIL_countPhysicalCores() -> c_int;
#[cfg(feature = "compression")]
fn UTIL_countLogicalCores() -> c_int;
fn UTIL_getFileSize(input_file_name: *const c_char) -> u64;
fn UTIL_isConsole(file: *mut libc::FILE) -> c_int;
fn UTIL_isLink(input_file_name: *const c_char) -> c_int;
fn UTIL_fakeStderrIsConsole();
fn UTIL_traceFileStat();
static mut stdin: *mut libc::FILE;
static mut stdout: *mut libc::FILE;
static mut stderr: *mut libc::FILE;
fn UTIL_createFileNamesTable_fromFileName(
input_file_name: *const c_char,
) -> *mut FileNamesTable;
fn UTIL_createExpandedFNT(
file_names: *const *const c_char,
nb_file_names: usize,
follow_links: c_int,
) -> *mut FileNamesTable;
fn UTIL_freeFileNamesTable(table: *mut FileNamesTable);
fn FIO_createPreferences() -> *mut FIO_prefs_t;
fn FIO_freePreferences(prefs: *mut FIO_prefs_t);
fn FIO_createContext() -> *mut FIO_ctx_t;
fn FIO_freeContext(ctx: *mut FIO_ctx_t);
fn FIO_addAbortHandler();
fn FIO_setCompressionType(prefs: *mut FIO_prefs_t, compression_type: c_int);
fn FIO_overwriteMode(prefs: *mut FIO_prefs_t);
fn FIO_setAdaptiveMode(prefs: *mut FIO_prefs_t, adapt: c_int);
#[cfg(feature = "compression")]
fn FIO_setAdaptMin(prefs: *mut FIO_prefs_t, level: c_int);
#[cfg(feature = "compression")]
fn FIO_setAdaptMax(prefs: *mut FIO_prefs_t, level: c_int);
fn FIO_setUseRowMatchFinder(prefs: *mut FIO_prefs_t, mode: c_int);
fn FIO_setBlockSize(prefs: *mut FIO_prefs_t, block_size: c_int);
fn FIO_setChecksumFlag(prefs: *mut FIO_prefs_t, checksum: c_int);
fn FIO_setDictIDFlag(prefs: *mut FIO_prefs_t, dict_id: c_int);
fn FIO_setLdmBucketSizeLog(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setLdmFlag(prefs: *mut FIO_prefs_t, value: c_uint);
fn FIO_setLdmHashRateLog(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setLdmHashLog(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setLdmMinMatch(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setMemLimit(prefs: *mut FIO_prefs_t, limit: c_uint);
#[cfg(feature = "compression")]
fn FIO_setNbWorkers(prefs: *mut FIO_prefs_t, workers: c_int);
fn FIO_setOverlapLog(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setRemoveSrcFile(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setSparseWrite(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setRsyncable(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setStreamSrcSize(prefs: *mut FIO_prefs_t, value: usize);
fn FIO_setTargetCBlockSize(prefs: *mut FIO_prefs_t, value: usize);
fn FIO_setSrcSizeHint(prefs: *mut FIO_prefs_t, value: usize);
#[cfg(feature = "decompression")]
fn FIO_setTestMode(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setLiteralCompressionMode(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setProgressSetting(value: c_int);
fn FIO_setNotificationLevel(value: c_int);
fn FIO_setExcludeCompressedFile(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setAllowBlockDevices(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setContentSize(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setAsyncIOFlag(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setPassThroughFlag(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setPatchFromMode(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setMMapDict(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setNbFilesTotal(ctx: *mut FIO_ctx_t, value: c_int);
fn FIO_setHasStdinInput(ctx: *mut FIO_ctx_t, value: c_int);
fn FIO_setHasStdoutOutput(ctx: *mut FIO_ctx_t, value: c_int);
fn TRACE_enable(filename: *const c_char);
fn TRACE_finish();
#[cfg(feature = "compression")]
fn FIO_compressFilename(
ctx: *mut FIO_ctx_t,
prefs: *mut FIO_prefs_t,
output: *const c_char,
input: *const c_char,
dict: *const c_char,
level: c_int,
params: ZSTD_compressionParameters,
) -> c_int;
#[cfg(feature = "decompression")]
fn FIO_decompressFilename(
ctx: *mut FIO_ctx_t,
prefs: *mut FIO_prefs_t,
output: *const c_char,
input: *const c_char,
dict: *const c_char,
) -> c_int;
#[cfg(feature = "compression")]
fn FIO_compressMultipleFilenames(
ctx: *mut FIO_ctx_t,
prefs: *mut FIO_prefs_t,
inputs: *const *const c_char,
output_mirror_dir: *const c_char,
output_dir: *const c_char,
output: *const c_char,
suffix: *const c_char,
dict: *const c_char,
level: c_int,
params: ZSTD_compressionParameters,
) -> c_int;
#[cfg(feature = "decompression")]
fn FIO_decompressMultipleFilenames(
ctx: *mut FIO_ctx_t,
prefs: *mut FIO_prefs_t,
inputs: *const *const c_char,
output_mirror_dir: *const c_char,
output_dir: *const c_char,
output: *const c_char,
dict: *const c_char,
) -> c_int;
#[cfg(feature = "decompression")]
fn FIO_listMultipleFiles(
nb_files: c_uint,
file_names: *const *const c_char,
display_level: c_int,
) -> c_int;
/// Narrow bridge to `programs/dibio.c`. The file loader and dictionary
/// algorithms remain on the C/Rust library side of this boundary.
#[cfg(all(feature = "compression", feature = "dict-builder"))]
fn DiB_trainFromFiles(
dict_file_name: *const c_char,
max_dict_size: usize,
file_names: *const *const c_char,
nb_files: c_int,
chunk_size: usize,
legacy_params: *mut ZDICT_legacy_params_t,
cover_params: *mut ZDICT_cover_params_t,
fast_cover_params: *mut ZDICT_fastCover_params_t,
optimize: c_int,
mem_limit: c_uint,
) -> c_int;
#[cfg(feature = "compression")]
fn ZSTD_getCParams(
compression_level: c_int,
estimated_src_size: u64,
dict_size: usize,
) -> ZSTD_compressionParameters;
}
const ZSTD_STRATEGY_NAMES: [&str; 10] = [
"",
"ZSTD_fast",
"ZSTD_dfast",
"ZSTD_greedy",
"ZSTD_lazy",
"ZSTD_lazy2",
"ZSTD_btlazy2",
"ZSTD_btopt",
"ZSTD_btultra",
"ZSTD_btultra2",
];
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum Operation {
Compress,
Decompress,
Test,
Bench,
List,
Train,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum CompressionFormat {
Zstd,
Gzip,
Xz,
Lzma,
Lz4,
}
impl CompressionFormat {
fn parse(value: &str) -> Result<Self, String> {
match value {
"zstd" => Ok(Self::Zstd),
"gzip" => Ok(Self::Gzip),
"xz" => Ok(Self::Xz),
"lzma" => Ok(Self::Lzma),
"lz4" => Ok(Self::Lz4),
_ => Err(format!("unknown compression format {value:?}")),
}
}
const fn fio_type(self) -> c_int {
match self {
Self::Zstd => FIO_ZSTD_COMPRESSION,
Self::Gzip => FIO_GZIP_COMPRESSION,
Self::Xz => FIO_XZ_COMPRESSION,
Self::Lzma => FIO_LZMA_COMPRESSION,
Self::Lz4 => FIO_LZ4_COMPRESSION,
}
}
const fn suffix(self) -> &'static [u8] {
match self {
Self::Zstd => ZSTD_SUFFIX,
Self::Gzip => GZ_SUFFIX,
Self::Xz => XZ_SUFFIX,
Self::Lzma => LZMA_SUFFIX,
Self::Lz4 => LZ4_SUFFIX,
}
}
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
enum TrainingAlgorithm {
Cover,
#[default]
FastCover,
Legacy,
}
#[derive(Debug)]
enum Action {
Run(Box<Cli>),
Help { advanced: bool },
Version { quiet: bool },
}
#[derive(Debug)]
struct Cli {
operation: Operation,
bench_decode: bool,
compression_format: CompressionFormat,
inputs: Vec<CString>,
output: Option<CString>,
dictionary: Option<CString>,
patch_from: Option<CString>,
level: i32,
ultra: bool,
display_level: i32,
force: bool,
force_stdout: bool,
remove_source: bool,
checksum: Option<i32>,
sparse: Option<i32>,
pass_through: Option<i32>,
content_size: i32,
dict_id: Option<i32>,
async_io: Option<i32>,
mmap_dict: i32,
progress: i32,
workers: Option<i32>,
single_thread: bool,
auto_threads_logical: bool,
block_size: Option<usize>,
mem_limit: Option<u32>,
ldm: bool,
ldm_hash_log: Option<i32>,
ldm_min_match: Option<i32>,
ldm_bucket_size_log: Option<i32>,
ldm_hash_rate_log: Option<i32>,
overlap_log: Option<i32>,
adapt: bool,
adapt_min: Option<i32>,
adapt_max: Option<i32>,
rsyncable: bool,
stream_src_size: Option<usize>,
target_cblock_size: Option<usize>,
src_size_hint: Option<usize>,
literal_compression: Option<i32>,
row_match_finder: i32,
exclude_compressed: bool,
compression_params: ZSTD_compressionParameters,
show_default_cparams: bool,
bench_end_level: Option<i32>,
bench_nb_seconds: Option<u32>,
recursive: bool,
file_lists: Vec<CString>,
output_dir_flat: Option<CString>,
output_dir_mirror: Option<CString>,
training_algorithm: TrainingAlgorithm,
max_dict_size: usize,
dictionary_id: u32,
dictionary_selectivity: u32,
trace: Option<CString>,
fake_stderr_is_console: bool,
trace_file_stat: bool,
gzip_compat: bool,
cover_params: ZDICT_cover_params_t,
fast_cover_params: ZDICT_fastCover_params_t,
}
impl Cli {
fn new(program_name: &str) -> Self {
let mut cli = Self {
operation: Operation::Compress,
bench_decode: false,
compression_format: CompressionFormat::Zstd,
inputs: Vec::new(),
output: None,
dictionary: None,
patch_from: None,
level: default_level(),
ultra: false,
display_level: 2,
force: false,
force_stdout: false,
remove_source: false,
checksum: None,
sparse: None,
pass_through: None,
content_size: 1,
dict_id: None,
async_io: None,
mmap_dict: ZSTD_PS_AUTO,
progress: FIO_PS_AUTO,
workers: None,
single_thread: false,
auto_threads_logical: false,
block_size: None,
mem_limit: None,
ldm: false,
ldm_hash_log: None,
ldm_min_match: None,
ldm_bucket_size_log: None,
ldm_hash_rate_log: None,
overlap_log: None,
adapt: false,
adapt_min: None,
adapt_max: None,
rsyncable: false,
stream_src_size: None,
target_cblock_size: None,
src_size_hint: None,
literal_compression: None,
row_match_finder: ZSTD_PS_AUTO,
exclude_compressed: false,
compression_params: ZSTD_compressionParameters::default(),
show_default_cparams: false,
bench_end_level: None,
bench_nb_seconds: None,
recursive: false,
file_lists: Vec::new(),
output_dir_flat: None,
output_dir_mirror: None,
training_algorithm: TrainingAlgorithm::default(),
max_dict_size: DEFAULT_MAX_DICT_SIZE,
dictionary_id: 0,
dictionary_selectivity: DEFAULT_DICT_SELECTIVITY,
trace: None,
fake_stderr_is_console: false,
trace_file_stat: false,
gzip_compat: false,
cover_params: ZDICT_cover_params_t::default(),
fast_cover_params: default_fast_cover_params(),
};
match program_name {
"unzstd" => cli.operation = Operation::Decompress,
"zstdmt" => cli.workers = Some(0),
"zstdcat" | "zcat" => {
cli.operation = Operation::Decompress;
cli.output = Some(cstring(STDOUT_MARK).expect("static stdout marker"));
cli.force = true;
cli.force_stdout = true;
cli.pass_through = Some(1);
cli.display_level = 1;
}
"gzip" => {
cli.compression_format = CompressionFormat::Gzip;
cli.remove_source = true;
cli.gzip_compat = true;
}
"gunzip" => {
cli.compression_format = CompressionFormat::Gzip;
cli.operation = Operation::Decompress;
cli.remove_source = true;
}
"gzcat" => {
cli.compression_format = CompressionFormat::Gzip;
cli.operation = Operation::Decompress;
cli.output = Some(cstring(STDOUT_MARK).expect("static stdout marker"));
cli.force = true;
cli.force_stdout = true;
cli.pass_through = Some(1);
cli.display_level = 1;
}
"xz" => {
cli.compression_format = CompressionFormat::Xz;
cli.remove_source = true;
}
"unxz" => {
cli.compression_format = CompressionFormat::Xz;
cli.operation = Operation::Decompress;
cli.remove_source = true;
}
"lzma" => {
cli.compression_format = CompressionFormat::Lzma;
cli.remove_source = true;
}
"unlzma" => {
cli.compression_format = CompressionFormat::Lzma;
cli.operation = Operation::Decompress;
cli.remove_source = true;
}
"lz4" => cli.compression_format = CompressionFormat::Lz4,
"unlz4" => {
cli.compression_format = CompressionFormat::Lz4;
cli.operation = Operation::Decompress;
}
_ => {}
}
cli
}
}
fn cstring(value: &str) -> Result<CString, String> {
CString::new(value).map_err(|_| format!("argument contains an interior NUL: {value:?}"))
}
fn os_cstring(value: &OsStr) -> Result<CString, String> {
#[cfg(unix)]
{
CString::new(value.as_bytes()).map_err(|_| "file name contains an interior NUL".to_owned())
}
#[cfg(not(unix))]
{
cstring(&value.to_string_lossy())
}
}
fn default_level() -> i32 {
let Ok(value) = env::var("ZSTD_CLEVEL") else {
return DEFAULT_CLEVEL;
};
match value.parse::<i128>() {
Ok(level) if (i32::MIN as i128..=i32::MAX as i128).contains(&level) => level as i32,
Ok(_) => {
eprintln!(
"Ignore environment variable setting ZSTD_CLEVEL={value}: numeric value too large "
);
DEFAULT_CLEVEL
}
Err(_) if value_has_only_sign_and_digits(&value) => {
eprintln!(
"Ignore environment variable setting ZSTD_CLEVEL={value}: numeric value too large "
);
DEFAULT_CLEVEL
}
Err(_) => {
eprintln!(
"Ignore environment variable setting ZSTD_CLEVEL={value}: not a valid integer value "
);
DEFAULT_CLEVEL
}
}
}
fn value_has_only_sign_and_digits(value: &str) -> bool {
let digits = value
.strip_prefix('+')
.or_else(|| value.strip_prefix('-'))
.unwrap_or(value);
!digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit())
}
#[cfg(feature = "compression")]
unsafe fn default_worker_count() -> i32 {
if let Ok(value) = env::var("ZSTD_NBTHREADS") {
if let Ok(workers) = value.parse::<u32>() {
if let Ok(workers) = i32::try_from(workers) {
return workers;
}
}
}
let logical_cores = unsafe { UTIL_countLogicalCores() }.max(1);
(logical_cores / 4).clamp(1, 4)
}
#[cfg(feature = "compression")]
unsafe fn resolved_worker_count(
workers: Option<i32>,
single_thread: bool,
auto_threads_logical: bool,
) -> i32 {
match workers {
/* --single-thread pins zero workers; a bare zero (-T0 or the zstdmt
* program name) auto-detects the core count as in the C CLI. */
Some(0) if single_thread => 0,
Some(0) if auto_threads_logical => unsafe { UTIL_countLogicalCores() }.max(1),
Some(0) => unsafe { UTIL_countPhysicalCores() }.max(1),
Some(workers) => workers,
None => unsafe { default_worker_count() },
}
}
fn program_basename(value: &OsStr) -> String {
Path::new(value)
.file_name()
.unwrap_or(value)
.to_string_lossy()
.split('.')
.next()
.unwrap_or("zstd")
.to_owned()
}
#[cfg(feature = "compression")]
fn max_c_level_for_help() -> i32 {
DEFAULT_MAX_CLEVEL
}
#[cfg(not(feature = "compression"))]
const fn max_c_level_for_help() -> i32 {
19
}
#[cfg(feature = "compression")]
fn max_c_level_for_ultra() -> i32 {
unsafe { ZSTD_maxCLevel() }
}
#[cfg(not(feature = "compression"))]
const fn max_c_level_for_ultra() -> i32 {
19
}
fn write_basic_usage<W: Write>(out: &mut W, program_name: &str) {
let max_level = max_c_level_for_help();
let _ = writeln!(
out,
"Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided."
);
let _ = writeln!(
out,
"\nUsage: {program_name} [OPTIONS...] [INPUT... | -] [-o OUTPUT]"
);
let _ = writeln!(out, "\nOptions:");
let _ = writeln!(
out,
" -o OUTPUT Write output to a single file, OUTPUT."
);
let _ = writeln!(
out,
" -k, --keep Preserve INPUT file(s). [Default] "
);
let _ = writeln!(
out,
" --rm Remove INPUT file(s) after successful (de)compression."
);
if program_name == "gzip" {
let _ = writeln!(
out,
" -n, --no-name Do not store original filename when compressing."
);
}
let _ = writeln!(out);
let _ = writeln!(
out,
" -# Desired compression level, where `#` is a number between 1 and {max_level};"
);
let _ = writeln!(
out,
" lower numbers provide faster compression, higher numbers yield"
);
let _ = writeln!(
out,
" better compression ratios. [Default: {DEFAULT_CLEVEL}]"
);
let _ = writeln!(out);
let _ = writeln!(
out,
" -d, --decompress Perform decompression."
);
let _ = writeln!(
out,
" -D DICT Use DICT as the dictionary for compression or decompression."
);
let _ = writeln!(out);
let _ = writeln!(
out,
" -f, --force Disable input and output checks. Allows overwriting existing files,"
);
let _ = writeln!(
out,
" receiving input from the console, printing output to STDOUT, and"
);
let _ = writeln!(
out,
" operating on links, block devices, etc. Unrecognized formats will be"
);
let _ = writeln!(
out,
" passed-through through as-is."
);
let _ = writeln!(out);
let _ = writeln!(
out,
" -h Display short usage and exit."
);
let _ = writeln!(
out,
" -H, --help Display full help and exit."
);
let _ = writeln!(
out,
" -V, --version Display the program version and exit."
);
let _ = writeln!(out);
}
fn write_advanced_usage<W: Write>(out: &mut W, program_name: &str) {
let version = unsafe { CStr::from_ptr(ZSTD_versionString()) }.to_string_lossy();
let _ = writeln!(
out,
"*** Zstandard CLI ({}-bit) v{version}, by Yann Collet ***",
usize::BITS
);
let _ = writeln!(out);
write_basic_usage(out, program_name);
let _ = writeln!(out, "\nAdvanced options:");
let _ = writeln!(
out,
" -c, --stdout Write to STDOUT (even if it is a console) and keep the INPUT file(s)."
);
let _ = writeln!(
out,
" -v, --verbose Enable verbose output; pass multiple times to increase verbosity."
);
let _ = writeln!(
out,
" -q, --quiet Suppress warnings; pass twice to suppress errors."
);
let _ = writeln!(
out,
" --trace LOG Log tracing information to LOG."
);
let _ = writeln!(out);
let _ = writeln!(
out,
" --[no-]progress Forcibly show/hide the progress counter."
);
let _ = writeln!(out);
let _ = writeln!(
out,
" -r Operate recursively on directories."
);
let _ = writeln!(
out,
" --filelist LIST Read a list of files to operate on from LIST."
);
let _ = writeln!(
out,
" --output-dir-flat DIR Store processed files in DIR."
);
#[cfg(unix)]
let _ = writeln!(
out,
" --output-dir-mirror DIR Store processed files in DIR, respecting original directory structure."
);
let _ = writeln!(out);
let _ = writeln!(
out,
" -- Treat remaining arguments after `--` as files."
);
let _ = writeln!(out, "\nAdvanced compression options:");
let _ = writeln!(
out,
" --ultra Enable levels beyond 19, up to {}; requires more memory.",
max_c_level_for_ultra()
);
let _ = writeln!(
out,
" --fast[=#] Use very fast compression levels. [Default: 1]"
);
if program_name == "gzip" {
let _ = writeln!(
out,
" --best Compatibility alias for `-9`."
);
}
let _ = writeln!(
out,
" --adapt Dynamically adapt compression level to I/O conditions."
);
let _ = writeln!(
out,
" --long[=#] Enable long distance matching with window log #."
);
let _ = writeln!(out, " --patch-from=REF Use REF as the reference point for Zstandard's diff engine.");
let _ = writeln!(
out,
" -T# Spawn # compression threads."
);
let _ = writeln!(
out,
" --single-thread Share a single thread for I/O and compression."
);
let _ = writeln!(out, " --auto-threads={{physical|logical}}");
let _ = writeln!(out, " -B# Set job size to #.");
let _ = writeln!(
out,
" --rsyncable Compress using a rsync-friendly method."
);
let _ = writeln!(
out,
" --exclude-compressed Only compress files that are not already compressed."
);
let _ = writeln!(
out,
" --stream-size=# Specify size of streaming input from STDIN."
);
let _ = writeln!(
out,
" --size-hint=# Optimize compression parameters for streaming input."
);
let _ = writeln!(out, " --target-compressed-block-size=#");
let _ = writeln!(
out,
" Generate compressed blocks of approximately # size."
);
let _ = writeln!(
out,
" --no-dictID Don't write `dictID` into the header."
);
let _ = writeln!(
out,
" --[no-]compress-literals Force (un)compressed literals."
);
let _ = writeln!(
out,
" --[no-]row-match-finder Enable/disable the row-based matchfinder."
);
let _ = writeln!(
out,
" --format=zstd Compress files to the `.zst` format. [Default]"
);
let _ = writeln!(
out,
" --format=gzip Compress files to the `.gz` format."
);
let _ = writeln!(
out,
" --format=xz Compress files to the `.xz` format."
);
let _ = writeln!(
out,
" --format=lzma Compress files to the `.lzma` format."
);
let _ = writeln!(
out,
" --format=lz4 Compress files to the `.lz4` format."
);
let _ = writeln!(out, "\nAdvanced decompression options:");
let _ = writeln!(
out,
" -l Print information about compressed files."
);
let _ = writeln!(
out,
" --test Test compressed file integrity."
);
let _ = writeln!(
out,
" -M# Set the memory usage limit to # megabytes."
);
let _ = writeln!(
out,
" --[no-]sparse Enable/disable sparse mode."
);
let _ = writeln!(
out,
" --[no-]pass-through Pass through uncompressed files as-is."
);
let _ = writeln!(out, "\nDictionary builder:");
let _ = writeln!(
out,
" --train Create a dictionary from a training set of files."
);
let _ = writeln!(out, " --train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]]");
let _ = writeln!(
out,
" --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]]"
);
let _ = writeln!(
out,
" --train-legacy[=s=#] Use the legacy algorithm."
);
let _ = writeln!(
out,
" --maxdict=# Limit dictionary to specified size #."
);
let _ = writeln!(
out,
" --dictID=# Force dictionary ID to #."
);
let _ = writeln!(out, "\nBenchmark options:");
let _ = writeln!(
out,
" -b# Perform benchmarking with compression level #."
);
let _ = writeln!(
out,
" -e# Test all compression levels up to #."
);
let _ = writeln!(
out,
" -i# Set the minimum evaluation time to # seconds."
);
}
fn usage(advanced: bool) {
let mut out = io::stdout().lock();
if advanced {
write_advanced_usage(&mut out, "zstd");
} else {
write_basic_usage(&mut out, "zstd");
}
}
fn print_version(quiet: bool) {
let version = unsafe { CStr::from_ptr(ZSTD_versionString()) }
.to_string_lossy()
.into_owned();
if quiet {
println!("{version}");
} else {
println!(
"*** Zstandard CLI ({}-bit) v{version}, by Yann Collet ***",
usize::BITS
);
}
}
fn print_cli_header(display_level: i32) {
if display_level >= 3 {
let version = unsafe { CStr::from_ptr(ZSTD_versionString()) }.to_string_lossy();
eprintln!(
"*** Zstandard CLI ({}-bit) v{version}, by Yann Collet ***",
usize::BITS
);
}
}
fn check_lib_version() -> Result<(), String> {
let expected = unsafe { CStr::from_ptr(ZSTD_rust_cli_expected_version()) };
let actual = unsafe { CStr::from_ptr(ZSTD_versionString()) };
if expected == actual {
return Ok(());
}
Err(format!(
"incorrect library version (expecting: {}; actual: {})",
expected.to_string_lossy(),
actual.to_string_lossy()
))
}
fn parse_size(value: &str) -> Result<usize, String> {
let split = value
.find(|character: char| !character.is_ascii_digit())
.unwrap_or(value.len());
let (digits, suffix) = value.split_at(split);
if digits.is_empty() {
return Err(
"error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed "
.to_owned(),
);
}
let mut number = digits
.parse::<usize>()
.map_err(|_| "error: numeric value overflows size_t ".to_owned())?;
let shift = match suffix {
"" => 0,
"K" | "KB" | "KiB" => 10,
"M" | "MB" | "MiB" => 20,
_ => return Err(
"error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed "
.to_owned(),
),
};
number = number
.checked_shl(shift)
.ok_or_else(|| "error: numeric value overflows size_t ".to_owned())?;
Ok(number)
}
fn parse_u32(value: &str, name: &str) -> Result<u32, String> {
let _ = name;
let size = parse_size(value)?;
u32::try_from(size)
.map_err(|_| "error: numeric value overflows 32-bit unsigned int ".to_owned())
}
fn parse_i32(value: &str, name: &str) -> Result<i32, String> {
value
.parse::<i32>()
.map_err(|_| format!("invalid {name}: {value:?}"))
}
fn parse_compression_level(value: &str) -> Result<i32, String> {
/* The C CLI reads short-form levels as U32 before storing them in its
* signed level field. Preserve that conversion for values such as
* 4294967295, which represents -1 on the supported two's-complement
* targets and is a valid fast-compression level. */
Ok(parse_u32(value, "compression level")? as i32)
}
fn parse_worker_count(value: &str) -> Result<i32, String> {
let workers = parse_i32(value, "thread count")?;
if workers < 0 {
return Err(format!("thread count must not be negative: {value:?}"));
}
Ok(workers)
}
fn next_value(
attached: Option<&str>,
args: &[OsString],
index: &mut usize,
option: &str,
) -> Result<String, String> {
if let Some(value) = attached {
if !value.is_empty() {
return Ok(value.to_owned());
}
}
*index += 1;
let Some(value) = args.get(*index) else {
return Err(format!("missing argument for {option}"));
};
let rendered = value.to_string_lossy().into_owned();
if rendered.starts_with('-') {
return Err(format!(
"{option} cannot be separated from its argument by another option"
));
}
Ok(rendered)
}
fn next_os_value(args: &[OsString], index: &mut usize, option: &str) -> Result<OsString, String> {
*index += 1;
let Some(value) = args.get(*index) else {
return Err(format!("missing argument for {option}"));
};
if value.to_string_lossy().starts_with('-') {
return Err(format!(
"{option} cannot be separated from its argument by another option"
));
}
Ok(value.clone())
}
/// Mirrors the C CLI `NEXT_FIELD` macro: an attached `=value` is used as-is
/// (even when empty), otherwise the next argument is consumed. Unlike
/// `next_value`, an empty attached value never falls through to the next
/// argument, so `--output-dir-flat=` stays an empty (and rejected) name.
fn next_field(
attached: Option<&str>,
args: &[OsString],
index: &mut usize,
option: &str,
) -> Result<String, String> {
if let Some(value) = attached {
return Ok(value.to_owned());
}
*index += 1;
let Some(value) = args.get(*index) else {
return Err(format!("missing argument for {option}"));
};
let rendered = value.to_string_lossy().into_owned();
if rendered.starts_with('-') {
return Err(format!(
"{option} cannot be separated from its argument by another option"
));
}
Ok(rendered)
}
#[cfg(unix)]
fn short_attached_value(value: &OsStr, start: usize) -> Option<OsString> {
let bytes = &value.as_bytes()[start..];
if bytes.is_empty() {
return None;
}
let bytes = if bytes.first() == Some(&b'=') {
&bytes[1..]
} else {
bytes
};
Some(OsString::from_vec(bytes.to_vec()))
}
#[cfg(not(unix))]
fn short_attached_value(value: &OsStr, start: usize) -> Option<OsString> {
let rendered = value.to_string_lossy();
let attached = &rendered[start..];
if attached.is_empty() {
return None;
}
Some(OsString::from(
attached.strip_prefix('=').unwrap_or(attached),
))
}
fn parse_compression_parameters(value: &str, cli: &mut Cli) -> Result<(), String> {
for item in value.split(',') {
let Some((name, raw)) = item.split_once('=') else {
return Err(format!("invalid --zstd parameter {item:?}"));
};
let parsed = parse_u32(raw, name)?;
match name {
"windowLog" | "wlog" => cli.compression_params.windowLog = parsed,
"chainLog" | "clog" => cli.compression_params.chainLog = parsed,
"hashLog" | "hlog" => cli.compression_params.hashLog = parsed,
"searchLog" | "slog" => cli.compression_params.searchLog = parsed,
"minMatch" | "mml" => cli.compression_params.minMatch = parsed,
"targetLength" | "tlen" => cli.compression_params.targetLength = parsed,
"strategy" | "strat" => cli.compression_params.strategy = parsed as c_int,
"overlapLog" | "ovlog" => cli.overlap_log = Some(parsed as i32),
"ldmHashLog" | "lhlog" => cli.ldm_hash_log = Some(parsed as i32),
"ldmMinMatch" | "lmml" => cli.ldm_min_match = Some(parsed as i32),
"ldmBucketSizeLog" | "lblog" => cli.ldm_bucket_size_log = Some(parsed as i32),
"ldmHashRateLog" | "lhrlog" => cli.ldm_hash_rate_log = Some(parsed as i32),
_ => return Err(format!("unknown --zstd parameter {name:?}")),
}
}
Ok(())
}
fn parse_adapt(value: &str, cli: &mut Cli) -> Result<(), String> {
cli.adapt = true;
if value.is_empty() {
return Err("invalid --adapt parameter".to_owned());
}
for item in value.split(',') {
let Some((name, raw)) = item.split_once('=') else {
return Err(format!("invalid --adapt parameter {item:?}"));
};
match name {
"min" => cli.adapt_min = Some(parse_i32(raw, "adapt minimum")?),
"max" => cli.adapt_max = Some(parse_i32(raw, "adapt maximum")?),
_ => return Err(format!("unknown --adapt parameter {name:?}")),
}
}
if let (Some(minimum), Some(maximum)) = (cli.adapt_min, cli.adapt_max) {
if minimum > maximum {
return Err("--adapt minimum must not exceed its maximum".to_owned());
}
}
Ok(())
}
fn default_fast_cover_params() -> ZDICT_fastCover_params_t {
ZDICT_fastCover_params_t {
d: 8,
f: 20,
steps: 4,
splitPoint: 0.75,
accel: DEFAULT_FASTCOVER_ACCEL,
shrinkDictMaxRegression: DEFAULT_SHRINK_DICT_REGRESSION,
..ZDICT_fastCover_params_t::default()
}
}
fn parse_training_u32(value: &str, option: &str, field: &str) -> Result<u32, String> {
parse_u32(value, &format!("{option} {field}"))
}
fn parse_cover_parameters(value: &str) -> Result<ZDICT_cover_params_t, String> {
let mut params = ZDICT_cover_params_t::default();
for item in value.split(',') {
if item == "shrink" {
params.shrinkDict = 1;
params.shrinkDictMaxRegression = DEFAULT_SHRINK_DICT_REGRESSION;
continue;
}
if let Some(raw) = item.strip_prefix("shrink=") {
params.shrinkDict = 1;
params.shrinkDictMaxRegression =
parse_training_u32(raw, "--train-cover", "shrink regression")?;
continue;
}
let Some((field, raw)) = item.split_once('=') else {
return Err(format!("invalid --train-cover parameter {item:?}"));
};
let parsed = parse_training_u32(raw, "--train-cover", field)?;
match field {
"k" => params.k = parsed,
"d" => params.d = parsed,
"steps" => params.steps = parsed,
"split" => params.splitPoint = f64::from(parsed) / 100.0,
_ => return Err(format!("unknown --train-cover parameter {field:?}")),
}
}
Ok(params)
}
fn parse_fast_cover_parameters(value: &str) -> Result<ZDICT_fastCover_params_t, String> {
let mut params = ZDICT_fastCover_params_t::default();
for item in value.split(',') {
if item == "shrink" {
params.shrinkDict = 1;
params.shrinkDictMaxRegression = DEFAULT_SHRINK_DICT_REGRESSION;
continue;
}
if let Some(raw) = item.strip_prefix("shrink=") {
params.shrinkDict = 1;
params.shrinkDictMaxRegression =
parse_training_u32(raw, "--train-fastcover", "shrink regression")?;
continue;
}
let Some((field, raw)) = item.split_once('=') else {
return Err(format!("invalid --train-fastcover parameter {item:?}"));
};
let parsed = parse_training_u32(raw, "--train-fastcover", field)?;
match field {
"k" => params.k = parsed,
"d" => params.d = parsed,
"f" => params.f = parsed,
"steps" => params.steps = parsed,
"split" => params.splitPoint = f64::from(parsed) / 100.0,
"accel" => params.accel = parsed,
_ => return Err(format!("unknown --train-fastcover parameter {field:?}")),
}
}
Ok(params)
}
fn parse_legacy_parameters(value: &str) -> Result<u32, String> {
let Some(raw) = value
.strip_prefix("s=")
.or_else(|| value.strip_prefix("selectivity="))
else {
return Err(format!("invalid --train-legacy parameter {value:?}"));
};
parse_training_u32(raw, "--train-legacy", "selectivity")
}
fn select_training(cli: &mut Cli, algorithm: TrainingAlgorithm) -> Result<(), String> {
cli.operation = Operation::Train;
cli.training_algorithm = algorithm;
if cli.output.is_none() {
cli.output = Some(cstring(DEFAULT_DICT_NAME)?);
}
Ok(())
}
fn unsupported(option: &str) -> Result<(), String> {
Err(format!(
"{option} is not yet implemented by the Rust CLI frontend"
))
}
fn set_max_compression(cli: &mut Cli) -> Result<(), String> {
if cfg!(target_pointer_width = "32") {
return Err("--max is incompatible with 32-bit mode".to_owned());
}
cli.ultra = true;
cli.ldm = true;
cli.compression_params = ZSTD_compressionParameters {
windowLog: MAX_WINDOW_LOG,
chainLog: MAX_CHAIN_LOG,
hashLog: MAX_HASH_LOG,
searchLog: MAX_SEARCH_LOG,
minMatch: MAX_MIN_MATCH,
targetLength: MAX_TARGET_LENGTH,
strategy: MAX_STRATEGY,
};
cli.overlap_log = Some(MAX_OVERLAP_LOG);
cli.ldm_hash_log = Some(MAX_LDM_HASH_LOG);
cli.ldm_hash_rate_log = Some(0);
cli.ldm_min_match = Some(MAX_LDM_MIN_MATCH);
cli.ldm_bucket_size_log = Some(MAX_LDM_BUCKET_SIZE_LOG);
Ok(())
}
fn parse_long_option(
option: &str,
args: &[OsString],
index: &mut usize,
cli: &mut Cli,
) -> Result<Option<Action>, String> {
let (name, attached) = option
.split_once('=')
.map_or((option, None), |(name, value)| (name, Some(value)));
if attached.is_some()
&& matches!(
name,
"--compress"
| "--decompress"
| "--uncompress"
| "--test"
| "--force"
| "--keep"
| "--rm"
| "--stdout"
| "--version"
| "--help"
| "--verbose"
| "--quiet"
| "--check"
| "--no-check"
| "--sparse"
| "--no-sparse"
| "--pass-through"
| "--no-pass-through"
| "--content-size"
| "--no-content-size"
| "--no-dictID"
| "--asyncio"
| "--no-asyncio"
| "--mmap-dict"
| "--no-mmap-dict"
| "--progress"
| "--no-progress"
| "--ultra"
| "--no-row-match-finder"
| "--row-match-finder"
| "--rsyncable"
| "--single-thread"
| "--compress-literals"
| "--no-compress-literals"
| "--exclude-compressed"
| "--no-name"
| "--best"
| "--list"
| "--show-default-cparams"
| "--train"
| "--max"
| "--fake-stderr-is-console"
| "--trace-file-stat"
)
{
return Err(format!("{name} does not take an argument"));
}
match name {
"--" => Ok(None),
"--compress" => {
cli.operation = Operation::Compress;
Ok(None)
}
"--decompress" | "--uncompress" => {
cli.bench_decode = true;
if cli.operation != Operation::Bench {
cli.operation = Operation::Decompress;
}
Ok(None)
}
"--test" => {
cli.operation = Operation::Test;
Ok(None)
}
"--force" => {
cli.force = true;
Ok(None)
}
"--keep" => {
cli.remove_source = false;
Ok(None)
}
"--no-name" => Ok(None),
"--best" if cli.gzip_compat => {
cli.level = 9;
Ok(None)
}
"--rm" => {
cli.remove_source = true;
Ok(None)
}
"--stdout" => {
cli.output = Some(cstring(STDOUT_MARK)?);
cli.force_stdout = true;
Ok(None)
}
"--version" => Ok(Some(Action::Version {
quiet: cli.display_level < 2,
})),
"--help" => Ok(Some(Action::Help { advanced: true })),
"--verbose" => {
cli.display_level += 1;
Ok(None)
}
"--quiet" => {
cli.display_level -= 1;
Ok(None)
}
"--check" => {
cli.checksum = Some(2);
Ok(None)
}
"--no-check" => {
cli.checksum = Some(0);
Ok(None)
}
"--sparse" => {
cli.sparse = Some(2);
Ok(None)
}
"--no-sparse" => {
cli.sparse = Some(0);
Ok(None)
}
"--pass-through" => {
cli.pass_through = Some(1);
Ok(None)
}
"--no-pass-through" => {
cli.pass_through = Some(0);
Ok(None)
}
"--content-size" => {
cli.content_size = 1;
Ok(None)
}
"--no-content-size" => {
cli.content_size = 0;
Ok(None)
}
"--no-dictID" => {
cli.dict_id = Some(0);
Ok(None)
}
"--asyncio" => {
cli.async_io = Some(1);
Ok(None)
}
"--no-asyncio" => {
cli.async_io = Some(0);
Ok(None)
}
"--mmap-dict" => {
cli.mmap_dict = ZSTD_PS_ENABLE;
Ok(None)
}
"--no-mmap-dict" => {
cli.mmap_dict = ZSTD_PS_DISABLE;
Ok(None)
}
"--progress" => {
cli.progress = FIO_PS_ALWAYS;
Ok(None)
}
"--no-progress" => {
cli.progress = FIO_PS_NEVER;
Ok(None)
}
"--ultra" => {
cli.ultra = true;
Ok(None)
}
"--fast" => {
let mut level = match attached {
Some(value) => parse_u32(value, "fast level")?,
None => 1,
};
if level == 0 {
return Err("fast level must be positive".to_owned());
}
level = level.min(MAX_FAST_ACCELERATION as u32);
cli.level = -(level as i32);
Ok(None)
}
"--long" => {
cli.ldm = true;
cli.ultra = true;
if let Some(value) = attached {
cli.compression_params.windowLog = parse_u32(value, "long window log")?;
} else if cli.compression_params.windowLog == 0 {
cli.compression_params.windowLog = DEFAULT_LONG_WINDOW_LOG;
}
Ok(None)
}
"--adapt" => {
if let Some(value) = attached {
parse_adapt(value, cli)?;
} else {
cli.adapt = true;
}
Ok(None)
}
"--no-row-match-finder" => {
cli.row_match_finder = ZSTD_PS_DISABLE;
Ok(None)
}
"--row-match-finder" => {
cli.row_match_finder = ZSTD_PS_ENABLE;
Ok(None)
}
"--rsyncable" => {
cli.rsyncable = true;
Ok(None)
}
"--compress-literals" => {
cli.literal_compression = Some(ZSTD_PS_ENABLE);
Ok(None)
}
"--no-compress-literals" => {
cli.literal_compression = Some(ZSTD_PS_DISABLE);
Ok(None)
}
"--exclude-compressed" => {
cli.exclude_compressed = true;
Ok(None)
}
"--threads" => {
let value = next_value(attached, args, index, "--threads")?;
cli.workers = Some(parse_worker_count(&value)?);
Ok(None)
}
"--auto-threads" => {
let value = next_field(attached, args, index, name)?;
cli.auto_threads_logical = match value.as_str() {
"physical" => false,
"logical" => true,
_ => return Err(format!("unknown auto-thread mode {value:?}")),
};
Ok(None)
}
"--single-thread" => {
/* As in the C CLI: zero workers plus a latch that suppresses the
* automatic core-count resolution, so fileio runs its
* single-thread streaming mode (slightly different from -T1). */
cli.workers = Some(0);
cli.single_thread = true;
Ok(None)
}
"--memlimit" | "--memory" | "--memlimit-decompress" => {
let value = next_value(attached, args, index, name)?;
cli.mem_limit = Some(parse_u32(&value, "memory limit")?);
Ok(None)
}
"--block-size" => {
let value = next_value(attached, args, index, name)?;
cli.block_size = Some(parse_size(&value)?);
Ok(None)
}
"--stream-size" => {
let value = next_value(attached, args, index, name)?;
cli.stream_src_size = Some(parse_size(&value)?);
Ok(None)
}
"--target-compressed-block-size" => {
let value = next_value(attached, args, index, name)?;
cli.target_cblock_size = Some(parse_size(&value)?);
Ok(None)
}
"--size-hint" => {
let value = next_value(attached, args, index, name)?;
cli.src_size_hint = Some(parse_size(&value)?);
Ok(None)
}
"--zstd" => {
let value = next_value(attached, args, index, "--zstd")?;
parse_compression_parameters(&value, cli)?;
Ok(None)
}
"--list" => {
cli.operation = Operation::List;
Ok(None)
}
"--show-default-cparams" => {
cli.show_default_cparams = true;
Ok(None)
}
"--train" => {
select_training(cli, cli.training_algorithm)?;
Ok(None)
}
"--train-cover" => {
let params = attached.map_or_else(
|| Ok(ZDICT_cover_params_t::default()),
parse_cover_parameters,
)?;
select_training(cli, TrainingAlgorithm::Cover)?;
cli.cover_params = params;
Ok(None)
}
"--train-fastcover" => {
let params = attached.map_or_else(
|| Ok(ZDICT_fastCover_params_t::default()),
parse_fast_cover_parameters,
)?;
select_training(cli, TrainingAlgorithm::FastCover)?;
cli.fast_cover_params = params;
Ok(None)
}
"--train-legacy" => {
let selectivity =
attached.map_or_else(|| Ok(cli.dictionary_selectivity), parse_legacy_parameters)?;
select_training(cli, TrainingAlgorithm::Legacy)?;
cli.dictionary_selectivity = selectivity;
Ok(None)
}
"--maxdict" => {
let value = next_field(attached, args, index, name)?;
cli.max_dict_size = parse_u32(&value, "maximum dictionary size")? as usize;
Ok(None)
}
"--dictID" => {
let value = next_field(attached, args, index, name)?;
cli.dictionary_id = parse_u32(&value, "dictionary ID")?;
Ok(None)
}
"--format" => {
let value = next_field(attached, args, index, name)?;
cli.compression_format = CompressionFormat::parse(&value)?;
Ok(None)
}
"--trace" => {
let value = next_field(attached, args, index, name)?;
cli.trace = Some(cstring(&value)?);
Ok(None)
}
"--filelist" => {
let value = next_field(attached, args, index, name)?;
cli.file_lists.push(cstring(&value)?);
Ok(None)
}
"--output-dir-flat" => {
let value = next_field(attached, args, index, name)?;
if value.is_empty() {
return Err(
"output dir cannot be empty string (did you mean to pass '.' instead?)"
.to_owned(),
);
}
cli.output_dir_flat = Some(cstring(&value)?);
Ok(None)
}
#[cfg(unix)]
"--output-dir-mirror" => {
/* Parsed only where the C CLI defines UTIL_HAS_MIRRORFILELIST
* (POSIX); elsewhere the option falls through as unknown. */
let value = next_field(attached, args, index, name)?;
if value.is_empty() {
return Err(
"output dir cannot be empty string (did you mean to pass '.' instead?)"
.to_owned(),
);
}
cli.output_dir_mirror = Some(cstring(&value)?);
Ok(None)
}
"--max" => {
set_max_compression(cli)?;
Ok(None)
}
"--fake-stderr-is-console" => {
cli.fake_stderr_is_console = true;
Ok(None)
}
"--trace-file-stat" => {
cli.trace_file_stat = true;
Ok(None)
}
"--patch-from" => {
if cli.dictionary.is_some() {
return Err("can't use -D and --patch-from=# at the same time".to_owned());
}
let value = next_field(attached, args, index, name)?;
cli.patch_from = Some(cstring(&value)?);
cli.ultra = true;
Ok(None)
}
"--priority" | "--fake-stdin-is-console" | "--fake-stdout-is-console" => {
unsupported(name)?;
Ok(None)
}
_ => Err(format!("unknown option {option}")),
}
}
fn parse_short_options(
value: &str,
raw_value: &OsStr,
args: &[OsString],
index: &mut usize,
cli: &mut Cli,
) -> Result<Option<Action>, String> {
let mut offset = 1usize;
let bytes = value.as_bytes();
while offset < bytes.len() {
let option = bytes[offset] as char;
match option {
'0'..='9' => {
let mut digits_end = offset;
while digits_end < bytes.len() && bytes[digits_end].is_ascii_digit() {
digits_end += 1;
}
cli.level = parse_compression_level(&value[offset..digits_end])?;
offset = digits_end;
continue;
}
'd' => {
cli.bench_decode = true;
if cli.operation != Operation::Bench {
cli.operation = Operation::Decompress;
}
}
'z' => cli.operation = Operation::Compress,
't' => cli.operation = Operation::Test,
'b' => cli.operation = Operation::Bench,
'l' => cli.operation = Operation::List,
#[cfg(any(unix, windows))]
'r' => cli.recursive = true,
'e' | 'i' => {
// Benchmark range end (-e#) and duration (-i#): like the C
// parser, digits attach directly and default to 0.
let mut digits_end = offset + 1;
while digits_end < bytes.len() && bytes[digits_end].is_ascii_digit() {
digits_end += 1;
}
let digits = &value[offset + 1..digits_end];
if option == 'e' {
cli.bench_end_level = Some(if digits.is_empty() {
0
} else {
parse_i32(digits, "benchmark end level")?
});
} else {
cli.bench_nb_seconds = Some(if digits.is_empty() {
0
} else {
digits
.parse::<u32>()
.map_err(|_| format!("invalid benchmark duration: {digits:?}"))?
});
}
offset = digits_end;
continue;
}
'c' => {
cli.output = Some(cstring(STDOUT_MARK)?);
cli.force_stdout = true;
}
'f' => cli.force = true,
'k' => cli.remove_source = false,
'n' => {}
'q' => cli.display_level -= 1,
'v' => cli.display_level += 1,
'C' => cli.checksum = Some(2),
'h' => return Ok(Some(Action::Help { advanced: false })),
'H' => return Ok(Some(Action::Help { advanced: true })),
'V' => {
return Ok(Some(Action::Version {
quiet: cli.display_level < 2,
}))
}
'o' | 'D' | 'T' | 'M' | 'B' => {
let attached = short_attached_value(raw_value, offset + 1);
let argument = attached
.map(Ok)
.unwrap_or_else(|| next_os_value(args, index, &format!("-{option}")))?;
match option {
'o' => cli.output = Some(os_cstring(&argument)?),
'D' => {
if cli.patch_from.is_some() {
return Err(
"can't use -D and --patch-from=# at the same time".to_owned()
);
}
cli.dictionary = Some(os_cstring(&argument)?);
}
'T' => cli.workers = Some(parse_worker_count(&argument.to_string_lossy())?),
'M' => {
cli.mem_limit =
Some(parse_u32(&argument.to_string_lossy(), "memory limit")?)
}
'B' => cli.block_size = Some(parse_size(&argument.to_string_lossy())?),
_ => unreachable!(),
}
break;
}
's' => {
let mut digits_end = offset + 1;
while digits_end < bytes.len() && bytes[digits_end].is_ascii_digit() {
digits_end += 1;
}
if digits_end == offset + 1 {
return Err("missing argument for -s".to_owned());
}
cli.dictionary_selectivity =
parse_u32(&value[offset + 1..digits_end], "dictionary selectivity")?;
offset = digits_end;
continue;
}
'p' | 'P' | 'S' => {
unsupported(&format!("-{option}"))?;
}
_ => return Err(format!("unknown option -{option}")),
}
offset += 1;
}
Ok(None)
}
#[derive(Debug)]
struct ParseError {
message: String,
#[allow(dead_code)]
details: String,
show_usage: bool,
}
impl ParseError {
fn direct(message: String) -> Self {
Self::direct_with_details(message.clone(), message)
}
fn direct_with_details(message: String, details: String) -> Self {
Self {
message,
details,
show_usage: false,
}
}
fn incorrect_parameter(parameter: &str, details: String, show_usage: bool) -> Self {
Self {
message: format!("Incorrect parameter: {parameter}"),
details,
show_usage,
}
}
#[cfg(test)]
fn contains(&self, needle: &str) -> bool {
self.message.contains(needle) || self.details.contains(needle)
}
}
fn classify_parse_error(error: String, parameter: &str, display_level: i32) -> ParseError {
if error.starts_with("error:") {
return ParseError::direct(error);
}
if error.starts_with("output dir cannot be empty string") {
return ParseError::direct(format!("error: {error}"));
}
if error.starts_with("missing argument for ") {
return ParseError::direct_with_details(
"error: missing command argument ".to_owned(),
error,
);
}
if error.contains("cannot be separated from its argument") {
return ParseError::direct_with_details(
"error: command cannot be separated from its argument by another command ".to_owned(),
error,
);
}
if let Some(option) = error.strip_prefix("unknown option ") {
return ParseError::incorrect_parameter(option, error.clone(), display_level >= 2);
}
ParseError::incorrect_parameter(parameter, error, display_level >= 2)
}
fn parse_args(args: Vec<OsString>) -> Result<Action, ParseError> {
let program_name = args
.first()
.map_or_else(|| "zstd".to_owned(), |value| program_basename(value));
let mut cli = Cli::new(&program_name);
let mut end_of_options = false;
let mut index = 1usize;
while index < args.len() {
let rendered = args[index].to_string_lossy().into_owned();
if end_of_options {
cli.inputs
.push(os_cstring(&args[index]).map_err(ParseError::direct)?);
} else if rendered == "--" {
end_of_options = true;
} else if rendered == "-" {
cli.inputs
.push(cstring(STDIN_MARK).map_err(ParseError::direct)?);
} else if rendered.starts_with("--") {
let action = parse_long_option(&rendered, &args, &mut index, &mut cli)
.map_err(|error| classify_parse_error(error, &rendered, cli.display_level))?;
if let Some(action) = action {
return Ok(action);
}
} else if rendered.starts_with('-') {
let action = parse_short_options(&rendered, &args[index], &args, &mut index, &mut cli)
.map_err(|error| classify_parse_error(error, &rendered, cli.display_level))?;
if let Some(action) = action {
return Ok(action);
}
} else {
cli.inputs
.push(os_cstring(&args[index]).map_err(ParseError::direct)?);
}
index += 1;
}
Ok(Action::Run(Box::new(cli)))
}
unsafe fn apply_preferences(cli: &Cli, prefs: *mut FIO_prefs_t, ctx: *mut FIO_ctx_t) {
let display_level = if is_stdout(cli.output.as_ref()) && cli.display_level == 2 {
1
} else {
cli.display_level
};
unsafe {
g_utilDisplayLevel = display_level;
FIO_setCompressionType(prefs, cli.compression_format.fio_type());
FIO_setNotificationLevel(display_level);
FIO_setProgressSetting(
if !cli.fake_stderr_is_console
&& !io::stderr().is_terminal()
&& cli.progress != FIO_PS_ALWAYS
{
FIO_PS_NEVER
} else {
cli.progress
},
);
FIO_setRemoveSrcFile(
prefs,
i32::from(
cli.remove_source
&& cli.operation != Operation::Test
&& !is_stdout(cli.output.as_ref()),
),
);
FIO_setAllowBlockDevices(prefs, i32::from(cli.force));
FIO_setMMapDict(prefs, cli.mmap_dict);
FIO_setUseRowMatchFinder(prefs, cli.row_match_finder);
FIO_setMemLimit(
prefs,
cli.mem_limit
.filter(|limit| *limit != 0)
.unwrap_or_else(|| {
if cli.compression_params.windowLog == 0 {
DEFAULT_MEM_LIMIT
} else {
1_u32 << (cli.compression_params.windowLog & 31)
}
}),
);
#[cfg(feature = "compression")]
FIO_setNbWorkers(
prefs,
resolved_worker_count(cli.workers, cli.single_thread, cli.auto_threads_logical),
);
FIO_setLdmFlag(prefs, u32::from(cli.ldm));
FIO_setAdaptiveMode(prefs, i32::from(cli.adapt));
FIO_setRsyncable(prefs, i32::from(cli.rsyncable));
FIO_setExcludeCompressedFile(prefs, i32::from(cli.exclude_compressed));
if cli.force {
FIO_overwriteMode(prefs);
}
if let Some(value) = cli.checksum {
FIO_setChecksumFlag(prefs, value);
}
if cli.operation == Operation::Compress {
FIO_setSparseWrite(prefs, 0);
} else if let Some(value) = cli.sparse {
FIO_setSparseWrite(prefs, value);
}
if let Some(value) = cli.pass_through {
FIO_setPassThroughFlag(prefs, value);
}
FIO_setPatchFromMode(prefs, i32::from(cli.patch_from.is_some()));
FIO_setContentSize(prefs, cli.content_size);
if let Some(value) = cli.dict_id {
FIO_setDictIDFlag(prefs, value);
}
if let Some(value) = cli.async_io {
FIO_setAsyncIOFlag(prefs, value);
}
if let Some(value) = cli.block_size {
FIO_setBlockSize(prefs, value as c_int);
}
if let Some(value) = cli.ldm_hash_log {
FIO_setLdmHashLog(prefs, value);
}
if let Some(value) = cli.ldm_min_match {
FIO_setLdmMinMatch(prefs, value);
}
if let Some(value) = cli.ldm_bucket_size_log {
FIO_setLdmBucketSizeLog(prefs, value);
}
if let Some(value) = cli.ldm_hash_rate_log {
FIO_setLdmHashRateLog(prefs, value);
}
if let Some(value) = cli.overlap_log {
FIO_setOverlapLog(prefs, value);
}
#[cfg(feature = "compression")]
{
FIO_setAdaptMin(prefs, cli.adapt_min.unwrap_or_else(|| ZSTD_minCLevel()));
FIO_setAdaptMax(prefs, cli.adapt_max.unwrap_or_else(|| ZSTD_maxCLevel()));
}
if let Some(value) = cli.stream_src_size {
FIO_setStreamSrcSize(prefs, value);
}
if let Some(value) = cli.target_cblock_size {
FIO_setTargetCBlockSize(prefs, value);
}
if let Some(value) = cli.src_size_hint {
FIO_setSrcSizeHint(prefs, value);
}
if let Some(value) = cli.literal_compression {
FIO_setLiteralCompressionMode(prefs, value);
}
FIO_setNbFilesTotal(ctx, cli.inputs.len() as c_int);
FIO_setHasStdinInput(ctx, i32::from(cli.inputs.iter().any(is_stdin)));
FIO_setHasStdoutOutput(ctx, i32::from(is_stdout(cli.output.as_ref())));
}
}
fn is_stdout(value: Option<&CString>) -> bool {
value.is_some_and(|value| value.as_bytes() == STDOUT_MARK.as_bytes())
}
fn is_stdin(value: &CString) -> bool {
value.as_bytes() == STDIN_MARK.as_bytes()
}
fn fileio_dictionary(cli: &Cli) -> Option<&CString> {
cli.patch_from.as_ref().or(cli.dictionary.as_ref())
}
#[cfg(unix)]
fn is_non_fifo_symlink(input: &CString) -> bool {
if unsafe { UTIL_isLink(input.as_ptr()) } == 0 {
return false;
}
let path = Path::new(OsStr::from_bytes(input.as_bytes()));
let Ok(metadata) = fs::symlink_metadata(path) else {
return false;
};
metadata.file_type().is_symlink()
&& !fs::metadata(path).is_ok_and(|target| target.file_type().is_fifo())
}
#[cfg(not(unix))]
fn is_non_fifo_symlink(input: &CString) -> bool {
if unsafe { UTIL_isLink(input.as_ptr()) } == 0 {
return false;
}
let path = Path::new(&input.to_string_lossy().into_owned());
fs::symlink_metadata(path).is_ok_and(|metadata| metadata.file_type().is_symlink())
}
fn filter_symlink_inputs(cli: &mut Cli) {
if cli.force {
return;
}
cli.inputs.retain(|input| {
if is_stdin(input) || !is_non_fifo_symlink(input) {
return true;
}
if cli.display_level >= 2 {
eprintln!(
"zstd: Warning : {} is a symbolic link, ignoring",
input.to_string_lossy()
);
}
false
});
}
fn check_terminal_safety(cli: &Cli) -> Result<(), String> {
let has_stdin = cli.inputs.iter().any(is_stdin);
let stdin_is_console = if has_stdin {
unsafe { UTIL_isConsole(stdin) != 0 }
} else {
false
};
let output_is_stdout = is_stdout(cli.output.as_ref());
let stdout_is_console =
if output_is_stdout || (cli.operation == Operation::Decompress && !has_stdin) {
unsafe { UTIL_isConsole(stdout) != 0 }
} else {
false
};
let stderr_is_console = unsafe { UTIL_isConsole(stderr) != 0 };
if has_stdin && !cli.force && stdin_is_console {
return Err("stdin is a console, aborting".to_owned());
}
if has_stdin
&& output_is_stdout
&& !cli.force
&& !cli.force_stdout
&& cli.operation != Operation::Decompress
&& stdout_is_console
{
return Err("stdout is a console, aborting".to_owned());
}
let _ = stderr_is_console;
Ok(())
}
#[cfg(feature = "compression")]
unsafe fn print_default_cparams(cli: &Cli) {
let dict_size = fileio_dictionary(cli).map_or(0, |dictionary| unsafe {
UTIL_getFileSize(dictionary.as_ptr()) as usize
});
for input in &cli.inputs {
let file_size = unsafe { UTIL_getFileSize(input.as_ptr()) };
let cparams = unsafe { ZSTD_getCParams(cli.level, file_size, dict_size) };
let strategy = ZSTD_STRATEGY_NAMES
.get(cparams.strategy as usize)
.copied()
.unwrap_or("");
if file_size != u64::MAX {
eprintln!("{} ({} bytes)", input.to_string_lossy(), file_size);
} else {
eprintln!("{} (src size unknown)", input.to_string_lossy());
}
eprintln!(" - windowLog : {}", cparams.windowLog);
eprintln!(" - chainLog : {}", cparams.chainLog);
eprintln!(" - hashLog : {}", cparams.hashLog);
eprintln!(" - searchLog : {}", cparams.searchLog);
eprintln!(" - minMatch : {}", cparams.minMatch);
eprintln!(" - targetLength : {}", cparams.targetLength);
eprintln!(" - strategy : {strategy} ({})", cparams.strategy);
}
}
#[cfg(feature = "compression")]
unsafe fn print_actual_cparams(cli: &Cli) {
let dict_size = fileio_dictionary(cli).map_or(0, |dictionary| unsafe {
UTIL_getFileSize(dictionary.as_ptr()) as usize
});
for input in &cli.inputs {
let file_size = unsafe { UTIL_getFileSize(input.as_ptr()) };
let mut actual = unsafe { ZSTD_getCParams(cli.level, file_size, dict_size) };
let requested = cli.compression_params;
if requested.windowLog != 0 {
actual.windowLog = requested.windowLog;
}
if requested.chainLog != 0 {
actual.chainLog = requested.chainLog;
}
if requested.hashLog != 0 {
actual.hashLog = requested.hashLog;
}
if requested.searchLog != 0 {
actual.searchLog = requested.searchLog;
}
if requested.minMatch != 0 {
actual.minMatch = requested.minMatch;
}
if requested.targetLength != 0 {
actual.targetLength = requested.targetLength;
}
if requested.strategy != 0 {
actual.strategy = requested.strategy;
}
eprintln!(
"--zstd=wlog={},clog={},hlog={},slog={},mml={},tlen={},strat={}",
actual.windowLog,
actual.chainLog,
actual.hashLog,
actual.searchLog,
actual.minMatch,
actual.targetLength,
actual.strategy
);
}
}
#[cfg(feature = "compression")]
unsafe fn run_compress(
cli: &Cli,
ctx: *mut FIO_ctx_t,
prefs: *mut FIO_prefs_t,
inputs: &[*const c_char],
output: *const c_char,
dictionary: *const c_char,
) -> c_int {
if inputs.len() == 1 && !output.is_null() {
unsafe {
FIO_compressFilename(
ctx,
prefs,
output,
inputs[0],
dictionary,
cli.level,
cli.compression_params,
)
}
} else {
unsafe {
FIO_compressMultipleFilenames(
ctx,
prefs,
inputs.as_ptr(),
output_dir_mirror(cli),
output_dir_flat(cli),
output,
cli.compression_format.suffix().as_ptr().cast(),
dictionary,
cli.level,
cli.compression_params,
)
}
}
}
fn output_dir_flat(cli: &Cli) -> *const c_char {
cli.output_dir_flat
.as_ref()
.map_or(ptr::null(), |value| value.as_ptr())
}
fn output_dir_mirror(cli: &Cli) -> *const c_char {
cli.output_dir_mirror
.as_ref()
.map_or(ptr::null(), |value| value.as_ptr())
}
#[cfg(feature = "decompression")]
unsafe fn run_decompress(
cli: &Cli,
ctx: *mut FIO_ctx_t,
prefs: *mut FIO_prefs_t,
inputs: &[*const c_char],
output: *const c_char,
dictionary: *const c_char,
) -> c_int {
match cli.operation {
Operation::Test => {
let null_output = cstring(NULL_MARK).expect("static null marker");
unsafe {
FIO_setTestMode(prefs, 1);
FIO_decompressMultipleFilenames(
ctx,
prefs,
inputs.as_ptr(),
output_dir_mirror(cli),
output_dir_flat(cli),
null_output.as_ptr(),
dictionary,
)
}
}
Operation::Decompress if inputs.len() == 1 && !output.is_null() => unsafe {
FIO_decompressFilename(ctx, prefs, output, inputs[0], dictionary)
},
Operation::Decompress => unsafe {
FIO_decompressMultipleFilenames(
ctx,
prefs,
inputs.as_ptr(),
output_dir_mirror(cli),
output_dir_flat(cli),
output,
dictionary,
)
},
Operation::Compress | Operation::Bench | Operation::List | Operation::Train => {
unreachable!("compression, benchmark, and list are dispatched separately")
}
}
}
/// Copies every name out of a C `FileNamesTable` and releases the table.
unsafe fn drain_file_names_table(table: *mut FileNamesTable, into: &mut Vec<CString>) {
unsafe {
for entry in 0..(*table).tableSize {
let name = *(*table).fileNames.add(entry);
if !name.is_null() {
into.push(CStr::from_ptr(name).to_owned());
}
}
UTIL_freeFileNamesTable(table);
}
}
/// Appends the names read from every `--filelist` argument to the inputs.
/// As in the C CLI, a list that cannot be read (missing, irregular, empty,
/// or with over-long lines) aborts the whole run.
fn expand_file_lists(cli: &mut Cli) -> Result<(), String> {
let file_lists = std::mem::take(&mut cli.file_lists);
for list in &file_lists {
let table = unsafe { UTIL_createFileNamesTable_fromFileName(list.as_ptr()) };
if table.is_null() {
return Err(format!("error reading {}", list.to_string_lossy()));
}
unsafe { drain_file_names_table(table, &mut cli.inputs) };
}
Ok(())
}
/// Replaces the inputs with their recursive directory expansion. Symbolic
/// links are followed only under --force, matching the C `followLinks` flag.
fn expand_recursive_inputs(cli: &mut Cli) -> Result<(), String> {
let pointers: Vec<*const c_char> = cli.inputs.iter().map(|value| value.as_ptr()).collect();
let names = if pointers.is_empty() {
ptr::null()
} else {
pointers.as_ptr()
};
let table = unsafe { UTIL_createExpandedFNT(names, pointers.len(), c_int::from(cli.force)) };
if table.is_null() {
/* The C CLI treats this allocation failure as fatal (CONTROL). */
return Err("recursive directory expansion failed".to_owned());
}
let mut expanded = Vec::new();
unsafe { drain_file_names_table(table, &mut expanded) };
cli.inputs = expanded;
Ok(())
}
/// Prints frame information for every input through `FIO_listMultipleFiles`,
/// which owns the display-level gating, the stdin refusal, and the totals.
#[cfg(feature = "decompression")]
fn run_list(cli: &Cli) -> i32 {
let inputs: Vec<*const c_char> = cli.inputs.iter().map(|value| value.as_ptr()).collect();
unsafe { FIO_listMultipleFiles(inputs.len() as c_uint, inputs.as_ptr(), cli.display_level) }
}
#[cfg(feature = "benchmark")]
fn normalize_benchmark_levels(
start_c_level: c_int,
end_c_level: c_int,
mode: c_int,
max_c_level: c_int,
) -> (c_int, c_int) {
let mut start_level = start_c_level;
let mut end_level = end_c_level;
if mode == crate::benchzstd::BMK_decodeOnly {
start_level = 0;
end_level = 0;
}
if start_level > max_c_level {
start_level = max_c_level;
}
if end_level > max_c_level {
end_level = max_c_level;
}
if end_level < start_level {
end_level = start_level;
}
(start_level, end_level)
}
/// Benchmark bridge formerly implemented by `programs/zstdcli.c`.
///
/// Keep the exported name because it is a useful program-side ABI boundary,
/// but make the Rust CLI archive own both the normalization and dispatch.
#[cfg(feature = "benchmark")]
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_cli_bench(
file_names: *const *const c_char,
nb_files: c_uint,
dict_file_name: *const c_char,
start_c_level: c_int,
end_c_level: c_int,
compression_params: *const ZSTD_compressionParameters,
display_level: c_int,
nb_seconds: c_uint,
block_size: usize,
nb_workers: c_int,
mode: c_int,
) -> c_int {
let mut advanced_params = crate::benchzstd::BMK_initAdvancedParams();
advanced_params.nbSeconds = nb_seconds;
advanced_params.blockSize = block_size;
advanced_params.nbWorkers = nb_workers;
advanced_params.mode = mode;
let (start_level, end_level) =
normalize_benchmark_levels(start_c_level, end_c_level, mode, unsafe {
ZSTD_maxCLevel()
});
let compression_params = compression_params.cast();
if nb_files == 0 {
/* No input file: benchmark a synthetic sample (lorem generator). */
unsafe {
crate::benchzstd::BMK_syntheticTest(
-1.0,
start_level,
end_level,
compression_params,
display_level,
&advanced_params,
)
}
} else {
unsafe {
crate::benchzstd::BMK_benchFilesAdvanced(
file_names,
nb_files,
dict_file_name,
start_level,
end_level,
compression_params,
display_level,
&advanced_params,
)
}
}
}
/// `ZSTD_NOBENCH` compatibility bridge for compact CLI archives.
#[cfg(not(feature = "benchmark"))]
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_cli_bench(
file_names: *const *const c_char,
nb_files: c_uint,
dict_file_name: *const c_char,
start_c_level: c_int,
end_c_level: c_int,
compression_params: *const ZSTD_compressionParameters,
display_level: c_int,
nb_seconds: c_uint,
block_size: usize,
nb_workers: c_int,
mode: c_int,
) -> c_int {
let _ = (
file_names,
nb_files,
dict_file_name,
start_c_level,
end_c_level,
compression_params,
display_level,
nb_seconds,
block_size,
nb_workers,
mode,
);
-1
}
/// Runs benchmark mode through the Rust bridge. No input file means a
/// synthetic-sample benchmark, matching the C CLI.
fn run_bench(cli: &Cli) -> Result<i32, String> {
let inputs: Vec<*const c_char> = cli.inputs.iter().map(|value| value.as_ptr()).collect();
let dictionary = cli
.dictionary
.as_ref()
.map_or(ptr::null(), |value| value.as_ptr());
let workers = {
#[cfg(feature = "compression")]
{
unsafe {
resolved_worker_count(cli.workers, cli.single_thread, cli.auto_threads_logical)
}
}
#[cfg(not(feature = "compression"))]
{
cli.workers.unwrap_or(1)
}
};
let result = unsafe {
ZSTD_rust_cli_bench(
inputs.as_ptr(),
inputs.len() as c_uint,
dictionary,
cli.level,
cli.bench_end_level.unwrap_or(cli.level),
&cli.compression_params,
cli.display_level,
cli.bench_nb_seconds.unwrap_or(DEFAULT_BENCH_NB_SECONDS),
cli.block_size.unwrap_or(0),
// The C CLI benchmarks single-threaded unless -T was given.
workers,
if cli.bench_decode { 1 } else { 0 },
)
};
if result < 0 {
return Err("benchmark mode is not available in this build".to_owned());
}
Ok(result)
}
/// Runs dictionary training through the narrow `programs/dibio.c` ABI. The
/// file loading and output path stay in that bridge for now; the selected
/// dictionary builder symbols are supplied by the Rust library archive.
fn run_train(cli: &Cli) -> Result<i32, String> {
#[cfg(not(all(feature = "compression", feature = "dict-builder")))]
{
let _ = cli;
Err("training mode not available".to_owned())
}
#[cfg(all(feature = "compression", feature = "dict-builder"))]
{
let default_output = cstring(DEFAULT_DICT_NAME).expect("static dictionary name");
let output = cli.output.as_ref().unwrap_or(&default_output);
let inputs: Vec<*const c_char> = cli.inputs.iter().map(|value| value.as_ptr()).collect();
let workers = unsafe {
resolved_worker_count(cli.workers, cli.single_thread, cli.auto_threads_logical)
} as c_uint;
let z_params = ZDICT_params_t {
compressionLevel: cli.level,
notificationLevel: cli.display_level as c_uint,
dictID: cli.dictionary_id,
};
let chunk_size = cli.block_size.unwrap_or(0);
let mem_limit = cli.mem_limit.unwrap_or(0);
let result = match cli.training_algorithm {
TrainingAlgorithm::Cover => {
let mut params = cli.cover_params;
params.nbThreads = workers;
params.zParams = z_params;
let optimize = c_int::from(params.k == 0 || params.d == 0);
unsafe {
DiB_trainFromFiles(
output.as_ptr(),
cli.max_dict_size,
inputs.as_ptr(),
inputs.len() as c_int,
chunk_size,
ptr::null_mut(),
&mut params,
ptr::null_mut(),
optimize,
mem_limit,
)
}
}
TrainingAlgorithm::FastCover => {
let mut params = cli.fast_cover_params;
params.nbThreads = workers;
params.zParams = z_params;
let optimize = c_int::from(params.k == 0 || params.d == 0);
unsafe {
DiB_trainFromFiles(
output.as_ptr(),
cli.max_dict_size,
inputs.as_ptr(),
inputs.len() as c_int,
chunk_size,
ptr::null_mut(),
ptr::null_mut(),
&mut params,
optimize,
mem_limit,
)
}
}
TrainingAlgorithm::Legacy => {
let mut params = ZDICT_legacy_params_t {
selectivityLevel: cli.dictionary_selectivity,
zParams: z_params,
};
unsafe {
DiB_trainFromFiles(
output.as_ptr(),
cli.max_dict_size,
inputs.as_ptr(),
inputs.len() as c_int,
chunk_size,
&mut params,
ptr::null_mut(),
ptr::null_mut(),
0,
mem_limit,
)
}
}
};
Ok(result)
}
}
struct TraceGuard {
enabled: bool,
}
impl TraceGuard {
fn new(filename: Option<&CString>) -> Self {
if let Some(filename) = filename {
unsafe { TRACE_enable(filename.as_ptr()) };
Self { enabled: true }
} else {
Self { enabled: false }
}
}
}
impl Drop for TraceGuard {
fn drop(&mut self) {
if self.enabled {
unsafe { TRACE_finish() };
}
}
}
fn run_cli(mut cli: Cli) -> Result<i32, String> {
if cli.fake_stderr_is_console {
unsafe { UTIL_fakeStderrIsConsole() };
}
if cli.trace_file_stat {
unsafe { UTIL_traceFileStat() };
}
let _trace_guard = TraceGuard::new(cli.trace.as_ref());
unsafe {
/* The C CLI publishes the display level to util.c before any table
* expansion, so traversal warnings obey -q/-v. */
g_utilDisplayLevel = cli.display_level;
}
let explicit_input_count = cli.inputs.len();
filter_symlink_inputs(&mut cli);
if explicit_input_count > 0 && cli.inputs.is_empty() {
return Ok(1);
}
expand_file_lists(&mut cli)?;
/* Names known before recursive expansion: an input set that named only
* (possibly empty) directories must not fall back to stdin below. */
let input_name_count = cli.inputs.len();
if cli.recursive {
expand_recursive_inputs(&mut cli)?;
}
if cli.operation == Operation::List {
#[cfg(feature = "decompression")]
return Ok(run_list(&cli));
#[cfg(not(feature = "decompression"))]
return Err("file information is not supported".to_owned());
}
if cli.operation == Operation::Bench {
if cli.compression_format != CompressionFormat::Zstd {
return Err("benchmark mode only supports the zstd format".to_owned());
}
return run_bench(&cli);
}
if cli.operation == Operation::Train {
return run_train(&cli);
}
if cli.operation == Operation::Test {
cli.output = Some(cstring(NULL_MARK)?);
cli.remove_source = false;
}
if cli.inputs.is_empty() {
if input_name_count > 0 {
if cli.display_level >= 1 {
eprintln!(
"please provide correct input file(s) or non-empty directories -- ignored"
);
}
return Ok(0);
}
cli.inputs.push(cstring(STDIN_MARK)?);
if cli.output.is_none() {
cli.output = Some(cstring(STDOUT_MARK)?);
}
}
if cli.inputs.len() == 1
&& cli.inputs[0].as_bytes() == STDIN_MARK.as_bytes()
&& cli.output.is_none()
{
cli.output = Some(cstring(STDOUT_MARK)?);
}
if cli.show_default_cparams && cli.operation == Operation::Decompress {
return Err("error : can't use --show-default-cparams in decompression mode".to_owned());
}
check_terminal_safety(&cli)?;
print_cli_header(cli.display_level);
#[cfg(feature = "compression")]
if cli.operation == Operation::Decompress
&& cli.display_level >= 2
&& cli.workers.is_some_and(|workers| workers > 1)
{
eprintln!("Warning : decompression does not support multi-threading");
}
if cli.operation == Operation::Compress {
#[cfg(not(feature = "compression"))]
return Err("Compression not supported".to_owned());
#[cfg(feature = "compression")]
{
let min_level = unsafe { ZSTD_minCLevel() };
let max_level = unsafe { ZSTD_maxCLevel() };
let ceiling = if cli.ultra {
max_level
} else {
DEFAULT_MAX_CLEVEL.min(max_level)
};
if cli.level > ceiling {
if cli.display_level >= 2 {
eprintln!("zstd: warning: compression level reduced to {ceiling}");
}
cli.level = ceiling;
}
if cli.level < min_level {
return Err(format!(
"compression level {} is below {min_level}",
cli.level
));
}
if let (Some(minimum), Some(maximum)) = (cli.adapt_min, cli.adapt_max) {
if minimum > maximum {
return Err("adaptation minimum exceeds maximum".to_owned());
}
cli.level = cli.level.clamp(minimum, maximum);
}
}
} else {
#[cfg(not(feature = "decompression"))]
return Err("Decompression not supported".to_owned());
}
if cli.patch_from.is_some() && cli.inputs.len() > 1 {
return Err("can't use --patch-from=# on multiple files".to_owned());
}
let prefs = unsafe { FIO_createPreferences() };
let ctx = unsafe { FIO_createContext() };
if prefs.is_null() || ctx.is_null() {
unsafe {
if !prefs.is_null() {
FIO_freePreferences(prefs);
}
if !ctx.is_null() {
FIO_freeContext(ctx);
}
}
return Err("could not allocate C file-I/O state".to_owned());
}
let result = {
unsafe {
FIO_addAbortHandler();
apply_preferences(&cli, prefs, ctx);
#[cfg(feature = "compression")]
if cli.operation == Operation::Compress {
if cli.show_default_cparams {
print_default_cparams(&cli);
}
if cli.display_level >= 4 {
print_actual_cparams(&cli);
}
}
}
let output = cli
.output
.as_ref()
.map_or(ptr::null(), |value| value.as_ptr());
let dictionary = fileio_dictionary(&cli).map_or(ptr::null(), |value| value.as_ptr());
let inputs: Vec<*const c_char> = cli.inputs.iter().map(|value| value.as_ptr()).collect();
match cli.operation {
Operation::Compress => {
#[cfg(feature = "compression")]
{
unsafe { run_compress(&cli, ctx, prefs, &inputs, output, dictionary) }
}
#[cfg(not(feature = "compression"))]
unreachable!("unsupported compression was rejected above")
}
Operation::Decompress | Operation::Test => {
#[cfg(feature = "decompression")]
{
unsafe { run_decompress(&cli, ctx, prefs, &inputs, output, dictionary) }
}
#[cfg(not(feature = "decompression"))]
unreachable!("unsupported decompression was rejected above")
}
Operation::Bench | Operation::List | Operation::Train => {
unreachable!("benchmark and list modes were dispatched earlier")
}
}
};
unsafe {
FIO_freePreferences(prefs);
FIO_freeContext(ctx);
}
Ok(result)
}
fn run_from_args(args: Vec<OsString>) -> c_int {
let program_name = args
.first()
.map_or_else(|| "zstd".to_owned(), |value| program_basename(value));
match parse_args(args) {
Ok(Action::Help { advanced }) => {
usage(advanced);
0
}
Ok(Action::Version { quiet }) => {
print_version(quiet);
0
}
Ok(Action::Run(cli)) => match run_cli(*cli) {
Ok(result) => result,
Err(error) => {
eprintln!("zstd: {error}");
1
}
},
Err(error) => {
eprintln!("{}", error.message);
if error.show_usage {
let mut out = io::stderr().lock();
write_basic_usage(&mut out, &program_name);
}
1
}
}
}
unsafe fn argv_to_os_strings(
arg_count: c_int,
argv: *const *const c_char,
) -> Result<Vec<OsString>, String> {
if arg_count <= 0 || argv.is_null() {
return Err("invalid argv supplied by C main".to_owned());
}
let count = arg_count as usize;
let mut args = Vec::with_capacity(count);
for index in 0..count {
let argument = unsafe { *argv.add(index) };
if argument.is_null() {
return Err(format!("argv[{index}] is null"));
}
let bytes = unsafe { CStr::from_ptr(argument) }.to_bytes();
#[cfg(unix)]
args.push(OsString::from_vec(bytes.to_vec()));
#[cfg(not(unix))]
args.push(OsString::from(String::from_utf8_lossy(bytes).into_owned()));
}
Ok(args)
}
/// C `main()` entry point retained by the small `programs/zstdcli.c` forwarder.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_cli_main(arg_count: c_int, argv: *const *const c_char) -> c_int {
if let Err(error) = check_lib_version() {
eprintln!("zstd: {error}");
return 1;
}
match unsafe { argv_to_os_strings(arg_count, argv) } {
Ok(args) => run_from_args(args),
Err(error) => {
eprintln!("zstd: {error}");
1
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn parse(values: &[&str]) -> Cli {
let args = values.iter().map(OsString::from).collect();
match parse_args(args).expect("arguments should parse") {
Action::Run(cli) => *cli,
Action::Help { .. } | Action::Version { .. } => panic!("expected a run action"),
}
}
#[test]
fn defaults_preserve_the_c_fileio_contract() {
let cli = parse(&["zstd", "input"]);
assert_eq!(cli.content_size, 1);
assert_eq!(cli.workers, None);
assert_eq!(cli.operation, Operation::Compress);
assert_eq!(cli.compression_format, CompressionFormat::Zstd);
}
#[test]
fn normal_help_ceiling_excludes_ultra_levels() {
assert_eq!(max_c_level_for_help(), 19);
}
#[test]
fn format_option_selects_frame_type_and_suffix() {
let cli = parse(&["zstd", "--format=gzip", "input"]);
assert_eq!(cli.compression_format, CompressionFormat::Gzip);
assert_eq!(cli.compression_format.fio_type(), FIO_GZIP_COMPRESSION);
assert_eq!(cli.compression_format.suffix(), b".gz\0");
let cli = parse(&["zstd", "--format=gzip", "--format=zstd", "input"]);
assert_eq!(cli.compression_format, CompressionFormat::Zstd);
}
#[test]
fn format_option_rejects_unknown_formats() {
let error = parse_args(vec![
OsString::from("zstd"),
OsString::from("--format=zip"),
OsString::from("input"),
])
.expect_err("unknown formats must be rejected");
assert!(error.contains("unknown compression format"));
}
#[test]
fn trace_option_accepts_attached_and_separate_paths() {
let cli = parse(&["zstd", "--trace=first.trace", "input"]);
assert_eq!(
cli.trace.as_deref().map(CStr::to_bytes),
Some(&b"first.trace"[..])
);
let cli = parse(&["zstd", "--trace", "second.trace", "input"]);
assert_eq!(
cli.trace.as_deref().map(CStr::to_bytes),
Some(&b"second.trace"[..])
);
}
#[test]
fn long_mode_uses_the_cli_default_window_and_enables_ultra() {
let cli = parse(&["zstd", "--long", "input"]);
assert!(cli.ldm);
assert!(cli.ultra);
assert_eq!(cli.compression_params.windowLog, DEFAULT_LONG_WINDOW_LOG);
}
#[test]
fn long_mode_does_not_replace_an_explicit_window_log() {
let cli = parse(&["zstd", "--zstd=wlog=25", "--long", "input"]);
assert_eq!(cli.compression_params.windowLog, 25);
}
#[test]
fn max_mode_sets_the_full_c_and_ldm_parameter_bundle() {
let cli = parse(&["zstd", "--max", "input"]);
assert!(cli.ultra);
assert!(cli.ldm);
assert_eq!(
cli.compression_params,
ZSTD_compressionParameters {
windowLog: MAX_WINDOW_LOG,
chainLog: MAX_CHAIN_LOG,
hashLog: MAX_HASH_LOG,
searchLog: MAX_SEARCH_LOG,
minMatch: MAX_MIN_MATCH,
targetLength: MAX_TARGET_LENGTH,
strategy: MAX_STRATEGY,
}
);
assert_eq!(cli.overlap_log, Some(MAX_OVERLAP_LOG));
assert_eq!(cli.ldm_hash_log, Some(MAX_LDM_HASH_LOG));
assert_eq!(cli.ldm_hash_rate_log, Some(0));
assert_eq!(cli.ldm_min_match, Some(MAX_LDM_MIN_MATCH));
assert_eq!(cli.ldm_bucket_size_log, Some(MAX_LDM_BUCKET_SIZE_LOG));
}
#[test]
fn max_mode_is_applied_at_the_option_position() {
let cli = parse(&["zstd", "--max", "--zstd=wlog=25", "input"]);
assert_eq!(cli.compression_params.windowLog, 25);
let cli = parse(&["zstd", "--zstd=wlog=25", "--max", "input"]);
assert_eq!(cli.compression_params.windowLog, MAX_WINDOW_LOG);
}
#[test]
fn console_and_file_stat_compatibility_switches_are_recorded() {
let cli = parse(&[
"zstd",
"--fake-stderr-is-console",
"--trace-file-stat",
"input",
]);
assert!(cli.fake_stderr_is_console);
assert!(cli.trace_file_stat);
}
#[test]
fn gzip_best_is_an_alias_for_level_nine() {
let cli = parse(&["gzip", "--best", "input"]);
assert!(cli.gzip_compat);
assert_eq!(cli.level, 9);
let error = parse_args(vec![OsString::from("zstd"), OsString::from("--best")])
.expect_err("--best is a gzip compatibility option");
assert!(error.contains("Incorrect parameter: --best"));
}
#[test]
fn stdio_and_dictionary_short_options_are_preserved() {
let cli = parse(&["zstd", "-dc", "-D", "dict", "-"]);
assert_eq!(cli.operation, Operation::Decompress);
assert!(is_stdout(cli.output.as_ref()));
assert_eq!(
cli.dictionary.as_deref().map(CStr::to_bytes),
Some(&b"dict"[..])
);
assert_eq!(
cli.inputs
.iter()
.map(|input| input.as_bytes())
.collect::<Vec<_>>(),
vec![STDIN_MARK.as_bytes()]
);
}
#[test]
fn patch_from_accepts_attached_and_separate_reference_paths() {
let attached = parse(&["zstd", "--patch-from=reference", "input"]);
assert_eq!(
attached.patch_from.as_deref().map(CStr::to_bytes),
Some(&b"reference"[..])
);
assert!(attached.ultra);
let separate = parse(&["zstd", "--patch-from", "reference", "input"]);
assert_eq!(
separate.patch_from.as_deref().map(CStr::to_bytes),
Some(&b"reference"[..])
);
assert!(separate.ultra);
}
#[test]
fn patch_from_rejects_conflicts_with_dictionary_mode_in_both_orders() {
for values in [
&["zstd", "-D", "dict", "--patch-from=reference", "input"][..],
&["zstd", "--patch-from", "reference", "-D", "dict", "input"][..],
] {
let args = values.iter().map(OsString::from).collect();
let error = parse_args(args).expect_err("-D and --patch-from must conflict");
assert!(error.contains("can't use -D and --patch-from=# at the same time"));
}
}
#[test]
fn stdout_selection_does_not_enable_force_or_pass_through() {
let cli = parse(&["zstd", "-c", "input"]);
assert!(cli.force_stdout);
assert!(!cli.force);
assert_eq!(cli.pass_through, None);
}
#[test]
fn short_level_can_be_combined_with_other_flags() {
let cli = parse(&["zstd", "-5q", "input"]);
assert_eq!(cli.level, 5);
assert_eq!(cli.display_level, 1);
}
#[test]
fn fast_levels_preserve_the_unsigned_32_bit_conversion() {
let cli = parse(&["zstd", "--fast", "-4294967295", "input"]);
assert_eq!(cli.level, -1);
let cli = parse(&["zstd", "--fast=4294967295", "input"]);
assert_eq!(cli.level, -MAX_FAST_ACCELERATION);
}
#[test]
fn short_option_equals_form_is_accepted() {
let cli = parse(&["zstd", "-T=2", "-M=64M", "-B=1M", "input"]);
assert_eq!(cli.workers, Some(2));
assert_eq!(cli.mem_limit, Some(64 << 20));
assert_eq!(cli.block_size, Some(1 << 20));
}
#[test]
fn valueless_long_flags_reject_attached_values() {
let error = parse_args(vec![OsString::from("zstd"), OsString::from("--rm=0")])
.expect_err("an attached value must not activate --rm");
assert!(error.contains("does not take an argument"));
}
#[test]
fn empty_attached_adapt_is_rejected_but_bare_adapt_is_enabled() {
let bare = parse(&["zstd", "--adapt", "input"]);
assert!(bare.adapt);
let error = parse_args(vec![OsString::from("zstd"), OsString::from("--adapt=")])
.expect_err("an empty attached adapt value must be rejected");
assert!(error.contains("Incorrect parameter: --adapt="));
}
#[test]
fn adapt_bounds_and_rsyncable_single_thread_are_parsed_for_runtime_checks() {
let error = parse_args(vec![
OsString::from("zstd"),
OsString::from("--adapt=min=10,max=9"),
])
.expect_err("incoherent adaptation bounds must be rejected");
assert!(error.contains("Incorrect parameter: --adapt=min=10,max=9"));
let cli = parse(&["zstd", "--rsyncable", "--single-thread", "input"]);
assert!(cli.rsyncable);
assert!(cli.single_thread);
assert_eq!(cli.workers, Some(0));
}
#[test]
fn zstdmt_uses_auto_threads() {
let cli = parse(&["zstdmt", "input"]);
assert_eq!(cli.workers, Some(0));
assert!(!cli.single_thread);
}
#[test]
fn auto_threads_selects_core_family() {
let physical = parse(&["zstd", "-T0", "--auto-threads=physical", "input"]);
assert_eq!(physical.workers, Some(0));
assert!(!physical.auto_threads_logical);
let logical = parse(&["zstd", "-T0", "--auto-threads", "logical", "input"]);
assert_eq!(logical.workers, Some(0));
assert!(logical.auto_threads_logical);
}
#[test]
fn auto_threads_rejects_unknown_core_family() {
let error = parse_args(vec![
OsString::from("zstd"),
OsString::from("--auto-threads=bogus"),
])
.expect_err("an unknown core family must be rejected");
assert!(error.contains("unknown auto-thread mode"));
}
#[test]
fn single_thread_pins_zero_workers() {
let cli = parse(&["zstd", "--single-thread", "input"]);
assert_eq!(cli.workers, Some(0));
assert!(cli.single_thread);
}
#[test]
fn a_later_thread_count_overrides_single_thread_workers() {
/* Mirrors the C CLI: -T after --single-thread wins the worker count,
* while the single-thread latch stays set. */
let cli = parse(&["zstd", "--single-thread", "-T2", "input"]);
assert_eq!(cli.workers, Some(2));
assert!(cli.single_thread);
}
#[test]
fn single_thread_rejects_attached_values() {
let error = parse_args(vec![
OsString::from("zstd"),
OsString::from("--single-thread=1"),
])
.expect_err("an attached value must not activate --single-thread");
assert!(error.contains("does not take an argument"));
}
#[test]
fn alternate_format_aliases_select_compatibility_modes() {
let cli = Cli::new("gzip");
assert_eq!(cli.compression_format, CompressionFormat::Gzip);
assert_eq!(cli.operation, Operation::Compress);
assert!(cli.remove_source);
let cli = Cli::new("unxz");
assert_eq!(cli.compression_format, CompressionFormat::Xz);
assert_eq!(cli.operation, Operation::Decompress);
assert!(cli.remove_source);
let cli = Cli::new("lzma");
assert_eq!(cli.compression_format, CompressionFormat::Lzma);
assert_eq!(cli.operation, Operation::Compress);
}
#[cfg(unix)]
#[test]
fn short_path_arguments_keep_non_utf8_bytes() {
let output = OsString::from_vec(vec![b'-', b'o', 0xff, b'.', b'z', b's', b't']);
let action = parse_args(vec![
OsString::from("zstd"),
output,
OsString::from("input"),
])
.expect("arguments should parse");
let Action::Run(cli) = action else {
panic!("expected a run action");
};
assert_eq!(
cli.output.as_deref().map(CStr::to_bytes),
Some(&b"\xff.zst"[..])
);
}
#[test]
fn train_defaults_to_fastcover_and_dictionary_output() {
let cli = parse(&["zstd", "--train", "sample"]);
assert_eq!(cli.operation, Operation::Train);
assert_eq!(cli.training_algorithm, TrainingAlgorithm::FastCover);
assert_eq!(
cli.output.as_deref().map(CStr::to_bytes),
Some(&b"dictionary"[..])
);
assert_eq!(cli.max_dict_size, DEFAULT_MAX_DICT_SIZE);
assert_eq!(cli.fast_cover_params.d, 8);
assert_eq!(cli.fast_cover_params.f, 20);
assert_eq!(cli.fast_cover_params.steps, 4);
assert_eq!(cli.fast_cover_params.splitPoint, 0.75);
assert_eq!(cli.fast_cover_params.accel, DEFAULT_FASTCOVER_ACCEL);
}
#[test]
fn train_cover_parameters_select_algorithm_and_parse_optional_fields() {
let cli = parse(&[
"zstd",
"--train-cover=k=48,d=8,steps=32,split=80,shrink=3",
"sample",
]);
assert_eq!(cli.operation, Operation::Train);
assert_eq!(cli.training_algorithm, TrainingAlgorithm::Cover);
assert_eq!(cli.cover_params.k, 48);
assert_eq!(cli.cover_params.d, 8);
assert_eq!(cli.cover_params.steps, 32);
assert_eq!(cli.cover_params.splitPoint, 0.8);
assert_eq!(cli.cover_params.shrinkDict, 1);
assert_eq!(cli.cover_params.shrinkDictMaxRegression, 3);
}
#[test]
fn train_fastcover_and_legacy_parameter_forms_share_training_options() {
let fast = parse(&[
"zstd",
"--train-fastcover=k=64,d=8,f=21,steps=12,split=75,accel=2,shrink",
"--maxdict",
"128K",
"--dictID=42",
"-B4K",
"-T2",
"-o",
"custom.dict",
"sample",
]);
assert_eq!(fast.training_algorithm, TrainingAlgorithm::FastCover);
assert_eq!(fast.fast_cover_params.k, 64);
assert_eq!(fast.fast_cover_params.d, 8);
assert_eq!(fast.fast_cover_params.f, 21);
assert_eq!(fast.fast_cover_params.steps, 12);
assert_eq!(fast.fast_cover_params.splitPoint, 0.75);
assert_eq!(fast.fast_cover_params.accel, 2);
assert_eq!(fast.fast_cover_params.shrinkDict, 1);
assert_eq!(fast.fast_cover_params.shrinkDictMaxRegression, 1);
assert_eq!(fast.max_dict_size, 128 << 10);
assert_eq!(fast.dictionary_id, 42);
assert_eq!(fast.block_size, Some(4 << 10));
assert_eq!(fast.workers, Some(2));
assert_eq!(
fast.output.as_deref().map(CStr::to_bytes),
Some(&b"custom.dict"[..])
);
let legacy = parse(&["zstd", "--train-legacy=selectivity=12", "sample"]);
assert_eq!(legacy.operation, Operation::Train);
assert_eq!(legacy.training_algorithm, TrainingAlgorithm::Legacy);
assert_eq!(legacy.dictionary_selectivity, 12);
let legacy_short = parse(&["zstd", "--train-legacy", "-s5", "sample"]);
assert_eq!(legacy_short.training_algorithm, TrainingAlgorithm::Legacy);
assert_eq!(legacy_short.dictionary_selectivity, 5);
}
#[test]
fn training_options_preserve_upstream_validation_boundaries() {
let attached = parse_args(vec![
OsString::from("zstd"),
OsString::from("--train=value"),
])
.expect_err("--train does not accept an attached value");
assert!(attached.contains("does not take an argument"));
let unknown = parse_args(vec![
OsString::from("zstd"),
OsString::from("--train-cover=unknown=1"),
])
.expect_err("unknown cover parameters must be rejected");
assert!(unknown.contains("unknown --train-cover parameter"));
let missing = parse_args(vec![OsString::from("zstd"), OsString::from("--dictID")])
.expect_err("--dictID requires a value");
assert!(missing.contains("missing argument for --dictID"));
}
#[test]
fn recursive_flag_can_be_aggregated_with_other_shorts() {
let cli = parse(&["zstd", "-rq", "dir"]);
assert!(cli.recursive);
assert_eq!(cli.display_level, 1);
}
#[test]
fn list_mode_is_selected_by_short_and_long_flags() {
let short = parse(&["zstd", "-l", "file.zst"]);
let long = parse(&["zstd", "--list", "file.zst"]);
assert_eq!(short.operation, Operation::List);
assert_eq!(long.operation, Operation::List);
}
#[test]
fn list_rejects_attached_values() {
let error = parse_args(vec![OsString::from("zstd"), OsString::from("--list=x")])
.expect_err("an attached value must not activate --list");
assert!(error.contains("does not take an argument"));
}
#[test]
fn show_default_cparams_is_a_compression_information_flag() {
let cli = parse(&["zstd", "--show-default-cparams", "input"]);
assert!(cli.show_default_cparams);
assert_eq!(cli.operation, Operation::Compress);
}
#[test]
fn show_default_cparams_rejects_attached_values() {
let error = parse_args(vec![
OsString::from("zstd"),
OsString::from("--show-default-cparams=x"),
])
.expect_err("show-default-cparams must not consume an attached value");
assert!(error.contains("does not take an argument"));
}
#[test]
fn filelist_accumulates_both_syntaxes() {
let cli = parse(&["zstd", "--filelist=one.txt", "--filelist", "two.txt", "in"]);
assert_eq!(
cli.file_lists
.iter()
.map(|list| list.as_bytes())
.collect::<Vec<_>>(),
vec![&b"one.txt"[..], &b"two.txt"[..]]
);
}
#[test]
fn filelist_argument_must_not_be_another_option() {
let error = parse_args(vec![
OsString::from("zstd"),
OsString::from("--filelist"),
OsString::from("-q"),
])
.expect_err("an option must not be consumed as the list name");
assert!(error.contains("cannot be separated from its argument"));
}
#[test]
fn output_dir_flat_accepts_attached_and_separate_values() {
let attached = parse(&["zstd", "--output-dir-flat=out", "in"]);
let separate = parse(&["zstd", "--output-dir-flat", "out", "in"]);
assert_eq!(
attached.output_dir_flat.as_deref().map(CStr::to_bytes),
Some(&b"out"[..])
);
assert_eq!(
separate.output_dir_flat.as_deref().map(CStr::to_bytes),
Some(&b"out"[..])
);
}
#[test]
fn output_dir_flat_rejects_an_empty_name() {
/* Mirrors the C NEXT_FIELD semantics: `--output-dir-flat=` keeps the
* empty attached value instead of consuming the next argument. */
let error = parse_args(vec![
OsString::from("zstd"),
OsString::from("--output-dir-flat="),
OsString::from("out"),
])
.expect_err("an empty output dir must be rejected");
assert!(error.contains("output dir cannot be empty string"));
}
#[cfg(unix)]
#[test]
fn output_dir_mirror_is_parsed_on_posix_targets() {
let cli = parse(&["zstd", "--output-dir-mirror", "out", "in"]);
assert_eq!(
cli.output_dir_mirror.as_deref().map(CStr::to_bytes),
Some(&b"out"[..])
);
}
#[test]
fn bench_mode_parses_level_duration_and_defaults() {
let cli = parse(&["zstd", "-b1", "-i0", "input"]);
assert_eq!(cli.operation, Operation::Bench);
assert_eq!(cli.level, 1);
assert_eq!(cli.bench_nb_seconds, Some(0));
assert_eq!(cli.bench_end_level, None);
assert_eq!(
cli.inputs
.iter()
.map(|input| input.as_bytes())
.collect::<Vec<_>>(),
vec![&b"input"[..]]
);
}
#[test]
fn bench_range_aggregates_within_a_single_argument() {
let cli = parse(&["zstd", "-b5e6i2", "input"]);
assert_eq!(cli.operation, Operation::Bench);
assert_eq!(cli.level, 5);
assert_eq!(cli.bench_end_level, Some(6));
assert_eq!(cli.bench_nb_seconds, Some(2));
}
#[test]
fn bench_duration_without_digits_defaults_to_zero() {
let cli = parse(&["zstd", "-b", "-e", "-i"]);
assert_eq!(cli.operation, Operation::Bench);
assert_eq!(cli.level, DEFAULT_CLEVEL);
assert_eq!(cli.bench_end_level, Some(0));
assert_eq!(cli.bench_nb_seconds, Some(0));
}
#[test]
fn bench_decompression_keeps_benchmark_operation() {
let cli = parse(&["zstd", "-b", "-d", "-i0", "input"]);
assert_eq!(cli.operation, Operation::Bench);
assert!(cli.bench_decode);
}
#[cfg(feature = "benchmark")]
#[test]
fn benchmark_level_normalization_matches_c_bridge() {
assert_eq!(
normalize_benchmark_levels(100, 4, crate::benchzstd::BMK_both, 22),
(22, 22)
);
assert_eq!(
normalize_benchmark_levels(7, 3, crate::benchzstd::BMK_both, 22),
(7, 7)
);
assert_eq!(
normalize_benchmark_levels(7, 3, crate::benchzstd::BMK_decodeOnly, 22),
(0, 0)
);
}
#[cfg(not(feature = "benchmark"))]
#[test]
fn benchmark_bridge_returns_unavailable_without_feature() {
assert_eq!(
unsafe {
ZSTD_rust_cli_bench(
ptr::null(),
0,
ptr::null(),
3,
3,
ptr::null(),
0,
3,
0,
1,
0,
)
},
-1
);
}
}
Reference in New Issue View Git Blame Copy Permalink