ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
cd7ae43da7dcf1f9c267a28d48eed872499a8d07
zstd-rs/rust/src/zstd_cli.rs
T
ddidderr 49454dada9 feat(cli): port dictionary-training dispatch to Rust 
Add Rust parsing and dispatch for the default, Cover, FastCover, and legacy
dictionary-training modes, including their parameter validation, output
defaults, and help text. Keep file loading behind the existing narrow dibio
bridge while the dictionary algorithms are supplied by the Rust library.

Test Plan:
- rustfmt +nightly --check --edition 2021 rust/src/zstd_cli.rs
- RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= cargo test --manifest-path rust/cli/Cargo.toml
- RUSTC_WRAPPER= CARGO_BUILD_RUSTC_WRAPPER= cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings
- git diff --cached --check
2026-07-12 10:44:57 +02:00

2577 lines
86 KiB
Rust
Raw Blame History

#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
//! Rust command-line frontend for zstd.
//!
//! This is intentionally a parser and dispatch layer, not a second file I/O
//! implementation. It reuses the mature C `fileio` layer through its narrow
//! public-in-the-programs-tree ABI: file opening, safe replacement, sparse
//! writes, dictionary loading, streaming, and metadata preservation remain in
//! `programs/fileio.c` for this first migration step.
//!
//! Benchmark mode (`-b`) parses here and dispatches through the
//! `ZSTD_rust_cli_bench` bridge in `programs/zstdcli.c`: the run/timing loop
//! (benchfn, timefn) is Rust, while orchestration and result formatting
//! (`benchzstd.c`) remain C behind the preprocessor-gated bridge, so builds
//! with `ZSTD_NOBENCH` never reference benchmark symbols.
//!
//! Recursive expansion (`-r`), `--filelist`, the output-directory modes, and
//! `--list` reuse the C `FileNamesTable` helpers from `programs/util.c` and
//! `FIO_listMultipleFiles`, so directory traversal and frame inspection stay
//! byte-identical with the C CLI.
//!
//! Remaining C-only CLI boundaries are called out in `unsupported()` below:
//! tracing and alternate-format selection. Dictionary training uses the
//! temporary `DiB_trainFromFiles` bridge while its algorithms are Rust-owned.
use std::env;
use std::ffi::{CStr, CString, OsStr, OsString};
use std::fs;
use std::io::{self, IsTerminal, Write};
use std::os::raw::{c_char, c_int, c_uint};
use std::path::Path;
use std::ptr;
#[cfg(unix)]
use std::os::unix::ffi::{OsStrExt, OsStringExt};
#[cfg(unix)]
use std::os::unix::fs::FileTypeExt;
const DEFAULT_CLEVEL: i32 = 3;
#[cfg(feature = "compression")]
const DEFAULT_MAX_CLEVEL: i32 = 19;
const DEFAULT_BENCH_NB_SECONDS: u32 = 3;
const DEFAULT_MEM_LIMIT: u32 = 1 << 27;
const DEFAULT_LONG_WINDOW_LOG: u32 = 27;
const DEFAULT_DICT_NAME: &str = "dictionary";
const DEFAULT_MAX_DICT_SIZE: usize = 110 << 10;
const DEFAULT_DICT_SELECTIVITY: u32 = 9;
const DEFAULT_SHRINK_DICT_REGRESSION: u32 = 1;
const DEFAULT_FASTCOVER_ACCEL: u32 = 1;
const MAX_FAST_ACCELERATION: i32 = 128 << 10;
const STDIN_MARK: &str = "/*stdin*\\";
const STDOUT_MARK: &str = "/*stdout*\\";
#[cfg(windows)]
const NULL_MARK: &str = "NUL";
#[cfg(not(windows))]
const NULL_MARK: &str = "/dev/null";
#[cfg(feature = "compression")]
const ZSTD_SUFFIX: &[u8] = b".zst\0";
const FIO_ZSTD_COMPRESSION: c_int = 0;
const FIO_PS_AUTO: c_int = 0;
const FIO_PS_NEVER: c_int = 1;
const FIO_PS_ALWAYS: c_int = 2;
const ZSTD_PS_AUTO: c_int = 0;
const ZSTD_PS_ENABLE: c_int = 1;
const ZSTD_PS_DISABLE: c_int = 2;
#[repr(C)]
struct FIO_prefs_t {
_private: [u8; 0],
}
#[repr(C)]
struct FIO_ctx_t {
_private: [u8; 0],
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
struct ZSTD_compressionParameters {
windowLog: u32,
chainLog: u32,
hashLog: u32,
searchLog: u32,
minMatch: u32,
targetLength: u32,
strategy: c_int,
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq)]
struct ZDICT_params_t {
compressionLevel: c_int,
notificationLevel: c_uint,
dictID: c_uint,
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq)]
struct ZDICT_cover_params_t {
k: c_uint,
d: c_uint,
steps: c_uint,
nbThreads: c_uint,
splitPoint: f64,
shrinkDict: c_uint,
shrinkDictMaxRegression: c_uint,
zParams: ZDICT_params_t,
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq)]
struct ZDICT_fastCover_params_t {
k: c_uint,
d: c_uint,
f: c_uint,
steps: c_uint,
nbThreads: c_uint,
splitPoint: f64,
accel: c_uint,
shrinkDict: c_uint,
shrinkDictMaxRegression: c_uint,
zParams: ZDICT_params_t,
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default, PartialEq)]
struct ZDICT_legacy_params_t {
selectivityLevel: c_uint,
zParams: ZDICT_params_t,
}
/// Mirror of the `FileNamesTable` in `programs/util.h`; the tables returned
/// by the `UTIL_*FNT` helpers are only read and released here, never resized.
#[repr(C)]
struct FileNamesTable {
fileNames: *mut *const c_char,
buf: *mut c_char,
tableSize: usize,
tableCapacity: usize,
}
unsafe extern "C" {
fn ZSTD_versionString() -> *const c_char;
fn ZSTD_rust_cli_expected_version() -> *const c_char;
static mut g_utilDisplayLevel: c_int;
#[cfg(feature = "compression")]
fn ZSTD_minCLevel() -> c_int;
#[cfg(feature = "compression")]
fn ZSTD_maxCLevel() -> c_int;
#[cfg(feature = "compression")]
fn UTIL_countPhysicalCores() -> c_int;
#[cfg(feature = "compression")]
fn UTIL_countLogicalCores() -> c_int;
fn UTIL_getFileSize(input_file_name: *const c_char) -> u64;
fn UTIL_createFileNamesTable_fromFileName(
input_file_name: *const c_char,
) -> *mut FileNamesTable;
fn UTIL_createExpandedFNT(
file_names: *const *const c_char,
nb_file_names: usize,
follow_links: c_int,
) -> *mut FileNamesTable;
fn UTIL_freeFileNamesTable(table: *mut FileNamesTable);
fn FIO_createPreferences() -> *mut FIO_prefs_t;
fn FIO_freePreferences(prefs: *mut FIO_prefs_t);
fn FIO_createContext() -> *mut FIO_ctx_t;
fn FIO_freeContext(ctx: *mut FIO_ctx_t);
fn FIO_addAbortHandler();
fn FIO_setCompressionType(prefs: *mut FIO_prefs_t, compression_type: c_int);
fn FIO_overwriteMode(prefs: *mut FIO_prefs_t);
fn FIO_setAdaptiveMode(prefs: *mut FIO_prefs_t, adapt: c_int);
#[cfg(feature = "compression")]
fn FIO_setAdaptMin(prefs: *mut FIO_prefs_t, level: c_int);
#[cfg(feature = "compression")]
fn FIO_setAdaptMax(prefs: *mut FIO_prefs_t, level: c_int);
fn FIO_setUseRowMatchFinder(prefs: *mut FIO_prefs_t, mode: c_int);
fn FIO_setBlockSize(prefs: *mut FIO_prefs_t, block_size: c_int);
fn FIO_setChecksumFlag(prefs: *mut FIO_prefs_t, checksum: c_int);
fn FIO_setDictIDFlag(prefs: *mut FIO_prefs_t, dict_id: c_int);
fn FIO_setLdmBucketSizeLog(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setLdmFlag(prefs: *mut FIO_prefs_t, value: c_uint);
fn FIO_setLdmHashRateLog(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setLdmHashLog(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setLdmMinMatch(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setMemLimit(prefs: *mut FIO_prefs_t, limit: c_uint);
#[cfg(feature = "compression")]
fn FIO_setNbWorkers(prefs: *mut FIO_prefs_t, workers: c_int);
fn FIO_setOverlapLog(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setRemoveSrcFile(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setSparseWrite(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setRsyncable(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setStreamSrcSize(prefs: *mut FIO_prefs_t, value: usize);
fn FIO_setTargetCBlockSize(prefs: *mut FIO_prefs_t, value: usize);
fn FIO_setSrcSizeHint(prefs: *mut FIO_prefs_t, value: usize);
#[cfg(feature = "decompression")]
fn FIO_setTestMode(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setLiteralCompressionMode(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setProgressSetting(value: c_int);
fn FIO_setNotificationLevel(value: c_int);
fn FIO_setExcludeCompressedFile(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setAllowBlockDevices(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setContentSize(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setAsyncIOFlag(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setPassThroughFlag(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setPatchFromMode(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setMMapDict(prefs: *mut FIO_prefs_t, value: c_int);
fn FIO_setNbFilesTotal(ctx: *mut FIO_ctx_t, value: c_int);
fn FIO_setHasStdinInput(ctx: *mut FIO_ctx_t, value: c_int);
fn FIO_setHasStdoutOutput(ctx: *mut FIO_ctx_t, value: c_int);
#[cfg(feature = "compression")]
fn FIO_compressFilename(
ctx: *mut FIO_ctx_t,
prefs: *mut FIO_prefs_t,
output: *const c_char,
input: *const c_char,
dict: *const c_char,
level: c_int,
params: ZSTD_compressionParameters,
) -> c_int;
#[cfg(feature = "decompression")]
fn FIO_decompressFilename(
ctx: *mut FIO_ctx_t,
prefs: *mut FIO_prefs_t,
output: *const c_char,
input: *const c_char,
dict: *const c_char,
) -> c_int;
#[cfg(feature = "compression")]
fn FIO_compressMultipleFilenames(
ctx: *mut FIO_ctx_t,
prefs: *mut FIO_prefs_t,
inputs: *const *const c_char,
output_mirror_dir: *const c_char,
output_dir: *const c_char,
output: *const c_char,
suffix: *const c_char,
dict: *const c_char,
level: c_int,
params: ZSTD_compressionParameters,
) -> c_int;
#[cfg(feature = "decompression")]
fn FIO_decompressMultipleFilenames(
ctx: *mut FIO_ctx_t,
prefs: *mut FIO_prefs_t,
inputs: *const *const c_char,
output_mirror_dir: *const c_char,
output_dir: *const c_char,
output: *const c_char,
dict: *const c_char,
) -> c_int;
#[cfg(feature = "decompression")]
fn FIO_listMultipleFiles(
nb_files: c_uint,
file_names: *const *const c_char,
display_level: c_int,
) -> c_int;
/// Benchmark bridge implemented by the `programs/zstdcli.c` shim, which
/// owns the `ZSTD_NOBENCH` preprocessor decision. Returns the benchmark
/// result (>= 0), or -1 when benchmarking is compiled out.
fn ZSTD_rust_cli_bench(
file_names: *const *const c_char,
nb_files: c_uint,
dict_file_name: *const c_char,
start_level: c_int,
end_level: c_int,
compression_params: *const ZSTD_compressionParameters,
display_level: c_int,
nb_seconds: c_uint,
block_size: usize,
nb_workers: c_int,
) -> c_int;
/// Narrow bridge to `programs/dibio.c`. The file loader and dictionary
/// algorithms remain on the C/Rust library side of this boundary.
#[cfg(feature = "compression")]
fn DiB_trainFromFiles(
dict_file_name: *const c_char,
max_dict_size: usize,
file_names: *const *const c_char,
nb_files: c_int,
chunk_size: usize,
legacy_params: *mut ZDICT_legacy_params_t,
cover_params: *mut ZDICT_cover_params_t,
fast_cover_params: *mut ZDICT_fastCover_params_t,
optimize: c_int,
mem_limit: c_uint,
) -> c_int;
#[cfg(feature = "compression")]
fn ZSTD_getCParams(
compression_level: c_int,
estimated_src_size: u64,
dict_size: usize,
) -> ZSTD_compressionParameters;
}
const ZSTD_STRATEGY_NAMES: [&str; 10] = [
"",
"ZSTD_fast",
"ZSTD_dfast",
"ZSTD_greedy",
"ZSTD_lazy",
"ZSTD_lazy2",
"ZSTD_btlazy2",
"ZSTD_btopt",
"ZSTD_btultra",
"ZSTD_btultra2",
];
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum Operation {
Compress,
Decompress,
Test,
Bench,
List,
Train,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
enum TrainingAlgorithm {
Cover,
#[default]
FastCover,
Legacy,
}
#[derive(Debug)]
enum Action {
Run(Box<Cli>),
Help { advanced: bool },
Version { quiet: bool },
}
#[derive(Debug)]
struct Cli {
operation: Operation,
inputs: Vec<CString>,
output: Option<CString>,
dictionary: Option<CString>,
level: i32,
ultra: bool,
display_level: i32,
force: bool,
force_stdout: bool,
remove_source: bool,
checksum: Option<i32>,
sparse: Option<i32>,
pass_through: Option<i32>,
content_size: i32,
dict_id: Option<i32>,
async_io: Option<i32>,
mmap_dict: i32,
progress: i32,
workers: Option<i32>,
single_thread: bool,
block_size: Option<usize>,
mem_limit: Option<u32>,
ldm: bool,
ldm_hash_log: Option<i32>,
ldm_min_match: Option<i32>,
ldm_bucket_size_log: Option<i32>,
ldm_hash_rate_log: Option<i32>,
overlap_log: Option<i32>,
adapt: bool,
adapt_min: Option<i32>,
adapt_max: Option<i32>,
rsyncable: bool,
stream_src_size: Option<usize>,
target_cblock_size: Option<usize>,
src_size_hint: Option<usize>,
literal_compression: Option<i32>,
row_match_finder: i32,
exclude_compressed: bool,
compression_params: ZSTD_compressionParameters,
show_default_cparams: bool,
bench_end_level: Option<i32>,
bench_nb_seconds: Option<u32>,
recursive: bool,
file_lists: Vec<CString>,
output_dir_flat: Option<CString>,
output_dir_mirror: Option<CString>,
training_algorithm: TrainingAlgorithm,
max_dict_size: usize,
dictionary_id: u32,
dictionary_selectivity: u32,
cover_params: ZDICT_cover_params_t,
fast_cover_params: ZDICT_fastCover_params_t,
unsupported_program: Option<String>,
}
impl Cli {
fn new(program_name: &str) -> Self {
let mut cli = Self {
operation: Operation::Compress,
inputs: Vec::new(),
output: None,
dictionary: None,
level: default_level(),
ultra: false,
display_level: 2,
force: false,
force_stdout: false,
remove_source: false,
checksum: None,
sparse: None,
pass_through: None,
content_size: 1,
dict_id: None,
async_io: None,
mmap_dict: ZSTD_PS_AUTO,
progress: FIO_PS_AUTO,
workers: None,
single_thread: false,
block_size: None,
mem_limit: None,
ldm: false,
ldm_hash_log: None,
ldm_min_match: None,
ldm_bucket_size_log: None,
ldm_hash_rate_log: None,
overlap_log: None,
adapt: false,
adapt_min: None,
adapt_max: None,
rsyncable: false,
stream_src_size: None,
target_cblock_size: None,
src_size_hint: None,
literal_compression: None,
row_match_finder: ZSTD_PS_AUTO,
exclude_compressed: false,
compression_params: ZSTD_compressionParameters::default(),
show_default_cparams: false,
bench_end_level: None,
bench_nb_seconds: None,
recursive: false,
file_lists: Vec::new(),
output_dir_flat: None,
output_dir_mirror: None,
training_algorithm: TrainingAlgorithm::default(),
max_dict_size: DEFAULT_MAX_DICT_SIZE,
dictionary_id: 0,
dictionary_selectivity: DEFAULT_DICT_SELECTIVITY,
cover_params: ZDICT_cover_params_t::default(),
fast_cover_params: default_fast_cover_params(),
unsupported_program: None,
};
match program_name {
"unzstd" => cli.operation = Operation::Decompress,
"zstdmt" => cli.workers = Some(0),
"zstdcat" | "zcat" => {
cli.operation = Operation::Decompress;
cli.output = Some(cstring(STDOUT_MARK).expect("static stdout marker"));
cli.force = true;
cli.force_stdout = true;
cli.pass_through = Some(1);
cli.display_level = 1;
}
"gzip" | "gunzip" | "gzcat" | "lzma" | "unlzma" | "xz" | "unxz" | "lz4" | "unlz4" => {
cli.unsupported_program = Some(program_name.to_owned())
}
_ => {}
}
cli
}
}
fn cstring(value: &str) -> Result<CString, String> {
CString::new(value).map_err(|_| format!("argument contains an interior NUL: {value:?}"))
}
fn os_cstring(value: &OsStr) -> Result<CString, String> {
#[cfg(unix)]
{
CString::new(value.as_bytes()).map_err(|_| "file name contains an interior NUL".to_owned())
}
#[cfg(not(unix))]
{
cstring(&value.to_string_lossy())
}
}
fn default_level() -> i32 {
match env::var("ZSTD_CLEVEL") {
Ok(value) => value.parse::<i32>().unwrap_or(DEFAULT_CLEVEL),
Err(_) => DEFAULT_CLEVEL,
}
}
#[cfg(feature = "compression")]
unsafe fn default_worker_count() -> i32 {
if let Ok(value) = env::var("ZSTD_NBTHREADS") {
if let Ok(workers) = value.parse::<u32>() {
if let Ok(workers) = i32::try_from(workers) {
return workers;
}
}
}
let logical_cores = unsafe { UTIL_countLogicalCores() }.max(1);
(logical_cores / 4).clamp(1, 4)
}
#[cfg(feature = "compression")]
unsafe fn resolved_worker_count(workers: Option<i32>, single_thread: bool) -> i32 {
match workers {
/* --single-thread pins zero workers; a bare zero (-T0 or the zstdmt
* program name) auto-detects the core count as in the C CLI. */
Some(0) if single_thread => 0,
Some(0) => unsafe { UTIL_countPhysicalCores() }.max(1),
Some(workers) => workers,
None => unsafe { default_worker_count() },
}
}
fn program_basename(value: &OsStr) -> String {
Path::new(value)
.file_name()
.unwrap_or(value)
.to_string_lossy()
.split('.')
.next()
.unwrap_or("zstd")
.to_owned()
}
fn usage(advanced: bool) {
let mut out = io::stdout().lock();
let _ = writeln!(
out,
"Compress or decompress INPUT file(s); reads stdin when INPUT is '-' or omitted."
);
let _ = writeln!(out, "\nUsage: zstd [OPTIONS...] [INPUT... | -] [-o OUTPUT]");
let _ = writeln!(out, "\nCore options:");
let _ = writeln!(
out,
" -o OUTPUT, -c, --stdout Select output file or stdout"
);
let _ = writeln!(out, " -d, --decompress Decompress");
let _ = writeln!(out, " -t, --test Test compressed input");
let _ = writeln!(
out,
" -# Compression level (default {DEFAULT_CLEVEL})"
);
let _ = writeln!(out, " -D DICT Use a dictionary");
let _ = writeln!(
out,
" -f, --force Overwrite output / allow stdio"
);
let _ = writeln!(
out,
" -k, --keep | --rm Preserve or remove source after success"
);
let _ = writeln!(out, " -q, --quiet | -v, --verbose Adjust display level");
let _ = writeln!(out, " -V, --version Print version");
let _ = writeln!(out, " -h | -H, --help Print help");
if advanced {
let _ = writeln!(out, "\nImplemented advanced compression controls:");
let _ = writeln!(
out,
" --fast[=#], --ultra, --long[=#], --threads=#, --single-thread, --block-size=#"
);
let _ = writeln!(
out,
" --zstd=wlog=#,clog=#,hlog=#,slog=#,mml=#,tlen=#,strat=#"
);
let _ = writeln!(
out,
" --[no-]check, --[no-]sparse, --[no-]progress, --[no-]asyncio"
);
let _ = writeln!(
out,
" --adapt[=min=#,max=#], --rsyncable, --[no-]row-match-finder"
);
let _ = writeln!(out, "\nFile selection and placement:");
let _ = writeln!(
out,
" -r Operate recursively on directories"
);
let _ = writeln!(
out,
" --filelist LIST Read a list of input files from LIST"
);
let _ = writeln!(
out,
" --output-dir-flat DIR Store processed files in DIR"
);
#[cfg(unix)]
let _ = writeln!(
out,
" --output-dir-mirror DIR Store processed files in DIR, mirroring the input tree"
);
let _ = writeln!(
out,
" -l, --list Print information about .zst file(s)"
);
let _ = writeln!(out, "\nBenchmark options:");
let _ = writeln!(
out,
" -b# Benchmark file(s) at compression level #"
);
let _ = writeln!(
out,
" -e# Test all levels from -b# up to # included"
);
let _ = writeln!(
out,
" -i# Set the minimum evaluation time to # seconds"
);
let _ = writeln!(
out,
"\nDictionary builder:\n --train Create a dictionary from training files\n --train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]]\n --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]]\n --train-legacy[=s=#] Use the legacy algorithm\n --maxdict=# Maximum dictionary size (default {DEFAULT_MAX_DICT_SIZE})\n --dictID=# Force the dictionary ID (default: random)"
);
let _ = writeln!(out, "\nNot yet migrated: trace and alternate formats.");
}
}
fn print_version(quiet: bool) {
let version = unsafe { CStr::from_ptr(ZSTD_versionString()) }
.to_string_lossy()
.into_owned();
if quiet {
println!("{version}");
} else {
println!(
"*** Zstandard CLI ({}-bit) v{version}, by Yann Collet ***",
usize::BITS
);
}
}
fn check_lib_version() -> Result<(), String> {
let expected = unsafe { CStr::from_ptr(ZSTD_rust_cli_expected_version()) };
let actual = unsafe { CStr::from_ptr(ZSTD_versionString()) };
if expected == actual {
return Ok(());
}
Err(format!(
"incorrect library version (expecting: {}; actual: {})",
expected.to_string_lossy(),
actual.to_string_lossy()
))
}
fn parse_size(value: &str) -> Result<usize, String> {
let split = value
.find(|character: char| !character.is_ascii_digit())
.unwrap_or(value.len());
let (digits, suffix) = value.split_at(split);
if digits.is_empty() {
return Err(format!("expected a numeric value, got {value:?}"));
}
let mut number = digits
.parse::<usize>()
.map_err(|_| format!("numeric value overflows size_t: {value:?}"))?;
let normalized = suffix.trim_end_matches('B').trim_end_matches('i');
let shift = match normalized {
"" => 0,
"K" | "k" => 10,
"M" | "m" => 20,
"G" | "g" => 30,
_ => return Err(format!("unsupported numeric suffix in {value:?}")),
};
number = number
.checked_shl(shift)
.ok_or_else(|| format!("numeric value overflows size_t: {value:?}"))?;
Ok(number)
}
fn parse_u32(value: &str, name: &str) -> Result<u32, String> {
let size = parse_size(value)?;
u32::try_from(size).map_err(|_| format!("{name} is too large: {value:?}"))
}
fn parse_i32(value: &str, name: &str) -> Result<i32, String> {
value
.parse::<i32>()
.map_err(|_| format!("invalid {name}: {value:?}"))
}
fn parse_compression_level(value: &str) -> Result<i32, String> {
/* The C CLI reads short-form levels as U32 before storing them in its
* signed level field. Preserve that conversion for values such as
* 4294967295, which represents -1 on the supported two's-complement
* targets and is a valid fast-compression level. */
Ok(parse_u32(value, "compression level")? as i32)
}
fn parse_worker_count(value: &str) -> Result<i32, String> {
let workers = parse_i32(value, "thread count")?;
if workers < 0 {
return Err(format!("thread count must not be negative: {value:?}"));
}
Ok(workers)
}
fn next_value(
attached: Option<&str>,
args: &[OsString],
index: &mut usize,
option: &str,
) -> Result<String, String> {
if let Some(value) = attached {
if !value.is_empty() {
return Ok(value.to_owned());
}
}
*index += 1;
let Some(value) = args.get(*index) else {
return Err(format!("missing argument for {option}"));
};
let rendered = value.to_string_lossy().into_owned();
if rendered.starts_with('-') {
return Err(format!(
"{option} cannot be separated from its argument by another option"
));
}
Ok(rendered)
}
fn next_os_value(args: &[OsString], index: &mut usize, option: &str) -> Result<OsString, String> {
*index += 1;
let Some(value) = args.get(*index) else {
return Err(format!("missing argument for {option}"));
};
if value.to_string_lossy().starts_with('-') {
return Err(format!(
"{option} cannot be separated from its argument by another option"
));
}
Ok(value.clone())
}
/// Mirrors the C CLI `NEXT_FIELD` macro: an attached `=value` is used as-is
/// (even when empty), otherwise the next argument is consumed. Unlike
/// `next_value`, an empty attached value never falls through to the next
/// argument, so `--output-dir-flat=` stays an empty (and rejected) name.
fn next_field(
attached: Option<&str>,
args: &[OsString],
index: &mut usize,
option: &str,
) -> Result<String, String> {
if let Some(value) = attached {
return Ok(value.to_owned());
}
*index += 1;
let Some(value) = args.get(*index) else {
return Err(format!("missing argument for {option}"));
};
let rendered = value.to_string_lossy().into_owned();
if rendered.starts_with('-') {
return Err(format!(
"{option} cannot be separated from its argument by another option"
));
}
Ok(rendered)
}
#[cfg(unix)]
fn short_attached_value(value: &OsStr, start: usize) -> Option<OsString> {
let bytes = &value.as_bytes()[start..];
if bytes.is_empty() {
return None;
}
let bytes = if bytes.first() == Some(&b'=') {
&bytes[1..]
} else {
bytes
};
Some(OsString::from_vec(bytes.to_vec()))
}
#[cfg(not(unix))]
fn short_attached_value(value: &OsStr, start: usize) -> Option<OsString> {
let rendered = value.to_string_lossy();
let attached = &rendered[start..];
if attached.is_empty() {
return None;
}
Some(OsString::from(
attached.strip_prefix('=').unwrap_or(attached),
))
}
fn parse_compression_parameters(value: &str, cli: &mut Cli) -> Result<(), String> {
for item in value.split(',') {
let Some((name, raw)) = item.split_once('=') else {
return Err(format!("invalid --zstd parameter {item:?}"));
};
let parsed = parse_u32(raw, name)?;
match name {
"windowLog" | "wlog" => cli.compression_params.windowLog = parsed,
"chainLog" | "clog" => cli.compression_params.chainLog = parsed,
"hashLog" | "hlog" => cli.compression_params.hashLog = parsed,
"searchLog" | "slog" => cli.compression_params.searchLog = parsed,
"minMatch" | "mml" => cli.compression_params.minMatch = parsed,
"targetLength" | "tlen" => cli.compression_params.targetLength = parsed,
"strategy" | "strat" => cli.compression_params.strategy = parsed as c_int,
"overlapLog" | "ovlog" => cli.overlap_log = Some(parsed as i32),
"ldmHashLog" | "lhlog" => cli.ldm_hash_log = Some(parsed as i32),
"ldmMinMatch" | "lmml" => cli.ldm_min_match = Some(parsed as i32),
"ldmBucketSizeLog" | "lblog" => cli.ldm_bucket_size_log = Some(parsed as i32),
"ldmHashRateLog" | "lhrlog" => cli.ldm_hash_rate_log = Some(parsed as i32),
_ => return Err(format!("unknown --zstd parameter {name:?}")),
}
}
Ok(())
}
fn parse_adapt(value: &str, cli: &mut Cli) -> Result<(), String> {
cli.adapt = true;
if value.is_empty() {
return Ok(());
}
for item in value.split(',') {
let Some((name, raw)) = item.split_once('=') else {
return Err(format!("invalid --adapt parameter {item:?}"));
};
match name {
"min" => cli.adapt_min = Some(parse_i32(raw, "adapt minimum")?),
"max" => cli.adapt_max = Some(parse_i32(raw, "adapt maximum")?),
_ => return Err(format!("unknown --adapt parameter {name:?}")),
}
}
if let (Some(minimum), Some(maximum)) = (cli.adapt_min, cli.adapt_max) {
if minimum > maximum {
return Err("--adapt minimum must not exceed its maximum".to_owned());
}
}
Ok(())
}
fn default_fast_cover_params() -> ZDICT_fastCover_params_t {
ZDICT_fastCover_params_t {
d: 8,
f: 20,
steps: 4,
splitPoint: 0.75,
accel: DEFAULT_FASTCOVER_ACCEL,
shrinkDictMaxRegression: DEFAULT_SHRINK_DICT_REGRESSION,
..ZDICT_fastCover_params_t::default()
}
}
fn parse_training_u32(value: &str, option: &str, field: &str) -> Result<u32, String> {
parse_u32(value, &format!("{option} {field}"))
}
fn parse_cover_parameters(value: &str) -> Result<ZDICT_cover_params_t, String> {
let mut params = ZDICT_cover_params_t::default();
for item in value.split(',') {
if item == "shrink" {
params.shrinkDict = 1;
params.shrinkDictMaxRegression = DEFAULT_SHRINK_DICT_REGRESSION;
continue;
}
if let Some(raw) = item.strip_prefix("shrink=") {
params.shrinkDict = 1;
params.shrinkDictMaxRegression =
parse_training_u32(raw, "--train-cover", "shrink regression")?;
continue;
}
let Some((field, raw)) = item.split_once('=') else {
return Err(format!("invalid --train-cover parameter {item:?}"));
};
let parsed = parse_training_u32(raw, "--train-cover", field)?;
match field {
"k" => params.k = parsed,
"d" => params.d = parsed,
"steps" => params.steps = parsed,
"split" => params.splitPoint = f64::from(parsed) / 100.0,
_ => return Err(format!("unknown --train-cover parameter {field:?}")),
}
}
Ok(params)
}
fn parse_fast_cover_parameters(value: &str) -> Result<ZDICT_fastCover_params_t, String> {
let mut params = ZDICT_fastCover_params_t::default();
for item in value.split(',') {
if item == "shrink" {
params.shrinkDict = 1;
params.shrinkDictMaxRegression = DEFAULT_SHRINK_DICT_REGRESSION;
continue;
}
if let Some(raw) = item.strip_prefix("shrink=") {
params.shrinkDict = 1;
params.shrinkDictMaxRegression =
parse_training_u32(raw, "--train-fastcover", "shrink regression")?;
continue;
}
let Some((field, raw)) = item.split_once('=') else {
return Err(format!("invalid --train-fastcover parameter {item:?}"));
};
let parsed = parse_training_u32(raw, "--train-fastcover", field)?;
match field {
"k" => params.k = parsed,
"d" => params.d = parsed,
"f" => params.f = parsed,
"steps" => params.steps = parsed,
"split" => params.splitPoint = f64::from(parsed) / 100.0,
"accel" => params.accel = parsed,
_ => return Err(format!("unknown --train-fastcover parameter {field:?}")),
}
}
Ok(params)
}
fn parse_legacy_parameters(value: &str) -> Result<u32, String> {
let Some(raw) = value
.strip_prefix("s=")
.or_else(|| value.strip_prefix("selectivity="))
else {
return Err(format!("invalid --train-legacy parameter {value:?}"));
};
parse_training_u32(raw, "--train-legacy", "selectivity")
}
fn select_training(cli: &mut Cli, algorithm: TrainingAlgorithm) -> Result<(), String> {
cli.operation = Operation::Train;
cli.training_algorithm = algorithm;
if cli.output.is_none() {
cli.output = Some(cstring(DEFAULT_DICT_NAME)?);
}
Ok(())
}
fn unsupported(option: &str) -> Result<(), String> {
Err(format!(
"{option} is not yet implemented by the Rust CLI frontend"
))
}
fn parse_long_option(
option: &str,
args: &[OsString],
index: &mut usize,
cli: &mut Cli,
) -> Result<Option<Action>, String> {
let (name, attached) = option
.split_once('=')
.map_or((option, None), |(name, value)| (name, Some(value)));
if attached.is_some()
&& matches!(
name,
"--compress"
| "--decompress"
| "--uncompress"
| "--test"
| "--force"
| "--keep"
| "--rm"
| "--stdout"
| "--version"
| "--help"
| "--verbose"
| "--quiet"
| "--check"
| "--no-check"
| "--sparse"
| "--no-sparse"
| "--pass-through"
| "--no-pass-through"
| "--content-size"
| "--no-content-size"
| "--no-dictID"
| "--asyncio"
| "--no-asyncio"
| "--mmap-dict"
| "--no-mmap-dict"
| "--progress"
| "--no-progress"
| "--ultra"
| "--no-row-match-finder"
| "--row-match-finder"
| "--rsyncable"
| "--single-thread"
| "--compress-literals"
| "--no-compress-literals"
| "--exclude-compressed"
| "--no-name"
| "--list"
| "--show-default-cparams"
| "--train"
)
{
return Err(format!("{name} does not take an argument"));
}
match name {
"--" => Ok(None),
"--compress" => {
cli.operation = Operation::Compress;
Ok(None)
}
"--decompress" | "--uncompress" => {
cli.operation = Operation::Decompress;
Ok(None)
}
"--test" => {
cli.operation = Operation::Test;
Ok(None)
}
"--force" => {
cli.force = true;
Ok(None)
}
"--keep" => {
cli.remove_source = false;
Ok(None)
}
"--no-name" => Ok(None),
"--rm" => {
cli.remove_source = true;
Ok(None)
}
"--stdout" => {
cli.output = Some(cstring(STDOUT_MARK)?);
cli.force_stdout = true;
Ok(None)
}
"--version" => Ok(Some(Action::Version {
quiet: cli.display_level < 2,
})),
"--help" => Ok(Some(Action::Help { advanced: true })),
"--verbose" => {
cli.display_level += 1;
Ok(None)
}
"--quiet" => {
cli.display_level -= 1;
Ok(None)
}
"--check" => {
cli.checksum = Some(2);
Ok(None)
}
"--no-check" => {
cli.checksum = Some(0);
Ok(None)
}
"--sparse" => {
cli.sparse = Some(2);
Ok(None)
}
"--no-sparse" => {
cli.sparse = Some(0);
Ok(None)
}
"--pass-through" => {
cli.pass_through = Some(1);
Ok(None)
}
"--no-pass-through" => {
cli.pass_through = Some(0);
Ok(None)
}
"--content-size" => {
cli.content_size = 1;
Ok(None)
}
"--no-content-size" => {
cli.content_size = 0;
Ok(None)
}
"--no-dictID" => {
cli.dict_id = Some(0);
Ok(None)
}
"--asyncio" => {
cli.async_io = Some(1);
Ok(None)
}
"--no-asyncio" => {
cli.async_io = Some(0);
Ok(None)
}
"--mmap-dict" => {
cli.mmap_dict = ZSTD_PS_ENABLE;
Ok(None)
}
"--no-mmap-dict" => {
cli.mmap_dict = ZSTD_PS_DISABLE;
Ok(None)
}
"--progress" => {
cli.progress = FIO_PS_ALWAYS;
Ok(None)
}
"--no-progress" => {
cli.progress = FIO_PS_NEVER;
Ok(None)
}
"--ultra" => {
cli.ultra = true;
Ok(None)
}
"--fast" => {
let mut level = match attached {
Some(value) => parse_u32(value, "fast level")?,
None => 1,
};
if level == 0 {
return Err("fast level must be positive".to_owned());
}
level = level.min(MAX_FAST_ACCELERATION as u32);
cli.level = -(level as i32);
Ok(None)
}
"--long" => {
cli.ldm = true;
cli.ultra = true;
if let Some(value) = attached {
cli.compression_params.windowLog = parse_u32(value, "long window log")?;
} else if cli.compression_params.windowLog == 0 {
cli.compression_params.windowLog = DEFAULT_LONG_WINDOW_LOG;
}
Ok(None)
}
"--adapt" => {
parse_adapt(attached.unwrap_or(""), cli)?;
Ok(None)
}
"--no-row-match-finder" => {
cli.row_match_finder = ZSTD_PS_DISABLE;
Ok(None)
}
"--row-match-finder" => {
cli.row_match_finder = ZSTD_PS_ENABLE;
Ok(None)
}
"--rsyncable" => {
cli.rsyncable = true;
Ok(None)
}
"--compress-literals" => {
cli.literal_compression = Some(ZSTD_PS_ENABLE);
Ok(None)
}
"--no-compress-literals" => {
cli.literal_compression = Some(ZSTD_PS_DISABLE);
Ok(None)
}
"--exclude-compressed" => {
cli.exclude_compressed = true;
Ok(None)
}
"--threads" => {
let value = next_value(attached, args, index, "--threads")?;
cli.workers = Some(parse_worker_count(&value)?);
Ok(None)
}
"--single-thread" => {
/* As in the C CLI: zero workers plus a latch that suppresses the
* automatic core-count resolution, so fileio runs its
* single-thread streaming mode (slightly different from -T1). */
cli.workers = Some(0);
cli.single_thread = true;
Ok(None)
}
"--memlimit" | "--memory" | "--memlimit-decompress" => {
let value = next_value(attached, args, index, name)?;
cli.mem_limit = Some(parse_u32(&value, "memory limit")?);
Ok(None)
}
"--block-size" => {
let value = next_value(attached, args, index, name)?;
cli.block_size = Some(parse_size(&value)?);
Ok(None)
}
"--stream-size" => {
let value = next_value(attached, args, index, name)?;
cli.stream_src_size = Some(parse_size(&value)?);
Ok(None)
}
"--target-compressed-block-size" => {
let value = next_value(attached, args, index, name)?;
cli.target_cblock_size = Some(parse_size(&value)?);
Ok(None)
}
"--size-hint" => {
let value = next_value(attached, args, index, name)?;
cli.src_size_hint = Some(parse_size(&value)?);
Ok(None)
}
"--zstd" => {
let value = next_value(attached, args, index, "--zstd")?;
parse_compression_parameters(&value, cli)?;
Ok(None)
}
"--list" => {
cli.operation = Operation::List;
Ok(None)
}
"--show-default-cparams" => {
cli.show_default_cparams = true;
Ok(None)
}
"--train" => {
select_training(cli, cli.training_algorithm)?;
Ok(None)
}
"--train-cover" => {
let params = attached.map_or_else(
|| Ok(ZDICT_cover_params_t::default()),
parse_cover_parameters,
)?;
select_training(cli, TrainingAlgorithm::Cover)?;
cli.cover_params = params;
Ok(None)
}
"--train-fastcover" => {
let params = attached.map_or_else(
|| Ok(ZDICT_fastCover_params_t::default()),
parse_fast_cover_parameters,
)?;
select_training(cli, TrainingAlgorithm::FastCover)?;
cli.fast_cover_params = params;
Ok(None)
}
"--train-legacy" => {
let selectivity =
attached.map_or_else(|| Ok(cli.dictionary_selectivity), parse_legacy_parameters)?;
select_training(cli, TrainingAlgorithm::Legacy)?;
cli.dictionary_selectivity = selectivity;
Ok(None)
}
"--maxdict" => {
let value = next_field(attached, args, index, name)?;
cli.max_dict_size = parse_u32(&value, "maximum dictionary size")? as usize;
Ok(None)
}
"--dictID" => {
let value = next_field(attached, args, index, name)?;
cli.dictionary_id = parse_u32(&value, "dictionary ID")?;
Ok(None)
}
"--filelist" => {
let value = next_field(attached, args, index, name)?;
cli.file_lists.push(cstring(&value)?);
Ok(None)
}
"--output-dir-flat" => {
let value = next_field(attached, args, index, name)?;
if value.is_empty() {
return Err(
"output dir cannot be empty string (did you mean to pass '.' instead?)"
.to_owned(),
);
}
cli.output_dir_flat = Some(cstring(&value)?);
Ok(None)
}
#[cfg(unix)]
"--output-dir-mirror" => {
/* Parsed only where the C CLI defines UTIL_HAS_MIRRORFILELIST
* (POSIX); elsewhere the option falls through as unknown. */
let value = next_field(attached, args, index, name)?;
if value.is_empty() {
return Err(
"output dir cannot be empty string (did you mean to pass '.' instead?)"
.to_owned(),
);
}
cli.output_dir_mirror = Some(cstring(&value)?);
Ok(None)
}
"--max"
| "--patch-from"
| "--trace"
| "--format"
| "--priority"
| "--auto-threads"
| "--fake-stdin-is-console"
| "--fake-stdout-is-console"
| "--fake-stderr-is-console"
| "--trace-file-stat" => {
unsupported(name)?;
Ok(None)
}
_ => Err(format!("unknown option {option:?}")),
}
}
fn parse_short_options(
value: &str,
raw_value: &OsStr,
args: &[OsString],
index: &mut usize,
cli: &mut Cli,
) -> Result<Option<Action>, String> {
let mut offset = 1usize;
let bytes = value.as_bytes();
while offset < bytes.len() {
let option = bytes[offset] as char;
match option {
'0'..='9' => {
let mut digits_end = offset;
while digits_end < bytes.len() && bytes[digits_end].is_ascii_digit() {
digits_end += 1;
}
cli.level = parse_compression_level(&value[offset..digits_end])?;
offset = digits_end;
continue;
}
'd' => cli.operation = Operation::Decompress,
'z' => cli.operation = Operation::Compress,
't' => cli.operation = Operation::Test,
'b' => cli.operation = Operation::Bench,
'l' => cli.operation = Operation::List,
#[cfg(any(unix, windows))]
'r' => cli.recursive = true,
'e' | 'i' => {
// Benchmark range end (-e#) and duration (-i#): like the C
// parser, digits attach directly and default to 0.
let mut digits_end = offset + 1;
while digits_end < bytes.len() && bytes[digits_end].is_ascii_digit() {
digits_end += 1;
}
let digits = &value[offset + 1..digits_end];
if option == 'e' {
cli.bench_end_level = Some(if digits.is_empty() {
0
} else {
parse_i32(digits, "benchmark end level")?
});
} else {
cli.bench_nb_seconds = Some(if digits.is_empty() {
0
} else {
digits
.parse::<u32>()
.map_err(|_| format!("invalid benchmark duration: {digits:?}"))?
});
}
offset = digits_end;
continue;
}
'c' => {
cli.output = Some(cstring(STDOUT_MARK)?);
cli.force_stdout = true;
}
'f' => cli.force = true,
'k' => cli.remove_source = false,
'n' => {}
'q' => cli.display_level -= 1,
'v' => cli.display_level += 1,
'C' => cli.checksum = Some(2),
'h' => return Ok(Some(Action::Help { advanced: false })),
'H' => return Ok(Some(Action::Help { advanced: true })),
'V' => {
return Ok(Some(Action::Version {
quiet: cli.display_level < 2,
}))
}
'o' | 'D' | 'T' | 'M' | 'B' => {
let attached = short_attached_value(raw_value, offset + 1);
let argument = attached
.map(Ok)
.unwrap_or_else(|| next_os_value(args, index, &format!("-{option}")))?;
match option {
'o' => cli.output = Some(os_cstring(&argument)?),
'D' => cli.dictionary = Some(os_cstring(&argument)?),
'T' => cli.workers = Some(parse_worker_count(&argument.to_string_lossy())?),
'M' => {
cli.mem_limit =
Some(parse_u32(&argument.to_string_lossy(), "memory limit")?)
}
'B' => cli.block_size = Some(parse_size(&argument.to_string_lossy())?),
_ => unreachable!(),
}
break;
}
'p' | 'P' | 's' | 'S' => {
unsupported(&format!("-{option}"))?;
}
_ => return Err(format!("unknown option -{option}")),
}
offset += 1;
}
Ok(None)
}
fn parse_args(args: Vec<OsString>) -> Result<Action, String> {
let program_name = args
.first()
.map_or_else(|| "zstd".to_owned(), |value| program_basename(value));
let mut cli = Cli::new(&program_name);
let mut end_of_options = false;
let mut index = 1usize;
while index < args.len() {
let rendered = args[index].to_string_lossy().into_owned();
if end_of_options {
cli.inputs.push(os_cstring(&args[index])?);
} else if rendered == "--" {
end_of_options = true;
} else if rendered == "-" {
cli.inputs.push(cstring(STDIN_MARK)?);
} else if rendered.starts_with("--") {
if let Some(action) = parse_long_option(&rendered, &args, &mut index, &mut cli)? {
return Ok(action);
}
} else if rendered.starts_with('-') {
if let Some(action) =
parse_short_options(&rendered, &args[index], &args, &mut index, &mut cli)?
{
return Ok(action);
}
} else {
cli.inputs.push(os_cstring(&args[index])?);
}
index += 1;
}
Ok(Action::Run(Box::new(cli)))
}
unsafe fn apply_preferences(cli: &Cli, prefs: *mut FIO_prefs_t, ctx: *mut FIO_ctx_t) {
let display_level = if is_stdout(cli.output.as_ref()) && cli.display_level == 2 {
1
} else {
cli.display_level
};
unsafe {
g_utilDisplayLevel = display_level;
FIO_setCompressionType(prefs, FIO_ZSTD_COMPRESSION);
FIO_setNotificationLevel(display_level);
FIO_setProgressSetting(
if !io::stderr().is_terminal() && cli.progress != FIO_PS_ALWAYS {
FIO_PS_NEVER
} else {
cli.progress
},
);
FIO_setRemoveSrcFile(
prefs,
i32::from(
cli.remove_source
&& cli.operation != Operation::Test
&& !is_stdout(cli.output.as_ref()),
),
);
FIO_setAllowBlockDevices(prefs, i32::from(cli.force));
FIO_setMMapDict(prefs, cli.mmap_dict);
FIO_setUseRowMatchFinder(prefs, cli.row_match_finder);
FIO_setMemLimit(
prefs,
cli.mem_limit
.filter(|limit| *limit != 0)
.unwrap_or_else(|| {
if cli.compression_params.windowLog == 0 {
DEFAULT_MEM_LIMIT
} else {
1_u32 << (cli.compression_params.windowLog & 31)
}
}),
);
#[cfg(feature = "compression")]
FIO_setNbWorkers(prefs, resolved_worker_count(cli.workers, cli.single_thread));
FIO_setLdmFlag(prefs, u32::from(cli.ldm));
FIO_setAdaptiveMode(prefs, i32::from(cli.adapt));
FIO_setRsyncable(prefs, i32::from(cli.rsyncable));
FIO_setExcludeCompressedFile(prefs, i32::from(cli.exclude_compressed));
if cli.force {
FIO_overwriteMode(prefs);
}
if let Some(value) = cli.checksum {
FIO_setChecksumFlag(prefs, value);
}
if cli.operation == Operation::Compress {
FIO_setSparseWrite(prefs, 0);
} else if let Some(value) = cli.sparse {
FIO_setSparseWrite(prefs, value);
}
if let Some(value) = cli.pass_through {
FIO_setPassThroughFlag(prefs, value);
}
/* `FIO_createPreferences()` leaves this field unspecified. The Rust
* frontend does not yet parse --patch-from, so explicitly keep the
* C file-I/O backend in ordinary dictionary mode. */
FIO_setPatchFromMode(prefs, 0);
FIO_setContentSize(prefs, cli.content_size);
if let Some(value) = cli.dict_id {
FIO_setDictIDFlag(prefs, value);
}
if let Some(value) = cli.async_io {
FIO_setAsyncIOFlag(prefs, value);
}
if let Some(value) = cli.block_size {
FIO_setBlockSize(prefs, value as c_int);
}
if let Some(value) = cli.ldm_hash_log {
FIO_setLdmHashLog(prefs, value);
}
if let Some(value) = cli.ldm_min_match {
FIO_setLdmMinMatch(prefs, value);
}
if let Some(value) = cli.ldm_bucket_size_log {
FIO_setLdmBucketSizeLog(prefs, value);
}
if let Some(value) = cli.ldm_hash_rate_log {
FIO_setLdmHashRateLog(prefs, value);
}
if let Some(value) = cli.overlap_log {
FIO_setOverlapLog(prefs, value);
}
#[cfg(feature = "compression")]
{
FIO_setAdaptMin(prefs, cli.adapt_min.unwrap_or_else(|| ZSTD_minCLevel()));
FIO_setAdaptMax(prefs, cli.adapt_max.unwrap_or_else(|| ZSTD_maxCLevel()));
}
if let Some(value) = cli.stream_src_size {
FIO_setStreamSrcSize(prefs, value);
}
if let Some(value) = cli.target_cblock_size {
FIO_setTargetCBlockSize(prefs, value);
}
if let Some(value) = cli.src_size_hint {
FIO_setSrcSizeHint(prefs, value);
}
if let Some(value) = cli.literal_compression {
FIO_setLiteralCompressionMode(prefs, value);
}
FIO_setNbFilesTotal(ctx, cli.inputs.len() as c_int);
FIO_setHasStdinInput(ctx, i32::from(cli.inputs.iter().any(is_stdin)));
FIO_setHasStdoutOutput(ctx, i32::from(is_stdout(cli.output.as_ref())));
}
}
fn is_stdout(value: Option<&CString>) -> bool {
value.is_some_and(|value| value.as_bytes() == STDOUT_MARK.as_bytes())
}
fn is_stdin(value: &CString) -> bool {
value.as_bytes() == STDIN_MARK.as_bytes()
}
#[cfg(unix)]
fn is_non_fifo_symlink(input: &CString) -> bool {
let path = Path::new(OsStr::from_bytes(input.as_bytes()));
let Ok(metadata) = fs::symlink_metadata(path) else {
return false;
};
metadata.file_type().is_symlink()
&& !fs::metadata(path).is_ok_and(|target| target.file_type().is_fifo())
}
#[cfg(not(unix))]
fn is_non_fifo_symlink(input: &CString) -> bool {
let path = Path::new(&input.to_string_lossy().into_owned());
fs::symlink_metadata(path).is_ok_and(|metadata| metadata.file_type().is_symlink())
}
fn filter_symlink_inputs(cli: &mut Cli) {
if cli.force {
return;
}
cli.inputs.retain(|input| {
if is_stdin(input) || !is_non_fifo_symlink(input) {
return true;
}
if cli.display_level >= 2 {
eprintln!(
"zstd: Warning : {} is a symbolic link, ignoring",
input.to_string_lossy()
);
}
false
});
}
fn check_terminal_safety(cli: &Cli) -> Result<(), String> {
let has_stdin = cli.inputs.iter().any(is_stdin);
if has_stdin && !cli.force && io::stdin().is_terminal() {
return Err("stdin is a console, aborting".to_owned());
}
if has_stdin
&& is_stdout(cli.output.as_ref())
&& !cli.force
&& !cli.force_stdout
&& cli.operation != Operation::Decompress
&& io::stdout().is_terminal()
{
return Err("stdout is a console, aborting".to_owned());
}
Ok(())
}
#[cfg(feature = "compression")]
unsafe fn print_default_cparams(cli: &Cli) {
let dict_size = cli.dictionary.as_ref().map_or(0, |dictionary| unsafe {
UTIL_getFileSize(dictionary.as_ptr()) as usize
});
for input in &cli.inputs {
let file_size = unsafe { UTIL_getFileSize(input.as_ptr()) };
let cparams = unsafe { ZSTD_getCParams(cli.level, file_size, dict_size) };
let strategy = ZSTD_STRATEGY_NAMES
.get(cparams.strategy as usize)
.copied()
.unwrap_or("");
if file_size != u64::MAX {
eprintln!("{} ({} bytes)", input.to_string_lossy(), file_size);
} else {
eprintln!("{} (src size unknown)", input.to_string_lossy());
}
eprintln!(" - windowLog : {}", cparams.windowLog);
eprintln!(" - chainLog : {}", cparams.chainLog);
eprintln!(" - hashLog : {}", cparams.hashLog);
eprintln!(" - searchLog : {}", cparams.searchLog);
eprintln!(" - minMatch : {}", cparams.minMatch);
eprintln!(" - targetLength : {}", cparams.targetLength);
eprintln!(" - strategy : {strategy} ({})", cparams.strategy);
}
}
#[cfg(feature = "compression")]
unsafe fn print_actual_cparams(cli: &Cli) {
let dict_size = cli.dictionary.as_ref().map_or(0, |dictionary| unsafe {
UTIL_getFileSize(dictionary.as_ptr()) as usize
});
for input in &cli.inputs {
let file_size = unsafe { UTIL_getFileSize(input.as_ptr()) };
let mut actual = unsafe { ZSTD_getCParams(cli.level, file_size, dict_size) };
let requested = cli.compression_params;
if requested.windowLog != 0 {
actual.windowLog = requested.windowLog;
}
if requested.chainLog != 0 {
actual.chainLog = requested.chainLog;
}
if requested.hashLog != 0 {
actual.hashLog = requested.hashLog;
}
if requested.searchLog != 0 {
actual.searchLog = requested.searchLog;
}
if requested.minMatch != 0 {
actual.minMatch = requested.minMatch;
}
if requested.targetLength != 0 {
actual.targetLength = requested.targetLength;
}
if requested.strategy != 0 {
actual.strategy = requested.strategy;
}
eprintln!(
"--zstd=wlog={},clog={},hlog={},slog={},mml={},tlen={},strat={}",
actual.windowLog,
actual.chainLog,
actual.hashLog,
actual.searchLog,
actual.minMatch,
actual.targetLength,
actual.strategy
);
}
}
#[cfg(feature = "compression")]
unsafe fn run_compress(
cli: &Cli,
ctx: *mut FIO_ctx_t,
prefs: *mut FIO_prefs_t,
inputs: &[*const c_char],
output: *const c_char,
dictionary: *const c_char,
) -> c_int {
if inputs.len() == 1 && !output.is_null() {
unsafe {
FIO_compressFilename(
ctx,
prefs,
output,
inputs[0],
dictionary,
cli.level,
cli.compression_params,
)
}
} else {
unsafe {
FIO_compressMultipleFilenames(
ctx,
prefs,
inputs.as_ptr(),
output_dir_mirror(cli),
output_dir_flat(cli),
output,
ZSTD_SUFFIX.as_ptr().cast(),
dictionary,
cli.level,
cli.compression_params,
)
}
}
}
fn output_dir_flat(cli: &Cli) -> *const c_char {
cli.output_dir_flat
.as_ref()
.map_or(ptr::null(), |value| value.as_ptr())
}
fn output_dir_mirror(cli: &Cli) -> *const c_char {
cli.output_dir_mirror
.as_ref()
.map_or(ptr::null(), |value| value.as_ptr())
}
#[cfg(feature = "decompression")]
unsafe fn run_decompress(
cli: &Cli,
ctx: *mut FIO_ctx_t,
prefs: *mut FIO_prefs_t,
inputs: &[*const c_char],
output: *const c_char,
dictionary: *const c_char,
) -> c_int {
match cli.operation {
Operation::Test => {
let null_output = cstring(NULL_MARK).expect("static null marker");
unsafe {
FIO_setTestMode(prefs, 1);
FIO_decompressMultipleFilenames(
ctx,
prefs,
inputs.as_ptr(),
output_dir_mirror(cli),
output_dir_flat(cli),
null_output.as_ptr(),
dictionary,
)
}
}
Operation::Decompress if inputs.len() == 1 && !output.is_null() => unsafe {
FIO_decompressFilename(ctx, prefs, output, inputs[0], dictionary)
},
Operation::Decompress => unsafe {
FIO_decompressMultipleFilenames(
ctx,
prefs,
inputs.as_ptr(),
output_dir_mirror(cli),
output_dir_flat(cli),
output,
dictionary,
)
},
Operation::Compress | Operation::Bench | Operation::List | Operation::Train => {
unreachable!("compression, benchmark, and list are dispatched separately")
}
}
}
/// Copies every name out of a C `FileNamesTable` and releases the table.
unsafe fn drain_file_names_table(table: *mut FileNamesTable, into: &mut Vec<CString>) {
unsafe {
for entry in 0..(*table).tableSize {
let name = *(*table).fileNames.add(entry);
if !name.is_null() {
into.push(CStr::from_ptr(name).to_owned());
}
}
UTIL_freeFileNamesTable(table);
}
}
/// Appends the names read from every `--filelist` argument to the inputs.
/// As in the C CLI, a list that cannot be read (missing, irregular, empty,
/// or with over-long lines) aborts the whole run.
fn expand_file_lists(cli: &mut Cli) -> Result<(), String> {
let file_lists = std::mem::take(&mut cli.file_lists);
for list in &file_lists {
let table = unsafe { UTIL_createFileNamesTable_fromFileName(list.as_ptr()) };
if table.is_null() {
return Err(format!("error reading {}", list.to_string_lossy()));
}
unsafe { drain_file_names_table(table, &mut cli.inputs) };
}
Ok(())
}
/// Replaces the inputs with their recursive directory expansion. Symbolic
/// links are followed only under --force, matching the C `followLinks` flag.
fn expand_recursive_inputs(cli: &mut Cli) -> Result<(), String> {
let pointers: Vec<*const c_char> = cli.inputs.iter().map(|value| value.as_ptr()).collect();
let names = if pointers.is_empty() {
ptr::null()
} else {
pointers.as_ptr()
};
let table = unsafe { UTIL_createExpandedFNT(names, pointers.len(), c_int::from(cli.force)) };
if table.is_null() {
/* The C CLI treats this allocation failure as fatal (CONTROL). */
return Err("recursive directory expansion failed".to_owned());
}
let mut expanded = Vec::new();
unsafe { drain_file_names_table(table, &mut expanded) };
cli.inputs = expanded;
Ok(())
}
/// Prints frame information for every input through `FIO_listMultipleFiles`,
/// which owns the display-level gating, the stdin refusal, and the totals.
#[cfg(feature = "decompression")]
fn run_list(cli: &Cli) -> i32 {
let inputs: Vec<*const c_char> = cli.inputs.iter().map(|value| value.as_ptr()).collect();
unsafe { FIO_listMultipleFiles(inputs.len() as c_uint, inputs.as_ptr(), cli.display_level) }
}
/// Runs benchmark mode through the C bridge. Level clamping against
/// `ZSTD_maxCLevel()` happens on the C side, where the symbol is always
/// available when benchmarking is compiled in. No input file means a
/// synthetic-sample benchmark, matching the C CLI.
fn run_bench(cli: &Cli) -> Result<i32, String> {
let inputs: Vec<*const c_char> = cli.inputs.iter().map(|value| value.as_ptr()).collect();
let dictionary = cli
.dictionary
.as_ref()
.map_or(ptr::null(), |value| value.as_ptr());
let result = unsafe {
ZSTD_rust_cli_bench(
inputs.as_ptr(),
inputs.len() as c_uint,
dictionary,
cli.level,
cli.bench_end_level.unwrap_or(cli.level),
&cli.compression_params,
cli.display_level,
cli.bench_nb_seconds.unwrap_or(DEFAULT_BENCH_NB_SECONDS),
cli.block_size.unwrap_or(0),
// The C CLI benchmarks single-threaded unless -T was given.
cli.workers.unwrap_or(1),
)
};
if result < 0 {
return Err("benchmark mode is not available in this build".to_owned());
}
Ok(result)
}
/// Runs dictionary training through the narrow `programs/dibio.c` ABI. The
/// file loading and output path stay in that bridge for now; the selected
/// dictionary builder symbols are supplied by the Rust library archive.
fn run_train(cli: &Cli) -> Result<i32, String> {
#[cfg(not(feature = "compression"))]
{
let _ = cli;
return Err("training mode not available".to_owned());
}
#[cfg(feature = "compression")]
{
let default_output = cstring(DEFAULT_DICT_NAME).expect("static dictionary name");
let output = cli.output.as_ref().unwrap_or(&default_output);
let inputs: Vec<*const c_char> = cli.inputs.iter().map(|value| value.as_ptr()).collect();
let workers = unsafe { resolved_worker_count(cli.workers, cli.single_thread) } as c_uint;
let z_params = ZDICT_params_t {
compressionLevel: cli.level,
notificationLevel: cli.display_level as c_uint,
dictID: cli.dictionary_id,
};
let chunk_size = cli.block_size.unwrap_or(0);
let mem_limit = cli.mem_limit.unwrap_or(0);
let result = match cli.training_algorithm {
TrainingAlgorithm::Cover => {
let mut params = cli.cover_params;
params.nbThreads = workers;
params.zParams = z_params;
let optimize = c_int::from(params.k == 0 || params.d == 0);
unsafe {
DiB_trainFromFiles(
output.as_ptr(),
cli.max_dict_size,
inputs.as_ptr(),
inputs.len() as c_int,
chunk_size,
ptr::null_mut(),
&mut params,
ptr::null_mut(),
optimize,
mem_limit,
)
}
}
TrainingAlgorithm::FastCover => {
let mut params = cli.fast_cover_params;
params.nbThreads = workers;
params.zParams = z_params;
let optimize = c_int::from(params.k == 0 || params.d == 0);
unsafe {
DiB_trainFromFiles(
output.as_ptr(),
cli.max_dict_size,
inputs.as_ptr(),
inputs.len() as c_int,
chunk_size,
ptr::null_mut(),
ptr::null_mut(),
&mut params,
optimize,
mem_limit,
)
}
}
TrainingAlgorithm::Legacy => {
let mut params = ZDICT_legacy_params_t {
selectivityLevel: cli.dictionary_selectivity,
zParams: z_params,
};
unsafe {
DiB_trainFromFiles(
output.as_ptr(),
cli.max_dict_size,
inputs.as_ptr(),
inputs.len() as c_int,
chunk_size,
&mut params,
ptr::null_mut(),
ptr::null_mut(),
0,
mem_limit,
)
}
}
};
Ok(result)
}
}
fn run_cli(mut cli: Cli) -> Result<i32, String> {
if let Some(program_name) = &cli.unsupported_program {
return Err(format!(
"{program_name} compatibility mode is not yet implemented by the Rust CLI frontend"
));
}
unsafe {
/* The C CLI publishes the display level to util.c before any table
* expansion, so traversal warnings obey -q/-v. */
g_utilDisplayLevel = cli.display_level;
}
let explicit_input_count = cli.inputs.len();
filter_symlink_inputs(&mut cli);
if explicit_input_count > 0 && cli.inputs.is_empty() {
return Ok(1);
}
expand_file_lists(&mut cli)?;
/* Names known before recursive expansion: an input set that named only
* (possibly empty) directories must not fall back to stdin below. */
let input_name_count = cli.inputs.len();
if cli.recursive {
expand_recursive_inputs(&mut cli)?;
}
if cli.operation == Operation::List {
#[cfg(feature = "decompression")]
return Ok(run_list(&cli));
#[cfg(not(feature = "decompression"))]
return Err("file information is not supported".to_owned());
}
if cli.operation == Operation::Bench {
return run_bench(&cli);
}
if cli.operation == Operation::Train {
return run_train(&cli);
}
if cli.operation == Operation::Test {
cli.output = Some(cstring(NULL_MARK)?);
cli.remove_source = false;
}
if cli.inputs.is_empty() {
if input_name_count > 0 {
if cli.display_level >= 1 {
eprintln!(
"please provide correct input file(s) or non-empty directories -- ignored"
);
}
return Ok(0);
}
cli.inputs.push(cstring(STDIN_MARK)?);
if cli.output.is_none() {
cli.output = Some(cstring(STDOUT_MARK)?);
}
}
if cli.inputs.len() == 1
&& cli.inputs[0].as_bytes() == STDIN_MARK.as_bytes()
&& cli.output.is_none()
{
cli.output = Some(cstring(STDOUT_MARK)?);
}
if cli.show_default_cparams && cli.operation == Operation::Decompress {
return Err("error : can't use --show-default-cparams in decompression mode".to_owned());
}
check_terminal_safety(&cli)?;
if cli.operation == Operation::Compress {
#[cfg(not(feature = "compression"))]
return Err("Compression not supported".to_owned());
#[cfg(feature = "compression")]
{
let min_level = unsafe { ZSTD_minCLevel() };
let max_level = unsafe { ZSTD_maxCLevel() };
let ceiling = if cli.ultra {
max_level
} else {
DEFAULT_MAX_CLEVEL.min(max_level)
};
if cli.level > ceiling {
eprintln!("zstd: warning: compression level reduced to {ceiling}");
cli.level = ceiling;
}
if cli.level < min_level {
return Err(format!(
"compression level {} is below {min_level}",
cli.level
));
}
if let (Some(minimum), Some(maximum)) = (cli.adapt_min, cli.adapt_max) {
if minimum > maximum {
return Err("adaptation minimum exceeds maximum".to_owned());
}
cli.level = cli.level.clamp(minimum, maximum);
}
}
} else {
#[cfg(not(feature = "decompression"))]
return Err("Decompression not supported".to_owned());
}
let prefs = unsafe { FIO_createPreferences() };
let ctx = unsafe { FIO_createContext() };
if prefs.is_null() || ctx.is_null() {
unsafe {
if !prefs.is_null() {
FIO_freePreferences(prefs);
}
if !ctx.is_null() {
FIO_freeContext(ctx);
}
}
return Err("could not allocate C file-I/O state".to_owned());
}
let result = {
unsafe {
FIO_addAbortHandler();
apply_preferences(&cli, prefs, ctx);
#[cfg(feature = "compression")]
if cli.operation == Operation::Compress {
if cli.show_default_cparams {
print_default_cparams(&cli);
}
if cli.display_level >= 4 {
print_actual_cparams(&cli);
}
}
}
let output = cli
.output
.as_ref()
.map_or(ptr::null(), |value| value.as_ptr());
let dictionary = cli
.dictionary
.as_ref()
.map_or(ptr::null(), |value| value.as_ptr());
let inputs: Vec<*const c_char> = cli.inputs.iter().map(|value| value.as_ptr()).collect();
match cli.operation {
Operation::Compress => {
#[cfg(feature = "compression")]
{
unsafe { run_compress(&cli, ctx, prefs, &inputs, output, dictionary) }
}
#[cfg(not(feature = "compression"))]
unreachable!("unsupported compression was rejected above")
}
Operation::Decompress | Operation::Test => {
#[cfg(feature = "decompression")]
{
unsafe { run_decompress(&cli, ctx, prefs, &inputs, output, dictionary) }
}
#[cfg(not(feature = "decompression"))]
unreachable!("unsupported decompression was rejected above")
}
Operation::Bench | Operation::List | Operation::Train => {
unreachable!("benchmark and list modes were dispatched earlier")
}
}
};
unsafe {
FIO_freePreferences(prefs);
FIO_freeContext(ctx);
}
Ok(result)
}
fn run_from_args(args: Vec<OsString>) -> c_int {
match parse_args(args) {
Ok(Action::Help { advanced }) => {
usage(advanced);
0
}
Ok(Action::Version { quiet }) => {
print_version(quiet);
0
}
Ok(Action::Run(cli)) => match run_cli(*cli) {
Ok(result) => result,
Err(error) => {
eprintln!("zstd: {error}");
1
}
},
Err(error) => {
eprintln!("zstd: {error}\nTry `zstd --help` for usage.");
1
}
}
}
unsafe fn argv_to_os_strings(
arg_count: c_int,
argv: *const *const c_char,
) -> Result<Vec<OsString>, String> {
if arg_count <= 0 || argv.is_null() {
return Err("invalid argv supplied by C main".to_owned());
}
let count = arg_count as usize;
let mut args = Vec::with_capacity(count);
for index in 0..count {
let argument = unsafe { *argv.add(index) };
if argument.is_null() {
return Err(format!("argv[{index}] is null"));
}
let bytes = unsafe { CStr::from_ptr(argument) }.to_bytes();
#[cfg(unix)]
args.push(OsString::from_vec(bytes.to_vec()));
#[cfg(not(unix))]
args.push(OsString::from(String::from_utf8_lossy(bytes).into_owned()));
}
Ok(args)
}
/// C `main()` entry point retained by the small `programs/zstdcli.c` forwarder.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_cli_main(arg_count: c_int, argv: *const *const c_char) -> c_int {
if let Err(error) = check_lib_version() {
eprintln!("zstd: {error}");
return 1;
}
match unsafe { argv_to_os_strings(arg_count, argv) } {
Ok(args) => run_from_args(args),
Err(error) => {
eprintln!("zstd: {error}");
1
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn parse(values: &[&str]) -> Cli {
let args = values.iter().map(OsString::from).collect();
match parse_args(args).expect("arguments should parse") {
Action::Run(cli) => *cli,
Action::Help { .. } | Action::Version { .. } => panic!("expected a run action"),
}
}
#[test]
fn defaults_preserve_the_c_fileio_contract() {
let cli = parse(&["zstd", "input"]);
assert_eq!(cli.content_size, 1);
assert_eq!(cli.workers, None);
assert_eq!(cli.operation, Operation::Compress);
}
#[test]
fn long_mode_uses_the_cli_default_window_and_enables_ultra() {
let cli = parse(&["zstd", "--long", "input"]);
assert!(cli.ldm);
assert!(cli.ultra);
assert_eq!(cli.compression_params.windowLog, DEFAULT_LONG_WINDOW_LOG);
}
#[test]
fn long_mode_does_not_replace_an_explicit_window_log() {
let cli = parse(&["zstd", "--zstd=wlog=25", "--long", "input"]);
assert_eq!(cli.compression_params.windowLog, 25);
}
#[test]
fn stdio_and_dictionary_short_options_are_preserved() {
let cli = parse(&["zstd", "-dc", "-D", "dict", "-"]);
assert_eq!(cli.operation, Operation::Decompress);
assert!(is_stdout(cli.output.as_ref()));
assert_eq!(
cli.dictionary.as_deref().map(CStr::to_bytes),
Some(&b"dict"[..])
);
assert_eq!(
cli.inputs
.iter()
.map(|input| input.as_bytes())
.collect::<Vec<_>>(),
vec![STDIN_MARK.as_bytes()]
);
}
#[test]
fn stdout_selection_does_not_enable_force_or_pass_through() {
let cli = parse(&["zstd", "-c", "input"]);
assert!(cli.force_stdout);
assert!(!cli.force);
assert_eq!(cli.pass_through, None);
}
#[test]
fn short_level_can_be_combined_with_other_flags() {
let cli = parse(&["zstd", "-5q", "input"]);
assert_eq!(cli.level, 5);
assert_eq!(cli.display_level, 1);
}
#[test]
fn fast_levels_preserve_the_unsigned_32_bit_conversion() {
let cli = parse(&["zstd", "--fast", "-4294967295", "input"]);
assert_eq!(cli.level, -1);
let cli = parse(&["zstd", "--fast=4294967295", "input"]);
assert_eq!(cli.level, -MAX_FAST_ACCELERATION);
}
#[test]
fn short_option_equals_form_is_accepted() {
let cli = parse(&["zstd", "-T=2", "-M=64M", "-B=1M", "input"]);
assert_eq!(cli.workers, Some(2));
assert_eq!(cli.mem_limit, Some(64 << 20));
assert_eq!(cli.block_size, Some(1 << 20));
}
#[test]
fn valueless_long_flags_reject_attached_values() {
let error = parse_args(vec![OsString::from("zstd"), OsString::from("--rm=0")])
.expect_err("an attached value must not activate --rm");
assert!(error.contains("does not take an argument"));
}
#[test]
fn zstdmt_uses_auto_threads() {
let cli = parse(&["zstdmt", "input"]);
assert_eq!(cli.workers, Some(0));
assert!(!cli.single_thread);
}
#[test]
fn single_thread_pins_zero_workers() {
let cli = parse(&["zstd", "--single-thread", "input"]);
assert_eq!(cli.workers, Some(0));
assert!(cli.single_thread);
}
#[test]
fn a_later_thread_count_overrides_single_thread_workers() {
/* Mirrors the C CLI: -T after --single-thread wins the worker count,
* while the single-thread latch stays set. */
let cli = parse(&["zstd", "--single-thread", "-T2", "input"]);
assert_eq!(cli.workers, Some(2));
assert!(cli.single_thread);
}
#[test]
fn single_thread_rejects_attached_values() {
let error = parse_args(vec![
OsString::from("zstd"),
OsString::from("--single-thread=1"),
])
.expect_err("an attached value must not activate --single-thread");
assert!(error.contains("does not take an argument"));
}
#[test]
fn alternate_format_aliases_fail_before_processing_files() {
let cli = Cli::new("gzip");
assert_eq!(cli.unsupported_program.as_deref(), Some("gzip"));
}
#[cfg(unix)]
#[test]
fn short_path_arguments_keep_non_utf8_bytes() {
let output = OsString::from_vec(vec![b'-', b'o', 0xff, b'.', b'z', b's', b't']);
let action = parse_args(vec![
OsString::from("zstd"),
output,
OsString::from("input"),
])
.expect("arguments should parse");
let Action::Run(cli) = action else {
panic!("expected a run action");
};
assert_eq!(
cli.output.as_deref().map(CStr::to_bytes),
Some(&b"\xff.zst"[..])
);
}
#[test]
fn train_defaults_to_fastcover_and_dictionary_output() {
let cli = parse(&["zstd", "--train", "sample"]);
assert_eq!(cli.operation, Operation::Train);
assert_eq!(cli.training_algorithm, TrainingAlgorithm::FastCover);
assert_eq!(
cli.output.as_deref().map(CStr::to_bytes),
Some(&b"dictionary"[..])
);
assert_eq!(cli.max_dict_size, DEFAULT_MAX_DICT_SIZE);
assert_eq!(cli.fast_cover_params.d, 8);
assert_eq!(cli.fast_cover_params.f, 20);
assert_eq!(cli.fast_cover_params.steps, 4);
assert_eq!(cli.fast_cover_params.splitPoint, 0.75);
assert_eq!(cli.fast_cover_params.accel, DEFAULT_FASTCOVER_ACCEL);
}
#[test]
fn train_cover_parameters_select_algorithm_and_parse_optional_fields() {
let cli = parse(&[
"zstd",
"--train-cover=k=48,d=8,steps=32,split=80,shrink=3",
"sample",
]);
assert_eq!(cli.operation, Operation::Train);
assert_eq!(cli.training_algorithm, TrainingAlgorithm::Cover);
assert_eq!(cli.cover_params.k, 48);
assert_eq!(cli.cover_params.d, 8);
assert_eq!(cli.cover_params.steps, 32);
assert_eq!(cli.cover_params.splitPoint, 0.8);
assert_eq!(cli.cover_params.shrinkDict, 1);
assert_eq!(cli.cover_params.shrinkDictMaxRegression, 3);
}
#[test]
fn train_fastcover_and_legacy_parameter_forms_share_training_options() {
let fast = parse(&[
"zstd",
"--train-fastcover=k=64,d=8,f=21,steps=12,split=75,accel=2,shrink",
"--maxdict",
"128K",
"--dictID=42",
"-B4K",
"-T2",
"-o",
"custom.dict",
"sample",
]);
assert_eq!(fast.training_algorithm, TrainingAlgorithm::FastCover);
assert_eq!(fast.fast_cover_params.k, 64);
assert_eq!(fast.fast_cover_params.d, 8);
assert_eq!(fast.fast_cover_params.f, 21);
assert_eq!(fast.fast_cover_params.steps, 12);
assert_eq!(fast.fast_cover_params.splitPoint, 0.75);
assert_eq!(fast.fast_cover_params.accel, 2);
assert_eq!(fast.fast_cover_params.shrinkDict, 1);
assert_eq!(fast.fast_cover_params.shrinkDictMaxRegression, 1);
assert_eq!(fast.max_dict_size, 128 << 10);
assert_eq!(fast.dictionary_id, 42);
assert_eq!(fast.block_size, Some(4 << 10));
assert_eq!(fast.workers, Some(2));
assert_eq!(
fast.output.as_deref().map(CStr::to_bytes),
Some(&b"custom.dict"[..])
);
let legacy = parse(&["zstd", "--train-legacy=selectivity=12", "sample"]);
assert_eq!(legacy.operation, Operation::Train);
assert_eq!(legacy.training_algorithm, TrainingAlgorithm::Legacy);
assert_eq!(legacy.dictionary_selectivity, 12);
}
#[test]
fn training_options_preserve_upstream_validation_boundaries() {
let attached = parse_args(vec![
OsString::from("zstd"),
OsString::from("--train=value"),
])
.expect_err("--train does not accept an attached value");
assert!(attached.contains("does not take an argument"));
let unknown = parse_args(vec![
OsString::from("zstd"),
OsString::from("--train-cover=unknown=1"),
])
.expect_err("unknown cover parameters must be rejected");
assert!(unknown.contains("unknown --train-cover parameter"));
let missing = parse_args(vec![OsString::from("zstd"), OsString::from("--dictID")])
.expect_err("--dictID requires a value");
assert!(missing.contains("missing argument for --dictID"));
}
#[test]
fn recursive_flag_can_be_aggregated_with_other_shorts() {
let cli = parse(&["zstd", "-rq", "dir"]);
assert!(cli.recursive);
assert_eq!(cli.display_level, 1);
}
#[test]
fn list_mode_is_selected_by_short_and_long_flags() {
let short = parse(&["zstd", "-l", "file.zst"]);
let long = parse(&["zstd", "--list", "file.zst"]);
assert_eq!(short.operation, Operation::List);
assert_eq!(long.operation, Operation::List);
}
#[test]
fn list_rejects_attached_values() {
let error = parse_args(vec![OsString::from("zstd"), OsString::from("--list=x")])
.expect_err("an attached value must not activate --list");
assert!(error.contains("does not take an argument"));
}
#[test]
fn show_default_cparams_is_a_compression_information_flag() {
let cli = parse(&["zstd", "--show-default-cparams", "input"]);
assert!(cli.show_default_cparams);
assert_eq!(cli.operation, Operation::Compress);
}
#[test]
fn show_default_cparams_rejects_attached_values() {
let error = parse_args(vec![
OsString::from("zstd"),
OsString::from("--show-default-cparams=x"),
])
.expect_err("show-default-cparams must not consume an attached value");
assert!(error.contains("does not take an argument"));
}
#[test]
fn filelist_accumulates_both_syntaxes() {
let cli = parse(&["zstd", "--filelist=one.txt", "--filelist", "two.txt", "in"]);
assert_eq!(
cli.file_lists
.iter()
.map(|list| list.as_bytes())
.collect::<Vec<_>>(),
vec![&b"one.txt"[..], &b"two.txt"[..]]
);
}
#[test]
fn filelist_argument_must_not_be_another_option() {
let error = parse_args(vec![
OsString::from("zstd"),
OsString::from("--filelist"),
OsString::from("-q"),
])
.expect_err("an option must not be consumed as the list name");
assert!(error.contains("cannot be separated from its argument"));
}
#[test]
fn output_dir_flat_accepts_attached_and_separate_values() {
let attached = parse(&["zstd", "--output-dir-flat=out", "in"]);
let separate = parse(&["zstd", "--output-dir-flat", "out", "in"]);
assert_eq!(
attached.output_dir_flat.as_deref().map(CStr::to_bytes),
Some(&b"out"[..])
);
assert_eq!(
separate.output_dir_flat.as_deref().map(CStr::to_bytes),
Some(&b"out"[..])
);
}
#[test]
fn output_dir_flat_rejects_an_empty_name() {
/* Mirrors the C NEXT_FIELD semantics: `--output-dir-flat=` keeps the
* empty attached value instead of consuming the next argument. */
let error = parse_args(vec![
OsString::from("zstd"),
OsString::from("--output-dir-flat="),
OsString::from("out"),
])
.expect_err("an empty output dir must be rejected");
assert!(error.contains("output dir cannot be empty string"));
}
#[cfg(unix)]
#[test]
fn output_dir_mirror_is_parsed_on_posix_targets() {
let cli = parse(&["zstd", "--output-dir-mirror", "out", "in"]);
assert_eq!(
cli.output_dir_mirror.as_deref().map(CStr::to_bytes),
Some(&b"out"[..])
);
}
#[test]
fn bench_mode_parses_level_duration_and_defaults() {
let cli = parse(&["zstd", "-b1", "-i0", "input"]);
assert_eq!(cli.operation, Operation::Bench);
assert_eq!(cli.level, 1);
assert_eq!(cli.bench_nb_seconds, Some(0));
assert_eq!(cli.bench_end_level, None);
assert_eq!(
cli.inputs
.iter()
.map(|input| input.as_bytes())
.collect::<Vec<_>>(),
vec![&b"input"[..]]
);
}
#[test]
fn bench_range_aggregates_within_a_single_argument() {
let cli = parse(&["zstd", "-b5e6i2", "input"]);
assert_eq!(cli.operation, Operation::Bench);
assert_eq!(cli.level, 5);
assert_eq!(cli.bench_end_level, Some(6));
assert_eq!(cli.bench_nb_seconds, Some(2));
}
#[test]
fn bench_duration_without_digits_defaults_to_zero() {
let cli = parse(&["zstd", "-b", "-e", "-i"]);
assert_eq!(cli.operation, Operation::Bench);
assert_eq!(cli.level, DEFAULT_CLEVEL);
assert_eq!(cli.bench_end_level, Some(0));
assert_eq!(cli.bench_nb_seconds, Some(0));
}
}
Reference in New Issue View Git Blame Copy Permalink