ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(cli): complete Rust argument and file-stat compatibility

Browse Source 
Port the remaining high-level CLI option, validation, environment, terminal-safety, and file-stat compatibility paths into Rust while keeping the existing C file-I/O ABI boundary. Match upstream quiet-mode warnings and exact error text for size and level handling.

Test Plan:

- cargo test --manifest-path rust/cli/Cargo.toml (90 passed)

- cargo clippy --manifest-path rust/Cargo.toml

- cargo clippy --manifest-path rust/Cargo.toml --benches

- cargo clippy --manifest-path rust/Cargo.toml --tests

- make -B -C tests -j2 test-cli-tests (41 passed)
This commit is contained in:
ddidderr
2026-07-18 02:35:21 +02:00
parent 23d45378bd
commit be02682a31
2 changed files with 755 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/util.rs
+109 -31
View File
@@ -113,6 +113,14 @@ fn display(message: &str) {
eprint!("{message}");
}
unsafe fn c_string(value: *const c_char) -> String {
if value.is_null() {
"(null)".to_owned()
} else {
String::from_utf8_lossy(CStr::from_ptr(value).to_bytes()).into_owned()
}
}
unsafe fn display_c_string(message: *const c_char) {
if !message.is_null() {
display(&String::from_utf8_lossy(CStr::from_ptr(message).to_bytes()));
@@ -128,11 +136,13 @@ fn trace_call(message: &str) {
fn trace_return(ret: c_int) {
if TRACE_FILE_STAT.load(Ordering::Relaxed) {
let depth = TRACE_DEPTH.fetch_sub(1, Ordering::Relaxed) - 1;
let depth = TRACE_DEPTH
.fetch_sub(1, Ordering::Relaxed)
.saturating_sub(1);
eprintln!(
"Trace:FileStat: {:width$}< {ret}",
"",
width = depth.max(0) as usize
width = depth as usize
);
}
}
@@ -294,7 +304,9 @@ pub unsafe extern "C" fn UTIL_fstat(
filename: *const c_char,
statbuf: *mut Stat,
) -> c_int {
trace_call("UTIL_stat");
trace_call(&format!("UTIL_stat({fd}, {})", unsafe {
c_string(filename)
}));
let result = if fd >= 0 {
libc::fstat(fd, statbuf)
} else {
@@ -324,12 +336,16 @@ pub unsafe extern "C" fn UTIL_isRegularFileStat(statbuf: *const Stat) -> c_int {
#[no_mangle]
pub unsafe extern "C" fn UTIL_isDirectoryStat(statbuf: *const Stat) -> c_int {
trace_call("UTIL_isDirectoryStat()");
#[cfg(any(unix, windows))]
{
mode_is(statbuf, libc::S_IFDIR) as c_int
let ret = mode_is(statbuf, libc::S_IFDIR) as c_int;
trace_return(ret);
ret
}
#[cfg(not(any(unix, windows)))]
{
trace_return(0);
0
}
}
@@ -362,20 +378,32 @@ pub unsafe extern "C" fn UTIL_isBlockDevStat(statbuf: *const Stat) -> c_int {
#[no_mangle]
pub unsafe extern "C" fn UTIL_isRegularFile(infilename: *const c_char) -> c_int {
trace_call(&format!("UTIL_isRegularFile({})", unsafe {
c_string(infilename)
}));
let mut statbuf = MaybeUninit::<Stat>::uninit();
if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 {
return 0;
}
UTIL_isRegularFileStat(statbuf.as_ptr())
let ret = if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 {
0
} else {
UTIL_isRegularFileStat(statbuf.as_ptr())
};
trace_return(ret);
ret
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isDirectory(infilename: *const c_char) -> c_int {
trace_call(&format!("UTIL_isDirectory({})", unsafe {
c_string(infilename)
}));
let mut statbuf = MaybeUninit::<Stat>::uninit();
if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 {
return 0;
}
UTIL_isDirectoryStat(statbuf.as_ptr())
let ret = if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 {
0
} else {
UTIL_isDirectoryStat(statbuf.as_ptr())
};
trace_return(ret);
ret
}
#[no_mangle]
@@ -393,34 +421,48 @@ pub unsafe extern "C" fn UTIL_isFIFO(infilename: *const c_char) -> c_int {
#[no_mangle]
pub unsafe extern "C" fn UTIL_isLink(infilename: *const c_char) -> c_int {
trace_call(&format!("UTIL_isLink({})", unsafe { c_string(infilename) }));
#[cfg(unix)]
{
let mut statbuf = MaybeUninit::<Stat>::uninit();
if libc::lstat(infilename, statbuf.as_mut_ptr()) == 0 {
return mode_is(statbuf.as_ptr(), libc::S_IFLNK) as c_int;
let ret = mode_is(statbuf.as_ptr(), libc::S_IFLNK) as c_int;
trace_return(ret);
return ret;
}
}
let _ = infilename;
trace_return(0);
0
}
#[no_mangle]
pub unsafe extern "C" fn UTIL_isSameFile(file1: *const c_char, file2: *const c_char) -> c_int {
assert!(!file1.is_null() && !file2.is_null());
trace_call(&format!(
"UTIL_isSameFile({}, {})",
unsafe { c_string(file1) },
unsafe { c_string(file2) }
));
#[cfg(windows)]
{
return (c_bytes(file1) == c_bytes(file2)) as c_int;
let ret = (c_bytes(file1) == c_bytes(file2)) as c_int;
trace_return(ret);
return ret;
}
#[cfg(not(windows))]
{
let mut file1_stat = MaybeUninit::<Stat>::uninit();
let mut file2_stat = MaybeUninit::<Stat>::uninit();
if UTIL_stat(file1, file1_stat.as_mut_ptr()) == 0
let ret = if UTIL_stat(file1, file1_stat.as_mut_ptr()) == 0
|| UTIL_stat(file2, file2_stat.as_mut_ptr()) == 0
{
return 0;
}
UTIL_isSameFileStat(file1, file2, file1_stat.as_ptr(), file2_stat.as_ptr())
0
} else {
UTIL_isSameFileStat(file1, file2, file1_stat.as_ptr(), file2_stat.as_ptr())
};
trace_return(ret);
ret
}
}
@@ -454,11 +496,17 @@ pub unsafe extern "C" fn UTIL_getFileSizeStat(statbuf: *const Stat) -> u64 {
#[no_mangle]
pub unsafe extern "C" fn UTIL_getFileSize(infilename: *const c_char) -> u64 {
trace_call(&format!("UTIL_getFileSize({})", unsafe {
c_string(infilename)
}));
let mut statbuf = MaybeUninit::<Stat>::uninit();
if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 {
trace_return(-1);
return UTIL_FILESIZE_UNKNOWN;
}
UTIL_getFileSizeStat(statbuf.as_ptr())
let size = UTIL_getFileSizeStat(statbuf.as_ptr());
trace_return(size as c_int);
size
}
#[no_mangle]
@@ -493,9 +541,15 @@ pub unsafe extern "C" fn UTIL_fchmod(
statbuf: *const Stat,
permissions: UtilMode,
) -> c_int {
trace_call(&format!(
"UTIL_chmod({}, {:04o})",
unsafe { c_string(filename) },
permissions
));
let mut local_stat = MaybeUninit::<Stat>::uninit();
let stat_ptr = if statbuf.is_null() {
if UTIL_fstat(fd, filename, local_stat.as_mut_ptr()) == 0 {
trace_return(0);
return 0;
}
local_stat.as_ptr()
@@ -503,24 +557,36 @@ pub unsafe extern "C" fn UTIL_fchmod(
statbuf
};
if UTIL_isRegularFileStat(stat_ptr) == 0 {
trace_return(0);
return 0;
}
#[cfg(unix)]
{
if fd >= 0 {
return libc::fchmod(fd, permissions);
trace_call("fchmod");
let ret = libc::fchmod(fd, permissions);
trace_return(ret);
trace_return(ret);
return ret;
}
libc::chmod(filename, permissions)
trace_call("chmod");
let ret = libc::chmod(filename, permissions);
trace_return(ret);
trace_return(ret);
ret
}
#[cfg(windows)]
{
let _ = fd;
libc::chmod(filename, permissions)
let ret = libc::chmod(filename, permissions);
trace_return(ret);
ret
}
#[cfg(not(any(unix, windows)))]
{
let _ = (fd, filename, permissions);
trace_return(0);
0
}
}
@@ -563,7 +629,10 @@ unsafe fn set_mtime(_filename: *const c_char, _statbuf: *const Stat) -> c_int {
#[no_mangle]
pub unsafe extern "C" fn UTIL_utime(filename: *const c_char, statbuf: *const Stat) -> c_int {
set_mtime(filename, statbuf)
trace_call(&format!("UTIL_utime({})", unsafe { c_string(filename) }));
let ret = set_mtime(filename, statbuf);
trace_return(ret);
ret
}
#[no_mangle]
@@ -577,10 +646,14 @@ pub unsafe extern "C" fn UTIL_setFDStat(
filename: *const c_char,
statbuf: *const Stat,
) -> c_int {
trace_call(&format!("UTIL_setFileStat({fd}, {})", unsafe {
c_string(filename)
}));
let mut current_stat = MaybeUninit::<Stat>::uninit();
if UTIL_fstat(fd, filename, current_stat.as_mut_ptr()) == 0
|| UTIL_isRegularFileStat(current_stat.as_ptr()) == 0
{
trace_return(-1);
return -1;
}
@@ -607,7 +680,9 @@ pub unsafe extern "C" fn UTIL_setFDStat(
result += libc::chown(filename, (*statbuf).st_uid, no_gid);
}
}
-result
let ret = -result;
trace_return(ret);
ret
}
#[no_mangle]
@@ -652,16 +727,19 @@ pub unsafe extern "C" fn UTIL_isConsole(file: *mut libc::FILE) -> c_int {
return 0;
}
let fd = libc::fileno(file);
if fd == 0 && FAKE_STDIN_IS_CONSOLE.load(Ordering::Relaxed)
trace_call(&format!("UTIL_isConsole({fd})"));
let ret = if fd == 0 && FAKE_STDIN_IS_CONSOLE.load(Ordering::Relaxed)
|| fd == 1 && FAKE_STDOUT_IS_CONSOLE.load(Ordering::Relaxed)
|| fd == 2 && FAKE_STDERR_IS_CONSOLE.load(Ordering::Relaxed)
{
return 1;
}
if fd < 0 {
return 0;
}
libc::isatty(fd)
1
} else if fd < 0 {
0
} else {
libc::isatty(fd)
};
trace_return(ret);
ret
}
#[no_mangle]
rust/src/zstd_cli.rs
+646 -130
View File
@@ -51,6 +51,23 @@ const DEFAULT_DICT_SELECTIVITY: u32 = 9;
const DEFAULT_SHRINK_DICT_REGRESSION: u32 = 1;
const DEFAULT_FASTCOVER_ACCEL: u32 = 1;
const MAX_FAST_ACCELERATION: i32 = 128 << 10;
#[cfg(target_pointer_width = "32")]
const MAX_WINDOW_LOG: u32 = 30;
#[cfg(not(target_pointer_width = "32"))]
const MAX_WINDOW_LOG: u32 = 31;
#[cfg(target_pointer_width = "32")]
const MAX_CHAIN_LOG: u32 = 29;
#[cfg(not(target_pointer_width = "32"))]
const MAX_CHAIN_LOG: u32 = 30;
const MAX_HASH_LOG: u32 = 30;
const MAX_SEARCH_LOG: u32 = MAX_WINDOW_LOG - 1;
const MAX_MIN_MATCH: u32 = 3;
const MAX_TARGET_LENGTH: u32 = 1 << 17;
const MAX_STRATEGY: c_int = 9;
const MAX_OVERLAP_LOG: i32 = 9;
const MAX_LDM_HASH_LOG: i32 = 30;
const MAX_LDM_MIN_MATCH: i32 = 16;
const MAX_LDM_BUCKET_SIZE_LOG: i32 = 8;
const STDIN_MARK: &str = "/*stdin*\\";
const STDOUT_MARK: &str = "/*stdout*\\";
#[cfg(windows)]
@@ -86,7 +103,7 @@ struct FIO_ctx_t {
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
#[derive(Clone, Copy, Debug, Default, PartialEq)]
struct ZSTD_compressionParameters {
windowLog: u32,
chainLog: u32,
@@ -163,6 +180,13 @@ unsafe extern "C" {
#[cfg(feature = "compression")]
fn UTIL_countLogicalCores() -> c_int;
fn UTIL_getFileSize(input_file_name: *const c_char) -> u64;
fn UTIL_isConsole(file: *mut libc::FILE) -> c_int;
fn UTIL_isLink(input_file_name: *const c_char) -> c_int;
fn UTIL_fakeStderrIsConsole();
fn UTIL_traceFileStat();
static mut stdin: *mut libc::FILE;
static mut stdout: *mut libc::FILE;
static mut stderr: *mut libc::FILE;
fn UTIL_createFileNamesTable_fromFileName(
input_file_name: *const c_char,
) -> *mut FileNamesTable;
@@ -446,6 +470,9 @@ struct Cli {
dictionary_id: u32,
dictionary_selectivity: u32,
trace: Option<CString>,
fake_stderr_is_console: bool,
trace_file_stat: bool,
gzip_compat: bool,
cover_params: ZDICT_cover_params_t,
fast_cover_params: ZDICT_fastCover_params_t,
}
@@ -507,6 +534,9 @@ impl Cli {
dictionary_id: 0,
dictionary_selectivity: DEFAULT_DICT_SELECTIVITY,
trace: None,
fake_stderr_is_console: false,
trace_file_stat: false,
gzip_compat: false,
cover_params: ZDICT_cover_params_t::default(),
fast_cover_params: default_fast_cover_params(),
};
@@ -525,6 +555,7 @@ impl Cli {
"gzip" => {
cli.compression_format = CompressionFormat::Gzip;
cli.remove_source = true;
cli.gzip_compat = true;
}
"gunzip" => {
cli.compression_format = CompressionFormat::Gzip;
@@ -585,12 +616,40 @@ fn os_cstring(value: &OsStr) -> Result<CString, String> {
}
fn default_level() -> i32 {
match env::var("ZSTD_CLEVEL") {
Ok(value) => value.parse::<i32>().unwrap_or(DEFAULT_CLEVEL),
Err(_) => DEFAULT_CLEVEL,
let Ok(value) = env::var("ZSTD_CLEVEL") else {
return DEFAULT_CLEVEL;
};
match value.parse::<i128>() {
Ok(level) if (i32::MIN as i128..=i32::MAX as i128).contains(&level) => level as i32,
Ok(_) => {
eprintln!(
"Ignore environment variable setting ZSTD_CLEVEL={value}: numeric value too large "
);
DEFAULT_CLEVEL
}
Err(_) if value_has_only_sign_and_digits(&value) => {
eprintln!(
"Ignore environment variable setting ZSTD_CLEVEL={value}: numeric value too large "
);
DEFAULT_CLEVEL
}
Err(_) => {
eprintln!(
"Ignore environment variable setting ZSTD_CLEVEL={value}: not a valid integer value "
);
DEFAULT_CLEVEL
}
}
}
fn value_has_only_sign_and_digits(value: &str) -> bool {
let digits = value
.strip_prefix('+')
.or_else(|| value.strip_prefix('-'))
.unwrap_or(value);
!digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit())
}
#[cfg(feature = "compression")]
unsafe fn default_worker_count() -> i32 {
if let Ok(value) = env::var("ZSTD_NBTHREADS") {
@@ -632,101 +691,306 @@ fn program_basename(value: &OsStr) -> String {
.to_owned()
}
#[cfg(feature = "compression")]
fn max_c_level_for_help() -> i32 {
unsafe { ZSTD_maxCLevel() }
}
#[cfg(not(feature = "compression"))]
const fn max_c_level_for_help() -> i32 {
19
}
fn write_basic_usage<W: Write>(out: &mut W, program_name: &str) {
let max_level = DEFAULT_MAX_CLEVEL;
let _ = writeln!(
out,
"Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided."
);
let _ = writeln!(
out,
"\nUsage: {program_name} [OPTIONS...] [INPUT... | -] [-o OUTPUT]"
);
let _ = writeln!(out, "\nOptions:");
let _ = writeln!(
out,
" -o OUTPUT Write output to a single file, OUTPUT."
);
let _ = writeln!(
out,
" -k, --keep Preserve INPUT file(s). [Default] "
);
let _ = writeln!(
out,
" --rm Remove INPUT file(s) after successful (de)compression."
);
if program_name == "gzip" {
let _ = writeln!(
out,
" -n, --no-name Do not store original filename when compressing."
);
}
let _ = writeln!(out);
let _ = writeln!(
out,
" -# Desired compression level, where `#` is a number between 1 and {max_level};"
);
let _ = writeln!(
out,
" lower numbers provide faster compression, higher numbers yield"
);
let _ = writeln!(
out,
" better compression ratios. [Default: {DEFAULT_CLEVEL}]"
);
let _ = writeln!(out);
let _ = writeln!(
out,
" -d, --decompress Perform decompression."
);
let _ = writeln!(
out,
" -D DICT Use DICT as the dictionary for compression or decompression."
);
let _ = writeln!(out);
let _ = writeln!(
out,
" -f, --force Disable input and output checks. Allows overwriting existing files,"
);
let _ = writeln!(
out,
" receiving input from the console, printing output to STDOUT, and"
);
let _ = writeln!(
out,
" operating on links, block devices, etc. Unrecognized formats will be"
);
let _ = writeln!(
out,
" passed-through through as-is."
);
let _ = writeln!(out);
let _ = writeln!(
out,
" -h Display short usage and exit."
);
let _ = writeln!(
out,
" -H, --help Display full help and exit."
);
let _ = writeln!(
out,
" -V, --version Display the program version and exit."
);
let _ = writeln!(out);
}
fn write_advanced_usage<W: Write>(out: &mut W, program_name: &str) {
let version = unsafe { CStr::from_ptr(ZSTD_versionString()) }.to_string_lossy();
let _ = writeln!(
out,
"*** Zstandard CLI ({}-bit) v{version}, by Yann Collet ***",
usize::BITS
);
let _ = writeln!(out);
write_basic_usage(out, program_name);
let _ = writeln!(out, "\nAdvanced options:");
let _ = writeln!(
out,
" -c, --stdout Write to STDOUT (even if it is a console) and keep the INPUT file(s)."
);
let _ = writeln!(
out,
" -v, --verbose Enable verbose output; pass multiple times to increase verbosity."
);
let _ = writeln!(
out,
" -q, --quiet Suppress warnings; pass twice to suppress errors."
);
let _ = writeln!(
out,
" --trace LOG Log tracing information to LOG."
);
let _ = writeln!(out);
let _ = writeln!(
out,
" --[no-]progress Forcibly show/hide the progress counter."
);
let _ = writeln!(out);
let _ = writeln!(
out,
" -r Operate recursively on directories."
);
let _ = writeln!(
out,
" --filelist LIST Read a list of files to operate on from LIST."
);
let _ = writeln!(
out,
" --output-dir-flat DIR Store processed files in DIR."
);
#[cfg(unix)]
let _ = writeln!(
out,
" --output-dir-mirror DIR Store processed files in DIR, respecting original directory structure."
);
let _ = writeln!(out);
let _ = writeln!(
out,
" -- Treat remaining arguments after `--` as files."
);
let _ = writeln!(out, "\nAdvanced compression options:");
let _ = writeln!(
out,
" --ultra Enable levels beyond 19, up to {}; requires more memory.",
max_c_level_for_help()
);
let _ = writeln!(
out,
" --fast[=#] Use very fast compression levels. [Default: 1]"
);
if program_name == "gzip" {
let _ = writeln!(
out,
" --best Compatibility alias for `-9`."
);
}
let _ = writeln!(
out,
" --adapt Dynamically adapt compression level to I/O conditions."
);
let _ = writeln!(
out,
" --long[=#] Enable long distance matching with window log #."
);
let _ = writeln!(out, " --patch-from=REF Use REF as the reference point for Zstandard's diff engine.");
let _ = writeln!(
out,
" -T# Spawn # compression threads."
);
let _ = writeln!(
out,
" --single-thread Share a single thread for I/O and compression."
);
let _ = writeln!(out, " --auto-threads={{physical|logical}}");
let _ = writeln!(out, " -B# Set job size to #.");
let _ = writeln!(
out,
" --rsyncable Compress using a rsync-friendly method."
);
let _ = writeln!(
out,
" --exclude-compressed Only compress files that are not already compressed."
);
let _ = writeln!(
out,
" --stream-size=# Specify size of streaming input from STDIN."
);
let _ = writeln!(
out,
" --size-hint=# Optimize compression parameters for streaming input."
);
let _ = writeln!(out, " --target-compressed-block-size=#");
let _ = writeln!(
out,
" Generate compressed blocks of approximately # size."
);
let _ = writeln!(
out,
" --no-dictID Don't write `dictID` into the header."
);
let _ = writeln!(
out,
" --[no-]compress-literals Force (un)compressed literals."
);
let _ = writeln!(
out,
" --[no-]row-match-finder Enable/disable the row-based matchfinder."
);
let _ = writeln!(
out,
" --format=zstd Compress files to the `.zst` format. [Default]"
);
let _ = writeln!(
out,
" --format=gzip Compress files to the `.gz` format."
);
let _ = writeln!(
out,
" --format=xz Compress files to the `.xz` format."
);
let _ = writeln!(
out,
" --format=lzma Compress files to the `.lzma` format."
);
let _ = writeln!(
out,
" --format=lz4 Compress files to the `.lz4` format."
);
let _ = writeln!(out, "\nAdvanced decompression options:");
let _ = writeln!(
out,
" -l Print information about compressed files."
);
let _ = writeln!(
out,
" --test Test compressed file integrity."
);
let _ = writeln!(
out,
" -M# Set the memory usage limit to # megabytes."
);
let _ = writeln!(
out,
" --[no-]sparse Enable/disable sparse mode."
);
let _ = writeln!(
out,
" --[no-]pass-through Pass through uncompressed files as-is."
);
let _ = writeln!(out, "\nDictionary builder:");
let _ = writeln!(
out,
" --train Create a dictionary from a training set of files."
);
let _ = writeln!(out, " --train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]]");
let _ = writeln!(
out,
" --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]]"
);
let _ = writeln!(
out,
" --train-legacy[=s=#] Use the legacy algorithm."
);
let _ = writeln!(
out,
" --maxdict=# Limit dictionary to specified size #."
);
let _ = writeln!(
out,
" --dictID=# Force dictionary ID to #."
);
let _ = writeln!(out, "\nBenchmark options:");
let _ = writeln!(
out,
" -b# Perform benchmarking with compression level #."
);
let _ = writeln!(
out,
" -e# Test all compression levels up to #."
);
let _ = writeln!(
out,
" -i# Set the minimum evaluation time to # seconds."
);
}
fn usage(advanced: bool) {
let mut out = io::stdout().lock();
let _ = writeln!(
out,
"Compress or decompress INPUT file(s); reads stdin when INPUT is '-' or omitted."
);
let _ = writeln!(out, "\nUsage: zstd [OPTIONS...] [INPUT... | -] [-o OUTPUT]");
let _ = writeln!(out, "\nCore options:");
let _ = writeln!(
out,
" -o OUTPUT, -c, --stdout Select output file or stdout"
);
let _ = writeln!(out, " -d, --decompress Decompress");
let _ = writeln!(out, " -t, --test Test compressed input");
let _ = writeln!(
out,
" -# Compression level (default {DEFAULT_CLEVEL})"
);
let _ = writeln!(out, " -D DICT Use a dictionary");
let _ = writeln!(
out,
" -f, --force Overwrite output / allow stdio"
);
let _ = writeln!(
out,
" -k, --keep | --rm Preserve or remove source after success"
);
let _ = writeln!(out, " -q, --quiet | -v, --verbose Adjust display level");
let _ = writeln!(out, " -V, --version Print version");
let _ = writeln!(out, " -h | -H, --help Print help");
if advanced {
let _ = writeln!(out, "\nImplemented advanced compression controls:");
let _ = writeln!(
out,
" --fast[=#], --ultra, --long[=#], --threads=#, --single-thread, --auto-threads={{physical|logical}}, --block-size=#"
);
let _ = writeln!(
out,
" --zstd=wlog=#,clog=#,hlog=#,slog=#,mml=#,tlen=#,strat=#"
);
let _ = writeln!(
out,
" --[no-]check, --[no-]sparse, --[no-]progress, --[no-]asyncio"
);
let _ = writeln!(
out,
" --adapt[=min=#,max=#], --rsyncable, --[no-]row-match-finder"
);
let _ = writeln!(out, "\nFile selection and placement:");
let _ = writeln!(
out,
" -r Operate recursively on directories"
);
let _ = writeln!(
out,
" --filelist LIST Read a list of input files from LIST"
);
let _ = writeln!(
out,
" --output-dir-flat DIR Store processed files in DIR"
);
#[cfg(unix)]
let _ = writeln!(
out,
" --output-dir-mirror DIR Store processed files in DIR, mirroring the input tree"
);
let _ = writeln!(
out,
" -l, --list Print information about .zst file(s)"
);
let _ = writeln!(out, "\nBenchmark options:");
let _ = writeln!(
out,
" -b# Benchmark file(s) at compression level #"
);
let _ = writeln!(
out,
" -e# Test all levels from -b# up to # included"
);
let _ = writeln!(
out,
" -i# Set the minimum evaluation time to # seconds"
);
let _ = writeln!(
out,
"\nDictionary builder:\n --train Create a dictionary from training files\n --train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]]\n --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]]\n --train-legacy[=s=#] Use the legacy algorithm\n --maxdict=# Maximum dictionary size (default {DEFAULT_MAX_DICT_SIZE})\n --dictID=# Force the dictionary ID (default: random)"
);
let _ = writeln!(
out,
"\n --format=zstd|gzip|xz|lzma|lz4 Select the compression frame format"
);
let _ = writeln!(
out,
" --trace LOG Log compression/decompression trace data"
);
write_advanced_usage(&mut out, "zstd");
} else {
write_basic_usage(&mut out, "zstd");
}
}
@@ -744,6 +1008,16 @@ fn print_version(quiet: bool) {
}
}
fn print_cli_header(display_level: i32) {
if display_level >= 3 {
let version = unsafe { CStr::from_ptr(ZSTD_versionString()) }.to_string_lossy();
eprintln!(
"*** Zstandard CLI ({}-bit) v{version}, by Yann Collet ***",
usize::BITS
);
}
}
fn check_lib_version() -> Result<(), String> {
let expected = unsafe { CStr::from_ptr(ZSTD_rust_cli_expected_version()) };
let actual = unsafe { CStr::from_ptr(ZSTD_versionString()) };
@@ -763,28 +1037,34 @@ fn parse_size(value: &str) -> Result<usize, String> {
.unwrap_or(value.len());
let (digits, suffix) = value.split_at(split);
if digits.is_empty() {
return Err(format!("expected a numeric value, got {value:?}"));
return Err(
"error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed "
.to_owned(),
);
}
let mut number = digits
.parse::<usize>()
.map_err(|_| format!("numeric value overflows size_t: {value:?}"))?;
let normalized = suffix.trim_end_matches('B').trim_end_matches('i');
let shift = match normalized {
.map_err(|_| "error: numeric value overflows size_t ".to_owned())?;
let shift = match suffix {
"" => 0,
"K" | "k" => 10,
"M" | "m" => 20,
"G" | "g" => 30,
_ => return Err(format!("unsupported numeric suffix in {value:?}")),
"K" | "KB" | "KiB" => 10,
"M" | "MB" | "MiB" => 20,
_ => return Err(
"error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed "
.to_owned(),
),
};
number = number
.checked_shl(shift)
.ok_or_else(|| format!("numeric value overflows size_t: {value:?}"))?;
.ok_or_else(|| "error: numeric value overflows size_t ".to_owned())?;
Ok(number)
}
fn parse_u32(value: &str, name: &str) -> Result<u32, String> {
let _ = name;
let size = parse_size(value)?;
u32::try_from(size).map_err(|_| format!("{name} is too large: {value:?}"))
u32::try_from(size)
.map_err(|_| "error: numeric value overflows 32-bit unsigned int ".to_owned())
}
fn parse_i32(value: &str, name: &str) -> Result<i32, String> {
@@ -926,7 +1206,7 @@ fn parse_compression_parameters(value: &str, cli: &mut Cli) -> Result<(), String
fn parse_adapt(value: &str, cli: &mut Cli) -> Result<(), String> {
cli.adapt = true;
if value.is_empty() {
return Ok(());
return Err("invalid --adapt parameter".to_owned());
}
for item in value.split(',') {
let Some((name, raw)) = item.split_once('=') else {
@@ -1047,6 +1327,29 @@ fn unsupported(option: &str) -> Result<(), String> {
))
}
fn set_max_compression(cli: &mut Cli) -> Result<(), String> {
if cfg!(target_pointer_width = "32") {
return Err("--max is incompatible with 32-bit mode".to_owned());
}
cli.ultra = true;
cli.ldm = true;
cli.compression_params = ZSTD_compressionParameters {
windowLog: MAX_WINDOW_LOG,
chainLog: MAX_CHAIN_LOG,
hashLog: MAX_HASH_LOG,
searchLog: MAX_SEARCH_LOG,
minMatch: MAX_MIN_MATCH,
targetLength: MAX_TARGET_LENGTH,
strategy: MAX_STRATEGY,
};
cli.overlap_log = Some(MAX_OVERLAP_LOG);
cli.ldm_hash_log = Some(MAX_LDM_HASH_LOG);
cli.ldm_hash_rate_log = Some(0);
cli.ldm_min_match = Some(MAX_LDM_MIN_MATCH);
cli.ldm_bucket_size_log = Some(MAX_LDM_BUCKET_SIZE_LOG);
Ok(())
}
fn parse_long_option(
option: &str,
args: &[OsString],
@@ -1095,9 +1398,13 @@ fn parse_long_option(
| "--no-compress-literals"
| "--exclude-compressed"
| "--no-name"
| "--best"
| "--list"
| "--show-default-cparams"
| "--train"
| "--max"
| "--fake-stderr-is-console"
| "--trace-file-stat"
)
{
return Err(format!("{name} does not take an argument"));
@@ -1128,6 +1435,10 @@ fn parse_long_option(
Ok(None)
}
"--no-name" => Ok(None),
"--best" if cli.gzip_compat => {
cli.level = 9;
Ok(None)
}
"--rm" => {
cli.remove_source = true;
Ok(None)
@@ -1236,7 +1547,11 @@ fn parse_long_option(
Ok(None)
}
"--adapt" => {
parse_adapt(attached.unwrap_or(""), cli)?;
if let Some(value) = attached {
parse_adapt(value, cli)?;
} else {
cli.adapt = true;
}
Ok(None)
}
"--no-row-match-finder" => {
@@ -1402,17 +1717,23 @@ fn parse_long_option(
cli.output_dir_mirror = Some(cstring(&value)?);
Ok(None)
}
"--max"
| "--patch-from"
| "--priority"
| "--fake-stdin-is-console"
| "--fake-stdout-is-console"
| "--fake-stderr-is-console"
| "--trace-file-stat" => {
"--max" => {
set_max_compression(cli)?;
Ok(None)
}
"--fake-stderr-is-console" => {
cli.fake_stderr_is_console = true;
Ok(None)
}
"--trace-file-stat" => {
cli.trace_file_stat = true;
Ok(None)
}
"--patch-from" | "--priority" | "--fake-stdin-is-console" | "--fake-stdout-is-console" => {
unsupported(name)?;
Ok(None)
}
_ => Err(format!("unknown option {option:?}")),
_ => Err(format!("unknown option {option}")),
}
}
@@ -1533,7 +1854,67 @@ fn parse_short_options(
Ok(None)
}
fn parse_args(args: Vec<OsString>) -> Result<Action, String> {
#[derive(Debug)]
struct ParseError {
message: String,
#[allow(dead_code)]
details: String,
show_usage: bool,
}
impl ParseError {
fn direct(message: String) -> Self {
Self::direct_with_details(message.clone(), message)
}
fn direct_with_details(message: String, details: String) -> Self {
Self {
message,
details,
show_usage: false,
}
}
fn incorrect_parameter(parameter: &str, details: String, show_usage: bool) -> Self {
Self {
message: format!("Incorrect parameter: {parameter}"),
details,
show_usage,
}
}
#[cfg(test)]
fn contains(&self, needle: &str) -> bool {
self.message.contains(needle) || self.details.contains(needle)
}
}
fn classify_parse_error(error: String, parameter: &str, display_level: i32) -> ParseError {
if error.starts_with("error:") {
return ParseError::direct(error);
}
if error.starts_with("output dir cannot be empty string") {
return ParseError::direct(format!("error: {error}"));
}
if error.starts_with("missing argument for ") {
return ParseError::direct_with_details(
"error: missing command argument ".to_owned(),
error,
);
}
if error.contains("cannot be separated from its argument") {
return ParseError::direct_with_details(
"error: command cannot be separated from its argument by another command ".to_owned(),
error,
);
}
if let Some(option) = error.strip_prefix("unknown option ") {
return ParseError::incorrect_parameter(option, error.clone(), display_level >= 2);
}
ParseError::incorrect_parameter(parameter, error, display_level >= 2)
}
fn parse_args(args: Vec<OsString>) -> Result<Action, ParseError> {
let program_name = args
.first()
.map_or_else(|| "zstd".to_owned(), |value| program_basename(value));
@@ -1544,23 +1925,28 @@ fn parse_args(args: Vec<OsString>) -> Result<Action, String> {
while index < args.len() {
let rendered = args[index].to_string_lossy().into_owned();
if end_of_options {
cli.inputs.push(os_cstring(&args[index])?);
cli.inputs
.push(os_cstring(&args[index]).map_err(ParseError::direct)?);
} else if rendered == "--" {
end_of_options = true;
} else if rendered == "-" {
cli.inputs.push(cstring(STDIN_MARK)?);
cli.inputs
.push(cstring(STDIN_MARK).map_err(ParseError::direct)?);
} else if rendered.starts_with("--") {
if let Some(action) = parse_long_option(&rendered, &args, &mut index, &mut cli)? {
let action = parse_long_option(&rendered, &args, &mut index, &mut cli)
.map_err(|error| classify_parse_error(error, &rendered, cli.display_level))?;
if let Some(action) = action {
return Ok(action);
}
} else if rendered.starts_with('-') {
if let Some(action) =
parse_short_options(&rendered, &args[index], &args, &mut index, &mut cli)?
{
let action = parse_short_options(&rendered, &args[index], &args, &mut index, &mut cli)
.map_err(|error| classify_parse_error(error, &rendered, cli.display_level))?;
if let Some(action) = action {
return Ok(action);
}
} else {
cli.inputs.push(os_cstring(&args[index])?);
cli.inputs
.push(os_cstring(&args[index]).map_err(ParseError::direct)?);
}
index += 1;
}
@@ -1578,7 +1964,10 @@ unsafe fn apply_preferences(cli: &Cli, prefs: *mut FIO_prefs_t, ctx: *mut FIO_ct
FIO_setCompressionType(prefs, cli.compression_format.fio_type());
FIO_setNotificationLevel(display_level);
FIO_setProgressSetting(
if !io::stderr().is_terminal() && cli.progress != FIO_PS_ALWAYS {
if !cli.fake_stderr_is_console
&& !io::stderr().is_terminal()
&& cli.progress != FIO_PS_ALWAYS
{
FIO_PS_NEVER
} else {
cli.progress
@@ -1694,6 +2083,9 @@ fn is_stdin(value: &CString) -> bool {
#[cfg(unix)]
fn is_non_fifo_symlink(input: &CString) -> bool {
if unsafe { UTIL_isLink(input.as_ptr()) } == 0 {
return false;
}
let path = Path::new(OsStr::from_bytes(input.as_bytes()));
let Ok(metadata) = fs::symlink_metadata(path) else {
return false;
@@ -1704,6 +2096,9 @@ fn is_non_fifo_symlink(input: &CString) -> bool {
#[cfg(not(unix))]
fn is_non_fifo_symlink(input: &CString) -> bool {
if unsafe { UTIL_isLink(input.as_ptr()) } == 0 {
return false;
}
let path = Path::new(&input.to_string_lossy().into_owned());
fs::symlink_metadata(path).is_ok_and(|metadata| metadata.file_type().is_symlink())
}
@@ -1728,18 +2123,32 @@ fn filter_symlink_inputs(cli: &mut Cli) {
fn check_terminal_safety(cli: &Cli) -> Result<(), String> {
let has_stdin = cli.inputs.iter().any(is_stdin);
if has_stdin && !cli.force && io::stdin().is_terminal() {
let stdin_is_console = if has_stdin {
unsafe { UTIL_isConsole(stdin) != 0 }
} else {
false
};
let output_is_stdout = is_stdout(cli.output.as_ref());
let stdout_is_console =
if output_is_stdout || (cli.operation == Operation::Decompress && !has_stdin) {
unsafe { UTIL_isConsole(stdout) != 0 }
} else {
false
};
let stderr_is_console = unsafe { UTIL_isConsole(stderr) != 0 };
if has_stdin && !cli.force && stdin_is_console {
return Err("stdin is a console, aborting".to_owned());
}
if has_stdin
&& is_stdout(cli.output.as_ref())
&& output_is_stdout
&& !cli.force
&& !cli.force_stdout
&& cli.operation != Operation::Decompress
&& io::stdout().is_terminal()
&& stdout_is_console
{
return Err("stdout is a console, aborting".to_owned());
}
let _ = stderr_is_console;
Ok(())
}
@@ -2124,6 +2533,12 @@ impl Drop for TraceGuard {
}
fn run_cli(mut cli: Cli) -> Result<i32, String> {
if cli.fake_stderr_is_console {
unsafe { UTIL_fakeStderrIsConsole() };
}
if cli.trace_file_stat {
unsafe { UTIL_traceFileStat() };
}
let _trace_guard = TraceGuard::new(cli.trace.as_ref());
unsafe {
/* The C CLI publishes the display level to util.c before any table
@@ -2187,6 +2602,15 @@ fn run_cli(mut cli: Cli) -> Result<i32, String> {
}
check_terminal_safety(&cli)?;
print_cli_header(cli.display_level);
#[cfg(feature = "compression")]
if cli.operation == Operation::Decompress
&& cli.display_level >= 2
&& cli.workers.is_some_and(|workers| workers > 1)
{
eprintln!("Warning : decompression does not support multi-threading");
}
if cli.operation == Operation::Compress {
#[cfg(not(feature = "compression"))]
@@ -2202,7 +2626,9 @@ fn run_cli(mut cli: Cli) -> Result<i32, String> {
DEFAULT_MAX_CLEVEL.min(max_level)
};
if cli.level > ceiling {
eprintln!("zstd: warning: compression level reduced to {ceiling}");
if cli.display_level >= 2 {
eprintln!("zstd: warning: compression level reduced to {ceiling}");
}
cli.level = ceiling;
}
if cli.level < min_level {
@@ -2291,6 +2717,9 @@ fn run_cli(mut cli: Cli) -> Result<i32, String> {
}
fn run_from_args(args: Vec<OsString>) -> c_int {
let program_name = args
.first()
.map_or_else(|| "zstd".to_owned(), |value| program_basename(value));
match parse_args(args) {
Ok(Action::Help { advanced }) => {
usage(advanced);
@@ -2308,7 +2737,11 @@ fn run_from_args(args: Vec<OsString>) -> c_int {
}
},
Err(error) => {
eprintln!("zstd: {error}\nTry `zstd --help` for usage.");
eprintln!("{}", error.message);
if error.show_usage {
let mut out = io::stderr().lock();
write_basic_usage(&mut out, &program_name);
}
1
}
}
@@ -2430,6 +2863,64 @@ mod tests {
assert_eq!(cli.compression_params.windowLog, 25);
}
#[test]
fn max_mode_sets_the_full_c_and_ldm_parameter_bundle() {
let cli = parse(&["zstd", "--max", "input"]);
assert!(cli.ultra);
assert!(cli.ldm);
assert_eq!(
cli.compression_params,
ZSTD_compressionParameters {
windowLog: MAX_WINDOW_LOG,
chainLog: MAX_CHAIN_LOG,
hashLog: MAX_HASH_LOG,
searchLog: MAX_SEARCH_LOG,
minMatch: MAX_MIN_MATCH,
targetLength: MAX_TARGET_LENGTH,
strategy: MAX_STRATEGY,
}
);
assert_eq!(cli.overlap_log, Some(MAX_OVERLAP_LOG));
assert_eq!(cli.ldm_hash_log, Some(MAX_LDM_HASH_LOG));
assert_eq!(cli.ldm_hash_rate_log, Some(0));
assert_eq!(cli.ldm_min_match, Some(MAX_LDM_MIN_MATCH));
assert_eq!(cli.ldm_bucket_size_log, Some(MAX_LDM_BUCKET_SIZE_LOG));
}
#[test]
fn max_mode_is_applied_at_the_option_position() {
let cli = parse(&["zstd", "--max", "--zstd=wlog=25", "input"]);
assert_eq!(cli.compression_params.windowLog, 25);
let cli = parse(&["zstd", "--zstd=wlog=25", "--max", "input"]);
assert_eq!(cli.compression_params.windowLog, MAX_WINDOW_LOG);
}
#[test]
fn console_and_file_stat_compatibility_switches_are_recorded() {
let cli = parse(&[
"zstd",
"--fake-stderr-is-console",
"--trace-file-stat",
"input",
]);
assert!(cli.fake_stderr_is_console);
assert!(cli.trace_file_stat);
}
#[test]
fn gzip_best_is_an_alias_for_level_nine() {
let cli = parse(&["gzip", "--best", "input"]);
assert!(cli.gzip_compat);
assert_eq!(cli.level, 9);
let error = parse_args(vec![OsString::from("zstd"), OsString::from("--best")])
.expect_err("--best is a gzip compatibility option");
assert!(error.contains("Incorrect parameter: --best"));
}
#[test]
fn stdio_and_dictionary_short_options_are_preserved() {
let cli = parse(&["zstd", "-dc", "-D", "dict", "-"]);
@@ -2494,6 +2985,31 @@ mod tests {
assert!(error.contains("does not take an argument"));
}
#[test]
fn empty_attached_adapt_is_rejected_but_bare_adapt_is_enabled() {
let bare = parse(&["zstd", "--adapt", "input"]);
assert!(bare.adapt);
let error = parse_args(vec![OsString::from("zstd"), OsString::from("--adapt=")])
.expect_err("an empty attached adapt value must be rejected");
assert!(error.contains("Incorrect parameter: --adapt="));
}
#[test]
fn adapt_bounds_and_rsyncable_single_thread_are_parsed_for_runtime_checks() {
let error = parse_args(vec![
OsString::from("zstd"),
OsString::from("--adapt=min=10,max=9"),
])
.expect_err("incoherent adaptation bounds must be rejected");
assert!(error.contains("Incorrect parameter: --adapt=min=10,max=9"));
let cli = parse(&["zstd", "--rsyncable", "--single-thread", "input"]);
assert!(cli.rsyncable);
assert!(cli.single_thread);
assert_eq!(cli.workers, Some(0));
}
#[test]
fn zstdmt_uses_auto_threads() {
let cli = parse(&["zstdmt", "input"]);