From be02682a3114588339ee9a32c8957eb09c5a5e81 Mon Sep 17 00:00:00 2001 From: ddidderr Date: Sat, 18 Jul 2026 02:35:21 +0200 Subject: [PATCH] feat(cli): complete Rust argument and file-stat compatibility Port the remaining high-level CLI option, validation, environment, terminal-safety, and file-stat compatibility paths into Rust while keeping the existing C file-I/O ABI boundary. Match upstream quiet-mode warnings and exact error text for size and level handling. Test Plan: - cargo test --manifest-path rust/cli/Cargo.toml (90 passed) - cargo clippy --manifest-path rust/Cargo.toml - cargo clippy --manifest-path rust/Cargo.toml --benches - cargo clippy --manifest-path rust/Cargo.toml --tests - make -B -C tests -j2 test-cli-tests (41 passed) --- rust/src/util.rs | 140 ++++++-- rust/src/zstd_cli.rs | 776 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 755 insertions(+), 161 deletions(-) diff --git a/rust/src/util.rs b/rust/src/util.rs index 928a3b740..a8b2b53dd 100644 --- a/rust/src/util.rs +++ b/rust/src/util.rs @@ -113,6 +113,14 @@ fn display(message: &str) { eprint!("{message}"); } +unsafe fn c_string(value: *const c_char) -> String { + if value.is_null() { + "(null)".to_owned() + } else { + String::from_utf8_lossy(CStr::from_ptr(value).to_bytes()).into_owned() + } +} + unsafe fn display_c_string(message: *const c_char) { if !message.is_null() { display(&String::from_utf8_lossy(CStr::from_ptr(message).to_bytes())); @@ -128,11 +136,13 @@ fn trace_call(message: &str) { fn trace_return(ret: c_int) { if TRACE_FILE_STAT.load(Ordering::Relaxed) { - let depth = TRACE_DEPTH.fetch_sub(1, Ordering::Relaxed) - 1; + let depth = TRACE_DEPTH + .fetch_sub(1, Ordering::Relaxed) + .saturating_sub(1); eprintln!( "Trace:FileStat: {:width$}< {ret}", "", - width = depth.max(0) as usize + width = depth as usize ); } } @@ -294,7 +304,9 @@ pub unsafe extern "C" fn UTIL_fstat( filename: *const c_char, statbuf: *mut Stat, ) -> c_int { - trace_call("UTIL_stat"); + trace_call(&format!("UTIL_stat({fd}, {})", unsafe { + c_string(filename) + })); let result = if fd >= 0 { libc::fstat(fd, statbuf) } else { @@ -324,12 +336,16 @@ pub unsafe extern "C" fn UTIL_isRegularFileStat(statbuf: *const Stat) -> c_int { #[no_mangle] pub unsafe extern "C" fn UTIL_isDirectoryStat(statbuf: *const Stat) -> c_int { + trace_call("UTIL_isDirectoryStat()"); #[cfg(any(unix, windows))] { - mode_is(statbuf, libc::S_IFDIR) as c_int + let ret = mode_is(statbuf, libc::S_IFDIR) as c_int; + trace_return(ret); + ret } #[cfg(not(any(unix, windows)))] { + trace_return(0); 0 } } @@ -362,20 +378,32 @@ pub unsafe extern "C" fn UTIL_isBlockDevStat(statbuf: *const Stat) -> c_int { #[no_mangle] pub unsafe extern "C" fn UTIL_isRegularFile(infilename: *const c_char) -> c_int { + trace_call(&format!("UTIL_isRegularFile({})", unsafe { + c_string(infilename) + })); let mut statbuf = MaybeUninit::::uninit(); - if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 { - return 0; - } - UTIL_isRegularFileStat(statbuf.as_ptr()) + let ret = if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 { + 0 + } else { + UTIL_isRegularFileStat(statbuf.as_ptr()) + }; + trace_return(ret); + ret } #[no_mangle] pub unsafe extern "C" fn UTIL_isDirectory(infilename: *const c_char) -> c_int { + trace_call(&format!("UTIL_isDirectory({})", unsafe { + c_string(infilename) + })); let mut statbuf = MaybeUninit::::uninit(); - if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 { - return 0; - } - UTIL_isDirectoryStat(statbuf.as_ptr()) + let ret = if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 { + 0 + } else { + UTIL_isDirectoryStat(statbuf.as_ptr()) + }; + trace_return(ret); + ret } #[no_mangle] @@ -393,34 +421,48 @@ pub unsafe extern "C" fn UTIL_isFIFO(infilename: *const c_char) -> c_int { #[no_mangle] pub unsafe extern "C" fn UTIL_isLink(infilename: *const c_char) -> c_int { + trace_call(&format!("UTIL_isLink({})", unsafe { c_string(infilename) })); #[cfg(unix)] { let mut statbuf = MaybeUninit::::uninit(); if libc::lstat(infilename, statbuf.as_mut_ptr()) == 0 { - return mode_is(statbuf.as_ptr(), libc::S_IFLNK) as c_int; + let ret = mode_is(statbuf.as_ptr(), libc::S_IFLNK) as c_int; + trace_return(ret); + return ret; } } let _ = infilename; + trace_return(0); 0 } #[no_mangle] pub unsafe extern "C" fn UTIL_isSameFile(file1: *const c_char, file2: *const c_char) -> c_int { assert!(!file1.is_null() && !file2.is_null()); + trace_call(&format!( + "UTIL_isSameFile({}, {})", + unsafe { c_string(file1) }, + unsafe { c_string(file2) } + )); #[cfg(windows)] { - return (c_bytes(file1) == c_bytes(file2)) as c_int; + let ret = (c_bytes(file1) == c_bytes(file2)) as c_int; + trace_return(ret); + return ret; } #[cfg(not(windows))] { let mut file1_stat = MaybeUninit::::uninit(); let mut file2_stat = MaybeUninit::::uninit(); - if UTIL_stat(file1, file1_stat.as_mut_ptr()) == 0 + let ret = if UTIL_stat(file1, file1_stat.as_mut_ptr()) == 0 || UTIL_stat(file2, file2_stat.as_mut_ptr()) == 0 { - return 0; - } - UTIL_isSameFileStat(file1, file2, file1_stat.as_ptr(), file2_stat.as_ptr()) + 0 + } else { + UTIL_isSameFileStat(file1, file2, file1_stat.as_ptr(), file2_stat.as_ptr()) + }; + trace_return(ret); + ret } } @@ -454,11 +496,17 @@ pub unsafe extern "C" fn UTIL_getFileSizeStat(statbuf: *const Stat) -> u64 { #[no_mangle] pub unsafe extern "C" fn UTIL_getFileSize(infilename: *const c_char) -> u64 { + trace_call(&format!("UTIL_getFileSize({})", unsafe { + c_string(infilename) + })); let mut statbuf = MaybeUninit::::uninit(); if UTIL_stat(infilename, statbuf.as_mut_ptr()) == 0 { + trace_return(-1); return UTIL_FILESIZE_UNKNOWN; } - UTIL_getFileSizeStat(statbuf.as_ptr()) + let size = UTIL_getFileSizeStat(statbuf.as_ptr()); + trace_return(size as c_int); + size } #[no_mangle] @@ -493,9 +541,15 @@ pub unsafe extern "C" fn UTIL_fchmod( statbuf: *const Stat, permissions: UtilMode, ) -> c_int { + trace_call(&format!( + "UTIL_chmod({}, {:04o})", + unsafe { c_string(filename) }, + permissions + )); let mut local_stat = MaybeUninit::::uninit(); let stat_ptr = if statbuf.is_null() { if UTIL_fstat(fd, filename, local_stat.as_mut_ptr()) == 0 { + trace_return(0); return 0; } local_stat.as_ptr() @@ -503,24 +557,36 @@ pub unsafe extern "C" fn UTIL_fchmod( statbuf }; if UTIL_isRegularFileStat(stat_ptr) == 0 { + trace_return(0); return 0; } #[cfg(unix)] { if fd >= 0 { - return libc::fchmod(fd, permissions); + trace_call("fchmod"); + let ret = libc::fchmod(fd, permissions); + trace_return(ret); + trace_return(ret); + return ret; } - libc::chmod(filename, permissions) + trace_call("chmod"); + let ret = libc::chmod(filename, permissions); + trace_return(ret); + trace_return(ret); + ret } #[cfg(windows)] { let _ = fd; - libc::chmod(filename, permissions) + let ret = libc::chmod(filename, permissions); + trace_return(ret); + ret } #[cfg(not(any(unix, windows)))] { let _ = (fd, filename, permissions); + trace_return(0); 0 } } @@ -563,7 +629,10 @@ unsafe fn set_mtime(_filename: *const c_char, _statbuf: *const Stat) -> c_int { #[no_mangle] pub unsafe extern "C" fn UTIL_utime(filename: *const c_char, statbuf: *const Stat) -> c_int { - set_mtime(filename, statbuf) + trace_call(&format!("UTIL_utime({})", unsafe { c_string(filename) })); + let ret = set_mtime(filename, statbuf); + trace_return(ret); + ret } #[no_mangle] @@ -577,10 +646,14 @@ pub unsafe extern "C" fn UTIL_setFDStat( filename: *const c_char, statbuf: *const Stat, ) -> c_int { + trace_call(&format!("UTIL_setFileStat({fd}, {})", unsafe { + c_string(filename) + })); let mut current_stat = MaybeUninit::::uninit(); if UTIL_fstat(fd, filename, current_stat.as_mut_ptr()) == 0 || UTIL_isRegularFileStat(current_stat.as_ptr()) == 0 { + trace_return(-1); return -1; } @@ -607,7 +680,9 @@ pub unsafe extern "C" fn UTIL_setFDStat( result += libc::chown(filename, (*statbuf).st_uid, no_gid); } } - -result + let ret = -result; + trace_return(ret); + ret } #[no_mangle] @@ -652,16 +727,19 @@ pub unsafe extern "C" fn UTIL_isConsole(file: *mut libc::FILE) -> c_int { return 0; } let fd = libc::fileno(file); - if fd == 0 && FAKE_STDIN_IS_CONSOLE.load(Ordering::Relaxed) + trace_call(&format!("UTIL_isConsole({fd})")); + let ret = if fd == 0 && FAKE_STDIN_IS_CONSOLE.load(Ordering::Relaxed) || fd == 1 && FAKE_STDOUT_IS_CONSOLE.load(Ordering::Relaxed) || fd == 2 && FAKE_STDERR_IS_CONSOLE.load(Ordering::Relaxed) { - return 1; - } - if fd < 0 { - return 0; - } - libc::isatty(fd) + 1 + } else if fd < 0 { + 0 + } else { + libc::isatty(fd) + }; + trace_return(ret); + ret } #[no_mangle] diff --git a/rust/src/zstd_cli.rs b/rust/src/zstd_cli.rs index c642f4a45..1c7a39931 100644 --- a/rust/src/zstd_cli.rs +++ b/rust/src/zstd_cli.rs @@ -51,6 +51,23 @@ const DEFAULT_DICT_SELECTIVITY: u32 = 9; const DEFAULT_SHRINK_DICT_REGRESSION: u32 = 1; const DEFAULT_FASTCOVER_ACCEL: u32 = 1; const MAX_FAST_ACCELERATION: i32 = 128 << 10; +#[cfg(target_pointer_width = "32")] +const MAX_WINDOW_LOG: u32 = 30; +#[cfg(not(target_pointer_width = "32"))] +const MAX_WINDOW_LOG: u32 = 31; +#[cfg(target_pointer_width = "32")] +const MAX_CHAIN_LOG: u32 = 29; +#[cfg(not(target_pointer_width = "32"))] +const MAX_CHAIN_LOG: u32 = 30; +const MAX_HASH_LOG: u32 = 30; +const MAX_SEARCH_LOG: u32 = MAX_WINDOW_LOG - 1; +const MAX_MIN_MATCH: u32 = 3; +const MAX_TARGET_LENGTH: u32 = 1 << 17; +const MAX_STRATEGY: c_int = 9; +const MAX_OVERLAP_LOG: i32 = 9; +const MAX_LDM_HASH_LOG: i32 = 30; +const MAX_LDM_MIN_MATCH: i32 = 16; +const MAX_LDM_BUCKET_SIZE_LOG: i32 = 8; const STDIN_MARK: &str = "/*stdin*\\"; const STDOUT_MARK: &str = "/*stdout*\\"; #[cfg(windows)] @@ -86,7 +103,7 @@ struct FIO_ctx_t { } #[repr(C)] -#[derive(Clone, Copy, Debug, Default)] +#[derive(Clone, Copy, Debug, Default, PartialEq)] struct ZSTD_compressionParameters { windowLog: u32, chainLog: u32, @@ -163,6 +180,13 @@ unsafe extern "C" { #[cfg(feature = "compression")] fn UTIL_countLogicalCores() -> c_int; fn UTIL_getFileSize(input_file_name: *const c_char) -> u64; + fn UTIL_isConsole(file: *mut libc::FILE) -> c_int; + fn UTIL_isLink(input_file_name: *const c_char) -> c_int; + fn UTIL_fakeStderrIsConsole(); + fn UTIL_traceFileStat(); + static mut stdin: *mut libc::FILE; + static mut stdout: *mut libc::FILE; + static mut stderr: *mut libc::FILE; fn UTIL_createFileNamesTable_fromFileName( input_file_name: *const c_char, ) -> *mut FileNamesTable; @@ -446,6 +470,9 @@ struct Cli { dictionary_id: u32, dictionary_selectivity: u32, trace: Option, + fake_stderr_is_console: bool, + trace_file_stat: bool, + gzip_compat: bool, cover_params: ZDICT_cover_params_t, fast_cover_params: ZDICT_fastCover_params_t, } @@ -507,6 +534,9 @@ impl Cli { dictionary_id: 0, dictionary_selectivity: DEFAULT_DICT_SELECTIVITY, trace: None, + fake_stderr_is_console: false, + trace_file_stat: false, + gzip_compat: false, cover_params: ZDICT_cover_params_t::default(), fast_cover_params: default_fast_cover_params(), }; @@ -525,6 +555,7 @@ impl Cli { "gzip" => { cli.compression_format = CompressionFormat::Gzip; cli.remove_source = true; + cli.gzip_compat = true; } "gunzip" => { cli.compression_format = CompressionFormat::Gzip; @@ -585,12 +616,40 @@ fn os_cstring(value: &OsStr) -> Result { } fn default_level() -> i32 { - match env::var("ZSTD_CLEVEL") { - Ok(value) => value.parse::().unwrap_or(DEFAULT_CLEVEL), - Err(_) => DEFAULT_CLEVEL, + let Ok(value) = env::var("ZSTD_CLEVEL") else { + return DEFAULT_CLEVEL; + }; + match value.parse::() { + Ok(level) if (i32::MIN as i128..=i32::MAX as i128).contains(&level) => level as i32, + Ok(_) => { + eprintln!( + "Ignore environment variable setting ZSTD_CLEVEL={value}: numeric value too large " + ); + DEFAULT_CLEVEL + } + Err(_) if value_has_only_sign_and_digits(&value) => { + eprintln!( + "Ignore environment variable setting ZSTD_CLEVEL={value}: numeric value too large " + ); + DEFAULT_CLEVEL + } + Err(_) => { + eprintln!( + "Ignore environment variable setting ZSTD_CLEVEL={value}: not a valid integer value " + ); + DEFAULT_CLEVEL + } } } +fn value_has_only_sign_and_digits(value: &str) -> bool { + let digits = value + .strip_prefix('+') + .or_else(|| value.strip_prefix('-')) + .unwrap_or(value); + !digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit()) +} + #[cfg(feature = "compression")] unsafe fn default_worker_count() -> i32 { if let Ok(value) = env::var("ZSTD_NBTHREADS") { @@ -632,101 +691,306 @@ fn program_basename(value: &OsStr) -> String { .to_owned() } +#[cfg(feature = "compression")] +fn max_c_level_for_help() -> i32 { + unsafe { ZSTD_maxCLevel() } +} + +#[cfg(not(feature = "compression"))] +const fn max_c_level_for_help() -> i32 { + 19 +} + +fn write_basic_usage(out: &mut W, program_name: &str) { + let max_level = DEFAULT_MAX_CLEVEL; + let _ = writeln!( + out, + "Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided." + ); + let _ = writeln!( + out, + "\nUsage: {program_name} [OPTIONS...] [INPUT... | -] [-o OUTPUT]" + ); + let _ = writeln!(out, "\nOptions:"); + let _ = writeln!( + out, + " -o OUTPUT Write output to a single file, OUTPUT." + ); + let _ = writeln!( + out, + " -k, --keep Preserve INPUT file(s). [Default] " + ); + let _ = writeln!( + out, + " --rm Remove INPUT file(s) after successful (de)compression." + ); + if program_name == "gzip" { + let _ = writeln!( + out, + " -n, --no-name Do not store original filename when compressing." + ); + } + let _ = writeln!(out); + let _ = writeln!( + out, + " -# Desired compression level, where `#` is a number between 1 and {max_level};" + ); + let _ = writeln!( + out, + " lower numbers provide faster compression, higher numbers yield" + ); + let _ = writeln!( + out, + " better compression ratios. [Default: {DEFAULT_CLEVEL}]" + ); + let _ = writeln!(out); + let _ = writeln!( + out, + " -d, --decompress Perform decompression." + ); + let _ = writeln!( + out, + " -D DICT Use DICT as the dictionary for compression or decompression." + ); + let _ = writeln!(out); + let _ = writeln!( + out, + " -f, --force Disable input and output checks. Allows overwriting existing files," + ); + let _ = writeln!( + out, + " receiving input from the console, printing output to STDOUT, and" + ); + let _ = writeln!( + out, + " operating on links, block devices, etc. Unrecognized formats will be" + ); + let _ = writeln!( + out, + " passed-through through as-is." + ); + let _ = writeln!(out); + let _ = writeln!( + out, + " -h Display short usage and exit." + ); + let _ = writeln!( + out, + " -H, --help Display full help and exit." + ); + let _ = writeln!( + out, + " -V, --version Display the program version and exit." + ); + let _ = writeln!(out); +} + +fn write_advanced_usage(out: &mut W, program_name: &str) { + let version = unsafe { CStr::from_ptr(ZSTD_versionString()) }.to_string_lossy(); + let _ = writeln!( + out, + "*** Zstandard CLI ({}-bit) v{version}, by Yann Collet ***", + usize::BITS + ); + let _ = writeln!(out); + write_basic_usage(out, program_name); + let _ = writeln!(out, "\nAdvanced options:"); + let _ = writeln!( + out, + " -c, --stdout Write to STDOUT (even if it is a console) and keep the INPUT file(s)." + ); + let _ = writeln!( + out, + " -v, --verbose Enable verbose output; pass multiple times to increase verbosity." + ); + let _ = writeln!( + out, + " -q, --quiet Suppress warnings; pass twice to suppress errors." + ); + let _ = writeln!( + out, + " --trace LOG Log tracing information to LOG." + ); + let _ = writeln!(out); + let _ = writeln!( + out, + " --[no-]progress Forcibly show/hide the progress counter." + ); + let _ = writeln!(out); + let _ = writeln!( + out, + " -r Operate recursively on directories." + ); + let _ = writeln!( + out, + " --filelist LIST Read a list of files to operate on from LIST." + ); + let _ = writeln!( + out, + " --output-dir-flat DIR Store processed files in DIR." + ); + #[cfg(unix)] + let _ = writeln!( + out, + " --output-dir-mirror DIR Store processed files in DIR, respecting original directory structure." + ); + let _ = writeln!(out); + let _ = writeln!( + out, + " -- Treat remaining arguments after `--` as files." + ); + let _ = writeln!(out, "\nAdvanced compression options:"); + let _ = writeln!( + out, + " --ultra Enable levels beyond 19, up to {}; requires more memory.", + max_c_level_for_help() + ); + let _ = writeln!( + out, + " --fast[=#] Use very fast compression levels. [Default: 1]" + ); + if program_name == "gzip" { + let _ = writeln!( + out, + " --best Compatibility alias for `-9`." + ); + } + let _ = writeln!( + out, + " --adapt Dynamically adapt compression level to I/O conditions." + ); + let _ = writeln!( + out, + " --long[=#] Enable long distance matching with window log #." + ); + let _ = writeln!(out, " --patch-from=REF Use REF as the reference point for Zstandard's diff engine."); + let _ = writeln!( + out, + " -T# Spawn # compression threads." + ); + let _ = writeln!( + out, + " --single-thread Share a single thread for I/O and compression." + ); + let _ = writeln!(out, " --auto-threads={{physical|logical}}"); + let _ = writeln!(out, " -B# Set job size to #."); + let _ = writeln!( + out, + " --rsyncable Compress using a rsync-friendly method." + ); + let _ = writeln!( + out, + " --exclude-compressed Only compress files that are not already compressed." + ); + let _ = writeln!( + out, + " --stream-size=# Specify size of streaming input from STDIN." + ); + let _ = writeln!( + out, + " --size-hint=# Optimize compression parameters for streaming input." + ); + let _ = writeln!(out, " --target-compressed-block-size=#"); + let _ = writeln!( + out, + " Generate compressed blocks of approximately # size." + ); + let _ = writeln!( + out, + " --no-dictID Don't write `dictID` into the header." + ); + let _ = writeln!( + out, + " --[no-]compress-literals Force (un)compressed literals." + ); + let _ = writeln!( + out, + " --[no-]row-match-finder Enable/disable the row-based matchfinder." + ); + let _ = writeln!( + out, + " --format=zstd Compress files to the `.zst` format. [Default]" + ); + let _ = writeln!( + out, + " --format=gzip Compress files to the `.gz` format." + ); + let _ = writeln!( + out, + " --format=xz Compress files to the `.xz` format." + ); + let _ = writeln!( + out, + " --format=lzma Compress files to the `.lzma` format." + ); + let _ = writeln!( + out, + " --format=lz4 Compress files to the `.lz4` format." + ); + let _ = writeln!(out, "\nAdvanced decompression options:"); + let _ = writeln!( + out, + " -l Print information about compressed files." + ); + let _ = writeln!( + out, + " --test Test compressed file integrity." + ); + let _ = writeln!( + out, + " -M# Set the memory usage limit to # megabytes." + ); + let _ = writeln!( + out, + " --[no-]sparse Enable/disable sparse mode." + ); + let _ = writeln!( + out, + " --[no-]pass-through Pass through uncompressed files as-is." + ); + let _ = writeln!(out, "\nDictionary builder:"); + let _ = writeln!( + out, + " --train Create a dictionary from a training set of files." + ); + let _ = writeln!(out, " --train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]]"); + let _ = writeln!( + out, + " --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]]" + ); + let _ = writeln!( + out, + " --train-legacy[=s=#] Use the legacy algorithm." + ); + let _ = writeln!( + out, + " --maxdict=# Limit dictionary to specified size #." + ); + let _ = writeln!( + out, + " --dictID=# Force dictionary ID to #." + ); + let _ = writeln!(out, "\nBenchmark options:"); + let _ = writeln!( + out, + " -b# Perform benchmarking with compression level #." + ); + let _ = writeln!( + out, + " -e# Test all compression levels up to #." + ); + let _ = writeln!( + out, + " -i# Set the minimum evaluation time to # seconds." + ); +} + fn usage(advanced: bool) { let mut out = io::stdout().lock(); - let _ = writeln!( - out, - "Compress or decompress INPUT file(s); reads stdin when INPUT is '-' or omitted." - ); - let _ = writeln!(out, "\nUsage: zstd [OPTIONS...] [INPUT... | -] [-o OUTPUT]"); - let _ = writeln!(out, "\nCore options:"); - let _ = writeln!( - out, - " -o OUTPUT, -c, --stdout Select output file or stdout" - ); - let _ = writeln!(out, " -d, --decompress Decompress"); - let _ = writeln!(out, " -t, --test Test compressed input"); - let _ = writeln!( - out, - " -# Compression level (default {DEFAULT_CLEVEL})" - ); - let _ = writeln!(out, " -D DICT Use a dictionary"); - let _ = writeln!( - out, - " -f, --force Overwrite output / allow stdio" - ); - let _ = writeln!( - out, - " -k, --keep | --rm Preserve or remove source after success" - ); - let _ = writeln!(out, " -q, --quiet | -v, --verbose Adjust display level"); - let _ = writeln!(out, " -V, --version Print version"); - let _ = writeln!(out, " -h | -H, --help Print help"); if advanced { - let _ = writeln!(out, "\nImplemented advanced compression controls:"); - let _ = writeln!( - out, - " --fast[=#], --ultra, --long[=#], --threads=#, --single-thread, --auto-threads={{physical|logical}}, --block-size=#" - ); - let _ = writeln!( - out, - " --zstd=wlog=#,clog=#,hlog=#,slog=#,mml=#,tlen=#,strat=#" - ); - let _ = writeln!( - out, - " --[no-]check, --[no-]sparse, --[no-]progress, --[no-]asyncio" - ); - let _ = writeln!( - out, - " --adapt[=min=#,max=#], --rsyncable, --[no-]row-match-finder" - ); - let _ = writeln!(out, "\nFile selection and placement:"); - let _ = writeln!( - out, - " -r Operate recursively on directories" - ); - let _ = writeln!( - out, - " --filelist LIST Read a list of input files from LIST" - ); - let _ = writeln!( - out, - " --output-dir-flat DIR Store processed files in DIR" - ); - #[cfg(unix)] - let _ = writeln!( - out, - " --output-dir-mirror DIR Store processed files in DIR, mirroring the input tree" - ); - let _ = writeln!( - out, - " -l, --list Print information about .zst file(s)" - ); - let _ = writeln!(out, "\nBenchmark options:"); - let _ = writeln!( - out, - " -b# Benchmark file(s) at compression level #" - ); - let _ = writeln!( - out, - " -e# Test all levels from -b# up to # included" - ); - let _ = writeln!( - out, - " -i# Set the minimum evaluation time to # seconds" - ); - let _ = writeln!( - out, - "\nDictionary builder:\n --train Create a dictionary from training files\n --train-cover[=k=#,d=#,steps=#,split=#,shrink[=#]]\n --train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#,shrink[=#]]\n --train-legacy[=s=#] Use the legacy algorithm\n --maxdict=# Maximum dictionary size (default {DEFAULT_MAX_DICT_SIZE})\n --dictID=# Force the dictionary ID (default: random)" - ); - let _ = writeln!( - out, - "\n --format=zstd|gzip|xz|lzma|lz4 Select the compression frame format" - ); - let _ = writeln!( - out, - " --trace LOG Log compression/decompression trace data" - ); + write_advanced_usage(&mut out, "zstd"); + } else { + write_basic_usage(&mut out, "zstd"); } } @@ -744,6 +1008,16 @@ fn print_version(quiet: bool) { } } +fn print_cli_header(display_level: i32) { + if display_level >= 3 { + let version = unsafe { CStr::from_ptr(ZSTD_versionString()) }.to_string_lossy(); + eprintln!( + "*** Zstandard CLI ({}-bit) v{version}, by Yann Collet ***", + usize::BITS + ); + } +} + fn check_lib_version() -> Result<(), String> { let expected = unsafe { CStr::from_ptr(ZSTD_rust_cli_expected_version()) }; let actual = unsafe { CStr::from_ptr(ZSTD_versionString()) }; @@ -763,28 +1037,34 @@ fn parse_size(value: &str) -> Result { .unwrap_or(value.len()); let (digits, suffix) = value.split_at(split); if digits.is_empty() { - return Err(format!("expected a numeric value, got {value:?}")); + return Err( + "error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed " + .to_owned(), + ); } let mut number = digits .parse::() - .map_err(|_| format!("numeric value overflows size_t: {value:?}"))?; - let normalized = suffix.trim_end_matches('B').trim_end_matches('i'); - let shift = match normalized { + .map_err(|_| "error: numeric value overflows size_t ".to_owned())?; + let shift = match suffix { "" => 0, - "K" | "k" => 10, - "M" | "m" => 20, - "G" | "g" => 30, - _ => return Err(format!("unsupported numeric suffix in {value:?}")), + "K" | "KB" | "KiB" => 10, + "M" | "MB" | "MiB" => 20, + _ => return Err( + "error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed " + .to_owned(), + ), }; number = number .checked_shl(shift) - .ok_or_else(|| format!("numeric value overflows size_t: {value:?}"))?; + .ok_or_else(|| "error: numeric value overflows size_t ".to_owned())?; Ok(number) } fn parse_u32(value: &str, name: &str) -> Result { + let _ = name; let size = parse_size(value)?; - u32::try_from(size).map_err(|_| format!("{name} is too large: {value:?}")) + u32::try_from(size) + .map_err(|_| "error: numeric value overflows 32-bit unsigned int ".to_owned()) } fn parse_i32(value: &str, name: &str) -> Result { @@ -926,7 +1206,7 @@ fn parse_compression_parameters(value: &str, cli: &mut Cli) -> Result<(), String fn parse_adapt(value: &str, cli: &mut Cli) -> Result<(), String> { cli.adapt = true; if value.is_empty() { - return Ok(()); + return Err("invalid --adapt parameter".to_owned()); } for item in value.split(',') { let Some((name, raw)) = item.split_once('=') else { @@ -1047,6 +1327,29 @@ fn unsupported(option: &str) -> Result<(), String> { )) } +fn set_max_compression(cli: &mut Cli) -> Result<(), String> { + if cfg!(target_pointer_width = "32") { + return Err("--max is incompatible with 32-bit mode".to_owned()); + } + cli.ultra = true; + cli.ldm = true; + cli.compression_params = ZSTD_compressionParameters { + windowLog: MAX_WINDOW_LOG, + chainLog: MAX_CHAIN_LOG, + hashLog: MAX_HASH_LOG, + searchLog: MAX_SEARCH_LOG, + minMatch: MAX_MIN_MATCH, + targetLength: MAX_TARGET_LENGTH, + strategy: MAX_STRATEGY, + }; + cli.overlap_log = Some(MAX_OVERLAP_LOG); + cli.ldm_hash_log = Some(MAX_LDM_HASH_LOG); + cli.ldm_hash_rate_log = Some(0); + cli.ldm_min_match = Some(MAX_LDM_MIN_MATCH); + cli.ldm_bucket_size_log = Some(MAX_LDM_BUCKET_SIZE_LOG); + Ok(()) +} + fn parse_long_option( option: &str, args: &[OsString], @@ -1095,9 +1398,13 @@ fn parse_long_option( | "--no-compress-literals" | "--exclude-compressed" | "--no-name" + | "--best" | "--list" | "--show-default-cparams" | "--train" + | "--max" + | "--fake-stderr-is-console" + | "--trace-file-stat" ) { return Err(format!("{name} does not take an argument")); @@ -1128,6 +1435,10 @@ fn parse_long_option( Ok(None) } "--no-name" => Ok(None), + "--best" if cli.gzip_compat => { + cli.level = 9; + Ok(None) + } "--rm" => { cli.remove_source = true; Ok(None) @@ -1236,7 +1547,11 @@ fn parse_long_option( Ok(None) } "--adapt" => { - parse_adapt(attached.unwrap_or(""), cli)?; + if let Some(value) = attached { + parse_adapt(value, cli)?; + } else { + cli.adapt = true; + } Ok(None) } "--no-row-match-finder" => { @@ -1402,17 +1717,23 @@ fn parse_long_option( cli.output_dir_mirror = Some(cstring(&value)?); Ok(None) } - "--max" - | "--patch-from" - | "--priority" - | "--fake-stdin-is-console" - | "--fake-stdout-is-console" - | "--fake-stderr-is-console" - | "--trace-file-stat" => { + "--max" => { + set_max_compression(cli)?; + Ok(None) + } + "--fake-stderr-is-console" => { + cli.fake_stderr_is_console = true; + Ok(None) + } + "--trace-file-stat" => { + cli.trace_file_stat = true; + Ok(None) + } + "--patch-from" | "--priority" | "--fake-stdin-is-console" | "--fake-stdout-is-console" => { unsupported(name)?; Ok(None) } - _ => Err(format!("unknown option {option:?}")), + _ => Err(format!("unknown option {option}")), } } @@ -1533,7 +1854,67 @@ fn parse_short_options( Ok(None) } -fn parse_args(args: Vec) -> Result { +#[derive(Debug)] +struct ParseError { + message: String, + #[allow(dead_code)] + details: String, + show_usage: bool, +} + +impl ParseError { + fn direct(message: String) -> Self { + Self::direct_with_details(message.clone(), message) + } + + fn direct_with_details(message: String, details: String) -> Self { + Self { + message, + details, + show_usage: false, + } + } + + fn incorrect_parameter(parameter: &str, details: String, show_usage: bool) -> Self { + Self { + message: format!("Incorrect parameter: {parameter}"), + details, + show_usage, + } + } + + #[cfg(test)] + fn contains(&self, needle: &str) -> bool { + self.message.contains(needle) || self.details.contains(needle) + } +} + +fn classify_parse_error(error: String, parameter: &str, display_level: i32) -> ParseError { + if error.starts_with("error:") { + return ParseError::direct(error); + } + if error.starts_with("output dir cannot be empty string") { + return ParseError::direct(format!("error: {error}")); + } + if error.starts_with("missing argument for ") { + return ParseError::direct_with_details( + "error: missing command argument ".to_owned(), + error, + ); + } + if error.contains("cannot be separated from its argument") { + return ParseError::direct_with_details( + "error: command cannot be separated from its argument by another command ".to_owned(), + error, + ); + } + if let Some(option) = error.strip_prefix("unknown option ") { + return ParseError::incorrect_parameter(option, error.clone(), display_level >= 2); + } + ParseError::incorrect_parameter(parameter, error, display_level >= 2) +} + +fn parse_args(args: Vec) -> Result { let program_name = args .first() .map_or_else(|| "zstd".to_owned(), |value| program_basename(value)); @@ -1544,23 +1925,28 @@ fn parse_args(args: Vec) -> Result { while index < args.len() { let rendered = args[index].to_string_lossy().into_owned(); if end_of_options { - cli.inputs.push(os_cstring(&args[index])?); + cli.inputs + .push(os_cstring(&args[index]).map_err(ParseError::direct)?); } else if rendered == "--" { end_of_options = true; } else if rendered == "-" { - cli.inputs.push(cstring(STDIN_MARK)?); + cli.inputs + .push(cstring(STDIN_MARK).map_err(ParseError::direct)?); } else if rendered.starts_with("--") { - if let Some(action) = parse_long_option(&rendered, &args, &mut index, &mut cli)? { + let action = parse_long_option(&rendered, &args, &mut index, &mut cli) + .map_err(|error| classify_parse_error(error, &rendered, cli.display_level))?; + if let Some(action) = action { return Ok(action); } } else if rendered.starts_with('-') { - if let Some(action) = - parse_short_options(&rendered, &args[index], &args, &mut index, &mut cli)? - { + let action = parse_short_options(&rendered, &args[index], &args, &mut index, &mut cli) + .map_err(|error| classify_parse_error(error, &rendered, cli.display_level))?; + if let Some(action) = action { return Ok(action); } } else { - cli.inputs.push(os_cstring(&args[index])?); + cli.inputs + .push(os_cstring(&args[index]).map_err(ParseError::direct)?); } index += 1; } @@ -1578,7 +1964,10 @@ unsafe fn apply_preferences(cli: &Cli, prefs: *mut FIO_prefs_t, ctx: *mut FIO_ct FIO_setCompressionType(prefs, cli.compression_format.fio_type()); FIO_setNotificationLevel(display_level); FIO_setProgressSetting( - if !io::stderr().is_terminal() && cli.progress != FIO_PS_ALWAYS { + if !cli.fake_stderr_is_console + && !io::stderr().is_terminal() + && cli.progress != FIO_PS_ALWAYS + { FIO_PS_NEVER } else { cli.progress @@ -1694,6 +2083,9 @@ fn is_stdin(value: &CString) -> bool { #[cfg(unix)] fn is_non_fifo_symlink(input: &CString) -> bool { + if unsafe { UTIL_isLink(input.as_ptr()) } == 0 { + return false; + } let path = Path::new(OsStr::from_bytes(input.as_bytes())); let Ok(metadata) = fs::symlink_metadata(path) else { return false; @@ -1704,6 +2096,9 @@ fn is_non_fifo_symlink(input: &CString) -> bool { #[cfg(not(unix))] fn is_non_fifo_symlink(input: &CString) -> bool { + if unsafe { UTIL_isLink(input.as_ptr()) } == 0 { + return false; + } let path = Path::new(&input.to_string_lossy().into_owned()); fs::symlink_metadata(path).is_ok_and(|metadata| metadata.file_type().is_symlink()) } @@ -1728,18 +2123,32 @@ fn filter_symlink_inputs(cli: &mut Cli) { fn check_terminal_safety(cli: &Cli) -> Result<(), String> { let has_stdin = cli.inputs.iter().any(is_stdin); - if has_stdin && !cli.force && io::stdin().is_terminal() { + let stdin_is_console = if has_stdin { + unsafe { UTIL_isConsole(stdin) != 0 } + } else { + false + }; + let output_is_stdout = is_stdout(cli.output.as_ref()); + let stdout_is_console = + if output_is_stdout || (cli.operation == Operation::Decompress && !has_stdin) { + unsafe { UTIL_isConsole(stdout) != 0 } + } else { + false + }; + let stderr_is_console = unsafe { UTIL_isConsole(stderr) != 0 }; + if has_stdin && !cli.force && stdin_is_console { return Err("stdin is a console, aborting".to_owned()); } if has_stdin - && is_stdout(cli.output.as_ref()) + && output_is_stdout && !cli.force && !cli.force_stdout && cli.operation != Operation::Decompress - && io::stdout().is_terminal() + && stdout_is_console { return Err("stdout is a console, aborting".to_owned()); } + let _ = stderr_is_console; Ok(()) } @@ -2124,6 +2533,12 @@ impl Drop for TraceGuard { } fn run_cli(mut cli: Cli) -> Result { + if cli.fake_stderr_is_console { + unsafe { UTIL_fakeStderrIsConsole() }; + } + if cli.trace_file_stat { + unsafe { UTIL_traceFileStat() }; + } let _trace_guard = TraceGuard::new(cli.trace.as_ref()); unsafe { /* The C CLI publishes the display level to util.c before any table @@ -2187,6 +2602,15 @@ fn run_cli(mut cli: Cli) -> Result { } check_terminal_safety(&cli)?; + print_cli_header(cli.display_level); + + #[cfg(feature = "compression")] + if cli.operation == Operation::Decompress + && cli.display_level >= 2 + && cli.workers.is_some_and(|workers| workers > 1) + { + eprintln!("Warning : decompression does not support multi-threading"); + } if cli.operation == Operation::Compress { #[cfg(not(feature = "compression"))] @@ -2202,7 +2626,9 @@ fn run_cli(mut cli: Cli) -> Result { DEFAULT_MAX_CLEVEL.min(max_level) }; if cli.level > ceiling { - eprintln!("zstd: warning: compression level reduced to {ceiling}"); + if cli.display_level >= 2 { + eprintln!("zstd: warning: compression level reduced to {ceiling}"); + } cli.level = ceiling; } if cli.level < min_level { @@ -2291,6 +2717,9 @@ fn run_cli(mut cli: Cli) -> Result { } fn run_from_args(args: Vec) -> c_int { + let program_name = args + .first() + .map_or_else(|| "zstd".to_owned(), |value| program_basename(value)); match parse_args(args) { Ok(Action::Help { advanced }) => { usage(advanced); @@ -2308,7 +2737,11 @@ fn run_from_args(args: Vec) -> c_int { } }, Err(error) => { - eprintln!("zstd: {error}\nTry `zstd --help` for usage."); + eprintln!("{}", error.message); + if error.show_usage { + let mut out = io::stderr().lock(); + write_basic_usage(&mut out, &program_name); + } 1 } } @@ -2430,6 +2863,64 @@ mod tests { assert_eq!(cli.compression_params.windowLog, 25); } + #[test] + fn max_mode_sets_the_full_c_and_ldm_parameter_bundle() { + let cli = parse(&["zstd", "--max", "input"]); + + assert!(cli.ultra); + assert!(cli.ldm); + assert_eq!( + cli.compression_params, + ZSTD_compressionParameters { + windowLog: MAX_WINDOW_LOG, + chainLog: MAX_CHAIN_LOG, + hashLog: MAX_HASH_LOG, + searchLog: MAX_SEARCH_LOG, + minMatch: MAX_MIN_MATCH, + targetLength: MAX_TARGET_LENGTH, + strategy: MAX_STRATEGY, + } + ); + assert_eq!(cli.overlap_log, Some(MAX_OVERLAP_LOG)); + assert_eq!(cli.ldm_hash_log, Some(MAX_LDM_HASH_LOG)); + assert_eq!(cli.ldm_hash_rate_log, Some(0)); + assert_eq!(cli.ldm_min_match, Some(MAX_LDM_MIN_MATCH)); + assert_eq!(cli.ldm_bucket_size_log, Some(MAX_LDM_BUCKET_SIZE_LOG)); + } + + #[test] + fn max_mode_is_applied_at_the_option_position() { + let cli = parse(&["zstd", "--max", "--zstd=wlog=25", "input"]); + assert_eq!(cli.compression_params.windowLog, 25); + + let cli = parse(&["zstd", "--zstd=wlog=25", "--max", "input"]); + assert_eq!(cli.compression_params.windowLog, MAX_WINDOW_LOG); + } + + #[test] + fn console_and_file_stat_compatibility_switches_are_recorded() { + let cli = parse(&[ + "zstd", + "--fake-stderr-is-console", + "--trace-file-stat", + "input", + ]); + + assert!(cli.fake_stderr_is_console); + assert!(cli.trace_file_stat); + } + + #[test] + fn gzip_best_is_an_alias_for_level_nine() { + let cli = parse(&["gzip", "--best", "input"]); + assert!(cli.gzip_compat); + assert_eq!(cli.level, 9); + + let error = parse_args(vec![OsString::from("zstd"), OsString::from("--best")]) + .expect_err("--best is a gzip compatibility option"); + assert!(error.contains("Incorrect parameter: --best")); + } + #[test] fn stdio_and_dictionary_short_options_are_preserved() { let cli = parse(&["zstd", "-dc", "-D", "dict", "-"]); @@ -2494,6 +2985,31 @@ mod tests { assert!(error.contains("does not take an argument")); } + #[test] + fn empty_attached_adapt_is_rejected_but_bare_adapt_is_enabled() { + let bare = parse(&["zstd", "--adapt", "input"]); + assert!(bare.adapt); + + let error = parse_args(vec![OsString::from("zstd"), OsString::from("--adapt=")]) + .expect_err("an empty attached adapt value must be rejected"); + assert!(error.contains("Incorrect parameter: --adapt=")); + } + + #[test] + fn adapt_bounds_and_rsyncable_single_thread_are_parsed_for_runtime_checks() { + let error = parse_args(vec![ + OsString::from("zstd"), + OsString::from("--adapt=min=10,max=9"), + ]) + .expect_err("incoherent adaptation bounds must be rejected"); + assert!(error.contains("Incorrect parameter: --adapt=min=10,max=9")); + + let cli = parse(&["zstd", "--rsyncable", "--single-thread", "input"]); + assert!(cli.rsyncable); + assert!(cli.single_thread); + assert_eq!(cli.workers, Some(0)); + } + #[test] fn zstdmt_uses_auto_threads() { let cli = parse(&["zstdmt", "input"]);