Move the implementation of programs/benchfn.c into rust/src/benchfn.rs and wire benchmark mode (-b/-e/-i) into the Rust CLI frontend, which previously rejected those options as not yet implemented. `zstd -b1 -i0 FILE` and range runs like `zstd -b5e6 -i0 FILE` work again, including the synthetic-sample benchmark when no file is given. benchfn.rs is a faithful port of the run/timing state machine: BMK_benchFunction keeps the exact loop accounting (first-loop blockResults and errorFn checks, dstSize summed on the first loop only, 0xE5 warm-up of result buffers, nbLoops minimum of 1) and BMK_benchTimedFn keeps the same convergence behavior (x10 workload growth for short runs, budget-based nbLoops estimation, runs below half the run budget re-tried rather than reported, best qualifying run returned). Arithmetic that C leaves to unsigned wrap-around uses wrapping operations so debug builds cannot panic where release C would wrap. ABI notes: BMK_runTime_t and BMK_runOutcome_t are returned by value across the C boundary and BMK_benchParams_t is passed by value, so all three are repr(C) mirrors of benchfn.h; their field offsets are pinned by const asserts in Rust and matching C static asserts in the benchfn.c shim, which is now declaration-only. BMK_timedFnState_t stays opaque, fits the 64-byte BMK_timedFnState_shell (compile-time checked), and is malloc/free-managed so creation and destruction remain interchangeable with C callers. The CLI parses -b (bench mode), -e (range end, digits attach directly, defaulting to 0 like readU32FromChar) and -i (duration in seconds), then dispatches through a new ZSTD_rust_cli_bench bridge in the zstdcli.c shim. The bridge exists because benchmark availability is a C preprocessor property (ZSTD_NOBENCH): orchestration and reporting stay in C benchzstd.c, stripped variants (zstd-small, zstd-compress, zstd-decompress) compile the stub branch and report "benchmark mode is not available in this build", and the Rust side never references benchmark symbols directly. Level clamping against ZSTD_maxCLevel() happens in the bridge, where the symbol is guaranteed to exist whenever benchmarking is compiled in. -T selects the worker count, defaulting to single-threaded like the C bench path; -S (separate files) and --priority=rt remain unimplemented. Makefile updates only extend the Rust source prerequisite lists with benchfn.rs; the helpers-archive plumbing from the timefn commit already links fullbench(-lib/-dll/32) and paramgrill, the benchfn consumers among the C tests. Original C test sources are untouched. Known pre-existing issues, unchanged by this commit: tests/fullbench-lib fails to link at the base commit too (libzstd.a precedes fullbench.c in its link line), and the cli-tests basic/help.sh, compression/levels.sh, compression/golden.sh, and decompression/pass-through.sh scripts fail identically with a base-commit binary because the Rust CLI frontend is still a partial reimplementation. Test Plan: - cd rust && cargo fmt --check && cargo clippy --all-targets -- -D warnings && cargo test --all-targets && cargo build --release - cd rust/cli && cargo fmt --check && cargo clippy --all-targets -- -D warnings && cargo test --all-targets; repeat tests with --no-default-features plus features compression / decompression / (none) - make -C programs zstd; ./programs/zstd -b1 -i0 lib/common/xxhash.c; ./programs/zstd -b5e6 -i0 programs/fileio.c; ./programs/zstd -b1 -i0 (synthetic); echo roundtrip via zstd | zstd -d - make -C programs zstd-small zstd-compress zstd-decompress zstd-nolegacy zstd-dictBuilder; zstd-small -b reports benchmark unavailable; compress/ decompress roundtrip across the split binaries - make -C tests fullbench fuzzer zstreamtest paramgrill decodecorpus poolTests fullbench32 fuzzer32; ./tests/fullbench -i0 (exercises Rust BMK_benchTimedFn from C); ./tests/fullbench32 -i0; ./tests/fuzzer -i1 --no-big-tests; ./tests/poolTests; make -C tests test-rust-lib-smoke - cli-tests subset: basic/version.sh, compression/basic.sh, compression/multiple-files.sh pass; failing scripts match the base commit Refs: rust/README.md
1727 lines
55 KiB
Rust
1727 lines
55 KiB
Rust
#![allow(non_camel_case_types)]
|
|
#![allow(non_snake_case)]
|
|
#![allow(clippy::missing_safety_doc)]
|
|
|
|
//! Rust command-line frontend for zstd.
|
|
//!
|
|
//! This is intentionally a parser and dispatch layer, not a second file I/O
|
|
//! implementation. It reuses the mature C `fileio` layer through its narrow
|
|
//! public-in-the-programs-tree ABI: file opening, safe replacement, sparse
|
|
//! writes, dictionary loading, streaming, and metadata preservation remain in
|
|
//! `programs/fileio.c` for this first migration step.
|
|
//!
|
|
//! Benchmark mode (`-b`) parses here and dispatches through the
|
|
//! `ZSTD_rust_cli_bench` bridge in `programs/zstdcli.c`: the run/timing loop
|
|
//! (benchfn, timefn) is Rust, while orchestration and result formatting
|
|
//! (`benchzstd.c`) remain C behind the preprocessor-gated bridge, so builds
|
|
//! with `ZSTD_NOBENCH` never reference benchmark symbols.
|
|
//!
|
|
//! Remaining C-only CLI boundaries are called out in `unsupported()` below:
|
|
//! dictionary training, recursive/file-list expansion, tracing,
|
|
//! alternate-format selection, and the advanced directory modes.
|
|
|
|
use std::env;
|
|
use std::ffi::{CStr, CString, OsStr, OsString};
|
|
use std::fs;
|
|
use std::io::{self, IsTerminal, Write};
|
|
use std::os::raw::{c_char, c_int, c_uint};
|
|
use std::path::Path;
|
|
use std::ptr;
|
|
|
|
#[cfg(unix)]
|
|
use std::os::unix::ffi::{OsStrExt, OsStringExt};
|
|
#[cfg(unix)]
|
|
use std::os::unix::fs::FileTypeExt;
|
|
|
|
const DEFAULT_CLEVEL: i32 = 3;
|
|
#[cfg(feature = "compression")]
|
|
const DEFAULT_MAX_CLEVEL: i32 = 19;
|
|
const DEFAULT_BENCH_NB_SECONDS: u32 = 3;
|
|
const DEFAULT_MEM_LIMIT: u32 = 1 << 27;
|
|
const DEFAULT_LONG_WINDOW_LOG: u32 = 27;
|
|
const MAX_FAST_ACCELERATION: i32 = 128 << 10;
|
|
const STDIN_MARK: &str = "/*stdin*\\";
|
|
const STDOUT_MARK: &str = "/*stdout*\\";
|
|
#[cfg(windows)]
|
|
const NULL_MARK: &str = "NUL";
|
|
#[cfg(not(windows))]
|
|
const NULL_MARK: &str = "/dev/null";
|
|
#[cfg(feature = "compression")]
|
|
const ZSTD_SUFFIX: &[u8] = b".zst\0";
|
|
|
|
const FIO_ZSTD_COMPRESSION: c_int = 0;
|
|
const FIO_PS_AUTO: c_int = 0;
|
|
const FIO_PS_NEVER: c_int = 1;
|
|
const FIO_PS_ALWAYS: c_int = 2;
|
|
const ZSTD_PS_AUTO: c_int = 0;
|
|
const ZSTD_PS_ENABLE: c_int = 1;
|
|
const ZSTD_PS_DISABLE: c_int = 2;
|
|
|
|
#[repr(C)]
|
|
struct FIO_prefs_t {
|
|
_private: [u8; 0],
|
|
}
|
|
|
|
#[repr(C)]
|
|
struct FIO_ctx_t {
|
|
_private: [u8; 0],
|
|
}
|
|
|
|
#[repr(C)]
|
|
#[derive(Clone, Copy, Debug, Default)]
|
|
struct ZSTD_compressionParameters {
|
|
windowLog: u32,
|
|
chainLog: u32,
|
|
hashLog: u32,
|
|
searchLog: u32,
|
|
minMatch: u32,
|
|
targetLength: u32,
|
|
strategy: c_int,
|
|
}
|
|
|
|
unsafe extern "C" {
|
|
fn ZSTD_versionString() -> *const c_char;
|
|
fn ZSTD_rust_cli_expected_version() -> *const c_char;
|
|
static mut g_utilDisplayLevel: c_int;
|
|
#[cfg(feature = "compression")]
|
|
fn ZSTD_minCLevel() -> c_int;
|
|
#[cfg(feature = "compression")]
|
|
fn ZSTD_maxCLevel() -> c_int;
|
|
#[cfg(feature = "compression")]
|
|
fn UTIL_countPhysicalCores() -> c_int;
|
|
#[cfg(feature = "compression")]
|
|
fn UTIL_countLogicalCores() -> c_int;
|
|
|
|
fn FIO_createPreferences() -> *mut FIO_prefs_t;
|
|
fn FIO_freePreferences(prefs: *mut FIO_prefs_t);
|
|
fn FIO_createContext() -> *mut FIO_ctx_t;
|
|
fn FIO_freeContext(ctx: *mut FIO_ctx_t);
|
|
fn FIO_addAbortHandler();
|
|
|
|
fn FIO_setCompressionType(prefs: *mut FIO_prefs_t, compression_type: c_int);
|
|
fn FIO_overwriteMode(prefs: *mut FIO_prefs_t);
|
|
fn FIO_setAdaptiveMode(prefs: *mut FIO_prefs_t, adapt: c_int);
|
|
#[cfg(feature = "compression")]
|
|
fn FIO_setAdaptMin(prefs: *mut FIO_prefs_t, level: c_int);
|
|
#[cfg(feature = "compression")]
|
|
fn FIO_setAdaptMax(prefs: *mut FIO_prefs_t, level: c_int);
|
|
fn FIO_setUseRowMatchFinder(prefs: *mut FIO_prefs_t, mode: c_int);
|
|
fn FIO_setBlockSize(prefs: *mut FIO_prefs_t, block_size: c_int);
|
|
fn FIO_setChecksumFlag(prefs: *mut FIO_prefs_t, checksum: c_int);
|
|
fn FIO_setDictIDFlag(prefs: *mut FIO_prefs_t, dict_id: c_int);
|
|
fn FIO_setLdmBucketSizeLog(prefs: *mut FIO_prefs_t, value: c_int);
|
|
fn FIO_setLdmFlag(prefs: *mut FIO_prefs_t, value: c_uint);
|
|
fn FIO_setLdmHashRateLog(prefs: *mut FIO_prefs_t, value: c_int);
|
|
fn FIO_setLdmHashLog(prefs: *mut FIO_prefs_t, value: c_int);
|
|
fn FIO_setLdmMinMatch(prefs: *mut FIO_prefs_t, value: c_int);
|
|
fn FIO_setMemLimit(prefs: *mut FIO_prefs_t, limit: c_uint);
|
|
#[cfg(feature = "compression")]
|
|
fn FIO_setNbWorkers(prefs: *mut FIO_prefs_t, workers: c_int);
|
|
fn FIO_setOverlapLog(prefs: *mut FIO_prefs_t, value: c_int);
|
|
fn FIO_setRemoveSrcFile(prefs: *mut FIO_prefs_t, value: c_int);
|
|
fn FIO_setSparseWrite(prefs: *mut FIO_prefs_t, value: c_int);
|
|
fn FIO_setRsyncable(prefs: *mut FIO_prefs_t, value: c_int);
|
|
fn FIO_setStreamSrcSize(prefs: *mut FIO_prefs_t, value: usize);
|
|
fn FIO_setTargetCBlockSize(prefs: *mut FIO_prefs_t, value: usize);
|
|
fn FIO_setSrcSizeHint(prefs: *mut FIO_prefs_t, value: usize);
|
|
#[cfg(feature = "decompression")]
|
|
fn FIO_setTestMode(prefs: *mut FIO_prefs_t, value: c_int);
|
|
fn FIO_setLiteralCompressionMode(prefs: *mut FIO_prefs_t, value: c_int);
|
|
fn FIO_setProgressSetting(value: c_int);
|
|
fn FIO_setNotificationLevel(value: c_int);
|
|
fn FIO_setExcludeCompressedFile(prefs: *mut FIO_prefs_t, value: c_int);
|
|
fn FIO_setAllowBlockDevices(prefs: *mut FIO_prefs_t, value: c_int);
|
|
fn FIO_setContentSize(prefs: *mut FIO_prefs_t, value: c_int);
|
|
fn FIO_setAsyncIOFlag(prefs: *mut FIO_prefs_t, value: c_int);
|
|
fn FIO_setPassThroughFlag(prefs: *mut FIO_prefs_t, value: c_int);
|
|
fn FIO_setMMapDict(prefs: *mut FIO_prefs_t, value: c_int);
|
|
fn FIO_setNbFilesTotal(ctx: *mut FIO_ctx_t, value: c_int);
|
|
fn FIO_setHasStdinInput(ctx: *mut FIO_ctx_t, value: c_int);
|
|
fn FIO_setHasStdoutOutput(ctx: *mut FIO_ctx_t, value: c_int);
|
|
|
|
#[cfg(feature = "compression")]
|
|
fn FIO_compressFilename(
|
|
ctx: *mut FIO_ctx_t,
|
|
prefs: *mut FIO_prefs_t,
|
|
output: *const c_char,
|
|
input: *const c_char,
|
|
dict: *const c_char,
|
|
level: c_int,
|
|
params: ZSTD_compressionParameters,
|
|
) -> c_int;
|
|
#[cfg(feature = "decompression")]
|
|
fn FIO_decompressFilename(
|
|
ctx: *mut FIO_ctx_t,
|
|
prefs: *mut FIO_prefs_t,
|
|
output: *const c_char,
|
|
input: *const c_char,
|
|
dict: *const c_char,
|
|
) -> c_int;
|
|
#[cfg(feature = "compression")]
|
|
fn FIO_compressMultipleFilenames(
|
|
ctx: *mut FIO_ctx_t,
|
|
prefs: *mut FIO_prefs_t,
|
|
inputs: *const *const c_char,
|
|
output_mirror_dir: *const c_char,
|
|
output_dir: *const c_char,
|
|
output: *const c_char,
|
|
suffix: *const c_char,
|
|
dict: *const c_char,
|
|
level: c_int,
|
|
params: ZSTD_compressionParameters,
|
|
) -> c_int;
|
|
#[cfg(feature = "decompression")]
|
|
fn FIO_decompressMultipleFilenames(
|
|
ctx: *mut FIO_ctx_t,
|
|
prefs: *mut FIO_prefs_t,
|
|
inputs: *const *const c_char,
|
|
output_mirror_dir: *const c_char,
|
|
output_dir: *const c_char,
|
|
output: *const c_char,
|
|
dict: *const c_char,
|
|
) -> c_int;
|
|
|
|
/// Benchmark bridge implemented by the `programs/zstdcli.c` shim, which
|
|
/// owns the `ZSTD_NOBENCH` preprocessor decision. Returns the benchmark
|
|
/// result (>= 0), or -1 when benchmarking is compiled out.
|
|
fn ZSTD_rust_cli_bench(
|
|
file_names: *const *const c_char,
|
|
nb_files: c_uint,
|
|
dict_file_name: *const c_char,
|
|
start_level: c_int,
|
|
end_level: c_int,
|
|
compression_params: *const ZSTD_compressionParameters,
|
|
display_level: c_int,
|
|
nb_seconds: c_uint,
|
|
block_size: usize,
|
|
nb_workers: c_int,
|
|
) -> c_int;
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
enum Operation {
|
|
Compress,
|
|
Decompress,
|
|
Test,
|
|
Bench,
|
|
}
|
|
|
|
#[derive(Debug)]
|
|
enum Action {
|
|
Run(Box<Cli>),
|
|
Help { advanced: bool },
|
|
Version { quiet: bool },
|
|
}
|
|
|
|
#[derive(Debug)]
|
|
struct Cli {
|
|
operation: Operation,
|
|
inputs: Vec<CString>,
|
|
output: Option<CString>,
|
|
dictionary: Option<CString>,
|
|
level: i32,
|
|
ultra: bool,
|
|
display_level: i32,
|
|
force: bool,
|
|
force_stdout: bool,
|
|
remove_source: bool,
|
|
checksum: Option<i32>,
|
|
sparse: Option<i32>,
|
|
pass_through: Option<i32>,
|
|
content_size: i32,
|
|
dict_id: Option<i32>,
|
|
async_io: Option<i32>,
|
|
mmap_dict: i32,
|
|
progress: i32,
|
|
workers: Option<i32>,
|
|
single_thread: bool,
|
|
block_size: Option<usize>,
|
|
mem_limit: Option<u32>,
|
|
ldm: bool,
|
|
ldm_hash_log: Option<i32>,
|
|
ldm_min_match: Option<i32>,
|
|
ldm_bucket_size_log: Option<i32>,
|
|
ldm_hash_rate_log: Option<i32>,
|
|
overlap_log: Option<i32>,
|
|
adapt: bool,
|
|
adapt_min: Option<i32>,
|
|
adapt_max: Option<i32>,
|
|
rsyncable: bool,
|
|
stream_src_size: Option<usize>,
|
|
target_cblock_size: Option<usize>,
|
|
src_size_hint: Option<usize>,
|
|
literal_compression: Option<i32>,
|
|
row_match_finder: i32,
|
|
exclude_compressed: bool,
|
|
compression_params: ZSTD_compressionParameters,
|
|
bench_end_level: Option<i32>,
|
|
bench_nb_seconds: Option<u32>,
|
|
unsupported_program: Option<String>,
|
|
}
|
|
|
|
impl Cli {
|
|
fn new(program_name: &str) -> Self {
|
|
let mut cli = Self {
|
|
operation: Operation::Compress,
|
|
inputs: Vec::new(),
|
|
output: None,
|
|
dictionary: None,
|
|
level: default_level(),
|
|
ultra: false,
|
|
display_level: 2,
|
|
force: false,
|
|
force_stdout: false,
|
|
remove_source: false,
|
|
checksum: None,
|
|
sparse: None,
|
|
pass_through: None,
|
|
content_size: 1,
|
|
dict_id: None,
|
|
async_io: None,
|
|
mmap_dict: ZSTD_PS_AUTO,
|
|
progress: FIO_PS_AUTO,
|
|
workers: None,
|
|
single_thread: false,
|
|
block_size: None,
|
|
mem_limit: None,
|
|
ldm: false,
|
|
ldm_hash_log: None,
|
|
ldm_min_match: None,
|
|
ldm_bucket_size_log: None,
|
|
ldm_hash_rate_log: None,
|
|
overlap_log: None,
|
|
adapt: false,
|
|
adapt_min: None,
|
|
adapt_max: None,
|
|
rsyncable: false,
|
|
stream_src_size: None,
|
|
target_cblock_size: None,
|
|
src_size_hint: None,
|
|
literal_compression: None,
|
|
row_match_finder: ZSTD_PS_AUTO,
|
|
exclude_compressed: false,
|
|
compression_params: ZSTD_compressionParameters::default(),
|
|
bench_end_level: None,
|
|
bench_nb_seconds: None,
|
|
unsupported_program: None,
|
|
};
|
|
|
|
match program_name {
|
|
"unzstd" => cli.operation = Operation::Decompress,
|
|
"zstdmt" => cli.workers = Some(0),
|
|
"zstdcat" | "zcat" => {
|
|
cli.operation = Operation::Decompress;
|
|
cli.output = Some(cstring(STDOUT_MARK).expect("static stdout marker"));
|
|
cli.force = true;
|
|
cli.force_stdout = true;
|
|
cli.pass_through = Some(1);
|
|
cli.display_level = 1;
|
|
}
|
|
"gzip" | "gunzip" | "gzcat" | "lzma" | "unlzma" | "xz" | "unxz" | "lz4" | "unlz4" => {
|
|
cli.unsupported_program = Some(program_name.to_owned())
|
|
}
|
|
_ => {}
|
|
}
|
|
cli
|
|
}
|
|
}
|
|
|
|
fn cstring(value: &str) -> Result<CString, String> {
|
|
CString::new(value).map_err(|_| format!("argument contains an interior NUL: {value:?}"))
|
|
}
|
|
|
|
fn os_cstring(value: &OsStr) -> Result<CString, String> {
|
|
#[cfg(unix)]
|
|
{
|
|
CString::new(value.as_bytes()).map_err(|_| "file name contains an interior NUL".to_owned())
|
|
}
|
|
#[cfg(not(unix))]
|
|
{
|
|
cstring(&value.to_string_lossy())
|
|
}
|
|
}
|
|
|
|
fn default_level() -> i32 {
|
|
match env::var("ZSTD_CLEVEL") {
|
|
Ok(value) => value.parse::<i32>().unwrap_or(DEFAULT_CLEVEL),
|
|
Err(_) => DEFAULT_CLEVEL,
|
|
}
|
|
}
|
|
|
|
#[cfg(feature = "compression")]
|
|
unsafe fn default_worker_count() -> i32 {
|
|
if let Ok(value) = env::var("ZSTD_NBTHREADS") {
|
|
if let Ok(workers) = value.parse::<u32>() {
|
|
if let Ok(workers) = i32::try_from(workers) {
|
|
return workers;
|
|
}
|
|
}
|
|
}
|
|
let logical_cores = unsafe { UTIL_countLogicalCores() }.max(1);
|
|
(logical_cores / 4).clamp(1, 4)
|
|
}
|
|
|
|
#[cfg(feature = "compression")]
|
|
unsafe fn resolved_worker_count(workers: Option<i32>, single_thread: bool) -> i32 {
|
|
match workers {
|
|
/* --single-thread pins zero workers; a bare zero (-T0 or the zstdmt
|
|
* program name) auto-detects the core count as in the C CLI. */
|
|
Some(0) if single_thread => 0,
|
|
Some(0) => unsafe { UTIL_countPhysicalCores() }.max(1),
|
|
Some(workers) => workers,
|
|
None => unsafe { default_worker_count() },
|
|
}
|
|
}
|
|
|
|
fn program_basename(value: &OsStr) -> String {
|
|
Path::new(value)
|
|
.file_name()
|
|
.unwrap_or(value)
|
|
.to_string_lossy()
|
|
.split('.')
|
|
.next()
|
|
.unwrap_or("zstd")
|
|
.to_owned()
|
|
}
|
|
|
|
fn usage(advanced: bool) {
|
|
let mut out = io::stdout().lock();
|
|
let _ = writeln!(
|
|
out,
|
|
"Compress or decompress INPUT file(s); reads stdin when INPUT is '-' or omitted."
|
|
);
|
|
let _ = writeln!(out, "\nUsage: zstd [OPTIONS...] [INPUT... | -] [-o OUTPUT]");
|
|
let _ = writeln!(out, "\nCore options:");
|
|
let _ = writeln!(
|
|
out,
|
|
" -o OUTPUT, -c, --stdout Select output file or stdout"
|
|
);
|
|
let _ = writeln!(out, " -d, --decompress Decompress");
|
|
let _ = writeln!(out, " -t, --test Test compressed input");
|
|
let _ = writeln!(
|
|
out,
|
|
" -# Compression level (default {DEFAULT_CLEVEL})"
|
|
);
|
|
let _ = writeln!(out, " -D DICT Use a dictionary");
|
|
let _ = writeln!(
|
|
out,
|
|
" -f, --force Overwrite output / allow stdio"
|
|
);
|
|
let _ = writeln!(
|
|
out,
|
|
" -k, --keep | --rm Preserve or remove source after success"
|
|
);
|
|
let _ = writeln!(out, " -q, --quiet | -v, --verbose Adjust display level");
|
|
let _ = writeln!(out, " -V, --version Print version");
|
|
let _ = writeln!(out, " -h | -H, --help Print help");
|
|
if advanced {
|
|
let _ = writeln!(out, "\nImplemented advanced compression controls:");
|
|
let _ = writeln!(
|
|
out,
|
|
" --fast[=#], --ultra, --long[=#], --threads=#, --single-thread, --block-size=#"
|
|
);
|
|
let _ = writeln!(
|
|
out,
|
|
" --zstd=wlog=#,clog=#,hlog=#,slog=#,mml=#,tlen=#,strat=#"
|
|
);
|
|
let _ = writeln!(
|
|
out,
|
|
" --[no-]check, --[no-]sparse, --[no-]progress, --[no-]asyncio"
|
|
);
|
|
let _ = writeln!(
|
|
out,
|
|
" --adapt[=min=#,max=#], --rsyncable, --[no-]row-match-finder"
|
|
);
|
|
let _ = writeln!(out, "\nBenchmark options:");
|
|
let _ = writeln!(
|
|
out,
|
|
" -b# Benchmark file(s) at compression level #"
|
|
);
|
|
let _ = writeln!(
|
|
out,
|
|
" -e# Test all levels from -b# up to # included"
|
|
);
|
|
let _ = writeln!(
|
|
out,
|
|
" -i# Set the minimum evaluation time to # seconds"
|
|
);
|
|
let _ = writeln!(
|
|
out,
|
|
"\nNot yet migrated: dictionary training, recursive/file-list expansion,"
|
|
);
|
|
let _ = writeln!(out, "trace, alternate formats, and output-directory modes.");
|
|
}
|
|
}
|
|
|
|
fn print_version(quiet: bool) {
|
|
let version = unsafe { CStr::from_ptr(ZSTD_versionString()) }
|
|
.to_string_lossy()
|
|
.into_owned();
|
|
if quiet {
|
|
println!("{version}");
|
|
} else {
|
|
println!(
|
|
"*** Zstandard CLI ({}-bit) v{version}, by Yann Collet ***",
|
|
usize::BITS
|
|
);
|
|
}
|
|
}
|
|
|
|
fn check_lib_version() -> Result<(), String> {
|
|
let expected = unsafe { CStr::from_ptr(ZSTD_rust_cli_expected_version()) };
|
|
let actual = unsafe { CStr::from_ptr(ZSTD_versionString()) };
|
|
if expected == actual {
|
|
return Ok(());
|
|
}
|
|
Err(format!(
|
|
"incorrect library version (expecting: {}; actual: {})",
|
|
expected.to_string_lossy(),
|
|
actual.to_string_lossy()
|
|
))
|
|
}
|
|
|
|
fn parse_size(value: &str) -> Result<usize, String> {
|
|
let split = value
|
|
.find(|character: char| !character.is_ascii_digit())
|
|
.unwrap_or(value.len());
|
|
let (digits, suffix) = value.split_at(split);
|
|
if digits.is_empty() {
|
|
return Err(format!("expected a numeric value, got {value:?}"));
|
|
}
|
|
let mut number = digits
|
|
.parse::<usize>()
|
|
.map_err(|_| format!("numeric value overflows size_t: {value:?}"))?;
|
|
let normalized = suffix.trim_end_matches('B').trim_end_matches('i');
|
|
let shift = match normalized {
|
|
"" => 0,
|
|
"K" | "k" => 10,
|
|
"M" | "m" => 20,
|
|
"G" | "g" => 30,
|
|
_ => return Err(format!("unsupported numeric suffix in {value:?}")),
|
|
};
|
|
number = number
|
|
.checked_shl(shift)
|
|
.ok_or_else(|| format!("numeric value overflows size_t: {value:?}"))?;
|
|
Ok(number)
|
|
}
|
|
|
|
fn parse_u32(value: &str, name: &str) -> Result<u32, String> {
|
|
let size = parse_size(value)?;
|
|
u32::try_from(size).map_err(|_| format!("{name} is too large: {value:?}"))
|
|
}
|
|
|
|
fn parse_i32(value: &str, name: &str) -> Result<i32, String> {
|
|
value
|
|
.parse::<i32>()
|
|
.map_err(|_| format!("invalid {name}: {value:?}"))
|
|
}
|
|
|
|
fn parse_worker_count(value: &str) -> Result<i32, String> {
|
|
let workers = parse_i32(value, "thread count")?;
|
|
if workers < 0 {
|
|
return Err(format!("thread count must not be negative: {value:?}"));
|
|
}
|
|
Ok(workers)
|
|
}
|
|
|
|
fn next_value(
|
|
attached: Option<&str>,
|
|
args: &[OsString],
|
|
index: &mut usize,
|
|
option: &str,
|
|
) -> Result<String, String> {
|
|
if let Some(value) = attached {
|
|
if !value.is_empty() {
|
|
return Ok(value.to_owned());
|
|
}
|
|
}
|
|
*index += 1;
|
|
let Some(value) = args.get(*index) else {
|
|
return Err(format!("missing argument for {option}"));
|
|
};
|
|
let rendered = value.to_string_lossy().into_owned();
|
|
if rendered.starts_with('-') {
|
|
return Err(format!(
|
|
"{option} cannot be separated from its argument by another option"
|
|
));
|
|
}
|
|
Ok(rendered)
|
|
}
|
|
|
|
fn next_os_value(args: &[OsString], index: &mut usize, option: &str) -> Result<OsString, String> {
|
|
*index += 1;
|
|
let Some(value) = args.get(*index) else {
|
|
return Err(format!("missing argument for {option}"));
|
|
};
|
|
if value.to_string_lossy().starts_with('-') {
|
|
return Err(format!(
|
|
"{option} cannot be separated from its argument by another option"
|
|
));
|
|
}
|
|
Ok(value.clone())
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
fn short_attached_value(value: &OsStr, start: usize) -> Option<OsString> {
|
|
let bytes = &value.as_bytes()[start..];
|
|
if bytes.is_empty() {
|
|
return None;
|
|
}
|
|
let bytes = if bytes.first() == Some(&b'=') {
|
|
&bytes[1..]
|
|
} else {
|
|
bytes
|
|
};
|
|
Some(OsString::from_vec(bytes.to_vec()))
|
|
}
|
|
|
|
#[cfg(not(unix))]
|
|
fn short_attached_value(value: &OsStr, start: usize) -> Option<OsString> {
|
|
let rendered = value.to_string_lossy();
|
|
let attached = &rendered[start..];
|
|
if attached.is_empty() {
|
|
return None;
|
|
}
|
|
Some(OsString::from(
|
|
attached.strip_prefix('=').unwrap_or(attached),
|
|
))
|
|
}
|
|
|
|
fn parse_compression_parameters(value: &str, cli: &mut Cli) -> Result<(), String> {
|
|
for item in value.split(',') {
|
|
let Some((name, raw)) = item.split_once('=') else {
|
|
return Err(format!("invalid --zstd parameter {item:?}"));
|
|
};
|
|
let parsed = parse_u32(raw, name)?;
|
|
match name {
|
|
"windowLog" | "wlog" => cli.compression_params.windowLog = parsed,
|
|
"chainLog" | "clog" => cli.compression_params.chainLog = parsed,
|
|
"hashLog" | "hlog" => cli.compression_params.hashLog = parsed,
|
|
"searchLog" | "slog" => cli.compression_params.searchLog = parsed,
|
|
"minMatch" | "mml" => cli.compression_params.minMatch = parsed,
|
|
"targetLength" | "tlen" => cli.compression_params.targetLength = parsed,
|
|
"strategy" | "strat" => cli.compression_params.strategy = parsed as c_int,
|
|
"overlapLog" | "ovlog" => cli.overlap_log = Some(parsed as i32),
|
|
"ldmHashLog" | "lhlog" => cli.ldm_hash_log = Some(parsed as i32),
|
|
"ldmMinMatch" | "lmml" => cli.ldm_min_match = Some(parsed as i32),
|
|
"ldmBucketSizeLog" | "lblog" => cli.ldm_bucket_size_log = Some(parsed as i32),
|
|
"ldmHashRateLog" | "lhrlog" => cli.ldm_hash_rate_log = Some(parsed as i32),
|
|
_ => return Err(format!("unknown --zstd parameter {name:?}")),
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn parse_adapt(value: &str, cli: &mut Cli) -> Result<(), String> {
|
|
cli.adapt = true;
|
|
if value.is_empty() {
|
|
return Ok(());
|
|
}
|
|
for item in value.split(',') {
|
|
let Some((name, raw)) = item.split_once('=') else {
|
|
return Err(format!("invalid --adapt parameter {item:?}"));
|
|
};
|
|
match name {
|
|
"min" => cli.adapt_min = Some(parse_i32(raw, "adapt minimum")?),
|
|
"max" => cli.adapt_max = Some(parse_i32(raw, "adapt maximum")?),
|
|
_ => return Err(format!("unknown --adapt parameter {name:?}")),
|
|
}
|
|
}
|
|
if let (Some(minimum), Some(maximum)) = (cli.adapt_min, cli.adapt_max) {
|
|
if minimum > maximum {
|
|
return Err("--adapt minimum must not exceed its maximum".to_owned());
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn unsupported(option: &str) -> Result<(), String> {
|
|
Err(format!(
|
|
"{option} is not yet implemented by the Rust CLI frontend"
|
|
))
|
|
}
|
|
|
|
fn parse_long_option(
|
|
option: &str,
|
|
args: &[OsString],
|
|
index: &mut usize,
|
|
cli: &mut Cli,
|
|
) -> Result<Option<Action>, String> {
|
|
let (name, attached) = option
|
|
.split_once('=')
|
|
.map_or((option, None), |(name, value)| (name, Some(value)));
|
|
if attached.is_some()
|
|
&& matches!(
|
|
name,
|
|
"--compress"
|
|
| "--decompress"
|
|
| "--uncompress"
|
|
| "--test"
|
|
| "--force"
|
|
| "--keep"
|
|
| "--rm"
|
|
| "--stdout"
|
|
| "--version"
|
|
| "--help"
|
|
| "--verbose"
|
|
| "--quiet"
|
|
| "--check"
|
|
| "--no-check"
|
|
| "--sparse"
|
|
| "--no-sparse"
|
|
| "--pass-through"
|
|
| "--no-pass-through"
|
|
| "--content-size"
|
|
| "--no-content-size"
|
|
| "--no-dictID"
|
|
| "--asyncio"
|
|
| "--no-asyncio"
|
|
| "--mmap-dict"
|
|
| "--no-mmap-dict"
|
|
| "--progress"
|
|
| "--no-progress"
|
|
| "--ultra"
|
|
| "--no-row-match-finder"
|
|
| "--row-match-finder"
|
|
| "--rsyncable"
|
|
| "--single-thread"
|
|
| "--compress-literals"
|
|
| "--no-compress-literals"
|
|
| "--exclude-compressed"
|
|
| "--no-name"
|
|
)
|
|
{
|
|
return Err(format!("{name} does not take an argument"));
|
|
}
|
|
match name {
|
|
"--" => Ok(None),
|
|
"--compress" => {
|
|
cli.operation = Operation::Compress;
|
|
Ok(None)
|
|
}
|
|
"--decompress" | "--uncompress" => {
|
|
cli.operation = Operation::Decompress;
|
|
Ok(None)
|
|
}
|
|
"--test" => {
|
|
cli.operation = Operation::Test;
|
|
Ok(None)
|
|
}
|
|
"--force" => {
|
|
cli.force = true;
|
|
Ok(None)
|
|
}
|
|
"--keep" => {
|
|
cli.remove_source = false;
|
|
Ok(None)
|
|
}
|
|
"--no-name" => Ok(None),
|
|
"--rm" => {
|
|
cli.remove_source = true;
|
|
Ok(None)
|
|
}
|
|
"--stdout" => {
|
|
cli.output = Some(cstring(STDOUT_MARK)?);
|
|
cli.force_stdout = true;
|
|
Ok(None)
|
|
}
|
|
"--version" => Ok(Some(Action::Version {
|
|
quiet: cli.display_level < 2,
|
|
})),
|
|
"--help" => Ok(Some(Action::Help { advanced: true })),
|
|
"--verbose" => {
|
|
cli.display_level += 1;
|
|
Ok(None)
|
|
}
|
|
"--quiet" => {
|
|
cli.display_level -= 1;
|
|
Ok(None)
|
|
}
|
|
"--check" => {
|
|
cli.checksum = Some(2);
|
|
Ok(None)
|
|
}
|
|
"--no-check" => {
|
|
cli.checksum = Some(0);
|
|
Ok(None)
|
|
}
|
|
"--sparse" => {
|
|
cli.sparse = Some(2);
|
|
Ok(None)
|
|
}
|
|
"--no-sparse" => {
|
|
cli.sparse = Some(0);
|
|
Ok(None)
|
|
}
|
|
"--pass-through" => {
|
|
cli.pass_through = Some(1);
|
|
Ok(None)
|
|
}
|
|
"--no-pass-through" => {
|
|
cli.pass_through = Some(0);
|
|
Ok(None)
|
|
}
|
|
"--content-size" => {
|
|
cli.content_size = 1;
|
|
Ok(None)
|
|
}
|
|
"--no-content-size" => {
|
|
cli.content_size = 0;
|
|
Ok(None)
|
|
}
|
|
"--no-dictID" => {
|
|
cli.dict_id = Some(0);
|
|
Ok(None)
|
|
}
|
|
"--asyncio" => {
|
|
cli.async_io = Some(1);
|
|
Ok(None)
|
|
}
|
|
"--no-asyncio" => {
|
|
cli.async_io = Some(0);
|
|
Ok(None)
|
|
}
|
|
"--mmap-dict" => {
|
|
cli.mmap_dict = ZSTD_PS_ENABLE;
|
|
Ok(None)
|
|
}
|
|
"--no-mmap-dict" => {
|
|
cli.mmap_dict = ZSTD_PS_DISABLE;
|
|
Ok(None)
|
|
}
|
|
"--progress" => {
|
|
cli.progress = FIO_PS_ALWAYS;
|
|
Ok(None)
|
|
}
|
|
"--no-progress" => {
|
|
cli.progress = FIO_PS_NEVER;
|
|
Ok(None)
|
|
}
|
|
"--ultra" => {
|
|
cli.ultra = true;
|
|
Ok(None)
|
|
}
|
|
"--fast" => {
|
|
let mut level = match attached {
|
|
Some(value) => parse_i32(value, "fast level")?,
|
|
None => 1,
|
|
};
|
|
if level <= 0 {
|
|
return Err("fast level must be positive".to_owned());
|
|
}
|
|
level = level.min(MAX_FAST_ACCELERATION);
|
|
cli.level = -level;
|
|
Ok(None)
|
|
}
|
|
"--long" => {
|
|
cli.ldm = true;
|
|
cli.ultra = true;
|
|
if let Some(value) = attached {
|
|
cli.compression_params.windowLog = parse_u32(value, "long window log")?;
|
|
} else if cli.compression_params.windowLog == 0 {
|
|
cli.compression_params.windowLog = DEFAULT_LONG_WINDOW_LOG;
|
|
}
|
|
Ok(None)
|
|
}
|
|
"--adapt" => {
|
|
parse_adapt(attached.unwrap_or(""), cli)?;
|
|
Ok(None)
|
|
}
|
|
"--no-row-match-finder" => {
|
|
cli.row_match_finder = ZSTD_PS_DISABLE;
|
|
Ok(None)
|
|
}
|
|
"--row-match-finder" => {
|
|
cli.row_match_finder = ZSTD_PS_ENABLE;
|
|
Ok(None)
|
|
}
|
|
"--rsyncable" => {
|
|
cli.rsyncable = true;
|
|
Ok(None)
|
|
}
|
|
"--compress-literals" => {
|
|
cli.literal_compression = Some(ZSTD_PS_ENABLE);
|
|
Ok(None)
|
|
}
|
|
"--no-compress-literals" => {
|
|
cli.literal_compression = Some(ZSTD_PS_DISABLE);
|
|
Ok(None)
|
|
}
|
|
"--exclude-compressed" => {
|
|
cli.exclude_compressed = true;
|
|
Ok(None)
|
|
}
|
|
"--threads" => {
|
|
let value = next_value(attached, args, index, "--threads")?;
|
|
cli.workers = Some(parse_worker_count(&value)?);
|
|
Ok(None)
|
|
}
|
|
"--single-thread" => {
|
|
/* As in the C CLI: zero workers plus a latch that suppresses the
|
|
* automatic core-count resolution, so fileio runs its
|
|
* single-thread streaming mode (slightly different from -T1). */
|
|
cli.workers = Some(0);
|
|
cli.single_thread = true;
|
|
Ok(None)
|
|
}
|
|
"--memlimit" | "--memory" | "--memlimit-decompress" => {
|
|
let value = next_value(attached, args, index, name)?;
|
|
cli.mem_limit = Some(parse_u32(&value, "memory limit")?);
|
|
Ok(None)
|
|
}
|
|
"--block-size" => {
|
|
let value = next_value(attached, args, index, name)?;
|
|
cli.block_size = Some(parse_size(&value)?);
|
|
Ok(None)
|
|
}
|
|
"--stream-size" => {
|
|
let value = next_value(attached, args, index, name)?;
|
|
cli.stream_src_size = Some(parse_size(&value)?);
|
|
Ok(None)
|
|
}
|
|
"--target-compressed-block-size" => {
|
|
let value = next_value(attached, args, index, name)?;
|
|
cli.target_cblock_size = Some(parse_size(&value)?);
|
|
Ok(None)
|
|
}
|
|
"--size-hint" => {
|
|
let value = next_value(attached, args, index, name)?;
|
|
cli.src_size_hint = Some(parse_size(&value)?);
|
|
Ok(None)
|
|
}
|
|
"--zstd" => {
|
|
let value = next_value(attached, args, index, "--zstd")?;
|
|
parse_compression_parameters(&value, cli)?;
|
|
Ok(None)
|
|
}
|
|
"--list"
|
|
| "--train"
|
|
| "--train-cover"
|
|
| "--train-fastcover"
|
|
| "--train-legacy"
|
|
| "--max"
|
|
| "--maxdict"
|
|
| "--dictID"
|
|
| "--filelist"
|
|
| "--output-dir-flat"
|
|
| "--output-dir-mirror"
|
|
| "--patch-from"
|
|
| "--trace"
|
|
| "--format"
|
|
| "--priority"
|
|
| "--auto-threads"
|
|
| "--fake-stdin-is-console"
|
|
| "--fake-stdout-is-console"
|
|
| "--fake-stderr-is-console"
|
|
| "--trace-file-stat"
|
|
| "--show-default-cparams" => {
|
|
unsupported(name)?;
|
|
Ok(None)
|
|
}
|
|
_ => Err(format!("unknown option {option:?}")),
|
|
}
|
|
}
|
|
|
|
fn parse_short_options(
|
|
value: &str,
|
|
raw_value: &OsStr,
|
|
args: &[OsString],
|
|
index: &mut usize,
|
|
cli: &mut Cli,
|
|
) -> Result<Option<Action>, String> {
|
|
let mut offset = 1usize;
|
|
let bytes = value.as_bytes();
|
|
while offset < bytes.len() {
|
|
let option = bytes[offset] as char;
|
|
match option {
|
|
'0'..='9' => {
|
|
let mut digits_end = offset;
|
|
while digits_end < bytes.len() && bytes[digits_end].is_ascii_digit() {
|
|
digits_end += 1;
|
|
}
|
|
cli.level = parse_i32(&value[offset..digits_end], "compression level")?;
|
|
offset = digits_end;
|
|
continue;
|
|
}
|
|
'd' => cli.operation = Operation::Decompress,
|
|
'z' => cli.operation = Operation::Compress,
|
|
't' => cli.operation = Operation::Test,
|
|
'b' => cli.operation = Operation::Bench,
|
|
'e' | 'i' => {
|
|
// Benchmark range end (-e#) and duration (-i#): like the C
|
|
// parser, digits attach directly and default to 0.
|
|
let mut digits_end = offset + 1;
|
|
while digits_end < bytes.len() && bytes[digits_end].is_ascii_digit() {
|
|
digits_end += 1;
|
|
}
|
|
let digits = &value[offset + 1..digits_end];
|
|
if option == 'e' {
|
|
cli.bench_end_level = Some(if digits.is_empty() {
|
|
0
|
|
} else {
|
|
parse_i32(digits, "benchmark end level")?
|
|
});
|
|
} else {
|
|
cli.bench_nb_seconds = Some(if digits.is_empty() {
|
|
0
|
|
} else {
|
|
digits
|
|
.parse::<u32>()
|
|
.map_err(|_| format!("invalid benchmark duration: {digits:?}"))?
|
|
});
|
|
}
|
|
offset = digits_end;
|
|
continue;
|
|
}
|
|
'c' => {
|
|
cli.output = Some(cstring(STDOUT_MARK)?);
|
|
cli.force_stdout = true;
|
|
}
|
|
'f' => cli.force = true,
|
|
'k' => cli.remove_source = false,
|
|
'n' => {}
|
|
'q' => cli.display_level -= 1,
|
|
'v' => cli.display_level += 1,
|
|
'C' => cli.checksum = Some(2),
|
|
'h' => return Ok(Some(Action::Help { advanced: false })),
|
|
'H' => return Ok(Some(Action::Help { advanced: true })),
|
|
'V' => {
|
|
return Ok(Some(Action::Version {
|
|
quiet: cli.display_level < 2,
|
|
}))
|
|
}
|
|
'o' | 'D' | 'T' | 'M' | 'B' => {
|
|
let attached = short_attached_value(raw_value, offset + 1);
|
|
let argument = attached
|
|
.map(Ok)
|
|
.unwrap_or_else(|| next_os_value(args, index, &format!("-{option}")))?;
|
|
match option {
|
|
'o' => cli.output = Some(os_cstring(&argument)?),
|
|
'D' => cli.dictionary = Some(os_cstring(&argument)?),
|
|
'T' => cli.workers = Some(parse_worker_count(&argument.to_string_lossy())?),
|
|
'M' => {
|
|
cli.mem_limit =
|
|
Some(parse_u32(&argument.to_string_lossy(), "memory limit")?)
|
|
}
|
|
'B' => cli.block_size = Some(parse_size(&argument.to_string_lossy())?),
|
|
_ => unreachable!(),
|
|
}
|
|
break;
|
|
}
|
|
'l' | 'p' | 'P' | 'r' | 's' | 'S' => {
|
|
unsupported(&format!("-{option}"))?;
|
|
}
|
|
_ => return Err(format!("unknown option -{option}")),
|
|
}
|
|
offset += 1;
|
|
}
|
|
Ok(None)
|
|
}
|
|
|
|
fn parse_args(args: Vec<OsString>) -> Result<Action, String> {
|
|
let program_name = args
|
|
.first()
|
|
.map_or_else(|| "zstd".to_owned(), |value| program_basename(value));
|
|
let mut cli = Cli::new(&program_name);
|
|
let mut end_of_options = false;
|
|
let mut index = 1usize;
|
|
|
|
while index < args.len() {
|
|
let rendered = args[index].to_string_lossy().into_owned();
|
|
if end_of_options {
|
|
cli.inputs.push(os_cstring(&args[index])?);
|
|
} else if rendered == "--" {
|
|
end_of_options = true;
|
|
} else if rendered == "-" {
|
|
cli.inputs.push(cstring(STDIN_MARK)?);
|
|
} else if rendered.starts_with("--") {
|
|
if let Some(action) = parse_long_option(&rendered, &args, &mut index, &mut cli)? {
|
|
return Ok(action);
|
|
}
|
|
} else if rendered.starts_with('-') {
|
|
if let Some(action) =
|
|
parse_short_options(&rendered, &args[index], &args, &mut index, &mut cli)?
|
|
{
|
|
return Ok(action);
|
|
}
|
|
} else {
|
|
cli.inputs.push(os_cstring(&args[index])?);
|
|
}
|
|
index += 1;
|
|
}
|
|
Ok(Action::Run(Box::new(cli)))
|
|
}
|
|
|
|
unsafe fn apply_preferences(cli: &Cli, prefs: *mut FIO_prefs_t, ctx: *mut FIO_ctx_t) {
|
|
let display_level = if is_stdout(cli.output.as_ref()) && cli.display_level == 2 {
|
|
1
|
|
} else {
|
|
cli.display_level
|
|
};
|
|
unsafe {
|
|
g_utilDisplayLevel = display_level;
|
|
FIO_setCompressionType(prefs, FIO_ZSTD_COMPRESSION);
|
|
FIO_setNotificationLevel(display_level);
|
|
FIO_setProgressSetting(
|
|
if !io::stderr().is_terminal() && cli.progress != FIO_PS_ALWAYS {
|
|
FIO_PS_NEVER
|
|
} else {
|
|
cli.progress
|
|
},
|
|
);
|
|
FIO_setRemoveSrcFile(
|
|
prefs,
|
|
i32::from(
|
|
cli.remove_source
|
|
&& cli.operation != Operation::Test
|
|
&& !is_stdout(cli.output.as_ref()),
|
|
),
|
|
);
|
|
FIO_setAllowBlockDevices(prefs, i32::from(cli.force));
|
|
FIO_setMMapDict(prefs, cli.mmap_dict);
|
|
FIO_setUseRowMatchFinder(prefs, cli.row_match_finder);
|
|
FIO_setMemLimit(
|
|
prefs,
|
|
cli.mem_limit
|
|
.filter(|limit| *limit != 0)
|
|
.unwrap_or_else(|| {
|
|
if cli.compression_params.windowLog == 0 {
|
|
DEFAULT_MEM_LIMIT
|
|
} else {
|
|
1_u32 << (cli.compression_params.windowLog & 31)
|
|
}
|
|
}),
|
|
);
|
|
#[cfg(feature = "compression")]
|
|
FIO_setNbWorkers(prefs, resolved_worker_count(cli.workers, cli.single_thread));
|
|
FIO_setLdmFlag(prefs, u32::from(cli.ldm));
|
|
FIO_setAdaptiveMode(prefs, i32::from(cli.adapt));
|
|
FIO_setRsyncable(prefs, i32::from(cli.rsyncable));
|
|
FIO_setExcludeCompressedFile(prefs, i32::from(cli.exclude_compressed));
|
|
|
|
if cli.force {
|
|
FIO_overwriteMode(prefs);
|
|
}
|
|
if let Some(value) = cli.checksum {
|
|
FIO_setChecksumFlag(prefs, value);
|
|
}
|
|
if cli.operation == Operation::Compress {
|
|
FIO_setSparseWrite(prefs, 0);
|
|
} else if let Some(value) = cli.sparse {
|
|
FIO_setSparseWrite(prefs, value);
|
|
}
|
|
if let Some(value) = cli.pass_through {
|
|
FIO_setPassThroughFlag(prefs, value);
|
|
}
|
|
FIO_setContentSize(prefs, cli.content_size);
|
|
if let Some(value) = cli.dict_id {
|
|
FIO_setDictIDFlag(prefs, value);
|
|
}
|
|
if let Some(value) = cli.async_io {
|
|
FIO_setAsyncIOFlag(prefs, value);
|
|
}
|
|
if let Some(value) = cli.block_size {
|
|
FIO_setBlockSize(prefs, value as c_int);
|
|
}
|
|
if let Some(value) = cli.ldm_hash_log {
|
|
FIO_setLdmHashLog(prefs, value);
|
|
}
|
|
if let Some(value) = cli.ldm_min_match {
|
|
FIO_setLdmMinMatch(prefs, value);
|
|
}
|
|
if let Some(value) = cli.ldm_bucket_size_log {
|
|
FIO_setLdmBucketSizeLog(prefs, value);
|
|
}
|
|
if let Some(value) = cli.ldm_hash_rate_log {
|
|
FIO_setLdmHashRateLog(prefs, value);
|
|
}
|
|
if let Some(value) = cli.overlap_log {
|
|
FIO_setOverlapLog(prefs, value);
|
|
}
|
|
#[cfg(feature = "compression")]
|
|
{
|
|
FIO_setAdaptMin(prefs, cli.adapt_min.unwrap_or_else(|| ZSTD_minCLevel()));
|
|
FIO_setAdaptMax(prefs, cli.adapt_max.unwrap_or_else(|| ZSTD_maxCLevel()));
|
|
}
|
|
if let Some(value) = cli.stream_src_size {
|
|
FIO_setStreamSrcSize(prefs, value);
|
|
}
|
|
if let Some(value) = cli.target_cblock_size {
|
|
FIO_setTargetCBlockSize(prefs, value);
|
|
}
|
|
if let Some(value) = cli.src_size_hint {
|
|
FIO_setSrcSizeHint(prefs, value);
|
|
}
|
|
if let Some(value) = cli.literal_compression {
|
|
FIO_setLiteralCompressionMode(prefs, value);
|
|
}
|
|
|
|
FIO_setNbFilesTotal(ctx, cli.inputs.len() as c_int);
|
|
FIO_setHasStdinInput(ctx, i32::from(cli.inputs.iter().any(is_stdin)));
|
|
FIO_setHasStdoutOutput(ctx, i32::from(is_stdout(cli.output.as_ref())));
|
|
}
|
|
}
|
|
|
|
fn is_stdout(value: Option<&CString>) -> bool {
|
|
value.is_some_and(|value| value.as_bytes() == STDOUT_MARK.as_bytes())
|
|
}
|
|
|
|
fn is_stdin(value: &CString) -> bool {
|
|
value.as_bytes() == STDIN_MARK.as_bytes()
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
fn is_non_fifo_symlink(input: &CString) -> bool {
|
|
let path = Path::new(OsStr::from_bytes(input.as_bytes()));
|
|
let Ok(metadata) = fs::symlink_metadata(path) else {
|
|
return false;
|
|
};
|
|
metadata.file_type().is_symlink()
|
|
&& !fs::metadata(path).is_ok_and(|target| target.file_type().is_fifo())
|
|
}
|
|
|
|
#[cfg(not(unix))]
|
|
fn is_non_fifo_symlink(input: &CString) -> bool {
|
|
let path = Path::new(&input.to_string_lossy().into_owned());
|
|
fs::symlink_metadata(path).is_ok_and(|metadata| metadata.file_type().is_symlink())
|
|
}
|
|
|
|
fn filter_symlink_inputs(cli: &mut Cli) {
|
|
if cli.force {
|
|
return;
|
|
}
|
|
cli.inputs.retain(|input| {
|
|
if is_stdin(input) || !is_non_fifo_symlink(input) {
|
|
return true;
|
|
}
|
|
if cli.display_level >= 2 {
|
|
eprintln!(
|
|
"zstd: Warning : {} is a symbolic link, ignoring",
|
|
input.to_string_lossy()
|
|
);
|
|
}
|
|
false
|
|
});
|
|
}
|
|
|
|
fn check_terminal_safety(cli: &Cli) -> Result<(), String> {
|
|
let has_stdin = cli.inputs.iter().any(is_stdin);
|
|
if has_stdin && !cli.force && io::stdin().is_terminal() {
|
|
return Err("stdin is a console, aborting".to_owned());
|
|
}
|
|
if has_stdin
|
|
&& is_stdout(cli.output.as_ref())
|
|
&& !cli.force
|
|
&& !cli.force_stdout
|
|
&& cli.operation != Operation::Decompress
|
|
&& io::stdout().is_terminal()
|
|
{
|
|
return Err("stdout is a console, aborting".to_owned());
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(feature = "compression")]
|
|
unsafe fn run_compress(
|
|
cli: &Cli,
|
|
ctx: *mut FIO_ctx_t,
|
|
prefs: *mut FIO_prefs_t,
|
|
inputs: &[*const c_char],
|
|
output: *const c_char,
|
|
dictionary: *const c_char,
|
|
) -> c_int {
|
|
if inputs.len() == 1 && !output.is_null() {
|
|
unsafe {
|
|
FIO_compressFilename(
|
|
ctx,
|
|
prefs,
|
|
output,
|
|
inputs[0],
|
|
dictionary,
|
|
cli.level,
|
|
cli.compression_params,
|
|
)
|
|
}
|
|
} else {
|
|
unsafe {
|
|
FIO_compressMultipleFilenames(
|
|
ctx,
|
|
prefs,
|
|
inputs.as_ptr(),
|
|
ptr::null(),
|
|
ptr::null(),
|
|
output,
|
|
ZSTD_SUFFIX.as_ptr().cast(),
|
|
dictionary,
|
|
cli.level,
|
|
cli.compression_params,
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(feature = "decompression")]
|
|
unsafe fn run_decompress(
|
|
operation: Operation,
|
|
ctx: *mut FIO_ctx_t,
|
|
prefs: *mut FIO_prefs_t,
|
|
inputs: &[*const c_char],
|
|
output: *const c_char,
|
|
dictionary: *const c_char,
|
|
) -> c_int {
|
|
match operation {
|
|
Operation::Test => {
|
|
let null_output = cstring(NULL_MARK).expect("static null marker");
|
|
unsafe {
|
|
FIO_setTestMode(prefs, 1);
|
|
FIO_decompressMultipleFilenames(
|
|
ctx,
|
|
prefs,
|
|
inputs.as_ptr(),
|
|
ptr::null(),
|
|
ptr::null(),
|
|
null_output.as_ptr(),
|
|
dictionary,
|
|
)
|
|
}
|
|
}
|
|
Operation::Decompress if inputs.len() == 1 && !output.is_null() => unsafe {
|
|
FIO_decompressFilename(ctx, prefs, output, inputs[0], dictionary)
|
|
},
|
|
Operation::Decompress => unsafe {
|
|
FIO_decompressMultipleFilenames(
|
|
ctx,
|
|
prefs,
|
|
inputs.as_ptr(),
|
|
ptr::null(),
|
|
ptr::null(),
|
|
output,
|
|
dictionary,
|
|
)
|
|
},
|
|
Operation::Compress | Operation::Bench => {
|
|
unreachable!("compression and benchmark are dispatched separately")
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Runs benchmark mode through the C bridge. Level clamping against
|
|
/// `ZSTD_maxCLevel()` happens on the C side, where the symbol is always
|
|
/// available when benchmarking is compiled in. No input file means a
|
|
/// synthetic-sample benchmark, matching the C CLI.
|
|
fn run_bench(cli: &Cli) -> Result<i32, String> {
|
|
let inputs: Vec<*const c_char> = cli.inputs.iter().map(|value| value.as_ptr()).collect();
|
|
let dictionary = cli
|
|
.dictionary
|
|
.as_ref()
|
|
.map_or(ptr::null(), |value| value.as_ptr());
|
|
let result = unsafe {
|
|
ZSTD_rust_cli_bench(
|
|
inputs.as_ptr(),
|
|
inputs.len() as c_uint,
|
|
dictionary,
|
|
cli.level,
|
|
cli.bench_end_level.unwrap_or(cli.level),
|
|
&cli.compression_params,
|
|
cli.display_level,
|
|
cli.bench_nb_seconds.unwrap_or(DEFAULT_BENCH_NB_SECONDS),
|
|
cli.block_size.unwrap_or(0),
|
|
// The C CLI benchmarks single-threaded unless -T was given.
|
|
cli.workers.unwrap_or(1),
|
|
)
|
|
};
|
|
if result < 0 {
|
|
return Err("benchmark mode is not available in this build".to_owned());
|
|
}
|
|
Ok(result)
|
|
}
|
|
|
|
fn run_cli(mut cli: Cli) -> Result<i32, String> {
|
|
if let Some(program_name) = &cli.unsupported_program {
|
|
return Err(format!(
|
|
"{program_name} compatibility mode is not yet implemented by the Rust CLI frontend"
|
|
));
|
|
}
|
|
if cli.operation == Operation::Bench {
|
|
return run_bench(&cli);
|
|
}
|
|
let explicit_input_count = cli.inputs.len();
|
|
filter_symlink_inputs(&mut cli);
|
|
if explicit_input_count > 0 && cli.inputs.is_empty() {
|
|
return Ok(1);
|
|
}
|
|
if cli.operation == Operation::Test {
|
|
cli.output = Some(cstring(NULL_MARK)?);
|
|
cli.remove_source = false;
|
|
}
|
|
if cli.inputs.is_empty() {
|
|
cli.inputs.push(cstring(STDIN_MARK)?);
|
|
if cli.output.is_none() {
|
|
cli.output = Some(cstring(STDOUT_MARK)?);
|
|
}
|
|
}
|
|
if cli.inputs.len() == 1
|
|
&& cli.inputs[0].as_bytes() == STDIN_MARK.as_bytes()
|
|
&& cli.output.is_none()
|
|
{
|
|
cli.output = Some(cstring(STDOUT_MARK)?);
|
|
}
|
|
|
|
check_terminal_safety(&cli)?;
|
|
|
|
if cli.operation == Operation::Compress {
|
|
#[cfg(not(feature = "compression"))]
|
|
return Err("Compression not supported".to_owned());
|
|
|
|
#[cfg(feature = "compression")]
|
|
{
|
|
let min_level = unsafe { ZSTD_minCLevel() };
|
|
let max_level = unsafe { ZSTD_maxCLevel() };
|
|
let ceiling = if cli.ultra {
|
|
max_level
|
|
} else {
|
|
DEFAULT_MAX_CLEVEL.min(max_level)
|
|
};
|
|
if cli.level > ceiling {
|
|
eprintln!("zstd: warning: compression level reduced to {ceiling}");
|
|
cli.level = ceiling;
|
|
}
|
|
if cli.level < min_level {
|
|
return Err(format!(
|
|
"compression level {} is below {min_level}",
|
|
cli.level
|
|
));
|
|
}
|
|
if let (Some(minimum), Some(maximum)) = (cli.adapt_min, cli.adapt_max) {
|
|
if minimum > maximum {
|
|
return Err("adaptation minimum exceeds maximum".to_owned());
|
|
}
|
|
cli.level = cli.level.clamp(minimum, maximum);
|
|
}
|
|
}
|
|
} else {
|
|
#[cfg(not(feature = "decompression"))]
|
|
return Err("Decompression not supported".to_owned());
|
|
}
|
|
|
|
let prefs = unsafe { FIO_createPreferences() };
|
|
let ctx = unsafe { FIO_createContext() };
|
|
if prefs.is_null() || ctx.is_null() {
|
|
unsafe {
|
|
if !prefs.is_null() {
|
|
FIO_freePreferences(prefs);
|
|
}
|
|
if !ctx.is_null() {
|
|
FIO_freeContext(ctx);
|
|
}
|
|
}
|
|
return Err("could not allocate C file-I/O state".to_owned());
|
|
}
|
|
|
|
let result = {
|
|
unsafe {
|
|
FIO_addAbortHandler();
|
|
apply_preferences(&cli, prefs, ctx);
|
|
}
|
|
let output = cli
|
|
.output
|
|
.as_ref()
|
|
.map_or(ptr::null(), |value| value.as_ptr());
|
|
let dictionary = cli
|
|
.dictionary
|
|
.as_ref()
|
|
.map_or(ptr::null(), |value| value.as_ptr());
|
|
let inputs: Vec<*const c_char> = cli.inputs.iter().map(|value| value.as_ptr()).collect();
|
|
match cli.operation {
|
|
Operation::Compress => {
|
|
#[cfg(feature = "compression")]
|
|
{
|
|
unsafe { run_compress(&cli, ctx, prefs, &inputs, output, dictionary) }
|
|
}
|
|
#[cfg(not(feature = "compression"))]
|
|
unreachable!("unsupported compression was rejected above")
|
|
}
|
|
Operation::Decompress | Operation::Test => {
|
|
#[cfg(feature = "decompression")]
|
|
{
|
|
unsafe {
|
|
run_decompress(cli.operation, ctx, prefs, &inputs, output, dictionary)
|
|
}
|
|
}
|
|
#[cfg(not(feature = "decompression"))]
|
|
unreachable!("unsupported decompression was rejected above")
|
|
}
|
|
Operation::Bench => unreachable!("benchmark mode was dispatched earlier"),
|
|
}
|
|
};
|
|
|
|
unsafe {
|
|
FIO_freePreferences(prefs);
|
|
FIO_freeContext(ctx);
|
|
}
|
|
Ok(result)
|
|
}
|
|
|
|
fn run_from_args(args: Vec<OsString>) -> c_int {
|
|
match parse_args(args) {
|
|
Ok(Action::Help { advanced }) => {
|
|
usage(advanced);
|
|
0
|
|
}
|
|
Ok(Action::Version { quiet }) => {
|
|
print_version(quiet);
|
|
0
|
|
}
|
|
Ok(Action::Run(cli)) => match run_cli(*cli) {
|
|
Ok(result) => result,
|
|
Err(error) => {
|
|
eprintln!("zstd: {error}");
|
|
1
|
|
}
|
|
},
|
|
Err(error) => {
|
|
eprintln!("zstd: {error}\nTry `zstd --help` for usage.");
|
|
1
|
|
}
|
|
}
|
|
}
|
|
|
|
unsafe fn argv_to_os_strings(
|
|
arg_count: c_int,
|
|
argv: *const *const c_char,
|
|
) -> Result<Vec<OsString>, String> {
|
|
if arg_count <= 0 || argv.is_null() {
|
|
return Err("invalid argv supplied by C main".to_owned());
|
|
}
|
|
let count = arg_count as usize;
|
|
let mut args = Vec::with_capacity(count);
|
|
for index in 0..count {
|
|
let argument = unsafe { *argv.add(index) };
|
|
if argument.is_null() {
|
|
return Err(format!("argv[{index}] is null"));
|
|
}
|
|
let bytes = unsafe { CStr::from_ptr(argument) }.to_bytes();
|
|
#[cfg(unix)]
|
|
args.push(OsString::from_vec(bytes.to_vec()));
|
|
#[cfg(not(unix))]
|
|
args.push(OsString::from(String::from_utf8_lossy(bytes).into_owned()));
|
|
}
|
|
Ok(args)
|
|
}
|
|
|
|
/// C `main()` entry point retained by the small `programs/zstdcli.c` forwarder.
|
|
#[no_mangle]
|
|
pub unsafe extern "C" fn ZSTD_rust_cli_main(arg_count: c_int, argv: *const *const c_char) -> c_int {
|
|
if let Err(error) = check_lib_version() {
|
|
eprintln!("zstd: {error}");
|
|
return 1;
|
|
}
|
|
match unsafe { argv_to_os_strings(arg_count, argv) } {
|
|
Ok(args) => run_from_args(args),
|
|
Err(error) => {
|
|
eprintln!("zstd: {error}");
|
|
1
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn parse(values: &[&str]) -> Cli {
|
|
let args = values.iter().map(OsString::from).collect();
|
|
match parse_args(args).expect("arguments should parse") {
|
|
Action::Run(cli) => *cli,
|
|
Action::Help { .. } | Action::Version { .. } => panic!("expected a run action"),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn defaults_preserve_the_c_fileio_contract() {
|
|
let cli = parse(&["zstd", "input"]);
|
|
|
|
assert_eq!(cli.content_size, 1);
|
|
assert_eq!(cli.workers, None);
|
|
assert_eq!(cli.operation, Operation::Compress);
|
|
}
|
|
|
|
#[test]
|
|
fn long_mode_uses_the_cli_default_window_and_enables_ultra() {
|
|
let cli = parse(&["zstd", "--long", "input"]);
|
|
|
|
assert!(cli.ldm);
|
|
assert!(cli.ultra);
|
|
assert_eq!(cli.compression_params.windowLog, DEFAULT_LONG_WINDOW_LOG);
|
|
}
|
|
|
|
#[test]
|
|
fn long_mode_does_not_replace_an_explicit_window_log() {
|
|
let cli = parse(&["zstd", "--zstd=wlog=25", "--long", "input"]);
|
|
|
|
assert_eq!(cli.compression_params.windowLog, 25);
|
|
}
|
|
|
|
#[test]
|
|
fn stdio_and_dictionary_short_options_are_preserved() {
|
|
let cli = parse(&["zstd", "-dc", "-D", "dict", "-"]);
|
|
|
|
assert_eq!(cli.operation, Operation::Decompress);
|
|
assert!(is_stdout(cli.output.as_ref()));
|
|
assert_eq!(
|
|
cli.dictionary.as_deref().map(CStr::to_bytes),
|
|
Some(&b"dict"[..])
|
|
);
|
|
assert_eq!(
|
|
cli.inputs
|
|
.iter()
|
|
.map(|input| input.as_bytes())
|
|
.collect::<Vec<_>>(),
|
|
vec![STDIN_MARK.as_bytes()]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn stdout_selection_does_not_enable_force_or_pass_through() {
|
|
let cli = parse(&["zstd", "-c", "input"]);
|
|
|
|
assert!(cli.force_stdout);
|
|
assert!(!cli.force);
|
|
assert_eq!(cli.pass_through, None);
|
|
}
|
|
|
|
#[test]
|
|
fn short_level_can_be_combined_with_other_flags() {
|
|
let cli = parse(&["zstd", "-5q", "input"]);
|
|
|
|
assert_eq!(cli.level, 5);
|
|
assert_eq!(cli.display_level, 1);
|
|
}
|
|
|
|
#[test]
|
|
fn short_option_equals_form_is_accepted() {
|
|
let cli = parse(&["zstd", "-T=2", "-M=64M", "-B=1M", "input"]);
|
|
|
|
assert_eq!(cli.workers, Some(2));
|
|
assert_eq!(cli.mem_limit, Some(64 << 20));
|
|
assert_eq!(cli.block_size, Some(1 << 20));
|
|
}
|
|
|
|
#[test]
|
|
fn valueless_long_flags_reject_attached_values() {
|
|
let error = parse_args(vec![OsString::from("zstd"), OsString::from("--rm=0")])
|
|
.expect_err("an attached value must not activate --rm");
|
|
|
|
assert!(error.contains("does not take an argument"));
|
|
}
|
|
|
|
#[test]
|
|
fn zstdmt_uses_auto_threads() {
|
|
let cli = parse(&["zstdmt", "input"]);
|
|
|
|
assert_eq!(cli.workers, Some(0));
|
|
assert!(!cli.single_thread);
|
|
}
|
|
|
|
#[test]
|
|
fn single_thread_pins_zero_workers() {
|
|
let cli = parse(&["zstd", "--single-thread", "input"]);
|
|
|
|
assert_eq!(cli.workers, Some(0));
|
|
assert!(cli.single_thread);
|
|
}
|
|
|
|
#[test]
|
|
fn a_later_thread_count_overrides_single_thread_workers() {
|
|
/* Mirrors the C CLI: -T after --single-thread wins the worker count,
|
|
* while the single-thread latch stays set. */
|
|
let cli = parse(&["zstd", "--single-thread", "-T2", "input"]);
|
|
|
|
assert_eq!(cli.workers, Some(2));
|
|
assert!(cli.single_thread);
|
|
}
|
|
|
|
#[test]
|
|
fn single_thread_rejects_attached_values() {
|
|
let error = parse_args(vec![
|
|
OsString::from("zstd"),
|
|
OsString::from("--single-thread=1"),
|
|
])
|
|
.expect_err("an attached value must not activate --single-thread");
|
|
|
|
assert!(error.contains("does not take an argument"));
|
|
}
|
|
|
|
#[test]
|
|
fn alternate_format_aliases_fail_before_processing_files() {
|
|
let cli = Cli::new("gzip");
|
|
|
|
assert_eq!(cli.unsupported_program.as_deref(), Some("gzip"));
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[test]
|
|
fn short_path_arguments_keep_non_utf8_bytes() {
|
|
let output = OsString::from_vec(vec![b'-', b'o', 0xff, b'.', b'z', b's', b't']);
|
|
let action = parse_args(vec![
|
|
OsString::from("zstd"),
|
|
output,
|
|
OsString::from("input"),
|
|
])
|
|
.expect("arguments should parse");
|
|
let Action::Run(cli) = action else {
|
|
panic!("expected a run action");
|
|
};
|
|
|
|
assert_eq!(
|
|
cli.output.as_deref().map(CStr::to_bytes),
|
|
Some(&b"\xff.zst"[..])
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn unsupported_modes_fail_during_parsing() {
|
|
let error = parse_args(vec![OsString::from("zstd"), OsString::from("--train")])
|
|
.expect_err("training has not yet been migrated");
|
|
|
|
assert!(error.contains("not yet implemented"));
|
|
}
|
|
|
|
#[test]
|
|
fn bench_mode_parses_level_duration_and_defaults() {
|
|
let cli = parse(&["zstd", "-b1", "-i0", "input"]);
|
|
|
|
assert_eq!(cli.operation, Operation::Bench);
|
|
assert_eq!(cli.level, 1);
|
|
assert_eq!(cli.bench_nb_seconds, Some(0));
|
|
assert_eq!(cli.bench_end_level, None);
|
|
assert_eq!(
|
|
cli.inputs
|
|
.iter()
|
|
.map(|input| input.as_bytes())
|
|
.collect::<Vec<_>>(),
|
|
vec![&b"input"[..]]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn bench_range_aggregates_within_a_single_argument() {
|
|
let cli = parse(&["zstd", "-b5e6i2", "input"]);
|
|
|
|
assert_eq!(cli.operation, Operation::Bench);
|
|
assert_eq!(cli.level, 5);
|
|
assert_eq!(cli.bench_end_level, Some(6));
|
|
assert_eq!(cli.bench_nb_seconds, Some(2));
|
|
}
|
|
|
|
#[test]
|
|
fn bench_duration_without_digits_defaults_to_zero() {
|
|
let cli = parse(&["zstd", "-b", "-e", "-i"]);
|
|
|
|
assert_eq!(cli.operation, Operation::Bench);
|
|
assert_eq!(cli.level, DEFAULT_CLEVEL);
|
|
assert_eq!(cli.bench_end_level, Some(0));
|
|
assert_eq!(cli.bench_nb_seconds, Some(0));
|
|
}
|
|
}
|