feat(cli): move patch-from parameter policy to Rust
Keep the public ZSTD_getCParams() selection and all DISPLAYLEVEL output in programs/fileio.c, but pass its by-value compression-parameter snapshot to a Rust policy leaf. Rust now preserves the original source-size highbit and window clamping rules, validates and updates the patch memory limit before success outputs, derives cycleLog, updates comprParams->windowLog, and applies automatic LDM without overwriting the explicit-LDM diagnostic semantics. The Rust ABI returns the raw file window and the two diagnostic predicates so C can retain its existing messages and optimal-parser note ordering. C layout assertions cover the seven-word compression-parameter snapshot on both platform-width variants; rejected unknown and oversized inputs leave all outputs unchanged. Test Plan: - Rust clippy pre/post nightly-format matrix for library compression and CLI cli,compression,decompression,benchmark features -- passed. - `cargo test --manifest-path rust/cli/Cargo.toml --no-default-features --features cli,compression,decompression,benchmark` -- 137 passed. - `make -C programs -j2 zstd` -- passed. - `make -C tests -j2 test-cli-tests` -- 41 passed. - `python3 tests/cli-tests/run.py decompression/pass-through.sh` -- passed. - Focused patch-from round trips, stream-size failure ordering, and automatic long-mode diagnostics -- passed. - `make -C tests -j2 test-rust-lib-smoke` -- passed. - No i686 Rust target is installed; 32-bit policy bounds are covered by size_of-based constants and C/Rust ABI assertions.
This commit is contained in:
+35
-28
@@ -282,6 +282,12 @@ typedef char FIO_rust_ctx_total_output_offset[
|
||||
typedef char FIO_rust_ctx_size[
|
||||
(sizeof(FIO_ctx_t)
|
||||
== offsetof(FIO_ctx_t, totalBytesOutput) + sizeof(size_t)) ? 1 : -1];
|
||||
typedef char FIO_rust_compression_params_strategy_offset[
|
||||
(offsetof(ZSTD_compressionParameters, strategy)
|
||||
== 6 * sizeof(unsigned)) ? 1 : -1];
|
||||
typedef char FIO_rust_compression_params_size[
|
||||
(sizeof(ZSTD_compressionParameters)
|
||||
== 7 * sizeof(unsigned)) ? 1 : -1];
|
||||
|
||||
int FIO_rust_shouldDisplayFileSummary(const FIO_ctx_t* fCtx);
|
||||
int FIO_rust_shouldDisplayMultipleFileSummary(const FIO_ctx_t* fCtx);
|
||||
@@ -328,11 +334,14 @@ static int FIO_shouldDisplayMultipleFileSummary(FIO_ctx_t const* fCtx)
|
||||
int FIO_rust_checkFilenameCollisions(const char** filenameTable, unsigned nbFiles);
|
||||
unsigned FIO_rust_highbit64(unsigned long long v);
|
||||
unsigned long long FIO_rust_getLargestFileSize(const char** inFileNames, unsigned nbFiles);
|
||||
unsigned FIO_rust_cycleLog(unsigned hashLog, int strategy);
|
||||
void FIO_rust_patchFromWindowPolicy(unsigned fileWindowLog,
|
||||
unsigned cycleLog,
|
||||
unsigned* windowLog,
|
||||
int* enableLdm);
|
||||
int FIO_rust_adjustParamsForPatchFromMode(FIO_prefs_t* prefs,
|
||||
ZSTD_compressionParameters* comprParams,
|
||||
unsigned long long dictSize,
|
||||
unsigned long long maxSrcFileSize,
|
||||
ZSTD_compressionParameters cParams,
|
||||
unsigned* fileWindowLog,
|
||||
int* autoLdm,
|
||||
int* optimalParser);
|
||||
void FIO_rust_setInBuffer(ZSTD_inBuffer* output, const void* buf, size_t s, size_t pos);
|
||||
void FIO_rust_setOutBuffer(ZSTD_outBuffer* output, void* buf, size_t s, size_t pos);
|
||||
const char* FIO_rust_determineCompressedName(const char* srcFileName, const char* outDirName, const char* suffix);
|
||||
@@ -865,38 +874,36 @@ typedef struct {
|
||||
ReadPoolCtx_t *readCtx;
|
||||
} cRess_t;
|
||||
|
||||
/** ZSTD_cycleLog() :
|
||||
* condition for correct operation : hashLog > 1 */
|
||||
static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
|
||||
{
|
||||
assert(hashLog > 1);
|
||||
return FIO_rust_cycleLog(hashLog, (int)strat);
|
||||
}
|
||||
|
||||
static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs,
|
||||
ZSTD_compressionParameters* comprParams,
|
||||
unsigned long long const dictSize,
|
||||
unsigned long long const maxSrcFileSize,
|
||||
int cLevel)
|
||||
{
|
||||
unsigned const fileWindowLog = FIO_highbit64(maxSrcFileSize) + 1;
|
||||
enum {
|
||||
FIO_PATCH_MEM_LIMIT_SUCCESS = 0,
|
||||
FIO_PATCH_MEM_LIMIT_UNKNOWN_SIZE = 1,
|
||||
FIO_PATCH_MEM_LIMIT_TOO_LARGE = 2
|
||||
};
|
||||
ZSTD_compressionParameters const cParams = ZSTD_getCParams(cLevel, (size_t)maxSrcFileSize, (size_t)dictSize);
|
||||
unsigned windowLog;
|
||||
int enableLdm;
|
||||
FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, maxSrcFileSize);
|
||||
FIO_rust_patchFromWindowPolicy(fileWindowLog,
|
||||
ZSTD_cycleLog(cParams.chainLog, cParams.strategy),
|
||||
&windowLog,
|
||||
&enableLdm);
|
||||
unsigned fileWindowLog;
|
||||
int autoLdm;
|
||||
int optimalParser;
|
||||
int const status = FIO_rust_adjustParamsForPatchFromMode(
|
||||
prefs, comprParams, dictSize, maxSrcFileSize, cParams,
|
||||
&fileWindowLog, &autoLdm, &optimalParser);
|
||||
if (status == FIO_PATCH_MEM_LIMIT_UNKNOWN_SIZE)
|
||||
EXM_THROW(42, "Using --patch-from with stdin requires --stream-size");
|
||||
if (status == FIO_PATCH_MEM_LIMIT_TOO_LARGE) {
|
||||
unsigned const maxWindowSize = (1U << ZSTD_WINDOWLOG_MAX);
|
||||
EXM_THROW(42, "Can't handle files larger than %u GB\n", maxWindowSize/(1 GB));
|
||||
}
|
||||
assert(status == FIO_PATCH_MEM_LIMIT_SUCCESS);
|
||||
if (fileWindowLog > ZSTD_WINDOWLOG_MAX)
|
||||
DISPLAYLEVEL(1, "Max window log exceeded by file (compression ratio will suffer)\n");
|
||||
comprParams->windowLog = windowLog;
|
||||
if (enableLdm) {
|
||||
if (!prefs->ldmFlag)
|
||||
DISPLAYLEVEL(2, "long mode automatically triggered\n");
|
||||
FIO_setLdmFlag(prefs, 1);
|
||||
}
|
||||
if (cParams.strategy >= ZSTD_btopt) {
|
||||
if (autoLdm)
|
||||
DISPLAYLEVEL(2, "long mode automatically triggered\n");
|
||||
if (optimalParser) {
|
||||
DISPLAYLEVEL(4, "[Optimal parser notes] Consider the following to improve patch size at the cost of speed:\n");
|
||||
DISPLAYLEVEL(4, "- Set a larger targetLength (e.g. --zstd=targetLength=4096)\n");
|
||||
DISPLAYLEVEL(4, "- Set a larger chainLog (e.g. --zstd=chainLog=%u)\n", ZSTD_CHAINLOG_MAX);
|
||||
|
||||
Reference in New Issue
Block a user