ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(cli): move compression format dispatch into Rust

Browse Source 
`FIO_compressFilename_internal()` still selected the zstd, gzip, xz/lzma, and
lz4 paths in a C-owned format switch and updated aggregate byte accounting
around those callbacks.  That left the mixed-format CLI dispatch loop outside
the Rust file-I/O orchestration even though the pools and decompression path
were already projected there.

Add a callback table for the codec leaves and move format selection, optional
codec status handling, post-success accounting, and progress-hook ordering to
Rust.  The existing C codec implementations, diagnostics, elapsed-time
formatting, and resource ownership remain unchanged; unsupported optional
formats return explicit statuses so the C wrapper preserves its original
EXM_THROW messages.

Test Plan:
- `cargo test --manifest-path rust/Cargo.toml --all-targets -- --test-threads=1` -- 473 passed.
- Native `test-cli-tests` -- all 41 passed.
- Native `test-zstd` -- passed, including async zstd/gzip/xz/lzma/lz4 round trips.
- `make -B -C programs -j2 zstd` and `make -B -C tests/fuzz -j2 all` -- passed.
This commit is contained in:
ddidderr
2026-07-18 22:41:39 +02:00
parent 4304c0af40
commit be7e656d8e
2 changed files with 657 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fileio.c
+167 -57
View File
@@ -975,6 +975,49 @@ typedef struct {
ReadPoolCtx_t *readCtx;
} cRess_t;
enum {
FIO_RUST_COMPRESS_OK = 0,
FIO_RUST_COMPRESS_GZIP_UNSUPPORTED = 1,
FIO_RUST_COMPRESS_LZMA_UNSUPPORTED = 2,
FIO_RUST_COMPRESS_LZ4_UNSUPPORTED = 3,
FIO_RUST_COMPRESS_ZSTD_UNSUPPORTED = 4
};
typedef unsigned long long (*FIO_rust_compress_zstd_fn)(
void* fCtx, void* prefs, void* ress,
const char* srcFileName, U64 srcFileSize,
int compressionLevel, U64* readsize);
typedef unsigned long long (*FIO_rust_compress_gzip_fn)(
void* ress, const char* srcFileName, U64 srcFileSize,
int compressionLevel, U64* readsize);
typedef unsigned long long (*FIO_rust_compress_lzma_fn)(
void* ress, const char* srcFileName, U64 srcFileSize,
int compressionLevel, U64* readsize, int plainLzma);
typedef unsigned long long (*FIO_rust_compress_lz4_fn)(
void* ress, const char* srcFileName, U64 srcFileSize,
int compressionLevel, int checksumFlag, U64* readsize);
typedef void (*FIO_rust_compress_input_display_fn)(
void* opaque, const char* srcFileName, U64 fileSize);
typedef void (*FIO_rust_compress_status_display_fn)(
void* opaque, void* fCtx, const char* dstFileName,
const char* srcFileName, U64 readsize, U64 compressedfilesize);
typedef struct {
void* opaque;
FIO_rust_compress_zstd_fn compress_zstd;
FIO_rust_compress_gzip_fn compress_gzip;
FIO_rust_compress_lzma_fn compress_lzma;
FIO_rust_compress_lz4_fn compress_lz4;
FIO_rust_compress_input_display_fn display_input;
FIO_rust_compress_status_display_fn display_status;
} FIO_rust_compress_callbacks_t;
int FIO_rust_compressFilenameInternal(
void* fCtx, FIO_prefs_t* prefs, void* ress,
const char* dstFileName, const char* srcFileName,
U64 fileSize, int compressionLevel,
const FIO_rust_compress_callbacks_t* callbacks);
static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs,
ZSTD_compressionParameters* comprParams,
unsigned long long const dictSize,
@@ -1632,71 +1675,79 @@ FIO_compressZstdFrame(FIO_ctx_t* const fCtx,
return compressedfilesize;
}
/*! FIO_compressFilename_internal() :
* same as FIO_compressFilename_extRess(), with `ress.desFile` already opened.
* @return : 0 : compression completed correctly,
* 1 : missing or pb opening srcFileName
*/
static int
FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
FIO_prefs_t* const prefs,
cRess_t ress,
const char* dstFileName, const char* srcFileName,
int compressionLevel)
static unsigned long long
FIO_rust_compressZstdCallback(void* fCtx, void* prefs, void* ress,
const char* srcFileName, U64 srcFileSize,
int compressionLevel, U64* readsize)
{
UTIL_time_t const timeStart = UTIL_getTime();
clock_t const cpuStart = clock();
U64 readsize = 0;
U64 compressedfilesize = 0;
U64 const fileSize = UTIL_getFileSize(srcFileName);
DISPLAYLEVEL(5, "%s: %llu bytes \n", srcFileName, (unsigned long long)fileSize);
return FIO_compressZstdFrame((FIO_ctx_t*)fCtx, (FIO_prefs_t*)prefs,
(const cRess_t*)ress, srcFileName,
srcFileSize, compressionLevel, readsize);
}
/* compression format selection */
switch (prefs->compressionType) {
default:
case FIO_zstdCompression:
compressedfilesize = FIO_compressZstdFrame(fCtx, prefs, &ress, srcFileName, fileSize, compressionLevel, &readsize);
break;
case FIO_gzipCompression:
#ifdef ZSTD_GZCOMPRESS
compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize);
#else
(void)compressionLevel;
EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n",
srcFileName);
static unsigned long long
FIO_rust_compressGzipCallback(void* ress, const char* srcFileName,
U64 srcFileSize, int compressionLevel,
U64* readsize)
{
return FIO_compressGzFrame((const cRess_t*)ress, srcFileName, srcFileSize,
compressionLevel, readsize);
}
#endif
break;
case FIO_xzCompression:
case FIO_lzmaCompression:
#ifdef ZSTD_LZMACOMPRESS
compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, prefs->compressionType==FIO_lzmaCompression);
#else
(void)compressionLevel;
EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n",
srcFileName);
static unsigned long long
FIO_rust_compressLzmaCallback(void* ress, const char* srcFileName,
U64 srcFileSize, int compressionLevel,
U64* readsize, int plainLzma)
{
return FIO_compressLzmaFrame((cRess_t*)ress, srcFileName, srcFileSize,
compressionLevel, readsize, plainLzma);
}
#endif
break;
case FIO_lz4Compression:
#ifdef ZSTD_LZ4COMPRESS
compressedfilesize = FIO_compressLz4Frame(&ress, srcFileName, fileSize, compressionLevel, prefs->checksumFlag, &readsize);
#else
(void)compressionLevel;
EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n",
srcFileName);
static unsigned long long
FIO_rust_compressLz4Callback(void* ress, const char* srcFileName,
U64 srcFileSize, int compressionLevel,
int checksumFlag, U64* readsize)
{
return FIO_compressLz4Frame((cRess_t*)ress, srcFileName, srcFileSize,
compressionLevel, checksumFlag, readsize);
}
#endif
break;
}
/* Status */
fCtx->totalBytesInput += (size_t)readsize;
fCtx->totalBytesOutput += (size_t)compressedfilesize;
typedef struct {
UTIL_time_t timeStart;
clock_t cpuStart;
} FIO_compressDisplayContext_t;
static void
FIO_rust_compressInputDisplayCallback(void* opaque, const char* srcFileName,
U64 fileSize)
{
(void)opaque;
DISPLAYLEVEL(5, "%s: %llu bytes \n", srcFileName,
(unsigned long long)fileSize);
}
static void
FIO_rust_compressStatusDisplayCallback(void* opaque, void* fCtxOpaque,
const char* dstFileName,
const char* srcFileName,
U64 readsize, U64 compressedfilesize)
{
FIO_compressDisplayContext_t const* const displayCtx =
(const FIO_compressDisplayContext_t*)opaque;
FIO_ctx_t* const fCtx = (FIO_ctx_t*)fCtxOpaque;
DISPLAY_PROGRESS("\r%79s\r", "");
if (FIO_shouldDisplayFileSummary(fCtx)) {
UTIL_HumanReadableSize_t hr_isize = UTIL_makeHumanReadableSize((U64) readsize);
UTIL_HumanReadableSize_t hr_osize = UTIL_makeHumanReadableSize((U64) compressedfilesize);
UTIL_HumanReadableSize_t hr_isize =
UTIL_makeHumanReadableSize((U64)readsize);
UTIL_HumanReadableSize_t hr_osize =
UTIL_makeHumanReadableSize((U64)compressedfilesize);
if (readsize == 0) {
DISPLAY_SUMMARY("%-20s : (%6.*f%s => %6.*f%s, %s) \n",
srcFileName,
@@ -1713,16 +1764,75 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
}
}
/* Elapsed Time and CPU Load */
/* Keep the original elapsed-time and CPU-load diagnostics in C so their
* formatting and clock source remain identical to the old loop. */
{ clock_t const cpuEnd = clock();
double const cpuLoad_s = (double)(cpuEnd - cpuStart) / CLOCKS_PER_SEC;
U64 const timeLength_ns = UTIL_clockSpanNano(timeStart);
double const cpuLoad_s = (double)(cpuEnd - displayCtx->cpuStart) / CLOCKS_PER_SEC;
U64 const timeLength_ns = UTIL_clockSpanNano(displayCtx->timeStart);
double const timeLength_s = (double)timeLength_ns / 1000000000;
double const cpuLoad_pct = (cpuLoad_s / timeLength_s) * 100;
DISPLAYLEVEL(4, "%-20s : Completed in %.2f sec (cpu load : %.0f%%)\n",
srcFileName, timeLength_s, cpuLoad_pct);
srcFileName, timeLength_s, cpuLoad_pct);
}
}
/*! FIO_compressFilename_internal() :
* same as FIO_compressFilename_extRess(), with `ress.desFile` already opened.
* @return : 0 : compression completed correctly,
* 1 : missing or pb opening srcFileName
*/
static int
FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
FIO_prefs_t* const prefs,
cRess_t ress,
const char* dstFileName, const char* srcFileName,
int compressionLevel)
{
FIO_compressDisplayContext_t displayCtx = {
UTIL_getTime(),
clock()
};
U64 const fileSize = UTIL_getFileSize(srcFileName);
FIO_rust_compress_callbacks_t callbacks;
int status;
memset(&callbacks, 0, sizeof(callbacks));
callbacks.opaque = &displayCtx;
callbacks.compress_zstd = FIO_rust_compressZstdCallback;
#ifdef ZSTD_GZCOMPRESS
callbacks.compress_gzip = FIO_rust_compressGzipCallback;
#endif
#ifdef ZSTD_LZMACOMPRESS
callbacks.compress_lzma = FIO_rust_compressLzmaCallback;
#endif
#ifdef ZSTD_LZ4COMPRESS
callbacks.compress_lz4 = FIO_rust_compressLz4Callback;
#endif
callbacks.display_input = FIO_rust_compressInputDisplayCallback;
callbacks.display_status = FIO_rust_compressStatusDisplayCallback;
status = FIO_rust_compressFilenameInternal(
fCtx, prefs, &ress, dstFileName, srcFileName, fileSize,
compressionLevel, &callbacks);
switch (status) {
case FIO_RUST_COMPRESS_OK:
return 0;
case FIO_RUST_COMPRESS_GZIP_UNSUPPORTED:
EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n",
srcFileName);
case FIO_RUST_COMPRESS_LZMA_UNSUPPORTED:
EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n",
srcFileName);
case FIO_RUST_COMPRESS_LZ4_UNSUPPORTED:
EXM_THROW(20, "zstd: %s: file cannot be compressed as lz4 (zstd compiled without ZSTD_LZ4COMPRESS) -- ignored \n",
srcFileName);
case FIO_RUST_COMPRESS_ZSTD_UNSUPPORTED:
default:
assert(status == FIO_RUST_COMPRESS_ZSTD_UNSUPPORTED);
EXM_THROW(20, "zstd: %s: file cannot be compressed as zstd -- ignored \n",
srcFileName);
}
return 0;
}