ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(cli): move mixed-format dispatch into Rust

Browse Source 
Move file I/O's format probing, frame classification, callback dispatch, and
size accounting into Rust. C retains the codec-specific decompression,
pass-through, metadata, and private resource adapters, while Rust coordinates
mixed-format streams and maps callback outcomes back to the CLI error paths.

Test Plan:
- cargo test --manifest-path rust/Cargo.toml --all-targets -- --test-threads=1
- cargo test --manifest-path rust/cli/Cargo.toml --all-targets -- --test-threads=1
- cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings
- cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings
- make -B -C programs -j2 zstd
- make -B -C tests -j2 test-cli-tests
- make -B -C tests -j2 test-zstd
This commit is contained in:
ddidderr
2026-07-18 22:09:05 +02:00
parent 2a588f89f2
commit 139b45832e
2 changed files with 471 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fileio.c
+166 -53
View File
@@ -421,6 +421,39 @@ int FIO_rust_decompressZstdFrames(void* fCtx,
U64* decodedSize,
size_t* zstdError,
FIO_rust_frame_progress_fn progress);
enum {
FIO_RUST_DECOMPRESS_OK = 0,
FIO_RUST_DECOMPRESS_PASS_THROUGH = 1,
FIO_RUST_DECOMPRESS_EMPTY_INPUT = 2,
FIO_RUST_DECOMPRESS_SHORT_INPUT = 3,
FIO_RUST_DECOMPRESS_GZIP_UNSUPPORTED = 4,
FIO_RUST_DECOMPRESS_LZMA_UNSUPPORTED = 5,
FIO_RUST_DECOMPRESS_LZ4_UNSUPPORTED = 6,
FIO_RUST_DECOMPRESS_FRAME_ERROR = 7,
FIO_RUST_DECOMPRESS_UNSUPPORTED_FORMAT = 8,
FIO_RUST_DECOMPRESS_PASS_THROUGH_ERROR = 9,
FIO_RUST_DECOMPRESS_ZSTD_UNSUPPORTED = 10
};
typedef int (*FIO_rust_decompress_frame_fn)(void* opaque,
const char* srcFileName,
U64 alreadyDecoded,
U64* frameSize,
size_t* errorCode,
int mode);
typedef int (*FIO_rust_pass_through_fn)(void* opaque);
typedef struct {
void* opaque;
FIO_rust_decompress_frame_fn decode_zstd;
FIO_rust_decompress_frame_fn decode_gzip;
FIO_rust_decompress_frame_fn decode_lzma;
FIO_rust_decompress_frame_fn decode_lz4;
FIO_rust_pass_through_fn pass_through;
} FIO_rust_decompress_callbacks_t;
int FIO_rust_decompressFrames(ReadPoolCtx_t* readCtx,
const char* srcFileName,
int passThrough,
U64* decodedSize,
const FIO_rust_decompress_callbacks_t* callbacks);
void FIO_rust_displayCompressionParameters(const FIO_prefs_t* prefs);
#ifdef ZSTD_LZ4COMPRESS
int FIO_rust_LZ4_GetBlockSize_FromBlockId(int id);
@@ -2474,6 +2507,93 @@ FIO_decompressLz4Frame(dRess_t* ress, const char* srcFileName)
}
#endif
/* Rust owns the mixed-format loop, while these adapters keep each codec and
* the private dRess_t layout in this C translation unit. */
typedef struct {
FIO_ctx_t* fCtx;
dRess_t* ress;
const FIO_prefs_t* prefs;
} FIO_rust_decompression_projection_t;
static int FIO_rust_decompressZstdFrameCallback(void* opaque,
const char* srcFileName,
U64 alreadyDecoded,
U64* frameSize,
size_t* errorCode,
int mode)
{
FIO_rust_decompression_projection_t* const projection =
(FIO_rust_decompression_projection_t*)opaque;
(void)mode;
*errorCode = 0;
*frameSize = FIO_decompressZstdFrames(projection->fCtx,
projection->ress,
projection->prefs,
srcFileName,
alreadyDecoded);
return *frameSize == FIO_ERROR_FRAME_DECODING;
}
#ifdef ZSTD_GZDECOMPRESS
static int FIO_rust_decompressGzFrameCallback(void* opaque,
const char* srcFileName,
U64 alreadyDecoded,
U64* frameSize,
size_t* errorCode,
int mode)
{
FIO_rust_decompression_projection_t* const projection =
(FIO_rust_decompression_projection_t*)opaque;
(void)alreadyDecoded;
(void)mode;
*errorCode = 0;
*frameSize = FIO_decompressGzFrame(projection->ress, srcFileName);
return *frameSize == FIO_ERROR_FRAME_DECODING;
}
#endif
#ifdef ZSTD_LZMADECOMPRESS
static int FIO_rust_decompressLzmaFrameCallback(void* opaque,
const char* srcFileName,
U64 alreadyDecoded,
U64* frameSize,
size_t* errorCode,
int mode)
{
FIO_rust_decompression_projection_t* const projection =
(FIO_rust_decompression_projection_t*)opaque;
(void)alreadyDecoded;
*errorCode = 0;
*frameSize = FIO_decompressLzmaFrame(projection->ress, srcFileName, mode);
return *frameSize == FIO_ERROR_FRAME_DECODING;
}
#endif
#ifdef ZSTD_LZ4DECOMPRESS
static int FIO_rust_decompressLz4FrameCallback(void* opaque,
const char* srcFileName,
U64 alreadyDecoded,
U64* frameSize,
size_t* errorCode,
int mode)
{
FIO_rust_decompression_projection_t* const projection =
(FIO_rust_decompression_projection_t*)opaque;
(void)alreadyDecoded;
(void)mode;
*errorCode = 0;
*frameSize = FIO_decompressLz4Frame(projection->ress, srcFileName);
return *frameSize == FIO_ERROR_FRAME_DECODING;
}
#endif
static int FIO_rust_decompressPassThroughCallback(void* opaque)
{
FIO_rust_decompression_projection_t* const projection =
(FIO_rust_decompression_projection_t*)opaque;
return FIO_passThrough(projection->ress);
}
/** FIO_decompressFrames() :
@@ -2486,9 +2606,11 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
dRess_t ress, const FIO_prefs_t* const prefs,
const char* dstFileName, const char* srcFileName)
{
unsigned readSomething = 0;
unsigned long long filesize = 0;
U64 filesize = 0;
int passThrough = prefs->passThrough;
FIO_rust_decompression_projection_t projection;
FIO_rust_decompress_callbacks_t callbacks;
int status;
if (passThrough == -1) {
/* If pass-through mode is not explicitly enabled or disabled,
@@ -2499,72 +2621,63 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
}
assert(passThrough == 0 || passThrough == 1);
/* for each frame */
for ( ; ; ) {
/* check magic number -> version */
size_t const toRead = 4;
const BYTE* buf;
AIO_ReadPool_fillBuffer(ress.readCtx, toRead);
buf = (const BYTE*)ress.readCtx->srcBuffer;
if (ress.readCtx->srcBufferLoaded==0) {
if (readSomething==0) { /* srcFile is empty (which is invalid) */
DISPLAYLEVEL(1, "zstd: %s: unexpected end of file \n", srcFileName);
return 1;
} /* else, just reached frame boundary */
break; /* no more input */
}
readSomething = 1; /* there is at least 1 byte in srcFile */
if (ress.readCtx->srcBufferLoaded < toRead) { /* not enough input to check magic number */
if (passThrough) {
return FIO_passThrough(&ress);
}
projection.fCtx = fCtx;
projection.ress = &ress;
projection.prefs = prefs;
memset(&callbacks, 0, sizeof(callbacks));
callbacks.opaque = &projection;
callbacks.decode_zstd = FIO_rust_decompressZstdFrameCallback;
#ifdef ZSTD_GZDECOMPRESS
callbacks.decode_gzip = FIO_rust_decompressGzFrameCallback;
#endif
#ifdef ZSTD_LZMADECOMPRESS
callbacks.decode_lzma = FIO_rust_decompressLzmaFrameCallback;
#endif
#ifdef ZSTD_LZ4DECOMPRESS
callbacks.decode_lz4 = FIO_rust_decompressLz4FrameCallback;
#endif
callbacks.pass_through = FIO_rust_decompressPassThroughCallback;
status = FIO_rust_decompressFrames(ress.readCtx, srcFileName, passThrough,
&filesize, &callbacks);
switch (status) {
case FIO_RUST_DECOMPRESS_OK:
break;
case FIO_RUST_DECOMPRESS_PASS_THROUGH:
return 0;
case FIO_RUST_DECOMPRESS_EMPTY_INPUT:
DISPLAYLEVEL(1, "zstd: %s: unexpected end of file \n", srcFileName);
return 1;
case FIO_RUST_DECOMPRESS_SHORT_INPUT:
DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName);
return 1;
}
if (ZSTD_isFrame(buf, ress.readCtx->srcBufferLoaded)) {
unsigned long long const frameSize = FIO_decompressZstdFrames(fCtx, &ress, prefs, srcFileName, filesize);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize;
} else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */
#ifdef ZSTD_GZDECOMPRESS
unsigned long long const frameSize = FIO_decompressGzFrame(&ress, srcFileName);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize;
#else
case FIO_RUST_DECOMPRESS_GZIP_UNSUPPORTED:
DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without HAVE_ZLIB) -- ignored \n", srcFileName);
return 1;
#endif
} else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */
|| (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */
#ifdef ZSTD_LZMADECOMPRESS
unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFileName, buf[0] != 0xFD);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize;
#else
case FIO_RUST_DECOMPRESS_LZMA_UNSUPPORTED:
DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without HAVE_LZMA) -- ignored \n", srcFileName);
return 1;
#endif
} else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) {
#ifdef ZSTD_LZ4DECOMPRESS
unsigned long long const frameSize = FIO_decompressLz4Frame(&ress, srcFileName);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize;
#else
case FIO_RUST_DECOMPRESS_LZ4_UNSUPPORTED:
DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName);
return 1;
#endif
} else if (passThrough) {
return FIO_passThrough(&ress);
} else {
case FIO_RUST_DECOMPRESS_UNSUPPORTED_FORMAT:
DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName);
return 1;
} } /* for each frame */
case FIO_RUST_DECOMPRESS_FRAME_ERROR:
case FIO_RUST_DECOMPRESS_PASS_THROUGH_ERROR:
case FIO_RUST_DECOMPRESS_ZSTD_UNSUPPORTED:
return 1;
default:
assert(0);
return 1;
}
/* Final Status */
fCtx->totalBytesOutput += (size_t)filesize;
DISPLAY_PROGRESS("\r%79s\r", "");
if (FIO_shouldDisplayFileSummary(fCtx))
DISPLAY_SUMMARY("%-20s: %llu bytes \n", srcFileName, filesize);
DISPLAY_SUMMARY("%-20s: %llu bytes \n", srcFileName,
(unsigned long long)filesize);
return 0;
}