ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(cli): move consecutive zstd frame dispatch into Rust

Browse Source 
The CLI already delegated one zstd frame at a time to Rust, but the C
FIO_decompressFrames loop still owned repeated-frame dispatch. That left
concatenated zstd streams split across the language boundary and made the
Rust frame helper unable to preserve the next mixed-format header itself.

Add a Rust multi-frame adapter that repeatedly probes four buffered bytes,
uses the existing one-frame decoder, accumulates decoded output and progress,
and leaves a following non-zstd header or short trailing input untouched for
C's format dispatcher. Keep C responsible for probing the first format,
non-zstd dispatch, diagnostics, cleanup, and final file accounting. Decoder
errors still preserve the current input for FIO_zstdErrorHelp().

Test Plan:
- `cargo test --manifest-path rust/Cargo.toml --lib fileio_asyncio -- --test-threads=1` -- 20 passed
- `cargo clippy --manifest-path rust/Cargo.toml --lib -- -D warnings` -- passed
- `cargo +nightly fmt --manifest-path rust/Cargo.toml -- --check` -- passed
- `make -B -C tests -j2 test-cli-tests` -- all 41 passed
- `git diff --cached --check` -- passed
- Full root `cargo clippy --all-targets/--benches/--tests -- -D warnings` remains blocked by the pre-existing `manual_repeat_n` warning in `rust/src/zstd_compress.rs` test code; no unrelated change was made.
This commit is contained in:
ddidderr
2026-07-18 19:34:33 +02:00
parent 6ea2ea887f
commit b0488f285a
2 changed files with 291 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fileio.c
+18 -18
View File
@@ -412,15 +412,15 @@ enum {
typedef void (*FIO_rust_frame_progress_fn)(void* opaque,
const char* srcFileName,
U64 decodedSize);
int FIO_rust_decompressZstdFrame(void* fCtx,
void* dctx,
ReadPoolCtx_t* readCtx,
WritePoolCtx_t* writeCtx,
const char* srcFileName,
U64 alreadyDecoded,
U64* frameSize,
size_t* zstdError,
FIO_rust_frame_progress_fn progress);
int FIO_rust_decompressZstdFrames(void* fCtx,
void* dctx,
ReadPoolCtx_t* readCtx,
WritePoolCtx_t* writeCtx,
const char* srcFileName,
U64 alreadyDecoded,
U64* decodedSize,
size_t* zstdError,
FIO_rust_frame_progress_fn progress);
void FIO_rust_displayCompressionParameters(const FIO_prefs_t* prefs);
#ifdef ZSTD_LZ4COMPRESS
int FIO_rust_LZ4_GetBlockSize_FromBlockId(int id);
@@ -2235,20 +2235,20 @@ FIO_decompressZstdFrameProgress(void* const opaque,
}
static unsigned long long
FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
const FIO_prefs_t* const prefs,
const char* srcFileName,
U64 alreadyDecoded) /* for multi-frames streams */
FIO_decompressZstdFrames(FIO_ctx_t* const fCtx, dRess_t* ress,
const FIO_prefs_t* const prefs,
const char* srcFileName,
U64 alreadyDecoded) /* for multi-frames streams */
{
U64 frameSize = 0;
U64 decodedSize = 0;
size_t zstdError = 0;
int const status = FIO_rust_decompressZstdFrame(
int const status = FIO_rust_decompressZstdFrames(
fCtx, ress->dctx, ress->readCtx, ress->writeCtx, srcFileName,
alreadyDecoded, &frameSize, &zstdError,
alreadyDecoded, &decodedSize, &zstdError,
FIO_decompressZstdFrameProgress);
if (status == FIO_RUST_ZSTD_FRAME_OK)
return frameSize;
return decodedSize;
if (status == FIO_RUST_ZSTD_FRAME_DECODING_ERROR) {
DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
srcFileName, ZSTD_getErrorName(zstdError));
@@ -2522,7 +2522,7 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
return 1;
}
if (ZSTD_isFrame(buf, ress.readCtx->srcBufferLoaded)) {
unsigned long long const frameSize = FIO_decompressZstdFrame(fCtx, &ress, prefs, srcFileName, filesize);
unsigned long long const frameSize = FIO_decompressZstdFrames(fCtx, &ress, prefs, srcFileName, filesize);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize;
} else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */