ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(cli): move zstd frame decode loop into Rust

Browse Source 
Move the zstd frame read/decode/write loop behind a narrow Rust ABI.  Rust
now owns decoder reset, asynchronous input/output buffer coordination, frame
progress accounting, frame-boundary handling, and status classification.  C
retains the mixed-format dispatcher, user-facing diagnostics, window-limit
help, progress formatting callback, source/destination policy, cleanup, and
metadata handling.

The bridge passes opaque decoder and I/O-pool pointers plus scalar counters and
out-parameters.  Decoder errors return before consuming the current input
buffer so the C diagnostic path can inspect it.  The Rust test seam covers
valid streaming, concatenated frame boundaries, pre-increment progress
semantics, decoder-error input preservation, and premature EOF.

Test Plan:
- cargo test --manifest-path rust/Cargo.toml --lib fileio_asyncio -- --test-threads=1
- cargo clippy --manifest-path rust/Cargo.toml --lib -- -D warnings
- cargo +nightly fmt --manifest-path rust/Cargo.toml -- --check
- cargo test --manifest-path rust/Cargo.toml --lib -- --test-threads=1
- make -B -C tests -j2 test-cli-tests (all 41 passed)
- git diff --cached --check
This commit is contained in:
ddidderr
2026-07-18 19:14:58 +02:00
parent 20db5861e5
commit bf364cce1c
2 changed files with 578 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fileio.c
+61 -59
View File
@@ -404,6 +404,23 @@ void FIO_rust_freeDict(int dictBufferType,
void** dictHandle);
int FIO_rust_removeFile(const char* path);
int FIO_rust_passThrough(ReadPoolCtx_t* readCtx, WritePoolCtx_t* writeCtx);
enum {
FIO_RUST_ZSTD_FRAME_OK = 0,
FIO_RUST_ZSTD_FRAME_DECODING_ERROR = 1,
FIO_RUST_ZSTD_FRAME_PREMATURE_END = 2,
};
typedef void (*FIO_rust_frame_progress_fn)(void* opaque,
const char* srcFileName,
U64 decodedSize);
int FIO_rust_decompressZstdFrame(void* fCtx,
void* dctx,
ReadPoolCtx_t* readCtx,
WritePoolCtx_t* writeCtx,
const char* srcFileName,
U64 alreadyDecoded,
U64* frameSize,
size_t* zstdError,
FIO_rust_frame_progress_fn progress);
void FIO_rust_displayCompressionParameters(const FIO_prefs_t* prefs);
#ifdef ZSTD_LZ4COMPRESS
int FIO_rust_LZ4_GetBlockSize_FromBlockId(int id);
@@ -2192,6 +2209,31 @@ FIO_zstdErrorHelp(const FIO_prefs_t* const prefs,
* @return : size of decoded zstd frame, or an error code
*/
#define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2))
static void
FIO_decompressZstdFrameProgress(void* const opaque,
const char* const srcFileName,
U64 const decodedSize)
{
FIO_ctx_t* const fCtx = (FIO_ctx_t*)opaque;
const char* srcFName20 = srcFileName;
size_t const srcFileLength = strlen(srcFileName);
UTIL_HumanReadableSize_t const hrs = UTIL_makeHumanReadableSize(decodedSize);
/* display last 20 characters only when not --verbose */
if ((srcFileLength>20) && (g_display_prefs.displayLevel<3))
srcFName20 += srcFileLength-20;
if (fCtx->nbFilesTotal > 1) {
DISPLAYUPDATE_PROGRESS(
"\rDecompress: %2u/%2u files. Current: %s : %.*f%s... ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFName20,
hrs.precision, hrs.value, hrs.suffix);
} else {
DISPLAYUPDATE_PROGRESS("\r%-20.20s : %.*f%s... ",
srcFName20, hrs.precision, hrs.value, hrs.suffix);
}
}
static unsigned long long
FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
const FIO_prefs_t* const prefs,
@@ -2199,68 +2241,28 @@ FIO_decompressZstdFrame(FIO_ctx_t* const fCtx, dRess_t* ress,
U64 alreadyDecoded) /* for multi-frames streams */
{
U64 frameSize = 0;
const char* srcFName20 = srcFileName;
IOJob_t* writeJob = AIO_WritePool_acquireJob(ress->writeCtx);
assert(writeJob);
size_t zstdError = 0;
int const status = FIO_rust_decompressZstdFrame(
fCtx, ress->dctx, ress->readCtx, ress->writeCtx, srcFileName,
alreadyDecoded, &frameSize, &zstdError,
FIO_decompressZstdFrameProgress);
/* display last 20 characters only when not --verbose */
{ size_t const srcFileLength = strlen(srcFileName);
if ((srcFileLength>20) && (g_display_prefs.displayLevel<3))
srcFName20 += srcFileLength-20;
if (status == FIO_RUST_ZSTD_FRAME_OK)
return frameSize;
if (status == FIO_RUST_ZSTD_FRAME_DECODING_ERROR) {
DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
srcFileName, ZSTD_getErrorName(zstdError));
FIO_zstdErrorHelp(prefs, ress, zstdError, srcFileName);
return FIO_ERROR_FRAME_DECODING;
}
if (status == FIO_RUST_ZSTD_FRAME_PREMATURE_END) {
DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n",
srcFileName);
return FIO_ERROR_FRAME_DECODING;
}
ZSTD_DCtx_reset(ress->dctx, ZSTD_reset_session_only);
/* Header loading : ensures ZSTD_getFrameHeader() will succeed */
AIO_ReadPool_fillBuffer(ress->readCtx, ZSTD_FRAMEHEADERSIZE_MAX);
/* Main decompression Loop */
while (1) {
ZSTD_inBuffer inBuff = setInBuffer( ress->readCtx->srcBuffer, ress->readCtx->srcBufferLoaded, 0 );
ZSTD_outBuffer outBuff= setOutBuffer( writeJob->buffer, writeJob->bufferSize, 0 );
size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff);
UTIL_HumanReadableSize_t const hrs = UTIL_makeHumanReadableSize(alreadyDecoded+frameSize);
if (ZSTD_isError(readSizeHint)) {
DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
srcFileName, ZSTD_getErrorName(readSizeHint));
FIO_zstdErrorHelp(prefs, ress, readSizeHint, srcFileName);
AIO_WritePool_releaseIoJob(writeJob);
return FIO_ERROR_FRAME_DECODING;
}
/* Write block */
writeJob->usedBufferSize = outBuff.pos;
AIO_WritePool_enqueueAndReacquireWriteJob(&writeJob);
frameSize += outBuff.pos;
if (fCtx->nbFilesTotal > 1) {
DISPLAYUPDATE_PROGRESS(
"\rDecompress: %2u/%2u files. Current: %s : %.*f%s... ",
fCtx->currFileIdx+1, fCtx->nbFilesTotal, srcFName20, hrs.precision, hrs.value, hrs.suffix);
} else {
DISPLAYUPDATE_PROGRESS("\r%-20.20s : %.*f%s... ",
srcFName20, hrs.precision, hrs.value, hrs.suffix);
}
AIO_ReadPool_consumeBytes(ress->readCtx, inBuff.pos);
if (readSizeHint == 0) break; /* end of frame */
/* Fill input buffer */
{ size_t const toDecode = MIN(readSizeHint, ZSTD_DStreamInSize()); /* support large skippable frames */
if (ress->readCtx->srcBufferLoaded < toDecode) {
size_t const readSize = AIO_ReadPool_fillBuffer(ress->readCtx, toDecode);
if (readSize==0) {
DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n",
srcFileName);
AIO_WritePool_releaseIoJob(writeJob);
return FIO_ERROR_FRAME_DECODING;
}
} } }
AIO_WritePool_releaseIoJob(writeJob);
AIO_WritePool_sparseWriteEnd(ress->writeCtx);
return frameSize;
assert(0);
return FIO_ERROR_FRAME_DECODING;
}