refactor(fileio): move zstd stream callback into Rust

The fileio zstd projection used to route its stream callback through a C
implementation that constructed the public input and output buffer views,
queried pending output, called the streaming codec, and copied four scalar
results back to the Rust-owned loop.  That left the central codec operation in
the CLI C translation unit even though the surrounding stream loop was already
in Rust.

Move that callback into Rust while keeping the C CCtx opaque across the
boundary.  Rust now builds the public ZSTD_inBuffer and ZSTD_outBuffer views,
invokes ZSTD_toFlushNow and ZSTD_compressStream2, publishes the original
positions and result, and preserves the existing success diagnostic through a
small C display callback.  The C projection passes the real CCtx as codecOpaque
and retains the adaptive iteration context and all private CLI state.  The
buffer structs are made public within the Rust crate so this seam can reuse the
existing C-compatible definitions without duplicating them.

Test Plan:
- `cargo +nightly fmt --manifest-path rust/Cargo.toml --all` -- passed
- `cc -fsyntax-only -Werror=incompatible-pointer-types -Ilib -Ilib/common -Ilib/compress -Ilib/decompress -Ilib/dict -Ilib/legacy programs/fileio.c` -- passed
- `git diff --check` and `git diff --cached --check` -- passed
- Full capped native and Rust verification remains to be run after this seam
This commit is contained in:
2026-07-20 15:09:23 +02:00
parent 0e93557879
commit 725877ad7a
3 changed files with 239 additions and 41 deletions
+14 -32
View File
@@ -331,8 +331,6 @@ int FIO_rust_adjustParamsForPatchFromMode(FIO_prefs_t* prefs,
unsigned* fileWindowLog,
int* autoLdm,
int* optimalParser);
void FIO_setInBuffer(ZSTD_inBuffer* output, const void* buf, size_t s, size_t pos);
void FIO_setOutBuffer(ZSTD_outBuffer* output, void* buf, size_t s, size_t pos);
const char* FIO_determineCompressedName(const char* srcFileName, const char* outDirName, const char* suffix);
const char* FIO_rust_determineDstName(const char* srcFileName, const char* outDirName,
const char* const* suffixList, const char* suffixListStr);
@@ -1333,6 +1331,14 @@ typedef int (*FIO_rust_zstd_compress_stream_fn)(
unsigned char* output, size_t outputSize,
size_t* inputPosAfter, size_t* outputProduced,
size_t* toFlushNow, size_t* zstdResult);
int FIO_rust_zstd_compressStream(
void* opaque, const char* srcFileName, int directive,
const unsigned char* input, size_t inputSize, size_t inputPos,
unsigned char* output, size_t outputSize,
size_t* inputPosAfter, size_t* outputProduced,
size_t* toFlushNow, size_t* zstdResult);
void FIO_rust_zstd_compressStreamDisplay(
int directive, size_t inputPos, size_t inputSize, size_t outputProduced);
typedef void (*FIO_rust_zstd_iteration_fn)(
void* opaque, const char* srcFileName, int* compressionLevel,
size_t oldInputPos, size_t newInputPos, size_t toFlushNow);
@@ -2524,36 +2530,12 @@ static void FIO_rust_zstd_sparseWriteEnd(void* opaque)
AIO_WritePool_sparseWriteEnd((WritePoolCtx_t*)opaque);
}
static int FIO_rust_zstd_compressStream(
void* opaque, const char* srcFileName, int directive,
const unsigned char* input, size_t inputSize, size_t inputPos,
unsigned char* output, size_t outputSize,
size_t* inputPosAfter, size_t* outputProduced,
size_t* toFlushNow, size_t* zstdResult)
void FIO_rust_zstd_compressStreamDisplay(
int directive, size_t inputPos, size_t inputSize, size_t outputProduced)
{
FIO_rust_zstd_projection_context_t* const context =
(FIO_rust_zstd_projection_context_t*)opaque;
ZSTD_inBuffer inBuff;
ZSTD_outBuffer outBuff;
size_t toFlush;
size_t result;
FIO_setInBuffer(&inBuff, input, inputSize, inputPos);
FIO_setOutBuffer(&outBuff, output, outputSize, 0);
toFlush = ZSTD_toFlushNow(context->cctx);
result = ZSTD_compressStream2(
context->cctx, &outBuff, &inBuff, (ZSTD_EndDirective)directive);
*inputPosAfter = inBuff.pos;
*outputProduced = outBuff.pos;
*toFlushNow = toFlush;
*zstdResult = result;
if (!ZSTD_isError(result)) {
DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n",
(unsigned)directive, (unsigned)inBuff.pos,
(unsigned)inBuff.size, (unsigned)outBuff.pos);
}
(void)srcFileName;
return ZSTD_isError(result);
DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n",
(unsigned)directive, (unsigned)inputPos,
(unsigned)inputSize, (unsigned)outputProduced);
}
static void FIO_rust_zstd_iteration(void* opaque, const char* srcFileName,
@@ -2766,7 +2748,7 @@ FIO_rust_compressZstdCallback(void* fCtx, void* prefs, void* ress,
memset(&projection, 0, sizeof(projection));
projection.readOpaque = (void*)ressPtr->readCtx;
projection.writeOpaque = (void*)ressPtr->writeCtx;
projection.codecOpaque = &context;
projection.codecOpaque = (void*)ressPtr->cctx;
projection.policyOpaque = &context;
projection.readBufferSize = ZSTD_CStreamInSize();
projection.readFill = FIO_rust_zstd_readFill;