refactor(fileio): move zstd stream callback into Rust
The fileio zstd projection used to route its stream callback through a C implementation that constructed the public input and output buffer views, queried pending output, called the streaming codec, and copied four scalar results back to the Rust-owned loop. That left the central codec operation in the CLI C translation unit even though the surrounding stream loop was already in Rust. Move that callback into Rust while keeping the C CCtx opaque across the boundary. Rust now builds the public ZSTD_inBuffer and ZSTD_outBuffer views, invokes ZSTD_toFlushNow and ZSTD_compressStream2, publishes the original positions and result, and preserves the existing success diagnostic through a small C display callback. The C projection passes the real CCtx as codecOpaque and retains the adaptive iteration context and all private CLI state. The buffer structs are made public within the Rust crate so this seam can reuse the existing C-compatible definitions without duplicating them. Test Plan: - `cargo +nightly fmt --manifest-path rust/Cargo.toml --all` -- passed - `cc -fsyntax-only -Werror=incompatible-pointer-types -Ilib -Ilib/common -Ilib/compress -Ilib/decompress -Ilib/dict -Ilib/legacy programs/fileio.c` -- passed - `git diff --check` and `git diff --cached --check` -- passed - Full capped native and Rust verification remains to be run after this seam
This commit is contained in:
+14
-32
@@ -331,8 +331,6 @@ int FIO_rust_adjustParamsForPatchFromMode(FIO_prefs_t* prefs,
|
||||
unsigned* fileWindowLog,
|
||||
int* autoLdm,
|
||||
int* optimalParser);
|
||||
void FIO_setInBuffer(ZSTD_inBuffer* output, const void* buf, size_t s, size_t pos);
|
||||
void FIO_setOutBuffer(ZSTD_outBuffer* output, void* buf, size_t s, size_t pos);
|
||||
const char* FIO_determineCompressedName(const char* srcFileName, const char* outDirName, const char* suffix);
|
||||
const char* FIO_rust_determineDstName(const char* srcFileName, const char* outDirName,
|
||||
const char* const* suffixList, const char* suffixListStr);
|
||||
@@ -1333,6 +1331,14 @@ typedef int (*FIO_rust_zstd_compress_stream_fn)(
|
||||
unsigned char* output, size_t outputSize,
|
||||
size_t* inputPosAfter, size_t* outputProduced,
|
||||
size_t* toFlushNow, size_t* zstdResult);
|
||||
int FIO_rust_zstd_compressStream(
|
||||
void* opaque, const char* srcFileName, int directive,
|
||||
const unsigned char* input, size_t inputSize, size_t inputPos,
|
||||
unsigned char* output, size_t outputSize,
|
||||
size_t* inputPosAfter, size_t* outputProduced,
|
||||
size_t* toFlushNow, size_t* zstdResult);
|
||||
void FIO_rust_zstd_compressStreamDisplay(
|
||||
int directive, size_t inputPos, size_t inputSize, size_t outputProduced);
|
||||
typedef void (*FIO_rust_zstd_iteration_fn)(
|
||||
void* opaque, const char* srcFileName, int* compressionLevel,
|
||||
size_t oldInputPos, size_t newInputPos, size_t toFlushNow);
|
||||
@@ -2524,36 +2530,12 @@ static void FIO_rust_zstd_sparseWriteEnd(void* opaque)
|
||||
AIO_WritePool_sparseWriteEnd((WritePoolCtx_t*)opaque);
|
||||
}
|
||||
|
||||
static int FIO_rust_zstd_compressStream(
|
||||
void* opaque, const char* srcFileName, int directive,
|
||||
const unsigned char* input, size_t inputSize, size_t inputPos,
|
||||
unsigned char* output, size_t outputSize,
|
||||
size_t* inputPosAfter, size_t* outputProduced,
|
||||
size_t* toFlushNow, size_t* zstdResult)
|
||||
void FIO_rust_zstd_compressStreamDisplay(
|
||||
int directive, size_t inputPos, size_t inputSize, size_t outputProduced)
|
||||
{
|
||||
FIO_rust_zstd_projection_context_t* const context =
|
||||
(FIO_rust_zstd_projection_context_t*)opaque;
|
||||
ZSTD_inBuffer inBuff;
|
||||
ZSTD_outBuffer outBuff;
|
||||
size_t toFlush;
|
||||
size_t result;
|
||||
FIO_setInBuffer(&inBuff, input, inputSize, inputPos);
|
||||
FIO_setOutBuffer(&outBuff, output, outputSize, 0);
|
||||
toFlush = ZSTD_toFlushNow(context->cctx);
|
||||
result = ZSTD_compressStream2(
|
||||
context->cctx, &outBuff, &inBuff, (ZSTD_EndDirective)directive);
|
||||
|
||||
*inputPosAfter = inBuff.pos;
|
||||
*outputProduced = outBuff.pos;
|
||||
*toFlushNow = toFlush;
|
||||
*zstdResult = result;
|
||||
if (!ZSTD_isError(result)) {
|
||||
DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n",
|
||||
(unsigned)directive, (unsigned)inBuff.pos,
|
||||
(unsigned)inBuff.size, (unsigned)outBuff.pos);
|
||||
}
|
||||
(void)srcFileName;
|
||||
return ZSTD_isError(result);
|
||||
DISPLAYLEVEL(6, "ZSTD_compress_generic(end:%u) => input pos(%u)<=(%u)size ; output generated %u bytes \n",
|
||||
(unsigned)directive, (unsigned)inputPos,
|
||||
(unsigned)inputSize, (unsigned)outputProduced);
|
||||
}
|
||||
|
||||
static void FIO_rust_zstd_iteration(void* opaque, const char* srcFileName,
|
||||
@@ -2766,7 +2748,7 @@ FIO_rust_compressZstdCallback(void* fCtx, void* prefs, void* ress,
|
||||
memset(&projection, 0, sizeof(projection));
|
||||
projection.readOpaque = (void*)ressPtr->readCtx;
|
||||
projection.writeOpaque = (void*)ressPtr->writeCtx;
|
||||
projection.codecOpaque = &context;
|
||||
projection.codecOpaque = (void*)ressPtr->cctx;
|
||||
projection.policyOpaque = &context;
|
||||
projection.readBufferSize = ZSTD_CStreamInSize();
|
||||
projection.readFill = FIO_rust_zstd_readFill;
|
||||
|
||||
Reference in New Issue
Block a user