refactor(fileio): move per-file decompression policy to Rust

Move source and destination lifecycle orchestration for single-file
 decompression into the Rust file-I/O layer.  Rust now owns async-selection,
resource attachment, destination opening, handler lifetime, metadata transfer,
close-error propagation, artefact cleanup, and successful-only source removal.
C retains filesystem/resource handles, format dispatch, diagnostics, and the
private callback implementations behind a narrow projection.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo fmt --manifest-path rust/Cargo.toml --all -- --check
- git diff --check and git diff --cached --check
- Added focused Rust callback-order, cleanup, and removal-policy tests
- Heavy Cargo, Make, native, and fuzzer verification deferred to the serial capped pass
This commit is contained in:
2026-07-20 03:02:07 +02:00
parent 73e0a5e629
commit afa2dde80d
2 changed files with 887 additions and 129 deletions
+285 -123
View File
@@ -1234,6 +1234,91 @@ typedef char FIO_rust_decompress_multiple_separate_projection_size[
int FIO_rust_decompressMultipleSeparateFilenames(
const FIO_rust_decompress_multiple_separate_projection_t* projection);
enum {
FIO_RUST_DECOMPRESS_SRC_OPEN_OK = 0,
};
typedef int (*FIO_rust_decompress_src_open_fn)(void* opaque, const char* srcFileName,
U64* fileSize, int* sourceIsRegular);
typedef void (*FIO_rust_decompress_src_async_fn)(void* opaque, int asyncMode);
typedef void (*FIO_rust_decompress_src_attach_fn)(void* opaque);
typedef int (*FIO_rust_decompress_src_close_fn)(void* opaque);
typedef int (*FIO_rust_decompress_dst_open_fn)(void* opaque,
const char* srcFileName,
const char* dstFileName,
int transferStat, int* dstFd);
typedef void (*FIO_rust_decompress_dst_attach_fn)(void* opaque);
typedef void (*FIO_rust_decompress_handler_fn)(void* opaque);
typedef int (*FIO_rust_decompress_file_fn)(void* opaque,
const char* dstFileName,
const char* srcFileName);
typedef void (*FIO_rust_decompress_dst_stat_fn)(void* opaque, int dstFd,
const char* dstFileName);
typedef int (*FIO_rust_decompress_dst_close_fn)(void* opaque);
typedef void (*FIO_rust_decompress_dst_remove_fn)(void* opaque, const char* dstFileName);
typedef int (*FIO_rust_decompress_src_remove_fn)(void* opaque, const char* srcFileName);
/* Rust owns per-file source/destination ordering; the callback context keeps
* dRess_t, FILE/pool handles, diagnostics, and the format dispatcher private. */
typedef struct {
void* opaque;
const char* dstFileName;
const char* srcFileName;
int destinationAlreadyOpen;
int testMode;
int sourceIsStdin;
int destinationIsStdout;
int removeSource;
FIO_rust_decompress_src_open_fn openSource;
FIO_rust_decompress_src_async_fn setAsync;
FIO_rust_decompress_src_attach_fn attachSource;
FIO_rust_decompress_src_attach_fn detachSource;
FIO_rust_decompress_src_close_fn closeSource;
FIO_rust_decompress_dst_open_fn openDestination;
FIO_rust_decompress_dst_attach_fn attachDestination;
FIO_rust_decompress_handler_fn addHandler;
FIO_rust_decompress_file_fn decompress;
FIO_rust_decompress_handler_fn clearHandler;
FIO_rust_decompress_dst_stat_fn setFDStat;
FIO_rust_decompress_dst_close_fn closeDestination;
FIO_rust_decompress_handler_fn utimeDestination;
FIO_rust_decompress_dst_remove_fn removeDestination;
FIO_rust_decompress_src_remove_fn removeSourceFile;
} FIO_rust_decompress_file_projection_t;
typedef char FIO_rust_decompress_file_opaque_offset[
(offsetof(FIO_rust_decompress_file_projection_t, opaque) == 0) ? 1 : -1];
typedef char FIO_rust_decompress_file_dst_name_offset[
(offsetof(FIO_rust_decompress_file_projection_t, dstFileName)
== sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_file_src_name_offset[
(offsetof(FIO_rust_decompress_file_projection_t, srcFileName)
== 2 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_file_destination_open_offset[
(offsetof(FIO_rust_decompress_file_projection_t, destinationAlreadyOpen)
== 3 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_file_test_mode_offset[
(offsetof(FIO_rust_decompress_file_projection_t, testMode)
== 3 * sizeof(void*) + sizeof(int)) ? 1 : -1];
typedef char FIO_rust_decompress_file_stdin_offset[
(offsetof(FIO_rust_decompress_file_projection_t, sourceIsStdin)
== 3 * sizeof(void*) + 2 * sizeof(int)) ? 1 : -1];
typedef char FIO_rust_decompress_file_stdout_offset[
(offsetof(FIO_rust_decompress_file_projection_t, destinationIsStdout)
== 3 * sizeof(void*) + 3 * sizeof(int)) ? 1 : -1];
typedef char FIO_rust_decompress_file_remove_source_offset[
(offsetof(FIO_rust_decompress_file_projection_t, removeSource)
== 3 * sizeof(void*) + 4 * sizeof(int)) ? 1 : -1];
typedef char FIO_rust_decompress_file_callback_offset[
(offsetof(FIO_rust_decompress_file_projection_t, openSource)
== ((3 * sizeof(void*) + 5 * sizeof(int) + sizeof(void*) - 1)
/ sizeof(void*)) * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_file_projection_size[
(sizeof(FIO_rust_decompress_file_projection_t)
== ((3 * sizeof(void*) + 5 * sizeof(int) + sizeof(void*) - 1)
/ sizeof(void*)) * sizeof(void*)
+ 15 * sizeof(FIO_rust_decompress_src_open_fn)) ? 1 : -1];
int FIO_rust_decompressFilename(
const FIO_rust_decompress_file_projection_t* projection);
enum {
FIO_RUST_ZSTD_OK = 0,
FIO_RUST_ZSTD_COMPRESS_ERROR = 1,
@@ -3896,132 +3981,209 @@ static int FIO_decompressFrames(FIO_ctx_t* const fCtx,
return FIO_rust_finishDecompressFrames(status, srcFileName, filesize, &callbacks);
}
/** FIO_decompressDstFile() :
open `dstFileName`, or pass-through if writeCtx's file is already != 0,
then start decompression process (FIO_decompressFrames()).
@return : 0 : OK
1 : operation aborted
*/
static int FIO_decompressDstFile(FIO_ctx_t* const fCtx,
FIO_prefs_t* const prefs,
dRess_t ress,
const char* dstFileName,
const char* srcFileName,
const stat_t* srcFileStat)
{
int result;
int releaseDstFile = 0;
int transferStat = 0;
int dstFd = 0;
if ((AIO_WritePool_getFile(ress.writeCtx) == NULL) && (prefs->testMode == 0)) {
FILE *dstFile;
int dstFilePermissions = DEFAULT_FILE_PERMISSIONS;
if ( strcmp(srcFileName, stdinmark) /* special case : don't transfer permissions from stdin */
&& strcmp(dstFileName, stdoutmark)
&& UTIL_isRegularFileStat(srcFileStat) ) {
transferStat = 1;
dstFilePermissions = TEMPORARY_FILE_PERMISSIONS;
}
releaseDstFile = 1;
dstFile = FIO_openDstFile(fCtx, prefs, srcFileName, dstFileName, dstFilePermissions);
if (dstFile==NULL) return 1;
dstFd = fileno(dstFile);
AIO_WritePool_setFile(ress.writeCtx, dstFile);
/* Must only be added after FIO_openDstFile() succeeds.
* Otherwise we may delete the destination file if it already exists,
* and the user presses Ctrl-C when asked if they wish to overwrite.
*/
addHandler(dstFileName);
}
result = FIO_decompressFrames(fCtx, ress, prefs, dstFileName, srcFileName);
if (releaseDstFile) {
clearHandler();
if (transferStat) {
UTIL_setFDStat(dstFd, dstFileName, srcFileStat);
}
if (AIO_WritePool_closeFile(ress.writeCtx)) {
DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno));
result = 1;
}
if (transferStat) {
UTIL_utime(dstFileName, srcFileStat);
}
if ( (result != 0) /* operation failure */
&& strcmp(dstFileName, stdoutmark) /* special case : don't remove() stdout */
) {
FIO_removeFile(dstFileName); /* remove decompression artefact; note: don't do anything special if remove() fails */
}
}
return result;
}
/** FIO_decompressSrcFile() :
Open `srcFileName`, transfer control to decompressDstFile()
@return : 0 : OK
1 : error
*/
static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName)
{
FILE* srcFile;
typedef struct {
FIO_ctx_t* fCtx;
FIO_prefs_t* prefs;
dRess_t* ress;
const char* dstFileName;
const char* srcFileName;
stat_t srcFileStat;
int result;
U64 fileSize = UTIL_FILESIZE_UNKNOWN;
FILE* srcFile;
FILE* dstFile;
} FIO_rust_decompress_file_context_t;
static int FIO_rust_decompressSourceOpen(void* opaque, const char* srcFileName,
U64* fileSize, int* sourceIsRegular)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
if (UTIL_isDirectory(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
return 1;
}
srcFile = FIO_openSrcFile(prefs, srcFileName, &srcFileStat);
if (srcFile==NULL) return 1;
context->srcFile = FIO_openSrcFile(context->prefs, srcFileName, &context->srcFileStat);
if (context->srcFile == NULL)
return 1;
/* Don't use AsyncIO for small files */
if (strcmp(srcFileName, stdinmark)) /* Stdin doesn't have stats */
fileSize = UTIL_getFileSizeStat(&srcFileStat);
if(fileSize != UTIL_FILESIZE_UNKNOWN && fileSize < ZSTD_BLOCKSIZE_MAX * 3) {
AIO_ReadPool_setAsync(ress.readCtx, 0);
AIO_WritePool_setAsync(ress.writeCtx, 0);
} else {
AIO_ReadPool_setAsync(ress.readCtx, 1);
AIO_WritePool_setAsync(ress.writeCtx, 1);
}
*fileSize = strcmp(srcFileName, stdinmark)
? UTIL_getFileSizeStat(&context->srcFileStat)
: UTIL_FILESIZE_UNKNOWN;
*sourceIsRegular = strcmp(srcFileName, stdinmark)
? UTIL_isRegularFileStat(&context->srcFileStat)
: 0;
return FIO_RUST_DECOMPRESS_SRC_OPEN_OK;
}
AIO_ReadPool_setFile(ress.readCtx, srcFile);
static void FIO_rust_decompressSourceSetAsync(void* opaque, int asyncMode)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
AIO_ReadPool_setAsync(context->ress->readCtx, asyncMode);
AIO_WritePool_setAsync(context->ress->writeCtx, asyncMode);
}
result = FIO_decompressDstFile(fCtx, prefs, ress, dstFileName, srcFileName, &srcFileStat);
static void FIO_rust_decompressSourceAttach(void* opaque)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
AIO_ReadPool_setFile(context->ress->readCtx, context->srcFile);
}
AIO_ReadPool_setFile(ress.readCtx, NULL);
static void FIO_rust_decompressSourceDetach(void* opaque)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
AIO_ReadPool_setFile(context->ress->readCtx, NULL);
}
/* Close file */
if (fclose(srcFile)) {
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); /* error should not happen */
static int FIO_rust_decompressSourceClose(void* opaque)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
int const result = fclose(context->srcFile);
context->srcFile = NULL;
if (result) {
DISPLAYLEVEL(1, "zstd: %s: %s \n", context->srcFileName, strerror(errno));
return 1;
}
if ( prefs->removeSrcFile /* --rm */
&& (result==0) /* decompression successful */
&& strcmp(srcFileName, stdinmark) ) /* not stdin */ {
/* We must clear the handler, since after this point calling it would
* delete both the source and destination files.
*/
clearHandler();
if (FIO_removeFile(srcFileName)) {
/* failed to remove src file */
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
return 1;
} }
return result;
return 0;
}
static int FIO_rust_decompressDestinationOpen(void* opaque,
const char* srcFileName,
const char* dstFileName,
int transferStat, int* dstFd)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
int const permissions = transferStat
? TEMPORARY_FILE_PERMISSIONS
: DEFAULT_FILE_PERMISSIONS;
context->dstFile = FIO_openDstFile(
context->fCtx, context->prefs, srcFileName, dstFileName, permissions);
if (context->dstFile == NULL)
return 1;
*dstFd = fileno(context->dstFile);
return 0;
}
static void FIO_rust_decompressDestinationAttach(void* opaque)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
AIO_WritePool_setFile(context->ress->writeCtx, context->dstFile);
}
static void FIO_rust_decompressAddHandler(void* opaque)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
addHandler(context->dstFileName);
}
static int FIO_rust_decompressFile(void* opaque,
const char* dstFileName,
const char* srcFileName)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
return FIO_decompressFrames(
context->fCtx, *context->ress, context->prefs, dstFileName, srcFileName);
}
static void FIO_rust_decompressClearHandler(void* opaque)
{
(void)opaque;
clearHandler();
}
static void FIO_rust_decompressSetFDStat(void* opaque, int dstFd,
const char* dstFileName)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
UTIL_setFDStat(dstFd, dstFileName, &context->srcFileStat);
}
static int FIO_rust_decompressDestinationClose(void* opaque)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
int const result = AIO_WritePool_closeFile(context->ress->writeCtx);
context->dstFile = NULL;
if (result) {
DISPLAYLEVEL(1, "zstd: %s: %s \n", context->dstFileName, strerror(errno));
return 1;
}
return 0;
}
static void FIO_rust_decompressDestinationUtime(void* opaque)
{
FIO_rust_decompress_file_context_t* const context =
(FIO_rust_decompress_file_context_t*)opaque;
UTIL_utime(context->dstFileName, &context->srcFileStat);
}
static void FIO_rust_decompressDestinationRemove(void* opaque, const char* dstFileName)
{
(void)opaque;
(void)FIO_removeFile(dstFileName);
}
static int FIO_rust_decompressSourceRemove(void* opaque, const char* srcFileName)
{
(void)opaque;
if (FIO_removeFile(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno));
return 1;
}
return 0;
}
static int FIO_decompressFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs,
dRess_t* const ress, const char* dstFileName,
const char* srcFileName)
{
FIO_rust_decompress_file_context_t context;
FIO_rust_decompress_file_projection_t projection;
memset(&context, 0, sizeof(context));
context.fCtx = fCtx;
context.prefs = prefs;
context.ress = ress;
context.dstFileName = dstFileName;
context.srcFileName = srcFileName;
memset(&projection, 0, sizeof(projection));
projection.opaque = &context;
projection.dstFileName = dstFileName;
projection.srcFileName = srcFileName;
projection.destinationAlreadyOpen = AIO_WritePool_getFile(ress->writeCtx) != NULL;
projection.testMode = prefs->testMode;
projection.sourceIsStdin = !strcmp(srcFileName, stdinmark);
projection.destinationIsStdout = !strcmp(dstFileName, stdoutmark);
projection.removeSource = prefs->removeSrcFile;
projection.openSource = FIO_rust_decompressSourceOpen;
projection.setAsync = FIO_rust_decompressSourceSetAsync;
projection.attachSource = FIO_rust_decompressSourceAttach;
projection.detachSource = FIO_rust_decompressSourceDetach;
projection.closeSource = FIO_rust_decompressSourceClose;
projection.openDestination = FIO_rust_decompressDestinationOpen;
projection.attachDestination = FIO_rust_decompressDestinationAttach;
projection.addHandler = FIO_rust_decompressAddHandler;
projection.decompress = FIO_rust_decompressFile;
projection.clearHandler = FIO_rust_decompressClearHandler;
projection.setFDStat = FIO_rust_decompressSetFDStat;
projection.closeDestination = FIO_rust_decompressDestinationClose;
projection.utimeDestination = FIO_rust_decompressDestinationUtime;
projection.removeDestination = FIO_rust_decompressDestinationRemove;
projection.removeSourceFile = FIO_rust_decompressSourceRemove;
return FIO_rust_decompressFilename(&projection);
}
typedef struct {
@@ -4030,15 +4192,16 @@ typedef struct {
dRess_t* ress;
} FIO_rust_decompress_multiple_context_t;
/* Keep source opening, format dispatch, diagnostics, and --rm in C. */
/* Keep private source/destination operations, format dispatch, and
* diagnostics in C; Rust owns per-file ordering and removal policy. */
static int FIO_rust_decompressMultipleFileCallback(void* opaque,
const char* outFileName,
const char* srcFileName)
{
FIO_rust_decompress_multiple_context_t* const context =
(FIO_rust_decompress_multiple_context_t*)opaque;
return FIO_decompressSrcFile(
context->fCtx, context->prefs, *context->ress, outFileName, srcFileName);
return FIO_decompressFile(
context->fCtx, context->prefs, context->ress, outFileName, srcFileName);
}
typedef struct {
@@ -4051,7 +4214,8 @@ typedef struct {
static const char* FIO_determineDstName(const char* srcFileName, const char* outDirName);
/* Keep destination-name construction, diagnostics, and decompression in C. */
/* Keep destination-name construction, diagnostics, and private operations in
* C; Rust owns per-file ordering and removal policy. */
static int FIO_rust_decompressMultipleSeparateFileCallback(void* opaque,
const char* srcFileName)
{
@@ -4074,8 +4238,8 @@ static int FIO_rust_decompressMultipleSeparateFileCallback(void* opaque,
}
if (dstFileName == NULL) return 1;
return FIO_decompressSrcFile(
context->fCtx, context->prefs, *context->ress, dstFileName, srcFileName);
return FIO_decompressFile(
context->fCtx, context->prefs, context->ress, dstFileName, srcFileName);
}
@@ -4084,11 +4248,9 @@ int FIO_decompressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs,
const char* dstFileName, const char* srcFileName,
const char* dictFileName)
{
dRess_t const ress = FIO_createDResources(prefs, dictFileName);
int const decodingError = FIO_decompressSrcFile(fCtx, prefs, ress, dstFileName, srcFileName);
dRess_t ress = FIO_createDResources(prefs, dictFileName);
int const decodingError = FIO_decompressFile(
fCtx, prefs, &ress, dstFileName, srcFileName);
FIO_freeDResources(ress);
return decodingError;