feat(fileio): move source compression policy into Rust
Move the ordered policy from FIO_compressFilename_srcFile into a narrow Rust projection: named-source stat and collision checks, compressed-file exclusion, source open and close, size-based async selection, source attachment, and successful-only --rm handling now run in Rust. The destination callback still hands compression to the existing Rust compression loop. Keep FIO_ctx_t, cRess_t, stat_t, FILE/pool handles, signal handling, and CLI diagnostics behind opaque C callbacks, so the ABI carries only filenames, scalar policy flags, and callback pointers. Focused Rust policy tests cover named-file short circuits, stdin behavior, async threshold selection, and the successful-compression removal gate. Test Plan: - `rustfmt --edition 2021 --check rust/src/fileio_asyncio.rs` — passed - `git diff --check` and path-limited `git diff --cached --check` — passed - Static ABI/order/diff inspection only; policy tests were not executed - cargo/make/native/fuzzer/heavy tests were not run per request
This commit is contained in:
+193
-62
@@ -1460,6 +1460,76 @@ int FIO_rust_compressFilenameInternal(
|
||||
U64 fileSize, int compressionLevel,
|
||||
const FIO_rust_compress_callbacks_t* callbacks);
|
||||
|
||||
enum {
|
||||
FIO_RUST_COMPRESS_SRC_STAT_FAILED = 0,
|
||||
FIO_RUST_COMPRESS_SRC_STAT_OK = 1,
|
||||
FIO_RUST_COMPRESS_SRC_DIRECTORY = 2,
|
||||
FIO_RUST_COMPRESS_SRC_DICT_COLLISION = 3,
|
||||
FIO_RUST_COMPRESS_SRC_OPEN_OK = 0,
|
||||
};
|
||||
typedef int (*FIO_rust_compress_src_stat_fn)(void* opaque, const char* srcFileName);
|
||||
typedef int (*FIO_rust_compress_src_excluded_fn)(void* opaque, const char* srcFileName);
|
||||
typedef int (*FIO_rust_compress_src_open_fn)(void* opaque, const char* srcFileName,
|
||||
U64* fileSize);
|
||||
typedef void (*FIO_rust_compress_src_async_fn)(void* opaque, int asyncMode);
|
||||
typedef void (*FIO_rust_compress_src_attach_fn)(void* opaque);
|
||||
typedef int (*FIO_rust_compress_src_compress_fn)(void* opaque,
|
||||
const char* dstFileName,
|
||||
const char* srcFileName,
|
||||
int compressionLevel);
|
||||
typedef void (*FIO_rust_compress_src_close_fn)(void* opaque);
|
||||
typedef void (*FIO_rust_compress_src_remove_fn)(void* opaque, const char* srcFileName);
|
||||
|
||||
/* Rust owns the source policy; this projection exposes only scalar names and
|
||||
* opaque C callbacks. FIO_ctx_t, cRess_t, stat_t, and all resource/diagnostic
|
||||
* state remain private to the callback context below. */
|
||||
typedef struct {
|
||||
void* opaque;
|
||||
const char* dstFileName;
|
||||
const char* srcFileName;
|
||||
int compressionLevel;
|
||||
int sourceIsStdin;
|
||||
int excludeCompressedFiles;
|
||||
int removeSrcFile;
|
||||
FIO_rust_compress_src_stat_fn statSource;
|
||||
FIO_rust_compress_src_excluded_fn sourceIsExcluded;
|
||||
FIO_rust_compress_src_open_fn openSource;
|
||||
FIO_rust_compress_src_async_fn setAsync;
|
||||
FIO_rust_compress_src_attach_fn attachSource;
|
||||
FIO_rust_compress_src_compress_fn compress;
|
||||
FIO_rust_compress_src_close_fn closeSource;
|
||||
FIO_rust_compress_src_remove_fn removeSource;
|
||||
} FIO_rust_compress_src_projection_t;
|
||||
typedef char FIO_rust_compress_src_opaque_offset[
|
||||
(offsetof(FIO_rust_compress_src_projection_t, opaque) == 0) ? 1 : -1];
|
||||
typedef char FIO_rust_compress_src_dst_name_offset[
|
||||
(offsetof(FIO_rust_compress_src_projection_t, dstFileName)
|
||||
== sizeof(void*)) ? 1 : -1];
|
||||
typedef char FIO_rust_compress_src_src_name_offset[
|
||||
(offsetof(FIO_rust_compress_src_projection_t, srcFileName)
|
||||
== 2 * sizeof(void*)) ? 1 : -1];
|
||||
typedef char FIO_rust_compress_src_level_offset[
|
||||
(offsetof(FIO_rust_compress_src_projection_t, compressionLevel)
|
||||
== 3 * sizeof(void*)) ? 1 : -1];
|
||||
typedef char FIO_rust_compress_src_stdin_offset[
|
||||
(offsetof(FIO_rust_compress_src_projection_t, sourceIsStdin)
|
||||
== 3 * sizeof(void*) + sizeof(int)) ? 1 : -1];
|
||||
typedef char FIO_rust_compress_src_exclude_offset[
|
||||
(offsetof(FIO_rust_compress_src_projection_t, excludeCompressedFiles)
|
||||
== 3 * sizeof(void*) + 2 * sizeof(int)) ? 1 : -1];
|
||||
typedef char FIO_rust_compress_src_remove_offset[
|
||||
(offsetof(FIO_rust_compress_src_projection_t, removeSrcFile)
|
||||
== 3 * sizeof(void*) + 3 * sizeof(int)) ? 1 : -1];
|
||||
typedef char FIO_rust_compress_src_stat_callback_offset[
|
||||
(offsetof(FIO_rust_compress_src_projection_t, statSource)
|
||||
== 3 * sizeof(void*) + 4 * sizeof(int)) ? 1 : -1];
|
||||
typedef char FIO_rust_compress_src_projection_size[
|
||||
(sizeof(FIO_rust_compress_src_projection_t)
|
||||
== 3 * sizeof(void*) + 4 * sizeof(int)
|
||||
+ 8 * sizeof(FIO_rust_compress_src_stat_fn)) ? 1 : -1];
|
||||
int FIO_rust_compressFilenameSrcFile(
|
||||
const FIO_rust_compress_src_projection_t* projection);
|
||||
|
||||
static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs,
|
||||
ZSTD_compressionParameters* comprParams,
|
||||
unsigned long long const dictSize,
|
||||
@@ -2891,6 +2961,105 @@ static const char *compressedFileExtensions[] = {
|
||||
NULL
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
FIO_ctx_t* fCtx;
|
||||
FIO_prefs_t* prefs;
|
||||
cRess_t* ress;
|
||||
stat_t srcFileStat;
|
||||
FILE* srcFile;
|
||||
} FIO_rust_compress_src_context_t;
|
||||
|
||||
static int FIO_rust_compressSourceStat(void* opaque, const char* srcFileName)
|
||||
{
|
||||
FIO_rust_compress_src_context_t* const context =
|
||||
(FIO_rust_compress_src_context_t*)opaque;
|
||||
|
||||
if (!UTIL_stat(srcFileName, &context->srcFileStat)) {
|
||||
/* Failure to stat at all is handled during opening. */
|
||||
return FIO_RUST_COMPRESS_SRC_STAT_FAILED;
|
||||
}
|
||||
|
||||
if (UTIL_isDirectoryStat(&context->srcFileStat)) {
|
||||
DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
|
||||
return FIO_RUST_COMPRESS_SRC_DIRECTORY;
|
||||
}
|
||||
|
||||
if (context->ress->dictFileName != NULL
|
||||
&& UTIL_isSameFileStat(srcFileName, context->ress->dictFileName,
|
||||
&context->srcFileStat, &context->ress->dictFileStat)) {
|
||||
DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName);
|
||||
return FIO_RUST_COMPRESS_SRC_DICT_COLLISION;
|
||||
}
|
||||
|
||||
return FIO_RUST_COMPRESS_SRC_STAT_OK;
|
||||
}
|
||||
|
||||
static int FIO_rust_compressSourceIsExcluded(void* opaque, const char* srcFileName)
|
||||
{
|
||||
int const isCompressed = UTIL_isCompressedFile(srcFileName, compressedFileExtensions);
|
||||
(void)opaque;
|
||||
if (isCompressed)
|
||||
DISPLAYLEVEL(4, "File is already compressed : %s \n", srcFileName);
|
||||
return isCompressed;
|
||||
}
|
||||
|
||||
static int FIO_rust_compressSourceOpen(void* opaque, const char* srcFileName,
|
||||
U64* fileSize)
|
||||
{
|
||||
FIO_rust_compress_src_context_t* const context =
|
||||
(FIO_rust_compress_src_context_t*)opaque;
|
||||
context->srcFile = FIO_openSrcFile(context->prefs, srcFileName, &context->srcFileStat);
|
||||
if (context->srcFile == NULL)
|
||||
return 1;
|
||||
|
||||
*fileSize = strcmp(srcFileName, stdinmark)
|
||||
? UTIL_getFileSizeStat(&context->srcFileStat)
|
||||
: UTIL_FILESIZE_UNKNOWN;
|
||||
return FIO_RUST_COMPRESS_SRC_OPEN_OK;
|
||||
}
|
||||
|
||||
static void FIO_rust_compressSourceSetAsync(void* opaque, int asyncMode)
|
||||
{
|
||||
FIO_rust_compress_src_context_t* const context =
|
||||
(FIO_rust_compress_src_context_t*)opaque;
|
||||
AIO_ReadPool_setAsync(context->ress->readCtx, asyncMode);
|
||||
AIO_WritePool_setAsync(context->ress->writeCtx, asyncMode);
|
||||
}
|
||||
|
||||
static void FIO_rust_compressSourceAttach(void* opaque)
|
||||
{
|
||||
FIO_rust_compress_src_context_t* const context =
|
||||
(FIO_rust_compress_src_context_t*)opaque;
|
||||
AIO_ReadPool_setFile(context->ress->readCtx, context->srcFile);
|
||||
}
|
||||
|
||||
static int FIO_rust_compressSourceCompress(void* opaque, const char* dstFileName,
|
||||
const char* srcFileName, int compressionLevel)
|
||||
{
|
||||
FIO_rust_compress_src_context_t* const context =
|
||||
(FIO_rust_compress_src_context_t*)opaque;
|
||||
return FIO_compressFilename_dstFile(
|
||||
context->fCtx, context->prefs, *context->ress,
|
||||
dstFileName, srcFileName, &context->srcFileStat, compressionLevel);
|
||||
}
|
||||
|
||||
static void FIO_rust_compressSourceClose(void* opaque)
|
||||
{
|
||||
FIO_rust_compress_src_context_t* const context =
|
||||
(FIO_rust_compress_src_context_t*)opaque;
|
||||
(void)AIO_ReadPool_closeFile(context->ress->readCtx);
|
||||
context->srcFile = NULL;
|
||||
}
|
||||
|
||||
static void FIO_rust_compressSourceRemove(void* opaque, const char* srcFileName)
|
||||
{
|
||||
(void)opaque;
|
||||
/* After this point Ctrl-C must not remove both source and destination. */
|
||||
clearHandler();
|
||||
if (FIO_removeFile(srcFileName))
|
||||
EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno));
|
||||
}
|
||||
|
||||
/*! FIO_compressFilename_srcFile() :
|
||||
* @return : 0 : compression completed correctly,
|
||||
* 1 : missing or pb opening srcFileName
|
||||
@@ -2903,72 +3072,34 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx,
|
||||
const char* srcFileName,
|
||||
int compressionLevel)
|
||||
{
|
||||
int result;
|
||||
FILE* srcFile;
|
||||
stat_t srcFileStat;
|
||||
U64 fileSize = UTIL_FILESIZE_UNKNOWN;
|
||||
FIO_rust_compress_src_context_t context;
|
||||
FIO_rust_compress_src_projection_t projection;
|
||||
|
||||
DISPLAYLEVEL(6, "FIO_compressFilename_srcFile: %s \n", srcFileName);
|
||||
|
||||
if (strcmp(srcFileName, stdinmark)) {
|
||||
if (UTIL_stat(srcFileName, &srcFileStat)) {
|
||||
/* failure to stat at all is handled during opening */
|
||||
memset(&context, 0, sizeof(context));
|
||||
context.fCtx = fCtx;
|
||||
context.prefs = prefs;
|
||||
context.ress = &ress;
|
||||
|
||||
/* ensure src is not a directory */
|
||||
if (UTIL_isDirectoryStat(&srcFileStat)) {
|
||||
DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
|
||||
return 1;
|
||||
}
|
||||
memset(&projection, 0, sizeof(projection));
|
||||
projection.opaque = &context;
|
||||
projection.dstFileName = dstFileName;
|
||||
projection.srcFileName = srcFileName;
|
||||
projection.compressionLevel = compressionLevel;
|
||||
projection.sourceIsStdin = !strcmp(srcFileName, stdinmark);
|
||||
projection.excludeCompressedFiles = prefs->excludeCompressedFiles == 1;
|
||||
projection.removeSrcFile = prefs->removeSrcFile;
|
||||
projection.statSource = FIO_rust_compressSourceStat;
|
||||
projection.sourceIsExcluded = FIO_rust_compressSourceIsExcluded;
|
||||
projection.openSource = FIO_rust_compressSourceOpen;
|
||||
projection.setAsync = FIO_rust_compressSourceSetAsync;
|
||||
projection.attachSource = FIO_rust_compressSourceAttach;
|
||||
projection.compress = FIO_rust_compressSourceCompress;
|
||||
projection.closeSource = FIO_rust_compressSourceClose;
|
||||
projection.removeSource = FIO_rust_compressSourceRemove;
|
||||
|
||||
/* ensure src is not the same as dict (if present) */
|
||||
if (ress.dictFileName != NULL && UTIL_isSameFileStat(srcFileName, ress.dictFileName, &srcFileStat, &ress.dictFileStat)) {
|
||||
DISPLAYLEVEL(1, "zstd: cannot use %s as an input file and dictionary \n", srcFileName);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Check if "srcFile" is compressed. Only done if --exclude-compressed flag is used
|
||||
* YES => ZSTD will skip compression of the file and will return 0.
|
||||
* NO => ZSTD will resume with compress operation.
|
||||
*/
|
||||
if (prefs->excludeCompressedFiles == 1 && UTIL_isCompressedFile(srcFileName, compressedFileExtensions)) {
|
||||
DISPLAYLEVEL(4, "File is already compressed : %s \n", srcFileName);
|
||||
return 0;
|
||||
}
|
||||
|
||||
srcFile = FIO_openSrcFile(prefs, srcFileName, &srcFileStat);
|
||||
if (srcFile == NULL) return 1; /* srcFile could not be opened */
|
||||
|
||||
/* Don't use AsyncIO for small files */
|
||||
if (strcmp(srcFileName, stdinmark)) /* Stdin doesn't have stats */
|
||||
fileSize = UTIL_getFileSizeStat(&srcFileStat);
|
||||
if(fileSize != UTIL_FILESIZE_UNKNOWN && fileSize < ZSTD_BLOCKSIZE_MAX * 3) {
|
||||
AIO_ReadPool_setAsync(ress.readCtx, 0);
|
||||
AIO_WritePool_setAsync(ress.writeCtx, 0);
|
||||
} else {
|
||||
AIO_ReadPool_setAsync(ress.readCtx, 1);
|
||||
AIO_WritePool_setAsync(ress.writeCtx, 1);
|
||||
}
|
||||
|
||||
AIO_ReadPool_setFile(ress.readCtx, srcFile);
|
||||
result = FIO_compressFilename_dstFile(
|
||||
fCtx, prefs, ress,
|
||||
dstFileName, srcFileName,
|
||||
&srcFileStat, compressionLevel);
|
||||
AIO_ReadPool_closeFile(ress.readCtx);
|
||||
|
||||
if ( prefs->removeSrcFile /* --rm */
|
||||
&& result == 0 /* success */
|
||||
&& strcmp(srcFileName, stdinmark) /* exception : don't erase stdin */
|
||||
) {
|
||||
/* We must clear the handler, since after this point calling it would
|
||||
* delete both the source and destination files.
|
||||
*/
|
||||
clearHandler();
|
||||
if (FIO_removeFile(srcFileName))
|
||||
EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno));
|
||||
}
|
||||
return result;
|
||||
return FIO_rust_compressFilenameSrcFile(&projection);
|
||||
}
|
||||
|
||||
void FIO_displayCompressionParameters(const FIO_prefs_t* prefs)
|
||||
|
||||
Reference in New Issue
Block a user