ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(cli): move separate-output scheduling into Rust

Browse Source 
Move the separate-destination branch of FIO_compressMultipleFilenames into a Rust-owned scheduler. Preserve source ordering, currFileIdx advancement, successful-file counting, aggregate error handling, and post-loop collision checks while retaining destination construction and private file/resource work in C callbacks.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml compression_multiple_separate -- --test-threads=1
- ulimit -v 41943040; make -B -C programs -j1 zstd
- ulimit -v 41943040; make -B -C tests -j1 test-cli-tests
This commit is contained in:
ddidderr
2026-07-19 10:52:31 +02:00
parent bd7fbc81fc
commit fbc80de7dc
2 changed files with 223 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fileio.c
+73 -21
View File
@@ -1154,6 +1154,31 @@ typedef char FIO_rust_compress_multiple_projection_size[
int FIO_rust_compressMultipleFilenames(
const FIO_rust_compress_multiple_projection_t* projection);
typedef int (*FIO_rust_compress_multiple_separate_file_fn)(
void* opaque, const char* srcFileName);
typedef struct {
void* fCtx;
const char** inFileNamesTable;
void* opaque;
FIO_rust_compress_multiple_separate_file_fn compressFile;
} FIO_rust_compress_multiple_separate_projection_t;
typedef char FIO_rust_compress_multiple_separate_fctx_offset[
(offsetof(FIO_rust_compress_multiple_separate_projection_t, fCtx) == 0) ? 1 : -1];
typedef char FIO_rust_compress_multiple_separate_input_names_offset[
(offsetof(FIO_rust_compress_multiple_separate_projection_t, inFileNamesTable)
== sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_separate_opaque_offset[
(offsetof(FIO_rust_compress_multiple_separate_projection_t, opaque)
== 2 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_separate_callback_offset[
(offsetof(FIO_rust_compress_multiple_separate_projection_t, compressFile)
== 3 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_compress_multiple_separate_projection_size[
(sizeof(FIO_rust_compress_multiple_separate_projection_t)
== 3 * sizeof(void*) + sizeof(FIO_rust_compress_multiple_separate_file_fn)) ? 1 : -1];
int FIO_rust_compressMultipleSeparateFilenames(
const FIO_rust_compress_multiple_separate_projection_t* projection);
enum {
FIO_RUST_ZSTD_OK = 0,
FIO_RUST_ZSTD_COMPRESS_ERROR = 1,
@@ -2944,6 +2969,45 @@ static int FIO_rust_compressMultipleFileCallback(void* opaque,
dstFileName, srcFileName, context->compressionLevel);
}
typedef struct {
FIO_ctx_t* fCtx;
FIO_prefs_t* prefs;
cRess_t* ress;
const char* outMirroredRootDirName;
const char* outDirName;
const char* suffix;
int compressionLevel;
} FIO_rust_compress_multiple_separate_context_t;
static int FIO_rust_compressMultipleSeparateFileCallback(void* opaque,
const char* srcFileName)
{
FIO_rust_compress_multiple_separate_context_t* const context =
(FIO_rust_compress_multiple_separate_context_t*)opaque;
const char* dstFileName;
if (context->outMirroredRootDirName) {
char* const validMirroredDirName = UTIL_createMirroredDestDirName(
srcFileName, context->outMirroredRootDirName);
if (validMirroredDirName) {
dstFileName = FIO_determineCompressedName(
srcFileName, validMirroredDirName, context->suffix);
free(validMirroredDirName);
} else {
DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot compress '%s' into '%s' \n",
srcFileName, context->outMirroredRootDirName);
return 1;
}
} else {
dstFileName = FIO_determineCompressedName(
srcFileName, context->outDirName, context->suffix);
}
return FIO_compressFilename_srcFile(
context->fCtx, context->prefs, *context->ress,
dstFileName, srcFileName, context->compressionLevel);
}
/* FIO_compressMultipleFilenames() :
* compress nbFiles files
* into either one destination (outFileName),
@@ -2959,7 +3023,6 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams)
{
int status;
int error = 0;
cRess_t ress = FIO_createCResources(prefs, dictFileName,
FIO_getLargestFileSize(inFileNamesTable, (unsigned)fCtx->nbFilesTotal),
@@ -2991,29 +3054,18 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
strerror(errno), outFileName);
}
} else {
FIO_rust_compress_multiple_separate_context_t callbackContext = {
fCtx, prefs, &ress, outMirroredRootDirName,
outDirName, suffix, compressionLevel
};
FIO_rust_compress_multiple_separate_projection_t projection = {
fCtx, inFileNamesTable, &callbackContext,
FIO_rust_compressMultipleSeparateFileCallback
};
if (outMirroredRootDirName)
UTIL_mirrorSourceFilesDirectories(inFileNamesTable, (unsigned)fCtx->nbFilesTotal, outMirroredRootDirName);
for (; fCtx->currFileIdx < fCtx->nbFilesTotal; ++fCtx->currFileIdx) {
const char* const srcFileName = inFileNamesTable[fCtx->currFileIdx];
const char* dstFileName = NULL;
if (outMirroredRootDirName) {
char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName);
if (validMirroredDirName) {
dstFileName = FIO_determineCompressedName(srcFileName, validMirroredDirName, suffix);
free(validMirroredDirName);
} else {
DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot compress '%s' into '%s' \n", srcFileName, outMirroredRootDirName);
error=1;
continue;
}
} else {
dstFileName = FIO_determineCompressedName(srcFileName, outDirName, suffix); /* cannot fail */
}
status = FIO_compressFilename_srcFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel);
if (!status) fCtx->nbFilesProcessed++;
error |= status;
}
error = FIO_rust_compressMultipleSeparateFilenames(&projection);
if (outDirName)
FIO_checkFilenameCollisions(inFileNamesTable , (unsigned)fCtx->nbFilesTotal);