ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(cli): move separate decompression scheduling into Rust

Browse Source 
Move the separate-destination multi-file decompression loop into the Rust
projection so Rust owns file iteration, progress counters, and aggregate
error handling. Keep destination-name construction, mirror setup, source
opening, format dispatch, diagnostics, and source removal in the C callback
boundary.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml decompression_multiple_separate -- --test-threads=1
- ulimit -v 41943040; make -B -C programs -j1 zstd
- ulimit -v 41943040; make -B -C tests -j1 test-cli-tests
This commit is contained in:
ddidderr
2026-07-19 11:25:59 +02:00
parent bc8c005c23
commit 845f77924a
2 changed files with 221 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fileio.c
+72 -20
View File
@@ -1208,6 +1208,32 @@ typedef char FIO_rust_decompress_multiple_projection_size[
int FIO_rust_decompressMultipleFilenames(
const FIO_rust_decompress_multiple_projection_t* projection);
typedef int (*FIO_rust_decompress_multiple_separate_file_fn)(
void* opaque, const char* srcFileName);
typedef struct {
void* fCtx;
const char** srcNamesTable;
void* opaque;
FIO_rust_decompress_multiple_separate_file_fn decompressFile;
} FIO_rust_decompress_multiple_separate_projection_t;
typedef char FIO_rust_decompress_multiple_separate_fctx_offset[
(offsetof(FIO_rust_decompress_multiple_separate_projection_t, fCtx) == 0) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_separate_input_names_offset[
(offsetof(FIO_rust_decompress_multiple_separate_projection_t, srcNamesTable)
== sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_separate_opaque_offset[
(offsetof(FIO_rust_decompress_multiple_separate_projection_t, opaque)
== 2 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_separate_callback_offset[
(offsetof(FIO_rust_decompress_multiple_separate_projection_t, decompressFile)
== 3 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_decompress_multiple_separate_projection_size[
(sizeof(FIO_rust_decompress_multiple_separate_projection_t)
== 3 * sizeof(void*)
+ sizeof(FIO_rust_decompress_multiple_separate_file_fn)) ? 1 : -1];
int FIO_rust_decompressMultipleSeparateFilenames(
const FIO_rust_decompress_multiple_separate_projection_t* projection);
enum {
FIO_RUST_ZSTD_OK = 0,
FIO_RUST_ZSTD_COMPRESS_ERROR = 1,
@@ -3884,6 +3910,43 @@ static int FIO_rust_decompressMultipleFileCallback(void* opaque,
context->fCtx, context->prefs, *context->ress, outFileName, srcFileName);
}
typedef struct {
FIO_ctx_t* fCtx;
FIO_prefs_t* prefs;
dRess_t* ress;
const char* outMirroredRootDirName;
const char* outDirName;
} FIO_rust_decompress_multiple_separate_context_t;
static const char* FIO_determineDstName(const char* srcFileName, const char* outDirName);
/* Keep destination-name construction, diagnostics, and decompression in C. */
static int FIO_rust_decompressMultipleSeparateFileCallback(void* opaque,
const char* srcFileName)
{
FIO_rust_decompress_multiple_separate_context_t* const context =
(FIO_rust_decompress_multiple_separate_context_t*)opaque;
const char* dstFileName = NULL;
if (context->outMirroredRootDirName) {
char* validMirroredDirName = UTIL_createMirroredDestDirName(
srcFileName, context->outMirroredRootDirName);
if (validMirroredDirName) {
dstFileName = FIO_determineDstName(srcFileName, validMirroredDirName);
free(validMirroredDirName);
} else {
DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot decompress '%s' into '%s'\n",
srcFileName, context->outMirroredRootDirName);
}
} else {
dstFileName = FIO_determineDstName(srcFileName, context->outDirName);
}
if (dstFileName == NULL) return 1;
return FIO_decompressSrcFile(
context->fCtx, context->prefs, *context->ress, dstFileName, srcFileName);
}
int FIO_decompressFilename(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs,
@@ -3953,7 +4016,6 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
const char* outDirName, const char* outFileName,
const char* dictFileName)
{
int status;
int error = 0;
dRess_t ress = FIO_createDResources(prefs, dictFileName);
@@ -3981,28 +4043,18 @@ FIO_decompressMultipleFilenames(FIO_ctx_t* const fCtx,
EXM_THROW(72, "Write error : %s : cannot properly close output file",
strerror(errno));
} else {
FIO_rust_decompress_multiple_separate_context_t callbackContext = {
fCtx, prefs, &ress, outMirroredRootDirName, outDirName
};
FIO_rust_decompress_multiple_separate_projection_t projection = {
fCtx, srcNamesTable, &callbackContext,
FIO_rust_decompressMultipleSeparateFileCallback
};
if (outMirroredRootDirName)
UTIL_mirrorSourceFilesDirectories(srcNamesTable, (unsigned)fCtx->nbFilesTotal, outMirroredRootDirName);
for (; fCtx->currFileIdx < fCtx->nbFilesTotal; fCtx->currFileIdx++) { /* create dstFileName */
const char* const srcFileName = srcNamesTable[fCtx->currFileIdx];
const char* dstFileName = NULL;
if (outMirroredRootDirName) {
char* validMirroredDirName = UTIL_createMirroredDestDirName(srcFileName, outMirroredRootDirName);
if (validMirroredDirName) {
dstFileName = FIO_determineDstName(srcFileName, validMirroredDirName);
free(validMirroredDirName);
} else {
DISPLAYLEVEL(2, "zstd: --output-dir-mirror cannot decompress '%s' into '%s'\n", srcFileName, outMirroredRootDirName);
}
} else {
dstFileName = FIO_determineDstName(srcFileName, outDirName);
}
if (dstFileName == NULL) { error=1; continue; }
status = FIO_decompressSrcFile(fCtx, prefs, ress, dstFileName, srcFileName);
if (!status) fCtx->nbFilesProcessed++;
error |= status;
}
error = FIO_rust_decompressMultipleSeparateFilenames(&projection);
if (outDirName)
FIO_checkFilenameCollisions(srcNamesTable , (unsigned)fCtx->nbFilesTotal);
}