ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(cli): move MT resource parameter policy to Rust

Browse Source 
File-resource creation already delegates the general compression-parameter
policy to Rust, but the multithreaded parameters were still applied in C in a
separate branch. That left parameter ordering, optional overlap handling, and
error short-circuiting outside the Rust policy boundary. Add a narrow ABI
projection with C callbacks for CCtx mutation and diagnostics, and let Rust
apply worker count, job size, optional overlap, and rsyncable in the original
order. The C90 declaration layout and compile-time ABI assertions keep the
existing native program configurations intact.

Test Plan:
- `cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings` -- passed
- `cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings` -- passed
- `cargo test --manifest-path rust/Cargo.toml --all-targets` -- 771 passed
- `cargo test --manifest-path rust/cli/Cargo.toml --all-targets` -- 179 passed
- `cargo +nightly fmt --manifest-path rust/Cargo.toml -- --check` -- passed
- `make -j1` under `ulimit -v 41943040` -- passed without the new C90 warning
- `make -j1 -C tests test` under `ulimit -v 41943040` -- passed
This commit is contained in:
ddidderr
2026-07-20 06:03:41 +02:00
parent 998c88c97b
commit a67e6a59df
2 changed files with 151 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fileio.c
+22 -16
View File
@@ -1733,6 +1733,7 @@ typedef size_t (*FIO_rust_createCResources_set_parameter_f)(void* context,
int parameter,
int value);
typedef int (*FIO_rust_createCResources_is_error_f)(size_t result);
typedef void (*FIO_rust_createCResources_display_overlap_f)(int overlapLog);
typedef struct {
void* callbackContext;
const FIO_prefs_t* prefs;
@@ -1740,6 +1741,7 @@ typedef struct {
int cLevel;
FIO_rust_createCResources_set_parameter_f setParameter;
FIO_rust_createCResources_is_error_f isError;
FIO_rust_createCResources_display_overlap_f displayOverlap;
} FIO_rust_createCResourcesState;
typedef char FIO_rust_create_c_resources_state_context_offset[
(offsetof(FIO_rust_createCResourcesState, callbackContext) == 0) ? 1 : -1];
@@ -1757,9 +1759,12 @@ typedef char FIO_rust_create_c_resources_state_set_parameter_offset[
typedef char FIO_rust_create_c_resources_state_is_error_offset[
(offsetof(FIO_rust_createCResourcesState, isError)
== offsetof(FIO_rust_createCResourcesState, setParameter) + sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_display_overlap_offset[
(offsetof(FIO_rust_createCResourcesState, displayOverlap)
== offsetof(FIO_rust_createCResourcesState, isError) + sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_size[
(sizeof(FIO_rust_createCResourcesState)
== offsetof(FIO_rust_createCResourcesState, setParameter) + 2 * sizeof(void*)) ? 1 : -1];
== offsetof(FIO_rust_createCResourcesState, setParameter) + 3 * sizeof(void*)) ? 1 : -1];
size_t FIO_rust_createCResources(const FIO_rust_createCResourcesState* state);
static size_t FIO_rust_createCResources_setParameter(void* context,
@@ -1774,6 +1779,13 @@ static int FIO_rust_createCResources_isError(size_t result)
return ZSTD_isError(result);
}
static void FIO_rust_createCResources_displayOverlap(int overlapLog)
{
DISPLAYLEVEL(3,"set overlapLog = %u \n", overlapLog);
}
size_t FIO_rust_setCResourcesMtParameters(const FIO_rust_createCResourcesState* state);
static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
const char* dictFileName, unsigned long long const maxSrcFileSize,
int cLevel, ZSTD_compressionParameters comprParams) {
@@ -1781,6 +1793,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
int forceNoUseMMap = prefs->mmapDict == ZSTD_ps_disable;
FIO_dictBufferType_t dictBufferType;
cRess_t ress;
FIO_rust_createCResourcesState policy;
memset(&ress, 0, sizeof(ress));
DISPLAYLEVEL(6, "FIO_createCResources \n");
@@ -1811,26 +1824,19 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
EXM_THROW(32, "allocation error : can't create dictBuffer");
ress.dictFileName = dictFileName;
FIO_rust_createCResourcesState const policy = {
ress.cctx,
prefs,
comprParams,
cLevel,
FIO_rust_createCResources_setParameter,
FIO_rust_createCResources_isError
};
policy.callbackContext = ress.cctx;
policy.prefs = prefs;
policy.comprParams = comprParams;
policy.cLevel = cLevel;
policy.setParameter = FIO_rust_createCResources_setParameter;
policy.isError = FIO_rust_createCResources_isError;
policy.displayOverlap = FIO_rust_createCResources_displayOverlap;
CHECK( FIO_rust_createCResources(&policy) );
/* multi-threading */
#ifdef ZSTD_MULTITHREAD
DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->blockSize) );
if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) {
DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) );
}
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, prefs->rsyncable) );
CHECK( FIO_rust_setCResourcesMtParameters(&policy) );
#endif
/* dictionary */
if (prefs->patchFromMode) {