ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(fileio): move compression parameter policy to Rust

Browse Source 
Move the non-threaded compression parameter sequence and adaptive window policy
out of FIO_createCResources.  C continues to own context and resource
allocation, dictionary and pool setup, and the multithread-specific settings;
Rust drives the ordered parameter callbacks and stops on the first error.
The projection mirrors the C ABI and has focused order and short-circuit tests.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/cli/Cargo.toml create_c_resources --lib
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040; make -j1
This commit is contained in:
ddidderr
2026-07-20 04:49:35 +02:00
parent 907c3af89e
commit bbe3a5dd74
2 changed files with 325 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fileio.c
+54 -32
View File
@@ -1729,6 +1729,51 @@ static void FIO_adjustParamsForPatchFromMode(FIO_prefs_t* const prefs,
}
}
typedef size_t (*FIO_rust_createCResources_set_parameter_f)(void* context,
int parameter,
int value);
typedef int (*FIO_rust_createCResources_is_error_f)(size_t result);
typedef struct {
void* callbackContext;
const FIO_prefs_t* prefs;
ZSTD_compressionParameters comprParams;
int cLevel;
FIO_rust_createCResources_set_parameter_f setParameter;
FIO_rust_createCResources_is_error_f isError;
} FIO_rust_createCResourcesState;
typedef char FIO_rust_create_c_resources_state_context_offset[
(offsetof(FIO_rust_createCResourcesState, callbackContext) == 0) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_prefs_offset[
(offsetof(FIO_rust_createCResourcesState, prefs) == sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_params_offset[
(offsetof(FIO_rust_createCResourcesState, comprParams) == 2 * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_level_offset[
(offsetof(FIO_rust_createCResourcesState, cLevel)
== 2 * sizeof(void*) + sizeof(ZSTD_compressionParameters)) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_set_parameter_offset[
(offsetof(FIO_rust_createCResourcesState, setParameter)
== ((2 * sizeof(void*) + sizeof(ZSTD_compressionParameters) + sizeof(int)
+ sizeof(void*) - 1) / sizeof(void*)) * sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_is_error_offset[
(offsetof(FIO_rust_createCResourcesState, isError)
== offsetof(FIO_rust_createCResourcesState, setParameter) + sizeof(void*)) ? 1 : -1];
typedef char FIO_rust_create_c_resources_state_size[
(sizeof(FIO_rust_createCResourcesState)
== offsetof(FIO_rust_createCResourcesState, setParameter) + 2 * sizeof(void*)) ? 1 : -1];
size_t FIO_rust_createCResources(const FIO_rust_createCResourcesState* state);
static size_t FIO_rust_createCResources_setParameter(void* context,
int parameter,
int value)
{
return ZSTD_CCtx_setParameter((ZSTD_CCtx*)context, (ZSTD_cParameter)parameter, value);
}
static int FIO_rust_createCResources_isError(size_t result)
{
return ZSTD_isError(result);
}
static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
const char* dictFileName, unsigned long long const maxSrcFileSize,
int cLevel, ZSTD_compressionParameters comprParams) {
@@ -1766,39 +1811,16 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
EXM_THROW(32, "allocation error : can't create dictBuffer");
ress.dictFileName = dictFileName;
if (prefs->adaptiveMode && !prefs->ldmFlag && !comprParams.windowLog)
comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT;
FIO_rust_createCResourcesState const policy = {
ress.cctx,
prefs,
comprParams,
cLevel,
FIO_rust_createCResources_setParameter,
FIO_rust_createCResources_isError
};
CHECK( FIO_rust_createCResources(&policy) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, prefs->contentSize) ); /* always enable content size when available (note: supposed to be default) */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, prefs->dictIDFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, prefs->checksumFlag) );
/* compression level */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) );
/* max compressed block size */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetCBlockSize, (int)prefs->targetCBlockSize) );
/* source size hint */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_srcSizeHint, (int)prefs->srcSizeHint) );
/* long distance matching */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, prefs->ldmFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, prefs->ldmHashLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, prefs->ldmMinMatch) );
if (prefs->ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, prefs->ldmBucketSizeLog) );
}
if (prefs->ldmHashRateLog != FIO_LDM_PARAM_NOTSET) {
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, prefs->ldmHashRateLog) );
}
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_useRowMatchFinder, prefs->useRowMatchFinder));
/* compression parameters */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, (int)comprParams.windowLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, (int)comprParams.chainLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_hashLog, (int)comprParams.hashLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_searchLog, (int)comprParams.searchLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_minMatch, (int)comprParams.minMatch) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_targetLength, (int)comprParams.targetLength) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, (int)comprParams.strategy) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_literalCompressionMode, (int)prefs->literalCompressionMode) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableDedicatedDictSearch, 1) );
/* multi-threading */
#ifdef ZSTD_MULTITHREAD
DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers);