ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(cli): move dictionary buffer selection policy to Rust

Browse Source 
Move the malloc-versus-mmap decision used by compression and decompression
resource construction into Rust.  The C adapter continues to own platform
loaders, allocation handles, diagnostics, and dictionary I/O; Rust only
combines the explicit mmap preference, patch-mode size threshold, and explicit
disable override.  Add enum-value ABI checks and focused policy coverage.

Also hoist the source-size declaration in the C adapter so the migration does
not introduce a C90 declaration-after-statement warning.

Test Plan:
- `ulimit -v 41943040; cargo +nightly fmt --manifest-path rust/Cargo.toml --all -- --check`
- `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings`
- `ulimit -v 41943040; make -j1`
- `ulimit -v 41943040; make -j1 -C tests test`
- `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings`
- `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/cli/Cargo.toml --all-targets`

The integrated checks ran at the combined working-tree tip under a serial
40 GiB virtual-memory limit. Standalone root Rust unit linking remains
unavailable because the crate imports C-owned bridge symbols without a Cargo
build/link setup.
This commit is contained in:
ddidderr
2026-07-20 09:50:12 +02:00
parent 7ea0cbd96e
commit ac4b7afca8
2 changed files with 66 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fileio.c
+18 -11
View File
@@ -351,6 +351,14 @@ int FIO_rust_adjustMemLimitForPatchFromMode(FIO_prefs_t* prefs,
unsigned long long dictSize,
unsigned long long maxSrcFileSize);
int FIO_rust_getDictFileStat(const char* fileName, stat_t* statBuf);
typedef char FIO_rust_dict_buffer_type_values[
(FIO_mallocDict == 0 && FIO_mmapDict == 1) ? 1 : -1];
typedef char FIO_rust_mmap_policy_values[
(ZSTD_ps_auto == 0 && ZSTD_ps_enable == 1 && ZSTD_ps_disable == 2) ? 1 : -1];
int FIO_rust_selectDictBufferType(int mmapDict,
int patchFromMode,
unsigned long long dictSize,
unsigned long long memLimit);
enum {
FIO_RUST_OPEN_SRC_SUCCESS = 0,
FIO_RUST_OPEN_SRC_STAT_FAILED = 1,
@@ -1833,8 +1841,8 @@ size_t FIO_rust_setCResourcesMtParameters(const FIO_rust_createCResourcesState*
static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
const char* dictFileName, unsigned long long const maxSrcFileSize,
int cLevel, ZSTD_compressionParameters comprParams) {
int useMMap = prefs->mmapDict == ZSTD_ps_enable;
int forceNoUseMMap = prefs->mmapDict == ZSTD_ps_disable;
unsigned long long dictSize = 0;
unsigned long long ssSize = 0;
FIO_dictBufferType_t dictBufferType;
cRess_t ress;
FIO_rust_createCResourcesState policy;
@@ -1851,13 +1859,13 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
/* need to update memLimit before calling createDictBuffer
* because of memLimit check inside it */
if (prefs->patchFromMode) {
U64 const dictSize = UTIL_getFileSizeStat(&ress.dictFileStat);
unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize;
useMMap |= dictSize > prefs->memLimit;
dictSize = (unsigned long long)UTIL_getFileSizeStat(&ress.dictFileStat);
ssSize = (unsigned long long)prefs->streamSrcSize;
FIO_adjustParamsForPatchFromMode(prefs, &comprParams, dictSize, ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
}
dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict;
dictBufferType = (FIO_dictBufferType_t)FIO_rust_selectDictBufferType(
prefs->mmapDict, prefs->patchFromMode, dictSize, prefs->memLimit);
FIO_initDict(&ress.dict, dictFileName, prefs, &ress.dictFileStat, dictBufferType); /* works with dictFileName==NULL */
ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_CStreamOutSize());
@@ -3715,8 +3723,7 @@ static void FIO_rust_freeDResources_readPool(void* context)
static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName)
{
int useMMap = prefs->mmapDict == ZSTD_ps_enable;
int forceNoUseMMap = prefs->mmapDict == ZSTD_ps_disable;
unsigned long long dictSize = 0;
stat_t statbuf;
dRess_t ress;
memset(&statbuf, 0, sizeof(statbuf));
@@ -3725,8 +3732,7 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
FIO_getDictFileStat(dictFileName, &statbuf);
if (prefs->patchFromMode){
U64 const dictSize = UTIL_getFileSizeStat(&statbuf);
useMMap |= dictSize > prefs->memLimit;
dictSize = (unsigned long long)UTIL_getFileSizeStat(&statbuf);
FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, 0 /* just use the dict size */);
}
@@ -3739,7 +3745,8 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
/* dictionary */
{
FIO_dictBufferType_t dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict;
FIO_dictBufferType_t const dictBufferType = (FIO_dictBufferType_t)FIO_rust_selectDictBufferType(
prefs->mmapDict, prefs->patchFromMode, dictSize, prefs->memLimit);
FIO_initDict(&ress.dict, dictFileName, prefs, &statbuf, dictBufferType);
CHECK(ZSTD_DCtx_reset(ress.dctx, ZSTD_reset_session_only) );