refactor(cli): move dictionary buffer selection policy to Rust
Move the malloc-versus-mmap decision used by compression and decompression resource construction into Rust. The C adapter continues to own platform loaders, allocation handles, diagnostics, and dictionary I/O; Rust only combines the explicit mmap preference, patch-mode size threshold, and explicit disable override. Add enum-value ABI checks and focused policy coverage. Also hoist the source-size declaration in the C adapter so the migration does not introduce a C90 declaration-after-statement warning. Test Plan: - `ulimit -v 41943040; cargo +nightly fmt --manifest-path rust/Cargo.toml --all -- --check` - `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings` - `ulimit -v 41943040; make -j1` - `ulimit -v 41943040; make -j1 -C tests test` - `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings` - `ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/cli/Cargo.toml --all-targets` The integrated checks ran at the combined working-tree tip under a serial 40 GiB virtual-memory limit. Standalone root Rust unit linking remains unavailable because the crate imports C-owned bridge symbols without a Cargo build/link setup.
This commit is contained in:
+18
-11
@@ -351,6 +351,14 @@ int FIO_rust_adjustMemLimitForPatchFromMode(FIO_prefs_t* prefs,
|
|||||||
unsigned long long dictSize,
|
unsigned long long dictSize,
|
||||||
unsigned long long maxSrcFileSize);
|
unsigned long long maxSrcFileSize);
|
||||||
int FIO_rust_getDictFileStat(const char* fileName, stat_t* statBuf);
|
int FIO_rust_getDictFileStat(const char* fileName, stat_t* statBuf);
|
||||||
|
typedef char FIO_rust_dict_buffer_type_values[
|
||||||
|
(FIO_mallocDict == 0 && FIO_mmapDict == 1) ? 1 : -1];
|
||||||
|
typedef char FIO_rust_mmap_policy_values[
|
||||||
|
(ZSTD_ps_auto == 0 && ZSTD_ps_enable == 1 && ZSTD_ps_disable == 2) ? 1 : -1];
|
||||||
|
int FIO_rust_selectDictBufferType(int mmapDict,
|
||||||
|
int patchFromMode,
|
||||||
|
unsigned long long dictSize,
|
||||||
|
unsigned long long memLimit);
|
||||||
enum {
|
enum {
|
||||||
FIO_RUST_OPEN_SRC_SUCCESS = 0,
|
FIO_RUST_OPEN_SRC_SUCCESS = 0,
|
||||||
FIO_RUST_OPEN_SRC_STAT_FAILED = 1,
|
FIO_RUST_OPEN_SRC_STAT_FAILED = 1,
|
||||||
@@ -1833,8 +1841,8 @@ size_t FIO_rust_setCResourcesMtParameters(const FIO_rust_createCResourcesState*
|
|||||||
static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
|
static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
|
||||||
const char* dictFileName, unsigned long long const maxSrcFileSize,
|
const char* dictFileName, unsigned long long const maxSrcFileSize,
|
||||||
int cLevel, ZSTD_compressionParameters comprParams) {
|
int cLevel, ZSTD_compressionParameters comprParams) {
|
||||||
int useMMap = prefs->mmapDict == ZSTD_ps_enable;
|
unsigned long long dictSize = 0;
|
||||||
int forceNoUseMMap = prefs->mmapDict == ZSTD_ps_disable;
|
unsigned long long ssSize = 0;
|
||||||
FIO_dictBufferType_t dictBufferType;
|
FIO_dictBufferType_t dictBufferType;
|
||||||
cRess_t ress;
|
cRess_t ress;
|
||||||
FIO_rust_createCResourcesState policy;
|
FIO_rust_createCResourcesState policy;
|
||||||
@@ -1851,13 +1859,13 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
|
|||||||
/* need to update memLimit before calling createDictBuffer
|
/* need to update memLimit before calling createDictBuffer
|
||||||
* because of memLimit check inside it */
|
* because of memLimit check inside it */
|
||||||
if (prefs->patchFromMode) {
|
if (prefs->patchFromMode) {
|
||||||
U64 const dictSize = UTIL_getFileSizeStat(&ress.dictFileStat);
|
dictSize = (unsigned long long)UTIL_getFileSizeStat(&ress.dictFileStat);
|
||||||
unsigned long long const ssSize = (unsigned long long)prefs->streamSrcSize;
|
ssSize = (unsigned long long)prefs->streamSrcSize;
|
||||||
useMMap |= dictSize > prefs->memLimit;
|
|
||||||
FIO_adjustParamsForPatchFromMode(prefs, &comprParams, dictSize, ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
|
FIO_adjustParamsForPatchFromMode(prefs, &comprParams, dictSize, ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
|
||||||
}
|
}
|
||||||
|
|
||||||
dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict;
|
dictBufferType = (FIO_dictBufferType_t)FIO_rust_selectDictBufferType(
|
||||||
|
prefs->mmapDict, prefs->patchFromMode, dictSize, prefs->memLimit);
|
||||||
FIO_initDict(&ress.dict, dictFileName, prefs, &ress.dictFileStat, dictBufferType); /* works with dictFileName==NULL */
|
FIO_initDict(&ress.dict, dictFileName, prefs, &ress.dictFileStat, dictBufferType); /* works with dictFileName==NULL */
|
||||||
|
|
||||||
ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_CStreamOutSize());
|
ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_CStreamOutSize());
|
||||||
@@ -3715,8 +3723,7 @@ static void FIO_rust_freeDResources_readPool(void* context)
|
|||||||
|
|
||||||
static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName)
|
static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName)
|
||||||
{
|
{
|
||||||
int useMMap = prefs->mmapDict == ZSTD_ps_enable;
|
unsigned long long dictSize = 0;
|
||||||
int forceNoUseMMap = prefs->mmapDict == ZSTD_ps_disable;
|
|
||||||
stat_t statbuf;
|
stat_t statbuf;
|
||||||
dRess_t ress;
|
dRess_t ress;
|
||||||
memset(&statbuf, 0, sizeof(statbuf));
|
memset(&statbuf, 0, sizeof(statbuf));
|
||||||
@@ -3725,8 +3732,7 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
|
|||||||
FIO_getDictFileStat(dictFileName, &statbuf);
|
FIO_getDictFileStat(dictFileName, &statbuf);
|
||||||
|
|
||||||
if (prefs->patchFromMode){
|
if (prefs->patchFromMode){
|
||||||
U64 const dictSize = UTIL_getFileSizeStat(&statbuf);
|
dictSize = (unsigned long long)UTIL_getFileSizeStat(&statbuf);
|
||||||
useMMap |= dictSize > prefs->memLimit;
|
|
||||||
FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, 0 /* just use the dict size */);
|
FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, 0 /* just use the dict size */);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3739,7 +3745,8 @@ static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFi
|
|||||||
|
|
||||||
/* dictionary */
|
/* dictionary */
|
||||||
{
|
{
|
||||||
FIO_dictBufferType_t dictBufferType = (useMMap && !forceNoUseMMap) ? FIO_mmapDict : FIO_mallocDict;
|
FIO_dictBufferType_t const dictBufferType = (FIO_dictBufferType_t)FIO_rust_selectDictBufferType(
|
||||||
|
prefs->mmapDict, prefs->patchFromMode, dictSize, prefs->memLimit);
|
||||||
FIO_initDict(&ress.dict, dictFileName, prefs, &statbuf, dictBufferType);
|
FIO_initDict(&ress.dict, dictFileName, prefs, &statbuf, dictBufferType);
|
||||||
|
|
||||||
CHECK(ZSTD_DCtx_reset(ress.dctx, ZSTD_reset_session_only) );
|
CHECK(ZSTD_DCtx_reset(ress.dctx, ZSTD_reset_session_only) );
|
||||||
|
|||||||
@@ -642,6 +642,30 @@ pub unsafe extern "C" fn FIO_rust_munmapDict(
|
|||||||
|
|
||||||
const FIO_MALLOC_DICT: c_int = 0;
|
const FIO_MALLOC_DICT: c_int = 0;
|
||||||
const FIO_MMAP_DICT: c_int = 1;
|
const FIO_MMAP_DICT: c_int = 1;
|
||||||
|
const FIO_MMAP_POLICY_ENABLE: c_int = 1;
|
||||||
|
const FIO_MMAP_POLICY_DISABLE: c_int = 2;
|
||||||
|
|
||||||
|
/// Selects the dictionary backing store without performing any filesystem I/O.
|
||||||
|
///
|
||||||
|
/// Explicit mmap enable wins for ordinary dictionaries, patch mode requests a
|
||||||
|
/// mapping when the dictionary exceeds the memory limit, and an explicit
|
||||||
|
/// disable always wins over both. The C adapter retains the platform-specific
|
||||||
|
/// loaders and all diagnostics; this function owns only that policy decision.
|
||||||
|
#[no_mangle]
|
||||||
|
pub extern "C" fn FIO_rust_selectDictBufferType(
|
||||||
|
mmap_dict: c_int,
|
||||||
|
patch_from_mode: c_int,
|
||||||
|
dict_size: u64,
|
||||||
|
mem_limit: u64,
|
||||||
|
) -> c_int {
|
||||||
|
let auto_mmap = patch_from_mode != 0 && dict_size > mem_limit;
|
||||||
|
let requested_mmap = mmap_dict == FIO_MMAP_POLICY_ENABLE || auto_mmap;
|
||||||
|
if requested_mmap && mmap_dict != FIO_MMAP_POLICY_DISABLE {
|
||||||
|
FIO_MMAP_DICT
|
||||||
|
} else {
|
||||||
|
FIO_MALLOC_DICT
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Releases either kind of dictionary buffer and clears its C-visible fields.
|
/// Releases either kind of dictionary buffer and clears its C-visible fields.
|
||||||
///
|
///
|
||||||
@@ -716,6 +740,30 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dict_buffer_selection_preserves_mmap_preference_and_patch_policy() {
|
||||||
|
assert_eq!(
|
||||||
|
FIO_rust_selectDictBufferType(0, 0, 4096, 1),
|
||||||
|
FIO_MALLOC_DICT
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
FIO_rust_selectDictBufferType(0, 1, 4096, 8192),
|
||||||
|
FIO_MALLOC_DICT
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
FIO_rust_selectDictBufferType(0, 1, 8193, 8192),
|
||||||
|
FIO_MMAP_DICT
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
FIO_rust_selectDictBufferType(FIO_MMAP_POLICY_ENABLE, 0, 0, 0),
|
||||||
|
FIO_MMAP_DICT
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
FIO_rust_selectDictBufferType(FIO_MMAP_POLICY_DISABLE, 1, u64::MAX, 0),
|
||||||
|
FIO_MALLOC_DICT
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
fn open_source(path: &Path) -> (c_int, *mut c_void) {
|
fn open_source(path: &Path) -> (c_int, *mut c_void) {
|
||||||
let path = c_path(path);
|
let path = c_path(path);
|
||||||
let mut stat_buf = MaybeUninit::<libc::stat>::uninit();
|
let mut stat_buf = MaybeUninit::<libc::stat>::uninit();
|
||||||
|
|||||||
Reference in New Issue
Block a user