ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(cli): move decompression resource orchestration to Rust

Browse Source 
Move the decompression resource creation order into the Rust CLI policy
layer: dictionary stat and patch-memory preparation, decoder allocation,
window/checksum configuration, dictionary reset/attachment, and asynchronous
pool creation now run through one Rust-owned sequence.  Keep dRess_t,
stat_t, dictionary buffers, decoder context, pools, and exact allocation
errors in C callbacks.

Preserve the original patch-from and dictionary-reference behavior while
adding ABI layout assertions, scalar policy coverage, and error short-circuit
tests for the new boundary.

Test Plan:
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo +nightly fmt --manifest-path rust/cli/Cargo.toml --all
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/cli/Cargo.toml --all-targets (207 passed)
- ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1
This commit is contained in:
ddidderr
2026-07-21 14:26:49 +02:00
parent 877eb89e6c
commit 1941154279
2 changed files with 398 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
programs/fileio.c
+147 -34
View File
@@ -3820,11 +3820,62 @@ int FIO_compressMultipleFilenames(FIO_ctx_t* const fCtx,
***************************************************************************/
typedef struct {
FIO_Dict_t dict;
stat_t dictFileStat;
ZSTD_DStream* dctx;
WritePoolCtx_t *writeCtx;
ReadPoolCtx_t *readCtx;
} dRess_t;
typedef void (*FIO_rust_createDResourcesPrepareDictionaryCallback_f)(
void* context, FIO_prefs_t* prefs, const char* dictFileName);
typedef void (*FIO_rust_createDResourcesCreateDctxCallback_f)(void* context);
typedef size_t (*FIO_rust_createDResourcesSetMaxWindowSizeCallback_f)(
void* context, unsigned memLimit);
typedef size_t (*FIO_rust_createDResourcesSetParameterCallback_f)(
void* context, int value);
typedef size_t (*FIO_rust_createDResourcesLoadDictionaryCallback_f)(
void* context, FIO_prefs_t* prefs, const char* dictFileName);
typedef void (*FIO_rust_createDResourcesCreatePoolCallback_f)(
void* context, const FIO_prefs_t* prefs);
typedef int (*FIO_rust_createDResourcesIsErrorCallback_f)(size_t result);
typedef struct {
void* callbackContext;
FIO_prefs_t* prefs;
const char* dictFileName;
FIO_rust_createDResourcesPrepareDictionaryCallback_f prepareDictionary;
FIO_rust_createDResourcesCreateDctxCallback_f createDctx;
FIO_rust_createDResourcesSetMaxWindowSizeCallback_f setMaxWindowSize;
FIO_rust_createDResourcesSetParameterCallback_f setParameter;
FIO_rust_createDResourcesLoadDictionaryCallback_f loadDictionary;
FIO_rust_createDResourcesCreatePoolCallback_f createWritePool;
FIO_rust_createDResourcesCreatePoolCallback_f createReadPool;
FIO_rust_createDResourcesIsErrorCallback_f isError;
} FIO_rust_createDResourcesState;
typedef char FIO_rust_create_d_resources_state_layout[
(offsetof(FIO_rust_createDResourcesState, callbackContext) == 0
&& offsetof(FIO_rust_createDResourcesState, prefs) == sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, dictFileName)
== 2 * sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, prepareDictionary)
== 3 * sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, createDctx)
== 4 * sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, setMaxWindowSize)
== 5 * sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, setParameter)
== 6 * sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, loadDictionary)
== 7 * sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, createWritePool)
== 8 * sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, createReadPool)
== 9 * sizeof(void*)
&& offsetof(FIO_rust_createDResourcesState, isError)
== 10 * sizeof(void*)
&& sizeof(FIO_rust_createDResourcesState) == 11 * sizeof(void*))
? 1 : -1];
size_t FIO_rust_createDResources(const FIO_rust_createDResourcesState* state);
typedef void (*FIO_rust_freeDResourcesCallback_f)(void* context);
typedef struct {
void* callbackContext;
@@ -3870,45 +3921,107 @@ static void FIO_rust_freeDResources_readPool(void* context)
AIO_ReadPool_free(ress->readCtx);
}
static void FIO_rust_createDResources_prepareDictionary(
void* context, FIO_prefs_t* prefs, const char* dictFileName)
{
dRess_t* const ress = (dRess_t*)context;
FIO_getDictFileStat(dictFileName, &ress->dictFileStat);
if (prefs->patchFromMode) {
unsigned long long const dictSize =
(unsigned long long)UTIL_getFileSizeStat(&ress->dictFileStat);
FIO_adjustMemLimitForPatchFromMode(
prefs, dictSize, 0 /* just use the dict size */);
}
}
static void FIO_rust_createDResources_createDctx(void* context)
{
dRess_t* const ress = (dRess_t*)context;
ress->dctx = ZSTD_createDStream();
if (ress->dctx == NULL)
EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno));
}
static size_t FIO_rust_createDResources_setMaxWindowSize(
void* context, unsigned memLimit)
{
dRess_t* const ress = (dRess_t*)context;
return ZSTD_DCtx_setMaxWindowSize(ress->dctx, memLimit);
}
static size_t FIO_rust_createDResources_setParameter(void* context, int value)
{
dRess_t* const ress = (dRess_t*)context;
return ZSTD_DCtx_setParameter(
ress->dctx, ZSTD_d_forceIgnoreChecksum, value);
}
static size_t FIO_rust_createDResources_loadDictionary(
void* context, FIO_prefs_t* prefs, const char* dictFileName)
{
dRess_t* const ress = (dRess_t*)context;
unsigned long long const dictSize = prefs->patchFromMode
? (unsigned long long)UTIL_getFileSizeStat(&ress->dictFileStat)
: 0;
FIO_dictBufferType_t const dictBufferType =
(FIO_dictBufferType_t)FIO_rust_selectDictBufferType(
prefs->mmapDict, prefs->patchFromMode, dictSize,
prefs->memLimit);
FIO_initDict(&ress->dict, dictFileName, prefs, &ress->dictFileStat,
dictBufferType);
{ size_t const result = ZSTD_DCtx_reset(
ress->dctx, ZSTD_reset_session_only);
if (ZSTD_isError(result)) return result;
}
if (prefs->patchFromMode) {
return ZSTD_DCtx_refPrefix(
ress->dctx, ress->dict.dictBuffer, ress->dict.dictBufferSize);
}
return ZSTD_DCtx_loadDictionary_byReference(
ress->dctx, ress->dict.dictBuffer, ress->dict.dictBufferSize);
}
static void FIO_rust_createDResources_createWritePool(
void* context, const FIO_prefs_t* prefs)
{
dRess_t* const ress = (dRess_t*)context;
ress->writeCtx = AIO_WritePool_create(prefs, ZSTD_DStreamOutSize());
}
static void FIO_rust_createDResources_createReadPool(
void* context, const FIO_prefs_t* prefs)
{
dRess_t* const ress = (dRess_t*)context;
ress->readCtx = AIO_ReadPool_create(prefs, ZSTD_DStreamInSize());
}
static int FIO_rust_createDResources_isError(size_t result)
{
return ZSTD_isError(result);
}
static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName)
{
unsigned long long dictSize = 0;
stat_t statbuf;
dRess_t ress;
memset(&statbuf, 0, sizeof(statbuf));
memset(&ress, 0, sizeof(ress));
FIO_getDictFileStat(dictFileName, &statbuf);
if (prefs->patchFromMode){
dictSize = (unsigned long long)UTIL_getFileSizeStat(&statbuf);
FIO_adjustMemLimitForPatchFromMode(prefs, dictSize, 0 /* just use the dict size */);
}
/* Allocation */
ress.dctx = ZSTD_createDStream();
if (ress.dctx==NULL)
EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno));
CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, prefs->memLimit) );
CHECK( ZSTD_DCtx_setParameter(ress.dctx, ZSTD_d_forceIgnoreChecksum, !prefs->checksumFlag));
/* dictionary */
{
FIO_dictBufferType_t const dictBufferType = (FIO_dictBufferType_t)FIO_rust_selectDictBufferType(
prefs->mmapDict, prefs->patchFromMode, dictSize, prefs->memLimit);
FIO_initDict(&ress.dict, dictFileName, prefs, &statbuf, dictBufferType);
CHECK(ZSTD_DCtx_reset(ress.dctx, ZSTD_reset_session_only) );
if (prefs->patchFromMode){
CHECK(ZSTD_DCtx_refPrefix(ress.dctx, ress.dict.dictBuffer, ress.dict.dictBufferSize));
} else {
CHECK(ZSTD_DCtx_loadDictionary_byReference(ress.dctx, ress.dict.dictBuffer, ress.dict.dictBufferSize));
}
}
ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_DStreamOutSize());
ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_DStreamInSize());
FIO_rust_createDResourcesState const state = {
&ress,
prefs,
dictFileName,
FIO_rust_createDResources_prepareDictionary,
FIO_rust_createDResources_createDctx,
FIO_rust_createDResources_setMaxWindowSize,
FIO_rust_createDResources_setParameter,
FIO_rust_createDResources_loadDictionary,
FIO_rust_createDResources_createWritePool,
FIO_rust_createDResources_createReadPool,
FIO_rust_createDResources_isError
};
CHECK(FIO_rust_createDResources(&state));
return ress;
}