feat(cli): move compression resource orchestration to Rust
Make the Rust policy layer own the ordered compression-resource lifecycle: create the CCtx, prepare patch/dictionary state, create the write and read pools, validate the dictionary, apply general and multithreaded parameters, and finally load the dictionary. Keep cRess_t, FIO_Dict_t, file statistics, AIO pools, and CLI diagnostics in C callbacks so the existing resource ownership and error behavior remain local to the C backend. Preserve adaptive window defaults and patch-from parameter adjustment while adding ABI layout assertions and a lifecycle-order test for the new boundary. Test Plan: - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo +nightly fmt --manifest-path rust/cli/Cargo.toml --all - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/cli/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/cli/Cargo.toml --all-targets (205 passed) - ulimit -v 41943040; CARGO_BUILD_JOBS=1 make -j1
This commit is contained in:
+120
-48
@@ -2063,11 +2063,29 @@ typedef size_t (*FIO_rust_createCResources_set_parameter_f)(void* context,
|
||||
int value);
|
||||
typedef int (*FIO_rust_createCResources_is_error_f)(size_t result);
|
||||
typedef void (*FIO_rust_createCResources_display_overlap_f)(int overlapLog);
|
||||
typedef void (*FIO_rust_createCResources_create_cctx_f)(void* context);
|
||||
typedef void (*FIO_rust_createCResources_prepare_dictionary_f)(
|
||||
void* context, const FIO_prefs_t* prefs, const char* dictFileName,
|
||||
U64 maxSrcFileSize, int cLevel, void* comprParams);
|
||||
typedef void (*FIO_rust_createCResources_create_pool_f)(
|
||||
void* context, const FIO_prefs_t* prefs);
|
||||
typedef void (*FIO_rust_createCResources_validate_dictionary_f)(void* context);
|
||||
typedef size_t (*FIO_rust_createCResources_load_dictionary_f)(
|
||||
void* context, int patchFromMode);
|
||||
typedef struct {
|
||||
void* callbackContext;
|
||||
const FIO_prefs_t* prefs;
|
||||
ZSTD_compressionParameters comprParams;
|
||||
int cLevel;
|
||||
const char* dictFileName;
|
||||
U64 maxSrcFileSize;
|
||||
int multithreaded;
|
||||
FIO_rust_createCResources_create_cctx_f createCctx;
|
||||
FIO_rust_createCResources_prepare_dictionary_f prepareDictionary;
|
||||
FIO_rust_createCResources_create_pool_f createWritePool;
|
||||
FIO_rust_createCResources_create_pool_f createReadPool;
|
||||
FIO_rust_createCResources_validate_dictionary_f validateDictionary;
|
||||
FIO_rust_createCResources_load_dictionary_f loadDictionary;
|
||||
FIO_rust_createCResources_set_parameter_f setParameter;
|
||||
FIO_rust_createCResources_is_error_f isError;
|
||||
FIO_rust_createCResources_display_overlap_f displayOverlap;
|
||||
@@ -2081,10 +2099,21 @@ typedef char FIO_rust_create_c_resources_state_params_offset[
|
||||
typedef char FIO_rust_create_c_resources_state_level_offset[
|
||||
(offsetof(FIO_rust_createCResourcesState, cLevel)
|
||||
== 2 * sizeof(void*) + sizeof(ZSTD_compressionParameters)) ? 1 : -1];
|
||||
typedef char FIO_rust_create_c_resources_state_set_parameter_offset[
|
||||
(offsetof(FIO_rust_createCResourcesState, setParameter)
|
||||
typedef char FIO_rust_create_c_resources_state_dict_name_offset[
|
||||
(offsetof(FIO_rust_createCResourcesState, dictFileName)
|
||||
== ((2 * sizeof(void*) + sizeof(ZSTD_compressionParameters) + sizeof(int)
|
||||
+ sizeof(void*) - 1) / sizeof(void*)) * sizeof(void*)) ? 1 : -1];
|
||||
typedef char FIO_rust_create_c_resources_state_max_src_offset[
|
||||
(offsetof(FIO_rust_createCResourcesState, maxSrcFileSize)
|
||||
== offsetof(FIO_rust_createCResourcesState, dictFileName) + sizeof(void*)) ? 1 : -1];
|
||||
typedef char FIO_rust_create_c_resources_state_mt_offset[
|
||||
(offsetof(FIO_rust_createCResourcesState, multithreaded)
|
||||
== offsetof(FIO_rust_createCResourcesState, maxSrcFileSize) + sizeof(U64)) ? 1 : -1];
|
||||
typedef char FIO_rust_create_c_resources_state_set_parameter_offset[
|
||||
(offsetof(FIO_rust_createCResourcesState, setParameter)
|
||||
== ((offsetof(FIO_rust_createCResourcesState, multithreaded) + sizeof(int)
|
||||
+ sizeof(void*) - 1) / sizeof(void*)) * sizeof(void*)
|
||||
+ 6 * sizeof(void*)) ? 1 : -1];
|
||||
typedef char FIO_rust_create_c_resources_state_is_error_offset[
|
||||
(offsetof(FIO_rust_createCResourcesState, isError)
|
||||
== offsetof(FIO_rust_createCResourcesState, setParameter) + sizeof(void*)) ? 1 : -1];
|
||||
@@ -2093,14 +2122,17 @@ typedef char FIO_rust_create_c_resources_state_display_overlap_offset[
|
||||
== offsetof(FIO_rust_createCResourcesState, isError) + sizeof(void*)) ? 1 : -1];
|
||||
typedef char FIO_rust_create_c_resources_state_size[
|
||||
(sizeof(FIO_rust_createCResourcesState)
|
||||
== offsetof(FIO_rust_createCResourcesState, setParameter) + 3 * sizeof(void*)) ? 1 : -1];
|
||||
== offsetof(FIO_rust_createCResourcesState, displayOverlap) + sizeof(void*)) ? 1 : -1];
|
||||
size_t FIO_rust_createCResources(const FIO_rust_createCResourcesState* state);
|
||||
|
||||
static size_t FIO_rust_createCResources_setParameter(void* context,
|
||||
int parameter,
|
||||
int value)
|
||||
{
|
||||
return ZSTD_CCtx_setParameter((ZSTD_CCtx*)context, (ZSTD_cParameter)parameter, value);
|
||||
cRess_t* const ress = (cRess_t*)context;
|
||||
if (parameter == ZSTD_c_nbWorkers)
|
||||
DISPLAYLEVEL(5, "set nb workers = %u \n", (unsigned)value);
|
||||
return ZSTD_CCtx_setParameter(ress->cctx, (ZSTD_cParameter)parameter, value);
|
||||
}
|
||||
|
||||
static int FIO_rust_createCResources_isError(size_t result)
|
||||
@@ -2113,67 +2145,107 @@ static void FIO_rust_createCResources_displayOverlap(int overlapLog)
|
||||
DISPLAYLEVEL(3,"set overlapLog = %u \n", overlapLog);
|
||||
}
|
||||
|
||||
size_t FIO_rust_setCResourcesMtParameters(const FIO_rust_createCResourcesState* state);
|
||||
static void FIO_rust_createCResources_createCctx(void* context)
|
||||
{
|
||||
cRess_t* const ress = (cRess_t*)context;
|
||||
ress->cctx = ZSTD_createCCtx();
|
||||
if (ress->cctx == NULL)
|
||||
EXM_THROW(30, "allocation error (%s): can't create ZSTD_CCtx",
|
||||
strerror(errno));
|
||||
}
|
||||
|
||||
static void FIO_rust_createCResources_prepareDictionary(
|
||||
void* context, const FIO_prefs_t* prefs, const char* dictFileName,
|
||||
U64 maxSrcFileSize, int cLevel, void* comprParams)
|
||||
{
|
||||
cRess_t* const ress = (cRess_t*)context;
|
||||
ZSTD_compressionParameters* const params =
|
||||
(ZSTD_compressionParameters*)comprParams;
|
||||
unsigned long long dictSize = 0;
|
||||
unsigned long long ssSize = 0;
|
||||
FIO_dictBufferType_t dictBufferType;
|
||||
|
||||
FIO_getDictFileStat(dictFileName, &ress->dictFileStat);
|
||||
|
||||
/* need to update memLimit before calling createDictBuffer
|
||||
* because of memLimit check inside it */
|
||||
if (prefs->patchFromMode) {
|
||||
dictSize = (unsigned long long)UTIL_getFileSizeStat(&ress->dictFileStat);
|
||||
ssSize = (unsigned long long)prefs->streamSrcSize;
|
||||
FIO_adjustParamsForPatchFromMode(
|
||||
(FIO_prefs_t*)prefs, params, dictSize,
|
||||
ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
|
||||
}
|
||||
|
||||
dictBufferType = (FIO_dictBufferType_t)FIO_rust_selectDictBufferType(
|
||||
prefs->mmapDict, prefs->patchFromMode, dictSize, prefs->memLimit);
|
||||
FIO_initDict(&ress->dict, dictFileName, (FIO_prefs_t*)prefs,
|
||||
&ress->dictFileStat, dictBufferType);
|
||||
ress->dictFileName = dictFileName;
|
||||
}
|
||||
|
||||
static void FIO_rust_createCResources_createWritePool(
|
||||
void* context, const FIO_prefs_t* prefs)
|
||||
{
|
||||
cRess_t* const ress = (cRess_t*)context;
|
||||
ress->writeCtx = AIO_WritePool_create(prefs, ZSTD_CStreamOutSize());
|
||||
}
|
||||
|
||||
static void FIO_rust_createCResources_createReadPool(
|
||||
void* context, const FIO_prefs_t* prefs)
|
||||
{
|
||||
cRess_t* const ress = (cRess_t*)context;
|
||||
ress->readCtx = AIO_ReadPool_create(prefs, ZSTD_CStreamInSize());
|
||||
}
|
||||
|
||||
static void FIO_rust_createCResources_validateDictionary(void* context)
|
||||
{
|
||||
cRess_t* const ress = (cRess_t*)context;
|
||||
if (ress->dictFileName && (ress->dict.dictBuffer == NULL))
|
||||
EXM_THROW(32, "allocation error : can't create dictBuffer");
|
||||
}
|
||||
|
||||
static size_t FIO_rust_createCResources_loadDictionary(
|
||||
void* context, int patchFromMode)
|
||||
{
|
||||
cRess_t* const ress = (cRess_t*)context;
|
||||
if (patchFromMode)
|
||||
return ZSTD_CCtx_refPrefix(
|
||||
ress->cctx, ress->dict.dictBuffer, ress->dict.dictBufferSize);
|
||||
return ZSTD_CCtx_loadDictionary_byReference(
|
||||
ress->cctx, ress->dict.dictBuffer, ress->dict.dictBufferSize);
|
||||
}
|
||||
|
||||
static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
|
||||
const char* dictFileName, unsigned long long const maxSrcFileSize,
|
||||
int cLevel, ZSTD_compressionParameters comprParams) {
|
||||
unsigned long long dictSize = 0;
|
||||
unsigned long long ssSize = 0;
|
||||
FIO_dictBufferType_t dictBufferType;
|
||||
cRess_t ress;
|
||||
FIO_rust_createCResourcesState policy;
|
||||
memset(&ress, 0, sizeof(ress));
|
||||
|
||||
DISPLAYLEVEL(6, "FIO_createCResources \n");
|
||||
ress.cctx = ZSTD_createCCtx();
|
||||
if (ress.cctx == NULL)
|
||||
EXM_THROW(30, "allocation error (%s): can't create ZSTD_CCtx",
|
||||
strerror(errno));
|
||||
|
||||
FIO_getDictFileStat(dictFileName, &ress.dictFileStat);
|
||||
|
||||
/* need to update memLimit before calling createDictBuffer
|
||||
* because of memLimit check inside it */
|
||||
if (prefs->patchFromMode) {
|
||||
dictSize = (unsigned long long)UTIL_getFileSizeStat(&ress.dictFileStat);
|
||||
ssSize = (unsigned long long)prefs->streamSrcSize;
|
||||
FIO_adjustParamsForPatchFromMode(prefs, &comprParams, dictSize, ssSize > 0 ? ssSize : maxSrcFileSize, cLevel);
|
||||
}
|
||||
|
||||
dictBufferType = (FIO_dictBufferType_t)FIO_rust_selectDictBufferType(
|
||||
prefs->mmapDict, prefs->patchFromMode, dictSize, prefs->memLimit);
|
||||
FIO_initDict(&ress.dict, dictFileName, prefs, &ress.dictFileStat, dictBufferType); /* works with dictFileName==NULL */
|
||||
|
||||
ress.writeCtx = AIO_WritePool_create(prefs, ZSTD_CStreamOutSize());
|
||||
ress.readCtx = AIO_ReadPool_create(prefs, ZSTD_CStreamInSize());
|
||||
|
||||
/* Advanced parameters, including dictionary */
|
||||
if (dictFileName && (ress.dict.dictBuffer==NULL))
|
||||
EXM_THROW(32, "allocation error : can't create dictBuffer");
|
||||
ress.dictFileName = dictFileName;
|
||||
|
||||
policy.callbackContext = ress.cctx;
|
||||
policy.callbackContext = &ress;
|
||||
policy.prefs = prefs;
|
||||
policy.comprParams = comprParams;
|
||||
policy.cLevel = cLevel;
|
||||
policy.dictFileName = dictFileName;
|
||||
policy.maxSrcFileSize = maxSrcFileSize;
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
policy.multithreaded = 1;
|
||||
#else
|
||||
policy.multithreaded = 0;
|
||||
#endif
|
||||
policy.createCctx = FIO_rust_createCResources_createCctx;
|
||||
policy.prepareDictionary = FIO_rust_createCResources_prepareDictionary;
|
||||
policy.createWritePool = FIO_rust_createCResources_createWritePool;
|
||||
policy.createReadPool = FIO_rust_createCResources_createReadPool;
|
||||
policy.validateDictionary = FIO_rust_createCResources_validateDictionary;
|
||||
policy.loadDictionary = FIO_rust_createCResources_loadDictionary;
|
||||
policy.setParameter = FIO_rust_createCResources_setParameter;
|
||||
policy.isError = FIO_rust_createCResources_isError;
|
||||
policy.displayOverlap = FIO_rust_createCResources_displayOverlap;
|
||||
CHECK( FIO_rust_createCResources(&policy) );
|
||||
|
||||
/* multi-threading */
|
||||
#ifdef ZSTD_MULTITHREAD
|
||||
DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers);
|
||||
CHECK( FIO_rust_setCResourcesMtParameters(&policy) );
|
||||
#endif
|
||||
/* dictionary */
|
||||
if (prefs->patchFromMode) {
|
||||
CHECK( ZSTD_CCtx_refPrefix(ress.cctx, ress.dict.dictBuffer, ress.dict.dictBufferSize) );
|
||||
} else {
|
||||
CHECK( ZSTD_CCtx_loadDictionary_byReference(ress.cctx, ress.dict.dictBuffer, ress.dict.dictBufferSize) );
|
||||
}
|
||||
|
||||
return ress;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user