refactor(mt): move serial reset policy to Rust
Project the MT serial-reset scalar inputs into a narrow Rust ABI and let Rust own LDM normalization, table sizing, callback order, error short-circuiting, and parameter publication. Keep SerialState, ldmState_t, allocators, dictionary/window/checksum operations, and table mutation in C callbacks while preserving the previous bucket-log policy and the size_t-to-U32 job-size cast. Test Plan: - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo +nightly fmt --manifest-path rust/Cargo.toml --all -- --check - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo check --manifest-path rust/Cargo.toml --all-targets - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040; cc -std=c99 -fsyntax-only -Ilib -Ilib/common -Ilib/compress -Ilib/dict -Ilib/deprecated -Iprograms lib/compress/zstdmt_compress.c - ulimit -v 41943040; clang -std=c99 -fsyntax-only -Ilib -Ilib/common -Ilib/compress -Ilib/dict -Ilib/deprecated -Iprograms lib/compress/zstdmt_compress.c - Focused Rust tests added but not executed per the no-heavy-test instruction.
This commit is contained in:
+106
-42
@@ -786,19 +786,69 @@ size_t ZSTDMT_rust_updateCStreamDictionary(
|
||||
ZSTDMT_initResetStreamFn attachBorrowed,
|
||||
ZSTDMT_initResetStreamFn setRawPrefix,
|
||||
ZSTDMT_initDictionaryFn createReferenced);
|
||||
typedef struct {
|
||||
int enableLdm;
|
||||
int checksumEnabled;
|
||||
size_t jobSize;
|
||||
unsigned windowLog;
|
||||
int strategy;
|
||||
unsigned hashLog;
|
||||
unsigned bucketSizeLog;
|
||||
unsigned minMatchLength;
|
||||
unsigned hashRateLog;
|
||||
unsigned previousHashLog;
|
||||
unsigned previousBucketSizeLog;
|
||||
size_t ldmEntrySize;
|
||||
} ZSTDMT_RustSerialResetProjection;
|
||||
typedef char ZSTDMT_rust_serial_reset_projection_layout[
|
||||
(offsetof(ZSTDMT_RustSerialResetProjection, enableLdm) == 0
|
||||
&& offsetof(ZSTDMT_RustSerialResetProjection, checksumEnabled)
|
||||
== sizeof(int)
|
||||
&& offsetof(ZSTDMT_RustSerialResetProjection, jobSize) == 8
|
||||
&& offsetof(ZSTDMT_RustSerialResetProjection, windowLog)
|
||||
== (sizeof(void*) == 8 ? 16 : 12)
|
||||
&& offsetof(ZSTDMT_RustSerialResetProjection, strategy)
|
||||
== (sizeof(void*) == 8 ? 20 : 16)
|
||||
&& offsetof(ZSTDMT_RustSerialResetProjection, hashLog)
|
||||
== (sizeof(void*) == 8 ? 24 : 20)
|
||||
&& offsetof(ZSTDMT_RustSerialResetProjection, bucketSizeLog)
|
||||
== (sizeof(void*) == 8 ? 28 : 24)
|
||||
&& offsetof(ZSTDMT_RustSerialResetProjection, minMatchLength)
|
||||
== (sizeof(void*) == 8 ? 32 : 28)
|
||||
&& offsetof(ZSTDMT_RustSerialResetProjection, hashRateLog)
|
||||
== (sizeof(void*) == 8 ? 36 : 32)
|
||||
&& offsetof(ZSTDMT_RustSerialResetProjection, previousHashLog)
|
||||
== (sizeof(void*) == 8 ? 40 : 36)
|
||||
&& offsetof(ZSTDMT_RustSerialResetProjection, previousBucketSizeLog)
|
||||
== (sizeof(void*) == 8 ? 44 : 40)
|
||||
&& offsetof(ZSTDMT_RustSerialResetProjection, ldmEntrySize)
|
||||
== (sizeof(void*) == 8 ? 48 : 44)
|
||||
&& sizeof(ZSTDMT_RustSerialResetProjection)
|
||||
== (sizeof(void*) == 8 ? 56 : 48)) ? 1 : -1];
|
||||
typedef void (*ZSTDMT_serialResetSetLdmParamsFn)(
|
||||
void* opaque, int enableLdm, unsigned hashLog,
|
||||
unsigned bucketSizeLog, unsigned minMatchLength,
|
||||
unsigned hashRateLog, unsigned windowLog);
|
||||
typedef void (*ZSTDMT_serialResetVoidFn)(void* opaque);
|
||||
typedef void (*ZSTDMT_serialResetSetNbSeqFn)(void* opaque, size_t nbSeq);
|
||||
typedef int (*ZSTDMT_serialResetResizeFn)(void* opaque);
|
||||
typedef int (*ZSTDMT_serialResetResizeFn)(
|
||||
void* opaque, size_t hashSize, size_t numBuckets,
|
||||
unsigned bucketLog, unsigned previousBucketLog);
|
||||
typedef void (*ZSTDMT_serialResetZeroTablesFn)(
|
||||
void* opaque, size_t hashSize, size_t numBuckets);
|
||||
typedef void (*ZSTDMT_serialResetPublishFn)(void* opaque, size_t jobSize);
|
||||
int ZSTDMT_rust_serialStateReset(
|
||||
int enableLdm, int checksumEnabled, size_t maxNbSeq, void* opaque,
|
||||
const ZSTDMT_RustSerialResetProjection* projection, void* opaque,
|
||||
ZSTDMT_serialResetSetLdmParamsFn setLdmParams,
|
||||
ZSTDMT_serialResetVoidFn resetNextJob,
|
||||
ZSTDMT_serialResetVoidFn resetChecksum,
|
||||
ZSTDMT_serialResetSetNbSeqFn setNbSeq,
|
||||
ZSTDMT_serialResetVoidFn resetWindow,
|
||||
ZSTDMT_serialResetResizeFn resizeTables,
|
||||
ZSTDMT_serialResetVoidFn zeroTables,
|
||||
ZSTDMT_serialResetZeroTablesFn zeroTables,
|
||||
ZSTDMT_serialResetVoidFn loadDictionary,
|
||||
ZSTDMT_serialResetVoidFn copyWindow);
|
||||
ZSTDMT_serialResetVoidFn copyWindow,
|
||||
ZSTDMT_serialResetPublishFn publishParams);
|
||||
typedef void (*ZSTDMT_serialStateVoidFn)(void* opaque);
|
||||
typedef int (*ZSTDMT_serialStateInitFn)(void* opaque);
|
||||
int ZSTDMT_rust_serialStateInit(
|
||||
@@ -1360,13 +1410,24 @@ typedef struct {
|
||||
const void* dict;
|
||||
size_t dictSize;
|
||||
ZSTD_dictContentType_e dictContentType;
|
||||
size_t hashSize;
|
||||
size_t numBuckets;
|
||||
unsigned bucketLog;
|
||||
unsigned prevBucketLog;
|
||||
ZSTD_customMem cMem;
|
||||
} ZSTDMT_serialResetContext;
|
||||
|
||||
static void ZSTDMT_serialResetSetLdmParams(
|
||||
void* opaque, int enableLdm, unsigned hashLog,
|
||||
unsigned bucketSizeLog, unsigned minMatchLength,
|
||||
unsigned hashRateLog, unsigned windowLog)
|
||||
{
|
||||
ZSTDMT_serialResetContext* const context = (ZSTDMT_serialResetContext*)opaque;
|
||||
ldmParams_t* const ldmParams = &context->params->ldmParams;
|
||||
ldmParams->enableLdm = (ZSTD_ParamSwitch_e)enableLdm;
|
||||
ldmParams->hashLog = hashLog;
|
||||
ldmParams->bucketSizeLog = bucketSizeLog;
|
||||
ldmParams->minMatchLength = minMatchLength;
|
||||
ldmParams->hashRateLog = hashRateLog;
|
||||
ldmParams->windowLog = windowLog;
|
||||
}
|
||||
|
||||
static void ZSTDMT_serialResetNextJob(void* opaque)
|
||||
{
|
||||
ZSTDMT_serialResetContext* const context = (ZSTDMT_serialResetContext*)opaque;
|
||||
@@ -1391,7 +1452,9 @@ static void ZSTDMT_serialResetWindow(void* opaque)
|
||||
ZSTD_window_init(&context->serialState->ldmState.window);
|
||||
}
|
||||
|
||||
static int ZSTDMT_serialResetResizeTables(void* opaque)
|
||||
static int ZSTDMT_serialResetResizeTables(
|
||||
void* opaque, size_t hashSize, size_t numBuckets,
|
||||
unsigned bucketLog, unsigned previousBucketLog)
|
||||
{
|
||||
ZSTDMT_serialResetContext* const context = (ZSTDMT_serialResetContext*)opaque;
|
||||
SerialState* const serialState = context->serialState;
|
||||
@@ -1401,23 +1464,24 @@ static int ZSTDMT_serialResetResizeTables(void* opaque)
|
||||
serialState->params.ldmParams.hashLog < ldmParams->hashLog) {
|
||||
ZSTD_customFree(serialState->ldmState.hashTable, context->cMem);
|
||||
serialState->ldmState.hashTable = (ldmEntry_t*)ZSTD_customMalloc(
|
||||
context->hashSize, context->cMem);
|
||||
hashSize, context->cMem);
|
||||
}
|
||||
if (serialState->ldmState.bucketOffsets == NULL ||
|
||||
context->prevBucketLog < context->bucketLog) {
|
||||
previousBucketLog < bucketLog) {
|
||||
ZSTD_customFree(serialState->ldmState.bucketOffsets, context->cMem);
|
||||
serialState->ldmState.bucketOffsets = (BYTE*)ZSTD_customMalloc(
|
||||
context->numBuckets, context->cMem);
|
||||
numBuckets, context->cMem);
|
||||
}
|
||||
return !serialState->ldmState.hashTable || !serialState->ldmState.bucketOffsets;
|
||||
}
|
||||
|
||||
static void ZSTDMT_serialResetZeroTables(void* opaque)
|
||||
static void ZSTDMT_serialResetZeroTables(
|
||||
void* opaque, size_t hashSize, size_t numBuckets)
|
||||
{
|
||||
ZSTDMT_serialResetContext* const context = (ZSTDMT_serialResetContext*)opaque;
|
||||
SerialState* const serialState = context->serialState;
|
||||
ZSTD_memset(serialState->ldmState.hashTable, 0, context->hashSize);
|
||||
ZSTD_memset(serialState->ldmState.bucketOffsets, 0, context->numBuckets);
|
||||
ZSTD_memset(serialState->ldmState.hashTable, 0, hashSize);
|
||||
ZSTD_memset(serialState->ldmState.bucketOffsets, 0, numBuckets);
|
||||
}
|
||||
|
||||
static void ZSTDMT_serialResetLoadDictionary(void* opaque)
|
||||
@@ -1446,6 +1510,13 @@ static void ZSTDMT_serialResetCopyWindow(void* opaque)
|
||||
context->serialState->ldmWindow = context->serialState->ldmState.window;
|
||||
}
|
||||
|
||||
static void ZSTDMT_serialResetPublishParams(void* opaque, size_t jobSize)
|
||||
{
|
||||
ZSTDMT_serialResetContext* const context = (ZSTDMT_serialResetContext*)opaque;
|
||||
context->serialState->params = *context->params;
|
||||
context->serialState->params.jobSize = (U32)jobSize;
|
||||
}
|
||||
|
||||
static int
|
||||
ZSTDMT_serialState_reset(SerialState* serialState,
|
||||
ZSTDMT_seqPool* seqPool,
|
||||
@@ -1456,16 +1527,26 @@ ZSTDMT_serialState_reset(SerialState* serialState,
|
||||
{
|
||||
ZSTDMT_serialResetContext context;
|
||||
|
||||
/* Adjust parameters */
|
||||
/* Rust owns LDM normalization; retain the original diagnostic only. */
|
||||
if (params.ldmParams.enableLdm == ZSTD_ps_enable) {
|
||||
DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10);
|
||||
ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams);
|
||||
assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog);
|
||||
assert(params.ldmParams.hashRateLog < 32);
|
||||
} else {
|
||||
ZSTD_memset(¶ms.ldmParams, 0, sizeof(params.ldmParams));
|
||||
}
|
||||
|
||||
ZSTDMT_RustSerialResetProjection const projection = {
|
||||
params.ldmParams.enableLdm,
|
||||
params.fParams.checksumFlag,
|
||||
jobSize,
|
||||
params.cParams.windowLog,
|
||||
params.cParams.strategy,
|
||||
params.ldmParams.hashLog,
|
||||
params.ldmParams.bucketSizeLog,
|
||||
params.ldmParams.minMatchLength,
|
||||
params.ldmParams.hashRateLog,
|
||||
serialState->params.ldmParams.hashLog,
|
||||
serialState->params.ldmParams.bucketSizeLog,
|
||||
sizeof(ldmEntry_t)
|
||||
};
|
||||
|
||||
context.serialState = serialState;
|
||||
context.seqPool = seqPool;
|
||||
context.params = ¶ms;
|
||||
@@ -1473,25 +1554,10 @@ ZSTDMT_serialState_reset(SerialState* serialState,
|
||||
context.dictSize = dictSize;
|
||||
context.dictContentType = dictContentType;
|
||||
context.cMem = params.customMem;
|
||||
context.hashSize = 0;
|
||||
context.numBuckets = 0;
|
||||
context.bucketLog = 0;
|
||||
context.prevBucketLog = 0;
|
||||
if (params.ldmParams.enableLdm == ZSTD_ps_enable) {
|
||||
unsigned const hashLog = params.ldmParams.hashLog;
|
||||
context.hashSize = ((size_t)1 << hashLog) * sizeof(ldmEntry_t);
|
||||
context.bucketLog = params.ldmParams.hashLog - params.ldmParams.bucketSizeLog;
|
||||
context.prevBucketLog =
|
||||
serialState->params.ldmParams.hashLog -
|
||||
serialState->params.ldmParams.bucketSizeLog;
|
||||
context.numBuckets = (size_t)1 << context.bucketLog;
|
||||
}
|
||||
|
||||
if (ZSTDMT_rust_serialStateReset(
|
||||
params.ldmParams.enableLdm == ZSTD_ps_enable,
|
||||
params.fParams.checksumFlag,
|
||||
ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize),
|
||||
&context,
|
||||
&projection, &context,
|
||||
ZSTDMT_serialResetSetLdmParams,
|
||||
ZSTDMT_serialResetNextJob,
|
||||
ZSTDMT_serialResetChecksum,
|
||||
ZSTDMT_serialResetSetNbSeq,
|
||||
@@ -1499,12 +1565,10 @@ ZSTDMT_serialState_reset(SerialState* serialState,
|
||||
ZSTDMT_serialResetResizeTables,
|
||||
ZSTDMT_serialResetZeroTables,
|
||||
ZSTDMT_serialResetLoadDictionary,
|
||||
ZSTDMT_serialResetCopyWindow)) {
|
||||
ZSTDMT_serialResetCopyWindow,
|
||||
ZSTDMT_serialResetPublishParams)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
serialState->params = params;
|
||||
serialState->params.jobSize = (U32)jobSize;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user