feat(compress): move sequence-store branch policy into Rust
Move buildSeqStore's preloaded external-sequence, LDM, external-producer, fallback, and ordinary matchfinder branch ordering into Rust. Keep the private CCtx, matchfinder, LDM, and producer operations behind narrow C callbacks, add ABI layout assertions, update the Rust boundary documentation, and cover each policy branch with focused tests. Test Plan: - cargo fmt --manifest-path rust/Cargo.toml -- --check - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --lib (690 passed) - ulimit -v 41943040 && CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040 && MAKEFLAGS=-j1 make -B -C programs -j1 zstd - ulimit -v 41943040 && MAKEFLAGS=-j1 make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s (84 tests and both short fuzz rounds passed)
This commit is contained in:
+125
-98
@@ -1862,9 +1862,13 @@ typedef void (*ZSTD_rust_buildSeqStoreSkip_f)(void* context, size_t srcSize);
|
||||
typedef void (*ZSTD_rust_buildSeqStorePrepare_f)(void* context,
|
||||
const void* src,
|
||||
size_t srcSize);
|
||||
typedef size_t (*ZSTD_rust_buildSeqStoreSelect_f)(
|
||||
typedef size_t (*ZSTD_rust_buildSeqStoreCompress_f)(
|
||||
void* context, SeqStore_t* seqStore, U32 nextRep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize, int* seqStoreComplete);
|
||||
const void* src, size_t srcSize);
|
||||
typedef size_t (*ZSTD_rust_buildSeqStoreTryExternalProducer_f)(
|
||||
void* context, const void* src, size_t srcSize,
|
||||
int* seqStoreComplete, int* allowFallback);
|
||||
typedef void (*ZSTD_rust_buildSeqStoreClearLdm_f)(void* context);
|
||||
typedef struct {
|
||||
SeqStore_t* seqStore;
|
||||
ZSTD_compressedBlockState_t** prevCBlock;
|
||||
@@ -1874,7 +1878,15 @@ typedef struct {
|
||||
int validateSeqStore;
|
||||
ZSTD_rust_buildSeqStoreSkip_f skipSmallBlock;
|
||||
ZSTD_rust_buildSeqStorePrepare_f prepareMatchState;
|
||||
ZSTD_rust_buildSeqStoreSelect_f selectSequences;
|
||||
int hasExternalSequences;
|
||||
int ldmEnabled;
|
||||
int hasExternalSequenceProducer;
|
||||
int enableMatchFinderFallback;
|
||||
ZSTD_rust_buildSeqStoreCompress_f compressExternalSequences;
|
||||
ZSTD_rust_buildSeqStoreCompress_f compressLdm;
|
||||
ZSTD_rust_buildSeqStoreTryExternalProducer_f tryExternalSequenceProducer;
|
||||
ZSTD_rust_buildSeqStoreCompress_f compressInternal;
|
||||
ZSTD_rust_buildSeqStoreClearLdm_f clearLdmSeqStore;
|
||||
} ZSTD_rust_buildSeqStoreState;
|
||||
size_t ZSTD_rust_buildSeqStore(const ZSTD_rust_buildSeqStoreState* state,
|
||||
const void* src, size_t srcSize);
|
||||
@@ -1890,10 +1902,26 @@ typedef char ZSTD_rust_build_seq_store_state_layout[
|
||||
== 4 * sizeof(void*) + sizeof(U32) + sizeof(int)
|
||||
&& offsetof(ZSTD_rust_buildSeqStoreState, prepareMatchState)
|
||||
== 5 * sizeof(void*) + sizeof(U32) + sizeof(int)
|
||||
&& offsetof(ZSTD_rust_buildSeqStoreState, selectSequences)
|
||||
&& offsetof(ZSTD_rust_buildSeqStoreState, hasExternalSequences)
|
||||
== 6 * sizeof(void*) + sizeof(U32) + sizeof(int)
|
||||
&& offsetof(ZSTD_rust_buildSeqStoreState, ldmEnabled)
|
||||
== 6 * sizeof(void*) + sizeof(U32) + 2 * sizeof(int)
|
||||
&& offsetof(ZSTD_rust_buildSeqStoreState, hasExternalSequenceProducer)
|
||||
== 6 * sizeof(void*) + sizeof(U32) + 3 * sizeof(int)
|
||||
&& offsetof(ZSTD_rust_buildSeqStoreState, enableMatchFinderFallback)
|
||||
== 6 * sizeof(void*) + sizeof(U32) + 4 * sizeof(int)
|
||||
&& offsetof(ZSTD_rust_buildSeqStoreState, compressExternalSequences)
|
||||
== 6 * sizeof(void*) + sizeof(U32) + 5 * sizeof(int)
|
||||
&& offsetof(ZSTD_rust_buildSeqStoreState, compressLdm)
|
||||
== 7 * sizeof(void*) + sizeof(U32) + 5 * sizeof(int)
|
||||
&& offsetof(ZSTD_rust_buildSeqStoreState, tryExternalSequenceProducer)
|
||||
== 8 * sizeof(void*) + sizeof(U32) + 5 * sizeof(int)
|
||||
&& offsetof(ZSTD_rust_buildSeqStoreState, compressInternal)
|
||||
== 9 * sizeof(void*) + sizeof(U32) + 5 * sizeof(int)
|
||||
&& offsetof(ZSTD_rust_buildSeqStoreState, clearLdmSeqStore)
|
||||
== 10 * sizeof(void*) + sizeof(U32) + 5 * sizeof(int)
|
||||
&& sizeof(ZSTD_rust_buildSeqStoreState)
|
||||
== 7 * sizeof(void*) + sizeof(U32) + sizeof(int))
|
||||
== 11 * sizeof(void*) + sizeof(U32) + 5 * sizeof(int))
|
||||
? 1 : -1];
|
||||
|
||||
typedef size_t (*ZSTD_rust_externalSequenceTransfer_f)(
|
||||
@@ -4672,106 +4700,97 @@ static void ZSTD_rust_buildSeqStore_prepareMatchState(void* context,
|
||||
}
|
||||
}
|
||||
|
||||
static size_t ZSTD_rust_buildSeqStore_selectSequences(
|
||||
static size_t ZSTD_rust_buildSeqStore_compressExternalSequences(
|
||||
void* context, SeqStore_t* seqStore, U32 nextRep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize, int* seqStoreComplete)
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
ZSTD_CCtx* const zc = (ZSTD_CCtx*)context;
|
||||
ZSTD_MatchState_t* const ms = &zc->blockState.matchState;
|
||||
ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms);
|
||||
size_t lastLLSize;
|
||||
*seqStoreComplete = 0;
|
||||
|
||||
if (zc->externSeqStore.pos < zc->externSeqStore.size) {
|
||||
assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable);
|
||||
|
||||
/* External matchfinder + LDM is technically possible, just not
|
||||
* implemented yet. */
|
||||
RETURN_ERROR_IF(
|
||||
ZSTD_hasExtSeqProd(&zc->appliedParams),
|
||||
parameter_combination_unsupported,
|
||||
"Long-distance matching with external sequence producer enabled is not currently supported."
|
||||
);
|
||||
|
||||
lastLLSize = ZSTD_ldm_blockCompress(
|
||||
&zc->externSeqStore, ms, seqStore, nextRep,
|
||||
zc->appliedParams.useRowMatchFinder, src, srcSize);
|
||||
assert(zc->externSeqStore.pos <= zc->externSeqStore.size);
|
||||
} else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) {
|
||||
RawSeqStore_t ldmSeqStore = kNullRawSeqStore;
|
||||
|
||||
/* External matchfinder + LDM is technically possible, just not
|
||||
* implemented yet. */
|
||||
RETURN_ERROR_IF(
|
||||
ZSTD_hasExtSeqProd(&zc->appliedParams),
|
||||
parameter_combination_unsupported,
|
||||
"Long-distance matching with external sequence producer enabled is not currently supported."
|
||||
);
|
||||
|
||||
ldmSeqStore.seq = zc->ldmSequences;
|
||||
ldmSeqStore.capacity = zc->maxNbLdmSequences;
|
||||
FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(
|
||||
&zc->ldmState, &ldmSeqStore, &zc->appliedParams.ldmParams,
|
||||
src, srcSize), "");
|
||||
lastLLSize = ZSTD_ldm_blockCompress(
|
||||
&ldmSeqStore, ms, seqStore, nextRep,
|
||||
zc->appliedParams.useRowMatchFinder, src, srcSize);
|
||||
assert(ldmSeqStore.pos == ldmSeqStore.size);
|
||||
} else if (ZSTD_hasExtSeqProd(&zc->appliedParams)) {
|
||||
size_t nbExternalSeqs = 0;
|
||||
int allowFallback = 0;
|
||||
ZSTD_rust_externalSequenceProducerState state;
|
||||
size_t const windowSize = (size_t)1 << zc->appliedParams.cParams.windowLog;
|
||||
assert(zc->extSeqBufCapacity >= ZSTD_sequenceBound(srcSize));
|
||||
assert(zc->appliedParams.extSeqProdFunc != NULL);
|
||||
|
||||
state.callbackContext = zc;
|
||||
state.producerState = zc->appliedParams.extSeqProdState;
|
||||
state.producer = zc->appliedParams.extSeqProdFunc;
|
||||
state.extSeqBuf = zc->extSeqBuf;
|
||||
state.extSeqBufCapacity = &zc->extSeqBufCapacity;
|
||||
state.src = src;
|
||||
state.srcSize = &srcSize;
|
||||
state.compressionLevel = &zc->appliedParams.compressionLevel;
|
||||
state.windowSize = &windowSize;
|
||||
state.transfer = ZSTD_rust_externalSequenceProducer_transfer;
|
||||
state.externalSeqCount = &nbExternalSeqs;
|
||||
state.seqStoreComplete = seqStoreComplete;
|
||||
state.allowFallback = &allowFallback;
|
||||
|
||||
{ size_t const producerResult =
|
||||
ZSTD_rust_tryExternalSequenceProducer(&state);
|
||||
if (*seqStoreComplete) {
|
||||
ms->ldmSeqStore = NULL;
|
||||
DEBUGLOG(5, "Copied %lu sequences from external sequence producer to internal seqStore.",
|
||||
(unsigned long)nbExternalSeqs);
|
||||
return producerResult;
|
||||
}
|
||||
|
||||
if (!allowFallback || !zc->appliedParams.enableMatchFinderFallback)
|
||||
return producerResult;
|
||||
|
||||
{ ZSTD_BlockCompressor_f const blockCompressor =
|
||||
ZSTD_selectBlockCompressor(
|
||||
zc->appliedParams.cParams.strategy,
|
||||
zc->appliedParams.useRowMatchFinder,
|
||||
dictMode);
|
||||
ms->ldmSeqStore = NULL;
|
||||
DEBUGLOG(5, "External sequence producer returned error code %lu. Falling back to internal parser.",
|
||||
(unsigned long)nbExternalSeqs);
|
||||
lastLLSize = blockCompressor(ms, seqStore, nextRep, src, srcSize);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ZSTD_BlockCompressor_f const blockCompressor = ZSTD_selectBlockCompressor(
|
||||
zc->appliedParams.cParams.strategy,
|
||||
zc->appliedParams.useRowMatchFinder, dictMode);
|
||||
ms->ldmSeqStore = NULL;
|
||||
lastLLSize = blockCompressor(ms, seqStore, nextRep, src, srcSize);
|
||||
}
|
||||
assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable);
|
||||
lastLLSize = ZSTD_ldm_blockCompress(
|
||||
&zc->externSeqStore, ms, seqStore, nextRep,
|
||||
zc->appliedParams.useRowMatchFinder, src, srcSize);
|
||||
assert(zc->externSeqStore.pos <= zc->externSeqStore.size);
|
||||
return lastLLSize;
|
||||
}
|
||||
|
||||
static size_t ZSTD_rust_buildSeqStore_compressLdm(
|
||||
void* context, SeqStore_t* seqStore, U32 nextRep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
ZSTD_CCtx* const zc = (ZSTD_CCtx*)context;
|
||||
ZSTD_MatchState_t* const ms = &zc->blockState.matchState;
|
||||
RawSeqStore_t ldmSeqStore = kNullRawSeqStore;
|
||||
size_t result;
|
||||
|
||||
ldmSeqStore.seq = zc->ldmSequences;
|
||||
ldmSeqStore.capacity = zc->maxNbLdmSequences;
|
||||
result = ZSTD_ldm_generateSequences(
|
||||
&zc->ldmState, &ldmSeqStore, &zc->appliedParams.ldmParams,
|
||||
src, srcSize);
|
||||
if (ERR_isError(result)) return result;
|
||||
result = ZSTD_ldm_blockCompress(
|
||||
&ldmSeqStore, ms, seqStore, nextRep,
|
||||
zc->appliedParams.useRowMatchFinder, src, srcSize);
|
||||
assert(ldmSeqStore.pos == ldmSeqStore.size);
|
||||
return result;
|
||||
}
|
||||
|
||||
static size_t ZSTD_rust_buildSeqStore_tryExternalSequenceProducer(
|
||||
void* context, const void* src, size_t srcSize,
|
||||
int* seqStoreComplete, int* allowFallback)
|
||||
{
|
||||
ZSTD_CCtx* const zc = (ZSTD_CCtx*)context;
|
||||
size_t nbExternalSeqs = 0;
|
||||
ZSTD_rust_externalSequenceProducerState state;
|
||||
size_t const windowSize = (size_t)1 << zc->appliedParams.cParams.windowLog;
|
||||
|
||||
assert(zc->extSeqBufCapacity >= ZSTD_sequenceBound(srcSize));
|
||||
assert(zc->appliedParams.extSeqProdFunc != NULL);
|
||||
state.callbackContext = zc;
|
||||
state.producerState = zc->appliedParams.extSeqProdState;
|
||||
state.producer = zc->appliedParams.extSeqProdFunc;
|
||||
state.extSeqBuf = zc->extSeqBuf;
|
||||
state.extSeqBufCapacity = &zc->extSeqBufCapacity;
|
||||
state.src = src;
|
||||
state.srcSize = &srcSize;
|
||||
state.compressionLevel = &zc->appliedParams.compressionLevel;
|
||||
state.windowSize = &windowSize;
|
||||
state.transfer = ZSTD_rust_externalSequenceProducer_transfer;
|
||||
state.externalSeqCount = &nbExternalSeqs;
|
||||
state.seqStoreComplete = seqStoreComplete;
|
||||
state.allowFallback = allowFallback;
|
||||
|
||||
{
|
||||
size_t const producerResult = ZSTD_rust_tryExternalSequenceProducer(&state);
|
||||
if (*seqStoreComplete) {
|
||||
DEBUGLOG(5, "Copied %lu sequences from external sequence producer to internal seqStore.",
|
||||
(unsigned long)nbExternalSeqs);
|
||||
}
|
||||
return producerResult;
|
||||
}
|
||||
}
|
||||
|
||||
static size_t ZSTD_rust_buildSeqStore_compressInternal(
|
||||
void* context, SeqStore_t* seqStore, U32 nextRep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
ZSTD_CCtx* const zc = (ZSTD_CCtx*)context;
|
||||
ZSTD_MatchState_t* const ms = &zc->blockState.matchState;
|
||||
ZSTD_BlockCompressor_f const blockCompressor = ZSTD_selectBlockCompressor(
|
||||
zc->appliedParams.cParams.strategy,
|
||||
zc->appliedParams.useRowMatchFinder,
|
||||
ZSTD_matchState_dictMode(ms));
|
||||
return blockCompressor(ms, seqStore, nextRep, src, srcSize);
|
||||
}
|
||||
|
||||
static void ZSTD_rust_buildSeqStore_clearLdmSeqStore(void* context)
|
||||
{
|
||||
ZSTD_CCtx* const zc = (ZSTD_CCtx*)context;
|
||||
zc->blockState.matchState.ldmSeqStore = NULL;
|
||||
}
|
||||
|
||||
static void ZSTD_initBuildSeqStoreState(
|
||||
ZSTD_CCtx* zc, ZSTD_rust_buildSeqStoreState* state)
|
||||
{
|
||||
@@ -4787,7 +4806,15 @@ static void ZSTD_initBuildSeqStoreState(
|
||||
#endif
|
||||
state->skipSmallBlock = ZSTD_rust_buildSeqStore_skipSmallBlock;
|
||||
state->prepareMatchState = ZSTD_rust_buildSeqStore_prepareMatchState;
|
||||
state->selectSequences = ZSTD_rust_buildSeqStore_selectSequences;
|
||||
state->hasExternalSequences = zc->externSeqStore.pos < zc->externSeqStore.size;
|
||||
state->ldmEnabled = zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable;
|
||||
state->hasExternalSequenceProducer = ZSTD_hasExtSeqProd(&zc->appliedParams);
|
||||
state->enableMatchFinderFallback = zc->appliedParams.enableMatchFinderFallback;
|
||||
state->compressExternalSequences = ZSTD_rust_buildSeqStore_compressExternalSequences;
|
||||
state->compressLdm = ZSTD_rust_buildSeqStore_compressLdm;
|
||||
state->tryExternalSequenceProducer = ZSTD_rust_buildSeqStore_tryExternalSequenceProducer;
|
||||
state->compressInternal = ZSTD_rust_buildSeqStore_compressInternal;
|
||||
state->clearLdmSeqStore = ZSTD_rust_buildSeqStore_clearLdmSeqStore;
|
||||
}
|
||||
|
||||
static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
|
||||
|
||||
Reference in New Issue
Block a user