feat(compress): move match-state reset policy into Rust
Move the reset decision tree for match-state tables from C into a Rust-owned callback projection. Rust now decides chain/hash3/row/optimal allocations, reset versus continue behavior, clean versus leave-dirty policy, scalar publication, and allocation-error propagation. C retains the cwksp phases, window invalidation, private pointer assignments, zeroing, and hash-salt implementation behind callbacks, so the private match-state and workspace layouts remain C-owned. Update the migration boundary documentation and add callback-order probes for CCtx optimal storage, CCtx row salting, CDict row zeroing, and table allocation failure. Test Plan: - cargo fmt --manifest-path rust/Cargo.toml -- --check - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml --lib (677 passed) - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040; make -B -C programs -j1 zstd - ulimit -v 41943040; make -C tests -j1 test-zstream ZSTREAM_TESTTIME=-T1s (84 tests and both short fuzzer rounds)
This commit is contained in:
+234
-76
@@ -614,6 +614,65 @@ U64 ZSTD_rust_advanceHashSalt(U64 hashSalt, U64 hashSaltEntropy);
|
|||||||
int ZSTD_rust_indexTooCloseToMax(size_t nextSrcBaseOffset);
|
int ZSTD_rust_indexTooCloseToMax(size_t nextSrcBaseOffset);
|
||||||
int ZSTD_rust_dictTooBig(size_t loadedDictSize);
|
int ZSTD_rust_dictTooBig(size_t loadedDictSize);
|
||||||
|
|
||||||
|
/* Match-state reset orchestration lives in Rust. The callbacks keep the
|
||||||
|
* workspace implementation, window representation, and private pointer
|
||||||
|
* fields in C. */
|
||||||
|
typedef void (*ZSTD_rust_resetMatchStateCallback_f)(void* context);
|
||||||
|
typedef void (*ZSTD_rust_resetMatchStateSetPointer_f)(
|
||||||
|
void* context, int pointerKind, void* pointer);
|
||||||
|
typedef void* (*ZSTD_rust_resetMatchStateReserve_f)(
|
||||||
|
void* context, int reserveKind, size_t size);
|
||||||
|
typedef int (*ZSTD_rust_resetMatchStateReserveFailed_f)(void* context);
|
||||||
|
typedef void (*ZSTD_rust_resetMatchStateZero_f)(
|
||||||
|
void* context, void* pointer, size_t size);
|
||||||
|
typedef struct {
|
||||||
|
void* callbackContext;
|
||||||
|
ZSTD_compressionParameters cParams;
|
||||||
|
int dedicatedDictSearch;
|
||||||
|
int useRowMatchFinder;
|
||||||
|
int compResetPolicy;
|
||||||
|
int indexResetPolicy;
|
||||||
|
int resetTarget;
|
||||||
|
unsigned hashLog3Max;
|
||||||
|
size_t litFreqSize;
|
||||||
|
size_t litLengthFreqSize;
|
||||||
|
size_t matchLengthFreqSize;
|
||||||
|
size_t offCodeFreqSize;
|
||||||
|
size_t matchSize;
|
||||||
|
size_t optimalSize;
|
||||||
|
void* hashLog3;
|
||||||
|
void* rowHashLog;
|
||||||
|
void* hashSalt;
|
||||||
|
void* lazySkipping;
|
||||||
|
void* cParamsOut;
|
||||||
|
ZSTD_rust_resetMatchStateSetPointer_f setPointer;
|
||||||
|
ZSTD_rust_resetMatchStateCallback_f windowInit;
|
||||||
|
ZSTD_rust_resetMatchStateCallback_f markTablesDirty;
|
||||||
|
ZSTD_rust_resetMatchStateCallback_f invalidateMatchState;
|
||||||
|
ZSTD_rust_resetMatchStateCallback_f clearTables;
|
||||||
|
ZSTD_rust_resetMatchStateCallback_f cleanTables;
|
||||||
|
ZSTD_rust_resetMatchStateCallback_f advanceHashSalt;
|
||||||
|
ZSTD_rust_resetMatchStateReserve_f reserve;
|
||||||
|
ZSTD_rust_resetMatchStateReserveFailed_f reserveFailed;
|
||||||
|
ZSTD_rust_resetMatchStateZero_f zero;
|
||||||
|
} ZSTD_rust_resetMatchStateState;
|
||||||
|
size_t ZSTD_rust_resetMatchState(const ZSTD_rust_resetMatchStateState* state);
|
||||||
|
typedef char ZSTD_rust_reset_match_state_layout[
|
||||||
|
(offsetof(ZSTD_rust_resetMatchStateState, callbackContext) == 0
|
||||||
|
&& offsetof(ZSTD_rust_resetMatchStateState, cParams) == sizeof(void*)
|
||||||
|
&& offsetof(ZSTD_rust_resetMatchStateState, litFreqSize)
|
||||||
|
> offsetof(ZSTD_rust_resetMatchStateState, hashLog3Max)
|
||||||
|
&& offsetof(ZSTD_rust_resetMatchStateState, hashLog3)
|
||||||
|
== offsetof(ZSTD_rust_resetMatchStateState, optimalSize)
|
||||||
|
+ sizeof(size_t)
|
||||||
|
&& offsetof(ZSTD_rust_resetMatchStateState, setPointer)
|
||||||
|
== offsetof(ZSTD_rust_resetMatchStateState, cParamsOut)
|
||||||
|
+ sizeof(void*)
|
||||||
|
&& sizeof(ZSTD_rust_resetMatchStateState)
|
||||||
|
== offsetof(ZSTD_rust_resetMatchStateState, zero)
|
||||||
|
+ sizeof(void*))
|
||||||
|
? 1 : -1];
|
||||||
|
|
||||||
/* The frame-chunk loop is Rust-owned. Its callbacks keep the private
|
/* The frame-chunk loop is Rust-owned. Its callbacks keep the private
|
||||||
* ZSTD_CCtx and match-state layout in C: Rust only drives the block loop and
|
* ZSTD_CCtx and match-state layout in C: Rust only drives the block loop and
|
||||||
* passes this context back to these C-owned state-preparation/dispatch seams. */
|
* passes this context back to these C-owned state-preparation/dispatch seams. */
|
||||||
@@ -3308,6 +3367,147 @@ static void ZSTD_advanceHashSalt(ZSTD_MatchState_t* ms) {
|
|||||||
ms->hashSalt = ZSTD_rust_advanceHashSalt(ms->hashSalt, (U64)ms->hashSaltEntropy);
|
ms->hashSalt = ZSTD_rust_advanceHashSalt(ms->hashSalt, (U64)ms->hashSaltEntropy);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
ZSTD_MatchState_t* ms;
|
||||||
|
ZSTD_cwksp* ws;
|
||||||
|
} ZSTD_rust_resetMatchStateContext;
|
||||||
|
|
||||||
|
enum {
|
||||||
|
ZSTD_RUST_RESET_MATCH_RESERVE_TABLE = 0,
|
||||||
|
ZSTD_RUST_RESET_MATCH_RESERVE_ALIGNED_INIT_ONCE = 1,
|
||||||
|
ZSTD_RUST_RESET_MATCH_RESERVE_ALIGNED64 = 2
|
||||||
|
};
|
||||||
|
|
||||||
|
enum {
|
||||||
|
ZSTD_RUST_RESET_MATCH_POINTER_HASH_TABLE = 0,
|
||||||
|
ZSTD_RUST_RESET_MATCH_POINTER_CHAIN_TABLE = 1,
|
||||||
|
ZSTD_RUST_RESET_MATCH_POINTER_HASH_TABLE3 = 2,
|
||||||
|
ZSTD_RUST_RESET_MATCH_POINTER_TAG_TABLE = 3,
|
||||||
|
ZSTD_RUST_RESET_MATCH_POINTER_LIT_FREQ = 4,
|
||||||
|
ZSTD_RUST_RESET_MATCH_POINTER_LIT_LENGTH_FREQ = 5,
|
||||||
|
ZSTD_RUST_RESET_MATCH_POINTER_MATCH_LENGTH_FREQ = 6,
|
||||||
|
ZSTD_RUST_RESET_MATCH_POINTER_OFF_CODE_FREQ = 7,
|
||||||
|
ZSTD_RUST_RESET_MATCH_POINTER_MATCH_TABLE = 8,
|
||||||
|
ZSTD_RUST_RESET_MATCH_POINTER_PRICE_TABLE = 9
|
||||||
|
};
|
||||||
|
|
||||||
|
static void ZSTD_rust_resetMatchState_windowInit(void* opaque)
|
||||||
|
{
|
||||||
|
ZSTD_rust_resetMatchStateContext* const context =
|
||||||
|
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||||
|
ZSTD_window_init(&context->ms->window);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void ZSTD_rust_resetMatchState_markTablesDirty(void* opaque)
|
||||||
|
{
|
||||||
|
ZSTD_rust_resetMatchStateContext* const context =
|
||||||
|
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||||
|
ZSTD_cwksp_mark_tables_dirty(context->ws);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void ZSTD_rust_resetMatchState_invalidate(void* opaque)
|
||||||
|
{
|
||||||
|
ZSTD_rust_resetMatchStateContext* const context =
|
||||||
|
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||||
|
ZSTD_invalidateMatchState(context->ms);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void ZSTD_rust_resetMatchState_clearTables(void* opaque)
|
||||||
|
{
|
||||||
|
ZSTD_rust_resetMatchStateContext* const context =
|
||||||
|
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||||
|
ZSTD_cwksp_clear_tables(context->ws);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void ZSTD_rust_resetMatchState_cleanTables(void* opaque)
|
||||||
|
{
|
||||||
|
ZSTD_rust_resetMatchStateContext* const context =
|
||||||
|
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||||
|
ZSTD_cwksp_clean_tables(context->ws);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void ZSTD_rust_resetMatchState_advanceHashSalt(void* opaque)
|
||||||
|
{
|
||||||
|
ZSTD_rust_resetMatchStateContext* const context =
|
||||||
|
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||||
|
ZSTD_advanceHashSalt(context->ms);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void* ZSTD_rust_resetMatchState_reserve(
|
||||||
|
void* opaque, int reserveKind, size_t size)
|
||||||
|
{
|
||||||
|
ZSTD_rust_resetMatchStateContext* const context =
|
||||||
|
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||||
|
switch (reserveKind) {
|
||||||
|
case ZSTD_RUST_RESET_MATCH_RESERVE_TABLE:
|
||||||
|
return ZSTD_cwksp_reserve_table(context->ws, size);
|
||||||
|
case ZSTD_RUST_RESET_MATCH_RESERVE_ALIGNED_INIT_ONCE:
|
||||||
|
return ZSTD_cwksp_reserve_aligned_init_once(context->ws, size);
|
||||||
|
case ZSTD_RUST_RESET_MATCH_RESERVE_ALIGNED64:
|
||||||
|
return ZSTD_cwksp_reserve_aligned64(context->ws, size);
|
||||||
|
default:
|
||||||
|
assert(0);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static int ZSTD_rust_resetMatchState_reserveFailed(void* opaque)
|
||||||
|
{
|
||||||
|
ZSTD_rust_resetMatchStateContext* const context =
|
||||||
|
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||||
|
return ZSTD_cwksp_reserve_failed(context->ws);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void ZSTD_rust_resetMatchState_setPointer(
|
||||||
|
void* opaque, int pointerKind, void* pointer)
|
||||||
|
{
|
||||||
|
ZSTD_rust_resetMatchStateContext* const context =
|
||||||
|
(ZSTD_rust_resetMatchStateContext*)opaque;
|
||||||
|
ZSTD_MatchState_t* const ms = context->ms;
|
||||||
|
switch (pointerKind) {
|
||||||
|
case ZSTD_RUST_RESET_MATCH_POINTER_HASH_TABLE:
|
||||||
|
ms->hashTable = (U32*)pointer;
|
||||||
|
break;
|
||||||
|
case ZSTD_RUST_RESET_MATCH_POINTER_CHAIN_TABLE:
|
||||||
|
ms->chainTable = (U32*)pointer;
|
||||||
|
break;
|
||||||
|
case ZSTD_RUST_RESET_MATCH_POINTER_HASH_TABLE3:
|
||||||
|
ms->hashTable3 = (U32*)pointer;
|
||||||
|
break;
|
||||||
|
case ZSTD_RUST_RESET_MATCH_POINTER_TAG_TABLE:
|
||||||
|
ms->tagTable = (BYTE*)pointer;
|
||||||
|
break;
|
||||||
|
case ZSTD_RUST_RESET_MATCH_POINTER_LIT_FREQ:
|
||||||
|
ms->opt.litFreq = (unsigned*)pointer;
|
||||||
|
break;
|
||||||
|
case ZSTD_RUST_RESET_MATCH_POINTER_LIT_LENGTH_FREQ:
|
||||||
|
ms->opt.litLengthFreq = (unsigned*)pointer;
|
||||||
|
break;
|
||||||
|
case ZSTD_RUST_RESET_MATCH_POINTER_MATCH_LENGTH_FREQ:
|
||||||
|
ms->opt.matchLengthFreq = (unsigned*)pointer;
|
||||||
|
break;
|
||||||
|
case ZSTD_RUST_RESET_MATCH_POINTER_OFF_CODE_FREQ:
|
||||||
|
ms->opt.offCodeFreq = (unsigned*)pointer;
|
||||||
|
break;
|
||||||
|
case ZSTD_RUST_RESET_MATCH_POINTER_MATCH_TABLE:
|
||||||
|
ms->opt.matchTable = (ZSTD_match_t*)pointer;
|
||||||
|
break;
|
||||||
|
case ZSTD_RUST_RESET_MATCH_POINTER_PRICE_TABLE:
|
||||||
|
ms->opt.priceTable = (ZSTD_optimal_t*)pointer;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
assert(0);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void ZSTD_rust_resetMatchState_zero(
|
||||||
|
void* opaque, void* pointer, size_t size)
|
||||||
|
{
|
||||||
|
(void)opaque;
|
||||||
|
ZSTD_memset(pointer, 0, size);
|
||||||
|
}
|
||||||
|
|
||||||
static size_t
|
static size_t
|
||||||
ZSTD_reset_matchState(ZSTD_MatchState_t* ms,
|
ZSTD_reset_matchState(ZSTD_MatchState_t* ms,
|
||||||
ZSTD_cwksp* ws,
|
ZSTD_cwksp* ws,
|
||||||
@@ -3317,82 +3517,40 @@ ZSTD_reset_matchState(ZSTD_MatchState_t* ms,
|
|||||||
const ZSTD_indexResetPolicy_e forceResetIndex,
|
const ZSTD_indexResetPolicy_e forceResetIndex,
|
||||||
const ZSTD_resetTarget_e forWho)
|
const ZSTD_resetTarget_e forWho)
|
||||||
{
|
{
|
||||||
/* disable chain table allocation for fast or row-based strategies */
|
ZSTD_rust_resetMatchStateContext context;
|
||||||
size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder,
|
ZSTD_rust_resetMatchStateState state;
|
||||||
ms->dedicatedDictSearch && (forWho == ZSTD_resetTarget_CDict))
|
context.ms = ms;
|
||||||
? ((size_t)1 << cParams->chainLog)
|
context.ws = ws;
|
||||||
: 0;
|
state.callbackContext = &context;
|
||||||
size_t const hSize = ((size_t)1) << cParams->hashLog;
|
state.cParams = *cParams;
|
||||||
U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
|
state.dedicatedDictSearch = ms->dedicatedDictSearch;
|
||||||
size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0;
|
state.useRowMatchFinder = (int)useRowMatchFinder;
|
||||||
|
state.compResetPolicy = (int)crp;
|
||||||
DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset);
|
state.indexResetPolicy = (int)forceResetIndex;
|
||||||
assert(useRowMatchFinder != ZSTD_ps_auto);
|
state.resetTarget = (int)forWho;
|
||||||
if (forceResetIndex == ZSTDirp_reset) {
|
state.hashLog3Max = ZSTD_HASHLOG3_MAX;
|
||||||
ZSTD_window_init(&ms->window);
|
state.litFreqSize = (1 << Litbits) * sizeof(unsigned);
|
||||||
ZSTD_cwksp_mark_tables_dirty(ws);
|
state.litLengthFreqSize = (MaxLL + 1) * sizeof(unsigned);
|
||||||
}
|
state.matchLengthFreqSize = (MaxML + 1) * sizeof(unsigned);
|
||||||
|
state.offCodeFreqSize = (MaxOff + 1) * sizeof(unsigned);
|
||||||
ms->hashLog3 = hashLog3;
|
state.matchSize = ZSTD_OPT_SIZE * sizeof(ZSTD_match_t);
|
||||||
ms->lazySkipping = 0;
|
state.optimalSize = ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t);
|
||||||
|
state.hashLog3 = &ms->hashLog3;
|
||||||
ZSTD_invalidateMatchState(ms);
|
state.rowHashLog = &ms->rowHashLog;
|
||||||
|
state.hashSalt = &ms->hashSalt;
|
||||||
assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */
|
state.lazySkipping = &ms->lazySkipping;
|
||||||
|
state.cParamsOut = &ms->cParams;
|
||||||
ZSTD_cwksp_clear_tables(ws);
|
state.setPointer = ZSTD_rust_resetMatchState_setPointer;
|
||||||
|
state.windowInit = ZSTD_rust_resetMatchState_windowInit;
|
||||||
DEBUGLOG(5, "reserving table space");
|
state.markTablesDirty = ZSTD_rust_resetMatchState_markTablesDirty;
|
||||||
/* table Space */
|
state.invalidateMatchState = ZSTD_rust_resetMatchState_invalidate;
|
||||||
ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32));
|
state.clearTables = ZSTD_rust_resetMatchState_clearTables;
|
||||||
ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32));
|
state.cleanTables = ZSTD_rust_resetMatchState_cleanTables;
|
||||||
ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32));
|
state.advanceHashSalt = ZSTD_rust_resetMatchState_advanceHashSalt;
|
||||||
RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation,
|
state.reserve = ZSTD_rust_resetMatchState_reserve;
|
||||||
"failed a workspace allocation in ZSTD_reset_matchState");
|
state.reserveFailed = ZSTD_rust_resetMatchState_reserveFailed;
|
||||||
|
state.zero = ZSTD_rust_resetMatchState_zero;
|
||||||
DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty);
|
return ZSTD_rust_resetMatchState(&state);
|
||||||
if (crp!=ZSTDcrp_leaveDirty) {
|
|
||||||
/* reset tables only */
|
|
||||||
ZSTD_cwksp_clean_tables(ws);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) {
|
|
||||||
/* Row match finder needs an additional table of hashes ("tags") */
|
|
||||||
size_t const tagTableSize = hSize;
|
|
||||||
/* We want to generate a new salt in case we reset a Cctx, but we always want to use
|
|
||||||
* 0 when we reset a Cdict */
|
|
||||||
if(forWho == ZSTD_resetTarget_CCtx) {
|
|
||||||
ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned_init_once(ws, tagTableSize);
|
|
||||||
ZSTD_advanceHashSalt(ms);
|
|
||||||
} else {
|
|
||||||
/* When we are not salting we want to always memset the memory */
|
|
||||||
ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned64(ws, tagTableSize);
|
|
||||||
ZSTD_memset(ms->tagTable, 0, tagTableSize);
|
|
||||||
ms->hashSalt = 0;
|
|
||||||
}
|
|
||||||
{ /* Switch to 32-entry rows if searchLog is 5 (or more) */
|
|
||||||
U32 const rowLog = BOUNDED(4, cParams->searchLog, 6);
|
|
||||||
assert(cParams->hashLog >= rowLog);
|
|
||||||
ms->rowHashLog = cParams->hashLog - rowLog;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* opt parser space */
|
|
||||||
if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) {
|
|
||||||
DEBUGLOG(4, "reserving optimal parser space");
|
|
||||||
ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (1<<Litbits) * sizeof(unsigned));
|
|
||||||
ms->opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxLL+1) * sizeof(unsigned));
|
|
||||||
ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxML+1) * sizeof(unsigned));
|
|
||||||
ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned64(ws, (MaxOff+1) * sizeof(unsigned));
|
|
||||||
ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_match_t));
|
|
||||||
ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned64(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t));
|
|
||||||
}
|
|
||||||
|
|
||||||
ms->cParams = *cParams;
|
|
||||||
|
|
||||||
RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation,
|
|
||||||
"failed a workspace allocation in ZSTD_reset_matchState");
|
|
||||||
return 0;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ZSTD_indexTooCloseToMax() :
|
/* ZSTD_indexTooCloseToMax() :
|
||||||
|
|||||||
+4
-4
@@ -166,10 +166,10 @@ CDict advanced private workspace construction, private static-CCtx and
|
|||||||
static-CDict workspace construction and dictionary-content allocation/loading,
|
static-CDict workspace construction and dictionary-content allocation/loading,
|
||||||
and advanced-CDict dictionary-content loading remain in C. Rust now owns
|
and advanced-CDict dictionary-content loading remain in C. Rust now owns
|
||||||
advanced-CDict custom-memory validation, workspace-size query/allocation,
|
advanced-CDict custom-memory validation, workspace-size query/allocation,
|
||||||
allocation/create/init cleanup ordering, and the CCtx workspace-size formula;
|
allocation/create/init cleanup ordering, the CCtx workspace-size formula, and
|
||||||
C retains private layout-size inputs, workspace layout, and allocator
|
the match-state reset policy/order; C retains private layout-size inputs,
|
||||||
callbacks. Reset policy,
|
workspace layout, and allocator callbacks. Private CCtx reset/matchfinder/
|
||||||
private CCtx/matchfinder/workspace operations, and codec/adaptive-policy
|
workspace operations and codec/adaptive-policy
|
||||||
callbacks remain in C. CDict initialization ordering and scalar publication,
|
callbacks remain in C. CDict initialization ordering and scalar publication,
|
||||||
shared compression-begin dictionary selection, CDict reset attach-versus-copy
|
shared compression-begin dictionary selection, CDict reset attach-versus-copy
|
||||||
selection, and CDict-begin parameter selection, initialization ordering, and
|
selection, and CDict-begin parameter selection, initialization ordering, and
|
||||||
|
|||||||
+696
-3
@@ -24,11 +24,12 @@ use crate::zstd_compress_frame::{
|
|||||||
use crate::zstd_compress_literals::min_gain;
|
use crate::zstd_compress_literals::min_gain;
|
||||||
use crate::zstd_compress_params::{
|
use crate::zstd_compress_params::{
|
||||||
ZSTD_compressionParameters, ZSTD_frameParameters, ZSTD_parameters, ZSTD_rustMatchStateSizing,
|
ZSTD_compressionParameters, ZSTD_frameParameters, ZSTD_parameters, ZSTD_rustMatchStateSizing,
|
||||||
ZSTD_rust_params_adjustCParams, ZSTD_rust_params_checkCParams, ZSTD_rust_params_defaultCLevel,
|
ZSTD_rust_params_adjustCParams, ZSTD_rust_params_allocateChainTable,
|
||||||
|
ZSTD_rust_params_checkCParams, ZSTD_rust_params_defaultCLevel,
|
||||||
ZSTD_rust_params_estimateMatchStateSize, ZSTD_rust_params_getParamsInternal,
|
ZSTD_rust_params_estimateMatchStateSize, ZSTD_rust_params_getParamsInternal,
|
||||||
ZSTD_rust_params_maxNbSeq, ZSTD_rust_params_resolveMaxBlockSize,
|
ZSTD_rust_params_maxNbSeq, ZSTD_rust_params_resolveMaxBlockSize,
|
||||||
ZSTD_rust_params_selectCParams, ZSTD_RUST_CPM_NO_ATTACH_DICT, ZSTD_RUST_PS_AUTO,
|
ZSTD_rust_params_rowMatchFinderUsed, ZSTD_rust_params_selectCParams,
|
||||||
ZSTD_RUST_PS_DISABLE, ZSTD_RUST_PS_ENABLE,
|
ZSTD_RUST_CPM_NO_ATTACH_DICT, ZSTD_RUST_PS_AUTO, ZSTD_RUST_PS_DISABLE, ZSTD_RUST_PS_ENABLE,
|
||||||
};
|
};
|
||||||
use crate::zstd_compress_params_api::{
|
use crate::zstd_compress_params_api::{
|
||||||
ZSTD_CCtxParams_setParameter, ZSTD_CCtx_params, ZSTD_rust_isUpdateAuthorized,
|
ZSTD_CCtxParams_setParameter, ZSTD_CCtx_params, ZSTD_rust_isUpdateAuthorized,
|
||||||
@@ -5221,6 +5222,356 @@ pub extern "C" fn ZSTD_rust_dictTooBig(loaded_dict_size: usize) -> c_int {
|
|||||||
dict_too_big(loaded_dict_size) as c_int
|
dict_too_big(loaded_dict_size) as c_int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ResetMatchStateCallback = unsafe extern "C" fn(*mut c_void);
|
||||||
|
type ResetMatchStateSetPointer = unsafe extern "C" fn(*mut c_void, c_int, *mut c_void);
|
||||||
|
type ResetMatchStateReserve = unsafe extern "C" fn(*mut c_void, c_int, usize) -> *mut c_void;
|
||||||
|
type ResetMatchStateReserveFailed = unsafe extern "C" fn(*mut c_void) -> c_int;
|
||||||
|
type ResetMatchStateZero = unsafe extern "C" fn(*mut c_void, *mut c_void, usize);
|
||||||
|
|
||||||
|
const RESET_MATCH_RESERVE_TABLE: c_int = 0;
|
||||||
|
const RESET_MATCH_RESERVE_ALIGNED_INIT_ONCE: c_int = 1;
|
||||||
|
const RESET_MATCH_RESERVE_ALIGNED64: c_int = 2;
|
||||||
|
|
||||||
|
const RESET_MATCH_POINTER_HASH_TABLE: c_int = 0;
|
||||||
|
const RESET_MATCH_POINTER_CHAIN_TABLE: c_int = 1;
|
||||||
|
const RESET_MATCH_POINTER_HASH_TABLE3: c_int = 2;
|
||||||
|
const RESET_MATCH_POINTER_TAG_TABLE: c_int = 3;
|
||||||
|
const RESET_MATCH_POINTER_LIT_FREQ: c_int = 4;
|
||||||
|
const RESET_MATCH_POINTER_LIT_LENGTH_FREQ: c_int = 5;
|
||||||
|
const RESET_MATCH_POINTER_MATCH_LENGTH_FREQ: c_int = 6;
|
||||||
|
const RESET_MATCH_POINTER_OFF_CODE_FREQ: c_int = 7;
|
||||||
|
const RESET_MATCH_POINTER_MATCH_TABLE: c_int = 8;
|
||||||
|
const RESET_MATCH_POINTER_PRICE_TABLE: c_int = 9;
|
||||||
|
|
||||||
|
/// Private C layout pointers and callback seams for match-state reset.
|
||||||
|
///
|
||||||
|
/// Rust owns the reset policy and allocation order. C retains the workspace,
|
||||||
|
/// window, and private match-state pointer representations behind callbacks.
|
||||||
|
#[repr(C)]
|
||||||
|
pub struct ZSTD_rust_resetMatchStateState {
|
||||||
|
callback_context: *mut c_void,
|
||||||
|
c_params: ZSTD_compressionParameters,
|
||||||
|
dedicated_dict_search: c_int,
|
||||||
|
use_row_match_finder: c_int,
|
||||||
|
comp_reset_policy: c_int,
|
||||||
|
index_reset_policy: c_int,
|
||||||
|
reset_target: c_int,
|
||||||
|
hash_log3_max: c_uint,
|
||||||
|
lit_freq_size: usize,
|
||||||
|
lit_length_freq_size: usize,
|
||||||
|
match_length_freq_size: usize,
|
||||||
|
off_code_freq_size: usize,
|
||||||
|
match_size: usize,
|
||||||
|
optimal_size: usize,
|
||||||
|
hash_log3: *mut c_uint,
|
||||||
|
row_hash_log: *mut c_uint,
|
||||||
|
hash_salt: *mut u64,
|
||||||
|
lazy_skipping: *mut c_int,
|
||||||
|
c_params_out: *mut ZSTD_compressionParameters,
|
||||||
|
set_pointer: Option<ResetMatchStateSetPointer>,
|
||||||
|
window_init: Option<ResetMatchStateCallback>,
|
||||||
|
mark_tables_dirty: Option<ResetMatchStateCallback>,
|
||||||
|
invalidate_match_state: Option<ResetMatchStateCallback>,
|
||||||
|
clear_tables: Option<ResetMatchStateCallback>,
|
||||||
|
clean_tables: Option<ResetMatchStateCallback>,
|
||||||
|
advance_hash_salt: Option<ResetMatchStateCallback>,
|
||||||
|
reserve: Option<ResetMatchStateReserve>,
|
||||||
|
reserve_failed: Option<ResetMatchStateReserveFailed>,
|
||||||
|
zero: Option<ResetMatchStateZero>,
|
||||||
|
}
|
||||||
|
|
||||||
|
const _: () = {
|
||||||
|
assert!(size_of::<ResetMatchStateCallback>() == size_of::<usize>());
|
||||||
|
assert!(size_of::<ResetMatchStateSetPointer>() == size_of::<usize>());
|
||||||
|
assert!(size_of::<ResetMatchStateReserve>() == size_of::<usize>());
|
||||||
|
assert!(size_of::<ResetMatchStateReserveFailed>() == size_of::<usize>());
|
||||||
|
assert!(size_of::<ResetMatchStateZero>() == size_of::<usize>());
|
||||||
|
assert!(offset_of!(ZSTD_rust_resetMatchStateState, callback_context) == 0);
|
||||||
|
assert!(offset_of!(ZSTD_rust_resetMatchStateState, c_params) == size_of::<usize>());
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_resetMatchStateState, lit_freq_size)
|
||||||
|
> offset_of!(ZSTD_rust_resetMatchStateState, hash_log3_max)
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_resetMatchStateState, hash_log3)
|
||||||
|
== offset_of!(ZSTD_rust_resetMatchStateState, optimal_size) + size_of::<usize>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
offset_of!(ZSTD_rust_resetMatchStateState, set_pointer)
|
||||||
|
== offset_of!(ZSTD_rust_resetMatchStateState, c_params_out) + size_of::<usize>()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
size_of::<ZSTD_rust_resetMatchStateState>()
|
||||||
|
== offset_of!(ZSTD_rust_resetMatchStateState, zero) + size_of::<usize>()
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn reset_match_state_table_size(log: u32) -> Option<usize> {
|
||||||
|
1usize.checked_shl(log)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reset and allocate one private match-state projection.
|
||||||
|
///
|
||||||
|
/// The callback context deliberately remains opaque here. This keeps C's
|
||||||
|
/// workspace phases and match-state layout out of the Rust ABI while making
|
||||||
|
/// the policy/order independently testable.
|
||||||
|
#[no_mangle]
|
||||||
|
pub unsafe extern "C" fn ZSTD_rust_resetMatchState(
|
||||||
|
state: *const ZSTD_rust_resetMatchStateState,
|
||||||
|
) -> usize {
|
||||||
|
if state.is_null() {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
}
|
||||||
|
let state = unsafe { &*state };
|
||||||
|
let Some(set_pointer) = state.set_pointer else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
let Some(window_init) = state.window_init else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
let Some(mark_tables_dirty) = state.mark_tables_dirty else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
let Some(invalidate_match_state) = state.invalidate_match_state else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
let Some(clear_tables) = state.clear_tables else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
let Some(clean_tables) = state.clean_tables else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
let Some(advance_hash_salt) = state.advance_hash_salt else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
let Some(reserve) = state.reserve else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
let Some(reserve_failed) = state.reserve_failed else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
let Some(zero) = state.zero else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
if state.callback_context.is_null()
|
||||||
|
|| state.hash_log3.is_null()
|
||||||
|
|| state.row_hash_log.is_null()
|
||||||
|
|| state.hash_salt.is_null()
|
||||||
|
|| state.lazy_skipping.is_null()
|
||||||
|
|| state.c_params_out.is_null()
|
||||||
|
{
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
}
|
||||||
|
if state.use_row_match_finder == ZSTD_RUST_PS_AUTO
|
||||||
|
|| !matches!(state.comp_reset_policy, 0..=1)
|
||||||
|
|| !matches!(state.index_reset_policy, 0..=1)
|
||||||
|
|| !matches!(state.reset_target, 0..=1)
|
||||||
|
{
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
}
|
||||||
|
|
||||||
|
let chain_size = if ZSTD_rust_params_allocateChainTable(
|
||||||
|
state.c_params.strategy,
|
||||||
|
state.use_row_match_finder,
|
||||||
|
c_int::from(state.dedicated_dict_search != 0 && state.reset_target == 0),
|
||||||
|
) != 0
|
||||||
|
{
|
||||||
|
let Some(size) = reset_match_state_table_size(state.c_params.chainLog) else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
size
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
let Some(hash_size) = reset_match_state_table_size(state.c_params.hashLog) else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
let hash_log3 = if state.reset_target == 1 && state.c_params.minMatch == 3 {
|
||||||
|
state.hash_log3_max.min(state.c_params.windowLog)
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
let hash3_size = if hash_log3 == 0 {
|
||||||
|
0
|
||||||
|
} else {
|
||||||
|
let Some(size) = reset_match_state_table_size(hash_log3) else {
|
||||||
|
return ERROR(ZstdErrorCode::Generic);
|
||||||
|
};
|
||||||
|
size
|
||||||
|
};
|
||||||
|
|
||||||
|
if state.index_reset_policy == 1 {
|
||||||
|
unsafe {
|
||||||
|
window_init(state.callback_context);
|
||||||
|
mark_tables_dirty(state.callback_context);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe {
|
||||||
|
*state.hash_log3 = hash_log3;
|
||||||
|
*state.lazy_skipping = 0;
|
||||||
|
invalidate_match_state(state.callback_context);
|
||||||
|
debug_assert_eq!(reserve_failed(state.callback_context), 0);
|
||||||
|
clear_tables(state.callback_context);
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe {
|
||||||
|
set_pointer(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_POINTER_HASH_TABLE,
|
||||||
|
reserve(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_RESERVE_TABLE,
|
||||||
|
hash_size.wrapping_mul(size_of::<c_uint>()),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
set_pointer(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_POINTER_CHAIN_TABLE,
|
||||||
|
reserve(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_RESERVE_TABLE,
|
||||||
|
chain_size.wrapping_mul(size_of::<c_uint>()),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
set_pointer(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_POINTER_HASH_TABLE3,
|
||||||
|
reserve(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_RESERVE_TABLE,
|
||||||
|
hash3_size.wrapping_mul(size_of::<c_uint>()),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if unsafe { reserve_failed(state.callback_context) } != 0 {
|
||||||
|
return ERROR(ZstdErrorCode::MemoryAllocation);
|
||||||
|
}
|
||||||
|
|
||||||
|
if state.comp_reset_policy != 1 {
|
||||||
|
unsafe { clean_tables(state.callback_context) };
|
||||||
|
}
|
||||||
|
|
||||||
|
if ZSTD_rust_params_rowMatchFinderUsed(state.c_params.strategy, state.use_row_match_finder) != 0
|
||||||
|
{
|
||||||
|
if state.reset_target == 1 {
|
||||||
|
let tag_table = unsafe {
|
||||||
|
reserve(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_RESERVE_ALIGNED_INIT_ONCE,
|
||||||
|
hash_size,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
unsafe {
|
||||||
|
set_pointer(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_POINTER_TAG_TABLE,
|
||||||
|
tag_table,
|
||||||
|
);
|
||||||
|
advance_hash_salt(state.callback_context);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
let tag_table = unsafe {
|
||||||
|
reserve(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_RESERVE_ALIGNED64,
|
||||||
|
hash_size,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
unsafe {
|
||||||
|
set_pointer(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_POINTER_TAG_TABLE,
|
||||||
|
tag_table,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if unsafe { reserve_failed(state.callback_context) } != 0 {
|
||||||
|
return ERROR(ZstdErrorCode::MemoryAllocation);
|
||||||
|
}
|
||||||
|
unsafe {
|
||||||
|
zero(state.callback_context, tag_table, hash_size);
|
||||||
|
*state.hash_salt = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if unsafe { reserve_failed(state.callback_context) } != 0 {
|
||||||
|
return ERROR(ZstdErrorCode::MemoryAllocation);
|
||||||
|
}
|
||||||
|
let row_log = state.c_params.searchLog.clamp(4, 6);
|
||||||
|
debug_assert!(state.c_params.hashLog >= row_log);
|
||||||
|
unsafe {
|
||||||
|
*state.row_hash_log = state.c_params.hashLog.wrapping_sub(row_log);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if state.reset_target == 1 && state.c_params.strategy >= 7 {
|
||||||
|
unsafe {
|
||||||
|
set_pointer(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_POINTER_LIT_FREQ,
|
||||||
|
reserve(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_RESERVE_ALIGNED64,
|
||||||
|
state.lit_freq_size,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
set_pointer(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_POINTER_LIT_LENGTH_FREQ,
|
||||||
|
reserve(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_RESERVE_ALIGNED64,
|
||||||
|
state.lit_length_freq_size,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
set_pointer(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_POINTER_MATCH_LENGTH_FREQ,
|
||||||
|
reserve(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_RESERVE_ALIGNED64,
|
||||||
|
state.match_length_freq_size,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
set_pointer(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_POINTER_OFF_CODE_FREQ,
|
||||||
|
reserve(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_RESERVE_ALIGNED64,
|
||||||
|
state.off_code_freq_size,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
set_pointer(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_POINTER_MATCH_TABLE,
|
||||||
|
reserve(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_RESERVE_ALIGNED64,
|
||||||
|
state.match_size,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
set_pointer(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_POINTER_PRICE_TABLE,
|
||||||
|
reserve(
|
||||||
|
state.callback_context,
|
||||||
|
RESET_MATCH_RESERVE_ALIGNED64,
|
||||||
|
state.optimal_size,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe {
|
||||||
|
*state.c_params_out = state.c_params;
|
||||||
|
}
|
||||||
|
if unsafe { reserve_failed(state.callback_context) } != 0 {
|
||||||
|
return ERROR(ZstdErrorCode::MemoryAllocation);
|
||||||
|
}
|
||||||
|
0
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
fn sizeof_local_dict(dict_buffer_present: c_int, dict_size: usize, cdict_size: usize) -> usize {
|
fn sizeof_local_dict(dict_buffer_present: c_int, dict_size: usize, cdict_size: usize) -> usize {
|
||||||
let buffer_size = if dict_buffer_present != 0 {
|
let buffer_size = if dict_buffer_present != 0 {
|
||||||
@@ -6821,6 +7172,348 @@ mod tests {
|
|||||||
const ZSTD_BTOPT: c_int = 7;
|
const ZSTD_BTOPT: c_int = 7;
|
||||||
const ZSTD_BTULTRA2: c_int = 9;
|
const ZSTD_BTULTRA2: c_int = 9;
|
||||||
|
|
||||||
|
#[derive(Default)]
|
||||||
|
struct ResetMatchStateTestContext {
|
||||||
|
events: Vec<c_int>,
|
||||||
|
reserves: Vec<(c_int, usize)>,
|
||||||
|
pointers: Vec<(c_int, *mut c_void)>,
|
||||||
|
zeroed: Vec<(usize, usize)>,
|
||||||
|
storage: [usize; 32],
|
||||||
|
fail_after: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe fn reset_match_state_test_context(
|
||||||
|
context: *mut c_void,
|
||||||
|
) -> &'static mut ResetMatchStateTestContext {
|
||||||
|
unsafe { &mut *context.cast::<ResetMatchStateTestContext>() }
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn reset_match_state_test_callback(context: *mut c_void) {
|
||||||
|
let context = unsafe { reset_match_state_test_context(context) };
|
||||||
|
context.events.push(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn reset_match_state_test_mark_dirty(context: *mut c_void) {
|
||||||
|
let context = unsafe { reset_match_state_test_context(context) };
|
||||||
|
context.events.push(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn reset_match_state_test_invalidate(context: *mut c_void) {
|
||||||
|
let context = unsafe { reset_match_state_test_context(context) };
|
||||||
|
context.events.push(3);
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn reset_match_state_test_clear(context: *mut c_void) {
|
||||||
|
let context = unsafe { reset_match_state_test_context(context) };
|
||||||
|
context.events.push(4);
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn reset_match_state_test_clean(context: *mut c_void) {
|
||||||
|
let context = unsafe { reset_match_state_test_context(context) };
|
||||||
|
context.events.push(5);
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn reset_match_state_test_advance(context: *mut c_void) {
|
||||||
|
let context = unsafe { reset_match_state_test_context(context) };
|
||||||
|
context.events.push(6);
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn reset_match_state_test_reserve(
|
||||||
|
context: *mut c_void,
|
||||||
|
reserve_kind: c_int,
|
||||||
|
size: usize,
|
||||||
|
) -> *mut c_void {
|
||||||
|
let context = unsafe { reset_match_state_test_context(context) };
|
||||||
|
context.events.push(100 + reserve_kind);
|
||||||
|
let index = context.reserves.len();
|
||||||
|
context.reserves.push((reserve_kind, size));
|
||||||
|
if context.fail_after != 0 && index + 1 >= context.fail_after {
|
||||||
|
ptr::null_mut()
|
||||||
|
} else {
|
||||||
|
unsafe { context.storage.as_mut_ptr().add(index).cast() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn reset_match_state_test_reserve_failed(context: *mut c_void) -> c_int {
|
||||||
|
let context = unsafe { reset_match_state_test_context(context) };
|
||||||
|
c_int::from(context.fail_after != 0 && context.reserves.len() >= context.fail_after)
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn reset_match_state_test_set_pointer(
|
||||||
|
context: *mut c_void,
|
||||||
|
pointer_kind: c_int,
|
||||||
|
pointer: *mut c_void,
|
||||||
|
) {
|
||||||
|
let context = unsafe { reset_match_state_test_context(context) };
|
||||||
|
context.events.push(200 + pointer_kind);
|
||||||
|
context.pointers.push((pointer_kind, pointer));
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe extern "C" fn reset_match_state_test_zero(
|
||||||
|
context: *mut c_void,
|
||||||
|
pointer: *mut c_void,
|
||||||
|
size: usize,
|
||||||
|
) {
|
||||||
|
let context = unsafe { reset_match_state_test_context(context) };
|
||||||
|
context.events.push(7);
|
||||||
|
context.zeroed.push((pointer as usize, size));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
fn reset_match_state_test_state(
|
||||||
|
context: &mut ResetMatchStateTestContext,
|
||||||
|
c_params: ZSTD_compressionParameters,
|
||||||
|
dedicated_dict_search: c_int,
|
||||||
|
use_row_match_finder: c_int,
|
||||||
|
comp_reset_policy: c_int,
|
||||||
|
index_reset_policy: c_int,
|
||||||
|
reset_target: c_int,
|
||||||
|
hash_log3: &mut c_uint,
|
||||||
|
row_hash_log: &mut c_uint,
|
||||||
|
hash_salt: &mut u64,
|
||||||
|
lazy_skipping: &mut c_int,
|
||||||
|
c_params_out: &mut ZSTD_compressionParameters,
|
||||||
|
) -> ZSTD_rust_resetMatchStateState {
|
||||||
|
ZSTD_rust_resetMatchStateState {
|
||||||
|
callback_context: (context as *mut ResetMatchStateTestContext).cast(),
|
||||||
|
c_params,
|
||||||
|
dedicated_dict_search,
|
||||||
|
use_row_match_finder,
|
||||||
|
comp_reset_policy,
|
||||||
|
index_reset_policy,
|
||||||
|
reset_target,
|
||||||
|
hash_log3_max: 17,
|
||||||
|
lit_freq_size: 256 * size_of::<c_uint>(),
|
||||||
|
lit_length_freq_size: 36 * size_of::<c_uint>(),
|
||||||
|
match_length_freq_size: 53 * size_of::<c_uint>(),
|
||||||
|
off_code_freq_size: 32 * size_of::<c_uint>(),
|
||||||
|
match_size: 4099 * 2 * size_of::<c_uint>(),
|
||||||
|
optimal_size: 4099 * 8 * size_of::<c_uint>(),
|
||||||
|
hash_log3,
|
||||||
|
row_hash_log,
|
||||||
|
hash_salt,
|
||||||
|
lazy_skipping,
|
||||||
|
c_params_out,
|
||||||
|
set_pointer: Some(reset_match_state_test_set_pointer),
|
||||||
|
window_init: Some(reset_match_state_test_callback),
|
||||||
|
mark_tables_dirty: Some(reset_match_state_test_mark_dirty),
|
||||||
|
invalidate_match_state: Some(reset_match_state_test_invalidate),
|
||||||
|
clear_tables: Some(reset_match_state_test_clear),
|
||||||
|
clean_tables: Some(reset_match_state_test_clean),
|
||||||
|
advance_hash_salt: Some(reset_match_state_test_advance),
|
||||||
|
reserve: Some(reset_match_state_test_reserve),
|
||||||
|
reserve_failed: Some(reset_match_state_test_reserve_failed),
|
||||||
|
zero: Some(reset_match_state_test_zero),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reset_match_state_test_params(
|
||||||
|
strategy: c_int,
|
||||||
|
search_log: u32,
|
||||||
|
min_match: u32,
|
||||||
|
) -> ZSTD_compressionParameters {
|
||||||
|
ZSTD_compressionParameters {
|
||||||
|
windowLog: 20,
|
||||||
|
chainLog: 11,
|
||||||
|
hashLog: 12,
|
||||||
|
searchLog: search_log,
|
||||||
|
minMatch: min_match,
|
||||||
|
targetLength: 16,
|
||||||
|
strategy,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reset_match_state_orders_cctx_opt_allocations() {
|
||||||
|
let c_params = reset_match_state_test_params(ZSTD_BTOPT, 5, 3);
|
||||||
|
let mut context = ResetMatchStateTestContext::default();
|
||||||
|
let mut hash_log3 = 99;
|
||||||
|
let mut row_hash_log = 99;
|
||||||
|
let mut hash_salt = 7;
|
||||||
|
let mut lazy_skipping = 9;
|
||||||
|
let mut c_params_out = ZSTD_compressionParameters::default();
|
||||||
|
let state = reset_match_state_test_state(
|
||||||
|
&mut context,
|
||||||
|
c_params,
|
||||||
|
0,
|
||||||
|
ZSTD_RUST_PS_DISABLE,
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
&mut hash_log3,
|
||||||
|
&mut row_hash_log,
|
||||||
|
&mut hash_salt,
|
||||||
|
&mut lazy_skipping,
|
||||||
|
&mut c_params_out,
|
||||||
|
);
|
||||||
|
|
||||||
|
let result = unsafe { ZSTD_rust_resetMatchState(&state) };
|
||||||
|
|
||||||
|
assert_eq!(result, 0);
|
||||||
|
assert_eq!(hash_log3, 17);
|
||||||
|
assert_eq!(row_hash_log, 99);
|
||||||
|
assert_eq!(hash_salt, 7);
|
||||||
|
assert_eq!(lazy_skipping, 0);
|
||||||
|
assert_eq!(c_params_out, c_params);
|
||||||
|
assert_eq!(
|
||||||
|
context.events,
|
||||||
|
[
|
||||||
|
1, 2, 3, 4, 100, 200, 100, 201, 100, 202, 5, 102, 204, 102, 205, 102, 206, 102,
|
||||||
|
207, 102, 208, 102, 209,
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
context.reserves,
|
||||||
|
[
|
||||||
|
(0, (1 << 12) * size_of::<c_uint>()),
|
||||||
|
(0, (1 << 11) * size_of::<c_uint>()),
|
||||||
|
(0, (1 << 17) * size_of::<c_uint>()),
|
||||||
|
(2, 256 * size_of::<c_uint>()),
|
||||||
|
(2, 36 * size_of::<c_uint>()),
|
||||||
|
(2, 53 * size_of::<c_uint>()),
|
||||||
|
(2, 32 * size_of::<c_uint>()),
|
||||||
|
(2, 4099 * 2 * size_of::<c_uint>()),
|
||||||
|
(2, 4099 * 8 * size_of::<c_uint>()),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert_eq!(context.pointers.len(), 9);
|
||||||
|
assert!(context.zeroed.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reset_match_state_salts_cctx_row_tables_without_zeroing() {
|
||||||
|
let c_params = reset_match_state_test_params(ZSTD_GREEDY, 6, 4);
|
||||||
|
let mut context = ResetMatchStateTestContext::default();
|
||||||
|
let mut hash_log3 = 99;
|
||||||
|
let mut row_hash_log = 99;
|
||||||
|
let mut hash_salt = 7;
|
||||||
|
let mut lazy_skipping = 9;
|
||||||
|
let mut c_params_out = ZSTD_compressionParameters::default();
|
||||||
|
let state = reset_match_state_test_state(
|
||||||
|
&mut context,
|
||||||
|
c_params,
|
||||||
|
0,
|
||||||
|
ZSTD_RUST_PS_ENABLE,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
&mut hash_log3,
|
||||||
|
&mut row_hash_log,
|
||||||
|
&mut hash_salt,
|
||||||
|
&mut lazy_skipping,
|
||||||
|
&mut c_params_out,
|
||||||
|
);
|
||||||
|
|
||||||
|
let result = unsafe { ZSTD_rust_resetMatchState(&state) };
|
||||||
|
|
||||||
|
assert_eq!(result, 0);
|
||||||
|
assert_eq!(hash_log3, 0);
|
||||||
|
assert_eq!(row_hash_log, 6);
|
||||||
|
assert_eq!(hash_salt, 7);
|
||||||
|
assert_eq!(c_params_out, c_params);
|
||||||
|
assert_eq!(
|
||||||
|
context.events,
|
||||||
|
[3, 4, 100, 200, 100, 201, 100, 202, 5, 101, 203, 6]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
context.reserves,
|
||||||
|
[
|
||||||
|
(0, (1 << 12) * size_of::<c_uint>()),
|
||||||
|
(0, 0),
|
||||||
|
(0, 0),
|
||||||
|
(1, 1 << 12),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert_eq!(context.pointers.len(), 4);
|
||||||
|
assert!(context.zeroed.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reset_match_state_uses_cdict_row_policy_without_opt_storage() {
|
||||||
|
let c_params = reset_match_state_test_params(ZSTD_GREEDY, 6, 4);
|
||||||
|
let mut context = ResetMatchStateTestContext::default();
|
||||||
|
let mut hash_log3 = 99;
|
||||||
|
let mut row_hash_log = 99;
|
||||||
|
let mut hash_salt = 7;
|
||||||
|
let mut lazy_skipping = 9;
|
||||||
|
let mut c_params_out = ZSTD_compressionParameters::default();
|
||||||
|
let state = reset_match_state_test_state(
|
||||||
|
&mut context,
|
||||||
|
c_params,
|
||||||
|
1,
|
||||||
|
ZSTD_RUST_PS_ENABLE,
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
&mut hash_log3,
|
||||||
|
&mut row_hash_log,
|
||||||
|
&mut hash_salt,
|
||||||
|
&mut lazy_skipping,
|
||||||
|
&mut c_params_out,
|
||||||
|
);
|
||||||
|
|
||||||
|
let result = unsafe { ZSTD_rust_resetMatchState(&state) };
|
||||||
|
|
||||||
|
assert_eq!(result, 0);
|
||||||
|
assert_eq!(hash_log3, 0);
|
||||||
|
assert_eq!(row_hash_log, 6);
|
||||||
|
assert_eq!(hash_salt, 0);
|
||||||
|
assert_eq!(c_params_out, c_params);
|
||||||
|
assert_eq!(
|
||||||
|
context.events,
|
||||||
|
[3, 4, 100, 200, 100, 201, 100, 202, 102, 203, 7]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
context.reserves,
|
||||||
|
[
|
||||||
|
(0, (1 << 12) * size_of::<c_uint>()),
|
||||||
|
(0, (1 << 11) * size_of::<c_uint>()),
|
||||||
|
(0, 0),
|
||||||
|
(2, 1 << 12),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert_eq!(context.pointers.len(), 4);
|
||||||
|
assert_eq!(context.zeroed.len(), 1);
|
||||||
|
assert_eq!(context.zeroed[0].1, 1 << 12);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reset_match_state_propagates_table_allocation_failure() {
|
||||||
|
let c_params = reset_match_state_test_params(ZSTD_GREEDY, 4, 4);
|
||||||
|
let mut context = ResetMatchStateTestContext {
|
||||||
|
fail_after: 1,
|
||||||
|
..ResetMatchStateTestContext::default()
|
||||||
|
};
|
||||||
|
let mut hash_log3 = 99;
|
||||||
|
let mut row_hash_log = 99;
|
||||||
|
let mut hash_salt = 7;
|
||||||
|
let mut lazy_skipping = 9;
|
||||||
|
let mut c_params_out = ZSTD_compressionParameters::default();
|
||||||
|
let state = reset_match_state_test_state(
|
||||||
|
&mut context,
|
||||||
|
c_params,
|
||||||
|
0,
|
||||||
|
ZSTD_RUST_PS_DISABLE,
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
1,
|
||||||
|
&mut hash_log3,
|
||||||
|
&mut row_hash_log,
|
||||||
|
&mut hash_salt,
|
||||||
|
&mut lazy_skipping,
|
||||||
|
&mut c_params_out,
|
||||||
|
);
|
||||||
|
|
||||||
|
let result = unsafe { ZSTD_rust_resetMatchState(&state) };
|
||||||
|
|
||||||
|
assert_eq!(result, ERROR(ZstdErrorCode::MemoryAllocation));
|
||||||
|
assert_eq!(context.events, [1, 2, 3, 4, 100, 200, 100, 201, 100, 202]);
|
||||||
|
assert_eq!(context.pointers.len(), 3);
|
||||||
|
assert_eq!(hash_log3, 0);
|
||||||
|
assert_eq!(lazy_skipping, 0);
|
||||||
|
assert_eq!(c_params_out, ZSTD_compressionParameters::default());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn reference_external_sequences_resets_store_state() {
|
fn reference_external_sequences_resets_store_state() {
|
||||||
let mut source = [0u8; 3];
|
let mut source = [0u8; 3];
|
||||||
|
|||||||
Reference in New Issue
Block a user