feat(compress): move CDict table copies into Rust
Move the CDict-to-CCtx hash, chain, and row-match table copies into the Rust reset orchestration. The C bridge now exposes table-field addresses and scalar policy inputs while retaining private reset and hashTable3 operations; Rust loads destination pointers only after reset and strips short-cache tags through the existing Rust leaf. The destination chain policy must be read after the reset callback updates appliedParams. Reading it before reset skipped DFAST chain copies and caused the dictionary source-size zstream test to segfault in the double-fast external-dictionary matcher. Focused tests cover tagged hash/chain copies, row-match tag/salt copies, callback ordering, and reset-error short-circuiting. Test Plan: - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo test --manifest-path rust/Cargo.toml - ulimit -v 41943040; CARGO_BUILD_JOBS=1 cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - ulimit -v 41943040; make -j1 - ulimit -v 41943040; make -j1 -C tests test-zstream ZSTREAM_TESTTIME=-T2s - ulimit -v 41943040; make -j1 -C tests test-fuzzer FUZZERTEST=-T3s FUZZER_FLAGS=--no-big-tests
This commit is contained in:
@@ -2511,9 +2511,23 @@ typedef struct {
|
||||
U64 pledgedSrcSize;
|
||||
ZSTD_rust_resetCCtxByCopyingCDictReset_f reset;
|
||||
ZSTD_rust_resetCCtxByCopyingCDictMarkTables_f markTablesDirty;
|
||||
ZSTD_rust_resetCCtxByCopyingCDictState_f copyTables;
|
||||
ZSTD_rust_resetCCtxByCopyingCDictState_f zeroHashTable3;
|
||||
ZSTD_rust_resetCCtxByCopyingCDictMarkTables_f markTablesClean;
|
||||
U32** destinationHashTable;
|
||||
const U32* sourceHashTable;
|
||||
const U32* sourceHashLog;
|
||||
U32** destinationChainTable;
|
||||
const U32* sourceChainTable;
|
||||
const U32* sourceChainLog;
|
||||
const int* sourceStrategy;
|
||||
const int* sourceUseRowMatchFinder;
|
||||
const int* sourceIndicesTagged;
|
||||
const int* destinationStrategy;
|
||||
const int* destinationUseRowMatchFinder;
|
||||
BYTE** destinationTagTable;
|
||||
const BYTE* sourceTagTable;
|
||||
U64* destinationHashSalt;
|
||||
const U64* sourceHashSalt;
|
||||
void* destinationWindow;
|
||||
const void* sourceWindow;
|
||||
U32* destinationNextToUpdate;
|
||||
@@ -2541,7 +2555,7 @@ typedef char ZSTD_rust_reset_cctx_by_copying_cdict_state_layout[
|
||||
&& offsetof(ZSTD_rust_resetCCtxByCopyingCDictState, reset)
|
||||
== 3 * sizeof(void*) + sizeof(U64)
|
||||
&& offsetof(ZSTD_rust_resetCCtxByCopyingCDictState, zbuff)
|
||||
== 3 * sizeof(void*) + sizeof(U64) + 17 * sizeof(void*)
|
||||
== 3 * sizeof(void*) + sizeof(U64) + 31 * sizeof(void*)
|
||||
&& sizeof(ZSTD_rust_resetCCtxByCopyingCDictState)
|
||||
== ((offsetof(ZSTD_rust_resetCCtxByCopyingCDictState, zbuff)
|
||||
+ sizeof(int) + sizeof(void*) - 1) / sizeof(void*))
|
||||
@@ -3089,14 +3103,6 @@ static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) {
|
||||
return ZSTD_rust_params_rowMatchFinderSupported((int)strategy);
|
||||
}
|
||||
|
||||
/* Returns true if the strategy and useRowMatchFinder mode indicate that we will use the row based matchfinder
|
||||
* for this compression.
|
||||
*/
|
||||
static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_ParamSwitch_e mode) {
|
||||
assert(mode != ZSTD_ps_auto);
|
||||
return ZSTD_rust_params_rowMatchFinderUsed((int)strategy, (int)mode);
|
||||
}
|
||||
|
||||
/* Returns row matchfinder usage given an initial mode and cParams */
|
||||
static ZSTD_ParamSwitch_e ZSTD_resolveRowMatchFinderMode(ZSTD_ParamSwitch_e mode,
|
||||
const ZSTD_compressionParameters* const cParams) {
|
||||
@@ -3111,15 +3117,6 @@ static ZSTD_ParamSwitch_e ZSTD_resolveBlockSplitterMode(ZSTD_ParamSwitch_e mode,
|
||||
(int)mode, *cParams);
|
||||
}
|
||||
|
||||
/* Returns 1 if the arguments indicate that we should allocate a chainTable, 0 otherwise */
|
||||
static int ZSTD_allocateChainTable(const ZSTD_strategy strategy,
|
||||
const ZSTD_ParamSwitch_e useRowMatchFinder,
|
||||
const U32 forDDSDict) {
|
||||
assert(useRowMatchFinder != ZSTD_ps_auto);
|
||||
return ZSTD_rust_params_allocateChainTable(
|
||||
(int)strategy, (int)useRowMatchFinder, (int)forDDSDict);
|
||||
}
|
||||
|
||||
/* Returns ZSTD_ps_enable if compression parameters are such that we should
|
||||
* enable long distance matching (wlog >= 27, strategy >= btopt).
|
||||
* Returns ZSTD_ps_disable otherwise.
|
||||
@@ -4563,13 +4560,6 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
|
||||
return ZSTD_rust_resetCCtxByAttachingCDict(&state);
|
||||
}
|
||||
|
||||
static void ZSTD_copyCDictTableIntoCCtx(U32* dst, U32 const* src, size_t tableSize,
|
||||
ZSTD_compressionParameters const* cParams) {
|
||||
ZSTD_STATIC_ASSERT(ZSTD_SHORT_CACHE_TAG_BITS == 8);
|
||||
ZSTD_rust_copyCDictTableIntoCCtx(dst, src, tableSize,
|
||||
ZSTD_CDictIndicesAreTagged(cParams));
|
||||
}
|
||||
|
||||
static size_t ZSTD_rust_resetCCtx_byCopyingCDict_reset(
|
||||
void* context, const void* cdictOpaque, const void* paramsOpaque,
|
||||
U64 pledgedSrcSize, int zbuff)
|
||||
@@ -4610,40 +4600,6 @@ static void ZSTD_rust_resetCCtx_byCopyingCDict_mark_tables_dirty(void* context)
|
||||
ZSTD_cwksp_mark_tables_dirty(&cctx->workspace);
|
||||
}
|
||||
|
||||
static void ZSTD_rust_resetCCtx_byCopyingCDict_copy_tables(
|
||||
void* context, const void* cdictOpaque)
|
||||
{
|
||||
ZSTD_CCtx* const cctx = (ZSTD_CCtx*)context;
|
||||
const ZSTD_CDict* const cdict = (const ZSTD_CDict*)cdictOpaque;
|
||||
const ZSTD_compressionParameters* const cdict_cParams =
|
||||
&cdict->matchState.cParams;
|
||||
size_t const chainSize = ZSTD_allocateChainTable(
|
||||
cdict_cParams->strategy, cdict->useRowMatchFinder,
|
||||
0 /* DDS guaranteed disabled */)
|
||||
? ((size_t)1 << cdict_cParams->chainLog)
|
||||
: 0;
|
||||
size_t const hSize = (size_t)1 << cdict_cParams->hashLog;
|
||||
|
||||
ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.hashTable,
|
||||
cdict->matchState.hashTable,
|
||||
hSize, cdict_cParams);
|
||||
|
||||
/* Do not copy cdict's chainTable if cctx will not use a chainTable. */
|
||||
if (ZSTD_allocateChainTable(cctx->appliedParams.cParams.strategy,
|
||||
cctx->appliedParams.useRowMatchFinder,
|
||||
0 /* forDDSDict */)) {
|
||||
ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.chainTable,
|
||||
cdict->matchState.chainTable,
|
||||
chainSize, cdict_cParams);
|
||||
}
|
||||
if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy,
|
||||
cdict->useRowMatchFinder)) {
|
||||
ZSTD_memcpy(cctx->blockState.matchState.tagTable,
|
||||
cdict->matchState.tagTable, hSize);
|
||||
cctx->blockState.matchState.hashSalt = cdict->matchState.hashSalt;
|
||||
}
|
||||
}
|
||||
|
||||
static void ZSTD_rust_resetCCtx_byCopyingCDict_zero_hash_table3(
|
||||
void* context, const void* cdictOpaque)
|
||||
{
|
||||
@@ -4668,6 +4624,14 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
|
||||
U64 pledgedSrcSize,
|
||||
ZSTD_buffered_policy_e zbuff)
|
||||
{
|
||||
const ZSTD_compressionParameters* const cdict_cParams =
|
||||
&cdict->matchState.cParams;
|
||||
U32 const sourceHashLog = cdict_cParams->hashLog;
|
||||
U32 const sourceChainLog = cdict_cParams->chainLog;
|
||||
int const sourceStrategy = (int)cdict_cParams->strategy;
|
||||
int const sourceUseRowMatchFinder = (int)cdict->useRowMatchFinder;
|
||||
int const sourceIndicesTagged =
|
||||
ZSTD_CDictIndicesAreTagged(cdict_cParams);
|
||||
ZSTD_rust_resetCCtxByCopyingCDictState state;
|
||||
state.callbackContext = cctx;
|
||||
state.cdict = cdict;
|
||||
@@ -4676,11 +4640,31 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
|
||||
state.reset = ZSTD_rust_resetCCtx_byCopyingCDict_reset;
|
||||
state.markTablesDirty =
|
||||
ZSTD_rust_resetCCtx_byCopyingCDict_mark_tables_dirty;
|
||||
state.copyTables = ZSTD_rust_resetCCtx_byCopyingCDict_copy_tables;
|
||||
state.zeroHashTable3 =
|
||||
ZSTD_rust_resetCCtx_byCopyingCDict_zero_hash_table3;
|
||||
state.markTablesClean =
|
||||
ZSTD_rust_resetCCtx_byCopyingCDict_mark_tables_clean;
|
||||
state.destinationHashTable =
|
||||
&cctx->blockState.matchState.hashTable;
|
||||
state.sourceHashTable = cdict->matchState.hashTable;
|
||||
state.sourceHashLog = &sourceHashLog;
|
||||
state.destinationChainTable =
|
||||
&cctx->blockState.matchState.chainTable;
|
||||
state.sourceChainTable = cdict->matchState.chainTable;
|
||||
state.sourceChainLog = &sourceChainLog;
|
||||
state.sourceStrategy = &sourceStrategy;
|
||||
state.sourceUseRowMatchFinder = &sourceUseRowMatchFinder;
|
||||
state.sourceIndicesTagged = &sourceIndicesTagged;
|
||||
state.destinationStrategy =
|
||||
(const int*)&cctx->appliedParams.cParams.strategy;
|
||||
state.destinationUseRowMatchFinder =
|
||||
(const int*)&cctx->appliedParams.useRowMatchFinder;
|
||||
state.destinationTagTable =
|
||||
&cctx->blockState.matchState.tagTable;
|
||||
state.sourceTagTable = cdict->matchState.tagTable;
|
||||
state.destinationHashSalt =
|
||||
&cctx->blockState.matchState.hashSalt;
|
||||
state.sourceHashSalt = &cdict->matchState.hashSalt;
|
||||
state.destinationWindow = &cctx->blockState.matchState.window;
|
||||
state.sourceWindow = &cdict->matchState.window;
|
||||
state.destinationNextToUpdate = &cctx->blockState.matchState.nextToUpdate;
|
||||
|
||||
Reference in New Issue
Block a user