ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(rust): port lazy block matching

Browse Source 
Move greedy, lazy, lazy2, and binary-tree block matchers into Rust. A narrow
C state projection preserves configuration-specific context layout while Rust
owns hash-chain, row-based, attached-dictionary, and external-dictionary
searches and sequence-store updates.

The component map now records the lazy family as migrated; the optimized
matcher and high-level contexts remain outside this commit.

Test Plan:
- cargo test --all-targets
- cargo test --target i686-unknown-linux-gnu --all-targets
- cargo clippy && cargo clippy --benches && cargo clippy --tests
- cargo +nightly fmt
- make -B -C tests -j2 fuzzer zstreamtest invalidDictionaries poolTests
- ./tests/fuzzer -s5346 -i1 --no-big-tests
- ./tests/zstreamtest -i3000 -s334462
- ./tests/invalidDictionaries
- timeout 20s stdbuf -oL ./tests/poolTests

Refs: rust/README.md
This commit is contained in:
ddidderr
2026-07-11 08:06:30 +02:00
parent f9adcf6aef
commit 7f16a07375
4 changed files with 2828 additions and 2147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_lazy.c
+197 -2143
View File
@@ -8,2192 +8,246 @@
* You may select, at your option, one of the above-listed licenses. * You may select, at your option, one of the above-listed licenses.
*/ */
/* The lazy matchers are implemented in rust/src/zstd_lazy.rs. Keep only the
* private match-state projection and the established C entry points here. */
#include "zstd_compress_internal.h" #include "zstd_compress_internal.h"
#include "zstd_lazy.h" #include "zstd_lazy.h"
#include "../common/bits.h" /* ZSTD_countTrailingZeros64 */
#if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ #if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \
|| !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ || !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \
|| !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \ || !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \
|| !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) || !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR)
#define kLazySkippingStep 8 typedef char ZSTD_rust_lazy_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
typedef char ZSTD_rust_lazy_seqstore_long_length_pos[
(offsetof(SeqStore_t, longLengthPos) == 9 * sizeof(size_t) + 4) ? 1 : -1];
typedef char ZSTD_rust_lazy_seqstore_layout[
(sizeof(SeqStore_t) == 9 * sizeof(size_t) + 8) ? 1 : -1];
typedef char ZSTD_rust_lazy_rep_count[(ZSTD_REP_NUM == 3) ? 1 : -1];
/* This must remain in lock-step with `ZSTD_RustLazyState` in Rust. The large
* and private `ZSTD_MatchState_t` itself deliberately never crosses FFI. */
typedef struct ZSTD_RustLazyState_s {
U32* hashTable;
U32* chainTable;
BYTE* tagTable;
U32* hashCache;
const BYTE* base;
const BYTE* dictBase;
const BYTE* nextSrc;
U32 dictLimit;
U32 lowLimit;
U32 loadedDictEnd;
U32* nextToUpdate;
int* lazySkipping;
U32 hashLog;
U32 chainLog;
U32 minMatch;
U32 searchLog;
U32 windowLog;
U32 rowHashLog;
U64 hashSalt;
U32* hashSaltEntropy;
const struct ZSTD_RustLazyState_s* dictMatchState;
} ZSTD_RustLazyState;
/*-************************************* U32 ZSTD_rust_lazy_insertAndFindFirstIndex(ZSTD_RustLazyState* state,
* Binary Tree search const void* ip);
***************************************/ void ZSTD_rust_lazy_row_update(ZSTD_RustLazyState* state, const void* ip);
void ZSTD_rust_lazy_loadDedicatedDict(ZSTD_RustLazyState* state,
const void* ip);
size_t ZSTD_rust_compressBlock_lazy(
ZSTD_RustLazyState* state, void* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize, int searchMethod, U32 depth,
int dictMode);
static enum {
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR ZSTD_rust_search_hashChain = 0,
void ZSTD_updateDUBT(ZSTD_MatchState_t* ms, ZSTD_rust_search_binaryTree = 1,
const BYTE* ip, const BYTE* iend, ZSTD_rust_search_rowHash = 2,
U32 mls) };
enum {
ZSTD_rust_dict_noDict = 0,
ZSTD_rust_dict_extDict = 1,
ZSTD_rust_dict_dictMatchState = 2,
ZSTD_rust_dict_dedicatedDictSearch = 3,
};
static void ZSTD_rustLazyState_init(
ZSTD_RustLazyState* const out,
ZSTD_MatchState_t* const ms,
const ZSTD_RustLazyState* const dms)
{ {
const ZSTD_compressionParameters* const cParams = &ms->cParams; out->hashTable = ms->hashTable;
U32* const hashTable = ms->hashTable; out->chainTable = ms->chainTable;
U32 const hashLog = cParams->hashLog; out->tagTable = ms->tagTable;
out->hashCache = ms->hashCache;
U32* const bt = ms->chainTable; out->base = ms->window.base;
U32 const btLog = cParams->chainLog - 1; out->dictBase = ms->window.dictBase;
U32 const btMask = (1 << btLog) - 1; out->nextSrc = ms->window.nextSrc;
out->dictLimit = ms->window.dictLimit;
const BYTE* const base = ms->window.base; out->lowLimit = ms->window.lowLimit;
U32 const target = (U32)(ip - base); out->loadedDictEnd = ms->loadedDictEnd;
U32 idx = ms->nextToUpdate; out->nextToUpdate = &ms->nextToUpdate;
out->lazySkipping = &ms->lazySkipping;
if (idx != target) out->hashLog = ms->cParams.hashLog;
DEBUGLOG(7, "ZSTD_updateDUBT, from %u to %u (dictLimit:%u)", out->chainLog = ms->cParams.chainLog;
idx, target, ms->window.dictLimit); out->minMatch = ms->cParams.minMatch;
assert(ip + 8 <= iend); /* condition for ZSTD_hashPtr */ out->searchLog = ms->cParams.searchLog;
(void)iend; out->windowLog = ms->cParams.windowLog;
out->rowHashLog = ms->rowHashLog;
assert(idx >= ms->window.dictLimit); /* condition for valid base+idx */ out->hashSalt = ms->hashSalt;
for ( ; idx < target ; idx++) { out->hashSaltEntropy = &ms->hashSaltEntropy;
size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); /* assumption : ip + 8 <= iend */ out->dictMatchState = dms;
U32 const matchIndex = hashTable[h];
U32* const nextCandidatePtr = bt + 2*(idx&btMask);
U32* const sortMarkPtr = nextCandidatePtr + 1;
DEBUGLOG(8, "ZSTD_updateDUBT: insert %u", idx);
hashTable[h] = idx; /* Update Hash Table */
*nextCandidatePtr = matchIndex; /* update BT like a chain */
*sortMarkPtr = ZSTD_DUBT_UNSORTED_MARK;
}
ms->nextToUpdate = target;
} }
/* Dictionary match states are searched but never updated by this module. */
/** ZSTD_insertDUBT1() : static void ZSTD_rustLazyState_initDict(
* sort one already inserted but unsorted position ZSTD_RustLazyState* const out,
* assumption : curr >= btlow == (curr - btmask) const ZSTD_MatchState_t* const ms)
* doesn't fail */
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_insertDUBT1(const ZSTD_MatchState_t* ms,
U32 curr, const BYTE* inputEnd,
U32 nbCompares, U32 btLow,
const ZSTD_dictMode_e dictMode)
{ {
const ZSTD_compressionParameters* const cParams = &ms->cParams; out->hashTable = ms->hashTable;
U32* const bt = ms->chainTable; out->chainTable = ms->chainTable;
U32 const btLog = cParams->chainLog - 1; out->tagTable = ms->tagTable;
U32 const btMask = (1 << btLog) - 1; out->hashCache = NULL;
size_t commonLengthSmaller=0, commonLengthLarger=0; out->base = ms->window.base;
const BYTE* const base = ms->window.base; out->dictBase = ms->window.dictBase;
const BYTE* const dictBase = ms->window.dictBase; out->nextSrc = ms->window.nextSrc;
const U32 dictLimit = ms->window.dictLimit; out->dictLimit = ms->window.dictLimit;
const BYTE* const ip = (curr>=dictLimit) ? base + curr : dictBase + curr; out->lowLimit = ms->window.lowLimit;
const BYTE* const iend = (curr>=dictLimit) ? inputEnd : dictBase + dictLimit; out->loadedDictEnd = ms->loadedDictEnd;
const BYTE* const dictEnd = dictBase + dictLimit; out->nextToUpdate = NULL;
const BYTE* const prefixStart = base + dictLimit; out->lazySkipping = NULL;
const BYTE* match; out->hashLog = ms->cParams.hashLog;
U32* smallerPtr = bt + 2*(curr&btMask); out->chainLog = ms->cParams.chainLog;
U32* largerPtr = smallerPtr + 1; out->minMatch = ms->cParams.minMatch;
U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */ out->searchLog = ms->cParams.searchLog;
U32 dummy32; /* to be nullified at the end */ out->windowLog = ms->cParams.windowLog;
U32 const windowValid = ms->window.lowLimit; out->rowHashLog = ms->rowHashLog;
U32 const maxDistance = 1U << cParams->windowLog; out->hashSalt = ms->hashSalt;
U32 const windowLow = (curr - windowValid > maxDistance) ? curr - maxDistance : windowValid; out->hashSaltEntropy = NULL;
out->dictMatchState = NULL;
DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)",
curr, dictLimit, windowLow);
assert(curr >= btLow);
assert(ip < iend); /* condition for ZSTD_count */
for (; nbCompares && (matchIndex > windowLow); --nbCompares) {
U32* const nextPtr = bt + 2*(matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
assert(matchIndex < curr);
/* note : all candidates are now supposed sorted,
* but it's still possible to have nextPtr[1] == ZSTD_DUBT_UNSORTED_MARK
* when a real index has the same value as ZSTD_DUBT_UNSORTED_MARK */
if ( (dictMode != ZSTD_extDict)
|| (matchIndex+matchLength >= dictLimit) /* both in current segment*/
|| (curr < dictLimit) /* both in extDict */) {
const BYTE* const mBase = ( (dictMode != ZSTD_extDict)
|| (matchIndex+matchLength >= dictLimit)) ?
base : dictBase;
assert( (matchIndex+matchLength >= dictLimit) /* might be wrong if extDict is incorrectly set to 0 */
|| (curr < dictLimit) );
match = mBase + matchIndex;
matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend);
} else {
match = dictBase + matchIndex;
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
if (matchIndex+matchLength >= dictLimit)
match = base + matchIndex; /* preparation for next read of match[matchLength] */
}
DEBUGLOG(8, "ZSTD_insertDUBT1: comparing %u with %u : found %u common bytes ",
curr, matchIndex, (U32)matchLength);
if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */
break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */
}
if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */
/* match is smaller than current */
*smallerPtr = matchIndex; /* update smaller idx */
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */
DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is smaller : next => %u",
matchIndex, btLow, nextPtr[1]);
smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */
matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */
} else {
/* match is larger than current */
*largerPtr = matchIndex;
commonLengthLarger = matchLength;
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */
DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is larger => %u",
matchIndex, btLow, nextPtr[0]);
largerPtr = nextPtr;
matchIndex = nextPtr[0];
} }
*smallerPtr = *largerPtr = 0;
} }
static size_t ZSTD_rust_compressBlock_lazy_call(
static ZSTD_MatchState_t* const ms, SeqStore_t* const seqStore,
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR U32 rep[ZSTD_REP_NUM], const void* const src, size_t const srcSize,
size_t ZSTD_DUBT_findBetterDictMatch ( int const searchMethod, U32 const depth, int const dictMode)
const ZSTD_MatchState_t* ms,
const BYTE* const ip, const BYTE* const iend,
size_t* offsetPtr,
size_t bestLength,
U32 nbCompares,
U32 const mls,
const ZSTD_dictMode_e dictMode)
{ {
const ZSTD_MatchState_t * const dms = ms->dictMatchState; ZSTD_RustLazyState state;
const ZSTD_compressionParameters* const dmsCParams = &dms->cParams; ZSTD_RustLazyState dictState;
const U32 * const dictHashTable = dms->hashTable; const ZSTD_RustLazyState* dms = NULL;
U32 const hashLog = dmsCParams->hashLog;
size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
U32 dictMatchIndex = dictHashTable[h];
const BYTE* const base = ms->window.base; if (dictMode == ZSTD_rust_dict_dictMatchState
const BYTE* const prefixStart = base + ms->window.dictLimit; || dictMode == ZSTD_rust_dict_dedicatedDictSearch) {
U32 const curr = (U32)(ip-base); const ZSTD_MatchState_t* const rawDms = ms->dictMatchState;
const BYTE* const dictBase = dms->window.base; assert(rawDms != NULL);
const BYTE* const dictEnd = dms->window.nextSrc; ZSTD_rustLazyState_initDict(&dictState, rawDms);
U32 const dictHighLimit = (U32)(dms->window.nextSrc - dms->window.base); dms = &dictState;
U32 const dictLowLimit = dms->window.lowLimit;
U32 const dictIndexDelta = ms->window.lowLimit - dictHighLimit;
U32* const dictBt = dms->chainTable;
U32 const btLog = dmsCParams->chainLog - 1;
U32 const btMask = (1 << btLog) - 1;
U32 const btLow = (btMask >= dictHighLimit - dictLowLimit) ? dictLowLimit : dictHighLimit - btMask;
size_t commonLengthSmaller=0, commonLengthLarger=0;
(void)dictMode;
assert(dictMode == ZSTD_dictMatchState);
for (; nbCompares && (dictMatchIndex > dictLowLimit); --nbCompares) {
U32* const nextPtr = dictBt + 2*(dictMatchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
const BYTE* match = dictBase + dictMatchIndex;
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
if (dictMatchIndex+matchLength >= dictHighLimit)
match = base + dictMatchIndex + dictIndexDelta; /* to prepare for next usage of match[matchLength] */
if (matchLength > bestLength) {
U32 matchIndex = dictMatchIndex + dictIndexDelta;
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) {
DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)",
curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, OFFSET_TO_OFFBASE(curr - matchIndex), dictMatchIndex, matchIndex);
bestLength = matchLength, *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex);
}
if (ip+matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */
break; /* drop, to guarantee consistency (miss a little bit of compression) */
}
}
if (match[matchLength] < ip[matchLength]) {
if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
} else {
/* match is larger than current */
if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */
commonLengthLarger = matchLength;
dictMatchIndex = nextPtr[0];
}
} }
ZSTD_rustLazyState_init(&state, ms, dms);
if (bestLength >= MINMATCH) { return ZSTD_rust_compressBlock_lazy(
U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offsetPtr); (void)mIndex; &state, seqStore, rep, src, srcSize, searchMethod, depth, dictMode);
DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
curr, (U32)bestLength, (U32)*offsetPtr, mIndex);
}
return bestLength;
} }
U32 ZSTD_insertAndFindFirstIndex(ZSTD_MatchState_t* ms, const BYTE* ip)
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_DUBT_findBestMatch(ZSTD_MatchState_t* ms,
const BYTE* const ip, const BYTE* const iend,
size_t* offBasePtr,
U32 const mls,
const ZSTD_dictMode_e dictMode)
{ {
const ZSTD_compressionParameters* const cParams = &ms->cParams; ZSTD_RustLazyState state;
U32* const hashTable = ms->hashTable; ZSTD_rustLazyState_init(&state, ms, NULL);
U32 const hashLog = cParams->hashLog; return ZSTD_rust_lazy_insertAndFindFirstIndex(&state, ip);
size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
U32 matchIndex = hashTable[h];
const BYTE* const base = ms->window.base;
U32 const curr = (U32)(ip-base);
U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog);
U32* const bt = ms->chainTable;
U32 const btLog = cParams->chainLog - 1;
U32 const btMask = (1 << btLog) - 1;
U32 const btLow = (btMask >= curr) ? 0 : curr - btMask;
U32 const unsortLimit = MAX(btLow, windowLow);
U32* nextCandidate = bt + 2*(matchIndex&btMask);
U32* unsortedMark = bt + 2*(matchIndex&btMask) + 1;
U32 nbCompares = 1U << cParams->searchLog;
U32 nbCandidates = nbCompares;
U32 previousCandidate = 0;
DEBUGLOG(7, "ZSTD_DUBT_findBestMatch (%u) ", curr);
assert(ip <= iend-8); /* required for h calculation */
assert(dictMode != ZSTD_dedicatedDictSearch);
/* reach end of unsorted candidates list */
while ( (matchIndex > unsortLimit)
&& (*unsortedMark == ZSTD_DUBT_UNSORTED_MARK)
&& (nbCandidates > 1) ) {
DEBUGLOG(8, "ZSTD_DUBT_findBestMatch: candidate %u is unsorted",
matchIndex);
*unsortedMark = previousCandidate; /* the unsortedMark becomes a reversed chain, to move up back to original position */
previousCandidate = matchIndex;
matchIndex = *nextCandidate;
nextCandidate = bt + 2*(matchIndex&btMask);
unsortedMark = bt + 2*(matchIndex&btMask) + 1;
nbCandidates --;
}
/* nullify last candidate if it's still unsorted
* simplification, detrimental to compression ratio, beneficial for speed */
if ( (matchIndex > unsortLimit)
&& (*unsortedMark==ZSTD_DUBT_UNSORTED_MARK) ) {
DEBUGLOG(7, "ZSTD_DUBT_findBestMatch: nullify last unsorted candidate %u",
matchIndex);
*nextCandidate = *unsortedMark = 0;
}
/* batch sort stacked candidates */
matchIndex = previousCandidate;
while (matchIndex) { /* will end on matchIndex == 0 */
U32* const nextCandidateIdxPtr = bt + 2*(matchIndex&btMask) + 1;
U32 const nextCandidateIdx = *nextCandidateIdxPtr;
ZSTD_insertDUBT1(ms, matchIndex, iend,
nbCandidates, unsortLimit, dictMode);
matchIndex = nextCandidateIdx;
nbCandidates++;
}
/* find longest match */
{ size_t commonLengthSmaller = 0, commonLengthLarger = 0;
const BYTE* const dictBase = ms->window.dictBase;
const U32 dictLimit = ms->window.dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const prefixStart = base + dictLimit;
U32* smallerPtr = bt + 2*(curr&btMask);
U32* largerPtr = bt + 2*(curr&btMask) + 1;
U32 matchEndIdx = curr + 8 + 1;
U32 dummy32; /* to be nullified at the end */
size_t bestLength = 0;
matchIndex = hashTable[h];
hashTable[h] = curr; /* Update Hash Table */
for (; nbCompares && (matchIndex > windowLow); --nbCompares) {
U32* const nextPtr = bt + 2*(matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
const BYTE* match;
if ((dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit)) {
match = base + matchIndex;
matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend);
} else {
match = dictBase + matchIndex;
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
if (matchIndex+matchLength >= dictLimit)
match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
}
if (matchLength > bestLength) {
if (matchLength > matchEndIdx - matchIndex)
matchEndIdx = matchIndex + (U32)matchLength;
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr - matchIndex + 1) - ZSTD_highbit32((U32)*offBasePtr)) )
bestLength = matchLength, *offBasePtr = OFFSET_TO_OFFBASE(curr - matchIndex);
if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */
if (dictMode == ZSTD_dictMatchState) {
nbCompares = 0; /* in addition to avoiding checking any
* further in this loop, make sure we
* skip checking in the dictionary. */
}
break; /* drop, to guarantee consistency (miss a little bit of compression) */
}
}
if (match[matchLength] < ip[matchLength]) {
/* match is smaller than current */
*smallerPtr = matchIndex; /* update smaller idx */
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */
smallerPtr = nextPtr+1; /* new "smaller" => larger of match */
matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
} else {
/* match is larger than current */
*largerPtr = matchIndex;
commonLengthLarger = matchLength;
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */
largerPtr = nextPtr;
matchIndex = nextPtr[0];
} }
*smallerPtr = *largerPtr = 0;
assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */
if (dictMode == ZSTD_dictMatchState && nbCompares) {
bestLength = ZSTD_DUBT_findBetterDictMatch(
ms, ip, iend,
offBasePtr, bestLength, nbCompares,
mls, dictMode);
}
assert(matchEndIdx > curr+8); /* ensure nextToUpdate is increased */
ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */
if (bestLength >= MINMATCH) {
U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offBasePtr); (void)mIndex;
DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
curr, (U32)bestLength, (U32)*offBasePtr, mIndex);
}
return bestLength;
}
} }
void ZSTD_row_update(ZSTD_MatchState_t* const ms, const BYTE* const ip)
/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_BtFindBestMatch( ZSTD_MatchState_t* ms,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offBasePtr,
const U32 mls /* template */,
const ZSTD_dictMode_e dictMode)
{ {
DEBUGLOG(7, "ZSTD_BtFindBestMatch"); ZSTD_RustLazyState state;
if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ ZSTD_rustLazyState_init(&state, ms, NULL);
ZSTD_updateDUBT(ms, ip, iLimit, mls); ZSTD_rust_lazy_row_update(&state, ip);
return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offBasePtr, mls, dictMode);
} }
/*********************************** void ZSTD_dedicatedDictSearch_lazy_loadDictionary(
* Dedicated dict search ZSTD_MatchState_t* ms, const BYTE* const ip)
***********************************/
void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_MatchState_t* ms, const BYTE* const ip)
{ {
const BYTE* const base = ms->window.base; ZSTD_RustLazyState state;
U32 const target = (U32)(ip - base); ZSTD_rustLazyState_init(&state, ms, NULL);
U32* const hashTable = ms->hashTable; ZSTD_rust_lazy_loadDedicatedDict(&state, ip);
U32* const chainTable = ms->chainTable;
U32 const chainSize = 1 << ms->cParams.chainLog;
U32 idx = ms->nextToUpdate;
U32 const minChain = chainSize < target - idx ? target - chainSize : idx;
U32 const bucketSize = 1 << ZSTD_LAZY_DDSS_BUCKET_LOG;
U32 const cacheSize = bucketSize - 1;
U32 const chainAttempts = (1 << ms->cParams.searchLog) - cacheSize;
U32 const chainLimit = chainAttempts > 255 ? 255 : chainAttempts;
/* We know the hashtable is oversized by a factor of `bucketSize`.
* We are going to temporarily pretend `bucketSize == 1`, keeping only a
* single entry. We will use the rest of the space to construct a temporary
* chaintable.
*/
U32 const hashLog = ms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG;
U32* const tmpHashTable = hashTable;
U32* const tmpChainTable = hashTable + ((size_t)1 << hashLog);
U32 const tmpChainSize = (U32)((1 << ZSTD_LAZY_DDSS_BUCKET_LOG) - 1) << hashLog;
U32 const tmpMinChain = tmpChainSize < target ? target - tmpChainSize : idx;
U32 hashIdx;
assert(ms->cParams.chainLog <= 24);
assert(ms->cParams.hashLog > ms->cParams.chainLog);
assert(idx != 0);
assert(tmpMinChain <= minChain);
/* fill conventional hash table and conventional chain table */
for ( ; idx < target; idx++) {
U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch);
if (idx >= tmpMinChain) {
tmpChainTable[idx - tmpMinChain] = hashTable[h];
}
tmpHashTable[h] = idx;
}
/* sort chains into ddss chain table */
{
U32 chainPos = 0;
for (hashIdx = 0; hashIdx < (1U << hashLog); hashIdx++) {
U32 count;
U32 countBeyondMinChain = 0;
U32 i = tmpHashTable[hashIdx];
for (count = 0; i >= tmpMinChain && count < cacheSize; count++) {
/* skip through the chain to the first position that won't be
* in the hash cache bucket */
if (i < minChain) {
countBeyondMinChain++;
}
i = tmpChainTable[i - tmpMinChain];
}
if (count == cacheSize) {
for (count = 0; count < chainLimit;) {
if (i < minChain) {
if (!i || ++countBeyondMinChain > cacheSize) {
/* only allow pulling `cacheSize` number of entries
* into the cache or chainTable beyond `minChain`,
* to replace the entries pulled out of the
* chainTable into the cache. This lets us reach
* back further without increasing the total number
* of entries in the chainTable, guaranteeing the
* DDSS chain table will fit into the space
* allocated for the regular one. */
break;
}
}
chainTable[chainPos++] = i;
count++;
if (i < tmpMinChain) {
break;
}
i = tmpChainTable[i - tmpMinChain];
}
} else {
count = 0;
}
if (count) {
tmpHashTable[hashIdx] = ((chainPos - count) << 8) + count;
} else {
tmpHashTable[hashIdx] = 0;
}
}
assert(chainPos <= chainSize); /* I believe this is guaranteed... */
}
/* move chain pointers into the last entry of each hash bucket */
for (hashIdx = (1 << hashLog); hashIdx; ) {
U32 const bucketIdx = --hashIdx << ZSTD_LAZY_DDSS_BUCKET_LOG;
U32 const chainPackedPointer = tmpHashTable[hashIdx];
U32 i;
for (i = 0; i < cacheSize; i++) {
hashTable[bucketIdx + i] = 0;
}
hashTable[bucketIdx + bucketSize - 1] = chainPackedPointer;
}
/* fill the buckets of the hash table */
for (idx = ms->nextToUpdate; idx < target; idx++) {
U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch)
<< ZSTD_LAZY_DDSS_BUCKET_LOG;
U32 i;
/* Shift hash cache down 1. */
for (i = cacheSize - 1; i; i--)
hashTable[h + i] = hashTable[h + i - 1];
hashTable[h] = idx;
}
ms->nextToUpdate = target;
} }
/* Returns the longest match length found in the dedicated dict search structure. #define ZSTD_RUST_LAZY_WRAPPER(name, method, parserDepth, mode) \
* If none are longer than the argument ml, then ml will be returned. size_t name( \
*/ ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
FORCE_INLINE_TEMPLATE const void* src, size_t srcSize) \
size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nbAttempts, { \
const ZSTD_MatchState_t* const dms, return ZSTD_rust_compressBlock_lazy_call( \
const BYTE* const ip, const BYTE* const iLimit, ms, seqStore, rep, src, srcSize, method, parserDepth, mode); \
const BYTE* const prefixStart, const U32 curr,
const U32 dictLimit, const size_t ddsIdx) {
const U32 ddsLowestIndex = dms->window.dictLimit;
const BYTE* const ddsBase = dms->window.base;
const BYTE* const ddsEnd = dms->window.nextSrc;
const U32 ddsSize = (U32)(ddsEnd - ddsBase);
const U32 ddsIndexDelta = dictLimit - ddsSize;
const U32 bucketSize = (1 << ZSTD_LAZY_DDSS_BUCKET_LOG);
const U32 bucketLimit = nbAttempts < bucketSize - 1 ? nbAttempts : bucketSize - 1;
U32 ddsAttempt;
U32 matchIndex;
for (ddsAttempt = 0; ddsAttempt < bucketSize - 1; ddsAttempt++) {
PREFETCH_L1(ddsBase + dms->hashTable[ddsIdx + ddsAttempt]);
} }
{
U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1];
U32 const chainIndex = chainPackedPointer >> 8;
PREFETCH_L1(&dms->chainTable[chainIndex]);
}
for (ddsAttempt = 0; ddsAttempt < bucketLimit; ddsAttempt++) {
size_t currentMl=0;
const BYTE* match;
matchIndex = dms->hashTable[ddsIdx + ddsAttempt];
match = ddsBase + matchIndex;
if (!matchIndex) {
return ml;
}
/* guaranteed by table construction */
(void)ddsLowestIndex;
assert(matchIndex >= ddsLowestIndex);
assert(match+4 <= ddsEnd);
if (MEM_read32(match) == MEM_read32(ip)) {
/* assumption : matchIndex <= dictLimit-4 (by table construction) */
currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4;
}
/* save best solution */
if (currentMl > ml) {
ml = currentMl;
*offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta));
if (ip+currentMl == iLimit) {
/* best possible, avoids read overflow on next attempt */
return ml;
}
}
}
{
U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1];
U32 chainIndex = chainPackedPointer >> 8;
U32 const chainLength = chainPackedPointer & 0xFF;
U32 const chainAttempts = nbAttempts - ddsAttempt;
U32 const chainLimit = chainAttempts > chainLength ? chainLength : chainAttempts;
U32 chainAttempt;
for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++) {
PREFETCH_L1(ddsBase + dms->chainTable[chainIndex + chainAttempt]);
}
for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++, chainIndex++) {
size_t currentMl=0;
const BYTE* match;
matchIndex = dms->chainTable[chainIndex];
match = ddsBase + matchIndex;
/* guaranteed by table construction */
assert(matchIndex >= ddsLowestIndex);
assert(match+4 <= ddsEnd);
if (MEM_read32(match) == MEM_read32(ip)) {
/* assumption : matchIndex <= dictLimit-4 (by table construction) */
currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4;
}
/* save best solution */
if (currentMl > ml) {
ml = currentMl;
*offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta));
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
}
}
}
return ml;
}
/* *********************************
* Hash Chain
***********************************/
#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)]
/* Update chains up to ip (excluded)
Assumption : always within prefix (i.e. not within extDict) */
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
U32 ZSTD_insertAndFindFirstIndex_internal(
ZSTD_MatchState_t* ms,
const ZSTD_compressionParameters* const cParams,
const BYTE* ip, U32 const mls, U32 const lazySkipping)
{
U32* const hashTable = ms->hashTable;
const U32 hashLog = cParams->hashLog;
U32* const chainTable = ms->chainTable;
const U32 chainMask = (1 << cParams->chainLog) - 1;
const BYTE* const base = ms->window.base;
const U32 target = (U32)(ip - base);
U32 idx = ms->nextToUpdate;
while(idx < target) { /* catch up */
size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls);
NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
hashTable[h] = idx;
idx++;
/* Stop inserting every position when in the lazy skipping mode. */
if (lazySkipping)
break;
}
ms->nextToUpdate = target;
return hashTable[ZSTD_hashPtr(ip, hashLog, mls)];
}
U32 ZSTD_insertAndFindFirstIndex(ZSTD_MatchState_t* ms, const BYTE* ip) {
const ZSTD_compressionParameters* const cParams = &ms->cParams;
return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch, /* lazySkipping*/ 0);
}
/* inlining is important to hardwire a hot branch (template emulation) */
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_HcFindBestMatch(
ZSTD_MatchState_t* ms,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 mls, const ZSTD_dictMode_e dictMode)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const chainTable = ms->chainTable;
const U32 chainSize = (1 << cParams->chainLog);
const U32 chainMask = chainSize-1;
const BYTE* const base = ms->window.base;
const BYTE* const dictBase = ms->window.dictBase;
const U32 dictLimit = ms->window.dictLimit;
const BYTE* const prefixStart = base + dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const U32 curr = (U32)(ip-base);
const U32 maxDistance = 1U << cParams->windowLog;
const U32 lowestValid = ms->window.lowLimit;
const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid;
const U32 isDictionary = (ms->loadedDictEnd != 0);
const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance;
const U32 minChain = curr > chainSize ? curr - chainSize : 0;
U32 nbAttempts = 1U << cParams->searchLog;
size_t ml=4-1;
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
const U32 ddsHashLog = dictMode == ZSTD_dedicatedDictSearch
? dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG : 0;
const size_t ddsIdx = dictMode == ZSTD_dedicatedDictSearch
? ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG : 0;
U32 matchIndex;
if (dictMode == ZSTD_dedicatedDictSearch) {
const U32* entry = &dms->hashTable[ddsIdx];
PREFETCH_L1(entry);
}
/* HC4 match finder */
matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls, ms->lazySkipping);
for ( ; (matchIndex>=lowLimit) & (nbAttempts>0) ; nbAttempts--) {
size_t currentMl=0;
if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
const BYTE* const match = base + matchIndex;
assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */
/* read 4B starting from (match + ml + 1 - sizeof(U32)) */
if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */
currentMl = ZSTD_count(ip, match, iLimit);
} else {
const BYTE* const match = dictBase + matchIndex;
assert(match+4 <= dictEnd);
if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */
currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4;
}
/* save best solution */
if (currentMl > ml) {
ml = currentMl;
*offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex);
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
}
if (matchIndex <= minChain) break;
matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask);
}
assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */
if (dictMode == ZSTD_dedicatedDictSearch) {
ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts, dms,
ip, iLimit, prefixStart, curr, dictLimit, ddsIdx);
} else if (dictMode == ZSTD_dictMatchState) {
const U32* const dmsChainTable = dms->chainTable;
const U32 dmsChainSize = (1 << dms->cParams.chainLog);
const U32 dmsChainMask = dmsChainSize - 1;
const U32 dmsLowestIndex = dms->window.dictLimit;
const BYTE* const dmsBase = dms->window.base;
const BYTE* const dmsEnd = dms->window.nextSrc;
const U32 dmsSize = (U32)(dmsEnd - dmsBase);
const U32 dmsIndexDelta = dictLimit - dmsSize;
const U32 dmsMinChain = dmsSize > dmsChainSize ? dmsSize - dmsChainSize : 0;
matchIndex = dms->hashTable[ZSTD_hashPtr(ip, dms->cParams.hashLog, mls)];
for ( ; (matchIndex>=dmsLowestIndex) & (nbAttempts>0) ; nbAttempts--) {
size_t currentMl=0;
const BYTE* const match = dmsBase + matchIndex;
assert(match+4 <= dmsEnd);
if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */
currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4;
/* save best solution */
if (currentMl > ml) {
ml = currentMl;
assert(curr > matchIndex + dmsIndexDelta);
*offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta));
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
}
if (matchIndex <= dmsMinChain) break;
matchIndex = dmsChainTable[matchIndex & dmsChainMask];
}
}
return ml;
}
/* *********************************
* (SIMD) Row-based matchfinder
***********************************/
/* Constants for row-based hash */
#define ZSTD_ROW_HASH_TAG_MASK ((1u << ZSTD_ROW_HASH_TAG_BITS) - 1)
#define ZSTD_ROW_HASH_MAX_ENTRIES 64 /* absolute maximum number of entries per row, for all configurations */
#define ZSTD_ROW_HASH_CACHE_MASK (ZSTD_ROW_HASH_CACHE_SIZE - 1)
typedef U64 ZSTD_VecMask; /* Clarifies when we are interacting with a U64 representing a mask of matches */
/* ZSTD_VecMask_next():
* Starting from the LSB, returns the idx of the next non-zero bit.
* Basically counting the nb of trailing zeroes.
*/
MEM_STATIC U32 ZSTD_VecMask_next(ZSTD_VecMask val) {
return ZSTD_countTrailingZeros64(val);
}
/* ZSTD_row_nextIndex():
* Returns the next index to insert at within a tagTable row, and updates the "head"
* value to reflect the update. Essentially cycles backwards from [1, {entries per row})
*/
FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextIndex(BYTE* const tagRow, U32 const rowMask) {
U32 next = (*tagRow-1) & rowMask;
next += (next == 0) ? rowMask : 0; /* skip first position */
*tagRow = (BYTE)next;
return next;
}
/* ZSTD_isAligned():
* Checks that a pointer is aligned to "align" bytes which must be a power of 2.
*/
MEM_STATIC int ZSTD_isAligned(void const* ptr, size_t align) {
assert((align & (align - 1)) == 0);
return (((size_t)ptr) & (align - 1)) == 0;
}
/* ZSTD_row_prefetch():
* Performs prefetching for the hashTable and tagTable at a given row.
*/
FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, BYTE const* tagTable, U32 const relRow, U32 const rowLog) {
PREFETCH_L1(hashTable + relRow);
if (rowLog >= 5) {
PREFETCH_L1(hashTable + relRow + 16);
/* Note: prefetching more of the hash table does not appear to be beneficial for 128-entry rows */
}
PREFETCH_L1(tagTable + relRow);
if (rowLog == 6) {
PREFETCH_L1(tagTable + relRow + 32);
}
assert(rowLog == 4 || rowLog == 5 || rowLog == 6);
assert(ZSTD_isAligned(hashTable + relRow, 64)); /* prefetched hash row always 64-byte aligned */
assert(ZSTD_isAligned(tagTable + relRow, (size_t)1 << rowLog)); /* prefetched tagRow sits on correct multiple of bytes (32,64,128) */
}
/* ZSTD_row_fillHashCache():
* Fill up the hash cache starting at idx, prefetching up to ZSTD_ROW_HASH_CACHE_SIZE entries,
* but not beyond iLimit.
*/
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_row_fillHashCache(ZSTD_MatchState_t* ms, const BYTE* base,
U32 const rowLog, U32 const mls,
U32 idx, const BYTE* const iLimit)
{
U32 const* const hashTable = ms->hashTable;
BYTE const* const tagTable = ms->tagTable;
U32 const hashLog = ms->rowHashLog;
U32 const maxElemsToPrefetch = (base + idx) > iLimit ? 0 : (U32)(iLimit - (base + idx) + 1);
U32 const lim = idx + MIN(ZSTD_ROW_HASH_CACHE_SIZE, maxElemsToPrefetch);
for (; idx < lim; ++idx) {
U32 const hash = (U32)ZSTD_hashPtrSalted(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt);
U32 const row = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
ZSTD_row_prefetch(hashTable, tagTable, row, rowLog);
ms->hashCache[idx & ZSTD_ROW_HASH_CACHE_MASK] = hash;
}
DEBUGLOG(6, "ZSTD_row_fillHashCache(): [%u %u %u %u %u %u %u %u]", ms->hashCache[0], ms->hashCache[1],
ms->hashCache[2], ms->hashCache[3], ms->hashCache[4],
ms->hashCache[5], ms->hashCache[6], ms->hashCache[7]);
}
/* ZSTD_row_nextCachedHash():
* Returns the hash of base + idx, and replaces the hash in the hash cache with the byte at
* base + idx + ZSTD_ROW_HASH_CACHE_SIZE. Also prefetches the appropriate rows from hashTable and tagTable.
*/
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
U32 ZSTD_row_nextCachedHash(U32* cache, U32 const* hashTable,
BYTE const* tagTable, BYTE const* base,
U32 idx, U32 const hashLog,
U32 const rowLog, U32 const mls,
U64 const hashSalt)
{
U32 const newHash = (U32)ZSTD_hashPtrSalted(base+idx+ZSTD_ROW_HASH_CACHE_SIZE, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt);
U32 const row = (newHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
ZSTD_row_prefetch(hashTable, tagTable, row, rowLog);
{ U32 const hash = cache[idx & ZSTD_ROW_HASH_CACHE_MASK];
cache[idx & ZSTD_ROW_HASH_CACHE_MASK] = newHash;
return hash;
}
}
/* ZSTD_row_update_internalImpl():
* Updates the hash table with positions starting from updateStartIdx until updateEndIdx.
*/
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_row_update_internalImpl(ZSTD_MatchState_t* ms,
U32 updateStartIdx, U32 const updateEndIdx,
U32 const mls, U32 const rowLog,
U32 const rowMask, U32 const useCache)
{
U32* const hashTable = ms->hashTable;
BYTE* const tagTable = ms->tagTable;
U32 const hashLog = ms->rowHashLog;
const BYTE* const base = ms->window.base;
DEBUGLOG(6, "ZSTD_row_update_internalImpl(): updateStartIdx=%u, updateEndIdx=%u", updateStartIdx, updateEndIdx);
for (; updateStartIdx < updateEndIdx; ++updateStartIdx) {
U32 const hash = useCache ? ZSTD_row_nextCachedHash(ms->hashCache, hashTable, tagTable, base, updateStartIdx, hashLog, rowLog, mls, ms->hashSalt)
: (U32)ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt);
U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
U32* const row = hashTable + relRow;
BYTE* tagRow = tagTable + relRow;
U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask);
assert(hash == ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt));
tagRow[pos] = hash & ZSTD_ROW_HASH_TAG_MASK;
row[pos] = updateStartIdx;
}
}
/* ZSTD_row_update_internal():
* Inserts the byte at ip into the appropriate position in the hash table, and updates ms->nextToUpdate.
* Skips sections of long matches as is necessary.
*/
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_row_update_internal(ZSTD_MatchState_t* ms, const BYTE* ip,
U32 const mls, U32 const rowLog,
U32 const rowMask, U32 const useCache)
{
U32 idx = ms->nextToUpdate;
const BYTE* const base = ms->window.base;
const U32 target = (U32)(ip - base);
const U32 kSkipThreshold = 384;
const U32 kMaxMatchStartPositionsToUpdate = 96;
const U32 kMaxMatchEndPositionsToUpdate = 32;
if (useCache) {
/* Only skip positions when using hash cache, i.e.
* if we are loading a dict, don't skip anything.
* If we decide to skip, then we only update a set number
* of positions at the beginning and end of the match.
*/
if (UNLIKELY(target - idx > kSkipThreshold)) {
U32 const bound = idx + kMaxMatchStartPositionsToUpdate;
ZSTD_row_update_internalImpl(ms, idx, bound, mls, rowLog, rowMask, useCache);
idx = target - kMaxMatchEndPositionsToUpdate;
ZSTD_row_fillHashCache(ms, base, rowLog, mls, idx, ip+1);
}
}
assert(target >= idx);
ZSTD_row_update_internalImpl(ms, idx, target, mls, rowLog, rowMask, useCache);
ms->nextToUpdate = target;
}
/* ZSTD_row_update():
* External wrapper for ZSTD_row_update_internal(). Used for filling the hashtable during dictionary
* processing.
*/
void ZSTD_row_update(ZSTD_MatchState_t* const ms, const BYTE* ip) {
const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6);
const U32 rowMask = (1u << rowLog) - 1;
const U32 mls = MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */);
DEBUGLOG(5, "ZSTD_row_update(), rowLog=%u", rowLog);
ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 0 /* don't use cache */);
}
/* Returns the mask width of bits group of which will be set to 1. Given not all
* architectures have easy movemask instruction, this helps to iterate over
* groups of bits easier and faster.
*/
FORCE_INLINE_TEMPLATE U32
ZSTD_row_matchMaskGroupWidth(const U32 rowEntries)
{
assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64);
assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES);
(void)rowEntries;
#if defined(ZSTD_ARCH_ARM_NEON)
/* NEON path only works for little endian */
if (!MEM_isLittleEndian()) {
return 1;
}
if (rowEntries == 16) {
return 4;
}
if (rowEntries == 32) {
return 2;
}
if (rowEntries == 64) {
return 1;
}
#endif
return 1;
}
#if defined(ZSTD_ARCH_X86_SSE2)
FORCE_INLINE_TEMPLATE ZSTD_VecMask
ZSTD_row_getSSEMask(int nbChunks, const BYTE* const src, const BYTE tag, const U32 head)
{
const __m128i comparisonMask = _mm_set1_epi8((char)tag);
int matches[4] = {0};
int i;
assert(nbChunks == 1 || nbChunks == 2 || nbChunks == 4);
for (i=0; i<nbChunks; i++) {
const __m128i chunk = _mm_loadu_si128((const __m128i*)(const void*)(src + 16*i));
const __m128i equalMask = _mm_cmpeq_epi8(chunk, comparisonMask);
matches[i] = _mm_movemask_epi8(equalMask);
}
if (nbChunks == 1) return ZSTD_rotateRight_U16((U16)matches[0], head);
if (nbChunks == 2) return ZSTD_rotateRight_U32((U32)matches[1] << 16 | (U32)matches[0], head);
assert(nbChunks == 4);
return ZSTD_rotateRight_U64((U64)matches[3] << 48 | (U64)matches[2] << 32 | (U64)matches[1] << 16 | (U64)matches[0], head);
}
#endif
#if defined(ZSTD_ARCH_ARM_NEON)
FORCE_INLINE_TEMPLATE ZSTD_VecMask
ZSTD_row_getNEONMask(const U32 rowEntries, const BYTE* const src, const BYTE tag, const U32 headGrouped)
{
assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64);
if (rowEntries == 16) {
/* vshrn_n_u16 shifts by 4 every u16 and narrows to 8 lower bits.
* After that groups of 4 bits represent the equalMask. We lower
* all bits except the highest in these groups by doing AND with
* 0x88 = 0b10001000.
*/
const uint8x16_t chunk = vld1q_u8(src);
const uint16x8_t equalMask = vreinterpretq_u16_u8(vceqq_u8(chunk, vdupq_n_u8(tag)));
const uint8x8_t res = vshrn_n_u16(equalMask, 4);
const U64 matches = vget_lane_u64(vreinterpret_u64_u8(res), 0);
return ZSTD_rotateRight_U64(matches, headGrouped) & 0x8888888888888888ull;
} else if (rowEntries == 32) {
/* Same idea as with rowEntries == 16 but doing AND with
* 0x55 = 0b01010101.
*/
const uint16x8x2_t chunk = vld2q_u16((const uint16_t*)(const void*)src);
const uint8x16_t chunk0 = vreinterpretq_u8_u16(chunk.val[0]);
const uint8x16_t chunk1 = vreinterpretq_u8_u16(chunk.val[1]);
const uint8x16_t dup = vdupq_n_u8(tag);
const uint8x8_t t0 = vshrn_n_u16(vreinterpretq_u16_u8(vceqq_u8(chunk0, dup)), 6);
const uint8x8_t t1 = vshrn_n_u16(vreinterpretq_u16_u8(vceqq_u8(chunk1, dup)), 6);
const uint8x8_t res = vsli_n_u8(t0, t1, 4);
const U64 matches = vget_lane_u64(vreinterpret_u64_u8(res), 0) ;
return ZSTD_rotateRight_U64(matches, headGrouped) & 0x5555555555555555ull;
} else { /* rowEntries == 64 */
const uint8x16x4_t chunk = vld4q_u8(src);
const uint8x16_t dup = vdupq_n_u8(tag);
const uint8x16_t cmp0 = vceqq_u8(chunk.val[0], dup);
const uint8x16_t cmp1 = vceqq_u8(chunk.val[1], dup);
const uint8x16_t cmp2 = vceqq_u8(chunk.val[2], dup);
const uint8x16_t cmp3 = vceqq_u8(chunk.val[3], dup);
const uint8x16_t t0 = vsriq_n_u8(cmp1, cmp0, 1);
const uint8x16_t t1 = vsriq_n_u8(cmp3, cmp2, 1);
const uint8x16_t t2 = vsriq_n_u8(t1, t0, 2);
const uint8x16_t t3 = vsriq_n_u8(t2, t2, 4);
const uint8x8_t t4 = vshrn_n_u16(vreinterpretq_u16_u8(t3), 4);
const U64 matches = vget_lane_u64(vreinterpret_u64_u8(t4), 0);
return ZSTD_rotateRight_U64(matches, headGrouped);
}
}
#endif
/* Returns a ZSTD_VecMask (U64) that has the nth group (determined by
* ZSTD_row_matchMaskGroupWidth) of bits set to 1 if the newly-computed "tag"
* matches the hash at the nth position in a row of the tagTable.
* Each row is a circular buffer beginning at the value of "headGrouped". So we
* must rotate the "matches" bitfield to match up with the actual layout of the
* entries within the hashTable */
FORCE_INLINE_TEMPLATE ZSTD_VecMask
ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGrouped, const U32 rowEntries)
{
const BYTE* const src = tagRow;
assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64);
assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES);
assert(ZSTD_row_matchMaskGroupWidth(rowEntries) * rowEntries <= sizeof(ZSTD_VecMask) * 8);
#if defined(ZSTD_ARCH_X86_SSE2)
return ZSTD_row_getSSEMask(rowEntries / 16, src, tag, headGrouped);
#else /* SW or NEON-LE */
# if defined(ZSTD_ARCH_ARM_NEON)
/* This NEON path only works for little endian - otherwise use SWAR below */
if (MEM_isLittleEndian()) {
return ZSTD_row_getNEONMask(rowEntries, src, tag, headGrouped);
}
# endif /* ZSTD_ARCH_ARM_NEON */
/* SWAR */
{ const int chunkSize = sizeof(size_t);
const size_t shiftAmount = ((chunkSize * 8) - chunkSize);
const size_t xFF = ~((size_t)0);
const size_t x01 = xFF / 0xFF;
const size_t x80 = x01 << 7;
const size_t splatChar = tag * x01;
ZSTD_VecMask matches = 0;
int i = rowEntries - chunkSize;
assert((sizeof(size_t) == 4) || (sizeof(size_t) == 8));
if (MEM_isLittleEndian()) { /* runtime check so have two loops */
const size_t extractMagic = (xFF / 0x7F) >> chunkSize;
do {
size_t chunk = MEM_readST(&src[i]);
chunk ^= splatChar;
chunk = (((chunk | x80) - x01) | chunk) & x80;
matches <<= chunkSize;
matches |= (chunk * extractMagic) >> shiftAmount;
i -= chunkSize;
} while (i >= 0);
} else { /* big endian: reverse bits during extraction */
const size_t msb = xFF ^ (xFF >> 1);
const size_t extractMagic = (msb / 0x1FF) | msb;
do {
size_t chunk = MEM_readST(&src[i]);
chunk ^= splatChar;
chunk = (((chunk | x80) - x01) | chunk) & x80;
matches <<= chunkSize;
matches |= ((chunk >> 7) * extractMagic) >> shiftAmount;
i -= chunkSize;
} while (i >= 0);
}
matches = ~matches;
if (rowEntries == 16) {
return ZSTD_rotateRight_U16((U16)matches, headGrouped);
} else if (rowEntries == 32) {
return ZSTD_rotateRight_U32((U32)matches, headGrouped);
} else {
return ZSTD_rotateRight_U64((U64)matches, headGrouped);
}
}
#endif
}
/* The high-level approach of the SIMD row based match finder is as follows:
* - Figure out where to insert the new entry:
* - Generate a hash for current input position and split it into a one byte of tag and `rowHashLog` bits of index.
* - The hash is salted by a value that changes on every context reset, so when the same table is used
* we will avoid collisions that would otherwise slow us down by introducing phantom matches.
* - The hashTable is effectively split into groups or "rows" of 15 or 31 entries of U32, and the index determines
* which row to insert into.
* - Determine the correct position within the row to insert the entry into. Each row of 15 or 31 can
* be considered as a circular buffer with a "head" index that resides in the tagTable (overall 16 or 32 bytes
* per row).
* - Use SIMD to efficiently compare the tags in the tagTable to the 1-byte tag calculated for the position and
* generate a bitfield that we can cycle through to check the collisions in the hash table.
* - Pick the longest match.
* - Insert the tag into the equivalent row and position in the tagTable.
*/
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_RowFindBestMatch(
ZSTD_MatchState_t* ms,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 mls, const ZSTD_dictMode_e dictMode,
const U32 rowLog)
{
U32* const hashTable = ms->hashTable;
BYTE* const tagTable = ms->tagTable;
U32* const hashCache = ms->hashCache;
const U32 hashLog = ms->rowHashLog;
const ZSTD_compressionParameters* const cParams = &ms->cParams;
const BYTE* const base = ms->window.base;
const BYTE* const dictBase = ms->window.dictBase;
const U32 dictLimit = ms->window.dictLimit;
const BYTE* const prefixStart = base + dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const U32 curr = (U32)(ip-base);
const U32 maxDistance = 1U << cParams->windowLog;
const U32 lowestValid = ms->window.lowLimit;
const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid;
const U32 isDictionary = (ms->loadedDictEnd != 0);
const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance;
const U32 rowEntries = (1U << rowLog);
const U32 rowMask = rowEntries - 1;
const U32 cappedSearchLog = MIN(cParams->searchLog, rowLog); /* nb of searches is capped at nb entries per row */
const U32 groupWidth = ZSTD_row_matchMaskGroupWidth(rowEntries);
const U64 hashSalt = ms->hashSalt;
U32 nbAttempts = 1U << cappedSearchLog;
size_t ml=4-1;
U32 hash;
/* DMS/DDS variables that may be referenced laster */
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
/* Initialize the following variables to satisfy static analyzer */
size_t ddsIdx = 0;
U32 ddsExtraAttempts = 0; /* cctx hash tables are limited in searches, but allow extra searches into DDS */
U32 dmsTag = 0;
U32* dmsRow = NULL;
BYTE* dmsTagRow = NULL;
if (dictMode == ZSTD_dedicatedDictSearch) {
const U32 ddsHashLog = dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG;
{ /* Prefetch DDS hashtable entry */
ddsIdx = ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG;
PREFETCH_L1(&dms->hashTable[ddsIdx]);
}
ddsExtraAttempts = cParams->searchLog > rowLog ? 1U << (cParams->searchLog - rowLog) : 0;
}
if (dictMode == ZSTD_dictMatchState) {
/* Prefetch DMS rows */
U32* const dmsHashTable = dms->hashTable;
BYTE* const dmsTagTable = dms->tagTable;
U32 const dmsHash = (U32)ZSTD_hashPtr(ip, dms->rowHashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
U32 const dmsRelRow = (dmsHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
dmsTag = dmsHash & ZSTD_ROW_HASH_TAG_MASK;
dmsTagRow = (BYTE*)(dmsTagTable + dmsRelRow);
dmsRow = dmsHashTable + dmsRelRow;
ZSTD_row_prefetch(dmsHashTable, dmsTagTable, dmsRelRow, rowLog);
}
/* Update the hashTable and tagTable up to (but not including) ip */
if (!ms->lazySkipping) {
ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 1 /* useCache */);
hash = ZSTD_row_nextCachedHash(hashCache, hashTable, tagTable, base, curr, hashLog, rowLog, mls, hashSalt);
} else {
/* Stop inserting every position when in the lazy skipping mode.
* The hash cache is also not kept up to date in this mode.
*/
hash = (U32)ZSTD_hashPtrSalted(ip, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt);
ms->nextToUpdate = curr;
}
ms->hashSaltEntropy += hash; /* collect salt entropy */
{ /* Get the hash for ip, compute the appropriate row */
U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
U32 const tag = hash & ZSTD_ROW_HASH_TAG_MASK;
U32* const row = hashTable + relRow;
BYTE* tagRow = (BYTE*)(tagTable + relRow);
U32 const headGrouped = (*tagRow & rowMask) * groupWidth;
U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES];
size_t numMatches = 0;
size_t currMatch = 0;
ZSTD_VecMask matches = ZSTD_row_getMatchMask(tagRow, (BYTE)tag, headGrouped, rowEntries);
/* Cycle through the matches and prefetch */
for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) {
U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask;
U32 const matchIndex = row[matchPos];
if(matchPos == 0) continue;
assert(numMatches < rowEntries);
if (matchIndex < lowLimit)
break;
if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
PREFETCH_L1(base + matchIndex);
} else {
PREFETCH_L1(dictBase + matchIndex);
}
matchBuffer[numMatches++] = matchIndex;
--nbAttempts;
}
/* Speed opt: insert current byte into hashtable too. This allows us to avoid one iteration of the loop
in ZSTD_row_update_internal() at the next search. */
{
U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask);
tagRow[pos] = (BYTE)tag;
row[pos] = ms->nextToUpdate++;
}
/* Return the longest match */
for (; currMatch < numMatches; ++currMatch) {
U32 const matchIndex = matchBuffer[currMatch];
size_t currentMl=0;
assert(matchIndex < curr);
assert(matchIndex >= lowLimit);
if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
const BYTE* const match = base + matchIndex;
assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */
/* read 4B starting from (match + ml + 1 - sizeof(U32)) */
if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */
currentMl = ZSTD_count(ip, match, iLimit);
} else {
const BYTE* const match = dictBase + matchIndex;
assert(match+4 <= dictEnd);
if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */
currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4;
}
/* Save best solution */
if (currentMl > ml) {
ml = currentMl;
*offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex);
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
}
}
}
assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */
if (dictMode == ZSTD_dedicatedDictSearch) {
ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts + ddsExtraAttempts, dms,
ip, iLimit, prefixStart, curr, dictLimit, ddsIdx);
} else if (dictMode == ZSTD_dictMatchState) {
/* TODO: Measure and potentially add prefetching to DMS */
const U32 dmsLowestIndex = dms->window.dictLimit;
const BYTE* const dmsBase = dms->window.base;
const BYTE* const dmsEnd = dms->window.nextSrc;
const U32 dmsSize = (U32)(dmsEnd - dmsBase);
const U32 dmsIndexDelta = dictLimit - dmsSize;
{ U32 const headGrouped = (*dmsTagRow & rowMask) * groupWidth;
U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES];
size_t numMatches = 0;
size_t currMatch = 0;
ZSTD_VecMask matches = ZSTD_row_getMatchMask(dmsTagRow, (BYTE)dmsTag, headGrouped, rowEntries);
for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) {
U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask;
U32 const matchIndex = dmsRow[matchPos];
if(matchPos == 0) continue;
if (matchIndex < dmsLowestIndex)
break;
PREFETCH_L1(dmsBase + matchIndex);
matchBuffer[numMatches++] = matchIndex;
--nbAttempts;
}
/* Return the longest match */
for (; currMatch < numMatches; ++currMatch) {
U32 const matchIndex = matchBuffer[currMatch];
size_t currentMl=0;
assert(matchIndex >= dmsLowestIndex);
assert(matchIndex < curr);
{ const BYTE* const match = dmsBase + matchIndex;
assert(match+4 <= dmsEnd);
if (MEM_read32(match) == MEM_read32(ip))
currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4;
}
if (currentMl > ml) {
ml = currentMl;
assert(curr > matchIndex + dmsIndexDelta);
*offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta));
if (ip+currentMl == iLimit) break;
}
}
}
}
return ml;
}
/**
* Generate search functions templated on (dictMode, mls, rowLog).
* These functions are outlined for code size & compilation time.
* ZSTD_searchMax() dispatches to the correct implementation function.
*
* TODO: The start of the search function involves loading and calculating a
* bunch of constants from the ZSTD_MatchState_t. These computations could be
* done in an initialization function, and saved somewhere in the match state.
* Then we could pass a pointer to the saved state instead of the match state,
* and avoid duplicate computations.
*
* TODO: Move the match re-winding into searchMax. This improves compression
* ratio, and unlocks further simplifications with the next TODO.
*
* TODO: Try moving the repcode search into searchMax. After the re-winding
* and repcode search are in searchMax, there is no more logic in the match
* finder loop that requires knowledge about the dictMode. So we should be
* able to avoid force inlining it, and we can join the extDict loop with
* the single segment loop. It should go in searchMax instead of its own
* function to avoid having multiple virtual function calls per search.
*/
#define ZSTD_BT_SEARCH_FN(dictMode, mls) ZSTD_BtFindBestMatch_##dictMode##_##mls
#define ZSTD_HC_SEARCH_FN(dictMode, mls) ZSTD_HcFindBestMatch_##dictMode##_##mls
#define ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) ZSTD_RowFindBestMatch_##dictMode##_##mls##_##rowLog
#define ZSTD_SEARCH_FN_ATTRS FORCE_NOINLINE
#define GEN_ZSTD_BT_SEARCH_FN(dictMode, mls) \
ZSTD_SEARCH_FN_ATTRS size_t ZSTD_BT_SEARCH_FN(dictMode, mls)( \
ZSTD_MatchState_t* ms, \
const BYTE* ip, const BYTE* const iLimit, \
size_t* offBasePtr) \
{ \
assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \
return ZSTD_BtFindBestMatch(ms, ip, iLimit, offBasePtr, mls, ZSTD_##dictMode); \
} \
#define GEN_ZSTD_HC_SEARCH_FN(dictMode, mls) \
ZSTD_SEARCH_FN_ATTRS size_t ZSTD_HC_SEARCH_FN(dictMode, mls)( \
ZSTD_MatchState_t* ms, \
const BYTE* ip, const BYTE* const iLimit, \
size_t* offsetPtr) \
{ \
assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \
return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \
} \
#define GEN_ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) \
ZSTD_SEARCH_FN_ATTRS size_t ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)( \
ZSTD_MatchState_t* ms, \
const BYTE* ip, const BYTE* const iLimit, \
size_t* offsetPtr) \
{ \
assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \
assert(MAX(4, MIN(6, ms->cParams.searchLog)) == rowLog); \
return ZSTD_RowFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode, rowLog); \
} \
#define ZSTD_FOR_EACH_ROWLOG(X, dictMode, mls) \
X(dictMode, mls, 4) \
X(dictMode, mls, 5) \
X(dictMode, mls, 6)
#define ZSTD_FOR_EACH_MLS_ROWLOG(X, dictMode) \
ZSTD_FOR_EACH_ROWLOG(X, dictMode, 4) \
ZSTD_FOR_EACH_ROWLOG(X, dictMode, 5) \
ZSTD_FOR_EACH_ROWLOG(X, dictMode, 6)
#define ZSTD_FOR_EACH_MLS(X, dictMode) \
X(dictMode, 4) \
X(dictMode, 5) \
X(dictMode, 6)
#define ZSTD_FOR_EACH_DICT_MODE(X, ...) \
X(__VA_ARGS__, noDict) \
X(__VA_ARGS__, extDict) \
X(__VA_ARGS__, dictMatchState) \
X(__VA_ARGS__, dedicatedDictSearch)
/* Generate row search fns for each combination of (dictMode, mls, rowLog) */
ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS_ROWLOG, GEN_ZSTD_ROW_SEARCH_FN)
/* Generate binary Tree search fns for each combination of (dictMode, mls) */
ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_SEARCH_FN)
/* Generate hash chain search fns for each combination of (dictMode, mls) */
ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_SEARCH_FN)
typedef enum { search_hashChain=0, search_binaryTree=1, search_rowHash=2 } searchMethod_e;
#define GEN_ZSTD_CALL_BT_SEARCH_FN(dictMode, mls) \
case mls: \
return ZSTD_BT_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr);
#define GEN_ZSTD_CALL_HC_SEARCH_FN(dictMode, mls) \
case mls: \
return ZSTD_HC_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr);
#define GEN_ZSTD_CALL_ROW_SEARCH_FN(dictMode, mls, rowLog) \
case rowLog: \
return ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)(ms, ip, iend, offsetPtr);
#define ZSTD_SWITCH_MLS(X, dictMode) \
switch (mls) { \
ZSTD_FOR_EACH_MLS(X, dictMode) \
}
#define ZSTD_SWITCH_ROWLOG(dictMode, mls) \
case mls: \
switch (rowLog) { \
ZSTD_FOR_EACH_ROWLOG(GEN_ZSTD_CALL_ROW_SEARCH_FN, dictMode, mls) \
} \
ZSTD_UNREACHABLE; \
break;
#define ZSTD_SWITCH_SEARCH_METHOD(dictMode) \
switch (searchMethod) { \
case search_hashChain: \
ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_HC_SEARCH_FN, dictMode) \
break; \
case search_binaryTree: \
ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_BT_SEARCH_FN, dictMode) \
break; \
case search_rowHash: \
ZSTD_SWITCH_MLS(ZSTD_SWITCH_ROWLOG, dictMode) \
break; \
} \
ZSTD_UNREACHABLE;
/**
* Searches for the longest match at @p ip.
* Dispatches to the correct implementation function based on the
* (searchMethod, dictMode, mls, rowLog). We use switch statements
* here instead of using an indirect function call through a function
* pointer because after Spectre and Meltdown mitigations, indirect
* function calls can be very costly, especially in the kernel.
*
* NOTE: dictMode and searchMethod should be templated, so those switch
* statements should be optimized out. Only the mls & rowLog switches
* should be left.
*
* @param ms The match state.
* @param ip The position to search at.
* @param iend The end of the input data.
* @param[out] offsetPtr Stores the match offset into this pointer.
* @param mls The minimum search length, in the range [4, 6].
* @param rowLog The row log (if applicable), in the range [4, 6].
* @param searchMethod The search method to use (templated).
* @param dictMode The dictMode (templated).
*
* @returns The length of the longest match found, or < mls if no match is found.
* If a match is found its offset is stored in @p offsetPtr.
*/
FORCE_INLINE_TEMPLATE size_t ZSTD_searchMax(
ZSTD_MatchState_t* ms,
const BYTE* ip,
const BYTE* iend,
size_t* offsetPtr,
U32 const mls,
U32 const rowLog,
searchMethod_e const searchMethod,
ZSTD_dictMode_e const dictMode)
{
if (dictMode == ZSTD_noDict) {
ZSTD_SWITCH_SEARCH_METHOD(noDict)
} else if (dictMode == ZSTD_extDict) {
ZSTD_SWITCH_SEARCH_METHOD(extDict)
} else if (dictMode == ZSTD_dictMatchState) {
ZSTD_SWITCH_SEARCH_METHOD(dictMatchState)
} else if (dictMode == ZSTD_dedicatedDictSearch) {
ZSTD_SWITCH_SEARCH_METHOD(dedicatedDictSearch)
}
ZSTD_UNREACHABLE;
return 0;
}
/* *******************************
* Common parser - lazy strategy
*********************************/
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_lazy_generic(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize,
const searchMethod_e searchMethod, const U32 depth,
ZSTD_dictMode_e const dictMode)
{
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = (searchMethod == search_rowHash) ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8;
const BYTE* const base = ms->window.base;
const U32 prefixLowestIndex = ms->window.dictLimit;
const BYTE* const prefixLowest = base + prefixLowestIndex;
const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6);
const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6);
U32 offset_1 = rep[0], offset_2 = rep[1];
U32 offsetSaved1 = 0, offsetSaved2 = 0;
const int isDMS = dictMode == ZSTD_dictMatchState;
const int isDDS = dictMode == ZSTD_dedicatedDictSearch;
const int isDxS = isDMS || isDDS;
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
const U32 dictLowestIndex = isDxS ? dms->window.dictLimit : 0;
const BYTE* const dictBase = isDxS ? dms->window.base : NULL;
const BYTE* const dictLowest = isDxS ? dictBase + dictLowestIndex : NULL;
const BYTE* const dictEnd = isDxS ? dms->window.nextSrc : NULL;
const U32 dictIndexDelta = isDxS ?
prefixLowestIndex - (U32)(dictEnd - dictBase) :
0;
const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest));
DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u) (searchFunc=%u)", (U32)dictMode, (U32)searchMethod);
ip += (dictAndPrefixLength == 0);
if (dictMode == ZSTD_noDict) {
U32 const curr = (U32)(ip - base);
U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, ms->cParams.windowLog);
U32 const maxRep = curr - windowLow;
if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0;
if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0;
}
if (isDxS) {
/* dictMatchState repCode checks don't currently handle repCode == 0
* disabling. */
assert(offset_1 <= dictAndPrefixLength);
assert(offset_2 <= dictAndPrefixLength);
}
/* Reset the lazy skipping state */
ms->lazySkipping = 0;
if (searchMethod == search_rowHash) {
ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit);
}
/* Match Loop */
#if defined(__GNUC__) && defined(__x86_64__)
/* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the
* code alignment is perturbed. To fix the instability align the loop on 32-bytes.
*/
__asm__(".p2align 5");
#endif
while (ip < ilimit) {
size_t matchLength=0;
size_t offBase = REPCODE1_TO_OFFBASE;
const BYTE* start=ip+1;
DEBUGLOG(7, "search baseline (depth 0)");
/* check repCode */
if (isDxS) {
const U32 repIndex = (U32)(ip - base) + 1 - offset_1;
const BYTE* repMatch = ((dictMode == ZSTD_dictMatchState || dictMode == ZSTD_dedicatedDictSearch)
&& repIndex < prefixLowestIndex) ?
dictBase + (repIndex - dictIndexDelta) :
base + repIndex;
if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
if (depth==0) goto _storeSequence;
}
}
if ( dictMode == ZSTD_noDict
&& ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) {
matchLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
if (depth==0) goto _storeSequence;
}
/* first search (depth 0) */
{ size_t offbaseFound = 999999999;
size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &offbaseFound, mls, rowLog, searchMethod, dictMode);
if (ml2 > matchLength)
matchLength = ml2, start = ip, offBase = offbaseFound;
}
if (matchLength < 4) {
size_t const step = ((size_t)(ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */;
ip += step;
/* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time.
* In this mode we stop inserting every position into our tables, and only insert
* positions that we search, which is one in step positions.
* The exact cutoff is flexible, I've just chosen a number that is reasonably high,
* so we minimize the compression ratio loss in "normal" scenarios. This mode gets
* triggered once we've gone 2KB without finding any matches.
*/
ms->lazySkipping = step > kLazySkippingStep;
continue;
}
/* let's try to find a better solution */
if (depth>=1)
while (ip<ilimit) {
DEBUGLOG(7, "search depth 1");
ip ++;
if ( (dictMode == ZSTD_noDict)
&& (offBase) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) {
size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4;
int const gain2 = (int)(mlRep * 3);
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1);
if ((mlRep >= 4) && (gain2 > gain1))
matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip;
}
if (isDxS) {
const U32 repIndex = (U32)(ip - base) - offset_1;
const BYTE* repMatch = repIndex < prefixLowestIndex ?
dictBase + (repIndex - dictIndexDelta) :
base + repIndex;
if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
&& (MEM_read32(repMatch) == MEM_read32(ip)) ) {
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
int const gain2 = (int)(mlRep * 3);
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1);
if ((mlRep >= 4) && (gain2 > gain1))
matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip;
}
}
{ size_t ofbCandidate=999999999;
size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode);
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4);
if ((ml2 >= 4) && (gain2 > gain1)) {
matchLength = ml2, offBase = ofbCandidate, start = ip;
continue; /* search a better one */
} }
/* let's find an even better one */
if ((depth==2) && (ip<ilimit)) {
DEBUGLOG(7, "search depth 2");
ip ++;
if ( (dictMode == ZSTD_noDict)
&& (offBase) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) {
size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4;
int const gain2 = (int)(mlRep * 4);
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1);
if ((mlRep >= 4) && (gain2 > gain1))
matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip;
}
if (isDxS) {
const U32 repIndex = (U32)(ip - base) - offset_1;
const BYTE* repMatch = repIndex < prefixLowestIndex ?
dictBase + (repIndex - dictIndexDelta) :
base + repIndex;
if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
&& (MEM_read32(repMatch) == MEM_read32(ip)) ) {
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
int const gain2 = (int)(mlRep * 4);
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1);
if ((mlRep >= 4) && (gain2 > gain1))
matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip;
}
}
{ size_t ofbCandidate=999999999;
size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode);
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7);
if ((ml2 >= 4) && (gain2 > gain1)) {
matchLength = ml2, offBase = ofbCandidate, start = ip;
continue;
} } }
break; /* nothing found : store previous solution */
}
/* NOTE:
* Pay attention that `start[-value]` can lead to strange undefined behavior
* notably if `value` is unsigned, resulting in a large positive `-value`.
*/
/* catch up */
if (OFFBASE_IS_OFFSET(offBase)) {
if (dictMode == ZSTD_noDict) {
while ( ((start > anchor) & (start - OFFBASE_TO_OFFSET(offBase) > prefixLowest))
&& (start[-1] == (start-OFFBASE_TO_OFFSET(offBase))[-1]) ) /* only search for offset within prefix */
{ start--; matchLength++; }
}
if (isDxS) {
U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase));
const BYTE* match = (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex;
const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest;
while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */
}
offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase);
}
/* store sequence */
_storeSequence:
{ size_t const litLength = (size_t)(start - anchor);
ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength);
anchor = ip = start + matchLength;
}
if (ms->lazySkipping) {
/* We've found a match, disable lazy skipping mode, and refill the hash cache. */
if (searchMethod == search_rowHash) {
ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit);
}
ms->lazySkipping = 0;
}
/* check immediate repcode */
if (isDxS) {
while (ip <= ilimit) {
U32 const current2 = (U32)(ip-base);
U32 const repIndex = current2 - offset_2;
const BYTE* repMatch = repIndex < prefixLowestIndex ?
dictBase - dictIndexDelta + repIndex :
base + repIndex;
if ( (ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
&& (MEM_read32(repMatch) == MEM_read32(ip)) ) {
const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4;
offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength);
ip += matchLength;
anchor = ip;
continue;
}
break;
}
}
if (dictMode == ZSTD_noDict) {
while ( ((ip <= ilimit) & (offset_2>0))
&& (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) {
/* store sequence */
matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap repcodes */
ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength);
ip += matchLength;
anchor = ip;
continue; /* faster when present ... (?) */
} } }
/* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0),
* rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */
offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2;
/* save reps for next block */
rep[0] = offset_1 ? offset_1 : offsetSaved1;
rep[1] = offset_2 ? offset_2 : offsetSaved2;
/* Return the last literals size */
return (size_t)(iend - anchor);
}
#endif /* build exclusions */
#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR #ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_greedy( ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_greedy,
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_rust_search_hashChain, 0, ZSTD_rust_dict_noDict)
void const* src, size_t srcSize) ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_greedy_dictMatchState,
{ ZSTD_rust_search_hashChain, 0, ZSTD_rust_dict_dictMatchState)
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_noDict); ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_greedy_dedicatedDictSearch,
} ZSTD_rust_search_hashChain, 0, ZSTD_rust_dict_dedicatedDictSearch)
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_greedy_row,
size_t ZSTD_compressBlock_greedy_dictMatchState( ZSTD_rust_search_rowHash, 0, ZSTD_rust_dict_noDict)
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_greedy_dictMatchState_row,
void const* src, size_t srcSize) ZSTD_rust_search_rowHash, 0, ZSTD_rust_dict_dictMatchState)
{ ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_greedy_dedicatedDictSearch_row,
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dictMatchState); ZSTD_rust_search_rowHash, 0, ZSTD_rust_dict_dedicatedDictSearch)
} ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_greedy_extDict,
ZSTD_rust_search_hashChain, 0, ZSTD_rust_dict_extDict)
size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_greedy_extDict_row,
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_rust_search_rowHash, 0, ZSTD_rust_dict_extDict)
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dedicatedDictSearch);
}
size_t ZSTD_compressBlock_greedy_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_noDict);
}
size_t ZSTD_compressBlock_greedy_dictMatchState_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dedicatedDictSearch);
}
#endif #endif
#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR #ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_lazy( ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy,
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_rust_search_hashChain, 1, ZSTD_rust_dict_noDict)
void const* src, size_t srcSize) ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy_dictMatchState,
{ ZSTD_rust_search_hashChain, 1, ZSTD_rust_dict_dictMatchState)
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_noDict); ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy_dedicatedDictSearch,
} ZSTD_rust_search_hashChain, 1, ZSTD_rust_dict_dedicatedDictSearch)
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy_row,
size_t ZSTD_compressBlock_lazy_dictMatchState( ZSTD_rust_search_rowHash, 1, ZSTD_rust_dict_noDict)
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy_dictMatchState_row,
void const* src, size_t srcSize) ZSTD_rust_search_rowHash, 1, ZSTD_rust_dict_dictMatchState)
{ ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy_dedicatedDictSearch_row,
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dictMatchState); ZSTD_rust_search_rowHash, 1, ZSTD_rust_dict_dedicatedDictSearch)
} ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy_extDict,
ZSTD_rust_search_hashChain, 1, ZSTD_rust_dict_extDict)
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy_extDict_row,
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_rust_search_rowHash, 1, ZSTD_rust_dict_extDict)
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dedicatedDictSearch);
}
size_t ZSTD_compressBlock_lazy_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_noDict);
}
size_t ZSTD_compressBlock_lazy_dictMatchState_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dedicatedDictSearch);
}
#endif #endif
#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR #ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_lazy2( ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy2,
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_rust_search_hashChain, 2, ZSTD_rust_dict_noDict)
void const* src, size_t srcSize) ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy2_dictMatchState,
{ ZSTD_rust_search_hashChain, 2, ZSTD_rust_dict_dictMatchState)
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_noDict); ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy2_dedicatedDictSearch,
} ZSTD_rust_search_hashChain, 2, ZSTD_rust_dict_dedicatedDictSearch)
ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy2_row,
size_t ZSTD_compressBlock_lazy2_dictMatchState( ZSTD_rust_search_rowHash, 2, ZSTD_rust_dict_noDict)
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy2_dictMatchState_row,
void const* src, size_t srcSize) ZSTD_rust_search_rowHash, 2, ZSTD_rust_dict_dictMatchState)
{ ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy2_dedicatedDictSearch_row,
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dictMatchState); ZSTD_rust_search_rowHash, 2, ZSTD_rust_dict_dedicatedDictSearch)
} ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy2_extDict,
ZSTD_rust_search_hashChain, 2, ZSTD_rust_dict_extDict)
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_lazy2_extDict_row,
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_rust_search_rowHash, 2, ZSTD_rust_dict_extDict)
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dedicatedDictSearch);
}
size_t ZSTD_compressBlock_lazy2_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_noDict);
}
size_t ZSTD_compressBlock_lazy2_dictMatchState_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dedicatedDictSearch);
}
#endif #endif
#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR #ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btlazy2( ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_btlazy2,
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], ZSTD_rust_search_binaryTree, 2, ZSTD_rust_dict_noDict)
void const* src, size_t srcSize) ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_btlazy2_dictMatchState,
{ ZSTD_rust_search_binaryTree, 2, ZSTD_rust_dict_dictMatchState)
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_noDict); ZSTD_RUST_LAZY_WRAPPER(ZSTD_compressBlock_btlazy2_extDict,
} ZSTD_rust_search_binaryTree, 2, ZSTD_rust_dict_extDict)
size_t ZSTD_compressBlock_btlazy2_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_dictMatchState);
}
#endif #endif
#if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ #undef ZSTD_RUST_LAZY_WRAPPER
|| !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \
|| !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \
|| !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR)
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_lazy_extDict_generic(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize,
const searchMethod_e searchMethod, const U32 depth)
{
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = searchMethod == search_rowHash ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8;
const BYTE* const base = ms->window.base;
const U32 dictLimit = ms->window.dictLimit;
const BYTE* const prefixStart = base + dictLimit;
const BYTE* const dictBase = ms->window.dictBase;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const dictStart = dictBase + ms->window.lowLimit;
const U32 windowLog = ms->cParams.windowLog;
const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6);
const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6);
U32 offset_1 = rep[0], offset_2 = rep[1]; #endif /* lazy matcher build exclusions */
DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic (searchFunc=%u)", (U32)searchMethod);
/* Reset the lazy skipping state */
ms->lazySkipping = 0;
/* init */
ip += (ip == prefixStart);
if (searchMethod == search_rowHash) {
ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit);
}
/* Match Loop */
#if defined(__GNUC__) && defined(__x86_64__)
/* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the
* code alignment is perturbed. To fix the instability align the loop on 32-bytes.
*/
__asm__(".p2align 5");
#endif
while (ip < ilimit) {
size_t matchLength=0;
size_t offBase = REPCODE1_TO_OFFBASE;
const BYTE* start=ip+1;
U32 curr = (U32)(ip-base);
/* check repCode */
{ const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr+1, windowLog);
const U32 repIndex = (U32)(curr+1 - offset_1);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ( (ZSTD_index_overlap_check(dictLimit, repIndex))
& (offset_1 <= curr+1 - windowLow) ) /* note: we are searching at curr+1 */
if (MEM_read32(ip+1) == MEM_read32(repMatch)) {
/* repcode detected we should take it */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repEnd, prefixStart) + 4;
if (depth==0) goto _storeSequence;
} }
/* first search (depth 0) */
{ size_t ofbCandidate = 999999999;
size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict);
if (ml2 > matchLength)
matchLength = ml2, start = ip, offBase = ofbCandidate;
}
if (matchLength < 4) {
size_t const step = ((size_t)(ip-anchor) >> kSearchStrength);
ip += step + 1; /* jump faster over incompressible sections */
/* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time.
* In this mode we stop inserting every position into our tables, and only insert
* positions that we search, which is one in step positions.
* The exact cutoff is flexible, I've just chosen a number that is reasonably high,
* so we minimize the compression ratio loss in "normal" scenarios. This mode gets
* triggered once we've gone 2KB without finding any matches.
*/
ms->lazySkipping = step > kLazySkippingStep;
continue;
}
/* let's try to find a better solution */
if (depth>=1)
while (ip<ilimit) {
ip ++;
curr++;
/* check repCode */
if (offBase) {
const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog);
const U32 repIndex = (U32)(curr - offset_1);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ( (ZSTD_index_overlap_check(dictLimit, repIndex))
& (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
if (MEM_read32(ip) == MEM_read32(repMatch)) {
/* repcode detected */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
int const gain2 = (int)(repLength * 3);
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1);
if ((repLength >= 4) && (gain2 > gain1))
matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip;
} }
/* search match, depth 1 */
{ size_t ofbCandidate = 999999999;
size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict);
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4);
if ((ml2 >= 4) && (gain2 > gain1)) {
matchLength = ml2, offBase = ofbCandidate, start = ip;
continue; /* search a better one */
} }
/* let's find an even better one */
if ((depth==2) && (ip<ilimit)) {
ip ++;
curr++;
/* check repCode */
if (offBase) {
const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog);
const U32 repIndex = (U32)(curr - offset_1);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ( (ZSTD_index_overlap_check(dictLimit, repIndex))
& (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
if (MEM_read32(ip) == MEM_read32(repMatch)) {
/* repcode detected */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
int const gain2 = (int)(repLength * 4);
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1);
if ((repLength >= 4) && (gain2 > gain1))
matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip;
} }
/* search match, depth 2 */
{ size_t ofbCandidate = 999999999;
size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict);
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7);
if ((ml2 >= 4) && (gain2 > gain1)) {
matchLength = ml2, offBase = ofbCandidate, start = ip;
continue;
} } }
break; /* nothing found : store previous solution */
}
/* catch up */
if (OFFBASE_IS_OFFSET(offBase)) {
U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase));
const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */
offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase);
}
/* store sequence */
_storeSequence:
{ size_t const litLength = (size_t)(start - anchor);
ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength);
anchor = ip = start + matchLength;
}
if (ms->lazySkipping) {
/* We've found a match, disable lazy skipping mode, and refill the hash cache. */
if (searchMethod == search_rowHash) {
ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit);
}
ms->lazySkipping = 0;
}
/* check immediate repcode */
while (ip <= ilimit) {
const U32 repCurrent = (U32)(ip-base);
const U32 windowLow = ZSTD_getLowestMatchIndex(ms, repCurrent, windowLog);
const U32 repIndex = repCurrent - offset_2;
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if ( (ZSTD_index_overlap_check(dictLimit, repIndex))
& (offset_2 <= repCurrent - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
if (MEM_read32(ip) == MEM_read32(repMatch)) {
/* repcode detected we should take it */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset history */
ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength);
ip += matchLength;
anchor = ip;
continue; /* faster when present ... (?) */
}
break;
} }
/* Save reps for next block */
rep[0] = offset_1;
rep[1] = offset_2;
/* Return the last literals size */
return (size_t)(iend - anchor);
}
#endif /* build exclusions */
#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_greedy_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0);
}
size_t ZSTD_compressBlock_greedy_extDict_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0);
}
#endif
#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_lazy_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1);
}
size_t ZSTD_compressBlock_lazy_extDict_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1);
}
#endif
#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_lazy2_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2);
}
size_t ZSTD_compressBlock_lazy2_extDict_row(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2);
}
#endif
#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR
size_t ZSTD_compressBlock_btlazy2_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2);
}
#endif
rust/README.md
+6 -4
View File
@@ -33,16 +33,18 @@ zstd ABI:
while preserving the compressor's Huffman-table repeat state. while preserving the compressor's Huffman-table repeat state.
- `zstd_fast` and `zstd_double_fast` implement the single- and two-table - `zstd_fast` and `zstd_double_fast` implement the single- and two-table
fast block match finders, including attached and external dictionary paths. fast block match finders, including attached and external dictionary paths.
- `zstd_lazy` implements greedy, lazy, lazy2, and binary-tree matching,
including row-based and dictionary search variants.
- Runtime support - Runtime support
- `threading` provides platform pthread wrappers required by zstd headers. - `threading` provides platform pthread wrappers required by zstd headers.
- `pool` implements the bounded worker pool used by multithreaded compression. - `pool` implements the bounded worker pool used by multithreaded compression.
- Dictionary support - Dictionary support
- `zstd_ddict` owns, loads, copies, and references decode dictionaries. - `zstd_ddict` owns, loads, copies, and references decode dictionaries.
The lazy and optimal block matchers, general decompression, dictionary-building, The optimal block matcher, general decompression, dictionary-building, legacy,
legacy, and CLI translation units are still C. They must move before the and CLI translation units are still C. They must move before the rewrite is
rewrite is complete. Keeping that boundary explicit prevents a passing hybrid complete. Keeping that boundary explicit prevents a passing hybrid build from
build from being mistaken for the final all-Rust result. being mistaken for the final all-Rust result.
## Compatibility boundary ## Compatibility boundary
rust/src/lib.rs
+2
View File
@@ -36,4 +36,6 @@ pub mod zstd_double_fast;
#[cfg(feature = "compression")] #[cfg(feature = "compression")]
pub mod zstd_fast; pub mod zstd_fast;
#[cfg(feature = "compression")] #[cfg(feature = "compression")]
pub mod zstd_lazy;
#[cfg(feature = "compression")]
pub mod zstd_presplit; pub mod zstd_presplit;
rust/src/zstd_lazy.rs
+2623
View File
@@ -0,0 +1,2623 @@
#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::too_many_arguments)]
#![allow(clippy::not_unsafe_ptr_arg_deref)]
//! Lazy, lazy2, greedy, and binary-tree block match finders.
//!
//! `ZSTD_MatchState_t` deliberately stays on the C side of the boundary. The
//! small representation below contains only the fields that the match finders
//! use, with mutable scalar fields represented by pointers back into the C
//! match state. This keeps the matching and sequence-generation loops in
//! Rust without making a private C layout part of the Rust ABI.
use crate::bits::ZSTD_highbit32;
use crate::mem::{
MEM_64bits, MEM_isLittleEndian, MEM_read16, MEM_read32, MEM_readLE32, MEM_readLE64, MEM_readST,
};
use std::ffi::c_void;
use std::mem::size_of;
use std::os::raw::c_int;
use std::ptr;
const ZSTD_REP_NUM: usize = 3;
const MINMATCH: usize = 3;
const HASH_READ_SIZE: usize = 8;
const K_LAZY_SKIPPING_STEP: usize = 8;
const K_SEARCH_STRENGTH: usize = 8;
const ZSTD_DUBT_UNSORTED_MARK: u32 = 1;
const ZSTD_LAZY_DDSS_BUCKET_LOG: u32 = 2;
const ZSTD_ROW_HASH_TAG_BITS: u32 = 8;
const ZSTD_ROW_HASH_TAG_MASK: u32 = (1 << ZSTD_ROW_HASH_TAG_BITS) - 1;
const ZSTD_ROW_HASH_CACHE_SIZE: usize = 8;
const ZSTD_ROW_HASH_MAX_ENTRIES: usize = 64;
const REPCODE1_TO_OFFBASE: u32 = 1;
const OFFSET_OFFBASE: u32 = ZSTD_REP_NUM as u32;
const SEARCH_HASH_CHAIN: c_int = 0;
const SEARCH_BINARY_TREE: c_int = 1;
const SEARCH_ROW_HASH: c_int = 2;
const DICT_NO_DICT: c_int = 0;
const DICT_EXT: c_int = 1;
const DICT_MATCH_STATE: c_int = 2;
const DICT_DEDICATED: c_int = 3;
#[repr(C)]
#[derive(Clone, Copy)]
struct SeqDef {
offBase: u32,
litLength: u16,
mlBase: u16,
}
/// The C shim verifies the layout of this leaf structure. The matcher only
/// needs its sequence and literal append cursors.
#[repr(C)]
struct SeqStore_t {
sequencesStart: *mut SeqDef,
sequences: *mut SeqDef,
litStart: *mut u8,
lit: *mut u8,
llCode: *mut u8,
mlCode: *mut u8,
ofCode: *mut u8,
maxNbSeq: usize,
maxNbLit: usize,
longLengthType: c_int,
longLengthPos: u32,
}
/// A field-level view of `ZSTD_MatchState_t` used by the lazy match finders.
///
/// All pointers remain owned by C. `dictMatchState` points at a second view
/// constructed by the C shim for calls that search an attached dictionary.
#[repr(C)]
pub struct ZSTD_RustLazyState {
hashTable: *mut u32,
chainTable: *mut u32,
tagTable: *mut u8,
hashCache: *mut u32,
base: *const u8,
dictBase: *const u8,
nextSrc: *const u8,
dictLimit: u32,
lowLimit: u32,
loadedDictEnd: u32,
nextToUpdate: *mut u32,
lazySkipping: *mut c_int,
hashLog: u32,
chainLog: u32,
minMatch: u32,
searchLog: u32,
windowLog: u32,
rowHashLog: u32,
hashSalt: u64,
hashSaltEntropy: *mut u32,
dictMatchState: *const ZSTD_RustLazyState,
}
#[inline]
fn ptr_lt(left: *const u8, right: *const u8) -> bool {
(left as usize) < (right as usize)
}
#[inline]
fn ptr_le(left: *const u8, right: *const u8) -> bool {
(left as usize) <= (right as usize)
}
#[inline]
fn ptr_gt(left: *const u8, right: *const u8) -> bool {
(left as usize) > (right as usize)
}
#[inline]
unsafe fn index_from(base: *const u8, value: *const u8) -> u32 {
(value as usize).wrapping_sub(base as usize) as u32
}
#[inline]
unsafe fn read32(value: *const u8) -> u32 {
unsafe { MEM_read32(value.cast::<c_void>()) }
}
#[inline]
unsafe fn table_get(table: *const u32, index: usize) -> u32 {
unsafe { *table.add(index) }
}
#[inline]
unsafe fn table_set(table: *mut u32, index: usize, value: u32) {
unsafe { *table.add(index) = value };
}
#[inline]
unsafe fn byte_get(table: *const u8, index: usize) -> u8 {
unsafe { *table.add(index) }
}
#[inline]
unsafe fn byte_set(table: *mut u8, index: usize, value: u8) {
unsafe { *table.add(index) = value };
}
#[inline]
unsafe fn next_to_update(state: &ZSTD_RustLazyState) -> u32 {
unsafe { *state.nextToUpdate }
}
#[inline]
unsafe fn set_next_to_update(state: &ZSTD_RustLazyState, value: u32) {
unsafe { *state.nextToUpdate = value };
}
#[inline]
unsafe fn lazy_skipping(state: &ZSTD_RustLazyState) -> bool {
unsafe { *state.lazySkipping != 0 }
}
#[inline]
unsafe fn set_lazy_skipping(state: &ZSTD_RustLazyState, value: bool) {
unsafe { *state.lazySkipping = c_int::from(value) };
}
#[inline]
unsafe fn add_hash_salt_entropy(state: &ZSTD_RustLazyState, value: u32) {
unsafe {
*state.hashSaltEntropy = (*state.hashSaltEntropy).wrapping_add(value);
}
}
#[inline]
unsafe fn dict_state<'a>(state: &ZSTD_RustLazyState) -> &'a ZSTD_RustLazyState {
debug_assert!(!state.dictMatchState.is_null());
unsafe { &*state.dictMatchState }
}
#[inline]
fn bounded(low: u32, value: u32, high: u32) -> u32 {
value.clamp(low, high)
}
#[inline]
fn hash_shift32(value: u32, hbits: u32) -> usize {
if hbits == 0 {
0
} else {
(value >> (32 - hbits)) as usize
}
}
#[inline]
fn hash_shift64(value: u64, hbits: u32) -> usize {
if hbits == 0 {
0
} else {
(value >> (64 - hbits)) as usize
}
}
#[inline]
unsafe fn hash_ptr(ptr: *const u8, hbits: u32, mls: u32) -> usize {
match mls {
5 => {
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
hash_shift64(value.wrapping_shl(24).wrapping_mul(889_523_592_379), hbits)
}
6 => {
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
hash_shift64(
value.wrapping_shl(16).wrapping_mul(227_718_039_650_203),
hbits,
)
}
7 => {
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
hash_shift64(
value.wrapping_shl(8).wrapping_mul(58_295_818_150_454_627),
hbits,
)
}
8 => {
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
hash_shift64(value.wrapping_mul(0xCF1B_BCDC_B7A5_6463), hbits)
}
_ => {
let value = unsafe { MEM_readLE32(ptr.cast::<c_void>()) };
hash_shift32(value.wrapping_mul(2_654_435_761), hbits)
}
}
}
#[inline]
unsafe fn hash_ptr_salted(ptr: *const u8, hbits: u32, mls: u32, salt: u64) -> usize {
match mls {
5 => {
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
hash_shift64(
value.wrapping_shl(24).wrapping_mul(889_523_592_379) ^ salt,
hbits,
)
}
6 => {
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
hash_shift64(
value.wrapping_shl(16).wrapping_mul(227_718_039_650_203) ^ salt,
hbits,
)
}
7 => {
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
hash_shift64(
value.wrapping_shl(8).wrapping_mul(58_295_818_150_454_627) ^ salt,
hbits,
)
}
8 => {
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
hash_shift64(value.wrapping_mul(0xCF1B_BCDC_B7A5_6463) ^ salt, hbits)
}
_ => {
let value = unsafe { MEM_readLE32(ptr.cast::<c_void>()) };
hash_shift32(value.wrapping_mul(2_654_435_761) ^ salt as u32, hbits)
}
}
}
#[inline]
fn common_bytes(word: usize) -> usize {
let zeros = if MEM_isLittleEndian() {
word.trailing_zeros()
} else {
word.leading_zeros()
};
(zeros / 8) as usize
}
/// Equivalent to C's `ZSTD_count()`.
unsafe fn count(mut input: *const u8, mut matched: *const u8, input_limit: *const u8) -> usize {
let input_start = input;
let word_size = size_of::<usize>();
while (input_limit as usize).wrapping_sub(input as usize) >= word_size {
let diff =
unsafe { MEM_readST(matched.cast::<c_void>()) ^ MEM_readST(input.cast::<c_void>()) };
if diff != 0 {
return (input as usize).wrapping_sub(input_start as usize) + common_bytes(diff);
}
input = input.wrapping_add(word_size);
matched = matched.wrapping_add(word_size);
}
if MEM_64bits()
&& (input_limit as usize).wrapping_sub(input as usize) >= 4
&& unsafe { MEM_read32(matched.cast::<c_void>()) == MEM_read32(input.cast::<c_void>()) }
{
input = input.wrapping_add(4);
matched = matched.wrapping_add(4);
}
if (input_limit as usize).wrapping_sub(input as usize) >= 2
&& unsafe { MEM_read16(matched.cast::<c_void>()) == MEM_read16(input.cast::<c_void>()) }
{
input = input.wrapping_add(2);
matched = matched.wrapping_add(2);
}
if ptr_lt(input, input_limit) && unsafe { *matched == *input } {
input = input.wrapping_add(1);
}
(input as usize).wrapping_sub(input_start as usize)
}
unsafe fn count_2segments(
input: *const u8,
matched: *const u8,
input_end: *const u8,
match_end: *const u8,
input_start: *const u8,
) -> usize {
let match_remaining = (match_end as usize).wrapping_sub(matched as usize);
let input_remaining = (input_end as usize).wrapping_sub(input as usize);
let first_end = input.wrapping_add(match_remaining.min(input_remaining));
let first_count = unsafe { count(input, matched, first_end) };
if matched.wrapping_add(first_count) != match_end {
return first_count;
}
first_count + unsafe { count(input.wrapping_add(first_count), input_start, input_end) }
}
#[inline]
fn lowest_prefix_index(dict_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 {
let max_distance = 1u32.wrapping_shl(window_log);
let within_window = if curr.wrapping_sub(dict_limit) > max_distance {
curr.wrapping_sub(max_distance)
} else {
dict_limit
};
if loaded_dict_end != 0 {
dict_limit
} else {
within_window
}
}
#[inline]
fn lowest_match_index(low_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 {
let max_distance = 1u32.wrapping_shl(window_log);
let within_window = if curr.wrapping_sub(low_limit) > max_distance {
curr.wrapping_sub(max_distance)
} else {
low_limit
};
if loaded_dict_end != 0 {
low_limit
} else {
within_window
}
}
#[inline]
fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool {
prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3
}
/// Stores exactly the observable bytes that C's `ZSTD_storeSeq()` writes.
unsafe fn store_seq(
seq_store: *mut SeqStore_t,
lit_length: usize,
literals: *const u8,
_lit_limit: *const u8,
off_base: u32,
match_length: usize,
) {
let seq_store = unsafe { &mut *seq_store };
let sequence = seq_store.sequences;
debug_assert!(
(sequence as usize).wrapping_sub(seq_store.sequencesStart as usize) / size_of::<SeqDef>()
< seq_store.maxNbSeq
);
debug_assert!(lit_length <= seq_store.maxNbLit);
debug_assert!(match_length >= MINMATCH);
if lit_length != 0 {
unsafe { ptr::copy_nonoverlapping(literals, seq_store.lit, lit_length) };
}
seq_store.lit = seq_store.lit.wrapping_add(lit_length);
let sequence_index =
(sequence as usize).wrapping_sub(seq_store.sequencesStart as usize) / size_of::<SeqDef>();
if lit_length > u16::MAX as usize {
debug_assert_eq!(seq_store.longLengthType, 0);
seq_store.longLengthType = 1;
seq_store.longLengthPos = sequence_index as u32;
}
unsafe { (*sequence).litLength = lit_length as u16 };
unsafe { (*sequence).offBase = off_base };
let match_base = match_length - MINMATCH;
if match_base > u16::MAX as usize {
debug_assert_eq!(seq_store.longLengthType, 0);
seq_store.longLengthType = 2;
seq_store.longLengthPos = sequence_index as u32;
}
unsafe { (*sequence).mlBase = match_base as u16 };
seq_store.sequences = sequence.wrapping_add(1);
}
#[inline]
fn offset_to_offbase(offset: u32) -> u32 {
offset.wrapping_add(OFFSET_OFFBASE)
}
#[inline]
fn offbase_is_offset(offbase: usize) -> bool {
offbase > OFFSET_OFFBASE as usize
}
#[inline]
fn offbase_to_offset(offbase: usize) -> u32 {
(offbase as u32).wrapping_sub(OFFSET_OFFBASE)
}
#[inline]
fn match_improves(
match_length: usize,
best_length: usize,
curr: u32,
match_index: u32,
old_offbase: usize,
old_offbase_plus_one: bool,
) -> bool {
let lhs = (match_length.wrapping_sub(best_length) as i32).wrapping_mul(4);
let new_hb = ZSTD_highbit32(curr.wrapping_sub(match_index).wrapping_add(1)) as i32;
let old_value = (old_offbase as u32).wrapping_add(u32::from(old_offbase_plus_one));
let old_hb = ZSTD_highbit32(old_value) as i32;
lhs > new_hb.wrapping_sub(old_hb)
}
/* ------------------------------------------------------------------------- */
/* Binary tree search */
/* ------------------------------------------------------------------------- */
unsafe fn update_dubt(state: &ZSTD_RustLazyState, ip: *const u8, _iend: *const u8, mls: u32) {
let target = unsafe { index_from(state.base, ip) };
let mut idx = unsafe { next_to_update(state) };
let bt_log = state.chainLog - 1;
let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1);
while idx < target {
let hash = unsafe { hash_ptr(state.base.wrapping_add(idx as usize), state.hashLog, mls) };
let match_index = unsafe { table_get(state.hashTable, hash) };
let cell = 2 * (idx & bt_mask) as usize;
unsafe {
table_set(state.hashTable, hash, idx);
table_set(state.chainTable, cell, match_index);
table_set(state.chainTable, cell + 1, ZSTD_DUBT_UNSORTED_MARK);
}
idx = idx.wrapping_add(1);
}
unsafe { set_next_to_update(state, target) };
}
unsafe fn insert_dubt1(
state: &ZSTD_RustLazyState,
curr: u32,
input_end: *const u8,
mut nb_compares: u32,
bt_low: u32,
dict_mode: c_int,
) {
let bt_log = state.chainLog - 1;
let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1);
let mut common_smaller = 0usize;
let mut common_larger = 0usize;
let ip = if curr >= state.dictLimit {
state.base.wrapping_add(curr as usize)
} else {
state.dictBase.wrapping_add(curr as usize)
};
let iend = if curr >= state.dictLimit {
input_end
} else {
state.dictBase.wrapping_add(state.dictLimit as usize)
};
let dict_end = state.dictBase.wrapping_add(state.dictLimit as usize);
let prefix_start = state.base.wrapping_add(state.dictLimit as usize);
let window_valid = state.lowLimit;
let max_distance = 1u32.wrapping_shl(state.windowLog);
let window_low = if curr.wrapping_sub(window_valid) > max_distance {
curr.wrapping_sub(max_distance)
} else {
window_valid
};
let mut smaller_slot = 2 * (curr & bt_mask) as usize;
let mut larger_slot = smaller_slot + 1;
let mut smaller_real = true;
let mut larger_real = true;
let mut match_index = unsafe { table_get(state.chainTable, smaller_slot) };
while nb_compares != 0 && match_index > window_low {
let next_cell = 2 * (match_index & bt_mask) as usize;
let mut match_length = common_smaller.min(common_larger);
let mut matched;
if dict_mode != DICT_EXT || match_index.wrapping_add(match_length as u32) >= state.dictLimit
{
matched = state.base.wrapping_add(match_index as usize);
match_length += unsafe {
count(
ip.wrapping_add(match_length),
matched.wrapping_add(match_length),
iend,
)
};
} else {
matched = state.dictBase.wrapping_add(match_index as usize);
match_length += unsafe {
count_2segments(
ip.wrapping_add(match_length),
matched.wrapping_add(match_length),
iend,
dict_end,
prefix_start,
)
};
if match_index.wrapping_add(match_length as u32) >= state.dictLimit {
matched = state.base.wrapping_add(match_index as usize);
}
}
if ip.wrapping_add(match_length) == iend {
break;
}
if unsafe { *matched.wrapping_add(match_length) < *ip.wrapping_add(match_length) } {
if smaller_real {
unsafe { table_set(state.chainTable, smaller_slot, match_index) };
}
common_smaller = match_length;
if match_index <= bt_low {
smaller_real = false;
break;
}
smaller_slot = next_cell + 1;
match_index = unsafe { table_get(state.chainTable, next_cell + 1) };
} else {
if larger_real {
unsafe { table_set(state.chainTable, larger_slot, match_index) };
}
common_larger = match_length;
if match_index <= bt_low {
larger_real = false;
break;
}
larger_slot = next_cell;
match_index = unsafe { table_get(state.chainTable, next_cell) };
}
nb_compares = nb_compares.wrapping_sub(1);
}
if smaller_real {
unsafe { table_set(state.chainTable, smaller_slot, 0) };
}
if larger_real {
unsafe { table_set(state.chainTable, larger_slot, 0) };
}
}
unsafe fn dubt_find_better_dict_match(
state: &ZSTD_RustLazyState,
ip: *const u8,
iend: *const u8,
offbase: &mut usize,
mut best_length: usize,
mut nb_compares: u32,
mls: u32,
) -> usize {
let dms = unsafe { dict_state(state) };
let dict_hash = unsafe { hash_ptr(ip, dms.hashLog, mls) };
let mut dict_match_index = unsafe { table_get(dms.hashTable, dict_hash) };
let prefix_start = state.base.wrapping_add(state.dictLimit as usize);
let curr = unsafe { index_from(state.base, ip) };
let dict_base = dms.base;
let dict_end = dms.nextSrc;
let dict_high_limit = unsafe { index_from(dms.base, dms.nextSrc) };
let dict_low_limit = dms.lowLimit;
let dict_index_delta = state.lowLimit.wrapping_sub(dict_high_limit);
let bt_log = dms.chainLog - 1;
let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1);
let bt_low = if bt_mask >= dict_high_limit.wrapping_sub(dict_low_limit) {
dict_low_limit
} else {
dict_high_limit.wrapping_sub(bt_mask)
};
let mut common_smaller = 0usize;
let mut common_larger = 0usize;
while nb_compares != 0 && dict_match_index > dict_low_limit {
let next_cell = 2 * (dict_match_index & bt_mask) as usize;
let mut match_length = common_smaller.min(common_larger);
let mut matched = dict_base.wrapping_add(dict_match_index as usize);
match_length += unsafe {
count_2segments(
ip.wrapping_add(match_length),
matched.wrapping_add(match_length),
iend,
dict_end,
prefix_start,
)
};
if dict_match_index.wrapping_add(match_length as u32) >= dict_high_limit {
matched = state
.base
.wrapping_add(dict_match_index.wrapping_add(dict_index_delta) as usize);
}
if match_length > best_length {
let match_index = dict_match_index.wrapping_add(dict_index_delta);
if match_improves(match_length, best_length, curr, match_index, *offbase, true) {
best_length = match_length;
*offbase = offset_to_offbase(curr.wrapping_sub(match_index)) as usize;
}
if ip.wrapping_add(match_length) == iend {
break;
}
}
if unsafe { *matched.wrapping_add(match_length) < *ip.wrapping_add(match_length) } {
if dict_match_index <= bt_low {
break;
}
common_smaller = match_length;
dict_match_index = unsafe { table_get(dms.chainTable, next_cell + 1) };
} else {
if dict_match_index <= bt_low {
break;
}
common_larger = match_length;
dict_match_index = unsafe { table_get(dms.chainTable, next_cell) };
}
nb_compares = nb_compares.wrapping_sub(1);
}
best_length
}
unsafe fn dubt_find_best_match(
state: &ZSTD_RustLazyState,
ip: *const u8,
iend: *const u8,
offbase: &mut usize,
mls: u32,
dict_mode: c_int,
) -> usize {
let hash = unsafe { hash_ptr(ip, state.hashLog, mls) };
let mut match_index = unsafe { table_get(state.hashTable, hash) };
let curr = unsafe { index_from(state.base, ip) };
let window_low = lowest_match_index(state.lowLimit, state.loadedDictEnd, curr, state.windowLog);
let bt_log = state.chainLog - 1;
let bt_mask = 1u32.wrapping_shl(bt_log).wrapping_sub(1);
let bt_low = if bt_mask >= curr {
0
} else {
curr.wrapping_sub(bt_mask)
};
let unsort_limit = bt_low.max(window_low);
let mut next_candidate = 2 * (match_index & bt_mask) as usize;
let mut unsorted_mark = next_candidate + 1;
let mut nb_compares = 1u32.wrapping_shl(state.searchLog);
let mut nb_candidates = nb_compares;
let mut previous_candidate = 0u32;
while match_index > unsort_limit
&& unsafe { table_get(state.chainTable, unsorted_mark) == ZSTD_DUBT_UNSORTED_MARK }
&& nb_candidates > 1
{
unsafe { table_set(state.chainTable, unsorted_mark, previous_candidate) };
previous_candidate = match_index;
match_index = unsafe { table_get(state.chainTable, next_candidate) };
next_candidate = 2 * (match_index & bt_mask) as usize;
unsorted_mark = next_candidate + 1;
nb_candidates = nb_candidates.wrapping_sub(1);
}
if match_index > unsort_limit
&& unsafe { table_get(state.chainTable, unsorted_mark) == ZSTD_DUBT_UNSORTED_MARK }
{
unsafe {
table_set(state.chainTable, next_candidate, 0);
table_set(state.chainTable, unsorted_mark, 0);
}
}
match_index = previous_candidate;
while match_index != 0 {
let next_candidate_idx =
unsafe { table_get(state.chainTable, 2 * (match_index & bt_mask) as usize + 1) };
unsafe {
insert_dubt1(
state,
match_index,
iend,
nb_candidates,
unsort_limit,
dict_mode,
)
};
match_index = next_candidate_idx;
nb_candidates = nb_candidates.wrapping_add(1);
}
let mut common_smaller = 0usize;
let mut common_larger = 0usize;
let dict_base = state.dictBase;
let dict_limit = state.dictLimit;
let dict_end = dict_base.wrapping_add(dict_limit as usize);
let prefix_start = state.base.wrapping_add(dict_limit as usize);
let mut smaller_slot = 2 * (curr & bt_mask) as usize;
let mut larger_slot = smaller_slot + 1;
let mut smaller_real = true;
let mut larger_real = true;
let mut match_end_idx = curr.wrapping_add(HASH_READ_SIZE as u32 + 1);
let mut best_length = 0usize;
match_index = unsafe { table_get(state.hashTable, hash) };
unsafe { table_set(state.hashTable, hash, curr) };
while nb_compares != 0 && match_index > window_low {
let next_cell = 2 * (match_index & bt_mask) as usize;
let mut match_length = common_smaller.min(common_larger);
let mut matched;
if dict_mode != DICT_EXT || match_index.wrapping_add(match_length as u32) >= dict_limit {
matched = state.base.wrapping_add(match_index as usize);
match_length += unsafe {
count(
ip.wrapping_add(match_length),
matched.wrapping_add(match_length),
iend,
)
};
} else {
matched = dict_base.wrapping_add(match_index as usize);
match_length += unsafe {
count_2segments(
ip.wrapping_add(match_length),
matched.wrapping_add(match_length),
iend,
dict_end,
prefix_start,
)
};
if match_index.wrapping_add(match_length as u32) >= dict_limit {
matched = state.base.wrapping_add(match_index as usize);
}
}
if match_length > best_length {
if match_length > match_end_idx.wrapping_sub(match_index) as usize {
match_end_idx = match_index.wrapping_add(match_length as u32);
}
if match_improves(
match_length,
best_length,
curr,
match_index,
*offbase,
false,
) {
best_length = match_length;
*offbase = offset_to_offbase(curr.wrapping_sub(match_index)) as usize;
}
if ip.wrapping_add(match_length) == iend {
if dict_mode == DICT_MATCH_STATE {
nb_compares = 0;
}
break;
}
}
if unsafe { *matched.wrapping_add(match_length) < *ip.wrapping_add(match_length) } {
if smaller_real {
unsafe { table_set(state.chainTable, smaller_slot, match_index) };
}
common_smaller = match_length;
if match_index <= bt_low {
smaller_real = false;
break;
}
smaller_slot = next_cell + 1;
match_index = unsafe { table_get(state.chainTable, next_cell + 1) };
} else {
if larger_real {
unsafe { table_set(state.chainTable, larger_slot, match_index) };
}
common_larger = match_length;
if match_index <= bt_low {
larger_real = false;
break;
}
larger_slot = next_cell;
match_index = unsafe { table_get(state.chainTable, next_cell) };
}
nb_compares = nb_compares.wrapping_sub(1);
}
if smaller_real {
unsafe { table_set(state.chainTable, smaller_slot, 0) };
}
if larger_real {
unsafe { table_set(state.chainTable, larger_slot, 0) };
}
if dict_mode == DICT_MATCH_STATE && nb_compares != 0 {
best_length = unsafe {
dubt_find_better_dict_match(state, ip, iend, offbase, best_length, nb_compares, mls)
};
}
unsafe { set_next_to_update(state, match_end_idx.wrapping_sub(HASH_READ_SIZE as u32)) };
best_length
}
unsafe fn bt_find_best_match(
state: &ZSTD_RustLazyState,
ip: *const u8,
iend: *const u8,
offbase: &mut usize,
mls: u32,
dict_mode: c_int,
) -> usize {
if ptr_lt(
ip,
state
.base
.wrapping_add(unsafe { next_to_update(state) } as usize),
) {
return 0;
}
unsafe { update_dubt(state, ip, iend, mls) };
unsafe { dubt_find_best_match(state, ip, iend, offbase, mls, dict_mode) }
}
/* ------------------------------------------------------------------------- */
/* Dedicated dictionary search */
/* ------------------------------------------------------------------------- */
unsafe fn dedicated_dict_search_load_dictionary(state: &ZSTD_RustLazyState, ip: *const u8) {
let target = unsafe { index_from(state.base, ip) };
let chain_size = 1u32.wrapping_shl(state.chainLog);
let mut idx = unsafe { next_to_update(state) };
let min_chain = if chain_size < target.wrapping_sub(idx) {
target.wrapping_sub(chain_size)
} else {
idx
};
let bucket_size = 1u32 << ZSTD_LAZY_DDSS_BUCKET_LOG;
let cache_size = bucket_size - 1;
let chain_attempts = (1u32 << state.searchLog).wrapping_sub(cache_size);
let chain_limit = chain_attempts.min(255);
let hash_log = state.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG;
let tmp_hash_table = state.hashTable;
let tmp_chain_table = state
.hashTable
.wrapping_add((1usize).wrapping_shl(hash_log));
let tmp_chain_size = ((1u32 << ZSTD_LAZY_DDSS_BUCKET_LOG) - 1).wrapping_shl(hash_log);
let tmp_min_chain = if tmp_chain_size < target {
target.wrapping_sub(tmp_chain_size)
} else {
idx
};
while idx < target {
let hash = unsafe {
hash_ptr(
state.base.wrapping_add(idx as usize),
hash_log,
state.minMatch,
)
};
if idx >= tmp_min_chain {
let previous = unsafe { table_get(state.hashTable, hash) };
unsafe {
table_set(
tmp_chain_table,
idx.wrapping_sub(tmp_min_chain) as usize,
previous,
)
};
}
unsafe { table_set(tmp_hash_table, hash, idx) };
idx = idx.wrapping_add(1);
}
let mut chain_pos = 0u32;
let mut hash_idx = 0u32;
while hash_idx < (1u32 << hash_log) {
let mut count = 0u32;
let mut count_beyond_min_chain = 0u32;
let mut chain_index = unsafe { table_get(tmp_hash_table, hash_idx as usize) };
while chain_index >= tmp_min_chain && count < cache_size {
if chain_index < min_chain {
count_beyond_min_chain = count_beyond_min_chain.wrapping_add(1);
}
chain_index = unsafe {
table_get(
tmp_chain_table,
chain_index.wrapping_sub(tmp_min_chain) as usize,
)
};
count = count.wrapping_add(1);
}
if count == cache_size {
count = 0;
while count < chain_limit {
if chain_index < min_chain {
if chain_index == 0 || count_beyond_min_chain.wrapping_add(1) > cache_size {
break;
}
count_beyond_min_chain = count_beyond_min_chain.wrapping_add(1);
}
unsafe { table_set(state.chainTable, chain_pos as usize, chain_index) };
chain_pos = chain_pos.wrapping_add(1);
count = count.wrapping_add(1);
if chain_index < tmp_min_chain {
break;
}
chain_index = unsafe {
table_get(
tmp_chain_table,
chain_index.wrapping_sub(tmp_min_chain) as usize,
)
};
}
} else {
count = 0;
}
if count != 0 {
unsafe {
table_set(
tmp_hash_table,
hash_idx as usize,
((chain_pos.wrapping_sub(count)) << 8).wrapping_add(count),
)
};
} else {
unsafe { table_set(tmp_hash_table, hash_idx as usize, 0) };
}
hash_idx = hash_idx.wrapping_add(1);
}
hash_idx = 1u32 << hash_log;
while hash_idx != 0 {
hash_idx = hash_idx.wrapping_sub(1);
let bucket_idx = hash_idx << ZSTD_LAZY_DDSS_BUCKET_LOG;
let packed = unsafe { table_get(tmp_hash_table, hash_idx as usize) };
let mut i = 0u32;
while i < cache_size {
unsafe { table_set(state.hashTable, bucket_idx.wrapping_add(i) as usize, 0) };
i = i.wrapping_add(1);
}
unsafe {
table_set(
state.hashTable,
bucket_idx.wrapping_add(bucket_size).wrapping_sub(1) as usize,
packed,
)
};
}
idx = unsafe { next_to_update(state) };
while idx < target {
let hash = unsafe {
hash_ptr(
state.base.wrapping_add(idx as usize),
hash_log,
state.minMatch,
)
} << ZSTD_LAZY_DDSS_BUCKET_LOG;
let mut i = cache_size.wrapping_sub(1);
while i != 0 {
let previous =
unsafe { table_get(state.hashTable, hash.wrapping_add((i - 1) as usize)) };
unsafe { table_set(state.hashTable, hash.wrapping_add(i as usize), previous) };
i = i.wrapping_sub(1);
}
unsafe { table_set(state.hashTable, hash, idx) };
idx = idx.wrapping_add(1);
}
unsafe { set_next_to_update(state, target) };
}
unsafe fn dedicated_dict_search(
offset: &mut usize,
mut ml: usize,
nb_attempts: u32,
dms: &ZSTD_RustLazyState,
ip: *const u8,
ilimit: *const u8,
prefix_start: *const u8,
curr: u32,
dict_limit: u32,
dds_idx: usize,
) -> usize {
let dds_lowest_index = dms.dictLimit;
let dds_base = dms.base;
let dds_end = dms.nextSrc;
let dds_size = unsafe { index_from(dds_base, dds_end) };
let dds_index_delta = dict_limit.wrapping_sub(dds_size);
let bucket_size = 1u32 << ZSTD_LAZY_DDSS_BUCKET_LOG;
let bucket_limit = nb_attempts.min(bucket_size - 1);
let mut dds_attempt = 0u32;
while dds_attempt < bucket_limit {
let match_index = unsafe { table_get(dms.hashTable, dds_idx + dds_attempt as usize) };
if match_index == 0 {
return ml;
}
debug_assert!(match_index >= dds_lowest_index);
let matched = dds_base.wrapping_add(match_index as usize);
let mut current_ml = 0usize;
if unsafe { read32(matched) == read32(ip) } {
current_ml = unsafe {
count_2segments(
ip.wrapping_add(4),
matched.wrapping_add(4),
ilimit,
dds_end,
prefix_start,
) + 4
};
}
if current_ml > ml {
ml = current_ml;
*offset =
offset_to_offbase(curr.wrapping_sub(match_index.wrapping_add(dds_index_delta)))
as usize;
if ip.wrapping_add(current_ml) == ilimit {
return ml;
}
}
dds_attempt = dds_attempt.wrapping_add(1);
}
let packed = unsafe { table_get(dms.hashTable, dds_idx + (bucket_size - 1) as usize) };
let mut chain_index = packed >> 8;
let chain_length = packed & 0xff;
let chain_attempts = nb_attempts.wrapping_sub(dds_attempt);
let chain_limit = chain_attempts.min(chain_length);
let mut chain_attempt = 0u32;
while chain_attempt < chain_limit {
let match_index = unsafe { table_get(dms.chainTable, chain_index as usize) };
let matched = dds_base.wrapping_add(match_index as usize);
let mut current_ml = 0usize;
if unsafe { read32(matched) == read32(ip) } {
current_ml = unsafe {
count_2segments(
ip.wrapping_add(4),
matched.wrapping_add(4),
ilimit,
dds_end,
prefix_start,
) + 4
};
}
if current_ml > ml {
ml = current_ml;
*offset =
offset_to_offbase(curr.wrapping_sub(match_index.wrapping_add(dds_index_delta)))
as usize;
if ip.wrapping_add(current_ml) == ilimit {
break;
}
}
chain_index = chain_index.wrapping_add(1);
chain_attempt = chain_attempt.wrapping_add(1);
}
ml
}
/* ------------------------------------------------------------------------- */
/* Hash-chain search */
/* ------------------------------------------------------------------------- */
unsafe fn insert_and_find_first_index_internal(
state: &ZSTD_RustLazyState,
ip: *const u8,
mls: u32,
lazy_skip: bool,
) -> u32 {
let chain_mask = 1u32.wrapping_shl(state.chainLog).wrapping_sub(1);
let target = unsafe { index_from(state.base, ip) };
let mut idx = unsafe { next_to_update(state) };
while idx < target {
let hash = unsafe { hash_ptr(state.base.wrapping_add(idx as usize), state.hashLog, mls) };
let previous = unsafe { table_get(state.hashTable, hash) };
unsafe {
table_set(state.chainTable, (idx & chain_mask) as usize, previous);
table_set(state.hashTable, hash, idx);
}
idx = idx.wrapping_add(1);
if lazy_skip {
break;
}
}
unsafe { set_next_to_update(state, target) };
let hash = unsafe { hash_ptr(ip, state.hashLog, mls) };
unsafe { table_get(state.hashTable, hash) }
}
unsafe fn hc_find_best_match(
state: &ZSTD_RustLazyState,
ip: *const u8,
ilimit: *const u8,
offset: &mut usize,
mls: u32,
dict_mode: c_int,
) -> usize {
let chain_size = 1u32.wrapping_shl(state.chainLog);
let chain_mask = chain_size.wrapping_sub(1);
let base = state.base;
let dict_base = state.dictBase;
let dict_limit = state.dictLimit;
let prefix_start = base.wrapping_add(dict_limit as usize);
let dict_end = dict_base.wrapping_add(dict_limit as usize);
let curr = unsafe { index_from(base, ip) };
let max_distance = 1u32.wrapping_shl(state.windowLog);
let lowest_valid = state.lowLimit;
let within_max_distance = if curr.wrapping_sub(lowest_valid) > max_distance {
curr.wrapping_sub(max_distance)
} else {
lowest_valid
};
let low_limit = if state.loadedDictEnd != 0 {
lowest_valid
} else {
within_max_distance
};
let min_chain = if curr > chain_size {
curr.wrapping_sub(chain_size)
} else {
0
};
let mut nb_attempts = 1u32.wrapping_shl(state.searchLog);
let mut ml = 3usize;
let dds_hash_log = if dict_mode == DICT_DEDICATED {
unsafe { dict_state(state) }.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG
} else {
0
};
let dds_idx = if dict_mode == DICT_DEDICATED {
(unsafe { hash_ptr(ip, dds_hash_log, mls) }) << ZSTD_LAZY_DDSS_BUCKET_LOG
} else {
0
};
let mut match_index =
unsafe { insert_and_find_first_index_internal(state, ip, mls, lazy_skipping(state)) };
while match_index >= low_limit && nb_attempts != 0 {
let mut current_ml = 0usize;
if dict_mode != DICT_EXT || match_index >= dict_limit {
let matched = base.wrapping_add(match_index as usize);
if unsafe { read32(matched.wrapping_add(ml - 3)) == read32(ip.wrapping_add(ml - 3)) } {
current_ml = unsafe { count(ip, matched, ilimit) };
}
} else {
let matched = dict_base.wrapping_add(match_index as usize);
if unsafe { read32(matched) == read32(ip) } {
current_ml = unsafe {
count_2segments(
ip.wrapping_add(4),
matched.wrapping_add(4),
ilimit,
dict_end,
prefix_start,
) + 4
};
}
}
if current_ml > ml {
ml = current_ml;
*offset = offset_to_offbase(curr.wrapping_sub(match_index)) as usize;
if ip.wrapping_add(current_ml) == ilimit {
break;
}
}
if match_index <= min_chain {
break;
}
match_index = unsafe { table_get(state.chainTable, (match_index & chain_mask) as usize) };
nb_attempts = nb_attempts.wrapping_sub(1);
}
if dict_mode == DICT_DEDICATED {
ml = unsafe {
dedicated_dict_search(
offset,
ml,
nb_attempts,
dict_state(state),
ip,
ilimit,
prefix_start,
curr,
dict_limit,
dds_idx,
)
};
} else if dict_mode == DICT_MATCH_STATE {
let dms = unsafe { dict_state(state) };
let dms_chain_size = 1u32.wrapping_shl(dms.chainLog);
let dms_chain_mask = dms_chain_size.wrapping_sub(1);
let dms_lowest_index = dms.dictLimit;
let dms_base = dms.base;
let dms_end = dms.nextSrc;
let dms_size = unsafe { index_from(dms_base, dms_end) };
let dms_index_delta = dict_limit.wrapping_sub(dms_size);
let dms_min_chain = if dms_size > dms_chain_size {
dms_size.wrapping_sub(dms_chain_size)
} else {
0
};
match_index = unsafe { table_get(dms.hashTable, hash_ptr(ip, dms.hashLog, mls)) };
while match_index >= dms_lowest_index && nb_attempts != 0 {
let matched = dms_base.wrapping_add(match_index as usize);
let mut current_ml = 0usize;
if unsafe { read32(matched) == read32(ip) } {
current_ml = unsafe {
count_2segments(
ip.wrapping_add(4),
matched.wrapping_add(4),
ilimit,
dms_end,
prefix_start,
) + 4
};
}
if current_ml > ml {
ml = current_ml;
*offset =
offset_to_offbase(curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)))
as usize;
if ip.wrapping_add(current_ml) == ilimit {
break;
}
}
if match_index <= dms_min_chain {
break;
}
match_index =
unsafe { table_get(dms.chainTable, (match_index & dms_chain_mask) as usize) };
nb_attempts = nb_attempts.wrapping_sub(1);
}
}
ml
}
/* ------------------------------------------------------------------------- */
/* Row hash search */
/* ------------------------------------------------------------------------- */
#[inline]
unsafe fn row_next_index(tag_row: *mut u8, row_mask: u32) -> u32 {
let mut next = unsafe { ((*tag_row).wrapping_sub(1) as u32) & row_mask };
if next == 0 {
next = next.wrapping_add(row_mask);
}
unsafe { *tag_row = next as u8 };
next
}
unsafe fn row_fill_hash_cache(
state: &ZSTD_RustLazyState,
base: *const u8,
_row_log: u32,
mls: u32,
mut idx: u32,
ilimit: *const u8,
) {
let hash_log = state.rowHashLog;
let at = base.wrapping_add(idx as usize);
let max_to_fill = if ptr_gt(at, ilimit) {
0
} else {
((ilimit as usize).wrapping_sub(at as usize)).wrapping_add(1) as u32
};
let limit = idx.wrapping_add((ZSTD_ROW_HASH_CACHE_SIZE as u32).min(max_to_fill));
while idx < limit {
let hash = unsafe {
hash_ptr_salted(
base.wrapping_add(idx as usize),
hash_log + ZSTD_ROW_HASH_TAG_BITS,
mls,
state.hashSalt,
) as u32
};
unsafe {
table_set(
state.hashCache,
(idx as usize) & (ZSTD_ROW_HASH_CACHE_SIZE - 1),
hash,
)
};
idx = idx.wrapping_add(1);
}
}
unsafe fn row_next_cached_hash(
cache: *mut u32,
_hash_table: *const u32,
_tag_table: *const u8,
base: *const u8,
idx: u32,
hash_log: u32,
_row_log: u32,
mls: u32,
hash_salt: u64,
) -> u32 {
let new_hash = unsafe {
hash_ptr_salted(
base.wrapping_add(idx as usize + ZSTD_ROW_HASH_CACHE_SIZE),
hash_log + ZSTD_ROW_HASH_TAG_BITS,
mls,
hash_salt,
) as u32
};
let slot = (idx as usize) & (ZSTD_ROW_HASH_CACHE_SIZE - 1);
let hash = unsafe { table_get(cache, slot) };
unsafe { table_set(cache, slot, new_hash) };
hash
}
unsafe fn row_update_internal_impl(
state: &ZSTD_RustLazyState,
mut update_start: u32,
update_end: u32,
mls: u32,
row_log: u32,
row_mask: u32,
use_cache: bool,
) {
while update_start < update_end {
let hash = if use_cache {
unsafe {
row_next_cached_hash(
state.hashCache,
state.hashTable,
state.tagTable,
state.base,
update_start,
state.rowHashLog,
row_log,
mls,
state.hashSalt,
)
}
} else {
unsafe {
hash_ptr_salted(
state.base.wrapping_add(update_start as usize),
state.rowHashLog + ZSTD_ROW_HASH_TAG_BITS,
mls,
state.hashSalt,
) as u32
}
};
let rel_row = (hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log;
let row = state.hashTable.wrapping_add(rel_row as usize);
let tag_row = state.tagTable.wrapping_add(rel_row as usize);
let pos = unsafe { row_next_index(tag_row, row_mask) };
unsafe {
byte_set(tag_row, pos as usize, (hash & ZSTD_ROW_HASH_TAG_MASK) as u8);
table_set(row, pos as usize, update_start);
}
update_start = update_start.wrapping_add(1);
}
}
unsafe fn row_update_internal(
state: &ZSTD_RustLazyState,
ip: *const u8,
mls: u32,
row_log: u32,
row_mask: u32,
use_cache: bool,
) {
let mut idx = unsafe { next_to_update(state) };
let target = unsafe { index_from(state.base, ip) };
const SKIP_THRESHOLD: u32 = 384;
const MAX_START: u32 = 96;
const MAX_END: u32 = 32;
if use_cache && target.wrapping_sub(idx) > SKIP_THRESHOLD {
let bound = idx.wrapping_add(MAX_START);
unsafe { row_update_internal_impl(state, idx, bound, mls, row_log, row_mask, use_cache) };
idx = target.wrapping_sub(MAX_END);
unsafe { row_fill_hash_cache(state, state.base, row_log, mls, idx, ip.wrapping_add(1)) };
}
unsafe { row_update_internal_impl(state, idx, target, mls, row_log, row_mask, use_cache) };
unsafe { set_next_to_update(state, target) };
}
/// Returns matching tag positions in the same circular-buffer order as the
/// C SSE/SWAR mask iterator. The scalar version avoids architecture-specific
/// vector intrinsics while preserving the selected candidate order.
unsafe fn row_matching_positions(
tag_row: *const u8,
tag: u8,
head: u32,
row_mask: u32,
row_entries: u32,
out: &mut [u32; ZSTD_ROW_HASH_MAX_ENTRIES],
) -> usize {
let mut len = 0usize;
let mut relative = 0u32;
while relative < row_entries {
let pos = head.wrapping_add(relative) & row_mask;
if unsafe { byte_get(tag_row, pos as usize) } == tag {
out[len] = pos;
len += 1;
}
relative = relative.wrapping_add(1);
}
len
}
unsafe fn row_find_best_match(
state: &ZSTD_RustLazyState,
ip: *const u8,
ilimit: *const u8,
offset: &mut usize,
mls: u32,
dict_mode: c_int,
row_log: u32,
) -> usize {
let base = state.base;
let dict_base = state.dictBase;
let dict_limit = state.dictLimit;
let prefix_start = base.wrapping_add(dict_limit as usize);
let dict_end = dict_base.wrapping_add(dict_limit as usize);
let curr = unsafe { index_from(base, ip) };
let max_distance = 1u32.wrapping_shl(state.windowLog);
let lowest_valid = state.lowLimit;
let within_max_distance = if curr.wrapping_sub(lowest_valid) > max_distance {
curr.wrapping_sub(max_distance)
} else {
lowest_valid
};
let low_limit = if state.loadedDictEnd != 0 {
lowest_valid
} else {
within_max_distance
};
let row_entries = 1u32 << row_log;
let row_mask = row_entries - 1;
let capped_search_log = state.searchLog.min(row_log);
let mut nb_attempts = 1u32 << capped_search_log;
let mut ml = 3usize;
let (mut dds_idx, mut dds_extra_attempts) = (0usize, 0u32);
if dict_mode == DICT_DEDICATED {
let dms = unsafe { dict_state(state) };
let dds_hash_log = dms.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG;
dds_idx = unsafe { hash_ptr(ip, dds_hash_log, mls) } << ZSTD_LAZY_DDSS_BUCKET_LOG;
dds_extra_attempts = if state.searchLog > row_log {
1u32 << (state.searchLog - row_log)
} else {
0
};
}
let mut dms_tag = 0u32;
let mut dms_row = ptr::null_mut::<u32>();
let mut dms_tag_row = ptr::null_mut::<u8>();
if dict_mode == DICT_MATCH_STATE {
let dms = unsafe { dict_state(state) };
let dms_hash = unsafe { hash_ptr(ip, dms.rowHashLog + ZSTD_ROW_HASH_TAG_BITS, mls) as u32 };
let dms_rel_row = (dms_hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log;
dms_tag = dms_hash & ZSTD_ROW_HASH_TAG_MASK;
dms_tag_row = dms.tagTable.wrapping_add(dms_rel_row as usize);
dms_row = dms.hashTable.wrapping_add(dms_rel_row as usize);
}
let hash = if !unsafe { lazy_skipping(state) } {
unsafe { row_update_internal(state, ip, mls, row_log, row_mask, true) };
unsafe {
row_next_cached_hash(
state.hashCache,
state.hashTable,
state.tagTable,
base,
curr,
state.rowHashLog,
row_log,
mls,
state.hashSalt,
)
}
} else {
let h = unsafe {
hash_ptr_salted(
ip,
state.rowHashLog + ZSTD_ROW_HASH_TAG_BITS,
mls,
state.hashSalt,
) as u32
};
unsafe { set_next_to_update(state, curr) };
h
};
unsafe { add_hash_salt_entropy(state, hash) };
let rel_row = (hash >> ZSTD_ROW_HASH_TAG_BITS) << row_log;
let tag = (hash & ZSTD_ROW_HASH_TAG_MASK) as u8;
let row = state.hashTable.wrapping_add(rel_row as usize);
let tag_row = state.tagTable.wrapping_add(rel_row as usize);
let head = unsafe { byte_get(tag_row, 0) as u32 } & row_mask;
let mut positions = [0u32; ZSTD_ROW_HASH_MAX_ENTRIES];
let position_len = unsafe {
row_matching_positions(tag_row, tag, head, row_mask, row_entries, &mut positions)
};
let mut match_buffer = [0u32; ZSTD_ROW_HASH_MAX_ENTRIES];
let mut num_matches = 0usize;
for pos in positions[..position_len].iter().copied() {
if nb_attempts == 0 {
break;
}
if pos == 0 {
continue;
}
let match_index = unsafe { table_get(row, pos as usize) };
if match_index < low_limit {
break;
}
match_buffer[num_matches] = match_index;
num_matches += 1;
nb_attempts = nb_attempts.wrapping_sub(1);
}
let insert_pos = unsafe { row_next_index(tag_row, row_mask) };
unsafe {
byte_set(tag_row, insert_pos as usize, tag);
let index = next_to_update(state);
table_set(row, insert_pos as usize, index);
set_next_to_update(state, index.wrapping_add(1));
}
for match_index in match_buffer[..num_matches].iter().copied() {
let mut current_ml = 0usize;
if dict_mode != DICT_EXT || match_index >= dict_limit {
let matched = base.wrapping_add(match_index as usize);
if unsafe { read32(matched.wrapping_add(ml - 3)) == read32(ip.wrapping_add(ml - 3)) } {
current_ml = unsafe { count(ip, matched, ilimit) };
}
} else {
let matched = dict_base.wrapping_add(match_index as usize);
if unsafe { read32(matched) == read32(ip) } {
current_ml = unsafe {
count_2segments(
ip.wrapping_add(4),
matched.wrapping_add(4),
ilimit,
dict_end,
prefix_start,
) + 4
};
}
}
if current_ml > ml {
ml = current_ml;
*offset = offset_to_offbase(curr.wrapping_sub(match_index)) as usize;
if ip.wrapping_add(current_ml) == ilimit {
break;
}
}
}
if dict_mode == DICT_DEDICATED {
ml = unsafe {
dedicated_dict_search(
offset,
ml,
nb_attempts.wrapping_add(dds_extra_attempts),
dict_state(state),
ip,
ilimit,
prefix_start,
curr,
dict_limit,
dds_idx,
)
};
} else if dict_mode == DICT_MATCH_STATE {
let dms = unsafe { dict_state(state) };
let dms_lowest_index = dms.dictLimit;
let dms_base = dms.base;
let dms_end = dms.nextSrc;
let dms_size = unsafe { index_from(dms_base, dms_end) };
let dms_index_delta = dict_limit.wrapping_sub(dms_size);
let dms_head = unsafe { byte_get(dms_tag_row, 0) as u32 } & row_mask;
let mut dms_positions = [0u32; ZSTD_ROW_HASH_MAX_ENTRIES];
let dms_position_len = unsafe {
row_matching_positions(
dms_tag_row,
dms_tag as u8,
dms_head,
row_mask,
row_entries,
&mut dms_positions,
)
};
let mut dms_matches = [0u32; ZSTD_ROW_HASH_MAX_ENTRIES];
let mut dms_count = 0usize;
for pos in dms_positions[..dms_position_len].iter().copied() {
if nb_attempts == 0 {
break;
}
if pos == 0 {
continue;
}
let match_index = unsafe { table_get(dms_row, pos as usize) };
if match_index < dms_lowest_index {
break;
}
dms_matches[dms_count] = match_index;
dms_count += 1;
nb_attempts = nb_attempts.wrapping_sub(1);
}
for match_index in dms_matches[..dms_count].iter().copied() {
let matched = dms_base.wrapping_add(match_index as usize);
let mut current_ml = 0usize;
if unsafe { read32(matched) == read32(ip) } {
current_ml = unsafe {
count_2segments(
ip.wrapping_add(4),
matched.wrapping_add(4),
ilimit,
dms_end,
prefix_start,
) + 4
};
}
if current_ml > ml {
ml = current_ml;
*offset =
offset_to_offbase(curr.wrapping_sub(match_index.wrapping_add(dms_index_delta)))
as usize;
if ip.wrapping_add(current_ml) == ilimit {
break;
}
}
}
}
ml
}
unsafe fn search_max(
state: &ZSTD_RustLazyState,
ip: *const u8,
iend: *const u8,
offset: &mut usize,
mls: u32,
row_log: u32,
search_method: c_int,
dict_mode: c_int,
) -> usize {
match search_method {
SEARCH_HASH_CHAIN => unsafe { hc_find_best_match(state, ip, iend, offset, mls, dict_mode) },
SEARCH_BINARY_TREE => unsafe {
bt_find_best_match(state, ip, iend, offset, mls, dict_mode)
},
SEARCH_ROW_HASH => unsafe {
row_find_best_match(state, ip, iend, offset, mls, dict_mode, row_log)
},
_ => 0,
}
}
#[inline]
fn score(length: usize, multiplier: usize, offbase: usize, adjustment: usize) -> i32 {
let raw = length
.wrapping_mul(multiplier)
.wrapping_sub(ZSTD_highbit32(offbase as u32) as usize)
.wrapping_add(adjustment);
raw as i32
}
/* ------------------------------------------------------------------------- */
/* Lazy parser for no dictionary, dictionary match state, and DDS */
/* ------------------------------------------------------------------------- */
unsafe fn compress_block_lazy_generic(
state: &ZSTD_RustLazyState,
seq_store: *mut SeqStore_t,
reps: *mut u32,
src: *const u8,
src_size: usize,
search_method: c_int,
depth: u32,
dict_mode: c_int,
) -> usize {
let istart = src;
let mut ip = istart;
let mut anchor = istart;
let iend = istart.wrapping_add(src_size);
let required = if search_method == SEARCH_ROW_HASH {
HASH_READ_SIZE + ZSTD_ROW_HASH_CACHE_SIZE
} else {
HASH_READ_SIZE
};
if src_size < required {
return src_size;
}
let ilimit = iend.wrapping_sub(required);
let base = state.base;
let prefix_lowest_index = state.dictLimit;
let prefix_lowest = base.wrapping_add(prefix_lowest_index as usize);
let mls = bounded(4, state.minMatch, 6);
let row_log = bounded(4, state.searchLog, 6);
let mut offset_1 = unsafe { *reps };
let mut offset_2 = unsafe { *reps.add(1) };
let mut offset_saved_1 = 0u32;
let mut offset_saved_2 = 0u32;
let is_dms = dict_mode == DICT_MATCH_STATE;
let is_dds = dict_mode == DICT_DEDICATED;
let is_dxs = is_dms || is_dds;
let dms = if is_dxs {
Some(unsafe { dict_state(state) })
} else {
None
};
let dict_lowest_index = dms.map_or(0, |value| value.dictLimit);
let dict_base = dms.map_or(ptr::null(), |value| value.base);
let dict_lowest = if is_dxs {
dict_base.wrapping_add(dict_lowest_index as usize)
} else {
ptr::null()
};
let dict_end = dms.map_or(ptr::null(), |value| value.nextSrc);
let dict_index_delta = if is_dxs {
prefix_lowest_index.wrapping_sub(unsafe { index_from(dict_base, dict_end) })
} else {
0
};
let dict_and_prefix_length = if is_dxs {
(ip as usize)
.wrapping_sub(prefix_lowest as usize)
.wrapping_add((dict_end as usize).wrapping_sub(dict_lowest as usize)) as u32
} else {
(ip as usize).wrapping_sub(prefix_lowest as usize) as u32
};
if dict_and_prefix_length == 0 {
ip = ip.wrapping_add(1);
}
if dict_mode == DICT_NO_DICT {
let curr = unsafe { index_from(base, ip) };
let window_low =
lowest_prefix_index(state.dictLimit, state.loadedDictEnd, curr, state.windowLog);
let max_rep = curr.wrapping_sub(window_low);
if offset_2 > max_rep {
offset_saved_2 = offset_2;
offset_2 = 0;
}
if offset_1 > max_rep {
offset_saved_1 = offset_1;
offset_1 = 0;
}
}
unsafe { set_lazy_skipping(state, false) };
if search_method == SEARCH_ROW_HASH {
unsafe { row_fill_hash_cache(state, base, row_log, mls, next_to_update(state), ilimit) };
}
while ptr_lt(ip, ilimit) {
let mut match_length = 0usize;
let mut offbase = REPCODE1_TO_OFFBASE as usize;
let mut start = ip.wrapping_add(1);
let mut store_directly = false;
if is_dxs {
let rep_index = unsafe { index_from(base, ip) }
.wrapping_add(1)
.wrapping_sub(offset_1);
let rep_match = if rep_index < prefix_lowest_index {
dict_base.wrapping_add(rep_index.wrapping_sub(dict_index_delta) as usize)
} else {
base.wrapping_add(rep_index as usize)
};
if index_overlap_check(prefix_lowest_index, rep_index)
&& unsafe { read32(rep_match) == read32(ip.wrapping_add(1)) }
{
let rep_end = if rep_index < prefix_lowest_index {
dict_end
} else {
iend
};
match_length = unsafe {
count_2segments(
ip.wrapping_add(5),
rep_match.wrapping_add(4),
iend,
rep_end,
prefix_lowest,
) + 4
};
if depth == 0 {
store_directly = true;
}
}
}
if dict_mode == DICT_NO_DICT
&& offset_1 > 0
&& unsafe {
read32(ip.wrapping_add(1).wrapping_sub(offset_1 as usize))
== read32(ip.wrapping_add(1))
}
{
match_length = unsafe {
count(
ip.wrapping_add(5),
ip.wrapping_add(5).wrapping_sub(offset_1 as usize),
iend,
) + 4
};
if depth == 0 {
store_directly = true;
}
}
if !store_directly {
let mut candidate = 999_999_999usize;
let found = unsafe {
search_max(
state,
ip,
iend,
&mut candidate,
mls,
row_log,
search_method,
dict_mode,
)
};
if found > match_length {
match_length = found;
start = ip;
offbase = candidate;
}
if match_length < 4 {
let step = (ip as usize).wrapping_sub(anchor as usize) >> K_SEARCH_STRENGTH;
ip = ip.wrapping_add(step.wrapping_add(1));
unsafe { set_lazy_skipping(state, step.wrapping_add(1) > K_LAZY_SKIPPING_STEP) };
continue;
}
if depth >= 1 {
while ptr_lt(ip, ilimit) {
ip = ip.wrapping_add(1);
if dict_mode == DICT_NO_DICT
&& offbase != 0
&& offset_1 > 0
&& unsafe { read32(ip) == read32(ip.wrapping_sub(offset_1 as usize)) }
{
let ml_rep = unsafe {
count(
ip.wrapping_add(4),
ip.wrapping_add(4).wrapping_sub(offset_1 as usize),
iend,
) + 4
};
if ml_rep >= 4
&& score(ml_rep, 3, REPCODE1_TO_OFFBASE as usize, 0)
> score(match_length, 3, offbase, 1)
{
match_length = ml_rep;
offbase = REPCODE1_TO_OFFBASE as usize;
start = ip;
}
}
if is_dxs {
let rep_index = unsafe { index_from(base, ip) }.wrapping_sub(offset_1);
let rep_match = if rep_index < prefix_lowest_index {
dict_base
.wrapping_add(rep_index.wrapping_sub(dict_index_delta) as usize)
} else {
base.wrapping_add(rep_index as usize)
};
if index_overlap_check(prefix_lowest_index, rep_index)
&& unsafe { read32(rep_match) == read32(ip) }
{
let rep_end = if rep_index < prefix_lowest_index {
dict_end
} else {
iend
};
let ml_rep = unsafe {
count_2segments(
ip.wrapping_add(4),
rep_match.wrapping_add(4),
iend,
rep_end,
prefix_lowest,
) + 4
};
if ml_rep >= 4
&& score(ml_rep, 3, REPCODE1_TO_OFFBASE as usize, 0)
> score(match_length, 3, offbase, 1)
{
match_length = ml_rep;
offbase = REPCODE1_TO_OFFBASE as usize;
start = ip;
}
}
}
let mut candidate = 999_999_999usize;
let found = unsafe {
search_max(
state,
ip,
iend,
&mut candidate,
mls,
row_log,
search_method,
dict_mode,
)
};
if found >= 4
&& score(found, 4, candidate, 0) > score(match_length, 4, offbase, 4)
{
match_length = found;
offbase = candidate;
start = ip;
continue;
}
if depth == 2 && ptr_lt(ip, ilimit) {
ip = ip.wrapping_add(1);
if dict_mode == DICT_NO_DICT
&& offbase != 0
&& offset_1 > 0
&& unsafe { read32(ip) == read32(ip.wrapping_sub(offset_1 as usize)) }
{
let ml_rep = unsafe {
count(
ip.wrapping_add(4),
ip.wrapping_add(4).wrapping_sub(offset_1 as usize),
iend,
) + 4
};
if ml_rep >= 4
&& score(ml_rep, 4, REPCODE1_TO_OFFBASE as usize, 0)
> score(match_length, 4, offbase, 1)
{
match_length = ml_rep;
offbase = REPCODE1_TO_OFFBASE as usize;
start = ip;
}
}
if is_dxs {
let rep_index = unsafe { index_from(base, ip) }.wrapping_sub(offset_1);
let rep_match = if rep_index < prefix_lowest_index {
dict_base
.wrapping_add(rep_index.wrapping_sub(dict_index_delta) as usize)
} else {
base.wrapping_add(rep_index as usize)
};
if index_overlap_check(prefix_lowest_index, rep_index)
&& unsafe { read32(rep_match) == read32(ip) }
{
let rep_end = if rep_index < prefix_lowest_index {
dict_end
} else {
iend
};
let ml_rep = unsafe {
count_2segments(
ip.wrapping_add(4),
rep_match.wrapping_add(4),
iend,
rep_end,
prefix_lowest,
) + 4
};
if ml_rep >= 4
&& score(ml_rep, 4, REPCODE1_TO_OFFBASE as usize, 0)
> score(match_length, 4, offbase, 1)
{
match_length = ml_rep;
offbase = REPCODE1_TO_OFFBASE as usize;
start = ip;
}
}
}
let mut second_candidate = 999_999_999usize;
let second_found = unsafe {
search_max(
state,
ip,
iend,
&mut second_candidate,
mls,
row_log,
search_method,
dict_mode,
)
};
if second_found >= 4
&& score(second_found, 4, second_candidate, 0)
> score(match_length, 4, offbase, 7)
{
match_length = second_found;
offbase = second_candidate;
start = ip;
continue;
}
}
break;
}
}
if offbase_is_offset(offbase) {
let offset = offbase_to_offset(offbase);
if dict_mode == DICT_NO_DICT {
while ptr_gt(start, anchor)
&& ptr_gt(start.wrapping_sub(offset as usize), prefix_lowest)
&& unsafe {
*start.wrapping_sub(1) == *start.wrapping_sub(offset as usize + 1)
}
{
start = start.wrapping_sub(1);
match_length += 1;
}
}
if is_dxs {
let match_index = unsafe { index_from(base, start) }.wrapping_sub(offset);
let mut matched = if match_index < prefix_lowest_index {
dict_base.wrapping_add(match_index.wrapping_sub(dict_index_delta) as usize)
} else {
base.wrapping_add(match_index as usize)
};
let match_start = if match_index < prefix_lowest_index {
dict_lowest
} else {
prefix_lowest
};
while ptr_gt(start, anchor)
&& ptr_gt(matched, match_start)
&& unsafe { *start.wrapping_sub(1) == *matched.wrapping_sub(1) }
{
start = start.wrapping_sub(1);
matched = matched.wrapping_sub(1);
match_length += 1;
}
}
offset_2 = offset_1;
offset_1 = offset;
}
}
let lit_length = (start as usize).wrapping_sub(anchor as usize);
unsafe {
store_seq(
seq_store,
lit_length,
anchor,
iend,
offbase as u32,
match_length,
)
};
anchor = start.wrapping_add(match_length);
ip = anchor;
if unsafe { lazy_skipping(state) } {
if search_method == SEARCH_ROW_HASH {
unsafe {
row_fill_hash_cache(state, base, row_log, mls, next_to_update(state), ilimit)
};
}
unsafe { set_lazy_skipping(state, false) };
}
if is_dxs {
while ptr_le(ip, ilimit) {
let current = unsafe { index_from(base, ip) };
let rep_index = current.wrapping_sub(offset_2);
let rep_match = if rep_index < prefix_lowest_index {
dict_base.wrapping_add(rep_index.wrapping_sub(dict_index_delta) as usize)
} else {
base.wrapping_add(rep_index as usize)
};
if index_overlap_check(prefix_lowest_index, rep_index)
&& unsafe { read32(rep_match) == read32(ip) }
{
let rep_end = if rep_index < prefix_lowest_index {
dict_end
} else {
iend
};
match_length = unsafe {
count_2segments(
ip.wrapping_add(4),
rep_match.wrapping_add(4),
iend,
rep_end,
prefix_lowest,
) + 4
};
std::mem::swap(&mut offset_2, &mut offset_1);
unsafe {
store_seq(
seq_store,
0,
anchor,
iend,
REPCODE1_TO_OFFBASE,
match_length,
)
};
ip = ip.wrapping_add(match_length);
anchor = ip;
continue;
}
break;
}
}
if dict_mode == DICT_NO_DICT {
while ptr_le(ip, ilimit)
&& offset_2 > 0
&& unsafe { read32(ip) == read32(ip.wrapping_sub(offset_2 as usize)) }
{
match_length = unsafe {
count(
ip.wrapping_add(4),
ip.wrapping_add(4).wrapping_sub(offset_2 as usize),
iend,
) + 4
};
std::mem::swap(&mut offset_2, &mut offset_1);
unsafe {
store_seq(
seq_store,
0,
anchor,
iend,
REPCODE1_TO_OFFBASE,
match_length,
)
};
ip = ip.wrapping_add(match_length);
anchor = ip;
}
}
}
if offset_saved_1 != 0 && offset_1 != 0 {
offset_saved_2 = offset_saved_1;
}
unsafe {
*reps = if offset_1 != 0 {
offset_1
} else {
offset_saved_1
};
*reps.add(1) = if offset_2 != 0 {
offset_2
} else {
offset_saved_2
};
}
(iend as usize).wrapping_sub(anchor as usize)
}
/* ------------------------------------------------------------------------- */
/* Lazy parser for external dictionaries */
/* ------------------------------------------------------------------------- */
unsafe fn compress_block_lazy_ext_dict_generic(
state: &ZSTD_RustLazyState,
seq_store: *mut SeqStore_t,
reps: *mut u32,
src: *const u8,
src_size: usize,
search_method: c_int,
depth: u32,
) -> usize {
let istart = src;
let mut ip = istart;
let mut anchor = istart;
let iend = istart.wrapping_add(src_size);
let required = if search_method == SEARCH_ROW_HASH {
HASH_READ_SIZE + ZSTD_ROW_HASH_CACHE_SIZE
} else {
HASH_READ_SIZE
};
if src_size < required {
return src_size;
}
let ilimit = iend.wrapping_sub(required);
let base = state.base;
let dict_limit = state.dictLimit;
let prefix_start = base.wrapping_add(dict_limit as usize);
let dict_base = state.dictBase;
let dict_end = dict_base.wrapping_add(dict_limit as usize);
let dict_start = dict_base.wrapping_add(state.lowLimit as usize);
let mls = bounded(4, state.minMatch, 6);
let row_log = bounded(4, state.searchLog, 6);
let mut offset_1 = unsafe { *reps };
let mut offset_2 = unsafe { *reps.add(1) };
if ip == prefix_start {
ip = ip.wrapping_add(1);
}
unsafe { set_lazy_skipping(state, false) };
if search_method == SEARCH_ROW_HASH {
unsafe { row_fill_hash_cache(state, base, row_log, mls, next_to_update(state), ilimit) };
}
while ptr_lt(ip, ilimit) {
let mut match_length = 0usize;
let mut offbase = REPCODE1_TO_OFFBASE as usize;
let mut start = ip.wrapping_add(1);
let mut curr = unsafe { index_from(base, ip) };
let mut store_directly = false;
let window_low = lowest_match_index(
state.lowLimit,
state.loadedDictEnd,
curr.wrapping_add(1),
state.windowLog,
);
let rep_index = curr.wrapping_add(1).wrapping_sub(offset_1);
let rep_base = if rep_index < dict_limit {
dict_base
} else {
base
};
let rep_match = rep_base.wrapping_add(rep_index as usize);
if index_overlap_check(dict_limit, rep_index)
&& offset_1 <= curr.wrapping_add(1).wrapping_sub(window_low)
&& unsafe { read32(ip.wrapping_add(1)) == read32(rep_match) }
{
let rep_end = if rep_index < dict_limit {
dict_end
} else {
iend
};
match_length = unsafe {
count_2segments(
ip.wrapping_add(5),
rep_match.wrapping_add(4),
iend,
rep_end,
prefix_start,
) + 4
};
if depth == 0 {
store_directly = true;
}
}
if !store_directly {
let mut candidate = 999_999_999usize;
let found = unsafe {
search_max(
state,
ip,
iend,
&mut candidate,
mls,
row_log,
search_method,
DICT_EXT,
)
};
if found > match_length {
match_length = found;
start = ip;
offbase = candidate;
}
if match_length < 4 {
let step = (ip as usize).wrapping_sub(anchor as usize) >> K_SEARCH_STRENGTH;
ip = ip.wrapping_add(step.wrapping_add(1));
unsafe { set_lazy_skipping(state, step > K_LAZY_SKIPPING_STEP) };
continue;
}
if depth >= 1 {
while ptr_lt(ip, ilimit) {
ip = ip.wrapping_add(1);
curr = curr.wrapping_add(1);
if offbase != 0 {
let window_low = lowest_match_index(
state.lowLimit,
state.loadedDictEnd,
curr,
state.windowLog,
);
let rep_index = curr.wrapping_sub(offset_1);
let rep_base = if rep_index < dict_limit {
dict_base
} else {
base
};
let rep_match = rep_base.wrapping_add(rep_index as usize);
if index_overlap_check(dict_limit, rep_index)
&& offset_1 <= curr.wrapping_sub(window_low)
&& unsafe { read32(ip) == read32(rep_match) }
{
let rep_end = if rep_index < dict_limit {
dict_end
} else {
iend
};
let rep_length = unsafe {
count_2segments(
ip.wrapping_add(4),
rep_match.wrapping_add(4),
iend,
rep_end,
prefix_start,
) + 4
};
if rep_length >= 4
&& score(rep_length, 3, REPCODE1_TO_OFFBASE as usize, 0)
> score(match_length, 3, offbase, 1)
{
match_length = rep_length;
offbase = REPCODE1_TO_OFFBASE as usize;
start = ip;
}
}
}
let mut candidate = 999_999_999usize;
let found = unsafe {
search_max(
state,
ip,
iend,
&mut candidate,
mls,
row_log,
search_method,
DICT_EXT,
)
};
if found >= 4
&& score(found, 4, candidate, 0) > score(match_length, 4, offbase, 4)
{
match_length = found;
offbase = candidate;
start = ip;
continue;
}
if depth == 2 && ptr_lt(ip, ilimit) {
ip = ip.wrapping_add(1);
curr = curr.wrapping_add(1);
if offbase != 0 {
let window_low = lowest_match_index(
state.lowLimit,
state.loadedDictEnd,
curr,
state.windowLog,
);
let rep_index = curr.wrapping_sub(offset_1);
let rep_base = if rep_index < dict_limit {
dict_base
} else {
base
};
let rep_match = rep_base.wrapping_add(rep_index as usize);
if index_overlap_check(dict_limit, rep_index)
&& offset_1 <= curr.wrapping_sub(window_low)
&& unsafe { read32(ip) == read32(rep_match) }
{
let rep_end = if rep_index < dict_limit {
dict_end
} else {
iend
};
let rep_length = unsafe {
count_2segments(
ip.wrapping_add(4),
rep_match.wrapping_add(4),
iend,
rep_end,
prefix_start,
) + 4
};
if rep_length >= 4
&& score(rep_length, 4, REPCODE1_TO_OFFBASE as usize, 0)
> score(match_length, 4, offbase, 1)
{
match_length = rep_length;
offbase = REPCODE1_TO_OFFBASE as usize;
start = ip;
}
}
}
let mut second_candidate = 999_999_999usize;
let second_found = unsafe {
search_max(
state,
ip,
iend,
&mut second_candidate,
mls,
row_log,
search_method,
DICT_EXT,
)
};
if second_found >= 4
&& score(second_found, 4, second_candidate, 0)
> score(match_length, 4, offbase, 7)
{
match_length = second_found;
offbase = second_candidate;
start = ip;
continue;
}
}
break;
}
}
if offbase_is_offset(offbase) {
let offset = offbase_to_offset(offbase);
let match_index = unsafe { index_from(base, start) }.wrapping_sub(offset);
let mut matched = if match_index < dict_limit {
dict_base.wrapping_add(match_index as usize)
} else {
base.wrapping_add(match_index as usize)
};
let match_start = if match_index < dict_limit {
dict_start
} else {
prefix_start
};
while ptr_gt(start, anchor)
&& ptr_gt(matched, match_start)
&& unsafe { *start.wrapping_sub(1) == *matched.wrapping_sub(1) }
{
start = start.wrapping_sub(1);
matched = matched.wrapping_sub(1);
match_length += 1;
}
offset_2 = offset_1;
offset_1 = offset;
}
}
let lit_length = (start as usize).wrapping_sub(anchor as usize);
unsafe {
store_seq(
seq_store,
lit_length,
anchor,
iend,
offbase as u32,
match_length,
)
};
anchor = start.wrapping_add(match_length);
ip = anchor;
if unsafe { lazy_skipping(state) } {
if search_method == SEARCH_ROW_HASH {
unsafe {
row_fill_hash_cache(state, base, row_log, mls, next_to_update(state), ilimit)
};
}
unsafe { set_lazy_skipping(state, false) };
}
while ptr_le(ip, ilimit) {
let rep_current = unsafe { index_from(base, ip) };
let window_low = lowest_match_index(
state.lowLimit,
state.loadedDictEnd,
rep_current,
state.windowLog,
);
let rep_index = rep_current.wrapping_sub(offset_2);
let rep_base = if rep_index < dict_limit {
dict_base
} else {
base
};
let rep_match = rep_base.wrapping_add(rep_index as usize);
if index_overlap_check(dict_limit, rep_index)
&& offset_2 <= rep_current.wrapping_sub(window_low)
&& unsafe { read32(ip) == read32(rep_match) }
{
let rep_end = if rep_index < dict_limit {
dict_end
} else {
iend
};
match_length = unsafe {
count_2segments(
ip.wrapping_add(4),
rep_match.wrapping_add(4),
iend,
rep_end,
prefix_start,
) + 4
};
std::mem::swap(&mut offset_2, &mut offset_1);
unsafe {
store_seq(
seq_store,
0,
anchor,
iend,
REPCODE1_TO_OFFBASE,
match_length,
)
};
ip = ip.wrapping_add(match_length);
anchor = ip;
continue;
}
break;
}
}
unsafe {
*reps = offset_1;
*reps.add(1) = offset_2;
}
(iend as usize).wrapping_sub(anchor as usize)
}
/* ------------------------------------------------------------------------- */
/* C ABI */
/* ------------------------------------------------------------------------- */
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_lazy_insertAndFindFirstIndex(
state: *mut ZSTD_RustLazyState,
ip: *const c_void,
) -> u32 {
let state = unsafe { &*state };
unsafe { insert_and_find_first_index_internal(state, ip.cast::<u8>(), state.minMatch, false) }
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_lazy_row_update(
state: *mut ZSTD_RustLazyState,
ip: *const c_void,
) {
let state = unsafe { &*state };
let row_log = bounded(4, state.searchLog, 6);
let row_mask = (1u32 << row_log) - 1;
let mls = state.minMatch.min(6);
unsafe { row_update_internal(state, ip.cast::<u8>(), mls, row_log, row_mask, false) };
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_lazy_loadDedicatedDict(
state: *mut ZSTD_RustLazyState,
ip: *const c_void,
) {
let state = unsafe { &*state };
unsafe { dedicated_dict_search_load_dictionary(state, ip.cast::<u8>()) };
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBlock_lazy(
state: *mut ZSTD_RustLazyState,
seq_store: *mut c_void,
reps: *mut u32,
src: *const c_void,
src_size: usize,
search_method: c_int,
depth: u32,
dict_mode: c_int,
) -> usize {
let state = unsafe { &*state };
if dict_mode == DICT_EXT {
unsafe {
compress_block_lazy_ext_dict_generic(
state,
seq_store.cast::<SeqStore_t>(),
reps,
src.cast::<u8>(),
src_size,
search_method,
depth,
)
}
} else {
unsafe {
compress_block_lazy_generic(
state,
seq_store.cast::<SeqStore_t>(),
reps,
src.cast::<u8>(),
src_size,
search_method,
depth,
dict_mode,
)
}
}
}