ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(rust): port double-fast block matching

Browse Source 
Move the two-table fast match finder into Rust while retaining a small C
projection of the private match-state. The Rust implementation handles normal,
attached-dictionary, and external-dictionary matching without exposing the
full C context ABI. The C entry points and build-time exclusion guards remain
unchanged for configured consumers.

Document the migrated matcher in the Rust component map and narrow the
remaining-C boundary accordingly.

Test Plan:
- cargo test --all-targets
- cargo test --target i686-unknown-linux-gnu --all-targets
- cargo clippy && cargo clippy --benches && cargo clippy --tests
- cargo +nightly fmt
- make -B -C tests -j2 fuzzer zstreamtest invalidDictionaries poolTests
- ./tests/fuzzer -s5346 -i1 --no-big-tests
- ./tests/zstreamtest -i3000 -s334462
- ./tests/invalidDictionaries
- timeout 20s stdbuf -oL ./tests/poolTests

Refs: rust/README.md
This commit is contained in:
ddidderr
2026-07-11 08:05:25 +02:00
parent d316a14ec3
commit f9adcf6aef
4 changed files with 1885 additions and 740 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_double_fast.c
+61 -739
View File
@@ -8,771 +8,93 @@
* You may select, at your option, one of the above-listed licenses.
*/
/* The double-fast algorithms live in rust/src/zstd_double_fast.rs. Keep the
* private match-state layout in C and pass only the matching leaf fields. */
#include "zstd_compress_internal.h"
#include "zstd_double_fast.h"
#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_fillDoubleHashTableForCDict(ZSTD_MatchState_t* ms,
void const* end, ZSTD_dictTableLoadMethod_e dtlm)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashLarge = ms->hashTable;
U32 const hBitsL = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
U32 const mls = cParams->minMatch;
U32* const hashSmall = ms->chainTable;
U32 const hBitsS = cParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS;
const BYTE* const base = ms->window.base;
const BYTE* ip = base + ms->nextToUpdate;
const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
const U32 fastHashFillStep = 3;
typedef char ZSTD_rust_double_fast_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
typedef char ZSTD_rust_double_fast_seqstore_long_length_pos[
(offsetof(SeqStore_t, longLengthPos) == 9 * sizeof(size_t) + 4) ? 1 : -1];
typedef char ZSTD_rust_double_fast_seqstore_layout[
(sizeof(SeqStore_t) == 9 * sizeof(size_t) + 8) ? 1 : -1];
typedef char ZSTD_rust_double_fast_rep_count[(ZSTD_REP_NUM == 3) ? 1 : -1];
/* Always insert every fastHashFillStep position into the hash tables.
* Insert the other positions into the large hash table if their entry
* is empty.
*/
for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) {
U32 const curr = (U32)(ip - base);
U32 i;
for (i = 0; i < fastHashFillStep; ++i) {
size_t const smHashAndTag = ZSTD_hashPtr(ip + i, hBitsS, mls);
size_t const lgHashAndTag = ZSTD_hashPtr(ip + i, hBitsL, 8);
if (i == 0) {
ZSTD_writeTaggedIndex(hashSmall, smHashAndTag, curr + i);
}
if (i == 0 || hashLarge[lgHashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) {
ZSTD_writeTaggedIndex(hashLarge, lgHashAndTag, curr + i);
}
/* Only load extra positions for ZSTD_dtlm_full */
if (dtlm == ZSTD_dtlm_fast)
break;
} }
}
void ZSTD_rust_fillDoubleHashTable(
U32* hashLong, U32* hashSmall, const BYTE* base, U32 nextToUpdate,
const void* end, U32 hashLog, U32 chainLog, U32 minMatch,
int fullTableLoad, int forCDict);
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_fillDoubleHashTableForCCtx(ZSTD_MatchState_t* ms,
void const* end, ZSTD_dictTableLoadMethod_e dtlm)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashLarge = ms->hashTable;
U32 const hBitsL = cParams->hashLog;
U32 const mls = cParams->minMatch;
U32* const hashSmall = ms->chainTable;
U32 const hBitsS = cParams->chainLog;
const BYTE* const base = ms->window.base;
const BYTE* ip = base + ms->nextToUpdate;
const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
const U32 fastHashFillStep = 3;
size_t ZSTD_rust_compressBlock_doubleFast(
U32* hashLong, U32* hashSmall, const BYTE* base,
U32 dictLimit, U32 loadedDictEnd,
U32 hashLog, U32 chainLog, U32 minMatch, U32 windowLog,
void* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize);
/* Always insert every fastHashFillStep position into the hash tables.
* Insert the other positions into the large hash table if their entry
* is empty.
*/
for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) {
U32 const curr = (U32)(ip - base);
U32 i;
for (i = 0; i < fastHashFillStep; ++i) {
size_t const smHash = ZSTD_hashPtr(ip + i, hBitsS, mls);
size_t const lgHash = ZSTD_hashPtr(ip + i, hBitsL, 8);
if (i == 0)
hashSmall[smHash] = curr + i;
if (i == 0 || hashLarge[lgHash] == 0)
hashLarge[lgHash] = curr + i;
/* Only load extra positions for ZSTD_dtlm_full */
if (dtlm == ZSTD_dtlm_fast)
break;
} }
}
size_t ZSTD_rust_compressBlock_doubleFast_dictMatchState(
U32* hashLong, U32* hashSmall, const BYTE* base, U32 prefixLowestIndex,
U32 hashLog, U32 chainLog, U32 minMatch,
void* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize,
const U32* dictHashLong, const U32* dictHashSmall, const BYTE* dictBase,
U32 dictStartIndex, const BYTE* dictEnd,
U32 dictHashLog, U32 dictChainLog, int prefetchCDictTables);
size_t ZSTD_rust_compressBlock_doubleFast_extDict(
U32* hashLong, U32* hashSmall, const BYTE* base, const BYTE* dictBase,
U32 dictLimit, U32 lowLimit, U32 loadedDictEnd,
U32 hashLog, U32 chainLog, U32 minMatch, U32 windowLog,
void* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize);
void ZSTD_fillDoubleHashTable(ZSTD_MatchState_t* ms,
const void* const end,
ZSTD_dictTableLoadMethod_e dtlm,
ZSTD_tableFillPurpose_e tfp)
const void* end,
ZSTD_dictTableLoadMethod_e dtlm,
ZSTD_tableFillPurpose_e tfp)
{
if (tfp == ZSTD_tfp_forCDict) {
ZSTD_fillDoubleHashTableForCDict(ms, end, dtlm);
} else {
ZSTD_fillDoubleHashTableForCCtx(ms, end, dtlm);
}
ZSTD_rust_fillDoubleHashTable(
ms->hashTable, ms->chainTable, ms->window.base, ms->nextToUpdate,
end, ms->cParams.hashLog, ms->cParams.chainLog, ms->cParams.minMatch,
dtlm == ZSTD_dtlm_full, tfp == ZSTD_tfp_forCDict);
}
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_doubleFast_noDict_generic(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize, U32 const mls /* template */)
{
ZSTD_compressionParameters const* cParams = &ms->cParams;
U32* const hashLong = ms->hashTable;
const U32 hBitsL = cParams->hashLog;
U32* const hashSmall = ms->chainTable;
const U32 hBitsS = cParams->chainLog;
const BYTE* const base = ms->window.base;
const BYTE* const istart = (const BYTE*)src;
const BYTE* anchor = istart;
const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
/* presumes that, if there is a dictionary, it must be using Attach mode */
const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog);
const BYTE* const prefixLowest = base + prefixLowestIndex;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - HASH_READ_SIZE;
U32 offset_1=rep[0], offset_2=rep[1];
U32 offsetSaved1 = 0, offsetSaved2 = 0;
size_t mLength;
U32 offset;
U32 curr;
/* how many positions to search before increasing step size */
const size_t kStepIncr = 1 << kSearchStrength;
/* the position at which to increment the step size if no match is found */
const BYTE* nextStep;
size_t step; /* the current step size */
size_t hl0; /* the long hash at ip */
size_t hl1; /* the long hash at ip1 */
U32 idxl0; /* the long match index for ip */
U32 idxl1; /* the long match index for ip1 */
const BYTE* matchl0; /* the long match for ip */
const BYTE* matchs0; /* the short match for ip */
const BYTE* matchl1; /* the long match for ip1 */
const BYTE* matchs0_safe; /* matchs0 or safe address */
const BYTE* ip = istart; /* the current position */
const BYTE* ip1; /* the next position */
/* Array of ~random data, should have low probability of matching data
* we load from here instead of from tables, if matchl0/matchl1 are
* invalid indices. Used to avoid unpredictable branches. */
const BYTE dummy[] = {0x12,0x34,0x56,0x78,0x9a,0xbc,0xde,0xf0,0xe2,0xb4};
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_noDict_generic");
/* init */
ip += ((ip - prefixLowest) == 0);
{
U32 const current = (U32)(ip - base);
U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog);
U32 const maxRep = current - windowLow;
if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0;
if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0;
}
/* Outer Loop: one iteration per match found and stored */
while (1) {
step = 1;
nextStep = ip + kStepIncr;
ip1 = ip + step;
if (ip1 > ilimit) {
goto _cleanup;
}
hl0 = ZSTD_hashPtr(ip, hBitsL, 8);
idxl0 = hashLong[hl0];
matchl0 = base + idxl0;
/* Inner Loop: one iteration per search / position */
do {
const size_t hs0 = ZSTD_hashPtr(ip, hBitsS, mls);
const U32 idxs0 = hashSmall[hs0];
curr = (U32)(ip-base);
matchs0 = base + idxs0;
hashLong[hl0] = hashSmall[hs0] = curr; /* update hash tables */
/* check noDict repcode */
if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {
mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
ip++;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
goto _match_stored;
}
hl1 = ZSTD_hashPtr(ip1, hBitsL, 8);
/* idxl0 > prefixLowestIndex is a (somewhat) unpredictable branch.
* However expression below complies into conditional move. Since
* match is unlikely and we only *branch* on idxl0 > prefixLowestIndex
* if there is a match, all branches become predictable. */
{ const BYTE* const matchl0_safe = ZSTD_selectAddr(idxl0, prefixLowestIndex, matchl0, &dummy[0]);
/* check prefix long match */
if (MEM_read64(matchl0_safe) == MEM_read64(ip) && matchl0_safe == matchl0) {
mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8;
offset = (U32)(ip-matchl0);
while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */
goto _match_found;
} }
idxl1 = hashLong[hl1];
matchl1 = base + idxl1;
/* Same optimization as matchl0 above */
matchs0_safe = ZSTD_selectAddr(idxs0, prefixLowestIndex, matchs0, &dummy[0]);
/* check prefix short match */
if(MEM_read32(matchs0_safe) == MEM_read32(ip) && matchs0_safe == matchs0) {
goto _search_next_long;
}
if (ip1 >= nextStep) {
PREFETCH_L1(ip1 + 64);
PREFETCH_L1(ip1 + 128);
step++;
nextStep += kStepIncr;
}
ip = ip1;
ip1 += step;
hl0 = hl1;
idxl0 = idxl1;
matchl0 = matchl1;
#if defined(__aarch64__)
PREFETCH_L1(ip+256);
#endif
} while (ip1 <= ilimit);
_cleanup:
/* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0),
* rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */
offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2;
/* save reps for next block */
rep[0] = offset_1 ? offset_1 : offsetSaved1;
rep[1] = offset_2 ? offset_2 : offsetSaved2;
/* Return the last literals size */
return (size_t)(iend - anchor);
_search_next_long:
/* short match found: let's check for a longer one */
mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4;
offset = (U32)(ip - matchs0);
/* check long match at +1 position */
if ((idxl1 > prefixLowestIndex) && (MEM_read64(matchl1) == MEM_read64(ip1))) {
size_t const l1len = ZSTD_count(ip1+8, matchl1+8, iend) + 8;
if (l1len > mLength) {
/* use the long match instead */
ip = ip1;
mLength = l1len;
offset = (U32)(ip-matchl1);
matchs0 = matchl1;
}
}
while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* complete backward */
/* fall-through */
_match_found: /* requires ip, offset, mLength */
offset_2 = offset_1;
offset_1 = offset;
if (step < 4) {
/* It is unsafe to write this value back to the hashtable when ip1 is
* greater than or equal to the new ip we will have after we're done
* processing this match. Rather than perform that test directly
* (ip1 >= ip + mLength), which costs speed in practice, we do a simpler
* more predictable test. The minmatch even if we take a short match is
* 4 bytes, so as long as step, the distance between ip and ip1
* (initially) is less than 4, we know ip1 < new ip. */
hashLong[hl1] = (U32)(ip1 - base);
}
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
_match_stored:
/* match found */
ip += mLength;
anchor = ip;
if (ip <= ilimit) {
/* Complementary insertion */
/* done after iLimit test, as candidates could be > iend-8 */
{ U32 const indexToInsert = curr+2;
hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert;
hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert;
hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base);
}
/* check immediate repcode */
while ( (ip <= ilimit)
&& ( (offset_2>0)
& (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
/* store sequence */
size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base);
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base);
ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, rLength);
ip += rLength;
anchor = ip;
continue; /* faster when present ... (?) */
}
}
}
}
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize,
U32 const mls /* template */)
{
ZSTD_compressionParameters const* cParams = &ms->cParams;
U32* const hashLong = ms->hashTable;
const U32 hBitsL = cParams->hashLog;
U32* const hashSmall = ms->chainTable;
const U32 hBitsS = cParams->chainLog;
const BYTE* const base = ms->window.base;
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
/* presumes that, if there is a dictionary, it must be using Attach mode */
const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog);
const BYTE* const prefixLowest = base + prefixLowestIndex;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - HASH_READ_SIZE;
U32 offset_1=rep[0], offset_2=rep[1];
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
const ZSTD_compressionParameters* const dictCParams = &dms->cParams;
const U32* const dictHashLong = dms->hashTable;
const U32* const dictHashSmall = dms->chainTable;
const U32 dictStartIndex = dms->window.dictLimit;
const BYTE* const dictBase = dms->window.base;
const BYTE* const dictStart = dictBase + dictStartIndex;
const BYTE* const dictEnd = dms->window.nextSrc;
const U32 dictIndexDelta = prefixLowestIndex - (U32)(dictEnd - dictBase);
const U32 dictHBitsL = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
const U32 dictHBitsS = dictCParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS;
const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart));
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_dictMatchState_generic");
/* if a dictionary is attached, it must be within window range */
assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex);
if (ms->prefetchCDictTables) {
size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32);
size_t const chainTableBytes = (((size_t)1) << dictCParams->chainLog) * sizeof(U32);
PREFETCH_AREA(dictHashLong, hashTableBytes);
PREFETCH_AREA(dictHashSmall, chainTableBytes);
}
/* init */
ip += (dictAndPrefixLength == 0);
/* dictMatchState repCode checks don't currently handle repCode == 0
* disabling. */
assert(offset_1 <= dictAndPrefixLength);
assert(offset_2 <= dictAndPrefixLength);
/* Main Search Loop */
while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
size_t mLength;
U32 offset;
size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8);
size_t const h = ZSTD_hashPtr(ip, hBitsS, mls);
size_t const dictHashAndTagL = ZSTD_hashPtr(ip, dictHBitsL, 8);
size_t const dictHashAndTagS = ZSTD_hashPtr(ip, dictHBitsS, mls);
U32 const dictMatchIndexAndTagL = dictHashLong[dictHashAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS];
U32 const dictMatchIndexAndTagS = dictHashSmall[dictHashAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS];
int const dictTagsMatchL = ZSTD_comparePackedTags(dictMatchIndexAndTagL, dictHashAndTagL);
int const dictTagsMatchS = ZSTD_comparePackedTags(dictMatchIndexAndTagS, dictHashAndTagS);
U32 const curr = (U32)(ip-base);
U32 const matchIndexL = hashLong[h2];
U32 matchIndexS = hashSmall[h];
const BYTE* matchLong = base + matchIndexL;
const BYTE* match = base + matchIndexS;
const U32 repIndex = curr + 1 - offset_1;
const BYTE* repMatch = (repIndex < prefixLowestIndex) ?
dictBase + (repIndex - dictIndexDelta) :
base + repIndex;
hashLong[h2] = hashSmall[h] = curr; /* update hash tables */
/* check repcode */
if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
ip++;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
goto _match_stored;
}
if ((matchIndexL >= prefixLowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) {
/* check prefix long match */
mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8;
offset = (U32)(ip-matchLong);
while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
goto _match_found;
} else if (dictTagsMatchL) {
/* check dictMatchState long match */
U32 const dictMatchIndexL = dictMatchIndexAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS;
const BYTE* dictMatchL = dictBase + dictMatchIndexL;
assert(dictMatchL < dictEnd);
if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) {
mLength = ZSTD_count_2segments(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8;
offset = (U32)(curr - dictMatchIndexL - dictIndexDelta);
while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */
goto _match_found;
} }
if (matchIndexS > prefixLowestIndex) {
/* short match candidate */
if (MEM_read32(match) == MEM_read32(ip)) {
goto _search_next_long;
}
} else if (dictTagsMatchS) {
/* check dictMatchState short match */
U32 const dictMatchIndexS = dictMatchIndexAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS;
match = dictBase + dictMatchIndexS;
matchIndexS = dictMatchIndexS + dictIndexDelta;
if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) {
goto _search_next_long;
} }
ip += ((ip-anchor) >> kSearchStrength) + 1;
#if defined(__aarch64__)
PREFETCH_L1(ip+256);
#endif
continue;
_search_next_long:
{ size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
size_t const dictHashAndTagL3 = ZSTD_hashPtr(ip+1, dictHBitsL, 8);
U32 const matchIndexL3 = hashLong[hl3];
U32 const dictMatchIndexAndTagL3 = dictHashLong[dictHashAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS];
int const dictTagsMatchL3 = ZSTD_comparePackedTags(dictMatchIndexAndTagL3, dictHashAndTagL3);
const BYTE* matchL3 = base + matchIndexL3;
hashLong[hl3] = curr + 1;
/* check prefix long +1 match */
if ((matchIndexL3 >= prefixLowestIndex) && (MEM_read64(matchL3) == MEM_read64(ip+1))) {
mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8;
ip++;
offset = (U32)(ip-matchL3);
while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */
goto _match_found;
} else if (dictTagsMatchL3) {
/* check dict long +1 match */
U32 const dictMatchIndexL3 = dictMatchIndexAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS;
const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3;
assert(dictMatchL3 < dictEnd);
if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) {
mLength = ZSTD_count_2segments(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8;
ip++;
offset = (U32)(curr + 1 - dictMatchIndexL3 - dictIndexDelta);
while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */
goto _match_found;
} } }
/* if no long +1 match, explore the short match we found */
if (matchIndexS < prefixLowestIndex) {
mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4;
offset = (U32)(curr - matchIndexS);
while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
} else {
mLength = ZSTD_count(ip+4, match+4, iend) + 4;
offset = (U32)(ip - match);
while (((ip>anchor) & (match>prefixLowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
}
_match_found:
offset_2 = offset_1;
offset_1 = offset;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
_match_stored:
/* match found */
ip += mLength;
anchor = ip;
if (ip <= ilimit) {
/* Complementary insertion */
/* done after iLimit test, as candidates could be > iend-8 */
{ U32 const indexToInsert = curr+2;
hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert;
hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert;
hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base);
}
/* check immediate repcode */
while (ip <= ilimit) {
U32 const current2 = (U32)(ip-base);
U32 const repIndex2 = current2 - offset_2;
const BYTE* repMatch2 = repIndex2 < prefixLowestIndex ?
dictBase + repIndex2 - dictIndexDelta :
base + repIndex2;
if ( (ZSTD_index_overlap_check(prefixLowestIndex, repIndex2))
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4;
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
ip += repLength2;
anchor = ip;
continue;
}
break;
}
}
} /* while (ip < ilimit) */
/* save reps for next block */
rep[0] = offset_1;
rep[1] = offset_2;
/* Return the last literals size */
return (size_t)(iend - anchor);
}
#define ZSTD_GEN_DFAST_FN(dictMode, mls) \
static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
void const* src, size_t srcSize) \
{ \
return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \
}
ZSTD_GEN_DFAST_FN(noDict, 4)
ZSTD_GEN_DFAST_FN(noDict, 5)
ZSTD_GEN_DFAST_FN(noDict, 6)
ZSTD_GEN_DFAST_FN(noDict, 7)
ZSTD_GEN_DFAST_FN(dictMatchState, 4)
ZSTD_GEN_DFAST_FN(dictMatchState, 5)
ZSTD_GEN_DFAST_FN(dictMatchState, 6)
ZSTD_GEN_DFAST_FN(dictMatchState, 7)
size_t ZSTD_compressBlock_doubleFast(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
const void* src, size_t srcSize)
{
const U32 mls = ms->cParams.minMatch;
switch(mls)
{
default: /* includes case 3 */
case 4 :
return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize);
case 5 :
return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize);
case 6 :
return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize);
case 7 :
return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize);
}
return ZSTD_rust_compressBlock_doubleFast(
ms->hashTable, ms->chainTable, ms->window.base,
ms->window.dictLimit, ms->loadedDictEnd,
ms->cParams.hashLog, ms->cParams.chainLog, ms->cParams.minMatch,
ms->cParams.windowLog,
seqStore, rep, src, srcSize);
}
size_t ZSTD_compressBlock_doubleFast_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
const void* src, size_t srcSize)
{
const U32 mls = ms->cParams.minMatch;
switch(mls)
{
default: /* includes case 3 */
case 4 :
return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize);
case 5 :
return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize);
case 6 :
return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize);
case 7 :
return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize);
}
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
assert(dms != NULL);
return ZSTD_rust_compressBlock_doubleFast_dictMatchState(
ms->hashTable, ms->chainTable, ms->window.base, ms->window.dictLimit,
ms->cParams.hashLog, ms->cParams.chainLog, ms->cParams.minMatch,
seqStore, rep, src, srcSize,
dms->hashTable, dms->chainTable, dms->window.base,
dms->window.dictLimit, dms->window.nextSrc,
dms->cParams.hashLog, dms->cParams.chainLog, ms->prefetchCDictTables);
}
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_doubleFast_extDict_generic(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize,
U32 const mls /* template */)
{
ZSTD_compressionParameters const* cParams = &ms->cParams;
U32* const hashLong = ms->hashTable;
U32 const hBitsL = cParams->hashLog;
U32* const hashSmall = ms->chainTable;
U32 const hBitsS = cParams->chainLog;
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip = istart;
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
const BYTE* const base = ms->window.base;
const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog);
const U32 dictStartIndex = lowLimit;
const U32 dictLimit = ms->window.dictLimit;
const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit;
const BYTE* const prefixStart = base + prefixStartIndex;
const BYTE* const dictBase = ms->window.dictBase;
const BYTE* const dictStart = dictBase + dictStartIndex;
const BYTE* const dictEnd = dictBase + prefixStartIndex;
U32 offset_1=rep[0], offset_2=rep[1];
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize);
/* if extDict is invalidated due to maxDistance, switch to "regular" variant */
if (prefixStartIndex == dictStartIndex)
return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize);
/* Search Loop */
while (ip < ilimit) { /* < instead of <=, because (ip+1) */
const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls);
const U32 matchIndex = hashSmall[hSmall];
const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base;
const BYTE* match = matchBase + matchIndex;
const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8);
const U32 matchLongIndex = hashLong[hLong];
const BYTE* const matchLongBase = matchLongIndex < prefixStartIndex ? dictBase : base;
const BYTE* matchLong = matchLongBase + matchLongIndex;
const U32 curr = (U32)(ip-base);
const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */
const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
size_t mLength;
hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */
if (((ZSTD_index_overlap_check(prefixStartIndex, repIndex))
& (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
ip++;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
} else {
if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) {
const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend;
const BYTE* const lowMatchPtr = matchLongIndex < prefixStartIndex ? dictStart : prefixStart;
U32 offset;
mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, prefixStart) + 8;
offset = curr - matchLongIndex;
while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
offset_2 = offset_1;
offset_1 = offset;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
} else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) {
size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
U32 const matchIndex3 = hashLong[h3];
const BYTE* const match3Base = matchIndex3 < prefixStartIndex ? dictBase : base;
const BYTE* match3 = match3Base + matchIndex3;
U32 offset;
hashLong[h3] = curr + 1;
if ( (matchIndex3 > dictStartIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) {
const BYTE* const matchEnd = matchIndex3 < prefixStartIndex ? dictEnd : iend;
const BYTE* const lowMatchPtr = matchIndex3 < prefixStartIndex ? dictStart : prefixStart;
mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, prefixStart) + 8;
ip++;
offset = curr+1 - matchIndex3;
while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */
} else {
const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend;
const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart;
mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4;
offset = curr - matchIndex;
while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
}
offset_2 = offset_1;
offset_1 = offset;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
} else {
ip += ((ip-anchor) >> kSearchStrength) + 1;
continue;
} }
/* move to next sequence start */
ip += mLength;
anchor = ip;
if (ip <= ilimit) {
/* Complementary insertion */
/* done after iLimit test, as candidates could be > iend-8 */
{ U32 const indexToInsert = curr+2;
hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert;
hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert;
hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base);
}
/* check immediate repcode */
while (ip <= ilimit) {
U32 const current2 = (U32)(ip-base);
U32 const repIndex2 = current2 - offset_2;
const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2))
& (offset_2 <= current2 - dictStartIndex))
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
ip += repLength2;
anchor = ip;
continue;
}
break;
} } }
/* save reps for next block */
rep[0] = offset_1;
rep[1] = offset_2;
/* Return the last literals size */
return (size_t)(iend - anchor);
}
ZSTD_GEN_DFAST_FN(extDict, 4)
ZSTD_GEN_DFAST_FN(extDict, 5)
ZSTD_GEN_DFAST_FN(extDict, 6)
ZSTD_GEN_DFAST_FN(extDict, 7)
size_t ZSTD_compressBlock_doubleFast_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
const void* src, size_t srcSize)
{
U32 const mls = ms->cParams.minMatch;
switch(mls)
{
default: /* includes case 3 */
case 4 :
return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize);
case 5 :
return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize);
case 6 :
return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize);
case 7 :
return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize);
}
return ZSTD_rust_compressBlock_doubleFast_extDict(
ms->hashTable, ms->chainTable, ms->window.base, ms->window.dictBase,
ms->window.dictLimit, ms->window.lowLimit, ms->loadedDictEnd,
ms->cParams.hashLog, ms->cParams.chainLog, ms->cParams.minMatch,
ms->cParams.windowLog,
seqStore, rep, src, srcSize);
}
#endif /* ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR */
rust/README.md
+3 -1
View File
@@ -31,13 +31,15 @@ zstd ABI:
- `zstd_presplit` chooses split points for full compression blocks.
- `zstd_compress_literals` emits raw, RLE, and Huffman literal sections
while preserving the compressor's Huffman-table repeat state.
- `zstd_fast` and `zstd_double_fast` implement the single- and two-table
fast block match finders, including attached and external dictionary paths.
- Runtime support
- `threading` provides platform pthread wrappers required by zstd headers.
- `pool` implements the bounded worker pool used by multithreaded compression.
- Dictionary support
- `zstd_ddict` owns, loads, copies, and references decode dictionaries.
The remaining block compression, general decompression, dictionary-building,
The lazy and optimal block matchers, general decompression, dictionary-building,
legacy, and CLI translation units are still C. They must move before the
rewrite is complete. Keeping that boundary explicit prevents a passing hybrid
build from being mistaken for the final all-Rust result.
rust/src/lib.rs
+2
View File
@@ -32,6 +32,8 @@ pub mod zstd_compress_superblock;
#[cfg(feature = "decompression")]
pub mod zstd_ddict;
#[cfg(feature = "compression")]
pub mod zstd_double_fast;
#[cfg(feature = "compression")]
pub mod zstd_fast;
#[cfg(feature = "compression")]
pub mod zstd_presplit;
rust/src/zstd_double_fast.rs
+1819
View File
@@ -0,0 +1,1819 @@
#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::too_many_arguments)]
//! Double-fast block match finder.
//!
//! The C shim keeps `ZSTD_MatchState_t` opaque and supplies only the leaf
//! fields used by this two-table matcher. The long (8-byte) and short
//! (min-match) search loops, table fills, and sequence writes live here.
use crate::mem::{
MEM_64bits, MEM_isLittleEndian, MEM_read16, MEM_read32, MEM_read64, MEM_readLE32, MEM_readLE64,
MEM_readST,
};
use std::ffi::c_void;
use std::mem::size_of;
use std::os::raw::c_int;
use std::ptr;
const ZSTD_REP_NUM: usize = 3;
const MINMATCH: usize = 3;
const HASH_READ_SIZE: usize = 8;
const SHORT_CACHE_TAG_BITS: u32 = 8;
const SHORT_CACHE_TAG_MASK: u32 = (1 << SHORT_CACHE_TAG_BITS) - 1;
const K_SEARCH_STRENGTH: usize = 8;
const REPCODE1_TO_OFFBASE: u32 = 1;
#[repr(C)]
#[derive(Clone, Copy)]
struct SeqDef {
offBase: u32,
litLength: u16,
mlBase: u16,
}
/// The C shim checks this leaf ABI. `ZSTD_MatchState_t` remains wholly in C.
#[repr(C)]
struct SeqStore_t {
sequencesStart: *mut SeqDef,
sequences: *mut SeqDef,
litStart: *mut u8,
lit: *mut u8,
llCode: *mut u8,
mlCode: *mut u8,
ofCode: *mut u8,
maxNbSeq: usize,
maxNbLit: usize,
longLengthType: c_int,
longLengthPos: u32,
}
#[inline]
fn ptr_lt(left: *const u8, right: *const u8) -> bool {
(left as usize) < (right as usize)
}
#[inline]
fn ptr_le(left: *const u8, right: *const u8) -> bool {
(left as usize) <= (right as usize)
}
#[inline]
fn ptr_gt(left: *const u8, right: *const u8) -> bool {
(left as usize) > (right as usize)
}
#[inline]
fn ptr_ge(left: *const u8, right: *const u8) -> bool {
(left as usize) >= (right as usize)
}
#[inline]
unsafe fn index_from(base: *const u8, ptr: *const u8) -> u32 {
unsafe { ptr.offset_from(base) as u32 }
}
#[inline]
unsafe fn read32(ptr: *const u8) -> u32 {
unsafe { MEM_read32(ptr.cast::<c_void>()) }
}
#[inline]
unsafe fn read64(ptr: *const u8) -> u64 {
unsafe { MEM_read64(ptr.cast::<c_void>()) }
}
#[inline]
unsafe fn table_get(table: *const u32, index: usize) -> u32 {
unsafe { *table.add(index) }
}
#[inline]
unsafe fn table_set(table: *mut u32, index: usize, value: u32) {
unsafe { *table.add(index) = value };
}
#[inline]
fn hash_shift32(value: u32, hbits: u32) -> usize {
if hbits == 0 {
0
} else {
(value >> (32 - hbits)) as usize
}
}
#[inline]
fn hash_shift64(value: u64, hbits: u32) -> usize {
if hbits == 0 {
0
} else {
(value >> (64 - hbits)) as usize
}
}
#[inline]
unsafe fn hash_ptr(ptr: *const u8, hbits: u32, mls: u32) -> usize {
match mls {
5 => {
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
hash_shift64(value.wrapping_shl(24).wrapping_mul(889_523_592_379), hbits)
}
6 => {
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
hash_shift64(
value.wrapping_shl(16).wrapping_mul(227_718_039_650_203),
hbits,
)
}
7 => {
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
hash_shift64(
value.wrapping_shl(8).wrapping_mul(58_295_818_150_454_627),
hbits,
)
}
8 => {
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
hash_shift64(value.wrapping_mul(0xCF1B_BCDC_B7A5_6463), hbits)
}
_ => {
let value = unsafe { MEM_readLE32(ptr.cast::<c_void>()) };
hash_shift32(value.wrapping_mul(2_654_435_761), hbits)
}
}
}
#[inline]
fn common_bytes(word: usize) -> usize {
let zeros = if MEM_isLittleEndian() {
word.trailing_zeros()
} else {
word.leading_zeros()
};
(zeros / 8) as usize
}
unsafe fn count(mut input: *const u8, mut matched: *const u8, input_limit: *const u8) -> usize {
let input_start = input;
let word_size = size_of::<usize>();
while unsafe { input_limit.offset_from(input) as usize } >= word_size {
let diff =
unsafe { MEM_readST(matched.cast::<c_void>()) ^ MEM_readST(input.cast::<c_void>()) };
if diff != 0 {
return unsafe { input.offset_from(input_start) as usize } + common_bytes(diff);
}
input = input.wrapping_add(word_size);
matched = matched.wrapping_add(word_size);
}
if MEM_64bits()
&& unsafe { input_limit.offset_from(input) as usize } >= 4
&& unsafe { MEM_read32(matched.cast::<c_void>()) == MEM_read32(input.cast::<c_void>()) }
{
input = input.wrapping_add(4);
matched = matched.wrapping_add(4);
}
if unsafe { input_limit.offset_from(input) as usize } >= 2
&& unsafe { MEM_read16(matched.cast::<c_void>()) == MEM_read16(input.cast::<c_void>()) }
{
input = input.wrapping_add(2);
matched = matched.wrapping_add(2);
}
if ptr_lt(input, input_limit) && unsafe { *input == *matched } {
input = input.wrapping_add(1);
}
unsafe { input.offset_from(input_start) as usize }
}
unsafe fn count_2segments(
input: *const u8,
matched: *const u8,
input_end: *const u8,
match_end: *const u8,
input_start: *const u8,
) -> usize {
let match_remaining = unsafe { match_end.offset_from(matched) as usize };
let input_remaining = unsafe { input_end.offset_from(input) as usize };
let first_end = input.wrapping_add(match_remaining.min(input_remaining));
let first_count = unsafe { count(input, matched, first_end) };
if matched.wrapping_add(first_count) != match_end {
return first_count;
}
first_count + unsafe { count(input.wrapping_add(first_count), input_start, input_end) }
}
#[inline]
fn lowest_prefix_index(dict_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 {
let max_distance = 1u32.wrapping_shl(window_log);
let within_window = if curr.wrapping_sub(dict_limit) > max_distance {
curr.wrapping_sub(max_distance)
} else {
dict_limit
};
if loaded_dict_end != 0 {
dict_limit
} else {
within_window
}
}
#[inline]
fn lowest_match_index(low_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 {
let max_distance = 1u32.wrapping_shl(window_log);
let within_window = if curr.wrapping_sub(low_limit) > max_distance {
curr.wrapping_sub(max_distance)
} else {
low_limit
};
if loaded_dict_end != 0 {
low_limit
} else {
within_window
}
}
#[inline]
fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool {
prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3
}
#[inline]
fn write_tagged_index(table: *mut u32, hash_and_tag: usize, index: u32) {
let hash = hash_and_tag >> SHORT_CACHE_TAG_BITS;
let tag = (hash_and_tag as u32) & SHORT_CACHE_TAG_MASK;
unsafe { table_set(table, hash, (index << SHORT_CACHE_TAG_BITS) | tag) };
}
#[inline]
fn packed_tags_match(first: u32, second: usize) -> bool {
(first & SHORT_CACHE_TAG_MASK) == ((second as u32) & SHORT_CACHE_TAG_MASK)
}
unsafe fn store_seq(
seq_store: *mut SeqStore_t,
lit_length: usize,
literals: *const u8,
_lit_limit: *const u8,
off_base: u32,
match_length: usize,
) {
let seq_store = unsafe { &mut *seq_store };
let sequence = seq_store.sequences;
debug_assert!(
unsafe { sequence.offset_from(seq_store.sequencesStart) as usize } < seq_store.maxNbSeq
);
debug_assert!(match_length >= MINMATCH);
if lit_length != 0 {
unsafe { ptr::copy_nonoverlapping(literals, seq_store.lit, lit_length) };
}
seq_store.lit = seq_store.lit.wrapping_add(lit_length);
let sequence_index = unsafe { sequence.offset_from(seq_store.sequencesStart) as u32 };
if lit_length > u16::MAX as usize {
debug_assert_eq!(seq_store.longLengthType, 0);
seq_store.longLengthType = 1;
seq_store.longLengthPos = sequence_index;
}
unsafe { (*sequence).litLength = lit_length as u16 };
unsafe { (*sequence).offBase = off_base };
let match_base = match_length - MINMATCH;
if match_base > u16::MAX as usize {
debug_assert_eq!(seq_store.longLengthType, 0);
seq_store.longLengthType = 2;
seq_store.longLengthPos = sequence_index;
}
unsafe { (*sequence).mlBase = match_base as u16 };
seq_store.sequences = sequence.wrapping_add(1);
}
#[inline]
fn fast_mls(min_match: u32) -> u32 {
match min_match {
5..=7 => min_match,
_ => 4,
}
}
unsafe fn fill_double_hash_table(
hash_long: *mut u32,
hash_small: *mut u32,
base: *const u8,
next_to_update: u32,
end: *const u8,
hash_log: u32,
chain_log: u32,
min_match: u32,
full_table_load: bool,
tagged_indices: bool,
) {
if unsafe { end.offset_from(base) } < HASH_READ_SIZE as isize {
return;
}
let hbits_long = hash_log
+ if tagged_indices {
SHORT_CACHE_TAG_BITS
} else {
0
};
let hbits_small = chain_log
+ if tagged_indices {
SHORT_CACHE_TAG_BITS
} else {
0
};
let mut input = base.wrapping_add(next_to_update as usize);
let input_end = end.wrapping_sub(HASH_READ_SIZE);
while ptr_le(input.wrapping_add(2), input_end) {
let current = unsafe { index_from(base, input) };
for position in 0..3usize {
let small_hash =
unsafe { hash_ptr(input.wrapping_add(position), hbits_small, min_match) };
let long_hash = unsafe { hash_ptr(input.wrapping_add(position), hbits_long, 8) };
let small_index = if tagged_indices {
small_hash >> SHORT_CACHE_TAG_BITS
} else {
small_hash
};
let long_index = if tagged_indices {
long_hash >> SHORT_CACHE_TAG_BITS
} else {
long_hash
};
if position == 0 {
if tagged_indices {
write_tagged_index(
hash_small,
small_hash,
current.wrapping_add(position as u32),
);
} else {
unsafe {
table_set(
hash_small,
small_index,
current.wrapping_add(position as u32),
)
};
}
}
if position == 0 || unsafe { table_get(hash_long, long_index) } == 0 {
if tagged_indices {
write_tagged_index(hash_long, long_hash, current.wrapping_add(position as u32));
} else {
unsafe {
table_set(hash_long, long_index, current.wrapping_add(position as u32))
};
}
}
if !full_table_load {
break;
}
}
input = input.wrapping_add(3);
}
}
/// Rust implementation called by the C ABI wrapper for `ZSTD_fillDoubleHashTable`.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_fillDoubleHashTable(
hash_long: *mut u32,
hash_small: *mut u32,
base: *const u8,
next_to_update: u32,
end: *const c_void,
hash_log: u32,
chain_log: u32,
min_match: u32,
full_table_load: c_int,
for_cdict: c_int,
) {
unsafe {
fill_double_hash_table(
hash_long,
hash_small,
base,
next_to_update,
end.cast::<u8>(),
hash_log,
chain_log,
min_match,
full_table_load != 0,
for_cdict != 0,
)
};
}
unsafe fn post_no_dict_match(
hash_long: *mut u32,
hash_small: *mut u32,
base: *const u8,
hbits_long: u32,
hbits_small: u32,
mls: u32,
seq_store: *mut SeqStore_t,
offset1: &mut u32,
offset2: &mut u32,
mut ip: *const u8,
match_length: usize,
current: u32,
iend: *const u8,
ilimit: *const u8,
) -> (*const u8, *const u8) {
ip = ip.wrapping_add(match_length);
let mut anchor = ip;
if ptr_le(ip, ilimit) {
let index_to_insert = current.wrapping_add(2);
let hash = unsafe { hash_ptr(base.wrapping_add(index_to_insert as usize), hbits_long, 8) };
unsafe { table_set(hash_long, hash, index_to_insert) };
let hash = unsafe { hash_ptr(ip.wrapping_sub(2), hbits_long, 8) };
unsafe { table_set(hash_long, hash, index_from(base, ip.wrapping_sub(2))) };
let hash = unsafe {
hash_ptr(
base.wrapping_add(index_to_insert as usize),
hbits_small,
mls,
)
};
unsafe { table_set(hash_small, hash, index_to_insert) };
let hash = unsafe { hash_ptr(ip.wrapping_sub(1), hbits_small, mls) };
unsafe { table_set(hash_small, hash, index_from(base, ip.wrapping_sub(1))) };
while ptr_le(ip, ilimit)
&& *offset2 > 0
&& unsafe { read32(ip) == read32(ip.wrapping_sub(*offset2 as usize)) }
{
let repeat_length = unsafe {
count(
ip.wrapping_add(4),
ip.wrapping_add(4).wrapping_sub(*offset2 as usize),
iend,
) + 4
};
std::mem::swap(offset1, offset2);
let hash = unsafe { hash_ptr(ip, hbits_small, mls) };
unsafe { table_set(hash_small, hash, index_from(base, ip)) };
let hash = unsafe { hash_ptr(ip, hbits_long, 8) };
unsafe { table_set(hash_long, hash, index_from(base, ip)) };
unsafe {
store_seq(
seq_store,
0,
anchor,
iend,
REPCODE1_TO_OFFBASE,
repeat_length,
)
};
ip = ip.wrapping_add(repeat_length);
anchor = ip;
}
}
(ip, anchor)
}
unsafe fn compress_block_double_fast_no_dict(
hash_long: *mut u32,
hash_small: *mut u32,
base: *const u8,
dict_limit: u32,
loaded_dict_end: u32,
hash_log: u32,
chain_log: u32,
window_log: u32,
seq_store: *mut SeqStore_t,
reps: *mut u32,
src: *const u8,
src_size: usize,
mls: u32,
) -> usize {
if src_size < HASH_READ_SIZE {
return src_size;
}
let istart = src;
let iend = istart.wrapping_add(src_size);
let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
let end_index = unsafe { index_from(base, istart) }.wrapping_add(src_size as u32);
let prefix_lowest_index =
lowest_prefix_index(dict_limit, loaded_dict_end, end_index, window_log);
let prefix_lowest = base.wrapping_add(prefix_lowest_index as usize);
let mut ip = istart;
let mut anchor = istart;
let mut offset1 = unsafe { *reps };
let mut offset2 = unsafe { *reps.add(1) };
let mut offset_saved1 = 0u32;
let mut offset_saved2 = 0u32;
if ip == prefix_lowest {
ip = ip.wrapping_add(1);
}
let current = unsafe { index_from(base, ip) };
let window_low = lowest_prefix_index(dict_limit, loaded_dict_end, current, window_log);
let max_rep = current.wrapping_sub(window_low);
if offset2 > max_rep {
offset_saved2 = offset2;
offset2 = 0;
}
if offset1 > max_rep {
offset_saved1 = offset1;
offset1 = 0;
}
'outer: loop {
let mut step = 1usize;
let mut next_step = ip.wrapping_add(1 << K_SEARCH_STRENGTH);
let mut ip1 = ip.wrapping_add(step);
if ptr_gt(ip1, ilimit) {
break;
}
let mut hash_long0 = unsafe { hash_ptr(ip, hash_log, 8) };
let mut index_long0 = unsafe { table_get(hash_long, hash_long0) };
let mut match_long0 = base.wrapping_add(index_long0 as usize);
loop {
let hash_small0 = unsafe { hash_ptr(ip, chain_log, mls) };
let index_small0 = unsafe { table_get(hash_small, hash_small0) };
let mut match_small0 = base.wrapping_add(index_small0 as usize);
let current = unsafe { index_from(base, ip) };
unsafe {
table_set(hash_long, hash_long0, current);
table_set(hash_small, hash_small0, current);
}
if offset1 > 0
&& unsafe {
read32(ip.wrapping_add(1).wrapping_sub(offset1 as usize))
== read32(ip.wrapping_add(1))
}
{
let match_length = unsafe {
count(
ip.wrapping_add(5),
ip.wrapping_add(5).wrapping_sub(offset1 as usize),
iend,
) + 4
};
ip = ip.wrapping_add(1);
unsafe {
store_seq(
seq_store,
ip.offset_from(anchor) as usize,
anchor,
iend,
REPCODE1_TO_OFFBASE,
match_length,
)
};
let (next_ip, next_anchor) = unsafe {
post_no_dict_match(
hash_long,
hash_small,
base,
hash_log,
chain_log,
mls,
seq_store,
&mut offset1,
&mut offset2,
ip,
match_length,
current,
iend,
ilimit,
)
};
ip = next_ip;
anchor = next_anchor;
continue 'outer;
}
let hash_long1 = unsafe { hash_ptr(ip1, hash_log, 8) };
if index_long0 > prefix_lowest_index && unsafe { read64(match_long0) == read64(ip) } {
let mut match_length =
unsafe { count(ip.wrapping_add(8), match_long0.wrapping_add(8), iend) + 8 };
let offset = unsafe { index_from(match_long0, ip) };
while ptr_gt(ip, anchor)
&& ptr_gt(match_long0, prefix_lowest)
&& unsafe { *ip.wrapping_sub(1) == *match_long0.wrapping_sub(1) }
{
ip = ip.wrapping_sub(1);
match_long0 = match_long0.wrapping_sub(1);
match_length += 1;
}
offset2 = offset1;
offset1 = offset;
if step < 4 {
unsafe { table_set(hash_long, hash_long1, index_from(base, ip1)) };
}
unsafe {
store_seq(
seq_store,
ip.offset_from(anchor) as usize,
anchor,
iend,
offset.wrapping_add(ZSTD_REP_NUM as u32),
match_length,
)
};
let (next_ip, next_anchor) = unsafe {
post_no_dict_match(
hash_long,
hash_small,
base,
hash_log,
chain_log,
mls,
seq_store,
&mut offset1,
&mut offset2,
ip,
match_length,
current,
iend,
ilimit,
)
};
ip = next_ip;
anchor = next_anchor;
continue 'outer;
}
let index_long1 = unsafe { table_get(hash_long, hash_long1) };
let match_long1 = base.wrapping_add(index_long1 as usize);
if index_small0 > prefix_lowest_index && unsafe { read32(match_small0) == read32(ip) } {
let mut match_length =
unsafe { count(ip.wrapping_add(4), match_small0.wrapping_add(4), iend) + 4 };
let mut offset = unsafe { index_from(match_small0, ip) };
if index_long1 > prefix_lowest_index
&& unsafe { read64(match_long1) == read64(ip1) }
{
let long_length = unsafe {
count(ip1.wrapping_add(8), match_long1.wrapping_add(8), iend) + 8
};
if long_length > match_length {
ip = ip1;
match_length = long_length;
offset = unsafe { index_from(match_long1, ip) };
match_small0 = match_long1;
}
}
while ptr_gt(ip, anchor)
&& ptr_gt(match_small0, prefix_lowest)
&& unsafe { *ip.wrapping_sub(1) == *match_small0.wrapping_sub(1) }
{
ip = ip.wrapping_sub(1);
match_small0 = match_small0.wrapping_sub(1);
match_length += 1;
}
offset2 = offset1;
offset1 = offset;
if step < 4 {
unsafe { table_set(hash_long, hash_long1, index_from(base, ip1)) };
}
unsafe {
store_seq(
seq_store,
ip.offset_from(anchor) as usize,
anchor,
iend,
offset.wrapping_add(ZSTD_REP_NUM as u32),
match_length,
)
};
let (next_ip, next_anchor) = unsafe {
post_no_dict_match(
hash_long,
hash_small,
base,
hash_log,
chain_log,
mls,
seq_store,
&mut offset1,
&mut offset2,
ip,
match_length,
current,
iend,
ilimit,
)
};
ip = next_ip;
anchor = next_anchor;
continue 'outer;
}
if ptr_ge(ip1, next_step) {
step += 1;
next_step = next_step.wrapping_add(1 << K_SEARCH_STRENGTH);
}
ip = ip1;
ip1 = ip1.wrapping_add(step);
hash_long0 = hash_long1;
index_long0 = index_long1;
match_long0 = match_long1;
if ptr_gt(ip1, ilimit) {
break 'outer;
}
}
}
offset_saved2 = if offset_saved1 != 0 && offset1 != 0 {
offset_saved1
} else {
offset_saved2
};
unsafe {
*reps = if offset1 != 0 { offset1 } else { offset_saved1 };
*reps.add(1) = if offset2 != 0 { offset2 } else { offset_saved2 };
}
unsafe { iend.offset_from(anchor) as usize }
}
/// Rust implementation called by the C ABI wrapper for `ZSTD_compressBlock_doubleFast`.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBlock_doubleFast(
hash_long: *mut u32,
hash_small: *mut u32,
base: *const u8,
dict_limit: u32,
loaded_dict_end: u32,
hash_log: u32,
chain_log: u32,
min_match: u32,
window_log: u32,
seq_store: *mut c_void,
reps: *mut u32,
src: *const c_void,
src_size: usize,
) -> usize {
unsafe {
compress_block_double_fast_no_dict(
hash_long,
hash_small,
base,
dict_limit,
loaded_dict_end,
hash_log,
chain_log,
window_log,
seq_store.cast::<SeqStore_t>(),
reps,
src.cast::<u8>(),
src_size,
fast_mls(min_match),
)
}
}
unsafe fn post_dict_match(
hash_long: *mut u32,
hash_small: *mut u32,
base: *const u8,
dict_base: *const u8,
prefix_lowest_index: u32,
dict_index_delta: u32,
dict_end: *const u8,
prefix_lowest: *const u8,
hbits_long: u32,
hbits_small: u32,
mls: u32,
seq_store: *mut SeqStore_t,
offset1: &mut u32,
offset2: &mut u32,
mut ip: *const u8,
match_length: usize,
current: u32,
iend: *const u8,
ilimit: *const u8,
) -> (*const u8, *const u8) {
ip = ip.wrapping_add(match_length);
let mut anchor = ip;
if ptr_le(ip, ilimit) {
let index_to_insert = current.wrapping_add(2);
let hash = unsafe { hash_ptr(base.wrapping_add(index_to_insert as usize), hbits_long, 8) };
unsafe { table_set(hash_long, hash, index_to_insert) };
let hash = unsafe { hash_ptr(ip.wrapping_sub(2), hbits_long, 8) };
unsafe { table_set(hash_long, hash, index_from(base, ip.wrapping_sub(2))) };
let hash = unsafe {
hash_ptr(
base.wrapping_add(index_to_insert as usize),
hbits_small,
mls,
)
};
unsafe { table_set(hash_small, hash, index_to_insert) };
let hash = unsafe { hash_ptr(ip.wrapping_sub(1), hbits_small, mls) };
unsafe { table_set(hash_small, hash, index_from(base, ip.wrapping_sub(1))) };
while ptr_le(ip, ilimit) {
let current2 = unsafe { index_from(base, ip) };
let rep_index2 = current2.wrapping_sub(*offset2);
let rep_match2 = if rep_index2 < prefix_lowest_index {
dict_base
.wrapping_sub(dict_index_delta as usize)
.wrapping_add(rep_index2 as usize)
} else {
base.wrapping_add(rep_index2 as usize)
};
if index_overlap_check(prefix_lowest_index, rep_index2)
&& unsafe { read32(rep_match2) == read32(ip) }
{
let rep_end2 = if rep_index2 < prefix_lowest_index {
dict_end
} else {
iend
};
let repeat_length = unsafe {
count_2segments(
ip.wrapping_add(4),
rep_match2.wrapping_add(4),
iend,
rep_end2,
prefix_lowest,
) + 4
};
std::mem::swap(offset1, offset2);
unsafe {
store_seq(
seq_store,
0,
anchor,
iend,
REPCODE1_TO_OFFBASE,
repeat_length,
)
};
let hash = unsafe { hash_ptr(ip, hbits_small, mls) };
unsafe { table_set(hash_small, hash, current2) };
let hash = unsafe { hash_ptr(ip, hbits_long, 8) };
unsafe { table_set(hash_long, hash, current2) };
ip = ip.wrapping_add(repeat_length);
anchor = ip;
continue;
}
break;
}
}
(ip, anchor)
}
unsafe fn compress_block_double_fast_dict_match_state(
hash_long: *mut u32,
hash_small: *mut u32,
base: *const u8,
prefix_lowest_index: u32,
hash_log: u32,
chain_log: u32,
seq_store: *mut SeqStore_t,
reps: *mut u32,
src: *const u8,
src_size: usize,
mls: u32,
dict_hash_long: *const u32,
dict_hash_small: *const u32,
dict_base: *const u8,
dict_start_index: u32,
dict_end: *const u8,
dict_hash_log: u32,
dict_chain_log: u32,
_prefetch_cdict_tables: bool,
) -> usize {
if src_size < HASH_READ_SIZE {
return src_size;
}
let istart = src;
let iend = istart.wrapping_add(src_size);
let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
let mut ip = istart;
let mut anchor = istart;
let prefix_lowest = base.wrapping_add(prefix_lowest_index as usize);
let dict_start = dict_base.wrapping_add(dict_start_index as usize);
let dict_index_delta =
prefix_lowest_index.wrapping_sub(unsafe { index_from(dict_base, dict_end) });
let dict_hbits_long = dict_hash_log + SHORT_CACHE_TAG_BITS;
let dict_hbits_small = dict_chain_log + SHORT_CACHE_TAG_BITS;
let dict_and_prefix_length =
unsafe { ip.offset_from(prefix_lowest) as u32 + dict_end.offset_from(dict_start) as u32 };
let mut offset1 = unsafe { *reps };
let mut offset2 = unsafe { *reps.add(1) };
if dict_and_prefix_length == 0 {
ip = ip.wrapping_add(1);
}
'outer: while ptr_lt(ip, ilimit) {
let h_long = unsafe { hash_ptr(ip, hash_log, 8) };
let h_small = unsafe { hash_ptr(ip, chain_log, mls) };
let dict_hash_and_tag_long = unsafe { hash_ptr(ip, dict_hbits_long, 8) };
let dict_hash_and_tag_small = unsafe { hash_ptr(ip, dict_hbits_small, mls) };
let dict_match_index_and_tag_long = unsafe {
table_get(
dict_hash_long,
dict_hash_and_tag_long >> SHORT_CACHE_TAG_BITS,
)
};
let dict_match_index_and_tag_small = unsafe {
table_get(
dict_hash_small,
dict_hash_and_tag_small >> SHORT_CACHE_TAG_BITS,
)
};
let dict_tags_match_long =
packed_tags_match(dict_match_index_and_tag_long, dict_hash_and_tag_long);
let dict_tags_match_small =
packed_tags_match(dict_match_index_and_tag_small, dict_hash_and_tag_small);
let current = unsafe { index_from(base, ip) };
let match_index_long = unsafe { table_get(hash_long, h_long) };
let mut match_index_small = unsafe { table_get(hash_small, h_small) };
let mut match_long = base.wrapping_add(match_index_long as usize);
let mut matched = base.wrapping_add(match_index_small as usize);
let rep_index = current.wrapping_add(1).wrapping_sub(offset1);
let rep_match = if rep_index < prefix_lowest_index {
dict_base.wrapping_add(rep_index.wrapping_sub(dict_index_delta) as usize)
} else {
base.wrapping_add(rep_index as usize)
};
unsafe {
table_set(hash_long, h_long, current);
table_set(hash_small, h_small, current);
}
if index_overlap_check(prefix_lowest_index, rep_index)
&& unsafe { read32(rep_match) == read32(ip.wrapping_add(1)) }
{
let rep_match_end = if rep_index < prefix_lowest_index {
dict_end
} else {
iend
};
let match_length = unsafe {
count_2segments(
ip.wrapping_add(5),
rep_match.wrapping_add(4),
iend,
rep_match_end,
prefix_lowest,
) + 4
};
ip = ip.wrapping_add(1);
unsafe {
store_seq(
seq_store,
ip.offset_from(anchor) as usize,
anchor,
iend,
REPCODE1_TO_OFFBASE,
match_length,
)
};
let (next_ip, next_anchor) = unsafe {
post_dict_match(
hash_long,
hash_small,
base,
dict_base,
prefix_lowest_index,
dict_index_delta,
dict_end,
prefix_lowest,
hash_log,
chain_log,
mls,
seq_store,
&mut offset1,
&mut offset2,
ip,
match_length,
current,
iend,
ilimit,
)
};
ip = next_ip;
anchor = next_anchor;
continue;
}
if match_index_long >= prefix_lowest_index && unsafe { read64(match_long) == read64(ip) } {
let mut match_length =
unsafe { count(ip.wrapping_add(8), match_long.wrapping_add(8), iend) + 8 };
let offset = unsafe { index_from(match_long, ip) };
while ptr_gt(ip, anchor)
&& ptr_gt(match_long, prefix_lowest)
&& unsafe { *ip.wrapping_sub(1) == *match_long.wrapping_sub(1) }
{
ip = ip.wrapping_sub(1);
match_long = match_long.wrapping_sub(1);
match_length += 1;
}
offset2 = offset1;
offset1 = offset;
unsafe {
store_seq(
seq_store,
ip.offset_from(anchor) as usize,
anchor,
iend,
offset.wrapping_add(ZSTD_REP_NUM as u32),
match_length,
)
};
let (next_ip, next_anchor) = unsafe {
post_dict_match(
hash_long,
hash_small,
base,
dict_base,
prefix_lowest_index,
dict_index_delta,
dict_end,
prefix_lowest,
hash_log,
chain_log,
mls,
seq_store,
&mut offset1,
&mut offset2,
ip,
match_length,
current,
iend,
ilimit,
)
};
ip = next_ip;
anchor = next_anchor;
continue;
}
if dict_tags_match_long {
let dict_match_index = dict_match_index_and_tag_long >> SHORT_CACHE_TAG_BITS;
let mut dict_match = dict_base.wrapping_add(dict_match_index as usize);
if ptr_gt(dict_match, dict_start) && unsafe { read64(dict_match) == read64(ip) } {
let mut match_length = unsafe {
count_2segments(
ip.wrapping_add(8),
dict_match.wrapping_add(8),
iend,
dict_end,
prefix_lowest,
) + 8
};
let offset = current
.wrapping_sub(dict_match_index)
.wrapping_sub(dict_index_delta);
while ptr_gt(ip, anchor)
&& ptr_gt(dict_match, dict_start)
&& unsafe { *ip.wrapping_sub(1) == *dict_match.wrapping_sub(1) }
{
ip = ip.wrapping_sub(1);
dict_match = dict_match.wrapping_sub(1);
match_length += 1;
}
offset2 = offset1;
offset1 = offset;
unsafe {
store_seq(
seq_store,
ip.offset_from(anchor) as usize,
anchor,
iend,
offset.wrapping_add(ZSTD_REP_NUM as u32),
match_length,
)
};
let (next_ip, next_anchor) = unsafe {
post_dict_match(
hash_long,
hash_small,
base,
dict_base,
prefix_lowest_index,
dict_index_delta,
dict_end,
prefix_lowest,
hash_log,
chain_log,
mls,
seq_store,
&mut offset1,
&mut offset2,
ip,
match_length,
current,
iend,
ilimit,
)
};
ip = next_ip;
anchor = next_anchor;
continue;
}
}
let mut short_found = false;
if match_index_small > prefix_lowest_index {
short_found = unsafe { read32(matched) == read32(ip) };
} else if dict_tags_match_small {
let dict_match_index = dict_match_index_and_tag_small >> SHORT_CACHE_TAG_BITS;
let dict_match = dict_base.wrapping_add(dict_match_index as usize);
match_index_small = dict_match_index.wrapping_add(dict_index_delta);
if ptr_gt(dict_match, dict_start) && unsafe { read32(dict_match) == read32(ip) } {
matched = dict_match;
short_found = true;
}
}
if !short_found {
ip = ip.wrapping_add(
(unsafe { ip.offset_from(anchor) as usize } >> K_SEARCH_STRENGTH) + 1,
);
continue;
}
let hash_long3 = unsafe { hash_ptr(ip.wrapping_add(1), hash_log, 8) };
let dict_hash_and_tag_long3 = unsafe { hash_ptr(ip.wrapping_add(1), dict_hbits_long, 8) };
let match_index_long3 = unsafe { table_get(hash_long, hash_long3) };
let dict_match_index_and_tag_long3 = unsafe {
table_get(
dict_hash_long,
dict_hash_and_tag_long3 >> SHORT_CACHE_TAG_BITS,
)
};
let dict_tags_match_long3 =
packed_tags_match(dict_match_index_and_tag_long3, dict_hash_and_tag_long3);
let mut match_long3 = base.wrapping_add(match_index_long3 as usize);
unsafe { table_set(hash_long, hash_long3, current.wrapping_add(1)) };
let (match_length, offset) = if match_index_long3 >= prefix_lowest_index
&& unsafe { read64(match_long3) == read64(ip.wrapping_add(1)) }
{
let mut length =
unsafe { count(ip.wrapping_add(9), match_long3.wrapping_add(8), iend) + 8 };
ip = ip.wrapping_add(1);
let offset = unsafe { index_from(match_long3, ip) };
while ptr_gt(ip, anchor)
&& ptr_gt(match_long3, prefix_lowest)
&& unsafe { *ip.wrapping_sub(1) == *match_long3.wrapping_sub(1) }
{
ip = ip.wrapping_sub(1);
match_long3 = match_long3.wrapping_sub(1);
length += 1;
}
(length, offset)
} else if dict_tags_match_long3 {
let dict_match_index = dict_match_index_and_tag_long3 >> SHORT_CACHE_TAG_BITS;
let mut dict_match = dict_base.wrapping_add(dict_match_index as usize);
if ptr_gt(dict_match, dict_start)
&& unsafe { read64(dict_match) == read64(ip.wrapping_add(1)) }
{
let mut length = unsafe {
count_2segments(
ip.wrapping_add(9),
dict_match.wrapping_add(8),
iend,
dict_end,
prefix_lowest,
) + 8
};
ip = ip.wrapping_add(1);
let offset = current
.wrapping_add(1)
.wrapping_sub(dict_match_index)
.wrapping_sub(dict_index_delta);
while ptr_gt(ip, anchor)
&& ptr_gt(dict_match, dict_start)
&& unsafe { *ip.wrapping_sub(1) == *dict_match.wrapping_sub(1) }
{
ip = ip.wrapping_sub(1);
dict_match = dict_match.wrapping_sub(1);
length += 1;
}
(length, offset)
} else if match_index_small < prefix_lowest_index {
let mut length = unsafe {
count_2segments(
ip.wrapping_add(4),
matched.wrapping_add(4),
iend,
dict_end,
prefix_lowest,
) + 4
};
let offset = current.wrapping_sub(match_index_small);
while ptr_gt(ip, anchor)
&& ptr_gt(matched, dict_start)
&& unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
{
ip = ip.wrapping_sub(1);
matched = matched.wrapping_sub(1);
length += 1;
}
(length, offset)
} else {
let mut length =
unsafe { count(ip.wrapping_add(4), matched.wrapping_add(4), iend) + 4 };
let offset = unsafe { index_from(matched, ip) };
while ptr_gt(ip, anchor)
&& ptr_gt(matched, prefix_lowest)
&& unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
{
ip = ip.wrapping_sub(1);
matched = matched.wrapping_sub(1);
length += 1;
}
(length, offset)
}
} else if match_index_small < prefix_lowest_index {
let mut length = unsafe {
count_2segments(
ip.wrapping_add(4),
matched.wrapping_add(4),
iend,
dict_end,
prefix_lowest,
) + 4
};
let offset = current.wrapping_sub(match_index_small);
while ptr_gt(ip, anchor)
&& ptr_gt(matched, dict_start)
&& unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
{
ip = ip.wrapping_sub(1);
matched = matched.wrapping_sub(1);
length += 1;
}
(length, offset)
} else {
let mut length =
unsafe { count(ip.wrapping_add(4), matched.wrapping_add(4), iend) + 4 };
let offset = unsafe { index_from(matched, ip) };
while ptr_gt(ip, anchor)
&& ptr_gt(matched, prefix_lowest)
&& unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
{
ip = ip.wrapping_sub(1);
matched = matched.wrapping_sub(1);
length += 1;
}
(length, offset)
};
offset2 = offset1;
offset1 = offset;
unsafe {
store_seq(
seq_store,
ip.offset_from(anchor) as usize,
anchor,
iend,
offset.wrapping_add(ZSTD_REP_NUM as u32),
match_length,
)
};
let (next_ip, next_anchor) = unsafe {
post_dict_match(
hash_long,
hash_small,
base,
dict_base,
prefix_lowest_index,
dict_index_delta,
dict_end,
prefix_lowest,
hash_log,
chain_log,
mls,
seq_store,
&mut offset1,
&mut offset2,
ip,
match_length,
current,
iend,
ilimit,
)
};
ip = next_ip;
anchor = next_anchor;
continue 'outer;
}
unsafe {
*reps = offset1;
*reps.add(1) = offset2;
}
unsafe { iend.offset_from(anchor) as usize }
}
/// Rust implementation called by the C ABI wrapper for attached dictionaries.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBlock_doubleFast_dictMatchState(
hash_long: *mut u32,
hash_small: *mut u32,
base: *const u8,
prefix_lowest_index: u32,
hash_log: u32,
chain_log: u32,
min_match: u32,
seq_store: *mut c_void,
reps: *mut u32,
src: *const c_void,
src_size: usize,
dict_hash_long: *const u32,
dict_hash_small: *const u32,
dict_base: *const u8,
dict_start_index: u32,
dict_end: *const u8,
dict_hash_log: u32,
dict_chain_log: u32,
prefetch_cdict_tables: c_int,
) -> usize {
unsafe {
compress_block_double_fast_dict_match_state(
hash_long,
hash_small,
base,
prefix_lowest_index,
hash_log,
chain_log,
seq_store.cast::<SeqStore_t>(),
reps,
src.cast::<u8>(),
src_size,
fast_mls(min_match),
dict_hash_long,
dict_hash_small,
dict_base,
dict_start_index,
dict_end,
dict_hash_log,
dict_chain_log,
prefetch_cdict_tables != 0,
)
}
}
unsafe fn post_ext_match(
hash_long: *mut u32,
hash_small: *mut u32,
base: *const u8,
dict_base: *const u8,
dict_start_index: u32,
prefix_start_index: u32,
dict_end: *const u8,
prefix_start: *const u8,
hbits_long: u32,
hbits_small: u32,
mls: u32,
seq_store: *mut SeqStore_t,
offset1: &mut u32,
offset2: &mut u32,
mut ip: *const u8,
match_length: usize,
current: u32,
iend: *const u8,
ilimit: *const u8,
) -> (*const u8, *const u8) {
ip = ip.wrapping_add(match_length);
let mut anchor = ip;
if ptr_le(ip, ilimit) {
let index_to_insert = current.wrapping_add(2);
let hash = unsafe { hash_ptr(base.wrapping_add(index_to_insert as usize), hbits_long, 8) };
unsafe { table_set(hash_long, hash, index_to_insert) };
let hash = unsafe { hash_ptr(ip.wrapping_sub(2), hbits_long, 8) };
unsafe { table_set(hash_long, hash, index_from(base, ip.wrapping_sub(2))) };
let hash = unsafe {
hash_ptr(
base.wrapping_add(index_to_insert as usize),
hbits_small,
mls,
)
};
unsafe { table_set(hash_small, hash, index_to_insert) };
let hash = unsafe { hash_ptr(ip.wrapping_sub(1), hbits_small, mls) };
unsafe { table_set(hash_small, hash, index_from(base, ip.wrapping_sub(1))) };
while ptr_le(ip, ilimit) {
let current2 = unsafe { index_from(base, ip) };
let rep_index2 = current2.wrapping_sub(*offset2);
let rep_match2 = if rep_index2 < prefix_start_index {
dict_base.wrapping_add(rep_index2 as usize)
} else {
base.wrapping_add(rep_index2 as usize)
};
if index_overlap_check(prefix_start_index, rep_index2)
&& *offset2 <= current2.wrapping_sub(dict_start_index)
&& unsafe { read32(rep_match2) == read32(ip) }
{
let rep_end2 = if rep_index2 < prefix_start_index {
dict_end
} else {
iend
};
let repeat_length = unsafe {
count_2segments(
ip.wrapping_add(4),
rep_match2.wrapping_add(4),
iend,
rep_end2,
prefix_start,
) + 4
};
std::mem::swap(offset1, offset2);
unsafe {
store_seq(
seq_store,
0,
anchor,
iend,
REPCODE1_TO_OFFBASE,
repeat_length,
)
};
let hash = unsafe { hash_ptr(ip, hbits_small, mls) };
unsafe { table_set(hash_small, hash, current2) };
let hash = unsafe { hash_ptr(ip, hbits_long, 8) };
unsafe { table_set(hash_long, hash, current2) };
ip = ip.wrapping_add(repeat_length);
anchor = ip;
continue;
}
break;
}
}
(ip, anchor)
}
unsafe fn compress_block_double_fast_ext_dict(
hash_long: *mut u32,
hash_small: *mut u32,
base: *const u8,
dict_base: *const u8,
dict_limit: u32,
low_limit: u32,
loaded_dict_end: u32,
hash_log: u32,
chain_log: u32,
window_log: u32,
seq_store: *mut SeqStore_t,
reps: *mut u32,
src: *const u8,
src_size: usize,
mls: u32,
) -> usize {
if src_size < HASH_READ_SIZE {
return src_size;
}
let istart = src;
let iend = istart.wrapping_add(src_size);
let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
let end_index = unsafe { index_from(base, istart) }.wrapping_add(src_size as u32);
let dict_start_index = lowest_match_index(low_limit, loaded_dict_end, end_index, window_log);
let prefix_start_index = dict_limit.max(dict_start_index);
if prefix_start_index == dict_start_index {
return unsafe {
compress_block_double_fast_no_dict(
hash_long,
hash_small,
base,
dict_limit,
loaded_dict_end,
hash_log,
chain_log,
window_log,
seq_store,
reps,
src,
src_size,
mls,
)
};
}
let prefix_start = base.wrapping_add(prefix_start_index as usize);
let dict_start = dict_base.wrapping_add(dict_start_index as usize);
let dict_end = dict_base.wrapping_add(prefix_start_index as usize);
let mut ip = istart;
let mut anchor = istart;
let mut offset1 = unsafe { *reps };
let mut offset2 = unsafe { *reps.add(1) };
while ptr_lt(ip, ilimit) {
let h_small = unsafe { hash_ptr(ip, chain_log, mls) };
let match_index = unsafe { table_get(hash_small, h_small) };
let match_base = if match_index < prefix_start_index {
dict_base
} else {
base
};
let mut matched = match_base.wrapping_add(match_index as usize);
let h_long = unsafe { hash_ptr(ip, hash_log, 8) };
let match_long_index = unsafe { table_get(hash_long, h_long) };
let match_long_base = if match_long_index < prefix_start_index {
dict_base
} else {
base
};
let mut match_long = match_long_base.wrapping_add(match_long_index as usize);
let current = unsafe { index_from(base, ip) };
let rep_index = current.wrapping_add(1).wrapping_sub(offset1);
let rep_base = if rep_index < prefix_start_index {
dict_base
} else {
base
};
let rep_match = rep_base.wrapping_add(rep_index as usize);
unsafe {
table_set(hash_small, h_small, current);
table_set(hash_long, h_long, current);
}
if index_overlap_check(prefix_start_index, rep_index)
&& offset1 <= current.wrapping_add(1).wrapping_sub(dict_start_index)
&& unsafe { read32(rep_match) == read32(ip.wrapping_add(1)) }
{
let rep_end = if rep_index < prefix_start_index {
dict_end
} else {
iend
};
let match_length = unsafe {
count_2segments(
ip.wrapping_add(5),
rep_match.wrapping_add(4),
iend,
rep_end,
prefix_start,
) + 4
};
ip = ip.wrapping_add(1);
unsafe {
store_seq(
seq_store,
ip.offset_from(anchor) as usize,
anchor,
iend,
REPCODE1_TO_OFFBASE,
match_length,
)
};
let (next_ip, next_anchor) = unsafe {
post_ext_match(
hash_long,
hash_small,
base,
dict_base,
dict_start_index,
prefix_start_index,
dict_end,
prefix_start,
hash_log,
chain_log,
mls,
seq_store,
&mut offset1,
&mut offset2,
ip,
match_length,
current,
iend,
ilimit,
)
};
ip = next_ip;
anchor = next_anchor;
continue;
}
if match_long_index > dict_start_index && unsafe { read64(match_long) == read64(ip) } {
let match_end = if match_long_index < prefix_start_index {
dict_end
} else {
iend
};
let low_match = if match_long_index < prefix_start_index {
dict_start
} else {
prefix_start
};
let mut match_length = unsafe {
count_2segments(
ip.wrapping_add(8),
match_long.wrapping_add(8),
iend,
match_end,
prefix_start,
) + 8
};
let offset = current.wrapping_sub(match_long_index);
while ptr_gt(ip, anchor)
&& ptr_gt(match_long, low_match)
&& unsafe { *ip.wrapping_sub(1) == *match_long.wrapping_sub(1) }
{
ip = ip.wrapping_sub(1);
match_long = match_long.wrapping_sub(1);
match_length += 1;
}
offset2 = offset1;
offset1 = offset;
unsafe {
store_seq(
seq_store,
ip.offset_from(anchor) as usize,
anchor,
iend,
offset.wrapping_add(ZSTD_REP_NUM as u32),
match_length,
)
};
let (next_ip, next_anchor) = unsafe {
post_ext_match(
hash_long,
hash_small,
base,
dict_base,
dict_start_index,
prefix_start_index,
dict_end,
prefix_start,
hash_log,
chain_log,
mls,
seq_store,
&mut offset1,
&mut offset2,
ip,
match_length,
current,
iend,
ilimit,
)
};
ip = next_ip;
anchor = next_anchor;
continue;
}
if match_index > dict_start_index && unsafe { read32(matched) == read32(ip) } {
let h_long3 = unsafe { hash_ptr(ip.wrapping_add(1), hash_log, 8) };
let match_index3 = unsafe { table_get(hash_long, h_long3) };
let match_base3 = if match_index3 < prefix_start_index {
dict_base
} else {
base
};
let mut match3 = match_base3.wrapping_add(match_index3 as usize);
unsafe { table_set(hash_long, h_long3, current.wrapping_add(1)) };
let (match_length, offset) = if match_index3 > dict_start_index
&& unsafe { read64(match3) == read64(ip.wrapping_add(1)) }
{
let match_end = if match_index3 < prefix_start_index {
dict_end
} else {
iend
};
let low_match = if match_index3 < prefix_start_index {
dict_start
} else {
prefix_start
};
let mut length = unsafe {
count_2segments(
ip.wrapping_add(9),
match3.wrapping_add(8),
iend,
match_end,
prefix_start,
) + 8
};
ip = ip.wrapping_add(1);
let offset = current.wrapping_add(1).wrapping_sub(match_index3);
while ptr_gt(ip, anchor)
&& ptr_gt(match3, low_match)
&& unsafe { *ip.wrapping_sub(1) == *match3.wrapping_sub(1) }
{
ip = ip.wrapping_sub(1);
match3 = match3.wrapping_sub(1);
length += 1;
}
(length, offset)
} else {
let match_end = if match_index < prefix_start_index {
dict_end
} else {
iend
};
let low_match = if match_index < prefix_start_index {
dict_start
} else {
prefix_start
};
let mut length = unsafe {
count_2segments(
ip.wrapping_add(4),
matched.wrapping_add(4),
iend,
match_end,
prefix_start,
) + 4
};
let offset = current.wrapping_sub(match_index);
while ptr_gt(ip, anchor)
&& ptr_gt(matched, low_match)
&& unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
{
ip = ip.wrapping_sub(1);
matched = matched.wrapping_sub(1);
length += 1;
}
(length, offset)
};
offset2 = offset1;
offset1 = offset;
unsafe {
store_seq(
seq_store,
ip.offset_from(anchor) as usize,
anchor,
iend,
offset.wrapping_add(ZSTD_REP_NUM as u32),
match_length,
)
};
let (next_ip, next_anchor) = unsafe {
post_ext_match(
hash_long,
hash_small,
base,
dict_base,
dict_start_index,
prefix_start_index,
dict_end,
prefix_start,
hash_log,
chain_log,
mls,
seq_store,
&mut offset1,
&mut offset2,
ip,
match_length,
current,
iend,
ilimit,
)
};
ip = next_ip;
anchor = next_anchor;
continue;
}
ip = ip.wrapping_add((unsafe { ip.offset_from(anchor) as usize } >> K_SEARCH_STRENGTH) + 1);
}
unsafe {
*reps = offset1;
*reps.add(1) = offset2;
}
unsafe { iend.offset_from(anchor) as usize }
}
/// Rust implementation called by the C ABI wrapper for external dictionaries.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBlock_doubleFast_extDict(
hash_long: *mut u32,
hash_small: *mut u32,
base: *const u8,
dict_base: *const u8,
dict_limit: u32,
low_limit: u32,
loaded_dict_end: u32,
hash_log: u32,
chain_log: u32,
min_match: u32,
window_log: u32,
seq_store: *mut c_void,
reps: *mut u32,
src: *const c_void,
src_size: usize,
) -> usize {
unsafe {
compress_block_double_fast_ext_dict(
hash_long,
hash_small,
base,
dict_base,
dict_limit,
low_limit,
loaded_dict_end,
hash_log,
chain_log,
window_log,
seq_store.cast::<SeqStore_t>(),
reps,
src.cast::<u8>(),
src_size,
fast_mls(min_match),
)
}
}