feat(rust): port double-fast block matching
Move the two-table fast match finder into Rust while retaining a small C projection of the private match-state. The Rust implementation handles normal, attached-dictionary, and external-dictionary matching without exposing the full C context ABI. The C entry points and build-time exclusion guards remain unchanged for configured consumers. Document the migrated matcher in the Rust component map and narrow the remaining-C boundary accordingly. Test Plan: - cargo test --all-targets - cargo test --target i686-unknown-linux-gnu --all-targets - cargo clippy && cargo clippy --benches && cargo clippy --tests - cargo +nightly fmt - make -B -C tests -j2 fuzzer zstreamtest invalidDictionaries poolTests - ./tests/fuzzer -s5346 -i1 --no-big-tests - ./tests/zstreamtest -i3000 -s334462 - ./tests/invalidDictionaries - timeout 20s stdbuf -oL ./tests/poolTests Refs: rust/README.md
This commit is contained in:
+61
-739
@@ -8,771 +8,93 @@
|
|||||||
* You may select, at your option, one of the above-listed licenses.
|
* You may select, at your option, one of the above-listed licenses.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
/* The double-fast algorithms live in rust/src/zstd_double_fast.rs. Keep the
|
||||||
|
* private match-state layout in C and pass only the matching leaf fields. */
|
||||||
#include "zstd_compress_internal.h"
|
#include "zstd_compress_internal.h"
|
||||||
#include "zstd_double_fast.h"
|
#include "zstd_double_fast.h"
|
||||||
|
|
||||||
#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR
|
#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR
|
||||||
|
|
||||||
static
|
typedef char ZSTD_rust_double_fast_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
|
||||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
typedef char ZSTD_rust_double_fast_seqstore_long_length_pos[
|
||||||
void ZSTD_fillDoubleHashTableForCDict(ZSTD_MatchState_t* ms,
|
(offsetof(SeqStore_t, longLengthPos) == 9 * sizeof(size_t) + 4) ? 1 : -1];
|
||||||
void const* end, ZSTD_dictTableLoadMethod_e dtlm)
|
typedef char ZSTD_rust_double_fast_seqstore_layout[
|
||||||
{
|
(sizeof(SeqStore_t) == 9 * sizeof(size_t) + 8) ? 1 : -1];
|
||||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
typedef char ZSTD_rust_double_fast_rep_count[(ZSTD_REP_NUM == 3) ? 1 : -1];
|
||||||
U32* const hashLarge = ms->hashTable;
|
|
||||||
U32 const hBitsL = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
|
|
||||||
U32 const mls = cParams->minMatch;
|
|
||||||
U32* const hashSmall = ms->chainTable;
|
|
||||||
U32 const hBitsS = cParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS;
|
|
||||||
const BYTE* const base = ms->window.base;
|
|
||||||
const BYTE* ip = base + ms->nextToUpdate;
|
|
||||||
const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
|
|
||||||
const U32 fastHashFillStep = 3;
|
|
||||||
|
|
||||||
/* Always insert every fastHashFillStep position into the hash tables.
|
void ZSTD_rust_fillDoubleHashTable(
|
||||||
* Insert the other positions into the large hash table if their entry
|
U32* hashLong, U32* hashSmall, const BYTE* base, U32 nextToUpdate,
|
||||||
* is empty.
|
const void* end, U32 hashLog, U32 chainLog, U32 minMatch,
|
||||||
*/
|
int fullTableLoad, int forCDict);
|
||||||
for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) {
|
|
||||||
U32 const curr = (U32)(ip - base);
|
|
||||||
U32 i;
|
|
||||||
for (i = 0; i < fastHashFillStep; ++i) {
|
|
||||||
size_t const smHashAndTag = ZSTD_hashPtr(ip + i, hBitsS, mls);
|
|
||||||
size_t const lgHashAndTag = ZSTD_hashPtr(ip + i, hBitsL, 8);
|
|
||||||
if (i == 0) {
|
|
||||||
ZSTD_writeTaggedIndex(hashSmall, smHashAndTag, curr + i);
|
|
||||||
}
|
|
||||||
if (i == 0 || hashLarge[lgHashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) {
|
|
||||||
ZSTD_writeTaggedIndex(hashLarge, lgHashAndTag, curr + i);
|
|
||||||
}
|
|
||||||
/* Only load extra positions for ZSTD_dtlm_full */
|
|
||||||
if (dtlm == ZSTD_dtlm_fast)
|
|
||||||
break;
|
|
||||||
} }
|
|
||||||
}
|
|
||||||
|
|
||||||
static
|
size_t ZSTD_rust_compressBlock_doubleFast(
|
||||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
U32* hashLong, U32* hashSmall, const BYTE* base,
|
||||||
void ZSTD_fillDoubleHashTableForCCtx(ZSTD_MatchState_t* ms,
|
U32 dictLimit, U32 loadedDictEnd,
|
||||||
void const* end, ZSTD_dictTableLoadMethod_e dtlm)
|
U32 hashLog, U32 chainLog, U32 minMatch, U32 windowLog,
|
||||||
{
|
void* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize);
|
||||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
|
||||||
U32* const hashLarge = ms->hashTable;
|
|
||||||
U32 const hBitsL = cParams->hashLog;
|
|
||||||
U32 const mls = cParams->minMatch;
|
|
||||||
U32* const hashSmall = ms->chainTable;
|
|
||||||
U32 const hBitsS = cParams->chainLog;
|
|
||||||
const BYTE* const base = ms->window.base;
|
|
||||||
const BYTE* ip = base + ms->nextToUpdate;
|
|
||||||
const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
|
|
||||||
const U32 fastHashFillStep = 3;
|
|
||||||
|
|
||||||
/* Always insert every fastHashFillStep position into the hash tables.
|
size_t ZSTD_rust_compressBlock_doubleFast_dictMatchState(
|
||||||
* Insert the other positions into the large hash table if their entry
|
U32* hashLong, U32* hashSmall, const BYTE* base, U32 prefixLowestIndex,
|
||||||
* is empty.
|
U32 hashLog, U32 chainLog, U32 minMatch,
|
||||||
*/
|
void* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize,
|
||||||
for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) {
|
const U32* dictHashLong, const U32* dictHashSmall, const BYTE* dictBase,
|
||||||
U32 const curr = (U32)(ip - base);
|
U32 dictStartIndex, const BYTE* dictEnd,
|
||||||
U32 i;
|
U32 dictHashLog, U32 dictChainLog, int prefetchCDictTables);
|
||||||
for (i = 0; i < fastHashFillStep; ++i) {
|
|
||||||
size_t const smHash = ZSTD_hashPtr(ip + i, hBitsS, mls);
|
size_t ZSTD_rust_compressBlock_doubleFast_extDict(
|
||||||
size_t const lgHash = ZSTD_hashPtr(ip + i, hBitsL, 8);
|
U32* hashLong, U32* hashSmall, const BYTE* base, const BYTE* dictBase,
|
||||||
if (i == 0)
|
U32 dictLimit, U32 lowLimit, U32 loadedDictEnd,
|
||||||
hashSmall[smHash] = curr + i;
|
U32 hashLog, U32 chainLog, U32 minMatch, U32 windowLog,
|
||||||
if (i == 0 || hashLarge[lgHash] == 0)
|
void* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize);
|
||||||
hashLarge[lgHash] = curr + i;
|
|
||||||
/* Only load extra positions for ZSTD_dtlm_full */
|
|
||||||
if (dtlm == ZSTD_dtlm_fast)
|
|
||||||
break;
|
|
||||||
} }
|
|
||||||
}
|
|
||||||
|
|
||||||
void ZSTD_fillDoubleHashTable(ZSTD_MatchState_t* ms,
|
void ZSTD_fillDoubleHashTable(ZSTD_MatchState_t* ms,
|
||||||
const void* const end,
|
const void* end,
|
||||||
ZSTD_dictTableLoadMethod_e dtlm,
|
ZSTD_dictTableLoadMethod_e dtlm,
|
||||||
ZSTD_tableFillPurpose_e tfp)
|
ZSTD_tableFillPurpose_e tfp)
|
||||||
{
|
{
|
||||||
if (tfp == ZSTD_tfp_forCDict) {
|
ZSTD_rust_fillDoubleHashTable(
|
||||||
ZSTD_fillDoubleHashTableForCDict(ms, end, dtlm);
|
ms->hashTable, ms->chainTable, ms->window.base, ms->nextToUpdate,
|
||||||
} else {
|
end, ms->cParams.hashLog, ms->cParams.chainLog, ms->cParams.minMatch,
|
||||||
ZSTD_fillDoubleHashTableForCCtx(ms, end, dtlm);
|
dtlm == ZSTD_dtlm_full, tfp == ZSTD_tfp_forCDict);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
FORCE_INLINE_TEMPLATE
|
|
||||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
|
||||||
size_t ZSTD_compressBlock_doubleFast_noDict_generic(
|
|
||||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
|
||||||
void const* src, size_t srcSize, U32 const mls /* template */)
|
|
||||||
{
|
|
||||||
ZSTD_compressionParameters const* cParams = &ms->cParams;
|
|
||||||
U32* const hashLong = ms->hashTable;
|
|
||||||
const U32 hBitsL = cParams->hashLog;
|
|
||||||
U32* const hashSmall = ms->chainTable;
|
|
||||||
const U32 hBitsS = cParams->chainLog;
|
|
||||||
const BYTE* const base = ms->window.base;
|
|
||||||
const BYTE* const istart = (const BYTE*)src;
|
|
||||||
const BYTE* anchor = istart;
|
|
||||||
const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
|
|
||||||
/* presumes that, if there is a dictionary, it must be using Attach mode */
|
|
||||||
const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog);
|
|
||||||
const BYTE* const prefixLowest = base + prefixLowestIndex;
|
|
||||||
const BYTE* const iend = istart + srcSize;
|
|
||||||
const BYTE* const ilimit = iend - HASH_READ_SIZE;
|
|
||||||
U32 offset_1=rep[0], offset_2=rep[1];
|
|
||||||
U32 offsetSaved1 = 0, offsetSaved2 = 0;
|
|
||||||
|
|
||||||
size_t mLength;
|
|
||||||
U32 offset;
|
|
||||||
U32 curr;
|
|
||||||
|
|
||||||
/* how many positions to search before increasing step size */
|
|
||||||
const size_t kStepIncr = 1 << kSearchStrength;
|
|
||||||
/* the position at which to increment the step size if no match is found */
|
|
||||||
const BYTE* nextStep;
|
|
||||||
size_t step; /* the current step size */
|
|
||||||
|
|
||||||
size_t hl0; /* the long hash at ip */
|
|
||||||
size_t hl1; /* the long hash at ip1 */
|
|
||||||
|
|
||||||
U32 idxl0; /* the long match index for ip */
|
|
||||||
U32 idxl1; /* the long match index for ip1 */
|
|
||||||
|
|
||||||
const BYTE* matchl0; /* the long match for ip */
|
|
||||||
const BYTE* matchs0; /* the short match for ip */
|
|
||||||
const BYTE* matchl1; /* the long match for ip1 */
|
|
||||||
const BYTE* matchs0_safe; /* matchs0 or safe address */
|
|
||||||
|
|
||||||
const BYTE* ip = istart; /* the current position */
|
|
||||||
const BYTE* ip1; /* the next position */
|
|
||||||
/* Array of ~random data, should have low probability of matching data
|
|
||||||
* we load from here instead of from tables, if matchl0/matchl1 are
|
|
||||||
* invalid indices. Used to avoid unpredictable branches. */
|
|
||||||
const BYTE dummy[] = {0x12,0x34,0x56,0x78,0x9a,0xbc,0xde,0xf0,0xe2,0xb4};
|
|
||||||
|
|
||||||
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_noDict_generic");
|
|
||||||
|
|
||||||
/* init */
|
|
||||||
ip += ((ip - prefixLowest) == 0);
|
|
||||||
{
|
|
||||||
U32 const current = (U32)(ip - base);
|
|
||||||
U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog);
|
|
||||||
U32 const maxRep = current - windowLow;
|
|
||||||
if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0;
|
|
||||||
if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Outer Loop: one iteration per match found and stored */
|
|
||||||
while (1) {
|
|
||||||
step = 1;
|
|
||||||
nextStep = ip + kStepIncr;
|
|
||||||
ip1 = ip + step;
|
|
||||||
|
|
||||||
if (ip1 > ilimit) {
|
|
||||||
goto _cleanup;
|
|
||||||
}
|
|
||||||
|
|
||||||
hl0 = ZSTD_hashPtr(ip, hBitsL, 8);
|
|
||||||
idxl0 = hashLong[hl0];
|
|
||||||
matchl0 = base + idxl0;
|
|
||||||
|
|
||||||
/* Inner Loop: one iteration per search / position */
|
|
||||||
do {
|
|
||||||
const size_t hs0 = ZSTD_hashPtr(ip, hBitsS, mls);
|
|
||||||
const U32 idxs0 = hashSmall[hs0];
|
|
||||||
curr = (U32)(ip-base);
|
|
||||||
matchs0 = base + idxs0;
|
|
||||||
|
|
||||||
hashLong[hl0] = hashSmall[hs0] = curr; /* update hash tables */
|
|
||||||
|
|
||||||
/* check noDict repcode */
|
|
||||||
if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {
|
|
||||||
mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
|
|
||||||
ip++;
|
|
||||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
|
|
||||||
goto _match_stored;
|
|
||||||
}
|
|
||||||
|
|
||||||
hl1 = ZSTD_hashPtr(ip1, hBitsL, 8);
|
|
||||||
|
|
||||||
/* idxl0 > prefixLowestIndex is a (somewhat) unpredictable branch.
|
|
||||||
* However expression below complies into conditional move. Since
|
|
||||||
* match is unlikely and we only *branch* on idxl0 > prefixLowestIndex
|
|
||||||
* if there is a match, all branches become predictable. */
|
|
||||||
{ const BYTE* const matchl0_safe = ZSTD_selectAddr(idxl0, prefixLowestIndex, matchl0, &dummy[0]);
|
|
||||||
|
|
||||||
/* check prefix long match */
|
|
||||||
if (MEM_read64(matchl0_safe) == MEM_read64(ip) && matchl0_safe == matchl0) {
|
|
||||||
mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8;
|
|
||||||
offset = (U32)(ip-matchl0);
|
|
||||||
while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */
|
|
||||||
goto _match_found;
|
|
||||||
} }
|
|
||||||
|
|
||||||
idxl1 = hashLong[hl1];
|
|
||||||
matchl1 = base + idxl1;
|
|
||||||
|
|
||||||
/* Same optimization as matchl0 above */
|
|
||||||
matchs0_safe = ZSTD_selectAddr(idxs0, prefixLowestIndex, matchs0, &dummy[0]);
|
|
||||||
|
|
||||||
/* check prefix short match */
|
|
||||||
if(MEM_read32(matchs0_safe) == MEM_read32(ip) && matchs0_safe == matchs0) {
|
|
||||||
goto _search_next_long;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (ip1 >= nextStep) {
|
|
||||||
PREFETCH_L1(ip1 + 64);
|
|
||||||
PREFETCH_L1(ip1 + 128);
|
|
||||||
step++;
|
|
||||||
nextStep += kStepIncr;
|
|
||||||
}
|
|
||||||
ip = ip1;
|
|
||||||
ip1 += step;
|
|
||||||
|
|
||||||
hl0 = hl1;
|
|
||||||
idxl0 = idxl1;
|
|
||||||
matchl0 = matchl1;
|
|
||||||
#if defined(__aarch64__)
|
|
||||||
PREFETCH_L1(ip+256);
|
|
||||||
#endif
|
|
||||||
} while (ip1 <= ilimit);
|
|
||||||
|
|
||||||
_cleanup:
|
|
||||||
/* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0),
|
|
||||||
* rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */
|
|
||||||
offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2;
|
|
||||||
|
|
||||||
/* save reps for next block */
|
|
||||||
rep[0] = offset_1 ? offset_1 : offsetSaved1;
|
|
||||||
rep[1] = offset_2 ? offset_2 : offsetSaved2;
|
|
||||||
|
|
||||||
/* Return the last literals size */
|
|
||||||
return (size_t)(iend - anchor);
|
|
||||||
|
|
||||||
_search_next_long:
|
|
||||||
|
|
||||||
/* short match found: let's check for a longer one */
|
|
||||||
mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4;
|
|
||||||
offset = (U32)(ip - matchs0);
|
|
||||||
|
|
||||||
/* check long match at +1 position */
|
|
||||||
if ((idxl1 > prefixLowestIndex) && (MEM_read64(matchl1) == MEM_read64(ip1))) {
|
|
||||||
size_t const l1len = ZSTD_count(ip1+8, matchl1+8, iend) + 8;
|
|
||||||
if (l1len > mLength) {
|
|
||||||
/* use the long match instead */
|
|
||||||
ip = ip1;
|
|
||||||
mLength = l1len;
|
|
||||||
offset = (U32)(ip-matchl1);
|
|
||||||
matchs0 = matchl1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* complete backward */
|
|
||||||
|
|
||||||
/* fall-through */
|
|
||||||
|
|
||||||
_match_found: /* requires ip, offset, mLength */
|
|
||||||
offset_2 = offset_1;
|
|
||||||
offset_1 = offset;
|
|
||||||
|
|
||||||
if (step < 4) {
|
|
||||||
/* It is unsafe to write this value back to the hashtable when ip1 is
|
|
||||||
* greater than or equal to the new ip we will have after we're done
|
|
||||||
* processing this match. Rather than perform that test directly
|
|
||||||
* (ip1 >= ip + mLength), which costs speed in practice, we do a simpler
|
|
||||||
* more predictable test. The minmatch even if we take a short match is
|
|
||||||
* 4 bytes, so as long as step, the distance between ip and ip1
|
|
||||||
* (initially) is less than 4, we know ip1 < new ip. */
|
|
||||||
hashLong[hl1] = (U32)(ip1 - base);
|
|
||||||
}
|
|
||||||
|
|
||||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
|
|
||||||
|
|
||||||
_match_stored:
|
|
||||||
/* match found */
|
|
||||||
ip += mLength;
|
|
||||||
anchor = ip;
|
|
||||||
|
|
||||||
if (ip <= ilimit) {
|
|
||||||
/* Complementary insertion */
|
|
||||||
/* done after iLimit test, as candidates could be > iend-8 */
|
|
||||||
{ U32 const indexToInsert = curr+2;
|
|
||||||
hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert;
|
|
||||||
hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
|
|
||||||
hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert;
|
|
||||||
hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* check immediate repcode */
|
|
||||||
while ( (ip <= ilimit)
|
|
||||||
&& ( (offset_2>0)
|
|
||||||
& (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) {
|
|
||||||
/* store sequence */
|
|
||||||
size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
|
|
||||||
U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */
|
|
||||||
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base);
|
|
||||||
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base);
|
|
||||||
ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, rLength);
|
|
||||||
ip += rLength;
|
|
||||||
anchor = ip;
|
|
||||||
continue; /* faster when present ... (?) */
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
FORCE_INLINE_TEMPLATE
|
|
||||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
|
||||||
size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
|
|
||||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
|
||||||
void const* src, size_t srcSize,
|
|
||||||
U32 const mls /* template */)
|
|
||||||
{
|
|
||||||
ZSTD_compressionParameters const* cParams = &ms->cParams;
|
|
||||||
U32* const hashLong = ms->hashTable;
|
|
||||||
const U32 hBitsL = cParams->hashLog;
|
|
||||||
U32* const hashSmall = ms->chainTable;
|
|
||||||
const U32 hBitsS = cParams->chainLog;
|
|
||||||
const BYTE* const base = ms->window.base;
|
|
||||||
const BYTE* const istart = (const BYTE*)src;
|
|
||||||
const BYTE* ip = istart;
|
|
||||||
const BYTE* anchor = istart;
|
|
||||||
const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
|
|
||||||
/* presumes that, if there is a dictionary, it must be using Attach mode */
|
|
||||||
const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog);
|
|
||||||
const BYTE* const prefixLowest = base + prefixLowestIndex;
|
|
||||||
const BYTE* const iend = istart + srcSize;
|
|
||||||
const BYTE* const ilimit = iend - HASH_READ_SIZE;
|
|
||||||
U32 offset_1=rep[0], offset_2=rep[1];
|
|
||||||
|
|
||||||
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
|
|
||||||
const ZSTD_compressionParameters* const dictCParams = &dms->cParams;
|
|
||||||
const U32* const dictHashLong = dms->hashTable;
|
|
||||||
const U32* const dictHashSmall = dms->chainTable;
|
|
||||||
const U32 dictStartIndex = dms->window.dictLimit;
|
|
||||||
const BYTE* const dictBase = dms->window.base;
|
|
||||||
const BYTE* const dictStart = dictBase + dictStartIndex;
|
|
||||||
const BYTE* const dictEnd = dms->window.nextSrc;
|
|
||||||
const U32 dictIndexDelta = prefixLowestIndex - (U32)(dictEnd - dictBase);
|
|
||||||
const U32 dictHBitsL = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
|
|
||||||
const U32 dictHBitsS = dictCParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS;
|
|
||||||
const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart));
|
|
||||||
|
|
||||||
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_dictMatchState_generic");
|
|
||||||
|
|
||||||
/* if a dictionary is attached, it must be within window range */
|
|
||||||
assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex);
|
|
||||||
|
|
||||||
if (ms->prefetchCDictTables) {
|
|
||||||
size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32);
|
|
||||||
size_t const chainTableBytes = (((size_t)1) << dictCParams->chainLog) * sizeof(U32);
|
|
||||||
PREFETCH_AREA(dictHashLong, hashTableBytes);
|
|
||||||
PREFETCH_AREA(dictHashSmall, chainTableBytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* init */
|
|
||||||
ip += (dictAndPrefixLength == 0);
|
|
||||||
|
|
||||||
/* dictMatchState repCode checks don't currently handle repCode == 0
|
|
||||||
* disabling. */
|
|
||||||
assert(offset_1 <= dictAndPrefixLength);
|
|
||||||
assert(offset_2 <= dictAndPrefixLength);
|
|
||||||
|
|
||||||
/* Main Search Loop */
|
|
||||||
while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */
|
|
||||||
size_t mLength;
|
|
||||||
U32 offset;
|
|
||||||
size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8);
|
|
||||||
size_t const h = ZSTD_hashPtr(ip, hBitsS, mls);
|
|
||||||
size_t const dictHashAndTagL = ZSTD_hashPtr(ip, dictHBitsL, 8);
|
|
||||||
size_t const dictHashAndTagS = ZSTD_hashPtr(ip, dictHBitsS, mls);
|
|
||||||
U32 const dictMatchIndexAndTagL = dictHashLong[dictHashAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS];
|
|
||||||
U32 const dictMatchIndexAndTagS = dictHashSmall[dictHashAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS];
|
|
||||||
int const dictTagsMatchL = ZSTD_comparePackedTags(dictMatchIndexAndTagL, dictHashAndTagL);
|
|
||||||
int const dictTagsMatchS = ZSTD_comparePackedTags(dictMatchIndexAndTagS, dictHashAndTagS);
|
|
||||||
U32 const curr = (U32)(ip-base);
|
|
||||||
U32 const matchIndexL = hashLong[h2];
|
|
||||||
U32 matchIndexS = hashSmall[h];
|
|
||||||
const BYTE* matchLong = base + matchIndexL;
|
|
||||||
const BYTE* match = base + matchIndexS;
|
|
||||||
const U32 repIndex = curr + 1 - offset_1;
|
|
||||||
const BYTE* repMatch = (repIndex < prefixLowestIndex) ?
|
|
||||||
dictBase + (repIndex - dictIndexDelta) :
|
|
||||||
base + repIndex;
|
|
||||||
hashLong[h2] = hashSmall[h] = curr; /* update hash tables */
|
|
||||||
|
|
||||||
/* check repcode */
|
|
||||||
if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex))
|
|
||||||
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
|
|
||||||
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
|
|
||||||
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
|
|
||||||
ip++;
|
|
||||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
|
|
||||||
goto _match_stored;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ((matchIndexL >= prefixLowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) {
|
|
||||||
/* check prefix long match */
|
|
||||||
mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8;
|
|
||||||
offset = (U32)(ip-matchLong);
|
|
||||||
while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
|
|
||||||
goto _match_found;
|
|
||||||
} else if (dictTagsMatchL) {
|
|
||||||
/* check dictMatchState long match */
|
|
||||||
U32 const dictMatchIndexL = dictMatchIndexAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS;
|
|
||||||
const BYTE* dictMatchL = dictBase + dictMatchIndexL;
|
|
||||||
assert(dictMatchL < dictEnd);
|
|
||||||
|
|
||||||
if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) {
|
|
||||||
mLength = ZSTD_count_2segments(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8;
|
|
||||||
offset = (U32)(curr - dictMatchIndexL - dictIndexDelta);
|
|
||||||
while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */
|
|
||||||
goto _match_found;
|
|
||||||
} }
|
|
||||||
|
|
||||||
if (matchIndexS > prefixLowestIndex) {
|
|
||||||
/* short match candidate */
|
|
||||||
if (MEM_read32(match) == MEM_read32(ip)) {
|
|
||||||
goto _search_next_long;
|
|
||||||
}
|
|
||||||
} else if (dictTagsMatchS) {
|
|
||||||
/* check dictMatchState short match */
|
|
||||||
U32 const dictMatchIndexS = dictMatchIndexAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS;
|
|
||||||
match = dictBase + dictMatchIndexS;
|
|
||||||
matchIndexS = dictMatchIndexS + dictIndexDelta;
|
|
||||||
|
|
||||||
if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) {
|
|
||||||
goto _search_next_long;
|
|
||||||
} }
|
|
||||||
|
|
||||||
ip += ((ip-anchor) >> kSearchStrength) + 1;
|
|
||||||
#if defined(__aarch64__)
|
|
||||||
PREFETCH_L1(ip+256);
|
|
||||||
#endif
|
|
||||||
continue;
|
|
||||||
|
|
||||||
_search_next_long:
|
|
||||||
{ size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
|
|
||||||
size_t const dictHashAndTagL3 = ZSTD_hashPtr(ip+1, dictHBitsL, 8);
|
|
||||||
U32 const matchIndexL3 = hashLong[hl3];
|
|
||||||
U32 const dictMatchIndexAndTagL3 = dictHashLong[dictHashAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS];
|
|
||||||
int const dictTagsMatchL3 = ZSTD_comparePackedTags(dictMatchIndexAndTagL3, dictHashAndTagL3);
|
|
||||||
const BYTE* matchL3 = base + matchIndexL3;
|
|
||||||
hashLong[hl3] = curr + 1;
|
|
||||||
|
|
||||||
/* check prefix long +1 match */
|
|
||||||
if ((matchIndexL3 >= prefixLowestIndex) && (MEM_read64(matchL3) == MEM_read64(ip+1))) {
|
|
||||||
mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8;
|
|
||||||
ip++;
|
|
||||||
offset = (U32)(ip-matchL3);
|
|
||||||
while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */
|
|
||||||
goto _match_found;
|
|
||||||
} else if (dictTagsMatchL3) {
|
|
||||||
/* check dict long +1 match */
|
|
||||||
U32 const dictMatchIndexL3 = dictMatchIndexAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS;
|
|
||||||
const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3;
|
|
||||||
assert(dictMatchL3 < dictEnd);
|
|
||||||
if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) {
|
|
||||||
mLength = ZSTD_count_2segments(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8;
|
|
||||||
ip++;
|
|
||||||
offset = (U32)(curr + 1 - dictMatchIndexL3 - dictIndexDelta);
|
|
||||||
while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */
|
|
||||||
goto _match_found;
|
|
||||||
} } }
|
|
||||||
|
|
||||||
/* if no long +1 match, explore the short match we found */
|
|
||||||
if (matchIndexS < prefixLowestIndex) {
|
|
||||||
mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4;
|
|
||||||
offset = (U32)(curr - matchIndexS);
|
|
||||||
while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
|
|
||||||
} else {
|
|
||||||
mLength = ZSTD_count(ip+4, match+4, iend) + 4;
|
|
||||||
offset = (U32)(ip - match);
|
|
||||||
while (((ip>anchor) & (match>prefixLowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
|
|
||||||
}
|
|
||||||
|
|
||||||
_match_found:
|
|
||||||
offset_2 = offset_1;
|
|
||||||
offset_1 = offset;
|
|
||||||
|
|
||||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
|
|
||||||
|
|
||||||
_match_stored:
|
|
||||||
/* match found */
|
|
||||||
ip += mLength;
|
|
||||||
anchor = ip;
|
|
||||||
|
|
||||||
if (ip <= ilimit) {
|
|
||||||
/* Complementary insertion */
|
|
||||||
/* done after iLimit test, as candidates could be > iend-8 */
|
|
||||||
{ U32 const indexToInsert = curr+2;
|
|
||||||
hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert;
|
|
||||||
hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
|
|
||||||
hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert;
|
|
||||||
hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* check immediate repcode */
|
|
||||||
while (ip <= ilimit) {
|
|
||||||
U32 const current2 = (U32)(ip-base);
|
|
||||||
U32 const repIndex2 = current2 - offset_2;
|
|
||||||
const BYTE* repMatch2 = repIndex2 < prefixLowestIndex ?
|
|
||||||
dictBase + repIndex2 - dictIndexDelta :
|
|
||||||
base + repIndex2;
|
|
||||||
if ( (ZSTD_index_overlap_check(prefixLowestIndex, repIndex2))
|
|
||||||
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
|
||||||
const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend;
|
|
||||||
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4;
|
|
||||||
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
|
|
||||||
ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
|
|
||||||
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
|
|
||||||
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
|
|
||||||
ip += repLength2;
|
|
||||||
anchor = ip;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} /* while (ip < ilimit) */
|
|
||||||
|
|
||||||
/* save reps for next block */
|
|
||||||
rep[0] = offset_1;
|
|
||||||
rep[1] = offset_2;
|
|
||||||
|
|
||||||
/* Return the last literals size */
|
|
||||||
return (size_t)(iend - anchor);
|
|
||||||
}
|
|
||||||
|
|
||||||
#define ZSTD_GEN_DFAST_FN(dictMode, mls) \
|
|
||||||
static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \
|
|
||||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
|
|
||||||
void const* src, size_t srcSize) \
|
|
||||||
{ \
|
|
||||||
return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \
|
|
||||||
}
|
|
||||||
|
|
||||||
ZSTD_GEN_DFAST_FN(noDict, 4)
|
|
||||||
ZSTD_GEN_DFAST_FN(noDict, 5)
|
|
||||||
ZSTD_GEN_DFAST_FN(noDict, 6)
|
|
||||||
ZSTD_GEN_DFAST_FN(noDict, 7)
|
|
||||||
|
|
||||||
ZSTD_GEN_DFAST_FN(dictMatchState, 4)
|
|
||||||
ZSTD_GEN_DFAST_FN(dictMatchState, 5)
|
|
||||||
ZSTD_GEN_DFAST_FN(dictMatchState, 6)
|
|
||||||
ZSTD_GEN_DFAST_FN(dictMatchState, 7)
|
|
||||||
|
|
||||||
|
|
||||||
size_t ZSTD_compressBlock_doubleFast(
|
size_t ZSTD_compressBlock_doubleFast(
|
||||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||||
void const* src, size_t srcSize)
|
const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
const U32 mls = ms->cParams.minMatch;
|
return ZSTD_rust_compressBlock_doubleFast(
|
||||||
switch(mls)
|
ms->hashTable, ms->chainTable, ms->window.base,
|
||||||
{
|
ms->window.dictLimit, ms->loadedDictEnd,
|
||||||
default: /* includes case 3 */
|
ms->cParams.hashLog, ms->cParams.chainLog, ms->cParams.minMatch,
|
||||||
case 4 :
|
ms->cParams.windowLog,
|
||||||
return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize);
|
seqStore, rep, src, srcSize);
|
||||||
case 5 :
|
|
||||||
return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize);
|
|
||||||
case 6 :
|
|
||||||
return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize);
|
|
||||||
case 7 :
|
|
||||||
return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
size_t ZSTD_compressBlock_doubleFast_dictMatchState(
|
size_t ZSTD_compressBlock_doubleFast_dictMatchState(
|
||||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||||
void const* src, size_t srcSize)
|
const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
const U32 mls = ms->cParams.minMatch;
|
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
|
||||||
switch(mls)
|
assert(dms != NULL);
|
||||||
{
|
return ZSTD_rust_compressBlock_doubleFast_dictMatchState(
|
||||||
default: /* includes case 3 */
|
ms->hashTable, ms->chainTable, ms->window.base, ms->window.dictLimit,
|
||||||
case 4 :
|
ms->cParams.hashLog, ms->cParams.chainLog, ms->cParams.minMatch,
|
||||||
return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize);
|
seqStore, rep, src, srcSize,
|
||||||
case 5 :
|
dms->hashTable, dms->chainTable, dms->window.base,
|
||||||
return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize);
|
dms->window.dictLimit, dms->window.nextSrc,
|
||||||
case 6 :
|
dms->cParams.hashLog, dms->cParams.chainLog, ms->prefetchCDictTables);
|
||||||
return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize);
|
|
||||||
case 7 :
|
|
||||||
return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static
|
|
||||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
|
||||||
size_t ZSTD_compressBlock_doubleFast_extDict_generic(
|
|
||||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
|
||||||
void const* src, size_t srcSize,
|
|
||||||
U32 const mls /* template */)
|
|
||||||
{
|
|
||||||
ZSTD_compressionParameters const* cParams = &ms->cParams;
|
|
||||||
U32* const hashLong = ms->hashTable;
|
|
||||||
U32 const hBitsL = cParams->hashLog;
|
|
||||||
U32* const hashSmall = ms->chainTable;
|
|
||||||
U32 const hBitsS = cParams->chainLog;
|
|
||||||
const BYTE* const istart = (const BYTE*)src;
|
|
||||||
const BYTE* ip = istart;
|
|
||||||
const BYTE* anchor = istart;
|
|
||||||
const BYTE* const iend = istart + srcSize;
|
|
||||||
const BYTE* const ilimit = iend - 8;
|
|
||||||
const BYTE* const base = ms->window.base;
|
|
||||||
const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
|
|
||||||
const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog);
|
|
||||||
const U32 dictStartIndex = lowLimit;
|
|
||||||
const U32 dictLimit = ms->window.dictLimit;
|
|
||||||
const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit;
|
|
||||||
const BYTE* const prefixStart = base + prefixStartIndex;
|
|
||||||
const BYTE* const dictBase = ms->window.dictBase;
|
|
||||||
const BYTE* const dictStart = dictBase + dictStartIndex;
|
|
||||||
const BYTE* const dictEnd = dictBase + prefixStartIndex;
|
|
||||||
U32 offset_1=rep[0], offset_2=rep[1];
|
|
||||||
|
|
||||||
DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize);
|
|
||||||
|
|
||||||
/* if extDict is invalidated due to maxDistance, switch to "regular" variant */
|
|
||||||
if (prefixStartIndex == dictStartIndex)
|
|
||||||
return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize);
|
|
||||||
|
|
||||||
/* Search Loop */
|
|
||||||
while (ip < ilimit) { /* < instead of <=, because (ip+1) */
|
|
||||||
const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls);
|
|
||||||
const U32 matchIndex = hashSmall[hSmall];
|
|
||||||
const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base;
|
|
||||||
const BYTE* match = matchBase + matchIndex;
|
|
||||||
|
|
||||||
const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8);
|
|
||||||
const U32 matchLongIndex = hashLong[hLong];
|
|
||||||
const BYTE* const matchLongBase = matchLongIndex < prefixStartIndex ? dictBase : base;
|
|
||||||
const BYTE* matchLong = matchLongBase + matchLongIndex;
|
|
||||||
|
|
||||||
const U32 curr = (U32)(ip-base);
|
|
||||||
const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */
|
|
||||||
const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base;
|
|
||||||
const BYTE* const repMatch = repBase + repIndex;
|
|
||||||
size_t mLength;
|
|
||||||
hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */
|
|
||||||
|
|
||||||
if (((ZSTD_index_overlap_check(prefixStartIndex, repIndex))
|
|
||||||
& (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */
|
|
||||||
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
|
|
||||||
const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
|
|
||||||
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
|
|
||||||
ip++;
|
|
||||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
|
|
||||||
} else {
|
|
||||||
if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) {
|
|
||||||
const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend;
|
|
||||||
const BYTE* const lowMatchPtr = matchLongIndex < prefixStartIndex ? dictStart : prefixStart;
|
|
||||||
U32 offset;
|
|
||||||
mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, prefixStart) + 8;
|
|
||||||
offset = curr - matchLongIndex;
|
|
||||||
while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
|
|
||||||
offset_2 = offset_1;
|
|
||||||
offset_1 = offset;
|
|
||||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
|
|
||||||
|
|
||||||
} else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) {
|
|
||||||
size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
|
|
||||||
U32 const matchIndex3 = hashLong[h3];
|
|
||||||
const BYTE* const match3Base = matchIndex3 < prefixStartIndex ? dictBase : base;
|
|
||||||
const BYTE* match3 = match3Base + matchIndex3;
|
|
||||||
U32 offset;
|
|
||||||
hashLong[h3] = curr + 1;
|
|
||||||
if ( (matchIndex3 > dictStartIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) {
|
|
||||||
const BYTE* const matchEnd = matchIndex3 < prefixStartIndex ? dictEnd : iend;
|
|
||||||
const BYTE* const lowMatchPtr = matchIndex3 < prefixStartIndex ? dictStart : prefixStart;
|
|
||||||
mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, prefixStart) + 8;
|
|
||||||
ip++;
|
|
||||||
offset = curr+1 - matchIndex3;
|
|
||||||
while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */
|
|
||||||
} else {
|
|
||||||
const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend;
|
|
||||||
const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart;
|
|
||||||
mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4;
|
|
||||||
offset = curr - matchIndex;
|
|
||||||
while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
|
|
||||||
}
|
|
||||||
offset_2 = offset_1;
|
|
||||||
offset_1 = offset;
|
|
||||||
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
|
|
||||||
|
|
||||||
} else {
|
|
||||||
ip += ((ip-anchor) >> kSearchStrength) + 1;
|
|
||||||
continue;
|
|
||||||
} }
|
|
||||||
|
|
||||||
/* move to next sequence start */
|
|
||||||
ip += mLength;
|
|
||||||
anchor = ip;
|
|
||||||
|
|
||||||
if (ip <= ilimit) {
|
|
||||||
/* Complementary insertion */
|
|
||||||
/* done after iLimit test, as candidates could be > iend-8 */
|
|
||||||
{ U32 const indexToInsert = curr+2;
|
|
||||||
hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert;
|
|
||||||
hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base);
|
|
||||||
hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert;
|
|
||||||
hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* check immediate repcode */
|
|
||||||
while (ip <= ilimit) {
|
|
||||||
U32 const current2 = (U32)(ip-base);
|
|
||||||
U32 const repIndex2 = current2 - offset_2;
|
|
||||||
const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
|
|
||||||
if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2))
|
|
||||||
& (offset_2 <= current2 - dictStartIndex))
|
|
||||||
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
|
|
||||||
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
|
|
||||||
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
|
|
||||||
U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
|
|
||||||
ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
|
|
||||||
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
|
|
||||||
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
|
|
||||||
ip += repLength2;
|
|
||||||
anchor = ip;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
} } }
|
|
||||||
|
|
||||||
/* save reps for next block */
|
|
||||||
rep[0] = offset_1;
|
|
||||||
rep[1] = offset_2;
|
|
||||||
|
|
||||||
/* Return the last literals size */
|
|
||||||
return (size_t)(iend - anchor);
|
|
||||||
}
|
|
||||||
|
|
||||||
ZSTD_GEN_DFAST_FN(extDict, 4)
|
|
||||||
ZSTD_GEN_DFAST_FN(extDict, 5)
|
|
||||||
ZSTD_GEN_DFAST_FN(extDict, 6)
|
|
||||||
ZSTD_GEN_DFAST_FN(extDict, 7)
|
|
||||||
|
|
||||||
size_t ZSTD_compressBlock_doubleFast_extDict(
|
size_t ZSTD_compressBlock_doubleFast_extDict(
|
||||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||||
void const* src, size_t srcSize)
|
const void* src, size_t srcSize)
|
||||||
{
|
{
|
||||||
U32 const mls = ms->cParams.minMatch;
|
return ZSTD_rust_compressBlock_doubleFast_extDict(
|
||||||
switch(mls)
|
ms->hashTable, ms->chainTable, ms->window.base, ms->window.dictBase,
|
||||||
{
|
ms->window.dictLimit, ms->window.lowLimit, ms->loadedDictEnd,
|
||||||
default: /* includes case 3 */
|
ms->cParams.hashLog, ms->cParams.chainLog, ms->cParams.minMatch,
|
||||||
case 4 :
|
ms->cParams.windowLog,
|
||||||
return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize);
|
seqStore, rep, src, srcSize);
|
||||||
case 5 :
|
|
||||||
return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize);
|
|
||||||
case 6 :
|
|
||||||
return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize);
|
|
||||||
case 7 :
|
|
||||||
return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#endif /* ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR */
|
#endif /* ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR */
|
||||||
|
|||||||
+3
-1
@@ -31,13 +31,15 @@ zstd ABI:
|
|||||||
- `zstd_presplit` chooses split points for full compression blocks.
|
- `zstd_presplit` chooses split points for full compression blocks.
|
||||||
- `zstd_compress_literals` emits raw, RLE, and Huffman literal sections
|
- `zstd_compress_literals` emits raw, RLE, and Huffman literal sections
|
||||||
while preserving the compressor's Huffman-table repeat state.
|
while preserving the compressor's Huffman-table repeat state.
|
||||||
|
- `zstd_fast` and `zstd_double_fast` implement the single- and two-table
|
||||||
|
fast block match finders, including attached and external dictionary paths.
|
||||||
- Runtime support
|
- Runtime support
|
||||||
- `threading` provides platform pthread wrappers required by zstd headers.
|
- `threading` provides platform pthread wrappers required by zstd headers.
|
||||||
- `pool` implements the bounded worker pool used by multithreaded compression.
|
- `pool` implements the bounded worker pool used by multithreaded compression.
|
||||||
- Dictionary support
|
- Dictionary support
|
||||||
- `zstd_ddict` owns, loads, copies, and references decode dictionaries.
|
- `zstd_ddict` owns, loads, copies, and references decode dictionaries.
|
||||||
|
|
||||||
The remaining block compression, general decompression, dictionary-building,
|
The lazy and optimal block matchers, general decompression, dictionary-building,
|
||||||
legacy, and CLI translation units are still C. They must move before the
|
legacy, and CLI translation units are still C. They must move before the
|
||||||
rewrite is complete. Keeping that boundary explicit prevents a passing hybrid
|
rewrite is complete. Keeping that boundary explicit prevents a passing hybrid
|
||||||
build from being mistaken for the final all-Rust result.
|
build from being mistaken for the final all-Rust result.
|
||||||
|
|||||||
@@ -32,6 +32,8 @@ pub mod zstd_compress_superblock;
|
|||||||
#[cfg(feature = "decompression")]
|
#[cfg(feature = "decompression")]
|
||||||
pub mod zstd_ddict;
|
pub mod zstd_ddict;
|
||||||
#[cfg(feature = "compression")]
|
#[cfg(feature = "compression")]
|
||||||
|
pub mod zstd_double_fast;
|
||||||
|
#[cfg(feature = "compression")]
|
||||||
pub mod zstd_fast;
|
pub mod zstd_fast;
|
||||||
#[cfg(feature = "compression")]
|
#[cfg(feature = "compression")]
|
||||||
pub mod zstd_presplit;
|
pub mod zstd_presplit;
|
||||||
|
|||||||
@@ -0,0 +1,1819 @@
|
|||||||
|
#![allow(non_camel_case_types)]
|
||||||
|
#![allow(non_snake_case)]
|
||||||
|
#![allow(clippy::missing_safety_doc)]
|
||||||
|
#![allow(clippy::too_many_arguments)]
|
||||||
|
|
||||||
|
//! Double-fast block match finder.
|
||||||
|
//!
|
||||||
|
//! The C shim keeps `ZSTD_MatchState_t` opaque and supplies only the leaf
|
||||||
|
//! fields used by this two-table matcher. The long (8-byte) and short
|
||||||
|
//! (min-match) search loops, table fills, and sequence writes live here.
|
||||||
|
|
||||||
|
use crate::mem::{
|
||||||
|
MEM_64bits, MEM_isLittleEndian, MEM_read16, MEM_read32, MEM_read64, MEM_readLE32, MEM_readLE64,
|
||||||
|
MEM_readST,
|
||||||
|
};
|
||||||
|
use std::ffi::c_void;
|
||||||
|
use std::mem::size_of;
|
||||||
|
use std::os::raw::c_int;
|
||||||
|
use std::ptr;
|
||||||
|
|
||||||
|
const ZSTD_REP_NUM: usize = 3;
|
||||||
|
const MINMATCH: usize = 3;
|
||||||
|
const HASH_READ_SIZE: usize = 8;
|
||||||
|
const SHORT_CACHE_TAG_BITS: u32 = 8;
|
||||||
|
const SHORT_CACHE_TAG_MASK: u32 = (1 << SHORT_CACHE_TAG_BITS) - 1;
|
||||||
|
const K_SEARCH_STRENGTH: usize = 8;
|
||||||
|
const REPCODE1_TO_OFFBASE: u32 = 1;
|
||||||
|
|
||||||
|
#[repr(C)]
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
struct SeqDef {
|
||||||
|
offBase: u32,
|
||||||
|
litLength: u16,
|
||||||
|
mlBase: u16,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The C shim checks this leaf ABI. `ZSTD_MatchState_t` remains wholly in C.
|
||||||
|
#[repr(C)]
|
||||||
|
struct SeqStore_t {
|
||||||
|
sequencesStart: *mut SeqDef,
|
||||||
|
sequences: *mut SeqDef,
|
||||||
|
litStart: *mut u8,
|
||||||
|
lit: *mut u8,
|
||||||
|
llCode: *mut u8,
|
||||||
|
mlCode: *mut u8,
|
||||||
|
ofCode: *mut u8,
|
||||||
|
maxNbSeq: usize,
|
||||||
|
maxNbLit: usize,
|
||||||
|
longLengthType: c_int,
|
||||||
|
longLengthPos: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn ptr_lt(left: *const u8, right: *const u8) -> bool {
|
||||||
|
(left as usize) < (right as usize)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn ptr_le(left: *const u8, right: *const u8) -> bool {
|
||||||
|
(left as usize) <= (right as usize)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn ptr_gt(left: *const u8, right: *const u8) -> bool {
|
||||||
|
(left as usize) > (right as usize)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn ptr_ge(left: *const u8, right: *const u8) -> bool {
|
||||||
|
(left as usize) >= (right as usize)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
unsafe fn index_from(base: *const u8, ptr: *const u8) -> u32 {
|
||||||
|
unsafe { ptr.offset_from(base) as u32 }
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
unsafe fn read32(ptr: *const u8) -> u32 {
|
||||||
|
unsafe { MEM_read32(ptr.cast::<c_void>()) }
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
unsafe fn read64(ptr: *const u8) -> u64 {
|
||||||
|
unsafe { MEM_read64(ptr.cast::<c_void>()) }
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
unsafe fn table_get(table: *const u32, index: usize) -> u32 {
|
||||||
|
unsafe { *table.add(index) }
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
unsafe fn table_set(table: *mut u32, index: usize, value: u32) {
|
||||||
|
unsafe { *table.add(index) = value };
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn hash_shift32(value: u32, hbits: u32) -> usize {
|
||||||
|
if hbits == 0 {
|
||||||
|
0
|
||||||
|
} else {
|
||||||
|
(value >> (32 - hbits)) as usize
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn hash_shift64(value: u64, hbits: u32) -> usize {
|
||||||
|
if hbits == 0 {
|
||||||
|
0
|
||||||
|
} else {
|
||||||
|
(value >> (64 - hbits)) as usize
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
unsafe fn hash_ptr(ptr: *const u8, hbits: u32, mls: u32) -> usize {
|
||||||
|
match mls {
|
||||||
|
5 => {
|
||||||
|
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
|
||||||
|
hash_shift64(value.wrapping_shl(24).wrapping_mul(889_523_592_379), hbits)
|
||||||
|
}
|
||||||
|
6 => {
|
||||||
|
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
|
||||||
|
hash_shift64(
|
||||||
|
value.wrapping_shl(16).wrapping_mul(227_718_039_650_203),
|
||||||
|
hbits,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
7 => {
|
||||||
|
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
|
||||||
|
hash_shift64(
|
||||||
|
value.wrapping_shl(8).wrapping_mul(58_295_818_150_454_627),
|
||||||
|
hbits,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
8 => {
|
||||||
|
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
|
||||||
|
hash_shift64(value.wrapping_mul(0xCF1B_BCDC_B7A5_6463), hbits)
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
let value = unsafe { MEM_readLE32(ptr.cast::<c_void>()) };
|
||||||
|
hash_shift32(value.wrapping_mul(2_654_435_761), hbits)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn common_bytes(word: usize) -> usize {
|
||||||
|
let zeros = if MEM_isLittleEndian() {
|
||||||
|
word.trailing_zeros()
|
||||||
|
} else {
|
||||||
|
word.leading_zeros()
|
||||||
|
};
|
||||||
|
(zeros / 8) as usize
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe fn count(mut input: *const u8, mut matched: *const u8, input_limit: *const u8) -> usize {
|
||||||
|
let input_start = input;
|
||||||
|
let word_size = size_of::<usize>();
|
||||||
|
while unsafe { input_limit.offset_from(input) as usize } >= word_size {
|
||||||
|
let diff =
|
||||||
|
unsafe { MEM_readST(matched.cast::<c_void>()) ^ MEM_readST(input.cast::<c_void>()) };
|
||||||
|
if diff != 0 {
|
||||||
|
return unsafe { input.offset_from(input_start) as usize } + common_bytes(diff);
|
||||||
|
}
|
||||||
|
input = input.wrapping_add(word_size);
|
||||||
|
matched = matched.wrapping_add(word_size);
|
||||||
|
}
|
||||||
|
if MEM_64bits()
|
||||||
|
&& unsafe { input_limit.offset_from(input) as usize } >= 4
|
||||||
|
&& unsafe { MEM_read32(matched.cast::<c_void>()) == MEM_read32(input.cast::<c_void>()) }
|
||||||
|
{
|
||||||
|
input = input.wrapping_add(4);
|
||||||
|
matched = matched.wrapping_add(4);
|
||||||
|
}
|
||||||
|
if unsafe { input_limit.offset_from(input) as usize } >= 2
|
||||||
|
&& unsafe { MEM_read16(matched.cast::<c_void>()) == MEM_read16(input.cast::<c_void>()) }
|
||||||
|
{
|
||||||
|
input = input.wrapping_add(2);
|
||||||
|
matched = matched.wrapping_add(2);
|
||||||
|
}
|
||||||
|
if ptr_lt(input, input_limit) && unsafe { *input == *matched } {
|
||||||
|
input = input.wrapping_add(1);
|
||||||
|
}
|
||||||
|
unsafe { input.offset_from(input_start) as usize }
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe fn count_2segments(
|
||||||
|
input: *const u8,
|
||||||
|
matched: *const u8,
|
||||||
|
input_end: *const u8,
|
||||||
|
match_end: *const u8,
|
||||||
|
input_start: *const u8,
|
||||||
|
) -> usize {
|
||||||
|
let match_remaining = unsafe { match_end.offset_from(matched) as usize };
|
||||||
|
let input_remaining = unsafe { input_end.offset_from(input) as usize };
|
||||||
|
let first_end = input.wrapping_add(match_remaining.min(input_remaining));
|
||||||
|
let first_count = unsafe { count(input, matched, first_end) };
|
||||||
|
if matched.wrapping_add(first_count) != match_end {
|
||||||
|
return first_count;
|
||||||
|
}
|
||||||
|
first_count + unsafe { count(input.wrapping_add(first_count), input_start, input_end) }
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn lowest_prefix_index(dict_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 {
|
||||||
|
let max_distance = 1u32.wrapping_shl(window_log);
|
||||||
|
let within_window = if curr.wrapping_sub(dict_limit) > max_distance {
|
||||||
|
curr.wrapping_sub(max_distance)
|
||||||
|
} else {
|
||||||
|
dict_limit
|
||||||
|
};
|
||||||
|
if loaded_dict_end != 0 {
|
||||||
|
dict_limit
|
||||||
|
} else {
|
||||||
|
within_window
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn lowest_match_index(low_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 {
|
||||||
|
let max_distance = 1u32.wrapping_shl(window_log);
|
||||||
|
let within_window = if curr.wrapping_sub(low_limit) > max_distance {
|
||||||
|
curr.wrapping_sub(max_distance)
|
||||||
|
} else {
|
||||||
|
low_limit
|
||||||
|
};
|
||||||
|
if loaded_dict_end != 0 {
|
||||||
|
low_limit
|
||||||
|
} else {
|
||||||
|
within_window
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool {
|
||||||
|
prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn write_tagged_index(table: *mut u32, hash_and_tag: usize, index: u32) {
|
||||||
|
let hash = hash_and_tag >> SHORT_CACHE_TAG_BITS;
|
||||||
|
let tag = (hash_and_tag as u32) & SHORT_CACHE_TAG_MASK;
|
||||||
|
unsafe { table_set(table, hash, (index << SHORT_CACHE_TAG_BITS) | tag) };
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn packed_tags_match(first: u32, second: usize) -> bool {
|
||||||
|
(first & SHORT_CACHE_TAG_MASK) == ((second as u32) & SHORT_CACHE_TAG_MASK)
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe fn store_seq(
|
||||||
|
seq_store: *mut SeqStore_t,
|
||||||
|
lit_length: usize,
|
||||||
|
literals: *const u8,
|
||||||
|
_lit_limit: *const u8,
|
||||||
|
off_base: u32,
|
||||||
|
match_length: usize,
|
||||||
|
) {
|
||||||
|
let seq_store = unsafe { &mut *seq_store };
|
||||||
|
let sequence = seq_store.sequences;
|
||||||
|
debug_assert!(
|
||||||
|
unsafe { sequence.offset_from(seq_store.sequencesStart) as usize } < seq_store.maxNbSeq
|
||||||
|
);
|
||||||
|
debug_assert!(match_length >= MINMATCH);
|
||||||
|
if lit_length != 0 {
|
||||||
|
unsafe { ptr::copy_nonoverlapping(literals, seq_store.lit, lit_length) };
|
||||||
|
}
|
||||||
|
seq_store.lit = seq_store.lit.wrapping_add(lit_length);
|
||||||
|
|
||||||
|
let sequence_index = unsafe { sequence.offset_from(seq_store.sequencesStart) as u32 };
|
||||||
|
if lit_length > u16::MAX as usize {
|
||||||
|
debug_assert_eq!(seq_store.longLengthType, 0);
|
||||||
|
seq_store.longLengthType = 1;
|
||||||
|
seq_store.longLengthPos = sequence_index;
|
||||||
|
}
|
||||||
|
unsafe { (*sequence).litLength = lit_length as u16 };
|
||||||
|
unsafe { (*sequence).offBase = off_base };
|
||||||
|
let match_base = match_length - MINMATCH;
|
||||||
|
if match_base > u16::MAX as usize {
|
||||||
|
debug_assert_eq!(seq_store.longLengthType, 0);
|
||||||
|
seq_store.longLengthType = 2;
|
||||||
|
seq_store.longLengthPos = sequence_index;
|
||||||
|
}
|
||||||
|
unsafe { (*sequence).mlBase = match_base as u16 };
|
||||||
|
seq_store.sequences = sequence.wrapping_add(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
fn fast_mls(min_match: u32) -> u32 {
|
||||||
|
match min_match {
|
||||||
|
5..=7 => min_match,
|
||||||
|
_ => 4,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe fn fill_double_hash_table(
|
||||||
|
hash_long: *mut u32,
|
||||||
|
hash_small: *mut u32,
|
||||||
|
base: *const u8,
|
||||||
|
next_to_update: u32,
|
||||||
|
end: *const u8,
|
||||||
|
hash_log: u32,
|
||||||
|
chain_log: u32,
|
||||||
|
min_match: u32,
|
||||||
|
full_table_load: bool,
|
||||||
|
tagged_indices: bool,
|
||||||
|
) {
|
||||||
|
if unsafe { end.offset_from(base) } < HASH_READ_SIZE as isize {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let hbits_long = hash_log
|
||||||
|
+ if tagged_indices {
|
||||||
|
SHORT_CACHE_TAG_BITS
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
let hbits_small = chain_log
|
||||||
|
+ if tagged_indices {
|
||||||
|
SHORT_CACHE_TAG_BITS
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
let mut input = base.wrapping_add(next_to_update as usize);
|
||||||
|
let input_end = end.wrapping_sub(HASH_READ_SIZE);
|
||||||
|
|
||||||
|
while ptr_le(input.wrapping_add(2), input_end) {
|
||||||
|
let current = unsafe { index_from(base, input) };
|
||||||
|
for position in 0..3usize {
|
||||||
|
let small_hash =
|
||||||
|
unsafe { hash_ptr(input.wrapping_add(position), hbits_small, min_match) };
|
||||||
|
let long_hash = unsafe { hash_ptr(input.wrapping_add(position), hbits_long, 8) };
|
||||||
|
let small_index = if tagged_indices {
|
||||||
|
small_hash >> SHORT_CACHE_TAG_BITS
|
||||||
|
} else {
|
||||||
|
small_hash
|
||||||
|
};
|
||||||
|
let long_index = if tagged_indices {
|
||||||
|
long_hash >> SHORT_CACHE_TAG_BITS
|
||||||
|
} else {
|
||||||
|
long_hash
|
||||||
|
};
|
||||||
|
if position == 0 {
|
||||||
|
if tagged_indices {
|
||||||
|
write_tagged_index(
|
||||||
|
hash_small,
|
||||||
|
small_hash,
|
||||||
|
current.wrapping_add(position as u32),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
unsafe {
|
||||||
|
table_set(
|
||||||
|
hash_small,
|
||||||
|
small_index,
|
||||||
|
current.wrapping_add(position as u32),
|
||||||
|
)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if position == 0 || unsafe { table_get(hash_long, long_index) } == 0 {
|
||||||
|
if tagged_indices {
|
||||||
|
write_tagged_index(hash_long, long_hash, current.wrapping_add(position as u32));
|
||||||
|
} else {
|
||||||
|
unsafe {
|
||||||
|
table_set(hash_long, long_index, current.wrapping_add(position as u32))
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !full_table_load {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
input = input.wrapping_add(3);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rust implementation called by the C ABI wrapper for `ZSTD_fillDoubleHashTable`.
|
||||||
|
#[no_mangle]
|
||||||
|
pub unsafe extern "C" fn ZSTD_rust_fillDoubleHashTable(
|
||||||
|
hash_long: *mut u32,
|
||||||
|
hash_small: *mut u32,
|
||||||
|
base: *const u8,
|
||||||
|
next_to_update: u32,
|
||||||
|
end: *const c_void,
|
||||||
|
hash_log: u32,
|
||||||
|
chain_log: u32,
|
||||||
|
min_match: u32,
|
||||||
|
full_table_load: c_int,
|
||||||
|
for_cdict: c_int,
|
||||||
|
) {
|
||||||
|
unsafe {
|
||||||
|
fill_double_hash_table(
|
||||||
|
hash_long,
|
||||||
|
hash_small,
|
||||||
|
base,
|
||||||
|
next_to_update,
|
||||||
|
end.cast::<u8>(),
|
||||||
|
hash_log,
|
||||||
|
chain_log,
|
||||||
|
min_match,
|
||||||
|
full_table_load != 0,
|
||||||
|
for_cdict != 0,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe fn post_no_dict_match(
|
||||||
|
hash_long: *mut u32,
|
||||||
|
hash_small: *mut u32,
|
||||||
|
base: *const u8,
|
||||||
|
hbits_long: u32,
|
||||||
|
hbits_small: u32,
|
||||||
|
mls: u32,
|
||||||
|
seq_store: *mut SeqStore_t,
|
||||||
|
offset1: &mut u32,
|
||||||
|
offset2: &mut u32,
|
||||||
|
mut ip: *const u8,
|
||||||
|
match_length: usize,
|
||||||
|
current: u32,
|
||||||
|
iend: *const u8,
|
||||||
|
ilimit: *const u8,
|
||||||
|
) -> (*const u8, *const u8) {
|
||||||
|
ip = ip.wrapping_add(match_length);
|
||||||
|
let mut anchor = ip;
|
||||||
|
if ptr_le(ip, ilimit) {
|
||||||
|
let index_to_insert = current.wrapping_add(2);
|
||||||
|
let hash = unsafe { hash_ptr(base.wrapping_add(index_to_insert as usize), hbits_long, 8) };
|
||||||
|
unsafe { table_set(hash_long, hash, index_to_insert) };
|
||||||
|
let hash = unsafe { hash_ptr(ip.wrapping_sub(2), hbits_long, 8) };
|
||||||
|
unsafe { table_set(hash_long, hash, index_from(base, ip.wrapping_sub(2))) };
|
||||||
|
let hash = unsafe {
|
||||||
|
hash_ptr(
|
||||||
|
base.wrapping_add(index_to_insert as usize),
|
||||||
|
hbits_small,
|
||||||
|
mls,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
unsafe { table_set(hash_small, hash, index_to_insert) };
|
||||||
|
let hash = unsafe { hash_ptr(ip.wrapping_sub(1), hbits_small, mls) };
|
||||||
|
unsafe { table_set(hash_small, hash, index_from(base, ip.wrapping_sub(1))) };
|
||||||
|
|
||||||
|
while ptr_le(ip, ilimit)
|
||||||
|
&& *offset2 > 0
|
||||||
|
&& unsafe { read32(ip) == read32(ip.wrapping_sub(*offset2 as usize)) }
|
||||||
|
{
|
||||||
|
let repeat_length = unsafe {
|
||||||
|
count(
|
||||||
|
ip.wrapping_add(4),
|
||||||
|
ip.wrapping_add(4).wrapping_sub(*offset2 as usize),
|
||||||
|
iend,
|
||||||
|
) + 4
|
||||||
|
};
|
||||||
|
std::mem::swap(offset1, offset2);
|
||||||
|
let hash = unsafe { hash_ptr(ip, hbits_small, mls) };
|
||||||
|
unsafe { table_set(hash_small, hash, index_from(base, ip)) };
|
||||||
|
let hash = unsafe { hash_ptr(ip, hbits_long, 8) };
|
||||||
|
unsafe { table_set(hash_long, hash, index_from(base, ip)) };
|
||||||
|
unsafe {
|
||||||
|
store_seq(
|
||||||
|
seq_store,
|
||||||
|
0,
|
||||||
|
anchor,
|
||||||
|
iend,
|
||||||
|
REPCODE1_TO_OFFBASE,
|
||||||
|
repeat_length,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
ip = ip.wrapping_add(repeat_length);
|
||||||
|
anchor = ip;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(ip, anchor)
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe fn compress_block_double_fast_no_dict(
|
||||||
|
hash_long: *mut u32,
|
||||||
|
hash_small: *mut u32,
|
||||||
|
base: *const u8,
|
||||||
|
dict_limit: u32,
|
||||||
|
loaded_dict_end: u32,
|
||||||
|
hash_log: u32,
|
||||||
|
chain_log: u32,
|
||||||
|
window_log: u32,
|
||||||
|
seq_store: *mut SeqStore_t,
|
||||||
|
reps: *mut u32,
|
||||||
|
src: *const u8,
|
||||||
|
src_size: usize,
|
||||||
|
mls: u32,
|
||||||
|
) -> usize {
|
||||||
|
if src_size < HASH_READ_SIZE {
|
||||||
|
return src_size;
|
||||||
|
}
|
||||||
|
let istart = src;
|
||||||
|
let iend = istart.wrapping_add(src_size);
|
||||||
|
let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
|
||||||
|
let end_index = unsafe { index_from(base, istart) }.wrapping_add(src_size as u32);
|
||||||
|
let prefix_lowest_index =
|
||||||
|
lowest_prefix_index(dict_limit, loaded_dict_end, end_index, window_log);
|
||||||
|
let prefix_lowest = base.wrapping_add(prefix_lowest_index as usize);
|
||||||
|
let mut ip = istart;
|
||||||
|
let mut anchor = istart;
|
||||||
|
let mut offset1 = unsafe { *reps };
|
||||||
|
let mut offset2 = unsafe { *reps.add(1) };
|
||||||
|
let mut offset_saved1 = 0u32;
|
||||||
|
let mut offset_saved2 = 0u32;
|
||||||
|
|
||||||
|
if ip == prefix_lowest {
|
||||||
|
ip = ip.wrapping_add(1);
|
||||||
|
}
|
||||||
|
let current = unsafe { index_from(base, ip) };
|
||||||
|
let window_low = lowest_prefix_index(dict_limit, loaded_dict_end, current, window_log);
|
||||||
|
let max_rep = current.wrapping_sub(window_low);
|
||||||
|
if offset2 > max_rep {
|
||||||
|
offset_saved2 = offset2;
|
||||||
|
offset2 = 0;
|
||||||
|
}
|
||||||
|
if offset1 > max_rep {
|
||||||
|
offset_saved1 = offset1;
|
||||||
|
offset1 = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
'outer: loop {
|
||||||
|
let mut step = 1usize;
|
||||||
|
let mut next_step = ip.wrapping_add(1 << K_SEARCH_STRENGTH);
|
||||||
|
let mut ip1 = ip.wrapping_add(step);
|
||||||
|
if ptr_gt(ip1, ilimit) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut hash_long0 = unsafe { hash_ptr(ip, hash_log, 8) };
|
||||||
|
let mut index_long0 = unsafe { table_get(hash_long, hash_long0) };
|
||||||
|
let mut match_long0 = base.wrapping_add(index_long0 as usize);
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let hash_small0 = unsafe { hash_ptr(ip, chain_log, mls) };
|
||||||
|
let index_small0 = unsafe { table_get(hash_small, hash_small0) };
|
||||||
|
let mut match_small0 = base.wrapping_add(index_small0 as usize);
|
||||||
|
let current = unsafe { index_from(base, ip) };
|
||||||
|
unsafe {
|
||||||
|
table_set(hash_long, hash_long0, current);
|
||||||
|
table_set(hash_small, hash_small0, current);
|
||||||
|
}
|
||||||
|
|
||||||
|
if offset1 > 0
|
||||||
|
&& unsafe {
|
||||||
|
read32(ip.wrapping_add(1).wrapping_sub(offset1 as usize))
|
||||||
|
== read32(ip.wrapping_add(1))
|
||||||
|
}
|
||||||
|
{
|
||||||
|
let match_length = unsafe {
|
||||||
|
count(
|
||||||
|
ip.wrapping_add(5),
|
||||||
|
ip.wrapping_add(5).wrapping_sub(offset1 as usize),
|
||||||
|
iend,
|
||||||
|
) + 4
|
||||||
|
};
|
||||||
|
ip = ip.wrapping_add(1);
|
||||||
|
unsafe {
|
||||||
|
store_seq(
|
||||||
|
seq_store,
|
||||||
|
ip.offset_from(anchor) as usize,
|
||||||
|
anchor,
|
||||||
|
iend,
|
||||||
|
REPCODE1_TO_OFFBASE,
|
||||||
|
match_length,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let (next_ip, next_anchor) = unsafe {
|
||||||
|
post_no_dict_match(
|
||||||
|
hash_long,
|
||||||
|
hash_small,
|
||||||
|
base,
|
||||||
|
hash_log,
|
||||||
|
chain_log,
|
||||||
|
mls,
|
||||||
|
seq_store,
|
||||||
|
&mut offset1,
|
||||||
|
&mut offset2,
|
||||||
|
ip,
|
||||||
|
match_length,
|
||||||
|
current,
|
||||||
|
iend,
|
||||||
|
ilimit,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
ip = next_ip;
|
||||||
|
anchor = next_anchor;
|
||||||
|
continue 'outer;
|
||||||
|
}
|
||||||
|
|
||||||
|
let hash_long1 = unsafe { hash_ptr(ip1, hash_log, 8) };
|
||||||
|
if index_long0 > prefix_lowest_index && unsafe { read64(match_long0) == read64(ip) } {
|
||||||
|
let mut match_length =
|
||||||
|
unsafe { count(ip.wrapping_add(8), match_long0.wrapping_add(8), iend) + 8 };
|
||||||
|
let offset = unsafe { index_from(match_long0, ip) };
|
||||||
|
while ptr_gt(ip, anchor)
|
||||||
|
&& ptr_gt(match_long0, prefix_lowest)
|
||||||
|
&& unsafe { *ip.wrapping_sub(1) == *match_long0.wrapping_sub(1) }
|
||||||
|
{
|
||||||
|
ip = ip.wrapping_sub(1);
|
||||||
|
match_long0 = match_long0.wrapping_sub(1);
|
||||||
|
match_length += 1;
|
||||||
|
}
|
||||||
|
offset2 = offset1;
|
||||||
|
offset1 = offset;
|
||||||
|
if step < 4 {
|
||||||
|
unsafe { table_set(hash_long, hash_long1, index_from(base, ip1)) };
|
||||||
|
}
|
||||||
|
unsafe {
|
||||||
|
store_seq(
|
||||||
|
seq_store,
|
||||||
|
ip.offset_from(anchor) as usize,
|
||||||
|
anchor,
|
||||||
|
iend,
|
||||||
|
offset.wrapping_add(ZSTD_REP_NUM as u32),
|
||||||
|
match_length,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let (next_ip, next_anchor) = unsafe {
|
||||||
|
post_no_dict_match(
|
||||||
|
hash_long,
|
||||||
|
hash_small,
|
||||||
|
base,
|
||||||
|
hash_log,
|
||||||
|
chain_log,
|
||||||
|
mls,
|
||||||
|
seq_store,
|
||||||
|
&mut offset1,
|
||||||
|
&mut offset2,
|
||||||
|
ip,
|
||||||
|
match_length,
|
||||||
|
current,
|
||||||
|
iend,
|
||||||
|
ilimit,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
ip = next_ip;
|
||||||
|
anchor = next_anchor;
|
||||||
|
continue 'outer;
|
||||||
|
}
|
||||||
|
|
||||||
|
let index_long1 = unsafe { table_get(hash_long, hash_long1) };
|
||||||
|
let match_long1 = base.wrapping_add(index_long1 as usize);
|
||||||
|
if index_small0 > prefix_lowest_index && unsafe { read32(match_small0) == read32(ip) } {
|
||||||
|
let mut match_length =
|
||||||
|
unsafe { count(ip.wrapping_add(4), match_small0.wrapping_add(4), iend) + 4 };
|
||||||
|
let mut offset = unsafe { index_from(match_small0, ip) };
|
||||||
|
if index_long1 > prefix_lowest_index
|
||||||
|
&& unsafe { read64(match_long1) == read64(ip1) }
|
||||||
|
{
|
||||||
|
let long_length = unsafe {
|
||||||
|
count(ip1.wrapping_add(8), match_long1.wrapping_add(8), iend) + 8
|
||||||
|
};
|
||||||
|
if long_length > match_length {
|
||||||
|
ip = ip1;
|
||||||
|
match_length = long_length;
|
||||||
|
offset = unsafe { index_from(match_long1, ip) };
|
||||||
|
match_small0 = match_long1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
while ptr_gt(ip, anchor)
|
||||||
|
&& ptr_gt(match_small0, prefix_lowest)
|
||||||
|
&& unsafe { *ip.wrapping_sub(1) == *match_small0.wrapping_sub(1) }
|
||||||
|
{
|
||||||
|
ip = ip.wrapping_sub(1);
|
||||||
|
match_small0 = match_small0.wrapping_sub(1);
|
||||||
|
match_length += 1;
|
||||||
|
}
|
||||||
|
offset2 = offset1;
|
||||||
|
offset1 = offset;
|
||||||
|
if step < 4 {
|
||||||
|
unsafe { table_set(hash_long, hash_long1, index_from(base, ip1)) };
|
||||||
|
}
|
||||||
|
unsafe {
|
||||||
|
store_seq(
|
||||||
|
seq_store,
|
||||||
|
ip.offset_from(anchor) as usize,
|
||||||
|
anchor,
|
||||||
|
iend,
|
||||||
|
offset.wrapping_add(ZSTD_REP_NUM as u32),
|
||||||
|
match_length,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let (next_ip, next_anchor) = unsafe {
|
||||||
|
post_no_dict_match(
|
||||||
|
hash_long,
|
||||||
|
hash_small,
|
||||||
|
base,
|
||||||
|
hash_log,
|
||||||
|
chain_log,
|
||||||
|
mls,
|
||||||
|
seq_store,
|
||||||
|
&mut offset1,
|
||||||
|
&mut offset2,
|
||||||
|
ip,
|
||||||
|
match_length,
|
||||||
|
current,
|
||||||
|
iend,
|
||||||
|
ilimit,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
ip = next_ip;
|
||||||
|
anchor = next_anchor;
|
||||||
|
continue 'outer;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ptr_ge(ip1, next_step) {
|
||||||
|
step += 1;
|
||||||
|
next_step = next_step.wrapping_add(1 << K_SEARCH_STRENGTH);
|
||||||
|
}
|
||||||
|
ip = ip1;
|
||||||
|
ip1 = ip1.wrapping_add(step);
|
||||||
|
hash_long0 = hash_long1;
|
||||||
|
index_long0 = index_long1;
|
||||||
|
match_long0 = match_long1;
|
||||||
|
if ptr_gt(ip1, ilimit) {
|
||||||
|
break 'outer;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
offset_saved2 = if offset_saved1 != 0 && offset1 != 0 {
|
||||||
|
offset_saved1
|
||||||
|
} else {
|
||||||
|
offset_saved2
|
||||||
|
};
|
||||||
|
unsafe {
|
||||||
|
*reps = if offset1 != 0 { offset1 } else { offset_saved1 };
|
||||||
|
*reps.add(1) = if offset2 != 0 { offset2 } else { offset_saved2 };
|
||||||
|
}
|
||||||
|
unsafe { iend.offset_from(anchor) as usize }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rust implementation called by the C ABI wrapper for `ZSTD_compressBlock_doubleFast`.
|
||||||
|
#[no_mangle]
|
||||||
|
pub unsafe extern "C" fn ZSTD_rust_compressBlock_doubleFast(
|
||||||
|
hash_long: *mut u32,
|
||||||
|
hash_small: *mut u32,
|
||||||
|
base: *const u8,
|
||||||
|
dict_limit: u32,
|
||||||
|
loaded_dict_end: u32,
|
||||||
|
hash_log: u32,
|
||||||
|
chain_log: u32,
|
||||||
|
min_match: u32,
|
||||||
|
window_log: u32,
|
||||||
|
seq_store: *mut c_void,
|
||||||
|
reps: *mut u32,
|
||||||
|
src: *const c_void,
|
||||||
|
src_size: usize,
|
||||||
|
) -> usize {
|
||||||
|
unsafe {
|
||||||
|
compress_block_double_fast_no_dict(
|
||||||
|
hash_long,
|
||||||
|
hash_small,
|
||||||
|
base,
|
||||||
|
dict_limit,
|
||||||
|
loaded_dict_end,
|
||||||
|
hash_log,
|
||||||
|
chain_log,
|
||||||
|
window_log,
|
||||||
|
seq_store.cast::<SeqStore_t>(),
|
||||||
|
reps,
|
||||||
|
src.cast::<u8>(),
|
||||||
|
src_size,
|
||||||
|
fast_mls(min_match),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe fn post_dict_match(
|
||||||
|
hash_long: *mut u32,
|
||||||
|
hash_small: *mut u32,
|
||||||
|
base: *const u8,
|
||||||
|
dict_base: *const u8,
|
||||||
|
prefix_lowest_index: u32,
|
||||||
|
dict_index_delta: u32,
|
||||||
|
dict_end: *const u8,
|
||||||
|
prefix_lowest: *const u8,
|
||||||
|
hbits_long: u32,
|
||||||
|
hbits_small: u32,
|
||||||
|
mls: u32,
|
||||||
|
seq_store: *mut SeqStore_t,
|
||||||
|
offset1: &mut u32,
|
||||||
|
offset2: &mut u32,
|
||||||
|
mut ip: *const u8,
|
||||||
|
match_length: usize,
|
||||||
|
current: u32,
|
||||||
|
iend: *const u8,
|
||||||
|
ilimit: *const u8,
|
||||||
|
) -> (*const u8, *const u8) {
|
||||||
|
ip = ip.wrapping_add(match_length);
|
||||||
|
let mut anchor = ip;
|
||||||
|
if ptr_le(ip, ilimit) {
|
||||||
|
let index_to_insert = current.wrapping_add(2);
|
||||||
|
let hash = unsafe { hash_ptr(base.wrapping_add(index_to_insert as usize), hbits_long, 8) };
|
||||||
|
unsafe { table_set(hash_long, hash, index_to_insert) };
|
||||||
|
let hash = unsafe { hash_ptr(ip.wrapping_sub(2), hbits_long, 8) };
|
||||||
|
unsafe { table_set(hash_long, hash, index_from(base, ip.wrapping_sub(2))) };
|
||||||
|
let hash = unsafe {
|
||||||
|
hash_ptr(
|
||||||
|
base.wrapping_add(index_to_insert as usize),
|
||||||
|
hbits_small,
|
||||||
|
mls,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
unsafe { table_set(hash_small, hash, index_to_insert) };
|
||||||
|
let hash = unsafe { hash_ptr(ip.wrapping_sub(1), hbits_small, mls) };
|
||||||
|
unsafe { table_set(hash_small, hash, index_from(base, ip.wrapping_sub(1))) };
|
||||||
|
|
||||||
|
while ptr_le(ip, ilimit) {
|
||||||
|
let current2 = unsafe { index_from(base, ip) };
|
||||||
|
let rep_index2 = current2.wrapping_sub(*offset2);
|
||||||
|
let rep_match2 = if rep_index2 < prefix_lowest_index {
|
||||||
|
dict_base
|
||||||
|
.wrapping_sub(dict_index_delta as usize)
|
||||||
|
.wrapping_add(rep_index2 as usize)
|
||||||
|
} else {
|
||||||
|
base.wrapping_add(rep_index2 as usize)
|
||||||
|
};
|
||||||
|
if index_overlap_check(prefix_lowest_index, rep_index2)
|
||||||
|
&& unsafe { read32(rep_match2) == read32(ip) }
|
||||||
|
{
|
||||||
|
let rep_end2 = if rep_index2 < prefix_lowest_index {
|
||||||
|
dict_end
|
||||||
|
} else {
|
||||||
|
iend
|
||||||
|
};
|
||||||
|
let repeat_length = unsafe {
|
||||||
|
count_2segments(
|
||||||
|
ip.wrapping_add(4),
|
||||||
|
rep_match2.wrapping_add(4),
|
||||||
|
iend,
|
||||||
|
rep_end2,
|
||||||
|
prefix_lowest,
|
||||||
|
) + 4
|
||||||
|
};
|
||||||
|
std::mem::swap(offset1, offset2);
|
||||||
|
unsafe {
|
||||||
|
store_seq(
|
||||||
|
seq_store,
|
||||||
|
0,
|
||||||
|
anchor,
|
||||||
|
iend,
|
||||||
|
REPCODE1_TO_OFFBASE,
|
||||||
|
repeat_length,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let hash = unsafe { hash_ptr(ip, hbits_small, mls) };
|
||||||
|
unsafe { table_set(hash_small, hash, current2) };
|
||||||
|
let hash = unsafe { hash_ptr(ip, hbits_long, 8) };
|
||||||
|
unsafe { table_set(hash_long, hash, current2) };
|
||||||
|
ip = ip.wrapping_add(repeat_length);
|
||||||
|
anchor = ip;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(ip, anchor)
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe fn compress_block_double_fast_dict_match_state(
|
||||||
|
hash_long: *mut u32,
|
||||||
|
hash_small: *mut u32,
|
||||||
|
base: *const u8,
|
||||||
|
prefix_lowest_index: u32,
|
||||||
|
hash_log: u32,
|
||||||
|
chain_log: u32,
|
||||||
|
seq_store: *mut SeqStore_t,
|
||||||
|
reps: *mut u32,
|
||||||
|
src: *const u8,
|
||||||
|
src_size: usize,
|
||||||
|
mls: u32,
|
||||||
|
dict_hash_long: *const u32,
|
||||||
|
dict_hash_small: *const u32,
|
||||||
|
dict_base: *const u8,
|
||||||
|
dict_start_index: u32,
|
||||||
|
dict_end: *const u8,
|
||||||
|
dict_hash_log: u32,
|
||||||
|
dict_chain_log: u32,
|
||||||
|
_prefetch_cdict_tables: bool,
|
||||||
|
) -> usize {
|
||||||
|
if src_size < HASH_READ_SIZE {
|
||||||
|
return src_size;
|
||||||
|
}
|
||||||
|
let istart = src;
|
||||||
|
let iend = istart.wrapping_add(src_size);
|
||||||
|
let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
|
||||||
|
let mut ip = istart;
|
||||||
|
let mut anchor = istart;
|
||||||
|
let prefix_lowest = base.wrapping_add(prefix_lowest_index as usize);
|
||||||
|
let dict_start = dict_base.wrapping_add(dict_start_index as usize);
|
||||||
|
let dict_index_delta =
|
||||||
|
prefix_lowest_index.wrapping_sub(unsafe { index_from(dict_base, dict_end) });
|
||||||
|
let dict_hbits_long = dict_hash_log + SHORT_CACHE_TAG_BITS;
|
||||||
|
let dict_hbits_small = dict_chain_log + SHORT_CACHE_TAG_BITS;
|
||||||
|
let dict_and_prefix_length =
|
||||||
|
unsafe { ip.offset_from(prefix_lowest) as u32 + dict_end.offset_from(dict_start) as u32 };
|
||||||
|
let mut offset1 = unsafe { *reps };
|
||||||
|
let mut offset2 = unsafe { *reps.add(1) };
|
||||||
|
if dict_and_prefix_length == 0 {
|
||||||
|
ip = ip.wrapping_add(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
'outer: while ptr_lt(ip, ilimit) {
|
||||||
|
let h_long = unsafe { hash_ptr(ip, hash_log, 8) };
|
||||||
|
let h_small = unsafe { hash_ptr(ip, chain_log, mls) };
|
||||||
|
let dict_hash_and_tag_long = unsafe { hash_ptr(ip, dict_hbits_long, 8) };
|
||||||
|
let dict_hash_and_tag_small = unsafe { hash_ptr(ip, dict_hbits_small, mls) };
|
||||||
|
let dict_match_index_and_tag_long = unsafe {
|
||||||
|
table_get(
|
||||||
|
dict_hash_long,
|
||||||
|
dict_hash_and_tag_long >> SHORT_CACHE_TAG_BITS,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let dict_match_index_and_tag_small = unsafe {
|
||||||
|
table_get(
|
||||||
|
dict_hash_small,
|
||||||
|
dict_hash_and_tag_small >> SHORT_CACHE_TAG_BITS,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let dict_tags_match_long =
|
||||||
|
packed_tags_match(dict_match_index_and_tag_long, dict_hash_and_tag_long);
|
||||||
|
let dict_tags_match_small =
|
||||||
|
packed_tags_match(dict_match_index_and_tag_small, dict_hash_and_tag_small);
|
||||||
|
let current = unsafe { index_from(base, ip) };
|
||||||
|
let match_index_long = unsafe { table_get(hash_long, h_long) };
|
||||||
|
let mut match_index_small = unsafe { table_get(hash_small, h_small) };
|
||||||
|
let mut match_long = base.wrapping_add(match_index_long as usize);
|
||||||
|
let mut matched = base.wrapping_add(match_index_small as usize);
|
||||||
|
let rep_index = current.wrapping_add(1).wrapping_sub(offset1);
|
||||||
|
let rep_match = if rep_index < prefix_lowest_index {
|
||||||
|
dict_base.wrapping_add(rep_index.wrapping_sub(dict_index_delta) as usize)
|
||||||
|
} else {
|
||||||
|
base.wrapping_add(rep_index as usize)
|
||||||
|
};
|
||||||
|
unsafe {
|
||||||
|
table_set(hash_long, h_long, current);
|
||||||
|
table_set(hash_small, h_small, current);
|
||||||
|
}
|
||||||
|
|
||||||
|
if index_overlap_check(prefix_lowest_index, rep_index)
|
||||||
|
&& unsafe { read32(rep_match) == read32(ip.wrapping_add(1)) }
|
||||||
|
{
|
||||||
|
let rep_match_end = if rep_index < prefix_lowest_index {
|
||||||
|
dict_end
|
||||||
|
} else {
|
||||||
|
iend
|
||||||
|
};
|
||||||
|
let match_length = unsafe {
|
||||||
|
count_2segments(
|
||||||
|
ip.wrapping_add(5),
|
||||||
|
rep_match.wrapping_add(4),
|
||||||
|
iend,
|
||||||
|
rep_match_end,
|
||||||
|
prefix_lowest,
|
||||||
|
) + 4
|
||||||
|
};
|
||||||
|
ip = ip.wrapping_add(1);
|
||||||
|
unsafe {
|
||||||
|
store_seq(
|
||||||
|
seq_store,
|
||||||
|
ip.offset_from(anchor) as usize,
|
||||||
|
anchor,
|
||||||
|
iend,
|
||||||
|
REPCODE1_TO_OFFBASE,
|
||||||
|
match_length,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let (next_ip, next_anchor) = unsafe {
|
||||||
|
post_dict_match(
|
||||||
|
hash_long,
|
||||||
|
hash_small,
|
||||||
|
base,
|
||||||
|
dict_base,
|
||||||
|
prefix_lowest_index,
|
||||||
|
dict_index_delta,
|
||||||
|
dict_end,
|
||||||
|
prefix_lowest,
|
||||||
|
hash_log,
|
||||||
|
chain_log,
|
||||||
|
mls,
|
||||||
|
seq_store,
|
||||||
|
&mut offset1,
|
||||||
|
&mut offset2,
|
||||||
|
ip,
|
||||||
|
match_length,
|
||||||
|
current,
|
||||||
|
iend,
|
||||||
|
ilimit,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
ip = next_ip;
|
||||||
|
anchor = next_anchor;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if match_index_long >= prefix_lowest_index && unsafe { read64(match_long) == read64(ip) } {
|
||||||
|
let mut match_length =
|
||||||
|
unsafe { count(ip.wrapping_add(8), match_long.wrapping_add(8), iend) + 8 };
|
||||||
|
let offset = unsafe { index_from(match_long, ip) };
|
||||||
|
while ptr_gt(ip, anchor)
|
||||||
|
&& ptr_gt(match_long, prefix_lowest)
|
||||||
|
&& unsafe { *ip.wrapping_sub(1) == *match_long.wrapping_sub(1) }
|
||||||
|
{
|
||||||
|
ip = ip.wrapping_sub(1);
|
||||||
|
match_long = match_long.wrapping_sub(1);
|
||||||
|
match_length += 1;
|
||||||
|
}
|
||||||
|
offset2 = offset1;
|
||||||
|
offset1 = offset;
|
||||||
|
unsafe {
|
||||||
|
store_seq(
|
||||||
|
seq_store,
|
||||||
|
ip.offset_from(anchor) as usize,
|
||||||
|
anchor,
|
||||||
|
iend,
|
||||||
|
offset.wrapping_add(ZSTD_REP_NUM as u32),
|
||||||
|
match_length,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let (next_ip, next_anchor) = unsafe {
|
||||||
|
post_dict_match(
|
||||||
|
hash_long,
|
||||||
|
hash_small,
|
||||||
|
base,
|
||||||
|
dict_base,
|
||||||
|
prefix_lowest_index,
|
||||||
|
dict_index_delta,
|
||||||
|
dict_end,
|
||||||
|
prefix_lowest,
|
||||||
|
hash_log,
|
||||||
|
chain_log,
|
||||||
|
mls,
|
||||||
|
seq_store,
|
||||||
|
&mut offset1,
|
||||||
|
&mut offset2,
|
||||||
|
ip,
|
||||||
|
match_length,
|
||||||
|
current,
|
||||||
|
iend,
|
||||||
|
ilimit,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
ip = next_ip;
|
||||||
|
anchor = next_anchor;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if dict_tags_match_long {
|
||||||
|
let dict_match_index = dict_match_index_and_tag_long >> SHORT_CACHE_TAG_BITS;
|
||||||
|
let mut dict_match = dict_base.wrapping_add(dict_match_index as usize);
|
||||||
|
if ptr_gt(dict_match, dict_start) && unsafe { read64(dict_match) == read64(ip) } {
|
||||||
|
let mut match_length = unsafe {
|
||||||
|
count_2segments(
|
||||||
|
ip.wrapping_add(8),
|
||||||
|
dict_match.wrapping_add(8),
|
||||||
|
iend,
|
||||||
|
dict_end,
|
||||||
|
prefix_lowest,
|
||||||
|
) + 8
|
||||||
|
};
|
||||||
|
let offset = current
|
||||||
|
.wrapping_sub(dict_match_index)
|
||||||
|
.wrapping_sub(dict_index_delta);
|
||||||
|
while ptr_gt(ip, anchor)
|
||||||
|
&& ptr_gt(dict_match, dict_start)
|
||||||
|
&& unsafe { *ip.wrapping_sub(1) == *dict_match.wrapping_sub(1) }
|
||||||
|
{
|
||||||
|
ip = ip.wrapping_sub(1);
|
||||||
|
dict_match = dict_match.wrapping_sub(1);
|
||||||
|
match_length += 1;
|
||||||
|
}
|
||||||
|
offset2 = offset1;
|
||||||
|
offset1 = offset;
|
||||||
|
unsafe {
|
||||||
|
store_seq(
|
||||||
|
seq_store,
|
||||||
|
ip.offset_from(anchor) as usize,
|
||||||
|
anchor,
|
||||||
|
iend,
|
||||||
|
offset.wrapping_add(ZSTD_REP_NUM as u32),
|
||||||
|
match_length,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let (next_ip, next_anchor) = unsafe {
|
||||||
|
post_dict_match(
|
||||||
|
hash_long,
|
||||||
|
hash_small,
|
||||||
|
base,
|
||||||
|
dict_base,
|
||||||
|
prefix_lowest_index,
|
||||||
|
dict_index_delta,
|
||||||
|
dict_end,
|
||||||
|
prefix_lowest,
|
||||||
|
hash_log,
|
||||||
|
chain_log,
|
||||||
|
mls,
|
||||||
|
seq_store,
|
||||||
|
&mut offset1,
|
||||||
|
&mut offset2,
|
||||||
|
ip,
|
||||||
|
match_length,
|
||||||
|
current,
|
||||||
|
iend,
|
||||||
|
ilimit,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
ip = next_ip;
|
||||||
|
anchor = next_anchor;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut short_found = false;
|
||||||
|
if match_index_small > prefix_lowest_index {
|
||||||
|
short_found = unsafe { read32(matched) == read32(ip) };
|
||||||
|
} else if dict_tags_match_small {
|
||||||
|
let dict_match_index = dict_match_index_and_tag_small >> SHORT_CACHE_TAG_BITS;
|
||||||
|
let dict_match = dict_base.wrapping_add(dict_match_index as usize);
|
||||||
|
match_index_small = dict_match_index.wrapping_add(dict_index_delta);
|
||||||
|
if ptr_gt(dict_match, dict_start) && unsafe { read32(dict_match) == read32(ip) } {
|
||||||
|
matched = dict_match;
|
||||||
|
short_found = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !short_found {
|
||||||
|
ip = ip.wrapping_add(
|
||||||
|
(unsafe { ip.offset_from(anchor) as usize } >> K_SEARCH_STRENGTH) + 1,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let hash_long3 = unsafe { hash_ptr(ip.wrapping_add(1), hash_log, 8) };
|
||||||
|
let dict_hash_and_tag_long3 = unsafe { hash_ptr(ip.wrapping_add(1), dict_hbits_long, 8) };
|
||||||
|
let match_index_long3 = unsafe { table_get(hash_long, hash_long3) };
|
||||||
|
let dict_match_index_and_tag_long3 = unsafe {
|
||||||
|
table_get(
|
||||||
|
dict_hash_long,
|
||||||
|
dict_hash_and_tag_long3 >> SHORT_CACHE_TAG_BITS,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let dict_tags_match_long3 =
|
||||||
|
packed_tags_match(dict_match_index_and_tag_long3, dict_hash_and_tag_long3);
|
||||||
|
let mut match_long3 = base.wrapping_add(match_index_long3 as usize);
|
||||||
|
unsafe { table_set(hash_long, hash_long3, current.wrapping_add(1)) };
|
||||||
|
|
||||||
|
let (match_length, offset) = if match_index_long3 >= prefix_lowest_index
|
||||||
|
&& unsafe { read64(match_long3) == read64(ip.wrapping_add(1)) }
|
||||||
|
{
|
||||||
|
let mut length =
|
||||||
|
unsafe { count(ip.wrapping_add(9), match_long3.wrapping_add(8), iend) + 8 };
|
||||||
|
ip = ip.wrapping_add(1);
|
||||||
|
let offset = unsafe { index_from(match_long3, ip) };
|
||||||
|
while ptr_gt(ip, anchor)
|
||||||
|
&& ptr_gt(match_long3, prefix_lowest)
|
||||||
|
&& unsafe { *ip.wrapping_sub(1) == *match_long3.wrapping_sub(1) }
|
||||||
|
{
|
||||||
|
ip = ip.wrapping_sub(1);
|
||||||
|
match_long3 = match_long3.wrapping_sub(1);
|
||||||
|
length += 1;
|
||||||
|
}
|
||||||
|
(length, offset)
|
||||||
|
} else if dict_tags_match_long3 {
|
||||||
|
let dict_match_index = dict_match_index_and_tag_long3 >> SHORT_CACHE_TAG_BITS;
|
||||||
|
let mut dict_match = dict_base.wrapping_add(dict_match_index as usize);
|
||||||
|
if ptr_gt(dict_match, dict_start)
|
||||||
|
&& unsafe { read64(dict_match) == read64(ip.wrapping_add(1)) }
|
||||||
|
{
|
||||||
|
let mut length = unsafe {
|
||||||
|
count_2segments(
|
||||||
|
ip.wrapping_add(9),
|
||||||
|
dict_match.wrapping_add(8),
|
||||||
|
iend,
|
||||||
|
dict_end,
|
||||||
|
prefix_lowest,
|
||||||
|
) + 8
|
||||||
|
};
|
||||||
|
ip = ip.wrapping_add(1);
|
||||||
|
let offset = current
|
||||||
|
.wrapping_add(1)
|
||||||
|
.wrapping_sub(dict_match_index)
|
||||||
|
.wrapping_sub(dict_index_delta);
|
||||||
|
while ptr_gt(ip, anchor)
|
||||||
|
&& ptr_gt(dict_match, dict_start)
|
||||||
|
&& unsafe { *ip.wrapping_sub(1) == *dict_match.wrapping_sub(1) }
|
||||||
|
{
|
||||||
|
ip = ip.wrapping_sub(1);
|
||||||
|
dict_match = dict_match.wrapping_sub(1);
|
||||||
|
length += 1;
|
||||||
|
}
|
||||||
|
(length, offset)
|
||||||
|
} else if match_index_small < prefix_lowest_index {
|
||||||
|
let mut length = unsafe {
|
||||||
|
count_2segments(
|
||||||
|
ip.wrapping_add(4),
|
||||||
|
matched.wrapping_add(4),
|
||||||
|
iend,
|
||||||
|
dict_end,
|
||||||
|
prefix_lowest,
|
||||||
|
) + 4
|
||||||
|
};
|
||||||
|
let offset = current.wrapping_sub(match_index_small);
|
||||||
|
while ptr_gt(ip, anchor)
|
||||||
|
&& ptr_gt(matched, dict_start)
|
||||||
|
&& unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
|
||||||
|
{
|
||||||
|
ip = ip.wrapping_sub(1);
|
||||||
|
matched = matched.wrapping_sub(1);
|
||||||
|
length += 1;
|
||||||
|
}
|
||||||
|
(length, offset)
|
||||||
|
} else {
|
||||||
|
let mut length =
|
||||||
|
unsafe { count(ip.wrapping_add(4), matched.wrapping_add(4), iend) + 4 };
|
||||||
|
let offset = unsafe { index_from(matched, ip) };
|
||||||
|
while ptr_gt(ip, anchor)
|
||||||
|
&& ptr_gt(matched, prefix_lowest)
|
||||||
|
&& unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
|
||||||
|
{
|
||||||
|
ip = ip.wrapping_sub(1);
|
||||||
|
matched = matched.wrapping_sub(1);
|
||||||
|
length += 1;
|
||||||
|
}
|
||||||
|
(length, offset)
|
||||||
|
}
|
||||||
|
} else if match_index_small < prefix_lowest_index {
|
||||||
|
let mut length = unsafe {
|
||||||
|
count_2segments(
|
||||||
|
ip.wrapping_add(4),
|
||||||
|
matched.wrapping_add(4),
|
||||||
|
iend,
|
||||||
|
dict_end,
|
||||||
|
prefix_lowest,
|
||||||
|
) + 4
|
||||||
|
};
|
||||||
|
let offset = current.wrapping_sub(match_index_small);
|
||||||
|
while ptr_gt(ip, anchor)
|
||||||
|
&& ptr_gt(matched, dict_start)
|
||||||
|
&& unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
|
||||||
|
{
|
||||||
|
ip = ip.wrapping_sub(1);
|
||||||
|
matched = matched.wrapping_sub(1);
|
||||||
|
length += 1;
|
||||||
|
}
|
||||||
|
(length, offset)
|
||||||
|
} else {
|
||||||
|
let mut length =
|
||||||
|
unsafe { count(ip.wrapping_add(4), matched.wrapping_add(4), iend) + 4 };
|
||||||
|
let offset = unsafe { index_from(matched, ip) };
|
||||||
|
while ptr_gt(ip, anchor)
|
||||||
|
&& ptr_gt(matched, prefix_lowest)
|
||||||
|
&& unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
|
||||||
|
{
|
||||||
|
ip = ip.wrapping_sub(1);
|
||||||
|
matched = matched.wrapping_sub(1);
|
||||||
|
length += 1;
|
||||||
|
}
|
||||||
|
(length, offset)
|
||||||
|
};
|
||||||
|
offset2 = offset1;
|
||||||
|
offset1 = offset;
|
||||||
|
unsafe {
|
||||||
|
store_seq(
|
||||||
|
seq_store,
|
||||||
|
ip.offset_from(anchor) as usize,
|
||||||
|
anchor,
|
||||||
|
iend,
|
||||||
|
offset.wrapping_add(ZSTD_REP_NUM as u32),
|
||||||
|
match_length,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let (next_ip, next_anchor) = unsafe {
|
||||||
|
post_dict_match(
|
||||||
|
hash_long,
|
||||||
|
hash_small,
|
||||||
|
base,
|
||||||
|
dict_base,
|
||||||
|
prefix_lowest_index,
|
||||||
|
dict_index_delta,
|
||||||
|
dict_end,
|
||||||
|
prefix_lowest,
|
||||||
|
hash_log,
|
||||||
|
chain_log,
|
||||||
|
mls,
|
||||||
|
seq_store,
|
||||||
|
&mut offset1,
|
||||||
|
&mut offset2,
|
||||||
|
ip,
|
||||||
|
match_length,
|
||||||
|
current,
|
||||||
|
iend,
|
||||||
|
ilimit,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
ip = next_ip;
|
||||||
|
anchor = next_anchor;
|
||||||
|
continue 'outer;
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe {
|
||||||
|
*reps = offset1;
|
||||||
|
*reps.add(1) = offset2;
|
||||||
|
}
|
||||||
|
unsafe { iend.offset_from(anchor) as usize }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rust implementation called by the C ABI wrapper for attached dictionaries.
|
||||||
|
#[no_mangle]
|
||||||
|
pub unsafe extern "C" fn ZSTD_rust_compressBlock_doubleFast_dictMatchState(
|
||||||
|
hash_long: *mut u32,
|
||||||
|
hash_small: *mut u32,
|
||||||
|
base: *const u8,
|
||||||
|
prefix_lowest_index: u32,
|
||||||
|
hash_log: u32,
|
||||||
|
chain_log: u32,
|
||||||
|
min_match: u32,
|
||||||
|
seq_store: *mut c_void,
|
||||||
|
reps: *mut u32,
|
||||||
|
src: *const c_void,
|
||||||
|
src_size: usize,
|
||||||
|
dict_hash_long: *const u32,
|
||||||
|
dict_hash_small: *const u32,
|
||||||
|
dict_base: *const u8,
|
||||||
|
dict_start_index: u32,
|
||||||
|
dict_end: *const u8,
|
||||||
|
dict_hash_log: u32,
|
||||||
|
dict_chain_log: u32,
|
||||||
|
prefetch_cdict_tables: c_int,
|
||||||
|
) -> usize {
|
||||||
|
unsafe {
|
||||||
|
compress_block_double_fast_dict_match_state(
|
||||||
|
hash_long,
|
||||||
|
hash_small,
|
||||||
|
base,
|
||||||
|
prefix_lowest_index,
|
||||||
|
hash_log,
|
||||||
|
chain_log,
|
||||||
|
seq_store.cast::<SeqStore_t>(),
|
||||||
|
reps,
|
||||||
|
src.cast::<u8>(),
|
||||||
|
src_size,
|
||||||
|
fast_mls(min_match),
|
||||||
|
dict_hash_long,
|
||||||
|
dict_hash_small,
|
||||||
|
dict_base,
|
||||||
|
dict_start_index,
|
||||||
|
dict_end,
|
||||||
|
dict_hash_log,
|
||||||
|
dict_chain_log,
|
||||||
|
prefetch_cdict_tables != 0,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe fn post_ext_match(
|
||||||
|
hash_long: *mut u32,
|
||||||
|
hash_small: *mut u32,
|
||||||
|
base: *const u8,
|
||||||
|
dict_base: *const u8,
|
||||||
|
dict_start_index: u32,
|
||||||
|
prefix_start_index: u32,
|
||||||
|
dict_end: *const u8,
|
||||||
|
prefix_start: *const u8,
|
||||||
|
hbits_long: u32,
|
||||||
|
hbits_small: u32,
|
||||||
|
mls: u32,
|
||||||
|
seq_store: *mut SeqStore_t,
|
||||||
|
offset1: &mut u32,
|
||||||
|
offset2: &mut u32,
|
||||||
|
mut ip: *const u8,
|
||||||
|
match_length: usize,
|
||||||
|
current: u32,
|
||||||
|
iend: *const u8,
|
||||||
|
ilimit: *const u8,
|
||||||
|
) -> (*const u8, *const u8) {
|
||||||
|
ip = ip.wrapping_add(match_length);
|
||||||
|
let mut anchor = ip;
|
||||||
|
if ptr_le(ip, ilimit) {
|
||||||
|
let index_to_insert = current.wrapping_add(2);
|
||||||
|
let hash = unsafe { hash_ptr(base.wrapping_add(index_to_insert as usize), hbits_long, 8) };
|
||||||
|
unsafe { table_set(hash_long, hash, index_to_insert) };
|
||||||
|
let hash = unsafe { hash_ptr(ip.wrapping_sub(2), hbits_long, 8) };
|
||||||
|
unsafe { table_set(hash_long, hash, index_from(base, ip.wrapping_sub(2))) };
|
||||||
|
let hash = unsafe {
|
||||||
|
hash_ptr(
|
||||||
|
base.wrapping_add(index_to_insert as usize),
|
||||||
|
hbits_small,
|
||||||
|
mls,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
unsafe { table_set(hash_small, hash, index_to_insert) };
|
||||||
|
let hash = unsafe { hash_ptr(ip.wrapping_sub(1), hbits_small, mls) };
|
||||||
|
unsafe { table_set(hash_small, hash, index_from(base, ip.wrapping_sub(1))) };
|
||||||
|
|
||||||
|
while ptr_le(ip, ilimit) {
|
||||||
|
let current2 = unsafe { index_from(base, ip) };
|
||||||
|
let rep_index2 = current2.wrapping_sub(*offset2);
|
||||||
|
let rep_match2 = if rep_index2 < prefix_start_index {
|
||||||
|
dict_base.wrapping_add(rep_index2 as usize)
|
||||||
|
} else {
|
||||||
|
base.wrapping_add(rep_index2 as usize)
|
||||||
|
};
|
||||||
|
if index_overlap_check(prefix_start_index, rep_index2)
|
||||||
|
&& *offset2 <= current2.wrapping_sub(dict_start_index)
|
||||||
|
&& unsafe { read32(rep_match2) == read32(ip) }
|
||||||
|
{
|
||||||
|
let rep_end2 = if rep_index2 < prefix_start_index {
|
||||||
|
dict_end
|
||||||
|
} else {
|
||||||
|
iend
|
||||||
|
};
|
||||||
|
let repeat_length = unsafe {
|
||||||
|
count_2segments(
|
||||||
|
ip.wrapping_add(4),
|
||||||
|
rep_match2.wrapping_add(4),
|
||||||
|
iend,
|
||||||
|
rep_end2,
|
||||||
|
prefix_start,
|
||||||
|
) + 4
|
||||||
|
};
|
||||||
|
std::mem::swap(offset1, offset2);
|
||||||
|
unsafe {
|
||||||
|
store_seq(
|
||||||
|
seq_store,
|
||||||
|
0,
|
||||||
|
anchor,
|
||||||
|
iend,
|
||||||
|
REPCODE1_TO_OFFBASE,
|
||||||
|
repeat_length,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let hash = unsafe { hash_ptr(ip, hbits_small, mls) };
|
||||||
|
unsafe { table_set(hash_small, hash, current2) };
|
||||||
|
let hash = unsafe { hash_ptr(ip, hbits_long, 8) };
|
||||||
|
unsafe { table_set(hash_long, hash, current2) };
|
||||||
|
ip = ip.wrapping_add(repeat_length);
|
||||||
|
anchor = ip;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(ip, anchor)
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe fn compress_block_double_fast_ext_dict(
|
||||||
|
hash_long: *mut u32,
|
||||||
|
hash_small: *mut u32,
|
||||||
|
base: *const u8,
|
||||||
|
dict_base: *const u8,
|
||||||
|
dict_limit: u32,
|
||||||
|
low_limit: u32,
|
||||||
|
loaded_dict_end: u32,
|
||||||
|
hash_log: u32,
|
||||||
|
chain_log: u32,
|
||||||
|
window_log: u32,
|
||||||
|
seq_store: *mut SeqStore_t,
|
||||||
|
reps: *mut u32,
|
||||||
|
src: *const u8,
|
||||||
|
src_size: usize,
|
||||||
|
mls: u32,
|
||||||
|
) -> usize {
|
||||||
|
if src_size < HASH_READ_SIZE {
|
||||||
|
return src_size;
|
||||||
|
}
|
||||||
|
let istart = src;
|
||||||
|
let iend = istart.wrapping_add(src_size);
|
||||||
|
let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
|
||||||
|
let end_index = unsafe { index_from(base, istart) }.wrapping_add(src_size as u32);
|
||||||
|
let dict_start_index = lowest_match_index(low_limit, loaded_dict_end, end_index, window_log);
|
||||||
|
let prefix_start_index = dict_limit.max(dict_start_index);
|
||||||
|
if prefix_start_index == dict_start_index {
|
||||||
|
return unsafe {
|
||||||
|
compress_block_double_fast_no_dict(
|
||||||
|
hash_long,
|
||||||
|
hash_small,
|
||||||
|
base,
|
||||||
|
dict_limit,
|
||||||
|
loaded_dict_end,
|
||||||
|
hash_log,
|
||||||
|
chain_log,
|
||||||
|
window_log,
|
||||||
|
seq_store,
|
||||||
|
reps,
|
||||||
|
src,
|
||||||
|
src_size,
|
||||||
|
mls,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
}
|
||||||
|
let prefix_start = base.wrapping_add(prefix_start_index as usize);
|
||||||
|
let dict_start = dict_base.wrapping_add(dict_start_index as usize);
|
||||||
|
let dict_end = dict_base.wrapping_add(prefix_start_index as usize);
|
||||||
|
let mut ip = istart;
|
||||||
|
let mut anchor = istart;
|
||||||
|
let mut offset1 = unsafe { *reps };
|
||||||
|
let mut offset2 = unsafe { *reps.add(1) };
|
||||||
|
|
||||||
|
while ptr_lt(ip, ilimit) {
|
||||||
|
let h_small = unsafe { hash_ptr(ip, chain_log, mls) };
|
||||||
|
let match_index = unsafe { table_get(hash_small, h_small) };
|
||||||
|
let match_base = if match_index < prefix_start_index {
|
||||||
|
dict_base
|
||||||
|
} else {
|
||||||
|
base
|
||||||
|
};
|
||||||
|
let mut matched = match_base.wrapping_add(match_index as usize);
|
||||||
|
let h_long = unsafe { hash_ptr(ip, hash_log, 8) };
|
||||||
|
let match_long_index = unsafe { table_get(hash_long, h_long) };
|
||||||
|
let match_long_base = if match_long_index < prefix_start_index {
|
||||||
|
dict_base
|
||||||
|
} else {
|
||||||
|
base
|
||||||
|
};
|
||||||
|
let mut match_long = match_long_base.wrapping_add(match_long_index as usize);
|
||||||
|
let current = unsafe { index_from(base, ip) };
|
||||||
|
let rep_index = current.wrapping_add(1).wrapping_sub(offset1);
|
||||||
|
let rep_base = if rep_index < prefix_start_index {
|
||||||
|
dict_base
|
||||||
|
} else {
|
||||||
|
base
|
||||||
|
};
|
||||||
|
let rep_match = rep_base.wrapping_add(rep_index as usize);
|
||||||
|
unsafe {
|
||||||
|
table_set(hash_small, h_small, current);
|
||||||
|
table_set(hash_long, h_long, current);
|
||||||
|
}
|
||||||
|
|
||||||
|
if index_overlap_check(prefix_start_index, rep_index)
|
||||||
|
&& offset1 <= current.wrapping_add(1).wrapping_sub(dict_start_index)
|
||||||
|
&& unsafe { read32(rep_match) == read32(ip.wrapping_add(1)) }
|
||||||
|
{
|
||||||
|
let rep_end = if rep_index < prefix_start_index {
|
||||||
|
dict_end
|
||||||
|
} else {
|
||||||
|
iend
|
||||||
|
};
|
||||||
|
let match_length = unsafe {
|
||||||
|
count_2segments(
|
||||||
|
ip.wrapping_add(5),
|
||||||
|
rep_match.wrapping_add(4),
|
||||||
|
iend,
|
||||||
|
rep_end,
|
||||||
|
prefix_start,
|
||||||
|
) + 4
|
||||||
|
};
|
||||||
|
ip = ip.wrapping_add(1);
|
||||||
|
unsafe {
|
||||||
|
store_seq(
|
||||||
|
seq_store,
|
||||||
|
ip.offset_from(anchor) as usize,
|
||||||
|
anchor,
|
||||||
|
iend,
|
||||||
|
REPCODE1_TO_OFFBASE,
|
||||||
|
match_length,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let (next_ip, next_anchor) = unsafe {
|
||||||
|
post_ext_match(
|
||||||
|
hash_long,
|
||||||
|
hash_small,
|
||||||
|
base,
|
||||||
|
dict_base,
|
||||||
|
dict_start_index,
|
||||||
|
prefix_start_index,
|
||||||
|
dict_end,
|
||||||
|
prefix_start,
|
||||||
|
hash_log,
|
||||||
|
chain_log,
|
||||||
|
mls,
|
||||||
|
seq_store,
|
||||||
|
&mut offset1,
|
||||||
|
&mut offset2,
|
||||||
|
ip,
|
||||||
|
match_length,
|
||||||
|
current,
|
||||||
|
iend,
|
||||||
|
ilimit,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
ip = next_ip;
|
||||||
|
anchor = next_anchor;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if match_long_index > dict_start_index && unsafe { read64(match_long) == read64(ip) } {
|
||||||
|
let match_end = if match_long_index < prefix_start_index {
|
||||||
|
dict_end
|
||||||
|
} else {
|
||||||
|
iend
|
||||||
|
};
|
||||||
|
let low_match = if match_long_index < prefix_start_index {
|
||||||
|
dict_start
|
||||||
|
} else {
|
||||||
|
prefix_start
|
||||||
|
};
|
||||||
|
let mut match_length = unsafe {
|
||||||
|
count_2segments(
|
||||||
|
ip.wrapping_add(8),
|
||||||
|
match_long.wrapping_add(8),
|
||||||
|
iend,
|
||||||
|
match_end,
|
||||||
|
prefix_start,
|
||||||
|
) + 8
|
||||||
|
};
|
||||||
|
let offset = current.wrapping_sub(match_long_index);
|
||||||
|
while ptr_gt(ip, anchor)
|
||||||
|
&& ptr_gt(match_long, low_match)
|
||||||
|
&& unsafe { *ip.wrapping_sub(1) == *match_long.wrapping_sub(1) }
|
||||||
|
{
|
||||||
|
ip = ip.wrapping_sub(1);
|
||||||
|
match_long = match_long.wrapping_sub(1);
|
||||||
|
match_length += 1;
|
||||||
|
}
|
||||||
|
offset2 = offset1;
|
||||||
|
offset1 = offset;
|
||||||
|
unsafe {
|
||||||
|
store_seq(
|
||||||
|
seq_store,
|
||||||
|
ip.offset_from(anchor) as usize,
|
||||||
|
anchor,
|
||||||
|
iend,
|
||||||
|
offset.wrapping_add(ZSTD_REP_NUM as u32),
|
||||||
|
match_length,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let (next_ip, next_anchor) = unsafe {
|
||||||
|
post_ext_match(
|
||||||
|
hash_long,
|
||||||
|
hash_small,
|
||||||
|
base,
|
||||||
|
dict_base,
|
||||||
|
dict_start_index,
|
||||||
|
prefix_start_index,
|
||||||
|
dict_end,
|
||||||
|
prefix_start,
|
||||||
|
hash_log,
|
||||||
|
chain_log,
|
||||||
|
mls,
|
||||||
|
seq_store,
|
||||||
|
&mut offset1,
|
||||||
|
&mut offset2,
|
||||||
|
ip,
|
||||||
|
match_length,
|
||||||
|
current,
|
||||||
|
iend,
|
||||||
|
ilimit,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
ip = next_ip;
|
||||||
|
anchor = next_anchor;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if match_index > dict_start_index && unsafe { read32(matched) == read32(ip) } {
|
||||||
|
let h_long3 = unsafe { hash_ptr(ip.wrapping_add(1), hash_log, 8) };
|
||||||
|
let match_index3 = unsafe { table_get(hash_long, h_long3) };
|
||||||
|
let match_base3 = if match_index3 < prefix_start_index {
|
||||||
|
dict_base
|
||||||
|
} else {
|
||||||
|
base
|
||||||
|
};
|
||||||
|
let mut match3 = match_base3.wrapping_add(match_index3 as usize);
|
||||||
|
unsafe { table_set(hash_long, h_long3, current.wrapping_add(1)) };
|
||||||
|
let (match_length, offset) = if match_index3 > dict_start_index
|
||||||
|
&& unsafe { read64(match3) == read64(ip.wrapping_add(1)) }
|
||||||
|
{
|
||||||
|
let match_end = if match_index3 < prefix_start_index {
|
||||||
|
dict_end
|
||||||
|
} else {
|
||||||
|
iend
|
||||||
|
};
|
||||||
|
let low_match = if match_index3 < prefix_start_index {
|
||||||
|
dict_start
|
||||||
|
} else {
|
||||||
|
prefix_start
|
||||||
|
};
|
||||||
|
let mut length = unsafe {
|
||||||
|
count_2segments(
|
||||||
|
ip.wrapping_add(9),
|
||||||
|
match3.wrapping_add(8),
|
||||||
|
iend,
|
||||||
|
match_end,
|
||||||
|
prefix_start,
|
||||||
|
) + 8
|
||||||
|
};
|
||||||
|
ip = ip.wrapping_add(1);
|
||||||
|
let offset = current.wrapping_add(1).wrapping_sub(match_index3);
|
||||||
|
while ptr_gt(ip, anchor)
|
||||||
|
&& ptr_gt(match3, low_match)
|
||||||
|
&& unsafe { *ip.wrapping_sub(1) == *match3.wrapping_sub(1) }
|
||||||
|
{
|
||||||
|
ip = ip.wrapping_sub(1);
|
||||||
|
match3 = match3.wrapping_sub(1);
|
||||||
|
length += 1;
|
||||||
|
}
|
||||||
|
(length, offset)
|
||||||
|
} else {
|
||||||
|
let match_end = if match_index < prefix_start_index {
|
||||||
|
dict_end
|
||||||
|
} else {
|
||||||
|
iend
|
||||||
|
};
|
||||||
|
let low_match = if match_index < prefix_start_index {
|
||||||
|
dict_start
|
||||||
|
} else {
|
||||||
|
prefix_start
|
||||||
|
};
|
||||||
|
let mut length = unsafe {
|
||||||
|
count_2segments(
|
||||||
|
ip.wrapping_add(4),
|
||||||
|
matched.wrapping_add(4),
|
||||||
|
iend,
|
||||||
|
match_end,
|
||||||
|
prefix_start,
|
||||||
|
) + 4
|
||||||
|
};
|
||||||
|
let offset = current.wrapping_sub(match_index);
|
||||||
|
while ptr_gt(ip, anchor)
|
||||||
|
&& ptr_gt(matched, low_match)
|
||||||
|
&& unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) }
|
||||||
|
{
|
||||||
|
ip = ip.wrapping_sub(1);
|
||||||
|
matched = matched.wrapping_sub(1);
|
||||||
|
length += 1;
|
||||||
|
}
|
||||||
|
(length, offset)
|
||||||
|
};
|
||||||
|
offset2 = offset1;
|
||||||
|
offset1 = offset;
|
||||||
|
unsafe {
|
||||||
|
store_seq(
|
||||||
|
seq_store,
|
||||||
|
ip.offset_from(anchor) as usize,
|
||||||
|
anchor,
|
||||||
|
iend,
|
||||||
|
offset.wrapping_add(ZSTD_REP_NUM as u32),
|
||||||
|
match_length,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
let (next_ip, next_anchor) = unsafe {
|
||||||
|
post_ext_match(
|
||||||
|
hash_long,
|
||||||
|
hash_small,
|
||||||
|
base,
|
||||||
|
dict_base,
|
||||||
|
dict_start_index,
|
||||||
|
prefix_start_index,
|
||||||
|
dict_end,
|
||||||
|
prefix_start,
|
||||||
|
hash_log,
|
||||||
|
chain_log,
|
||||||
|
mls,
|
||||||
|
seq_store,
|
||||||
|
&mut offset1,
|
||||||
|
&mut offset2,
|
||||||
|
ip,
|
||||||
|
match_length,
|
||||||
|
current,
|
||||||
|
iend,
|
||||||
|
ilimit,
|
||||||
|
)
|
||||||
|
};
|
||||||
|
ip = next_ip;
|
||||||
|
anchor = next_anchor;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
ip = ip.wrapping_add((unsafe { ip.offset_from(anchor) as usize } >> K_SEARCH_STRENGTH) + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
unsafe {
|
||||||
|
*reps = offset1;
|
||||||
|
*reps.add(1) = offset2;
|
||||||
|
}
|
||||||
|
unsafe { iend.offset_from(anchor) as usize }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rust implementation called by the C ABI wrapper for external dictionaries.
|
||||||
|
#[no_mangle]
|
||||||
|
pub unsafe extern "C" fn ZSTD_rust_compressBlock_doubleFast_extDict(
|
||||||
|
hash_long: *mut u32,
|
||||||
|
hash_small: *mut u32,
|
||||||
|
base: *const u8,
|
||||||
|
dict_base: *const u8,
|
||||||
|
dict_limit: u32,
|
||||||
|
low_limit: u32,
|
||||||
|
loaded_dict_end: u32,
|
||||||
|
hash_log: u32,
|
||||||
|
chain_log: u32,
|
||||||
|
min_match: u32,
|
||||||
|
window_log: u32,
|
||||||
|
seq_store: *mut c_void,
|
||||||
|
reps: *mut u32,
|
||||||
|
src: *const c_void,
|
||||||
|
src_size: usize,
|
||||||
|
) -> usize {
|
||||||
|
unsafe {
|
||||||
|
compress_block_double_fast_ext_dict(
|
||||||
|
hash_long,
|
||||||
|
hash_small,
|
||||||
|
base,
|
||||||
|
dict_base,
|
||||||
|
dict_limit,
|
||||||
|
low_limit,
|
||||||
|
loaded_dict_end,
|
||||||
|
hash_log,
|
||||||
|
chain_log,
|
||||||
|
window_log,
|
||||||
|
seq_store.cast::<SeqStore_t>(),
|
||||||
|
reps,
|
||||||
|
src.cast::<u8>(),
|
||||||
|
src_size,
|
||||||
|
fast_mls(min_match),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user