diff --git a/lib/compress/zstd_double_fast.c b/lib/compress/zstd_double_fast.c index 1a266e7d9..da4bd6aa8 100644 --- a/lib/compress/zstd_double_fast.c +++ b/lib/compress/zstd_double_fast.c @@ -8,771 +8,93 @@ * You may select, at your option, one of the above-listed licenses. */ +/* The double-fast algorithms live in rust/src/zstd_double_fast.rs. Keep the + * private match-state layout in C and pass only the matching leaf fields. */ #include "zstd_compress_internal.h" #include "zstd_double_fast.h" #ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR -static -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -void ZSTD_fillDoubleHashTableForCDict(ZSTD_MatchState_t* ms, - void const* end, ZSTD_dictTableLoadMethod_e dtlm) -{ - const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32* const hashLarge = ms->hashTable; - U32 const hBitsL = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; - U32 const mls = cParams->minMatch; - U32* const hashSmall = ms->chainTable; - U32 const hBitsS = cParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS; - const BYTE* const base = ms->window.base; - const BYTE* ip = base + ms->nextToUpdate; - const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; - const U32 fastHashFillStep = 3; +typedef char ZSTD_rust_double_fast_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1]; +typedef char ZSTD_rust_double_fast_seqstore_long_length_pos[ + (offsetof(SeqStore_t, longLengthPos) == 9 * sizeof(size_t) + 4) ? 1 : -1]; +typedef char ZSTD_rust_double_fast_seqstore_layout[ + (sizeof(SeqStore_t) == 9 * sizeof(size_t) + 8) ? 1 : -1]; +typedef char ZSTD_rust_double_fast_rep_count[(ZSTD_REP_NUM == 3) ? 1 : -1]; - /* Always insert every fastHashFillStep position into the hash tables. - * Insert the other positions into the large hash table if their entry - * is empty. - */ - for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) { - U32 const curr = (U32)(ip - base); - U32 i; - for (i = 0; i < fastHashFillStep; ++i) { - size_t const smHashAndTag = ZSTD_hashPtr(ip + i, hBitsS, mls); - size_t const lgHashAndTag = ZSTD_hashPtr(ip + i, hBitsL, 8); - if (i == 0) { - ZSTD_writeTaggedIndex(hashSmall, smHashAndTag, curr + i); - } - if (i == 0 || hashLarge[lgHashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { - ZSTD_writeTaggedIndex(hashLarge, lgHashAndTag, curr + i); - } - /* Only load extra positions for ZSTD_dtlm_full */ - if (dtlm == ZSTD_dtlm_fast) - break; - } } -} +void ZSTD_rust_fillDoubleHashTable( + U32* hashLong, U32* hashSmall, const BYTE* base, U32 nextToUpdate, + const void* end, U32 hashLog, U32 chainLog, U32 minMatch, + int fullTableLoad, int forCDict); -static -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -void ZSTD_fillDoubleHashTableForCCtx(ZSTD_MatchState_t* ms, - void const* end, ZSTD_dictTableLoadMethod_e dtlm) -{ - const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32* const hashLarge = ms->hashTable; - U32 const hBitsL = cParams->hashLog; - U32 const mls = cParams->minMatch; - U32* const hashSmall = ms->chainTable; - U32 const hBitsS = cParams->chainLog; - const BYTE* const base = ms->window.base; - const BYTE* ip = base + ms->nextToUpdate; - const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; - const U32 fastHashFillStep = 3; +size_t ZSTD_rust_compressBlock_doubleFast( + U32* hashLong, U32* hashSmall, const BYTE* base, + U32 dictLimit, U32 loadedDictEnd, + U32 hashLog, U32 chainLog, U32 minMatch, U32 windowLog, + void* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize); - /* Always insert every fastHashFillStep position into the hash tables. - * Insert the other positions into the large hash table if their entry - * is empty. - */ - for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) { - U32 const curr = (U32)(ip - base); - U32 i; - for (i = 0; i < fastHashFillStep; ++i) { - size_t const smHash = ZSTD_hashPtr(ip + i, hBitsS, mls); - size_t const lgHash = ZSTD_hashPtr(ip + i, hBitsL, 8); - if (i == 0) - hashSmall[smHash] = curr + i; - if (i == 0 || hashLarge[lgHash] == 0) - hashLarge[lgHash] = curr + i; - /* Only load extra positions for ZSTD_dtlm_full */ - if (dtlm == ZSTD_dtlm_fast) - break; - } } -} +size_t ZSTD_rust_compressBlock_doubleFast_dictMatchState( + U32* hashLong, U32* hashSmall, const BYTE* base, U32 prefixLowestIndex, + U32 hashLog, U32 chainLog, U32 minMatch, + void* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize, + const U32* dictHashLong, const U32* dictHashSmall, const BYTE* dictBase, + U32 dictStartIndex, const BYTE* dictEnd, + U32 dictHashLog, U32 dictChainLog, int prefetchCDictTables); + +size_t ZSTD_rust_compressBlock_doubleFast_extDict( + U32* hashLong, U32* hashSmall, const BYTE* base, const BYTE* dictBase, + U32 dictLimit, U32 lowLimit, U32 loadedDictEnd, + U32 hashLog, U32 chainLog, U32 minMatch, U32 windowLog, + void* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize); void ZSTD_fillDoubleHashTable(ZSTD_MatchState_t* ms, - const void* const end, - ZSTD_dictTableLoadMethod_e dtlm, - ZSTD_tableFillPurpose_e tfp) + const void* end, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) { - if (tfp == ZSTD_tfp_forCDict) { - ZSTD_fillDoubleHashTableForCDict(ms, end, dtlm); - } else { - ZSTD_fillDoubleHashTableForCCtx(ms, end, dtlm); - } + ZSTD_rust_fillDoubleHashTable( + ms->hashTable, ms->chainTable, ms->window.base, ms->nextToUpdate, + end, ms->cParams.hashLog, ms->cParams.chainLog, ms->cParams.minMatch, + dtlm == ZSTD_dtlm_full, tfp == ZSTD_tfp_forCDict); } - -FORCE_INLINE_TEMPLATE -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -size_t ZSTD_compressBlock_doubleFast_noDict_generic( - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize, U32 const mls /* template */) -{ - ZSTD_compressionParameters const* cParams = &ms->cParams; - U32* const hashLong = ms->hashTable; - const U32 hBitsL = cParams->hashLog; - U32* const hashSmall = ms->chainTable; - const U32 hBitsS = cParams->chainLog; - const BYTE* const base = ms->window.base; - const BYTE* const istart = (const BYTE*)src; - const BYTE* anchor = istart; - const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); - /* presumes that, if there is a dictionary, it must be using Attach mode */ - const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); - const BYTE* const prefixLowest = base + prefixLowestIndex; - const BYTE* const iend = istart + srcSize; - const BYTE* const ilimit = iend - HASH_READ_SIZE; - U32 offset_1=rep[0], offset_2=rep[1]; - U32 offsetSaved1 = 0, offsetSaved2 = 0; - - size_t mLength; - U32 offset; - U32 curr; - - /* how many positions to search before increasing step size */ - const size_t kStepIncr = 1 << kSearchStrength; - /* the position at which to increment the step size if no match is found */ - const BYTE* nextStep; - size_t step; /* the current step size */ - - size_t hl0; /* the long hash at ip */ - size_t hl1; /* the long hash at ip1 */ - - U32 idxl0; /* the long match index for ip */ - U32 idxl1; /* the long match index for ip1 */ - - const BYTE* matchl0; /* the long match for ip */ - const BYTE* matchs0; /* the short match for ip */ - const BYTE* matchl1; /* the long match for ip1 */ - const BYTE* matchs0_safe; /* matchs0 or safe address */ - - const BYTE* ip = istart; /* the current position */ - const BYTE* ip1; /* the next position */ - /* Array of ~random data, should have low probability of matching data - * we load from here instead of from tables, if matchl0/matchl1 are - * invalid indices. Used to avoid unpredictable branches. */ - const BYTE dummy[] = {0x12,0x34,0x56,0x78,0x9a,0xbc,0xde,0xf0,0xe2,0xb4}; - - DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_noDict_generic"); - - /* init */ - ip += ((ip - prefixLowest) == 0); - { - U32 const current = (U32)(ip - base); - U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog); - U32 const maxRep = current - windowLow; - if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0; - if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0; - } - - /* Outer Loop: one iteration per match found and stored */ - while (1) { - step = 1; - nextStep = ip + kStepIncr; - ip1 = ip + step; - - if (ip1 > ilimit) { - goto _cleanup; - } - - hl0 = ZSTD_hashPtr(ip, hBitsL, 8); - idxl0 = hashLong[hl0]; - matchl0 = base + idxl0; - - /* Inner Loop: one iteration per search / position */ - do { - const size_t hs0 = ZSTD_hashPtr(ip, hBitsS, mls); - const U32 idxs0 = hashSmall[hs0]; - curr = (U32)(ip-base); - matchs0 = base + idxs0; - - hashLong[hl0] = hashSmall[hs0] = curr; /* update hash tables */ - - /* check noDict repcode */ - if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { - mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; - ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); - goto _match_stored; - } - - hl1 = ZSTD_hashPtr(ip1, hBitsL, 8); - - /* idxl0 > prefixLowestIndex is a (somewhat) unpredictable branch. - * However expression below complies into conditional move. Since - * match is unlikely and we only *branch* on idxl0 > prefixLowestIndex - * if there is a match, all branches become predictable. */ - { const BYTE* const matchl0_safe = ZSTD_selectAddr(idxl0, prefixLowestIndex, matchl0, &dummy[0]); - - /* check prefix long match */ - if (MEM_read64(matchl0_safe) == MEM_read64(ip) && matchl0_safe == matchl0) { - mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8; - offset = (U32)(ip-matchl0); - while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */ - goto _match_found; - } } - - idxl1 = hashLong[hl1]; - matchl1 = base + idxl1; - - /* Same optimization as matchl0 above */ - matchs0_safe = ZSTD_selectAddr(idxs0, prefixLowestIndex, matchs0, &dummy[0]); - - /* check prefix short match */ - if(MEM_read32(matchs0_safe) == MEM_read32(ip) && matchs0_safe == matchs0) { - goto _search_next_long; - } - - if (ip1 >= nextStep) { - PREFETCH_L1(ip1 + 64); - PREFETCH_L1(ip1 + 128); - step++; - nextStep += kStepIncr; - } - ip = ip1; - ip1 += step; - - hl0 = hl1; - idxl0 = idxl1; - matchl0 = matchl1; - #if defined(__aarch64__) - PREFETCH_L1(ip+256); - #endif - } while (ip1 <= ilimit); - -_cleanup: - /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), - * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ - offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; - - /* save reps for next block */ - rep[0] = offset_1 ? offset_1 : offsetSaved1; - rep[1] = offset_2 ? offset_2 : offsetSaved2; - - /* Return the last literals size */ - return (size_t)(iend - anchor); - -_search_next_long: - - /* short match found: let's check for a longer one */ - mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4; - offset = (U32)(ip - matchs0); - - /* check long match at +1 position */ - if ((idxl1 > prefixLowestIndex) && (MEM_read64(matchl1) == MEM_read64(ip1))) { - size_t const l1len = ZSTD_count(ip1+8, matchl1+8, iend) + 8; - if (l1len > mLength) { - /* use the long match instead */ - ip = ip1; - mLength = l1len; - offset = (U32)(ip-matchl1); - matchs0 = matchl1; - } - } - - while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* complete backward */ - - /* fall-through */ - -_match_found: /* requires ip, offset, mLength */ - offset_2 = offset_1; - offset_1 = offset; - - if (step < 4) { - /* It is unsafe to write this value back to the hashtable when ip1 is - * greater than or equal to the new ip we will have after we're done - * processing this match. Rather than perform that test directly - * (ip1 >= ip + mLength), which costs speed in practice, we do a simpler - * more predictable test. The minmatch even if we take a short match is - * 4 bytes, so as long as step, the distance between ip and ip1 - * (initially) is less than 4, we know ip1 < new ip. */ - hashLong[hl1] = (U32)(ip1 - base); - } - - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); - -_match_stored: - /* match found */ - ip += mLength; - anchor = ip; - - if (ip <= ilimit) { - /* Complementary insertion */ - /* done after iLimit test, as candidates could be > iend-8 */ - { U32 const indexToInsert = curr+2; - hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; - hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); - hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; - hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); - } - - /* check immediate repcode */ - while ( (ip <= ilimit) - && ( (offset_2>0) - & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { - /* store sequence */ - size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; - U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ - hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); - hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); - ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, rLength); - ip += rLength; - anchor = ip; - continue; /* faster when present ... (?) */ - } - } - } -} - - -FORCE_INLINE_TEMPLATE -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize, - U32 const mls /* template */) -{ - ZSTD_compressionParameters const* cParams = &ms->cParams; - U32* const hashLong = ms->hashTable; - const U32 hBitsL = cParams->hashLog; - U32* const hashSmall = ms->chainTable; - const U32 hBitsS = cParams->chainLog; - const BYTE* const base = ms->window.base; - const BYTE* const istart = (const BYTE*)src; - const BYTE* ip = istart; - const BYTE* anchor = istart; - const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); - /* presumes that, if there is a dictionary, it must be using Attach mode */ - const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); - const BYTE* const prefixLowest = base + prefixLowestIndex; - const BYTE* const iend = istart + srcSize; - const BYTE* const ilimit = iend - HASH_READ_SIZE; - U32 offset_1=rep[0], offset_2=rep[1]; - - const ZSTD_MatchState_t* const dms = ms->dictMatchState; - const ZSTD_compressionParameters* const dictCParams = &dms->cParams; - const U32* const dictHashLong = dms->hashTable; - const U32* const dictHashSmall = dms->chainTable; - const U32 dictStartIndex = dms->window.dictLimit; - const BYTE* const dictBase = dms->window.base; - const BYTE* const dictStart = dictBase + dictStartIndex; - const BYTE* const dictEnd = dms->window.nextSrc; - const U32 dictIndexDelta = prefixLowestIndex - (U32)(dictEnd - dictBase); - const U32 dictHBitsL = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; - const U32 dictHBitsS = dictCParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS; - const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart)); - - DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_dictMatchState_generic"); - - /* if a dictionary is attached, it must be within window range */ - assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex); - - if (ms->prefetchCDictTables) { - size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32); - size_t const chainTableBytes = (((size_t)1) << dictCParams->chainLog) * sizeof(U32); - PREFETCH_AREA(dictHashLong, hashTableBytes); - PREFETCH_AREA(dictHashSmall, chainTableBytes); - } - - /* init */ - ip += (dictAndPrefixLength == 0); - - /* dictMatchState repCode checks don't currently handle repCode == 0 - * disabling. */ - assert(offset_1 <= dictAndPrefixLength); - assert(offset_2 <= dictAndPrefixLength); - - /* Main Search Loop */ - while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ - size_t mLength; - U32 offset; - size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); - size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); - size_t const dictHashAndTagL = ZSTD_hashPtr(ip, dictHBitsL, 8); - size_t const dictHashAndTagS = ZSTD_hashPtr(ip, dictHBitsS, mls); - U32 const dictMatchIndexAndTagL = dictHashLong[dictHashAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS]; - U32 const dictMatchIndexAndTagS = dictHashSmall[dictHashAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS]; - int const dictTagsMatchL = ZSTD_comparePackedTags(dictMatchIndexAndTagL, dictHashAndTagL); - int const dictTagsMatchS = ZSTD_comparePackedTags(dictMatchIndexAndTagS, dictHashAndTagS); - U32 const curr = (U32)(ip-base); - U32 const matchIndexL = hashLong[h2]; - U32 matchIndexS = hashSmall[h]; - const BYTE* matchLong = base + matchIndexL; - const BYTE* match = base + matchIndexS; - const U32 repIndex = curr + 1 - offset_1; - const BYTE* repMatch = (repIndex < prefixLowestIndex) ? - dictBase + (repIndex - dictIndexDelta) : - base + repIndex; - hashLong[h2] = hashSmall[h] = curr; /* update hash tables */ - - /* check repcode */ - if ((ZSTD_index_overlap_check(prefixLowestIndex, repIndex)) - && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { - const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; - ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); - goto _match_stored; - } - - if ((matchIndexL >= prefixLowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { - /* check prefix long match */ - mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; - offset = (U32)(ip-matchLong); - while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ - goto _match_found; - } else if (dictTagsMatchL) { - /* check dictMatchState long match */ - U32 const dictMatchIndexL = dictMatchIndexAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS; - const BYTE* dictMatchL = dictBase + dictMatchIndexL; - assert(dictMatchL < dictEnd); - - if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) { - mLength = ZSTD_count_2segments(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8; - offset = (U32)(curr - dictMatchIndexL - dictIndexDelta); - while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */ - goto _match_found; - } } - - if (matchIndexS > prefixLowestIndex) { - /* short match candidate */ - if (MEM_read32(match) == MEM_read32(ip)) { - goto _search_next_long; - } - } else if (dictTagsMatchS) { - /* check dictMatchState short match */ - U32 const dictMatchIndexS = dictMatchIndexAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS; - match = dictBase + dictMatchIndexS; - matchIndexS = dictMatchIndexS + dictIndexDelta; - - if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) { - goto _search_next_long; - } } - - ip += ((ip-anchor) >> kSearchStrength) + 1; -#if defined(__aarch64__) - PREFETCH_L1(ip+256); -#endif - continue; - -_search_next_long: - { size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); - size_t const dictHashAndTagL3 = ZSTD_hashPtr(ip+1, dictHBitsL, 8); - U32 const matchIndexL3 = hashLong[hl3]; - U32 const dictMatchIndexAndTagL3 = dictHashLong[dictHashAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS]; - int const dictTagsMatchL3 = ZSTD_comparePackedTags(dictMatchIndexAndTagL3, dictHashAndTagL3); - const BYTE* matchL3 = base + matchIndexL3; - hashLong[hl3] = curr + 1; - - /* check prefix long +1 match */ - if ((matchIndexL3 >= prefixLowestIndex) && (MEM_read64(matchL3) == MEM_read64(ip+1))) { - mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8; - ip++; - offset = (U32)(ip-matchL3); - while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */ - goto _match_found; - } else if (dictTagsMatchL3) { - /* check dict long +1 match */ - U32 const dictMatchIndexL3 = dictMatchIndexAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS; - const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3; - assert(dictMatchL3 < dictEnd); - if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) { - mLength = ZSTD_count_2segments(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8; - ip++; - offset = (U32)(curr + 1 - dictMatchIndexL3 - dictIndexDelta); - while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */ - goto _match_found; - } } } - - /* if no long +1 match, explore the short match we found */ - if (matchIndexS < prefixLowestIndex) { - mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4; - offset = (U32)(curr - matchIndexS); - while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ - } else { - mLength = ZSTD_count(ip+4, match+4, iend) + 4; - offset = (U32)(ip - match); - while (((ip>anchor) & (match>prefixLowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ - } - -_match_found: - offset_2 = offset_1; - offset_1 = offset; - - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); - -_match_stored: - /* match found */ - ip += mLength; - anchor = ip; - - if (ip <= ilimit) { - /* Complementary insertion */ - /* done after iLimit test, as candidates could be > iend-8 */ - { U32 const indexToInsert = curr+2; - hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; - hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); - hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; - hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); - } - - /* check immediate repcode */ - while (ip <= ilimit) { - U32 const current2 = (U32)(ip-base); - U32 const repIndex2 = current2 - offset_2; - const BYTE* repMatch2 = repIndex2 < prefixLowestIndex ? - dictBase + repIndex2 - dictIndexDelta : - base + repIndex2; - if ( (ZSTD_index_overlap_check(prefixLowestIndex, repIndex2)) - && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { - const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; - U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); - hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; - hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; - ip += repLength2; - anchor = ip; - continue; - } - break; - } - } - } /* while (ip < ilimit) */ - - /* save reps for next block */ - rep[0] = offset_1; - rep[1] = offset_2; - - /* Return the last literals size */ - return (size_t)(iend - anchor); -} - -#define ZSTD_GEN_DFAST_FN(dictMode, mls) \ - static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \ - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ - void const* src, size_t srcSize) \ - { \ - return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \ - } - -ZSTD_GEN_DFAST_FN(noDict, 4) -ZSTD_GEN_DFAST_FN(noDict, 5) -ZSTD_GEN_DFAST_FN(noDict, 6) -ZSTD_GEN_DFAST_FN(noDict, 7) - -ZSTD_GEN_DFAST_FN(dictMatchState, 4) -ZSTD_GEN_DFAST_FN(dictMatchState, 5) -ZSTD_GEN_DFAST_FN(dictMatchState, 6) -ZSTD_GEN_DFAST_FN(dictMatchState, 7) - - size_t ZSTD_compressBlock_doubleFast( ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) + const void* src, size_t srcSize) { - const U32 mls = ms->cParams.minMatch; - switch(mls) - { - default: /* includes case 3 */ - case 4 : - return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize); - case 5 : - return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize); - case 6 : - return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize); - case 7 : - return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize); - } + return ZSTD_rust_compressBlock_doubleFast( + ms->hashTable, ms->chainTable, ms->window.base, + ms->window.dictLimit, ms->loadedDictEnd, + ms->cParams.hashLog, ms->cParams.chainLog, ms->cParams.minMatch, + ms->cParams.windowLog, + seqStore, rep, src, srcSize); } - size_t ZSTD_compressBlock_doubleFast_dictMatchState( ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) + const void* src, size_t srcSize) { - const U32 mls = ms->cParams.minMatch; - switch(mls) - { - default: /* includes case 3 */ - case 4 : - return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize); - case 5 : - return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize); - case 6 : - return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize); - case 7 : - return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize); - } + const ZSTD_MatchState_t* const dms = ms->dictMatchState; + assert(dms != NULL); + return ZSTD_rust_compressBlock_doubleFast_dictMatchState( + ms->hashTable, ms->chainTable, ms->window.base, ms->window.dictLimit, + ms->cParams.hashLog, ms->cParams.chainLog, ms->cParams.minMatch, + seqStore, rep, src, srcSize, + dms->hashTable, dms->chainTable, dms->window.base, + dms->window.dictLimit, dms->window.nextSrc, + dms->cParams.hashLog, dms->cParams.chainLog, ms->prefetchCDictTables); } - -static -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -size_t ZSTD_compressBlock_doubleFast_extDict_generic( - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize, - U32 const mls /* template */) -{ - ZSTD_compressionParameters const* cParams = &ms->cParams; - U32* const hashLong = ms->hashTable; - U32 const hBitsL = cParams->hashLog; - U32* const hashSmall = ms->chainTable; - U32 const hBitsS = cParams->chainLog; - const BYTE* const istart = (const BYTE*)src; - const BYTE* ip = istart; - const BYTE* anchor = istart; - const BYTE* const iend = istart + srcSize; - const BYTE* const ilimit = iend - 8; - const BYTE* const base = ms->window.base; - const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); - const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); - const U32 dictStartIndex = lowLimit; - const U32 dictLimit = ms->window.dictLimit; - const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit; - const BYTE* const prefixStart = base + prefixStartIndex; - const BYTE* const dictBase = ms->window.dictBase; - const BYTE* const dictStart = dictBase + dictStartIndex; - const BYTE* const dictEnd = dictBase + prefixStartIndex; - U32 offset_1=rep[0], offset_2=rep[1]; - - DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize); - - /* if extDict is invalidated due to maxDistance, switch to "regular" variant */ - if (prefixStartIndex == dictStartIndex) - return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize); - - /* Search Loop */ - while (ip < ilimit) { /* < instead of <=, because (ip+1) */ - const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); - const U32 matchIndex = hashSmall[hSmall]; - const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base; - const BYTE* match = matchBase + matchIndex; - - const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8); - const U32 matchLongIndex = hashLong[hLong]; - const BYTE* const matchLongBase = matchLongIndex < prefixStartIndex ? dictBase : base; - const BYTE* matchLong = matchLongBase + matchLongIndex; - - const U32 curr = (U32)(ip-base); - const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */ - const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; - const BYTE* const repMatch = repBase + repIndex; - size_t mLength; - hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */ - - if (((ZSTD_index_overlap_check(prefixStartIndex, repIndex)) - & (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */ - && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { - const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; - ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); - } else { - if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { - const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; - const BYTE* const lowMatchPtr = matchLongIndex < prefixStartIndex ? dictStart : prefixStart; - U32 offset; - mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, prefixStart) + 8; - offset = curr - matchLongIndex; - while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); - - } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { - size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); - U32 const matchIndex3 = hashLong[h3]; - const BYTE* const match3Base = matchIndex3 < prefixStartIndex ? dictBase : base; - const BYTE* match3 = match3Base + matchIndex3; - U32 offset; - hashLong[h3] = curr + 1; - if ( (matchIndex3 > dictStartIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { - const BYTE* const matchEnd = matchIndex3 < prefixStartIndex ? dictEnd : iend; - const BYTE* const lowMatchPtr = matchIndex3 < prefixStartIndex ? dictStart : prefixStart; - mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, prefixStart) + 8; - ip++; - offset = curr+1 - matchIndex3; - while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ - } else { - const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; - const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; - mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; - offset = curr - matchIndex; - while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ - } - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); - - } else { - ip += ((ip-anchor) >> kSearchStrength) + 1; - continue; - } } - - /* move to next sequence start */ - ip += mLength; - anchor = ip; - - if (ip <= ilimit) { - /* Complementary insertion */ - /* done after iLimit test, as candidates could be > iend-8 */ - { U32 const indexToInsert = curr+2; - hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; - hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); - hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; - hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); - } - - /* check immediate repcode */ - while (ip <= ilimit) { - U32 const current2 = (U32)(ip-base); - U32 const repIndex2 = current2 - offset_2; - const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; - if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) - & (offset_2 <= current2 - dictStartIndex)) - && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { - const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; - U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); - hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; - hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; - ip += repLength2; - anchor = ip; - continue; - } - break; - } } } - - /* save reps for next block */ - rep[0] = offset_1; - rep[1] = offset_2; - - /* Return the last literals size */ - return (size_t)(iend - anchor); -} - -ZSTD_GEN_DFAST_FN(extDict, 4) -ZSTD_GEN_DFAST_FN(extDict, 5) -ZSTD_GEN_DFAST_FN(extDict, 6) -ZSTD_GEN_DFAST_FN(extDict, 7) - size_t ZSTD_compressBlock_doubleFast_extDict( ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) + const void* src, size_t srcSize) { - U32 const mls = ms->cParams.minMatch; - switch(mls) - { - default: /* includes case 3 */ - case 4 : - return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize); - case 5 : - return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize); - case 6 : - return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize); - case 7 : - return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize); - } + return ZSTD_rust_compressBlock_doubleFast_extDict( + ms->hashTable, ms->chainTable, ms->window.base, ms->window.dictBase, + ms->window.dictLimit, ms->window.lowLimit, ms->loadedDictEnd, + ms->cParams.hashLog, ms->cParams.chainLog, ms->cParams.minMatch, + ms->cParams.windowLog, + seqStore, rep, src, srcSize); } #endif /* ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR */ diff --git a/rust/README.md b/rust/README.md index d9ced35db..d0fe191e9 100644 --- a/rust/README.md +++ b/rust/README.md @@ -31,13 +31,15 @@ zstd ABI: - `zstd_presplit` chooses split points for full compression blocks. - `zstd_compress_literals` emits raw, RLE, and Huffman literal sections while preserving the compressor's Huffman-table repeat state. + - `zstd_fast` and `zstd_double_fast` implement the single- and two-table + fast block match finders, including attached and external dictionary paths. - Runtime support - `threading` provides platform pthread wrappers required by zstd headers. - `pool` implements the bounded worker pool used by multithreaded compression. - Dictionary support - `zstd_ddict` owns, loads, copies, and references decode dictionaries. -The remaining block compression, general decompression, dictionary-building, +The lazy and optimal block matchers, general decompression, dictionary-building, legacy, and CLI translation units are still C. They must move before the rewrite is complete. Keeping that boundary explicit prevents a passing hybrid build from being mistaken for the final all-Rust result. diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 54d299449..aa2954073 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -32,6 +32,8 @@ pub mod zstd_compress_superblock; #[cfg(feature = "decompression")] pub mod zstd_ddict; #[cfg(feature = "compression")] +pub mod zstd_double_fast; +#[cfg(feature = "compression")] pub mod zstd_fast; #[cfg(feature = "compression")] pub mod zstd_presplit; diff --git a/rust/src/zstd_double_fast.rs b/rust/src/zstd_double_fast.rs new file mode 100644 index 000000000..04f307230 --- /dev/null +++ b/rust/src/zstd_double_fast.rs @@ -0,0 +1,1819 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(clippy::missing_safety_doc)] +#![allow(clippy::too_many_arguments)] + +//! Double-fast block match finder. +//! +//! The C shim keeps `ZSTD_MatchState_t` opaque and supplies only the leaf +//! fields used by this two-table matcher. The long (8-byte) and short +//! (min-match) search loops, table fills, and sequence writes live here. + +use crate::mem::{ + MEM_64bits, MEM_isLittleEndian, MEM_read16, MEM_read32, MEM_read64, MEM_readLE32, MEM_readLE64, + MEM_readST, +}; +use std::ffi::c_void; +use std::mem::size_of; +use std::os::raw::c_int; +use std::ptr; + +const ZSTD_REP_NUM: usize = 3; +const MINMATCH: usize = 3; +const HASH_READ_SIZE: usize = 8; +const SHORT_CACHE_TAG_BITS: u32 = 8; +const SHORT_CACHE_TAG_MASK: u32 = (1 << SHORT_CACHE_TAG_BITS) - 1; +const K_SEARCH_STRENGTH: usize = 8; +const REPCODE1_TO_OFFBASE: u32 = 1; + +#[repr(C)] +#[derive(Clone, Copy)] +struct SeqDef { + offBase: u32, + litLength: u16, + mlBase: u16, +} + +/// The C shim checks this leaf ABI. `ZSTD_MatchState_t` remains wholly in C. +#[repr(C)] +struct SeqStore_t { + sequencesStart: *mut SeqDef, + sequences: *mut SeqDef, + litStart: *mut u8, + lit: *mut u8, + llCode: *mut u8, + mlCode: *mut u8, + ofCode: *mut u8, + maxNbSeq: usize, + maxNbLit: usize, + longLengthType: c_int, + longLengthPos: u32, +} + +#[inline] +fn ptr_lt(left: *const u8, right: *const u8) -> bool { + (left as usize) < (right as usize) +} + +#[inline] +fn ptr_le(left: *const u8, right: *const u8) -> bool { + (left as usize) <= (right as usize) +} + +#[inline] +fn ptr_gt(left: *const u8, right: *const u8) -> bool { + (left as usize) > (right as usize) +} + +#[inline] +fn ptr_ge(left: *const u8, right: *const u8) -> bool { + (left as usize) >= (right as usize) +} + +#[inline] +unsafe fn index_from(base: *const u8, ptr: *const u8) -> u32 { + unsafe { ptr.offset_from(base) as u32 } +} + +#[inline] +unsafe fn read32(ptr: *const u8) -> u32 { + unsafe { MEM_read32(ptr.cast::()) } +} + +#[inline] +unsafe fn read64(ptr: *const u8) -> u64 { + unsafe { MEM_read64(ptr.cast::()) } +} + +#[inline] +unsafe fn table_get(table: *const u32, index: usize) -> u32 { + unsafe { *table.add(index) } +} + +#[inline] +unsafe fn table_set(table: *mut u32, index: usize, value: u32) { + unsafe { *table.add(index) = value }; +} + +#[inline] +fn hash_shift32(value: u32, hbits: u32) -> usize { + if hbits == 0 { + 0 + } else { + (value >> (32 - hbits)) as usize + } +} + +#[inline] +fn hash_shift64(value: u64, hbits: u32) -> usize { + if hbits == 0 { + 0 + } else { + (value >> (64 - hbits)) as usize + } +} + +#[inline] +unsafe fn hash_ptr(ptr: *const u8, hbits: u32, mls: u32) -> usize { + match mls { + 5 => { + let value = unsafe { MEM_readLE64(ptr.cast::()) }; + hash_shift64(value.wrapping_shl(24).wrapping_mul(889_523_592_379), hbits) + } + 6 => { + let value = unsafe { MEM_readLE64(ptr.cast::()) }; + hash_shift64( + value.wrapping_shl(16).wrapping_mul(227_718_039_650_203), + hbits, + ) + } + 7 => { + let value = unsafe { MEM_readLE64(ptr.cast::()) }; + hash_shift64( + value.wrapping_shl(8).wrapping_mul(58_295_818_150_454_627), + hbits, + ) + } + 8 => { + let value = unsafe { MEM_readLE64(ptr.cast::()) }; + hash_shift64(value.wrapping_mul(0xCF1B_BCDC_B7A5_6463), hbits) + } + _ => { + let value = unsafe { MEM_readLE32(ptr.cast::()) }; + hash_shift32(value.wrapping_mul(2_654_435_761), hbits) + } + } +} + +#[inline] +fn common_bytes(word: usize) -> usize { + let zeros = if MEM_isLittleEndian() { + word.trailing_zeros() + } else { + word.leading_zeros() + }; + (zeros / 8) as usize +} + +unsafe fn count(mut input: *const u8, mut matched: *const u8, input_limit: *const u8) -> usize { + let input_start = input; + let word_size = size_of::(); + while unsafe { input_limit.offset_from(input) as usize } >= word_size { + let diff = + unsafe { MEM_readST(matched.cast::()) ^ MEM_readST(input.cast::()) }; + if diff != 0 { + return unsafe { input.offset_from(input_start) as usize } + common_bytes(diff); + } + input = input.wrapping_add(word_size); + matched = matched.wrapping_add(word_size); + } + if MEM_64bits() + && unsafe { input_limit.offset_from(input) as usize } >= 4 + && unsafe { MEM_read32(matched.cast::()) == MEM_read32(input.cast::()) } + { + input = input.wrapping_add(4); + matched = matched.wrapping_add(4); + } + if unsafe { input_limit.offset_from(input) as usize } >= 2 + && unsafe { MEM_read16(matched.cast::()) == MEM_read16(input.cast::()) } + { + input = input.wrapping_add(2); + matched = matched.wrapping_add(2); + } + if ptr_lt(input, input_limit) && unsafe { *input == *matched } { + input = input.wrapping_add(1); + } + unsafe { input.offset_from(input_start) as usize } +} + +unsafe fn count_2segments( + input: *const u8, + matched: *const u8, + input_end: *const u8, + match_end: *const u8, + input_start: *const u8, +) -> usize { + let match_remaining = unsafe { match_end.offset_from(matched) as usize }; + let input_remaining = unsafe { input_end.offset_from(input) as usize }; + let first_end = input.wrapping_add(match_remaining.min(input_remaining)); + let first_count = unsafe { count(input, matched, first_end) }; + if matched.wrapping_add(first_count) != match_end { + return first_count; + } + first_count + unsafe { count(input.wrapping_add(first_count), input_start, input_end) } +} + +#[inline] +fn lowest_prefix_index(dict_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32.wrapping_shl(window_log); + let within_window = if curr.wrapping_sub(dict_limit) > max_distance { + curr.wrapping_sub(max_distance) + } else { + dict_limit + }; + if loaded_dict_end != 0 { + dict_limit + } else { + within_window + } +} + +#[inline] +fn lowest_match_index(low_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32.wrapping_shl(window_log); + let within_window = if curr.wrapping_sub(low_limit) > max_distance { + curr.wrapping_sub(max_distance) + } else { + low_limit + }; + if loaded_dict_end != 0 { + low_limit + } else { + within_window + } +} + +#[inline] +fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool { + prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3 +} + +#[inline] +fn write_tagged_index(table: *mut u32, hash_and_tag: usize, index: u32) { + let hash = hash_and_tag >> SHORT_CACHE_TAG_BITS; + let tag = (hash_and_tag as u32) & SHORT_CACHE_TAG_MASK; + unsafe { table_set(table, hash, (index << SHORT_CACHE_TAG_BITS) | tag) }; +} + +#[inline] +fn packed_tags_match(first: u32, second: usize) -> bool { + (first & SHORT_CACHE_TAG_MASK) == ((second as u32) & SHORT_CACHE_TAG_MASK) +} + +unsafe fn store_seq( + seq_store: *mut SeqStore_t, + lit_length: usize, + literals: *const u8, + _lit_limit: *const u8, + off_base: u32, + match_length: usize, +) { + let seq_store = unsafe { &mut *seq_store }; + let sequence = seq_store.sequences; + debug_assert!( + unsafe { sequence.offset_from(seq_store.sequencesStart) as usize } < seq_store.maxNbSeq + ); + debug_assert!(match_length >= MINMATCH); + if lit_length != 0 { + unsafe { ptr::copy_nonoverlapping(literals, seq_store.lit, lit_length) }; + } + seq_store.lit = seq_store.lit.wrapping_add(lit_length); + + let sequence_index = unsafe { sequence.offset_from(seq_store.sequencesStart) as u32 }; + if lit_length > u16::MAX as usize { + debug_assert_eq!(seq_store.longLengthType, 0); + seq_store.longLengthType = 1; + seq_store.longLengthPos = sequence_index; + } + unsafe { (*sequence).litLength = lit_length as u16 }; + unsafe { (*sequence).offBase = off_base }; + let match_base = match_length - MINMATCH; + if match_base > u16::MAX as usize { + debug_assert_eq!(seq_store.longLengthType, 0); + seq_store.longLengthType = 2; + seq_store.longLengthPos = sequence_index; + } + unsafe { (*sequence).mlBase = match_base as u16 }; + seq_store.sequences = sequence.wrapping_add(1); +} + +#[inline] +fn fast_mls(min_match: u32) -> u32 { + match min_match { + 5..=7 => min_match, + _ => 4, + } +} + +unsafe fn fill_double_hash_table( + hash_long: *mut u32, + hash_small: *mut u32, + base: *const u8, + next_to_update: u32, + end: *const u8, + hash_log: u32, + chain_log: u32, + min_match: u32, + full_table_load: bool, + tagged_indices: bool, +) { + if unsafe { end.offset_from(base) } < HASH_READ_SIZE as isize { + return; + } + let hbits_long = hash_log + + if tagged_indices { + SHORT_CACHE_TAG_BITS + } else { + 0 + }; + let hbits_small = chain_log + + if tagged_indices { + SHORT_CACHE_TAG_BITS + } else { + 0 + }; + let mut input = base.wrapping_add(next_to_update as usize); + let input_end = end.wrapping_sub(HASH_READ_SIZE); + + while ptr_le(input.wrapping_add(2), input_end) { + let current = unsafe { index_from(base, input) }; + for position in 0..3usize { + let small_hash = + unsafe { hash_ptr(input.wrapping_add(position), hbits_small, min_match) }; + let long_hash = unsafe { hash_ptr(input.wrapping_add(position), hbits_long, 8) }; + let small_index = if tagged_indices { + small_hash >> SHORT_CACHE_TAG_BITS + } else { + small_hash + }; + let long_index = if tagged_indices { + long_hash >> SHORT_CACHE_TAG_BITS + } else { + long_hash + }; + if position == 0 { + if tagged_indices { + write_tagged_index( + hash_small, + small_hash, + current.wrapping_add(position as u32), + ); + } else { + unsafe { + table_set( + hash_small, + small_index, + current.wrapping_add(position as u32), + ) + }; + } + } + if position == 0 || unsafe { table_get(hash_long, long_index) } == 0 { + if tagged_indices { + write_tagged_index(hash_long, long_hash, current.wrapping_add(position as u32)); + } else { + unsafe { + table_set(hash_long, long_index, current.wrapping_add(position as u32)) + }; + } + } + if !full_table_load { + break; + } + } + input = input.wrapping_add(3); + } +} + +/// Rust implementation called by the C ABI wrapper for `ZSTD_fillDoubleHashTable`. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_fillDoubleHashTable( + hash_long: *mut u32, + hash_small: *mut u32, + base: *const u8, + next_to_update: u32, + end: *const c_void, + hash_log: u32, + chain_log: u32, + min_match: u32, + full_table_load: c_int, + for_cdict: c_int, +) { + unsafe { + fill_double_hash_table( + hash_long, + hash_small, + base, + next_to_update, + end.cast::(), + hash_log, + chain_log, + min_match, + full_table_load != 0, + for_cdict != 0, + ) + }; +} + +unsafe fn post_no_dict_match( + hash_long: *mut u32, + hash_small: *mut u32, + base: *const u8, + hbits_long: u32, + hbits_small: u32, + mls: u32, + seq_store: *mut SeqStore_t, + offset1: &mut u32, + offset2: &mut u32, + mut ip: *const u8, + match_length: usize, + current: u32, + iend: *const u8, + ilimit: *const u8, +) -> (*const u8, *const u8) { + ip = ip.wrapping_add(match_length); + let mut anchor = ip; + if ptr_le(ip, ilimit) { + let index_to_insert = current.wrapping_add(2); + let hash = unsafe { hash_ptr(base.wrapping_add(index_to_insert as usize), hbits_long, 8) }; + unsafe { table_set(hash_long, hash, index_to_insert) }; + let hash = unsafe { hash_ptr(ip.wrapping_sub(2), hbits_long, 8) }; + unsafe { table_set(hash_long, hash, index_from(base, ip.wrapping_sub(2))) }; + let hash = unsafe { + hash_ptr( + base.wrapping_add(index_to_insert as usize), + hbits_small, + mls, + ) + }; + unsafe { table_set(hash_small, hash, index_to_insert) }; + let hash = unsafe { hash_ptr(ip.wrapping_sub(1), hbits_small, mls) }; + unsafe { table_set(hash_small, hash, index_from(base, ip.wrapping_sub(1))) }; + + while ptr_le(ip, ilimit) + && *offset2 > 0 + && unsafe { read32(ip) == read32(ip.wrapping_sub(*offset2 as usize)) } + { + let repeat_length = unsafe { + count( + ip.wrapping_add(4), + ip.wrapping_add(4).wrapping_sub(*offset2 as usize), + iend, + ) + 4 + }; + std::mem::swap(offset1, offset2); + let hash = unsafe { hash_ptr(ip, hbits_small, mls) }; + unsafe { table_set(hash_small, hash, index_from(base, ip)) }; + let hash = unsafe { hash_ptr(ip, hbits_long, 8) }; + unsafe { table_set(hash_long, hash, index_from(base, ip)) }; + unsafe { + store_seq( + seq_store, + 0, + anchor, + iend, + REPCODE1_TO_OFFBASE, + repeat_length, + ) + }; + ip = ip.wrapping_add(repeat_length); + anchor = ip; + } + } + (ip, anchor) +} + +unsafe fn compress_block_double_fast_no_dict( + hash_long: *mut u32, + hash_small: *mut u32, + base: *const u8, + dict_limit: u32, + loaded_dict_end: u32, + hash_log: u32, + chain_log: u32, + window_log: u32, + seq_store: *mut SeqStore_t, + reps: *mut u32, + src: *const u8, + src_size: usize, + mls: u32, +) -> usize { + if src_size < HASH_READ_SIZE { + return src_size; + } + let istart = src; + let iend = istart.wrapping_add(src_size); + let ilimit = iend.wrapping_sub(HASH_READ_SIZE); + let end_index = unsafe { index_from(base, istart) }.wrapping_add(src_size as u32); + let prefix_lowest_index = + lowest_prefix_index(dict_limit, loaded_dict_end, end_index, window_log); + let prefix_lowest = base.wrapping_add(prefix_lowest_index as usize); + let mut ip = istart; + let mut anchor = istart; + let mut offset1 = unsafe { *reps }; + let mut offset2 = unsafe { *reps.add(1) }; + let mut offset_saved1 = 0u32; + let mut offset_saved2 = 0u32; + + if ip == prefix_lowest { + ip = ip.wrapping_add(1); + } + let current = unsafe { index_from(base, ip) }; + let window_low = lowest_prefix_index(dict_limit, loaded_dict_end, current, window_log); + let max_rep = current.wrapping_sub(window_low); + if offset2 > max_rep { + offset_saved2 = offset2; + offset2 = 0; + } + if offset1 > max_rep { + offset_saved1 = offset1; + offset1 = 0; + } + + 'outer: loop { + let mut step = 1usize; + let mut next_step = ip.wrapping_add(1 << K_SEARCH_STRENGTH); + let mut ip1 = ip.wrapping_add(step); + if ptr_gt(ip1, ilimit) { + break; + } + + let mut hash_long0 = unsafe { hash_ptr(ip, hash_log, 8) }; + let mut index_long0 = unsafe { table_get(hash_long, hash_long0) }; + let mut match_long0 = base.wrapping_add(index_long0 as usize); + + loop { + let hash_small0 = unsafe { hash_ptr(ip, chain_log, mls) }; + let index_small0 = unsafe { table_get(hash_small, hash_small0) }; + let mut match_small0 = base.wrapping_add(index_small0 as usize); + let current = unsafe { index_from(base, ip) }; + unsafe { + table_set(hash_long, hash_long0, current); + table_set(hash_small, hash_small0, current); + } + + if offset1 > 0 + && unsafe { + read32(ip.wrapping_add(1).wrapping_sub(offset1 as usize)) + == read32(ip.wrapping_add(1)) + } + { + let match_length = unsafe { + count( + ip.wrapping_add(5), + ip.wrapping_add(5).wrapping_sub(offset1 as usize), + iend, + ) + 4 + }; + ip = ip.wrapping_add(1); + unsafe { + store_seq( + seq_store, + ip.offset_from(anchor) as usize, + anchor, + iend, + REPCODE1_TO_OFFBASE, + match_length, + ) + }; + let (next_ip, next_anchor) = unsafe { + post_no_dict_match( + hash_long, + hash_small, + base, + hash_log, + chain_log, + mls, + seq_store, + &mut offset1, + &mut offset2, + ip, + match_length, + current, + iend, + ilimit, + ) + }; + ip = next_ip; + anchor = next_anchor; + continue 'outer; + } + + let hash_long1 = unsafe { hash_ptr(ip1, hash_log, 8) }; + if index_long0 > prefix_lowest_index && unsafe { read64(match_long0) == read64(ip) } { + let mut match_length = + unsafe { count(ip.wrapping_add(8), match_long0.wrapping_add(8), iend) + 8 }; + let offset = unsafe { index_from(match_long0, ip) }; + while ptr_gt(ip, anchor) + && ptr_gt(match_long0, prefix_lowest) + && unsafe { *ip.wrapping_sub(1) == *match_long0.wrapping_sub(1) } + { + ip = ip.wrapping_sub(1); + match_long0 = match_long0.wrapping_sub(1); + match_length += 1; + } + offset2 = offset1; + offset1 = offset; + if step < 4 { + unsafe { table_set(hash_long, hash_long1, index_from(base, ip1)) }; + } + unsafe { + store_seq( + seq_store, + ip.offset_from(anchor) as usize, + anchor, + iend, + offset.wrapping_add(ZSTD_REP_NUM as u32), + match_length, + ) + }; + let (next_ip, next_anchor) = unsafe { + post_no_dict_match( + hash_long, + hash_small, + base, + hash_log, + chain_log, + mls, + seq_store, + &mut offset1, + &mut offset2, + ip, + match_length, + current, + iend, + ilimit, + ) + }; + ip = next_ip; + anchor = next_anchor; + continue 'outer; + } + + let index_long1 = unsafe { table_get(hash_long, hash_long1) }; + let match_long1 = base.wrapping_add(index_long1 as usize); + if index_small0 > prefix_lowest_index && unsafe { read32(match_small0) == read32(ip) } { + let mut match_length = + unsafe { count(ip.wrapping_add(4), match_small0.wrapping_add(4), iend) + 4 }; + let mut offset = unsafe { index_from(match_small0, ip) }; + if index_long1 > prefix_lowest_index + && unsafe { read64(match_long1) == read64(ip1) } + { + let long_length = unsafe { + count(ip1.wrapping_add(8), match_long1.wrapping_add(8), iend) + 8 + }; + if long_length > match_length { + ip = ip1; + match_length = long_length; + offset = unsafe { index_from(match_long1, ip) }; + match_small0 = match_long1; + } + } + while ptr_gt(ip, anchor) + && ptr_gt(match_small0, prefix_lowest) + && unsafe { *ip.wrapping_sub(1) == *match_small0.wrapping_sub(1) } + { + ip = ip.wrapping_sub(1); + match_small0 = match_small0.wrapping_sub(1); + match_length += 1; + } + offset2 = offset1; + offset1 = offset; + if step < 4 { + unsafe { table_set(hash_long, hash_long1, index_from(base, ip1)) }; + } + unsafe { + store_seq( + seq_store, + ip.offset_from(anchor) as usize, + anchor, + iend, + offset.wrapping_add(ZSTD_REP_NUM as u32), + match_length, + ) + }; + let (next_ip, next_anchor) = unsafe { + post_no_dict_match( + hash_long, + hash_small, + base, + hash_log, + chain_log, + mls, + seq_store, + &mut offset1, + &mut offset2, + ip, + match_length, + current, + iend, + ilimit, + ) + }; + ip = next_ip; + anchor = next_anchor; + continue 'outer; + } + + if ptr_ge(ip1, next_step) { + step += 1; + next_step = next_step.wrapping_add(1 << K_SEARCH_STRENGTH); + } + ip = ip1; + ip1 = ip1.wrapping_add(step); + hash_long0 = hash_long1; + index_long0 = index_long1; + match_long0 = match_long1; + if ptr_gt(ip1, ilimit) { + break 'outer; + } + } + } + + offset_saved2 = if offset_saved1 != 0 && offset1 != 0 { + offset_saved1 + } else { + offset_saved2 + }; + unsafe { + *reps = if offset1 != 0 { offset1 } else { offset_saved1 }; + *reps.add(1) = if offset2 != 0 { offset2 } else { offset_saved2 }; + } + unsafe { iend.offset_from(anchor) as usize } +} + +/// Rust implementation called by the C ABI wrapper for `ZSTD_compressBlock_doubleFast`. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_compressBlock_doubleFast( + hash_long: *mut u32, + hash_small: *mut u32, + base: *const u8, + dict_limit: u32, + loaded_dict_end: u32, + hash_log: u32, + chain_log: u32, + min_match: u32, + window_log: u32, + seq_store: *mut c_void, + reps: *mut u32, + src: *const c_void, + src_size: usize, +) -> usize { + unsafe { + compress_block_double_fast_no_dict( + hash_long, + hash_small, + base, + dict_limit, + loaded_dict_end, + hash_log, + chain_log, + window_log, + seq_store.cast::(), + reps, + src.cast::(), + src_size, + fast_mls(min_match), + ) + } +} + +unsafe fn post_dict_match( + hash_long: *mut u32, + hash_small: *mut u32, + base: *const u8, + dict_base: *const u8, + prefix_lowest_index: u32, + dict_index_delta: u32, + dict_end: *const u8, + prefix_lowest: *const u8, + hbits_long: u32, + hbits_small: u32, + mls: u32, + seq_store: *mut SeqStore_t, + offset1: &mut u32, + offset2: &mut u32, + mut ip: *const u8, + match_length: usize, + current: u32, + iend: *const u8, + ilimit: *const u8, +) -> (*const u8, *const u8) { + ip = ip.wrapping_add(match_length); + let mut anchor = ip; + if ptr_le(ip, ilimit) { + let index_to_insert = current.wrapping_add(2); + let hash = unsafe { hash_ptr(base.wrapping_add(index_to_insert as usize), hbits_long, 8) }; + unsafe { table_set(hash_long, hash, index_to_insert) }; + let hash = unsafe { hash_ptr(ip.wrapping_sub(2), hbits_long, 8) }; + unsafe { table_set(hash_long, hash, index_from(base, ip.wrapping_sub(2))) }; + let hash = unsafe { + hash_ptr( + base.wrapping_add(index_to_insert as usize), + hbits_small, + mls, + ) + }; + unsafe { table_set(hash_small, hash, index_to_insert) }; + let hash = unsafe { hash_ptr(ip.wrapping_sub(1), hbits_small, mls) }; + unsafe { table_set(hash_small, hash, index_from(base, ip.wrapping_sub(1))) }; + + while ptr_le(ip, ilimit) { + let current2 = unsafe { index_from(base, ip) }; + let rep_index2 = current2.wrapping_sub(*offset2); + let rep_match2 = if rep_index2 < prefix_lowest_index { + dict_base + .wrapping_sub(dict_index_delta as usize) + .wrapping_add(rep_index2 as usize) + } else { + base.wrapping_add(rep_index2 as usize) + }; + if index_overlap_check(prefix_lowest_index, rep_index2) + && unsafe { read32(rep_match2) == read32(ip) } + { + let rep_end2 = if rep_index2 < prefix_lowest_index { + dict_end + } else { + iend + }; + let repeat_length = unsafe { + count_2segments( + ip.wrapping_add(4), + rep_match2.wrapping_add(4), + iend, + rep_end2, + prefix_lowest, + ) + 4 + }; + std::mem::swap(offset1, offset2); + unsafe { + store_seq( + seq_store, + 0, + anchor, + iend, + REPCODE1_TO_OFFBASE, + repeat_length, + ) + }; + let hash = unsafe { hash_ptr(ip, hbits_small, mls) }; + unsafe { table_set(hash_small, hash, current2) }; + let hash = unsafe { hash_ptr(ip, hbits_long, 8) }; + unsafe { table_set(hash_long, hash, current2) }; + ip = ip.wrapping_add(repeat_length); + anchor = ip; + continue; + } + break; + } + } + (ip, anchor) +} + +unsafe fn compress_block_double_fast_dict_match_state( + hash_long: *mut u32, + hash_small: *mut u32, + base: *const u8, + prefix_lowest_index: u32, + hash_log: u32, + chain_log: u32, + seq_store: *mut SeqStore_t, + reps: *mut u32, + src: *const u8, + src_size: usize, + mls: u32, + dict_hash_long: *const u32, + dict_hash_small: *const u32, + dict_base: *const u8, + dict_start_index: u32, + dict_end: *const u8, + dict_hash_log: u32, + dict_chain_log: u32, + _prefetch_cdict_tables: bool, +) -> usize { + if src_size < HASH_READ_SIZE { + return src_size; + } + let istart = src; + let iend = istart.wrapping_add(src_size); + let ilimit = iend.wrapping_sub(HASH_READ_SIZE); + let mut ip = istart; + let mut anchor = istart; + let prefix_lowest = base.wrapping_add(prefix_lowest_index as usize); + let dict_start = dict_base.wrapping_add(dict_start_index as usize); + let dict_index_delta = + prefix_lowest_index.wrapping_sub(unsafe { index_from(dict_base, dict_end) }); + let dict_hbits_long = dict_hash_log + SHORT_CACHE_TAG_BITS; + let dict_hbits_small = dict_chain_log + SHORT_CACHE_TAG_BITS; + let dict_and_prefix_length = + unsafe { ip.offset_from(prefix_lowest) as u32 + dict_end.offset_from(dict_start) as u32 }; + let mut offset1 = unsafe { *reps }; + let mut offset2 = unsafe { *reps.add(1) }; + if dict_and_prefix_length == 0 { + ip = ip.wrapping_add(1); + } + + 'outer: while ptr_lt(ip, ilimit) { + let h_long = unsafe { hash_ptr(ip, hash_log, 8) }; + let h_small = unsafe { hash_ptr(ip, chain_log, mls) }; + let dict_hash_and_tag_long = unsafe { hash_ptr(ip, dict_hbits_long, 8) }; + let dict_hash_and_tag_small = unsafe { hash_ptr(ip, dict_hbits_small, mls) }; + let dict_match_index_and_tag_long = unsafe { + table_get( + dict_hash_long, + dict_hash_and_tag_long >> SHORT_CACHE_TAG_BITS, + ) + }; + let dict_match_index_and_tag_small = unsafe { + table_get( + dict_hash_small, + dict_hash_and_tag_small >> SHORT_CACHE_TAG_BITS, + ) + }; + let dict_tags_match_long = + packed_tags_match(dict_match_index_and_tag_long, dict_hash_and_tag_long); + let dict_tags_match_small = + packed_tags_match(dict_match_index_and_tag_small, dict_hash_and_tag_small); + let current = unsafe { index_from(base, ip) }; + let match_index_long = unsafe { table_get(hash_long, h_long) }; + let mut match_index_small = unsafe { table_get(hash_small, h_small) }; + let mut match_long = base.wrapping_add(match_index_long as usize); + let mut matched = base.wrapping_add(match_index_small as usize); + let rep_index = current.wrapping_add(1).wrapping_sub(offset1); + let rep_match = if rep_index < prefix_lowest_index { + dict_base.wrapping_add(rep_index.wrapping_sub(dict_index_delta) as usize) + } else { + base.wrapping_add(rep_index as usize) + }; + unsafe { + table_set(hash_long, h_long, current); + table_set(hash_small, h_small, current); + } + + if index_overlap_check(prefix_lowest_index, rep_index) + && unsafe { read32(rep_match) == read32(ip.wrapping_add(1)) } + { + let rep_match_end = if rep_index < prefix_lowest_index { + dict_end + } else { + iend + }; + let match_length = unsafe { + count_2segments( + ip.wrapping_add(5), + rep_match.wrapping_add(4), + iend, + rep_match_end, + prefix_lowest, + ) + 4 + }; + ip = ip.wrapping_add(1); + unsafe { + store_seq( + seq_store, + ip.offset_from(anchor) as usize, + anchor, + iend, + REPCODE1_TO_OFFBASE, + match_length, + ) + }; + let (next_ip, next_anchor) = unsafe { + post_dict_match( + hash_long, + hash_small, + base, + dict_base, + prefix_lowest_index, + dict_index_delta, + dict_end, + prefix_lowest, + hash_log, + chain_log, + mls, + seq_store, + &mut offset1, + &mut offset2, + ip, + match_length, + current, + iend, + ilimit, + ) + }; + ip = next_ip; + anchor = next_anchor; + continue; + } + + if match_index_long >= prefix_lowest_index && unsafe { read64(match_long) == read64(ip) } { + let mut match_length = + unsafe { count(ip.wrapping_add(8), match_long.wrapping_add(8), iend) + 8 }; + let offset = unsafe { index_from(match_long, ip) }; + while ptr_gt(ip, anchor) + && ptr_gt(match_long, prefix_lowest) + && unsafe { *ip.wrapping_sub(1) == *match_long.wrapping_sub(1) } + { + ip = ip.wrapping_sub(1); + match_long = match_long.wrapping_sub(1); + match_length += 1; + } + offset2 = offset1; + offset1 = offset; + unsafe { + store_seq( + seq_store, + ip.offset_from(anchor) as usize, + anchor, + iend, + offset.wrapping_add(ZSTD_REP_NUM as u32), + match_length, + ) + }; + let (next_ip, next_anchor) = unsafe { + post_dict_match( + hash_long, + hash_small, + base, + dict_base, + prefix_lowest_index, + dict_index_delta, + dict_end, + prefix_lowest, + hash_log, + chain_log, + mls, + seq_store, + &mut offset1, + &mut offset2, + ip, + match_length, + current, + iend, + ilimit, + ) + }; + ip = next_ip; + anchor = next_anchor; + continue; + } + + if dict_tags_match_long { + let dict_match_index = dict_match_index_and_tag_long >> SHORT_CACHE_TAG_BITS; + let mut dict_match = dict_base.wrapping_add(dict_match_index as usize); + if ptr_gt(dict_match, dict_start) && unsafe { read64(dict_match) == read64(ip) } { + let mut match_length = unsafe { + count_2segments( + ip.wrapping_add(8), + dict_match.wrapping_add(8), + iend, + dict_end, + prefix_lowest, + ) + 8 + }; + let offset = current + .wrapping_sub(dict_match_index) + .wrapping_sub(dict_index_delta); + while ptr_gt(ip, anchor) + && ptr_gt(dict_match, dict_start) + && unsafe { *ip.wrapping_sub(1) == *dict_match.wrapping_sub(1) } + { + ip = ip.wrapping_sub(1); + dict_match = dict_match.wrapping_sub(1); + match_length += 1; + } + offset2 = offset1; + offset1 = offset; + unsafe { + store_seq( + seq_store, + ip.offset_from(anchor) as usize, + anchor, + iend, + offset.wrapping_add(ZSTD_REP_NUM as u32), + match_length, + ) + }; + let (next_ip, next_anchor) = unsafe { + post_dict_match( + hash_long, + hash_small, + base, + dict_base, + prefix_lowest_index, + dict_index_delta, + dict_end, + prefix_lowest, + hash_log, + chain_log, + mls, + seq_store, + &mut offset1, + &mut offset2, + ip, + match_length, + current, + iend, + ilimit, + ) + }; + ip = next_ip; + anchor = next_anchor; + continue; + } + } + + let mut short_found = false; + if match_index_small > prefix_lowest_index { + short_found = unsafe { read32(matched) == read32(ip) }; + } else if dict_tags_match_small { + let dict_match_index = dict_match_index_and_tag_small >> SHORT_CACHE_TAG_BITS; + let dict_match = dict_base.wrapping_add(dict_match_index as usize); + match_index_small = dict_match_index.wrapping_add(dict_index_delta); + if ptr_gt(dict_match, dict_start) && unsafe { read32(dict_match) == read32(ip) } { + matched = dict_match; + short_found = true; + } + } + if !short_found { + ip = ip.wrapping_add( + (unsafe { ip.offset_from(anchor) as usize } >> K_SEARCH_STRENGTH) + 1, + ); + continue; + } + + let hash_long3 = unsafe { hash_ptr(ip.wrapping_add(1), hash_log, 8) }; + let dict_hash_and_tag_long3 = unsafe { hash_ptr(ip.wrapping_add(1), dict_hbits_long, 8) }; + let match_index_long3 = unsafe { table_get(hash_long, hash_long3) }; + let dict_match_index_and_tag_long3 = unsafe { + table_get( + dict_hash_long, + dict_hash_and_tag_long3 >> SHORT_CACHE_TAG_BITS, + ) + }; + let dict_tags_match_long3 = + packed_tags_match(dict_match_index_and_tag_long3, dict_hash_and_tag_long3); + let mut match_long3 = base.wrapping_add(match_index_long3 as usize); + unsafe { table_set(hash_long, hash_long3, current.wrapping_add(1)) }; + + let (match_length, offset) = if match_index_long3 >= prefix_lowest_index + && unsafe { read64(match_long3) == read64(ip.wrapping_add(1)) } + { + let mut length = + unsafe { count(ip.wrapping_add(9), match_long3.wrapping_add(8), iend) + 8 }; + ip = ip.wrapping_add(1); + let offset = unsafe { index_from(match_long3, ip) }; + while ptr_gt(ip, anchor) + && ptr_gt(match_long3, prefix_lowest) + && unsafe { *ip.wrapping_sub(1) == *match_long3.wrapping_sub(1) } + { + ip = ip.wrapping_sub(1); + match_long3 = match_long3.wrapping_sub(1); + length += 1; + } + (length, offset) + } else if dict_tags_match_long3 { + let dict_match_index = dict_match_index_and_tag_long3 >> SHORT_CACHE_TAG_BITS; + let mut dict_match = dict_base.wrapping_add(dict_match_index as usize); + if ptr_gt(dict_match, dict_start) + && unsafe { read64(dict_match) == read64(ip.wrapping_add(1)) } + { + let mut length = unsafe { + count_2segments( + ip.wrapping_add(9), + dict_match.wrapping_add(8), + iend, + dict_end, + prefix_lowest, + ) + 8 + }; + ip = ip.wrapping_add(1); + let offset = current + .wrapping_add(1) + .wrapping_sub(dict_match_index) + .wrapping_sub(dict_index_delta); + while ptr_gt(ip, anchor) + && ptr_gt(dict_match, dict_start) + && unsafe { *ip.wrapping_sub(1) == *dict_match.wrapping_sub(1) } + { + ip = ip.wrapping_sub(1); + dict_match = dict_match.wrapping_sub(1); + length += 1; + } + (length, offset) + } else if match_index_small < prefix_lowest_index { + let mut length = unsafe { + count_2segments( + ip.wrapping_add(4), + matched.wrapping_add(4), + iend, + dict_end, + prefix_lowest, + ) + 4 + }; + let offset = current.wrapping_sub(match_index_small); + while ptr_gt(ip, anchor) + && ptr_gt(matched, dict_start) + && unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) } + { + ip = ip.wrapping_sub(1); + matched = matched.wrapping_sub(1); + length += 1; + } + (length, offset) + } else { + let mut length = + unsafe { count(ip.wrapping_add(4), matched.wrapping_add(4), iend) + 4 }; + let offset = unsafe { index_from(matched, ip) }; + while ptr_gt(ip, anchor) + && ptr_gt(matched, prefix_lowest) + && unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) } + { + ip = ip.wrapping_sub(1); + matched = matched.wrapping_sub(1); + length += 1; + } + (length, offset) + } + } else if match_index_small < prefix_lowest_index { + let mut length = unsafe { + count_2segments( + ip.wrapping_add(4), + matched.wrapping_add(4), + iend, + dict_end, + prefix_lowest, + ) + 4 + }; + let offset = current.wrapping_sub(match_index_small); + while ptr_gt(ip, anchor) + && ptr_gt(matched, dict_start) + && unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) } + { + ip = ip.wrapping_sub(1); + matched = matched.wrapping_sub(1); + length += 1; + } + (length, offset) + } else { + let mut length = + unsafe { count(ip.wrapping_add(4), matched.wrapping_add(4), iend) + 4 }; + let offset = unsafe { index_from(matched, ip) }; + while ptr_gt(ip, anchor) + && ptr_gt(matched, prefix_lowest) + && unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) } + { + ip = ip.wrapping_sub(1); + matched = matched.wrapping_sub(1); + length += 1; + } + (length, offset) + }; + offset2 = offset1; + offset1 = offset; + unsafe { + store_seq( + seq_store, + ip.offset_from(anchor) as usize, + anchor, + iend, + offset.wrapping_add(ZSTD_REP_NUM as u32), + match_length, + ) + }; + let (next_ip, next_anchor) = unsafe { + post_dict_match( + hash_long, + hash_small, + base, + dict_base, + prefix_lowest_index, + dict_index_delta, + dict_end, + prefix_lowest, + hash_log, + chain_log, + mls, + seq_store, + &mut offset1, + &mut offset2, + ip, + match_length, + current, + iend, + ilimit, + ) + }; + ip = next_ip; + anchor = next_anchor; + continue 'outer; + } + + unsafe { + *reps = offset1; + *reps.add(1) = offset2; + } + unsafe { iend.offset_from(anchor) as usize } +} + +/// Rust implementation called by the C ABI wrapper for attached dictionaries. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_compressBlock_doubleFast_dictMatchState( + hash_long: *mut u32, + hash_small: *mut u32, + base: *const u8, + prefix_lowest_index: u32, + hash_log: u32, + chain_log: u32, + min_match: u32, + seq_store: *mut c_void, + reps: *mut u32, + src: *const c_void, + src_size: usize, + dict_hash_long: *const u32, + dict_hash_small: *const u32, + dict_base: *const u8, + dict_start_index: u32, + dict_end: *const u8, + dict_hash_log: u32, + dict_chain_log: u32, + prefetch_cdict_tables: c_int, +) -> usize { + unsafe { + compress_block_double_fast_dict_match_state( + hash_long, + hash_small, + base, + prefix_lowest_index, + hash_log, + chain_log, + seq_store.cast::(), + reps, + src.cast::(), + src_size, + fast_mls(min_match), + dict_hash_long, + dict_hash_small, + dict_base, + dict_start_index, + dict_end, + dict_hash_log, + dict_chain_log, + prefetch_cdict_tables != 0, + ) + } +} + +unsafe fn post_ext_match( + hash_long: *mut u32, + hash_small: *mut u32, + base: *const u8, + dict_base: *const u8, + dict_start_index: u32, + prefix_start_index: u32, + dict_end: *const u8, + prefix_start: *const u8, + hbits_long: u32, + hbits_small: u32, + mls: u32, + seq_store: *mut SeqStore_t, + offset1: &mut u32, + offset2: &mut u32, + mut ip: *const u8, + match_length: usize, + current: u32, + iend: *const u8, + ilimit: *const u8, +) -> (*const u8, *const u8) { + ip = ip.wrapping_add(match_length); + let mut anchor = ip; + if ptr_le(ip, ilimit) { + let index_to_insert = current.wrapping_add(2); + let hash = unsafe { hash_ptr(base.wrapping_add(index_to_insert as usize), hbits_long, 8) }; + unsafe { table_set(hash_long, hash, index_to_insert) }; + let hash = unsafe { hash_ptr(ip.wrapping_sub(2), hbits_long, 8) }; + unsafe { table_set(hash_long, hash, index_from(base, ip.wrapping_sub(2))) }; + let hash = unsafe { + hash_ptr( + base.wrapping_add(index_to_insert as usize), + hbits_small, + mls, + ) + }; + unsafe { table_set(hash_small, hash, index_to_insert) }; + let hash = unsafe { hash_ptr(ip.wrapping_sub(1), hbits_small, mls) }; + unsafe { table_set(hash_small, hash, index_from(base, ip.wrapping_sub(1))) }; + + while ptr_le(ip, ilimit) { + let current2 = unsafe { index_from(base, ip) }; + let rep_index2 = current2.wrapping_sub(*offset2); + let rep_match2 = if rep_index2 < prefix_start_index { + dict_base.wrapping_add(rep_index2 as usize) + } else { + base.wrapping_add(rep_index2 as usize) + }; + if index_overlap_check(prefix_start_index, rep_index2) + && *offset2 <= current2.wrapping_sub(dict_start_index) + && unsafe { read32(rep_match2) == read32(ip) } + { + let rep_end2 = if rep_index2 < prefix_start_index { + dict_end + } else { + iend + }; + let repeat_length = unsafe { + count_2segments( + ip.wrapping_add(4), + rep_match2.wrapping_add(4), + iend, + rep_end2, + prefix_start, + ) + 4 + }; + std::mem::swap(offset1, offset2); + unsafe { + store_seq( + seq_store, + 0, + anchor, + iend, + REPCODE1_TO_OFFBASE, + repeat_length, + ) + }; + let hash = unsafe { hash_ptr(ip, hbits_small, mls) }; + unsafe { table_set(hash_small, hash, current2) }; + let hash = unsafe { hash_ptr(ip, hbits_long, 8) }; + unsafe { table_set(hash_long, hash, current2) }; + ip = ip.wrapping_add(repeat_length); + anchor = ip; + continue; + } + break; + } + } + (ip, anchor) +} + +unsafe fn compress_block_double_fast_ext_dict( + hash_long: *mut u32, + hash_small: *mut u32, + base: *const u8, + dict_base: *const u8, + dict_limit: u32, + low_limit: u32, + loaded_dict_end: u32, + hash_log: u32, + chain_log: u32, + window_log: u32, + seq_store: *mut SeqStore_t, + reps: *mut u32, + src: *const u8, + src_size: usize, + mls: u32, +) -> usize { + if src_size < HASH_READ_SIZE { + return src_size; + } + let istart = src; + let iend = istart.wrapping_add(src_size); + let ilimit = iend.wrapping_sub(HASH_READ_SIZE); + let end_index = unsafe { index_from(base, istart) }.wrapping_add(src_size as u32); + let dict_start_index = lowest_match_index(low_limit, loaded_dict_end, end_index, window_log); + let prefix_start_index = dict_limit.max(dict_start_index); + if prefix_start_index == dict_start_index { + return unsafe { + compress_block_double_fast_no_dict( + hash_long, + hash_small, + base, + dict_limit, + loaded_dict_end, + hash_log, + chain_log, + window_log, + seq_store, + reps, + src, + src_size, + mls, + ) + }; + } + let prefix_start = base.wrapping_add(prefix_start_index as usize); + let dict_start = dict_base.wrapping_add(dict_start_index as usize); + let dict_end = dict_base.wrapping_add(prefix_start_index as usize); + let mut ip = istart; + let mut anchor = istart; + let mut offset1 = unsafe { *reps }; + let mut offset2 = unsafe { *reps.add(1) }; + + while ptr_lt(ip, ilimit) { + let h_small = unsafe { hash_ptr(ip, chain_log, mls) }; + let match_index = unsafe { table_get(hash_small, h_small) }; + let match_base = if match_index < prefix_start_index { + dict_base + } else { + base + }; + let mut matched = match_base.wrapping_add(match_index as usize); + let h_long = unsafe { hash_ptr(ip, hash_log, 8) }; + let match_long_index = unsafe { table_get(hash_long, h_long) }; + let match_long_base = if match_long_index < prefix_start_index { + dict_base + } else { + base + }; + let mut match_long = match_long_base.wrapping_add(match_long_index as usize); + let current = unsafe { index_from(base, ip) }; + let rep_index = current.wrapping_add(1).wrapping_sub(offset1); + let rep_base = if rep_index < prefix_start_index { + dict_base + } else { + base + }; + let rep_match = rep_base.wrapping_add(rep_index as usize); + unsafe { + table_set(hash_small, h_small, current); + table_set(hash_long, h_long, current); + } + + if index_overlap_check(prefix_start_index, rep_index) + && offset1 <= current.wrapping_add(1).wrapping_sub(dict_start_index) + && unsafe { read32(rep_match) == read32(ip.wrapping_add(1)) } + { + let rep_end = if rep_index < prefix_start_index { + dict_end + } else { + iend + }; + let match_length = unsafe { + count_2segments( + ip.wrapping_add(5), + rep_match.wrapping_add(4), + iend, + rep_end, + prefix_start, + ) + 4 + }; + ip = ip.wrapping_add(1); + unsafe { + store_seq( + seq_store, + ip.offset_from(anchor) as usize, + anchor, + iend, + REPCODE1_TO_OFFBASE, + match_length, + ) + }; + let (next_ip, next_anchor) = unsafe { + post_ext_match( + hash_long, + hash_small, + base, + dict_base, + dict_start_index, + prefix_start_index, + dict_end, + prefix_start, + hash_log, + chain_log, + mls, + seq_store, + &mut offset1, + &mut offset2, + ip, + match_length, + current, + iend, + ilimit, + ) + }; + ip = next_ip; + anchor = next_anchor; + continue; + } + + if match_long_index > dict_start_index && unsafe { read64(match_long) == read64(ip) } { + let match_end = if match_long_index < prefix_start_index { + dict_end + } else { + iend + }; + let low_match = if match_long_index < prefix_start_index { + dict_start + } else { + prefix_start + }; + let mut match_length = unsafe { + count_2segments( + ip.wrapping_add(8), + match_long.wrapping_add(8), + iend, + match_end, + prefix_start, + ) + 8 + }; + let offset = current.wrapping_sub(match_long_index); + while ptr_gt(ip, anchor) + && ptr_gt(match_long, low_match) + && unsafe { *ip.wrapping_sub(1) == *match_long.wrapping_sub(1) } + { + ip = ip.wrapping_sub(1); + match_long = match_long.wrapping_sub(1); + match_length += 1; + } + offset2 = offset1; + offset1 = offset; + unsafe { + store_seq( + seq_store, + ip.offset_from(anchor) as usize, + anchor, + iend, + offset.wrapping_add(ZSTD_REP_NUM as u32), + match_length, + ) + }; + let (next_ip, next_anchor) = unsafe { + post_ext_match( + hash_long, + hash_small, + base, + dict_base, + dict_start_index, + prefix_start_index, + dict_end, + prefix_start, + hash_log, + chain_log, + mls, + seq_store, + &mut offset1, + &mut offset2, + ip, + match_length, + current, + iend, + ilimit, + ) + }; + ip = next_ip; + anchor = next_anchor; + continue; + } + + if match_index > dict_start_index && unsafe { read32(matched) == read32(ip) } { + let h_long3 = unsafe { hash_ptr(ip.wrapping_add(1), hash_log, 8) }; + let match_index3 = unsafe { table_get(hash_long, h_long3) }; + let match_base3 = if match_index3 < prefix_start_index { + dict_base + } else { + base + }; + let mut match3 = match_base3.wrapping_add(match_index3 as usize); + unsafe { table_set(hash_long, h_long3, current.wrapping_add(1)) }; + let (match_length, offset) = if match_index3 > dict_start_index + && unsafe { read64(match3) == read64(ip.wrapping_add(1)) } + { + let match_end = if match_index3 < prefix_start_index { + dict_end + } else { + iend + }; + let low_match = if match_index3 < prefix_start_index { + dict_start + } else { + prefix_start + }; + let mut length = unsafe { + count_2segments( + ip.wrapping_add(9), + match3.wrapping_add(8), + iend, + match_end, + prefix_start, + ) + 8 + }; + ip = ip.wrapping_add(1); + let offset = current.wrapping_add(1).wrapping_sub(match_index3); + while ptr_gt(ip, anchor) + && ptr_gt(match3, low_match) + && unsafe { *ip.wrapping_sub(1) == *match3.wrapping_sub(1) } + { + ip = ip.wrapping_sub(1); + match3 = match3.wrapping_sub(1); + length += 1; + } + (length, offset) + } else { + let match_end = if match_index < prefix_start_index { + dict_end + } else { + iend + }; + let low_match = if match_index < prefix_start_index { + dict_start + } else { + prefix_start + }; + let mut length = unsafe { + count_2segments( + ip.wrapping_add(4), + matched.wrapping_add(4), + iend, + match_end, + prefix_start, + ) + 4 + }; + let offset = current.wrapping_sub(match_index); + while ptr_gt(ip, anchor) + && ptr_gt(matched, low_match) + && unsafe { *ip.wrapping_sub(1) == *matched.wrapping_sub(1) } + { + ip = ip.wrapping_sub(1); + matched = matched.wrapping_sub(1); + length += 1; + } + (length, offset) + }; + offset2 = offset1; + offset1 = offset; + unsafe { + store_seq( + seq_store, + ip.offset_from(anchor) as usize, + anchor, + iend, + offset.wrapping_add(ZSTD_REP_NUM as u32), + match_length, + ) + }; + let (next_ip, next_anchor) = unsafe { + post_ext_match( + hash_long, + hash_small, + base, + dict_base, + dict_start_index, + prefix_start_index, + dict_end, + prefix_start, + hash_log, + chain_log, + mls, + seq_store, + &mut offset1, + &mut offset2, + ip, + match_length, + current, + iend, + ilimit, + ) + }; + ip = next_ip; + anchor = next_anchor; + continue; + } + + ip = ip.wrapping_add((unsafe { ip.offset_from(anchor) as usize } >> K_SEARCH_STRENGTH) + 1); + } + + unsafe { + *reps = offset1; + *reps.add(1) = offset2; + } + unsafe { iend.offset_from(anchor) as usize } +} + +/// Rust implementation called by the C ABI wrapper for external dictionaries. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_compressBlock_doubleFast_extDict( + hash_long: *mut u32, + hash_small: *mut u32, + base: *const u8, + dict_base: *const u8, + dict_limit: u32, + low_limit: u32, + loaded_dict_end: u32, + hash_log: u32, + chain_log: u32, + min_match: u32, + window_log: u32, + seq_store: *mut c_void, + reps: *mut u32, + src: *const c_void, + src_size: usize, +) -> usize { + unsafe { + compress_block_double_fast_ext_dict( + hash_long, + hash_small, + base, + dict_base, + dict_limit, + low_limit, + loaded_dict_end, + hash_log, + chain_log, + window_log, + seq_store.cast::(), + reps, + src.cast::(), + src_size, + fast_mls(min_match), + ) + } +}