From 74140c318f024a46ceaa5a83c082eaddf98dce6f Mon Sep 17 00:00:00 2001 From: ddidderr Date: Fri, 10 Jul 2026 22:46:15 +0200 Subject: [PATCH] feat(rust): port fast block matching Move fast hash-table filling and no-dictionary, external-dictionary, and attached-CDict match loops into Rust. The C shim keeps `ZSTD_MatchState_t` opaque, extracts only its needed fields, and asserts the mirrored SeqStore leaf layout so existing compression dispatch remains ABI-compatible. Test Plan: - cargo clippy - cargo clippy --benches - cargo clippy --tests - cargo +nightly fmt - cargo test zstd_fast::tests -- --nocapture - cargo test --target i686-unknown-linux-gnu zstd_fast::tests -- --nocapture - byte-identical C-control frames for dictionary and streaming variants - fuzzer -s2066 -i5 --no-big-tests - invalidDictionaries and zstreamtest -i1 Refs: Rust superblock port fde1d70c --- lib/compress/zstd_fast.c | 1002 ++---------------------- rust/src/lib.rs | 2 + rust/src/zstd_fast.rs | 1599 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 1657 insertions(+), 946 deletions(-) create mode 100644 rust/src/zstd_fast.rs diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c index ee25bcbac..190cd60d8 100644 --- a/lib/compress/zstd_fast.c +++ b/lib/compress/zstd_fast.c @@ -8,978 +8,88 @@ * You may select, at your option, one of the above-listed licenses. */ -#include "zstd_compress_internal.h" /* ZSTD_hashPtr, ZSTD_count, ZSTD_storeSeq */ +/* The matching algorithms live in rust/src/zstd_fast.rs. Keep match-state + * extraction here: it is a large private structure whose layout should not be + * duplicated in Rust. */ +#include "zstd_compress_internal.h" #include "zstd_fast.h" -static -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -void ZSTD_fillHashTableForCDict(ZSTD_MatchState_t* ms, - const void* const end, - ZSTD_dictTableLoadMethod_e dtlm) -{ - const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32* const hashTable = ms->hashTable; - U32 const hBits = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; - U32 const mls = cParams->minMatch; - const BYTE* const base = ms->window.base; - const BYTE* ip = base + ms->nextToUpdate; - const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; - const U32 fastHashFillStep = 3; +typedef char ZSTD_rust_fast_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1]; +typedef char ZSTD_rust_fast_seqstore_long_length_pos[ + (offsetof(SeqStore_t, longLengthPos) == 9 * sizeof(size_t) + 4) ? 1 : -1]; +typedef char ZSTD_rust_fast_seqstore_layout[ + (sizeof(SeqStore_t) == 9 * sizeof(size_t) + 8) ? 1 : -1]; +typedef char ZSTD_rust_fast_rep_count[(ZSTD_REP_NUM == 3) ? 1 : -1]; - /* Currently, we always use ZSTD_dtlm_full for filling CDict tables. - * Feel free to remove this assert if there's a good reason! */ - assert(dtlm == ZSTD_dtlm_full); +void ZSTD_rust_fillHashTable(U32* hashTable, const BYTE* base, U32 nextToUpdate, + const void* end, U32 hashLog, U32 minMatch, + int fullTableLoad, int forCDict); - /* Always insert every fastHashFillStep position into the hash table. - * Insert the other positions if their hash entry is empty. - */ - for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) { - U32 const curr = (U32)(ip - base); - { size_t const hashAndTag = ZSTD_hashPtr(ip, hBits, mls); - ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr); } +size_t ZSTD_rust_compressBlock_fast( + U32* hashTable, const BYTE* base, U32 dictLimit, U32 loadedDictEnd, + U32 hashLog, U32 minMatch, U32 targetLength, U32 windowLog, + void* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize); - if (dtlm == ZSTD_dtlm_fast) continue; - /* Only load extra positions for ZSTD_dtlm_full */ - { U32 p; - for (p = 1; p < fastHashFillStep; ++p) { - size_t const hashAndTag = ZSTD_hashPtr(ip + p, hBits, mls); - if (hashTable[hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { /* not yet filled */ - ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr + p); - } } } } -} +size_t ZSTD_rust_compressBlock_fast_dictMatchState( + U32* hashTable, const BYTE* base, U32 prefixStartIndex, + U32 hashLog, U32 minMatch, U32 targetLength, + void* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize, + const U32* dictHashTable, const BYTE* dictBase, U32 dictStartIndex, + const BYTE* dictEnd, U32 dictHashLog, int prefetchCDictTables); -static -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -void ZSTD_fillHashTableForCCtx(ZSTD_MatchState_t* ms, - const void* const end, - ZSTD_dictTableLoadMethod_e dtlm) -{ - const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32* const hashTable = ms->hashTable; - U32 const hBits = cParams->hashLog; - U32 const mls = cParams->minMatch; - const BYTE* const base = ms->window.base; - const BYTE* ip = base + ms->nextToUpdate; - const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; - const U32 fastHashFillStep = 3; - - /* Currently, we always use ZSTD_dtlm_fast for filling CCtx tables. - * Feel free to remove this assert if there's a good reason! */ - assert(dtlm == ZSTD_dtlm_fast); - - /* Always insert every fastHashFillStep position into the hash table. - * Insert the other positions if their hash entry is empty. - */ - for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) { - U32 const curr = (U32)(ip - base); - size_t const hash0 = ZSTD_hashPtr(ip, hBits, mls); - hashTable[hash0] = curr; - if (dtlm == ZSTD_dtlm_fast) continue; - /* Only load extra positions for ZSTD_dtlm_full */ - { U32 p; - for (p = 1; p < fastHashFillStep; ++p) { - size_t const hash = ZSTD_hashPtr(ip + p, hBits, mls); - if (hashTable[hash] == 0) { /* not yet filled */ - hashTable[hash] = curr + p; - } } } } -} +size_t ZSTD_rust_compressBlock_fast_extDict( + U32* hashTable, const BYTE* base, const BYTE* dictBase, + U32 dictLimit, U32 lowLimit, U32 loadedDictEnd, + U32 hashLog, U32 minMatch, U32 targetLength, U32 windowLog, + void* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize); void ZSTD_fillHashTable(ZSTD_MatchState_t* ms, - const void* const end, + const void* end, ZSTD_dictTableLoadMethod_e dtlm, ZSTD_tableFillPurpose_e tfp) { - if (tfp == ZSTD_tfp_forCDict) { - ZSTD_fillHashTableForCDict(ms, end, dtlm); - } else { - ZSTD_fillHashTableForCCtx(ms, end, dtlm); - } + assert((tfp == ZSTD_tfp_forCDict && dtlm == ZSTD_dtlm_full) + || (tfp != ZSTD_tfp_forCDict && dtlm == ZSTD_dtlm_fast)); + ZSTD_rust_fillHashTable(ms->hashTable, ms->window.base, ms->nextToUpdate, + end, ms->cParams.hashLog, ms->cParams.minMatch, + dtlm == ZSTD_dtlm_full, tfp == ZSTD_tfp_forCDict); } - -typedef int (*ZSTD_match4Found) (const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit); - -static int -ZSTD_match4Found_cmov(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) -{ - /* Array of ~random data, should have low probability of matching data. - * Load from here if the index is invalid. - * Used to avoid unpredictable branches. */ - static const BYTE dummy[] = {0x12,0x34,0x56,0x78}; - - /* currentIdx >= lowLimit is a (somewhat) unpredictable branch. - * However expression below compiles into conditional move. - */ - const BYTE* mvalAddr = ZSTD_selectAddr(matchIdx, idxLowLimit, matchAddress, dummy); - /* Note: this used to be written as : return test1 && test2; - * Unfortunately, once inlined, these tests become branches, - * in which case it becomes critical that they are executed in the right order (test1 then test2). - * So we have to write these tests in a specific manner to ensure their ordering. - */ - if (MEM_read32(currentPtr) != MEM_read32(mvalAddr)) return 0; - /* force ordering of these tests, which matters once the function is inlined, as they become branches */ -#if defined(__GNUC__) - __asm__(""); -#endif - return matchIdx >= idxLowLimit; -} - -static int -ZSTD_match4Found_branch(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit) -{ - /* using a branch instead of a cmov, - * because it's faster in scenarios where matchIdx >= idxLowLimit is generally true, - * aka almost all candidates are within range */ - U32 mval; - if (matchIdx >= idxLowLimit) { - mval = MEM_read32(matchAddress); - } else { - mval = MEM_read32(currentPtr) ^ 1; /* guaranteed to not match. */ - } - - return (MEM_read32(currentPtr) == mval); -} - - -/** - * If you squint hard enough (and ignore repcodes), the search operation at any - * given position is broken into 4 stages: - * - * 1. Hash (map position to hash value via input read) - * 2. Lookup (map hash val to index via hashtable read) - * 3. Load (map index to value at that position via input read) - * 4. Compare - * - * Each of these steps involves a memory read at an address which is computed - * from the previous step. This means these steps must be sequenced and their - * latencies are cumulative. - * - * Rather than do 1->2->3->4 sequentially for a single position before moving - * onto the next, this implementation interleaves these operations across the - * next few positions: - * - * R = Repcode Read & Compare - * H = Hash - * T = Table Lookup - * M = Match Read & Compare - * - * Pos | Time --> - * ----+------------------- - * N | ... M - * N+1 | ... TM - * N+2 | R H T M - * N+3 | H TM - * N+4 | R H T M - * N+5 | H ... - * N+6 | R ... - * - * This is very much analogous to the pipelining of execution in a CPU. And just - * like a CPU, we have to dump the pipeline when we find a match (i.e., take a - * branch). - * - * When this happens, we throw away our current state, and do the following prep - * to re-enter the loop: - * - * Pos | Time --> - * ----+------------------- - * N | H T - * N+1 | H - * - * This is also the work we do at the beginning to enter the loop initially. - */ -FORCE_INLINE_TEMPLATE -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -size_t ZSTD_compressBlock_fast_noDict_generic( - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize, - U32 const mls, int useCmov) -{ - const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32* const hashTable = ms->hashTable; - U32 const hlog = cParams->hashLog; - size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1; /* min 2 */ - const BYTE* const base = ms->window.base; - const BYTE* const istart = (const BYTE*)src; - const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); - const U32 prefixStartIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); - const BYTE* const prefixStart = base + prefixStartIndex; - const BYTE* const iend = istart + srcSize; - const BYTE* const ilimit = iend - HASH_READ_SIZE; - - const BYTE* anchor = istart; - const BYTE* ip0 = istart; - const BYTE* ip1; - const BYTE* ip2; - const BYTE* ip3; - U32 current0; - - U32 rep_offset1 = rep[0]; - U32 rep_offset2 = rep[1]; - U32 offsetSaved1 = 0, offsetSaved2 = 0; - - size_t hash0; /* hash for ip0 */ - size_t hash1; /* hash for ip1 */ - U32 matchIdx; /* match idx for ip0 */ - - U32 offcode; - const BYTE* match0; - size_t mLength; - - /* ip0 and ip1 are always adjacent. The targetLength skipping and - * uncompressibility acceleration is applied to every other position, - * matching the behavior of #1562. step therefore represents the gap - * between pairs of positions, from ip0 to ip2 or ip1 to ip3. */ - size_t step; - const BYTE* nextStep; - const size_t kStepIncr = (1 << (kSearchStrength - 1)); - const ZSTD_match4Found matchFound = useCmov ? ZSTD_match4Found_cmov : ZSTD_match4Found_branch; - - DEBUGLOG(5, "ZSTD_compressBlock_fast_generic"); - ip0 += (ip0 == prefixStart); - { U32 const curr = (U32)(ip0 - base); - U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog); - U32 const maxRep = curr - windowLow; - if (rep_offset2 > maxRep) offsetSaved2 = rep_offset2, rep_offset2 = 0; - if (rep_offset1 > maxRep) offsetSaved1 = rep_offset1, rep_offset1 = 0; - } - - /* start each op */ -_start: /* Requires: ip0 */ - - step = stepSize; - nextStep = ip0 + kStepIncr; - - /* calculate positions, ip0 - anchor == 0, so we skip step calc */ - ip1 = ip0 + 1; - ip2 = ip0 + step; - ip3 = ip2 + 1; - - if (ip3 >= ilimit) { - goto _cleanup; - } - - hash0 = ZSTD_hashPtr(ip0, hlog, mls); - hash1 = ZSTD_hashPtr(ip1, hlog, mls); - - matchIdx = hashTable[hash0]; - - do { - /* load repcode match for ip[2]*/ - const U32 rval = MEM_read32(ip2 - rep_offset1); - - /* write back hash table entry */ - current0 = (U32)(ip0 - base); - hashTable[hash0] = current0; - - /* check repcode at ip[2] */ - if ((MEM_read32(ip2) == rval) & (rep_offset1 > 0)) { - ip0 = ip2; - match0 = ip0 - rep_offset1; - mLength = ip0[-1] == match0[-1]; - ip0 -= mLength; - match0 -= mLength; - offcode = REPCODE1_TO_OFFBASE; - mLength += 4; - - /* Write next hash table entry: it's already calculated. - * This write is known to be safe because ip1 is before the - * repcode (ip2). */ - hashTable[hash1] = (U32)(ip1 - base); - - goto _match; - } - - if (matchFound(ip0, base + matchIdx, matchIdx, prefixStartIndex)) { - /* Write next hash table entry (it's already calculated). - * This write is known to be safe because the ip1 == ip0 + 1, - * so searching will resume after ip1 */ - hashTable[hash1] = (U32)(ip1 - base); - - goto _offset; - } - - /* lookup ip[1] */ - matchIdx = hashTable[hash1]; - - /* hash ip[2] */ - hash0 = hash1; - hash1 = ZSTD_hashPtr(ip2, hlog, mls); - - /* advance to next positions */ - ip0 = ip1; - ip1 = ip2; - ip2 = ip3; - - /* write back hash table entry */ - current0 = (U32)(ip0 - base); - hashTable[hash0] = current0; - - if (matchFound(ip0, base + matchIdx, matchIdx, prefixStartIndex)) { - /* Write next hash table entry, since it's already calculated */ - if (step <= 4) { - /* Avoid writing an index if it's >= position where search will resume. - * The minimum possible match has length 4, so search can resume at ip0 + 4. - */ - hashTable[hash1] = (U32)(ip1 - base); - } - goto _offset; - } - - /* lookup ip[1] */ - matchIdx = hashTable[hash1]; - - /* hash ip[2] */ - hash0 = hash1; - hash1 = ZSTD_hashPtr(ip2, hlog, mls); - - /* advance to next positions */ - ip0 = ip1; - ip1 = ip2; - ip2 = ip0 + step; - ip3 = ip1 + step; - - /* calculate step */ - if (ip2 >= nextStep) { - step++; - PREFETCH_L1(ip1 + 64); - PREFETCH_L1(ip1 + 128); - nextStep += kStepIncr; - } - } while (ip3 < ilimit); - -_cleanup: - /* Note that there are probably still a couple positions one could search. - * However, it seems to be a meaningful performance hit to try to search - * them. So let's not. */ - - /* When the repcodes are outside of the prefix, we set them to zero before the loop. - * When the offsets are still zero, we need to restore them after the block to have a correct - * repcode history. If only one offset was invalid, it is easy. The tricky case is when both - * offsets were invalid. We need to figure out which offset to refill with. - * - If both offsets are zero they are in the same order. - * - If both offsets are non-zero, we won't restore the offsets from `offsetSaved[12]`. - * - If only one is zero, we need to decide which offset to restore. - * - If rep_offset1 is non-zero, then rep_offset2 must be offsetSaved1. - * - It is impossible for rep_offset2 to be non-zero. - * - * So if rep_offset1 started invalid (offsetSaved1 != 0) and became valid (rep_offset1 != 0), then - * set rep[0] = rep_offset1 and rep[1] = offsetSaved1. - */ - offsetSaved2 = ((offsetSaved1 != 0) && (rep_offset1 != 0)) ? offsetSaved1 : offsetSaved2; - - /* save reps for next block */ - rep[0] = rep_offset1 ? rep_offset1 : offsetSaved1; - rep[1] = rep_offset2 ? rep_offset2 : offsetSaved2; - - /* Return the last literals size */ - return (size_t)(iend - anchor); - -_offset: /* Requires: ip0, idx */ - - /* Compute the offset code. */ - match0 = base + matchIdx; - rep_offset2 = rep_offset1; - rep_offset1 = (U32)(ip0-match0); - offcode = OFFSET_TO_OFFBASE(rep_offset1); - mLength = 4; - - /* Count the backwards match length. */ - while (((ip0>anchor) & (match0>prefixStart)) && (ip0[-1] == match0[-1])) { - ip0--; - match0--; - mLength++; - } - -_match: /* Requires: ip0, match0, offcode */ - - /* Count the forward length. */ - mLength += ZSTD_count(ip0 + mLength, match0 + mLength, iend); - - ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength); - - ip0 += mLength; - anchor = ip0; - - /* Fill table and check for immediate repcode. */ - if (ip0 <= ilimit) { - /* Fill Table */ - assert(base+current0+2 > istart); /* check base overflow */ - hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ - hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); - - if (rep_offset2 > 0) { /* rep_offset2==0 means rep_offset2 is invalidated */ - while ( (ip0 <= ilimit) && (MEM_read32(ip0) == MEM_read32(ip0 - rep_offset2)) ) { - /* store sequence */ - size_t const rLength = ZSTD_count(ip0+4, ip0+4-rep_offset2, iend) + 4; - { U32 const tmpOff = rep_offset2; rep_offset2 = rep_offset1; rep_offset1 = tmpOff; } /* swap rep_offset2 <=> rep_offset1 */ - hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); - ip0 += rLength; - ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, REPCODE1_TO_OFFBASE, rLength); - anchor = ip0; - continue; /* faster when present (confirmed on gcc-8) ... (?) */ - } } } - - goto _start; -} - -#define ZSTD_GEN_FAST_FN(dictMode, mml, cmov) \ - static size_t ZSTD_compressBlock_fast_##dictMode##_##mml##_##cmov( \ - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ - void const* src, size_t srcSize) \ - { \ - return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mml, cmov); \ - } - -ZSTD_GEN_FAST_FN(noDict, 4, 1) -ZSTD_GEN_FAST_FN(noDict, 5, 1) -ZSTD_GEN_FAST_FN(noDict, 6, 1) -ZSTD_GEN_FAST_FN(noDict, 7, 1) - -ZSTD_GEN_FAST_FN(noDict, 4, 0) -ZSTD_GEN_FAST_FN(noDict, 5, 0) -ZSTD_GEN_FAST_FN(noDict, 6, 0) -ZSTD_GEN_FAST_FN(noDict, 7, 0) - size_t ZSTD_compressBlock_fast( ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) + const void* src, size_t srcSize) { - U32 const mml = ms->cParams.minMatch; - /* use cmov when "candidate in range" branch is likely unpredictable */ - int const useCmov = ms->cParams.windowLog < 19; assert(ms->dictMatchState == NULL); - if (useCmov) { - switch(mml) - { - default: /* includes case 3 */ - case 4 : - return ZSTD_compressBlock_fast_noDict_4_1(ms, seqStore, rep, src, srcSize); - case 5 : - return ZSTD_compressBlock_fast_noDict_5_1(ms, seqStore, rep, src, srcSize); - case 6 : - return ZSTD_compressBlock_fast_noDict_6_1(ms, seqStore, rep, src, srcSize); - case 7 : - return ZSTD_compressBlock_fast_noDict_7_1(ms, seqStore, rep, src, srcSize); - } - } else { - /* use a branch instead */ - switch(mml) - { - default: /* includes case 3 */ - case 4 : - return ZSTD_compressBlock_fast_noDict_4_0(ms, seqStore, rep, src, srcSize); - case 5 : - return ZSTD_compressBlock_fast_noDict_5_0(ms, seqStore, rep, src, srcSize); - case 6 : - return ZSTD_compressBlock_fast_noDict_6_0(ms, seqStore, rep, src, srcSize); - case 7 : - return ZSTD_compressBlock_fast_noDict_7_0(ms, seqStore, rep, src, srcSize); - } - } + return ZSTD_rust_compressBlock_fast( + ms->hashTable, ms->window.base, ms->window.dictLimit, ms->loadedDictEnd, + ms->cParams.hashLog, ms->cParams.minMatch, + ms->cParams.targetLength, ms->cParams.windowLog, + seqStore, rep, src, srcSize); } -FORCE_INLINE_TEMPLATE -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -size_t ZSTD_compressBlock_fast_dictMatchState_generic( - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize, U32 const mls, U32 const hasStep) -{ - const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32* const hashTable = ms->hashTable; - U32 const hlog = cParams->hashLog; - /* support stepSize of 0 */ - U32 const stepSize = cParams->targetLength + !(cParams->targetLength); - const BYTE* const base = ms->window.base; - const BYTE* const istart = (const BYTE*)src; - const BYTE* ip0 = istart; - const BYTE* ip1 = ip0 + stepSize; /* we assert below that stepSize >= 1 */ - const BYTE* anchor = istart; - const U32 prefixStartIndex = ms->window.dictLimit; - const BYTE* const prefixStart = base + prefixStartIndex; - const BYTE* const iend = istart + srcSize; - const BYTE* const ilimit = iend - HASH_READ_SIZE; - U32 offset_1=rep[0], offset_2=rep[1]; - - const ZSTD_MatchState_t* const dms = ms->dictMatchState; - const ZSTD_compressionParameters* const dictCParams = &dms->cParams ; - const U32* const dictHashTable = dms->hashTable; - const U32 dictStartIndex = dms->window.dictLimit; - const BYTE* const dictBase = dms->window.base; - const BYTE* const dictStart = dictBase + dictStartIndex; - const BYTE* const dictEnd = dms->window.nextSrc; - const U32 dictIndexDelta = prefixStartIndex - (U32)(dictEnd - dictBase); - const U32 dictAndPrefixLength = (U32)(istart - prefixStart + dictEnd - dictStart); - const U32 dictHBits = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; - - /* if a dictionary is still attached, it necessarily means that - * it is within window size. So we just check it. */ - const U32 maxDistance = 1U << cParams->windowLog; - const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); - assert(endIndex - prefixStartIndex <= maxDistance); - (void)maxDistance; (void)endIndex; /* these variables are not used when assert() is disabled */ - - (void)hasStep; /* not currently specialized on whether it's accelerated */ - - /* ensure there will be no underflow - * when translating a dict index into a local index */ - assert(prefixStartIndex >= (U32)(dictEnd - dictBase)); - - if (ms->prefetchCDictTables) { - size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32); - PREFETCH_AREA(dictHashTable, hashTableBytes); - } - - /* init */ - DEBUGLOG(5, "ZSTD_compressBlock_fast_dictMatchState_generic"); - ip0 += (dictAndPrefixLength == 0); - /* dictMatchState repCode checks don't currently handle repCode == 0 - * disabling. */ - assert(offset_1 <= dictAndPrefixLength); - assert(offset_2 <= dictAndPrefixLength); - - /* Outer search loop */ - assert(stepSize >= 1); - while (ip1 <= ilimit) { /* repcode check at (ip0 + 1) is safe because ip0 < ip1 */ - size_t mLength; - size_t hash0 = ZSTD_hashPtr(ip0, hlog, mls); - - size_t const dictHashAndTag0 = ZSTD_hashPtr(ip0, dictHBits, mls); - U32 dictMatchIndexAndTag = dictHashTable[dictHashAndTag0 >> ZSTD_SHORT_CACHE_TAG_BITS]; - int dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag0); - - U32 matchIndex = hashTable[hash0]; - U32 curr = (U32)(ip0 - base); - size_t step = stepSize; - const size_t kStepIncr = 1 << kSearchStrength; - const BYTE* nextStep = ip0 + kStepIncr; - - /* Inner search loop */ - while (1) { - const BYTE* match = base + matchIndex; - const U32 repIndex = curr + 1 - offset_1; - const BYTE* repMatch = (repIndex < prefixStartIndex) ? - dictBase + (repIndex - dictIndexDelta) : - base + repIndex; - const size_t hash1 = ZSTD_hashPtr(ip1, hlog, mls); - size_t const dictHashAndTag1 = ZSTD_hashPtr(ip1, dictHBits, mls); - hashTable[hash0] = curr; /* update hash table */ - - if ((ZSTD_index_overlap_check(prefixStartIndex, repIndex)) - && (MEM_read32(repMatch) == MEM_read32(ip0 + 1))) { - const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip0 + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixStart) + 4; - ip0++; - ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); - break; - } - - if (dictTagsMatch) { - /* Found a possible dict match */ - const U32 dictMatchIndex = dictMatchIndexAndTag >> ZSTD_SHORT_CACHE_TAG_BITS; - const BYTE* dictMatch = dictBase + dictMatchIndex; - if (dictMatchIndex > dictStartIndex && - MEM_read32(dictMatch) == MEM_read32(ip0)) { - /* To replicate extDict parse behavior, we only use dict matches when the normal matchIndex is invalid */ - if (matchIndex <= prefixStartIndex) { - U32 const offset = (U32) (curr - dictMatchIndex - dictIndexDelta); - mLength = ZSTD_count_2segments(ip0 + 4, dictMatch + 4, iend, dictEnd, prefixStart) + 4; - while (((ip0 > anchor) & (dictMatch > dictStart)) - && (ip0[-1] == dictMatch[-1])) { - ip0--; - dictMatch--; - mLength++; - } /* catch up */ - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); - break; - } - } - } - - if (ZSTD_match4Found_cmov(ip0, match, matchIndex, prefixStartIndex)) { - /* found a regular match of size >= 4 */ - U32 const offset = (U32) (ip0 - match); - mLength = ZSTD_count(ip0 + 4, match + 4, iend) + 4; - while (((ip0 > anchor) & (match > prefixStart)) - && (ip0[-1] == match[-1])) { - ip0--; - match--; - mLength++; - } /* catch up */ - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); - break; - } - - /* Prepare for next iteration */ - dictMatchIndexAndTag = dictHashTable[dictHashAndTag1 >> ZSTD_SHORT_CACHE_TAG_BITS]; - dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag1); - matchIndex = hashTable[hash1]; - - if (ip1 >= nextStep) { - step++; - nextStep += kStepIncr; - } - ip0 = ip1; - ip1 = ip1 + step; - if (ip1 > ilimit) goto _cleanup; - - curr = (U32)(ip0 - base); - hash0 = hash1; - } /* end inner search loop */ - - /* match found */ - assert(mLength); - ip0 += mLength; - anchor = ip0; - - if (ip0 <= ilimit) { - /* Fill Table */ - assert(base+curr+2 > istart); /* check base overflow */ - hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; /* here because curr+2 could be > iend-8 */ - hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); - - /* check immediate repcode */ - while (ip0 <= ilimit) { - U32 const current2 = (U32)(ip0-base); - U32 const repIndex2 = current2 - offset_2; - const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? - dictBase - dictIndexDelta + repIndex2 : - base + repIndex2; - if ( (ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) - && (MEM_read32(repMatch2) == MEM_read32(ip0))) { - const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; - U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); - hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = current2; - ip0 += repLength2; - anchor = ip0; - continue; - } - break; - } - } - - /* Prepare for next iteration */ - assert(ip0 == anchor); - ip1 = ip0 + stepSize; - } - -_cleanup: - /* save reps for next block */ - rep[0] = offset_1; - rep[1] = offset_2; - - /* Return the last literals size */ - return (size_t)(iend - anchor); -} - - -ZSTD_GEN_FAST_FN(dictMatchState, 4, 0) -ZSTD_GEN_FAST_FN(dictMatchState, 5, 0) -ZSTD_GEN_FAST_FN(dictMatchState, 6, 0) -ZSTD_GEN_FAST_FN(dictMatchState, 7, 0) - size_t ZSTD_compressBlock_fast_dictMatchState( ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) + const void* src, size_t srcSize) { - U32 const mls = ms->cParams.minMatch; - assert(ms->dictMatchState != NULL); - switch(mls) - { - default: /* includes case 3 */ - case 4 : - return ZSTD_compressBlock_fast_dictMatchState_4_0(ms, seqStore, rep, src, srcSize); - case 5 : - return ZSTD_compressBlock_fast_dictMatchState_5_0(ms, seqStore, rep, src, srcSize); - case 6 : - return ZSTD_compressBlock_fast_dictMatchState_6_0(ms, seqStore, rep, src, srcSize); - case 7 : - return ZSTD_compressBlock_fast_dictMatchState_7_0(ms, seqStore, rep, src, srcSize); - } + const ZSTD_MatchState_t* const dms = ms->dictMatchState; + assert(dms != NULL); + return ZSTD_rust_compressBlock_fast_dictMatchState( + ms->hashTable, ms->window.base, ms->window.dictLimit, + ms->cParams.hashLog, ms->cParams.minMatch, ms->cParams.targetLength, + seqStore, rep, src, srcSize, + dms->hashTable, dms->window.base, dms->window.dictLimit, + dms->window.nextSrc, dms->cParams.hashLog, ms->prefetchCDictTables); } - -static -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -size_t ZSTD_compressBlock_fast_extDict_generic( - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize, U32 const mls, U32 const hasStep) -{ - const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32* const hashTable = ms->hashTable; - U32 const hlog = cParams->hashLog; - /* support stepSize of 0 */ - size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1; - const BYTE* const base = ms->window.base; - const BYTE* const dictBase = ms->window.dictBase; - const BYTE* const istart = (const BYTE*)src; - const BYTE* anchor = istart; - const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); - const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); - const U32 dictStartIndex = lowLimit; - const BYTE* const dictStart = dictBase + dictStartIndex; - const U32 dictLimit = ms->window.dictLimit; - const U32 prefixStartIndex = dictLimit < lowLimit ? lowLimit : dictLimit; - const BYTE* const prefixStart = base + prefixStartIndex; - const BYTE* const dictEnd = dictBase + prefixStartIndex; - const BYTE* const iend = istart + srcSize; - const BYTE* const ilimit = iend - 8; - U32 offset_1=rep[0], offset_2=rep[1]; - U32 offsetSaved1 = 0, offsetSaved2 = 0; - - const BYTE* ip0 = istart; - const BYTE* ip1; - const BYTE* ip2; - const BYTE* ip3; - U32 current0; - - - size_t hash0; /* hash for ip0 */ - size_t hash1; /* hash for ip1 */ - U32 idx; /* match idx for ip0 */ - const BYTE* idxBase; /* base pointer for idx */ - - U32 offcode; - const BYTE* match0; - size_t mLength; - const BYTE* matchEnd = 0; /* initialize to avoid warning, assert != 0 later */ - - size_t step; - const BYTE* nextStep; - const size_t kStepIncr = (1 << (kSearchStrength - 1)); - - (void)hasStep; /* not currently specialized on whether it's accelerated */ - - DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic (offset_1=%u)", offset_1); - - /* switch to "regular" variant if extDict is invalidated due to maxDistance */ - if (prefixStartIndex == dictStartIndex) - return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize); - - { U32 const curr = (U32)(ip0 - base); - U32 const maxRep = curr - dictStartIndex; - if (offset_2 >= maxRep) offsetSaved2 = offset_2, offset_2 = 0; - if (offset_1 >= maxRep) offsetSaved1 = offset_1, offset_1 = 0; - } - - /* start each op */ -_start: /* Requires: ip0 */ - - step = stepSize; - nextStep = ip0 + kStepIncr; - - /* calculate positions, ip0 - anchor == 0, so we skip step calc */ - ip1 = ip0 + 1; - ip2 = ip0 + step; - ip3 = ip2 + 1; - - if (ip3 >= ilimit) { - goto _cleanup; - } - - hash0 = ZSTD_hashPtr(ip0, hlog, mls); - hash1 = ZSTD_hashPtr(ip1, hlog, mls); - - idx = hashTable[hash0]; - idxBase = idx < prefixStartIndex ? dictBase : base; - - do { - { /* load repcode match for ip[2] */ - U32 const current2 = (U32)(ip2 - base); - U32 const repIndex = current2 - offset_1; - const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; - U32 rval; - if ( ((U32)(prefixStartIndex - repIndex) >= 4) /* intentional underflow */ - & (offset_1 > 0) ) { - rval = MEM_read32(repBase + repIndex); - } else { - rval = MEM_read32(ip2) ^ 1; /* guaranteed to not match. */ - } - - /* write back hash table entry */ - current0 = (U32)(ip0 - base); - hashTable[hash0] = current0; - - /* check repcode at ip[2] */ - if (MEM_read32(ip2) == rval) { - ip0 = ip2; - match0 = repBase + repIndex; - matchEnd = repIndex < prefixStartIndex ? dictEnd : iend; - assert((match0 != prefixStart) & (match0 != dictStart)); - mLength = ip0[-1] == match0[-1]; - ip0 -= mLength; - match0 -= mLength; - offcode = REPCODE1_TO_OFFBASE; - mLength += 4; - goto _match; - } } - - { /* load match for ip[0] */ - U32 const mval = idx >= dictStartIndex ? - MEM_read32(idxBase + idx) : - MEM_read32(ip0) ^ 1; /* guaranteed not to match */ - - /* check match at ip[0] */ - if (MEM_read32(ip0) == mval) { - /* found a match! */ - goto _offset; - } } - - /* lookup ip[1] */ - idx = hashTable[hash1]; - idxBase = idx < prefixStartIndex ? dictBase : base; - - /* hash ip[2] */ - hash0 = hash1; - hash1 = ZSTD_hashPtr(ip2, hlog, mls); - - /* advance to next positions */ - ip0 = ip1; - ip1 = ip2; - ip2 = ip3; - - /* write back hash table entry */ - current0 = (U32)(ip0 - base); - hashTable[hash0] = current0; - - { /* load match for ip[0] */ - U32 const mval = idx >= dictStartIndex ? - MEM_read32(idxBase + idx) : - MEM_read32(ip0) ^ 1; /* guaranteed not to match */ - - /* check match at ip[0] */ - if (MEM_read32(ip0) == mval) { - /* found a match! */ - goto _offset; - } } - - /* lookup ip[1] */ - idx = hashTable[hash1]; - idxBase = idx < prefixStartIndex ? dictBase : base; - - /* hash ip[2] */ - hash0 = hash1; - hash1 = ZSTD_hashPtr(ip2, hlog, mls); - - /* advance to next positions */ - ip0 = ip1; - ip1 = ip2; - ip2 = ip0 + step; - ip3 = ip1 + step; - - /* calculate step */ - if (ip2 >= nextStep) { - step++; - PREFETCH_L1(ip1 + 64); - PREFETCH_L1(ip1 + 128); - nextStep += kStepIncr; - } - } while (ip3 < ilimit); - -_cleanup: - /* Note that there are probably still a couple positions we could search. - * However, it seems to be a meaningful performance hit to try to search - * them. So let's not. */ - - /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), - * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ - offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; - - /* save reps for next block */ - rep[0] = offset_1 ? offset_1 : offsetSaved1; - rep[1] = offset_2 ? offset_2 : offsetSaved2; - - /* Return the last literals size */ - return (size_t)(iend - anchor); - -_offset: /* Requires: ip0, idx, idxBase */ - - /* Compute the offset code. */ - { U32 const offset = current0 - idx; - const BYTE* const lowMatchPtr = idx < prefixStartIndex ? dictStart : prefixStart; - matchEnd = idx < prefixStartIndex ? dictEnd : iend; - match0 = idxBase + idx; - offset_2 = offset_1; - offset_1 = offset; - offcode = OFFSET_TO_OFFBASE(offset); - mLength = 4; - - /* Count the backwards match length. */ - while (((ip0>anchor) & (match0>lowMatchPtr)) && (ip0[-1] == match0[-1])) { - ip0--; - match0--; - mLength++; - } } - -_match: /* Requires: ip0, match0, offcode, matchEnd */ - - /* Count the forward length. */ - assert(matchEnd != 0); - mLength += ZSTD_count_2segments(ip0 + mLength, match0 + mLength, iend, matchEnd, prefixStart); - - ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength); - - ip0 += mLength; - anchor = ip0; - - /* write next hash table entry */ - if (ip1 < ip0) { - hashTable[hash1] = (U32)(ip1 - base); - } - - /* Fill table and check for immediate repcode. */ - if (ip0 <= ilimit) { - /* Fill Table */ - assert(base+current0+2 > istart); /* check base overflow */ - hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ - hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); - - while (ip0 <= ilimit) { - U32 const repIndex2 = (U32)(ip0-base) - offset_2; - const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; - if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) & (offset_2 > 0)) - && (MEM_read32(repMatch2) == MEM_read32(ip0)) ) { - const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; - { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); - hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); - ip0 += repLength2; - anchor = ip0; - continue; - } - break; - } } - - goto _start; -} - -ZSTD_GEN_FAST_FN(extDict, 4, 0) -ZSTD_GEN_FAST_FN(extDict, 5, 0) -ZSTD_GEN_FAST_FN(extDict, 6, 0) -ZSTD_GEN_FAST_FN(extDict, 7, 0) - size_t ZSTD_compressBlock_fast_extDict( ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - void const* src, size_t srcSize) + const void* src, size_t srcSize) { - U32 const mls = ms->cParams.minMatch; assert(ms->dictMatchState == NULL); - switch(mls) - { - default: /* includes case 3 */ - case 4 : - return ZSTD_compressBlock_fast_extDict_4_0(ms, seqStore, rep, src, srcSize); - case 5 : - return ZSTD_compressBlock_fast_extDict_5_0(ms, seqStore, rep, src, srcSize); - case 6 : - return ZSTD_compressBlock_fast_extDict_6_0(ms, seqStore, rep, src, srcSize); - case 7 : - return ZSTD_compressBlock_fast_extDict_7_0(ms, seqStore, rep, src, srcSize); - } + return ZSTD_rust_compressBlock_fast_extDict( + ms->hashTable, ms->window.base, ms->window.dictBase, + ms->window.dictLimit, ms->window.lowLimit, ms->loadedDictEnd, + ms->cParams.hashLog, ms->cParams.minMatch, + ms->cParams.targetLength, ms->cParams.windowLog, + seqStore, rep, src, srcSize); } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 75df03764..bc7d8909d 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -30,4 +30,6 @@ pub mod zstd_compress_superblock; #[cfg(feature = "decompression")] pub mod zstd_ddict; #[cfg(feature = "compression")] +pub mod zstd_fast; +#[cfg(feature = "compression")] pub mod zstd_presplit; diff --git a/rust/src/zstd_fast.rs b/rust/src/zstd_fast.rs new file mode 100644 index 000000000..013889a30 --- /dev/null +++ b/rust/src/zstd_fast.rs @@ -0,0 +1,1599 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(clippy::missing_safety_doc)] +#![allow(clippy::too_many_arguments)] + +//! Fast block match finder. +//! +//! The C translation unit keeps the public/internal ABI entry points and +//! extracts the fields it needs from `ZSTD_MatchState_t`. This module owns +//! the hash-table search and sequence-generation algorithms themselves. That +//! boundary keeps the large, evolving match-state opaque while retaining the +//! hot matching loops in Rust. + +use crate::mem::{ + MEM_64bits, MEM_isLittleEndian, MEM_read16, MEM_read32, MEM_readLE32, MEM_readLE64, MEM_readST, +}; +use std::ffi::c_void; +use std::mem::size_of; +use std::os::raw::c_int; +use std::ptr; + +const ZSTD_REP_NUM: usize = 3; +const MINMATCH: usize = 3; +const HASH_READ_SIZE: usize = 8; +const SHORT_CACHE_TAG_BITS: u32 = 8; +const SHORT_CACHE_TAG_MASK: u32 = (1 << SHORT_CACHE_TAG_BITS) - 1; +const K_SEARCH_STRENGTH: usize = 8; +const REPCODE1_TO_OFFBASE: u32 = 1; + +#[repr(C)] +#[derive(Clone, Copy)] +struct SeqDef { + offBase: u32, + litLength: u16, + mlBase: u16, +} + +/// The only leaf layout written by the matcher. The C shim verifies this +/// shape at compile time; `ZSTD_MatchState_t` itself never crosses the FFI. +#[repr(C)] +struct SeqStore_t { + sequencesStart: *mut SeqDef, + sequences: *mut SeqDef, + litStart: *mut u8, + lit: *mut u8, + llCode: *mut u8, + mlCode: *mut u8, + ofCode: *mut u8, + maxNbSeq: usize, + maxNbLit: usize, + longLengthType: c_int, + longLengthPos: u32, +} + +#[inline] +fn ptr_lt(left: *const u8, right: *const u8) -> bool { + (left as usize) < (right as usize) +} + +#[inline] +fn ptr_le(left: *const u8, right: *const u8) -> bool { + (left as usize) <= (right as usize) +} + +#[inline] +fn ptr_ge(left: *const u8, right: *const u8) -> bool { + (left as usize) >= (right as usize) +} + +#[inline] +fn ptr_gt(left: *const u8, right: *const u8) -> bool { + (left as usize) > (right as usize) +} + +#[inline] +unsafe fn index_from(base: *const u8, ptr: *const u8) -> u32 { + unsafe { ptr.offset_from(base) as u32 } +} + +#[inline] +unsafe fn read32(ptr: *const u8) -> u32 { + unsafe { MEM_read32(ptr.cast::()) } +} + +#[inline] +unsafe fn table_get(table: *const u32, index: usize) -> u32 { + unsafe { *table.add(index) } +} + +#[inline] +unsafe fn table_set(table: *mut u32, index: usize, value: u32) { + unsafe { *table.add(index) = value }; +} + +#[inline] +fn hash_shift32(value: u32, hbits: u32) -> usize { + if hbits == 0 { + 0 + } else { + (value >> (32 - hbits)) as usize + } +} + +#[inline] +fn hash_shift64(value: u64, hbits: u32) -> usize { + if hbits == 0 { + 0 + } else { + (value >> (64 - hbits)) as usize + } +} + +#[inline] +unsafe fn hash_ptr(ptr: *const u8, hbits: u32, mls: u32) -> usize { + match mls { + 5 => { + let value = unsafe { MEM_readLE64(ptr.cast::()) }; + hash_shift64(value.wrapping_shl(24).wrapping_mul(889_523_592_379), hbits) + } + 6 => { + let value = unsafe { MEM_readLE64(ptr.cast::()) }; + hash_shift64( + value.wrapping_shl(16).wrapping_mul(227_718_039_650_203), + hbits, + ) + } + 7 => { + let value = unsafe { MEM_readLE64(ptr.cast::()) }; + hash_shift64( + value.wrapping_shl(8).wrapping_mul(58_295_818_150_454_627), + hbits, + ) + } + 8 => { + let value = unsafe { MEM_readLE64(ptr.cast::()) }; + hash_shift64(value.wrapping_mul(0xCF1B_BCDC_B7A5_6463), hbits) + } + _ => { + let value = unsafe { MEM_readLE32(ptr.cast::()) }; + hash_shift32(value.wrapping_mul(2_654_435_761), hbits) + } + } +} + +#[inline] +fn common_bytes(word: usize) -> usize { + let zeros = if MEM_isLittleEndian() { + word.trailing_zeros() + } else { + word.leading_zeros() + }; + (zeros / 8) as usize +} + +/// Equivalent to C's `ZSTD_count()`, including its word-at-a-time fast path. +unsafe fn count(mut input: *const u8, mut matched: *const u8, input_limit: *const u8) -> usize { + let input_start = input; + let word_size = size_of::(); + + while unsafe { input_limit.offset_from(input) as usize } >= word_size { + let diff = + unsafe { MEM_readST(matched.cast::()) ^ MEM_readST(input.cast::()) }; + if diff != 0 { + return unsafe { input.offset_from(input_start) as usize } + common_bytes(diff); + } + input = input.wrapping_add(word_size); + matched = matched.wrapping_add(word_size); + } + + if MEM_64bits() + && unsafe { input_limit.offset_from(input) as usize } >= 4 + && unsafe { MEM_read32(matched.cast::()) == MEM_read32(input.cast::()) } + { + input = input.wrapping_add(4); + matched = matched.wrapping_add(4); + } + if unsafe { input_limit.offset_from(input) as usize } >= 2 + && unsafe { MEM_read16(matched.cast::()) == MEM_read16(input.cast::()) } + { + input = input.wrapping_add(2); + matched = matched.wrapping_add(2); + } + if ptr_lt(input, input_limit) && unsafe { *matched == *input } { + input = input.wrapping_add(1); + } + unsafe { input.offset_from(input_start) as usize } +} + +unsafe fn count_2segments( + input: *const u8, + matched: *const u8, + input_end: *const u8, + match_end: *const u8, + input_start: *const u8, +) -> usize { + let match_remaining = unsafe { match_end.offset_from(matched) as usize }; + let input_remaining = unsafe { input_end.offset_from(input) as usize }; + let first_end = input.wrapping_add(match_remaining.min(input_remaining)); + let first_count = unsafe { count(input, matched, first_end) }; + if matched.wrapping_add(first_count) != match_end { + return first_count; + } + first_count + unsafe { count(input.wrapping_add(first_count), input_start, input_end) } +} + +#[inline] +fn lowest_prefix_index(dict_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32.wrapping_shl(window_log); + let within_window = if curr.wrapping_sub(dict_limit) > max_distance { + curr.wrapping_sub(max_distance) + } else { + dict_limit + }; + if loaded_dict_end != 0 { + dict_limit + } else { + within_window + } +} + +#[inline] +fn lowest_match_index(low_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 { + let max_distance = 1u32.wrapping_shl(window_log); + let within_window = if curr.wrapping_sub(low_limit) > max_distance { + curr.wrapping_sub(max_distance) + } else { + low_limit + }; + if loaded_dict_end != 0 { + low_limit + } else { + within_window + } +} + +#[inline] +fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool { + prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3 +} + +#[inline] +fn write_tagged_index(table: *mut u32, hash_and_tag: usize, index: u32) { + let hash = hash_and_tag >> SHORT_CACHE_TAG_BITS; + let tag = (hash_and_tag as u32) & SHORT_CACHE_TAG_MASK; + unsafe { table_set(table, hash, (index << SHORT_CACHE_TAG_BITS) | tag) }; +} + +#[inline] +fn packed_tags_match(first: u32, second: usize) -> bool { + (first & SHORT_CACHE_TAG_MASK) == ((second as u32) & SHORT_CACHE_TAG_MASK) +} + +/// Stores a sequence using the C `SeqStore_t` ABI. Copying exactly the +/// literal range is equivalent to C's over-copying wildcopy path for all +/// observable sequence-store bytes and avoids speculative reads in Rust. +unsafe fn store_seq( + seq_store: *mut SeqStore_t, + lit_length: usize, + literals: *const u8, + _lit_limit: *const u8, + off_base: u32, + match_length: usize, +) { + let seq_store = unsafe { &mut *seq_store }; + let sequence = seq_store.sequences; + debug_assert!( + unsafe { sequence.offset_from(seq_store.sequencesStart) as usize } < seq_store.maxNbSeq + ); + debug_assert!(lit_length <= seq_store.maxNbLit); + debug_assert!(match_length >= MINMATCH); + + if lit_length != 0 { + unsafe { ptr::copy_nonoverlapping(literals, seq_store.lit, lit_length) }; + } + seq_store.lit = seq_store.lit.wrapping_add(lit_length); + + let sequence_index = unsafe { sequence.offset_from(seq_store.sequencesStart) as u32 }; + if lit_length > u16::MAX as usize { + debug_assert_eq!(seq_store.longLengthType, 0); + seq_store.longLengthType = 1; + seq_store.longLengthPos = sequence_index; + } + unsafe { (*sequence).litLength = lit_length as u16 }; + unsafe { (*sequence).offBase = off_base }; + + let match_base = match_length - MINMATCH; + if match_base > u16::MAX as usize { + debug_assert_eq!(seq_store.longLengthType, 0); + seq_store.longLengthType = 2; + seq_store.longLengthPos = sequence_index; + } + unsafe { (*sequence).mlBase = match_base as u16 }; + seq_store.sequences = sequence.wrapping_add(1); +} + +#[inline] +unsafe fn match4_found( + current: *const u8, + match_address: *const u8, + match_index: u32, + low_limit: u32, +) -> bool { + match_index >= low_limit && unsafe { read32(current) == read32(match_address) } +} + +unsafe fn finish_no_dict_match( + hash_table: *mut u32, + base: *const u8, + hash_log: u32, + mls: u32, + seq_store: *mut SeqStore_t, + rep_offset1: &mut u32, + rep_offset2: &mut u32, + mut ip0: *const u8, + match0: *const u8, + anchor: *const u8, + iend: *const u8, + ilimit: *const u8, + current0: u32, + offcode: u32, + mut match_length: usize, +) -> (*const u8, *const u8) { + match_length += unsafe { + count( + ip0.wrapping_add(match_length), + match0.wrapping_add(match_length), + iend, + ) + }; + unsafe { + store_seq( + seq_store, + ip0.offset_from(anchor) as usize, + anchor, + iend, + offcode, + match_length, + ) + }; + ip0 = ip0.wrapping_add(match_length); + let mut new_anchor = ip0; + + if ptr_le(ip0, ilimit) { + let table_hash = + unsafe { hash_ptr(base.wrapping_add(current0 as usize + 2), hash_log, mls) }; + unsafe { table_set(hash_table, table_hash, current0.wrapping_add(2)) }; + let table_hash = unsafe { hash_ptr(ip0.wrapping_sub(2), hash_log, mls) }; + unsafe { + table_set( + hash_table, + table_hash, + index_from(base, ip0.wrapping_sub(2)), + ) + }; + + if *rep_offset2 > 0 { + while ptr_le(ip0, ilimit) + && unsafe { read32(ip0) == read32(ip0.wrapping_sub(*rep_offset2 as usize)) } + { + let repeat_length = unsafe { + count( + ip0.wrapping_add(4), + ip0.wrapping_add(4).wrapping_sub(*rep_offset2 as usize), + iend, + ) + 4 + }; + std::mem::swap(rep_offset1, rep_offset2); + let table_hash = unsafe { hash_ptr(ip0, hash_log, mls) }; + unsafe { table_set(hash_table, table_hash, index_from(base, ip0)) }; + unsafe { + store_seq( + seq_store, + 0, + new_anchor, + iend, + REPCODE1_TO_OFFBASE, + repeat_length, + ) + }; + ip0 = ip0.wrapping_add(repeat_length); + new_anchor = ip0; + } + } + } + + (ip0, new_anchor) +} + +unsafe fn compress_block_fast_no_dict( + hash_table: *mut u32, + base: *const u8, + dict_limit: u32, + loaded_dict_end: u32, + hash_log: u32, + target_length: u32, + window_log: u32, + seq_store: *mut SeqStore_t, + reps: *mut u32, + src: *const u8, + src_size: usize, + mls: u32, + _use_cmov: bool, +) -> usize { + if src_size < HASH_READ_SIZE { + return src_size; + } + let istart = src; + let iend = istart.wrapping_add(src_size); + let ilimit = iend.wrapping_sub(HASH_READ_SIZE); + let end_index = unsafe { index_from(base, istart) }.wrapping_add(src_size as u32); + let prefix_start_index = + lowest_prefix_index(dict_limit, loaded_dict_end, end_index, window_log); + let prefix_start = base.wrapping_add(prefix_start_index as usize); + let step_size = target_length as usize + usize::from(target_length == 0) + 1; + let mut ip0 = istart; + let mut anchor = istart; + let mut rep_offset1 = unsafe { *reps }; + let mut rep_offset2 = unsafe { *reps.add(1) }; + let mut offset_saved1 = 0u32; + let mut offset_saved2 = 0u32; + + if ip0 == prefix_start { + ip0 = ip0.wrapping_add(1); + } + let current = unsafe { index_from(base, ip0) }; + let window_low = lowest_prefix_index(dict_limit, loaded_dict_end, current, window_log); + let max_rep = current.wrapping_sub(window_low); + if rep_offset2 > max_rep { + offset_saved2 = rep_offset2; + rep_offset2 = 0; + } + if rep_offset1 > max_rep { + offset_saved1 = rep_offset1; + rep_offset1 = 0; + } + + 'start: loop { + let mut step = step_size; + let mut next_step = ip0.wrapping_add(1 << (K_SEARCH_STRENGTH - 1)); + let mut ip1 = ip0.wrapping_add(1); + let mut ip2 = ip0.wrapping_add(step); + let mut ip3 = ip2.wrapping_add(1); + if ptr_ge(ip3, ilimit) { + break; + } + + let mut hash0 = unsafe { hash_ptr(ip0, hash_log, mls) }; + let mut hash1 = unsafe { hash_ptr(ip1, hash_log, mls) }; + let mut match_index = unsafe { table_get(hash_table, hash0) }; + + loop { + let rval = if rep_offset1 > 0 { + unsafe { read32(ip2.wrapping_sub(rep_offset1 as usize)) } + } else { + unsafe { read32(ip2) } + }; + let mut current0 = unsafe { index_from(base, ip0) }; + unsafe { table_set(hash_table, hash0, current0) }; + + if rep_offset1 > 0 && unsafe { read32(ip2) == rval } { + ip0 = ip2; + let mut match0 = ip0.wrapping_sub(rep_offset1 as usize); + let match_length = + usize::from(unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) }); + ip0 = ip0.wrapping_sub(match_length); + match0 = match0.wrapping_sub(match_length); + unsafe { table_set(hash_table, hash1, index_from(base, ip1)) }; + let (new_ip0, new_anchor) = unsafe { + finish_no_dict_match( + hash_table, + base, + hash_log, + mls, + seq_store, + &mut rep_offset1, + &mut rep_offset2, + ip0, + match0, + anchor, + iend, + ilimit, + current0, + REPCODE1_TO_OFFBASE, + match_length + 4, + ) + }; + ip0 = new_ip0; + anchor = new_anchor; + continue 'start; + } + + if unsafe { + match4_found( + ip0, + base.wrapping_add(match_index as usize), + match_index, + prefix_start_index, + ) + } { + unsafe { table_set(hash_table, hash1, index_from(base, ip1)) }; + let mut match0 = base.wrapping_add(match_index as usize); + rep_offset2 = rep_offset1; + rep_offset1 = unsafe { index_from(match0, ip0) }; + let mut match_length = 4usize; + while ptr_gt(ip0, anchor) + && ptr_gt(match0, prefix_start) + && unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) } + { + ip0 = ip0.wrapping_sub(1); + match0 = match0.wrapping_sub(1); + match_length += 1; + } + let offcode = rep_offset1.wrapping_add(ZSTD_REP_NUM as u32); + let (new_ip0, new_anchor) = unsafe { + finish_no_dict_match( + hash_table, + base, + hash_log, + mls, + seq_store, + &mut rep_offset1, + &mut rep_offset2, + ip0, + match0, + anchor, + iend, + ilimit, + current0, + offcode, + match_length, + ) + }; + ip0 = new_ip0; + anchor = new_anchor; + continue 'start; + } + + match_index = unsafe { table_get(hash_table, hash1) }; + hash0 = hash1; + hash1 = unsafe { hash_ptr(ip2, hash_log, mls) }; + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + current0 = unsafe { index_from(base, ip0) }; + unsafe { table_set(hash_table, hash0, current0) }; + + if unsafe { + match4_found( + ip0, + base.wrapping_add(match_index as usize), + match_index, + prefix_start_index, + ) + } { + if step <= 4 { + unsafe { table_set(hash_table, hash1, index_from(base, ip1)) }; + } + let mut match0 = base.wrapping_add(match_index as usize); + rep_offset2 = rep_offset1; + rep_offset1 = unsafe { index_from(match0, ip0) }; + let mut match_length = 4usize; + while ptr_gt(ip0, anchor) + && ptr_gt(match0, prefix_start) + && unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) } + { + ip0 = ip0.wrapping_sub(1); + match0 = match0.wrapping_sub(1); + match_length += 1; + } + let offcode = rep_offset1.wrapping_add(ZSTD_REP_NUM as u32); + let (new_ip0, new_anchor) = unsafe { + finish_no_dict_match( + hash_table, + base, + hash_log, + mls, + seq_store, + &mut rep_offset1, + &mut rep_offset2, + ip0, + match0, + anchor, + iend, + ilimit, + current0, + offcode, + match_length, + ) + }; + ip0 = new_ip0; + anchor = new_anchor; + continue 'start; + } + + match_index = unsafe { table_get(hash_table, hash1) }; + hash0 = hash1; + hash1 = unsafe { hash_ptr(ip2, hash_log, mls) }; + ip0 = ip1; + ip1 = ip2; + ip2 = ip0.wrapping_add(step); + ip3 = ip1.wrapping_add(step); + if ptr_ge(ip2, next_step) { + step += 1; + next_step = next_step.wrapping_add(1 << (K_SEARCH_STRENGTH - 1)); + } + if ptr_ge(ip3, ilimit) { + break 'start; + } + } + } + + offset_saved2 = if offset_saved1 != 0 && rep_offset1 != 0 { + offset_saved1 + } else { + offset_saved2 + }; + unsafe { + *reps = if rep_offset1 != 0 { + rep_offset1 + } else { + offset_saved1 + }; + *reps.add(1) = if rep_offset2 != 0 { + rep_offset2 + } else { + offset_saved2 + }; + } + unsafe { iend.offset_from(anchor) as usize } +} + +unsafe fn fill_hash_table( + hash_table: *mut u32, + base: *const u8, + next_to_update: u32, + end: *const u8, + hash_log: u32, + min_match: u32, + full_table_load: bool, + tagged_indices: bool, +) { + if unsafe { end.offset_from(base) } < HASH_READ_SIZE as isize { + return; + } + let hbits = hash_log + + if tagged_indices { + SHORT_CACHE_TAG_BITS + } else { + 0 + }; + let mut input = base.wrapping_add(next_to_update as usize); + let input_end = end.wrapping_sub(HASH_READ_SIZE); + + while ptr_lt(input.wrapping_add(3), input_end.wrapping_add(2)) { + let current = unsafe { index_from(base, input) }; + let hash_and_tag = unsafe { hash_ptr(input, hbits, min_match) }; + if tagged_indices { + write_tagged_index(hash_table, hash_and_tag, current); + } else { + unsafe { table_set(hash_table, hash_and_tag, current) }; + } + + if full_table_load { + for position in 1..3usize { + let hash_and_tag = + unsafe { hash_ptr(input.wrapping_add(position), hbits, min_match) }; + let table_index = if tagged_indices { + hash_and_tag >> SHORT_CACHE_TAG_BITS + } else { + hash_and_tag + }; + if unsafe { table_get(hash_table, table_index) } == 0 { + if tagged_indices { + write_tagged_index( + hash_table, + hash_and_tag, + current.wrapping_add(position as u32), + ); + } else { + unsafe { + table_set( + hash_table, + table_index, + current.wrapping_add(position as u32), + ) + }; + } + } + } + } + input = input.wrapping_add(3); + } +} + +#[inline] +fn fast_mls(min_match: u32) -> u32 { + match min_match { + 5..=7 => min_match, + _ => 4, + } +} + +/// Rust implementation called by the C ABI wrapper for `ZSTD_fillHashTable`. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_fillHashTable( + hash_table: *mut u32, + base: *const u8, + next_to_update: u32, + end: *const c_void, + hash_log: u32, + min_match: u32, + full_table_load: c_int, + for_cdict: c_int, +) { + unsafe { + fill_hash_table( + hash_table, + base, + next_to_update, + end.cast::(), + hash_log, + min_match, + full_table_load != 0, + for_cdict != 0, + ) + }; +} + +/// Rust implementation called by the C ABI wrapper for `ZSTD_compressBlock_fast`. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_compressBlock_fast( + hash_table: *mut u32, + base: *const u8, + dict_limit: u32, + loaded_dict_end: u32, + hash_log: u32, + min_match: u32, + target_length: u32, + window_log: u32, + seq_store: *mut c_void, + reps: *mut u32, + src: *const c_void, + src_size: usize, +) -> usize { + unsafe { + compress_block_fast_no_dict( + hash_table, + base, + dict_limit, + loaded_dict_end, + hash_log, + target_length, + window_log, + seq_store.cast::(), + reps, + src.cast::(), + src_size, + fast_mls(min_match), + window_log < 19, + ) + } +} + +unsafe fn compress_block_fast_dict_match_state( + hash_table: *mut u32, + base: *const u8, + prefix_start_index: u32, + hash_log: u32, + target_length: u32, + seq_store: *mut SeqStore_t, + reps: *mut u32, + src: *const u8, + src_size: usize, + mls: u32, + dict_hash_table: *const u32, + dict_base: *const u8, + dict_start_index: u32, + dict_end: *const u8, + dict_hash_log: u32, + _prefetch_cdict_tables: bool, +) -> usize { + if src_size < HASH_READ_SIZE { + return src_size; + } + let istart = src; + let iend = istart.wrapping_add(src_size); + let ilimit = iend.wrapping_sub(HASH_READ_SIZE); + let prefix_start = base.wrapping_add(prefix_start_index as usize); + let dict_start = dict_base.wrapping_add(dict_start_index as usize); + let dict_index_delta = + prefix_start_index.wrapping_sub(unsafe { index_from(dict_base, dict_end) }); + let dict_and_prefix_length = unsafe { + istart.offset_from(prefix_start) as u32 + dict_end.offset_from(dict_start) as u32 + }; + let dict_hbits = dict_hash_log + SHORT_CACHE_TAG_BITS; + let step_size = target_length + u32::from(target_length == 0); + let mut ip0 = istart; + let mut ip1 = ip0.wrapping_add(step_size as usize); + let mut anchor = istart; + let mut offset1 = unsafe { *reps }; + let mut offset2 = unsafe { *reps.add(1) }; + + if dict_and_prefix_length == 0 { + ip0 = ip0.wrapping_add(1); + } + + 'outer: while ptr_le(ip1, ilimit) { + let mut match_length: usize; + let mut hash0 = unsafe { hash_ptr(ip0, hash_log, mls) }; + let dict_hash_and_tag0 = unsafe { hash_ptr(ip0, dict_hbits, mls) }; + let mut dict_match_index_and_tag = + unsafe { table_get(dict_hash_table, dict_hash_and_tag0 >> SHORT_CACHE_TAG_BITS) }; + let mut dict_tags_match = packed_tags_match(dict_match_index_and_tag, dict_hash_and_tag0); + let mut match_index = unsafe { table_get(hash_table, hash0) }; + let mut current = unsafe { index_from(base, ip0) }; + let mut step = step_size as usize; + let mut next_step = ip0.wrapping_add(1 << K_SEARCH_STRENGTH); + + loop { + let match_ptr = base.wrapping_add(match_index as usize); + let rep_index = current.wrapping_add(1).wrapping_sub(offset1); + let rep_match = if rep_index < prefix_start_index { + dict_base.wrapping_add(rep_index.wrapping_sub(dict_index_delta) as usize) + } else { + base.wrapping_add(rep_index as usize) + }; + let hash1 = unsafe { hash_ptr(ip1, hash_log, mls) }; + let dict_hash_and_tag1 = unsafe { hash_ptr(ip1, dict_hbits, mls) }; + unsafe { table_set(hash_table, hash0, current) }; + + if index_overlap_check(prefix_start_index, rep_index) + && unsafe { read32(rep_match) == read32(ip0.wrapping_add(1)) } + { + let rep_match_end = if rep_index < prefix_start_index { + dict_end + } else { + iend + }; + match_length = unsafe { + count_2segments( + ip0.wrapping_add(5), + rep_match.wrapping_add(4), + iend, + rep_match_end, + prefix_start, + ) + 4 + }; + ip0 = ip0.wrapping_add(1); + unsafe { + store_seq( + seq_store, + ip0.offset_from(anchor) as usize, + anchor, + iend, + REPCODE1_TO_OFFBASE, + match_length, + ) + }; + break; + } + + if dict_tags_match { + let dict_match_index = dict_match_index_and_tag >> SHORT_CACHE_TAG_BITS; + let mut dict_match = dict_base.wrapping_add(dict_match_index as usize); + if dict_match_index > dict_start_index + && unsafe { read32(dict_match) == read32(ip0) } + && match_index <= prefix_start_index + { + let offset = current + .wrapping_sub(dict_match_index) + .wrapping_sub(dict_index_delta); + match_length = unsafe { + count_2segments( + ip0.wrapping_add(4), + dict_match.wrapping_add(4), + iend, + dict_end, + prefix_start, + ) + 4 + }; + while ptr_gt(ip0, anchor) + && ptr_gt(dict_match, dict_start) + && unsafe { *ip0.wrapping_sub(1) == *dict_match.wrapping_sub(1) } + { + ip0 = ip0.wrapping_sub(1); + dict_match = dict_match.wrapping_sub(1); + match_length += 1; + } + offset2 = offset1; + offset1 = offset; + unsafe { + store_seq( + seq_store, + ip0.offset_from(anchor) as usize, + anchor, + iend, + offset.wrapping_add(ZSTD_REP_NUM as u32), + match_length, + ) + }; + break; + } + } + + if unsafe { match4_found(ip0, match_ptr, match_index, prefix_start_index) } { + let offset = unsafe { index_from(match_ptr, ip0) }; + match_length = + unsafe { count(ip0.wrapping_add(4), match_ptr.wrapping_add(4), iend) + 4 }; + let mut matched = match_ptr; + while ptr_gt(ip0, anchor) + && ptr_gt(matched, prefix_start) + && unsafe { *ip0.wrapping_sub(1) == *matched.wrapping_sub(1) } + { + ip0 = ip0.wrapping_sub(1); + matched = matched.wrapping_sub(1); + match_length += 1; + } + offset2 = offset1; + offset1 = offset; + unsafe { + store_seq( + seq_store, + ip0.offset_from(anchor) as usize, + anchor, + iend, + offset.wrapping_add(ZSTD_REP_NUM as u32), + match_length, + ) + }; + break; + } + + dict_match_index_and_tag = + unsafe { table_get(dict_hash_table, dict_hash_and_tag1 >> SHORT_CACHE_TAG_BITS) }; + dict_tags_match = packed_tags_match(dict_match_index_and_tag, dict_hash_and_tag1); + match_index = unsafe { table_get(hash_table, hash1) }; + if ptr_ge(ip1, next_step) { + step += 1; + next_step = next_step.wrapping_add(1 << K_SEARCH_STRENGTH); + } + ip0 = ip1; + ip1 = ip1.wrapping_add(step); + if ptr_gt(ip1, ilimit) { + break 'outer; + } + current = unsafe { index_from(base, ip0) }; + hash0 = hash1; + } + + ip0 = ip0.wrapping_add(match_length); + anchor = ip0; + if ptr_le(ip0, ilimit) { + let table_hash = + unsafe { hash_ptr(base.wrapping_add(current as usize + 2), hash_log, mls) }; + unsafe { table_set(hash_table, table_hash, current.wrapping_add(2)) }; + let table_hash = unsafe { hash_ptr(ip0.wrapping_sub(2), hash_log, mls) }; + unsafe { + table_set( + hash_table, + table_hash, + index_from(base, ip0.wrapping_sub(2)), + ) + }; + + while ptr_le(ip0, ilimit) { + let current2 = unsafe { index_from(base, ip0) }; + let rep_index2 = current2.wrapping_sub(offset2); + let rep_match2 = if rep_index2 < prefix_start_index { + dict_base + .wrapping_sub(dict_index_delta as usize) + .wrapping_add(rep_index2 as usize) + } else { + base.wrapping_add(rep_index2 as usize) + }; + if index_overlap_check(prefix_start_index, rep_index2) + && unsafe { read32(rep_match2) == read32(ip0) } + { + let rep_end2 = if rep_index2 < prefix_start_index { + dict_end + } else { + iend + }; + let repeat_length = unsafe { + count_2segments( + ip0.wrapping_add(4), + rep_match2.wrapping_add(4), + iend, + rep_end2, + prefix_start, + ) + 4 + }; + std::mem::swap(&mut offset1, &mut offset2); + unsafe { + store_seq( + seq_store, + 0, + anchor, + iend, + REPCODE1_TO_OFFBASE, + repeat_length, + ) + }; + let table_hash = unsafe { hash_ptr(ip0, hash_log, mls) }; + unsafe { table_set(hash_table, table_hash, current2) }; + ip0 = ip0.wrapping_add(repeat_length); + anchor = ip0; + continue; + } + break; + } + } + ip1 = ip0.wrapping_add(step_size as usize); + } + + unsafe { + *reps = offset1; + *reps.add(1) = offset2; + } + unsafe { iend.offset_from(anchor) as usize } +} + +/// Rust implementation called by the C ABI wrapper for attached dictionaries. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_compressBlock_fast_dictMatchState( + hash_table: *mut u32, + base: *const u8, + prefix_start_index: u32, + hash_log: u32, + min_match: u32, + target_length: u32, + seq_store: *mut c_void, + reps: *mut u32, + src: *const c_void, + src_size: usize, + dict_hash_table: *const u32, + dict_base: *const u8, + dict_start_index: u32, + dict_end: *const u8, + dict_hash_log: u32, + prefetch_cdict_tables: c_int, +) -> usize { + unsafe { + compress_block_fast_dict_match_state( + hash_table, + base, + prefix_start_index, + hash_log, + target_length, + seq_store.cast::(), + reps, + src.cast::(), + src_size, + fast_mls(min_match), + dict_hash_table, + dict_base, + dict_start_index, + dict_end, + dict_hash_log, + prefetch_cdict_tables != 0, + ) + } +} + +unsafe fn finish_ext_dict_match( + hash_table: *mut u32, + base: *const u8, + dict_base: *const u8, + prefix_start_index: u32, + dict_end: *const u8, + prefix_start: *const u8, + hash_log: u32, + mls: u32, + seq_store: *mut SeqStore_t, + offset1: &mut u32, + offset2: &mut u32, + mut ip0: *const u8, + match0: *const u8, + match_end: *const u8, + anchor: *const u8, + iend: *const u8, + ilimit: *const u8, + current0: u32, + hash1: usize, + ip1: *const u8, + offcode: u32, + mut match_length: usize, +) -> (*const u8, *const u8) { + match_length += unsafe { + count_2segments( + ip0.wrapping_add(match_length), + match0.wrapping_add(match_length), + iend, + match_end, + prefix_start, + ) + }; + unsafe { + store_seq( + seq_store, + ip0.offset_from(anchor) as usize, + anchor, + iend, + offcode, + match_length, + ) + }; + ip0 = ip0.wrapping_add(match_length); + let mut new_anchor = ip0; + + if ptr_lt(ip1, ip0) { + let table_hash = hash1; + unsafe { table_set(hash_table, table_hash, index_from(base, ip1)) }; + } + if ptr_le(ip0, ilimit) { + let table_hash = + unsafe { hash_ptr(base.wrapping_add(current0 as usize + 2), hash_log, mls) }; + unsafe { table_set(hash_table, table_hash, current0.wrapping_add(2)) }; + let table_hash = unsafe { hash_ptr(ip0.wrapping_sub(2), hash_log, mls) }; + unsafe { + table_set( + hash_table, + table_hash, + index_from(base, ip0.wrapping_sub(2)), + ) + }; + + while ptr_le(ip0, ilimit) { + let rep_index2 = unsafe { index_from(base, ip0) }.wrapping_sub(*offset2); + let rep_match2 = if rep_index2 < prefix_start_index { + dict_base.wrapping_add(rep_index2 as usize) + } else { + base.wrapping_add(rep_index2 as usize) + }; + if *offset2 > 0 + && index_overlap_check(prefix_start_index, rep_index2) + && unsafe { read32(rep_match2) == read32(ip0) } + { + let rep_end2 = if rep_index2 < prefix_start_index { + dict_end + } else { + iend + }; + let repeat_length = unsafe { + count_2segments( + ip0.wrapping_add(4), + rep_match2.wrapping_add(4), + iend, + rep_end2, + prefix_start, + ) + 4 + }; + std::mem::swap(offset1, offset2); + unsafe { + store_seq( + seq_store, + 0, + new_anchor, + iend, + REPCODE1_TO_OFFBASE, + repeat_length, + ) + }; + let table_hash = unsafe { hash_ptr(ip0, hash_log, mls) }; + unsafe { table_set(hash_table, table_hash, index_from(base, ip0)) }; + ip0 = ip0.wrapping_add(repeat_length); + new_anchor = ip0; + continue; + } + break; + } + } + (ip0, new_anchor) +} + +unsafe fn compress_block_fast_ext_dict( + hash_table: *mut u32, + base: *const u8, + dict_base: *const u8, + dict_limit: u32, + low_limit: u32, + loaded_dict_end: u32, + hash_log: u32, + target_length: u32, + window_log: u32, + seq_store: *mut SeqStore_t, + reps: *mut u32, + src: *const u8, + src_size: usize, + mls: u32, +) -> usize { + if src_size < HASH_READ_SIZE { + return src_size; + } + let istart = src; + let iend = istart.wrapping_add(src_size); + let ilimit = iend.wrapping_sub(HASH_READ_SIZE); + let end_index = unsafe { index_from(base, istart) }.wrapping_add(src_size as u32); + let dict_start_index = lowest_match_index(low_limit, loaded_dict_end, end_index, window_log); + let dict_start = dict_base.wrapping_add(dict_start_index as usize); + let prefix_start_index = dict_limit.max(dict_start_index); + let prefix_start = base.wrapping_add(prefix_start_index as usize); + let dict_end = dict_base.wrapping_add(prefix_start_index as usize); + if prefix_start_index == dict_start_index { + return unsafe { + compress_block_fast_no_dict( + hash_table, + base, + dict_limit, + loaded_dict_end, + hash_log, + target_length, + window_log, + seq_store, + reps, + src, + src_size, + mls, + window_log < 19, + ) + }; + } + + let step_size = target_length as usize + usize::from(target_length == 0) + 1; + let mut ip0 = istart; + let mut anchor = istart; + let mut offset1 = unsafe { *reps }; + let mut offset2 = unsafe { *reps.add(1) }; + let mut offset_saved1 = 0u32; + let mut offset_saved2 = 0u32; + let current = unsafe { index_from(base, ip0) }; + let max_rep = current.wrapping_sub(dict_start_index); + if offset2 >= max_rep { + offset_saved2 = offset2; + offset2 = 0; + } + if offset1 >= max_rep { + offset_saved1 = offset1; + offset1 = 0; + } + + 'start: loop { + let mut step = step_size; + let mut next_step = ip0.wrapping_add(1 << (K_SEARCH_STRENGTH - 1)); + let mut ip1 = ip0.wrapping_add(1); + let mut ip2 = ip0.wrapping_add(step); + let mut ip3 = ip2.wrapping_add(1); + if ptr_ge(ip3, ilimit) { + break; + } + + let mut hash0 = unsafe { hash_ptr(ip0, hash_log, mls) }; + let mut hash1 = unsafe { hash_ptr(ip1, hash_log, mls) }; + let mut index = unsafe { table_get(hash_table, hash0) }; + let mut index_base = if index < prefix_start_index { + dict_base + } else { + base + }; + + loop { + let current2 = unsafe { index_from(base, ip2) }; + let rep_index = current2.wrapping_sub(offset1); + let rep_base = if rep_index < prefix_start_index { + dict_base + } else { + base + }; + let rep_value = if offset1 > 0 && prefix_start_index.wrapping_sub(rep_index) >= 4 { + unsafe { read32(rep_base.wrapping_add(rep_index as usize)) } + } else { + unsafe { read32(ip2) ^ 1 } + }; + let mut current0 = unsafe { index_from(base, ip0) }; + unsafe { table_set(hash_table, hash0, current0) }; + + if unsafe { read32(ip2) == rep_value } { + ip0 = ip2; + let mut match0 = rep_base.wrapping_add(rep_index as usize); + let match_end = if rep_index < prefix_start_index { + dict_end + } else { + iend + }; + let match_length = + usize::from(unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) }); + ip0 = ip0.wrapping_sub(match_length); + match0 = match0.wrapping_sub(match_length); + let (new_ip0, new_anchor) = unsafe { + finish_ext_dict_match( + hash_table, + base, + dict_base, + prefix_start_index, + dict_end, + prefix_start, + hash_log, + mls, + seq_store, + &mut offset1, + &mut offset2, + ip0, + match0, + match_end, + anchor, + iend, + ilimit, + current0, + hash1, + ip1, + REPCODE1_TO_OFFBASE, + match_length + 4, + ) + }; + ip0 = new_ip0; + anchor = new_anchor; + continue 'start; + } + + if index >= dict_start_index + && unsafe { read32(index_base.wrapping_add(index as usize)) == read32(ip0) } + { + let offset = current0.wrapping_sub(index); + let low_match_ptr = if index < prefix_start_index { + dict_start + } else { + prefix_start + }; + let match_end = if index < prefix_start_index { + dict_end + } else { + iend + }; + let mut match0 = index_base.wrapping_add(index as usize); + offset2 = offset1; + offset1 = offset; + let mut match_length = 4usize; + while ptr_gt(ip0, anchor) + && ptr_gt(match0, low_match_ptr) + && unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) } + { + ip0 = ip0.wrapping_sub(1); + match0 = match0.wrapping_sub(1); + match_length += 1; + } + let (new_ip0, new_anchor) = unsafe { + finish_ext_dict_match( + hash_table, + base, + dict_base, + prefix_start_index, + dict_end, + prefix_start, + hash_log, + mls, + seq_store, + &mut offset1, + &mut offset2, + ip0, + match0, + match_end, + anchor, + iend, + ilimit, + current0, + hash1, + ip1, + offset.wrapping_add(ZSTD_REP_NUM as u32), + match_length, + ) + }; + ip0 = new_ip0; + anchor = new_anchor; + continue 'start; + } + + index = unsafe { table_get(hash_table, hash1) }; + index_base = if index < prefix_start_index { + dict_base + } else { + base + }; + hash0 = hash1; + hash1 = unsafe { hash_ptr(ip2, hash_log, mls) }; + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + current0 = unsafe { index_from(base, ip0) }; + unsafe { table_set(hash_table, hash0, current0) }; + + if index >= dict_start_index + && unsafe { read32(index_base.wrapping_add(index as usize)) == read32(ip0) } + { + let offset = current0.wrapping_sub(index); + let low_match_ptr = if index < prefix_start_index { + dict_start + } else { + prefix_start + }; + let match_end = if index < prefix_start_index { + dict_end + } else { + iend + }; + let mut match0 = index_base.wrapping_add(index as usize); + offset2 = offset1; + offset1 = offset; + let mut match_length = 4usize; + while ptr_gt(ip0, anchor) + && ptr_gt(match0, low_match_ptr) + && unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) } + { + ip0 = ip0.wrapping_sub(1); + match0 = match0.wrapping_sub(1); + match_length += 1; + } + let (new_ip0, new_anchor) = unsafe { + finish_ext_dict_match( + hash_table, + base, + dict_base, + prefix_start_index, + dict_end, + prefix_start, + hash_log, + mls, + seq_store, + &mut offset1, + &mut offset2, + ip0, + match0, + match_end, + anchor, + iend, + ilimit, + current0, + hash1, + ip1, + offset.wrapping_add(ZSTD_REP_NUM as u32), + match_length, + ) + }; + ip0 = new_ip0; + anchor = new_anchor; + continue 'start; + } + + index = unsafe { table_get(hash_table, hash1) }; + index_base = if index < prefix_start_index { + dict_base + } else { + base + }; + hash0 = hash1; + hash1 = unsafe { hash_ptr(ip2, hash_log, mls) }; + ip0 = ip1; + ip1 = ip2; + ip2 = ip0.wrapping_add(step); + ip3 = ip1.wrapping_add(step); + if ptr_ge(ip2, next_step) { + step += 1; + next_step = next_step.wrapping_add(1 << (K_SEARCH_STRENGTH - 1)); + } + if ptr_ge(ip3, ilimit) { + break 'start; + } + } + } + + offset_saved2 = if offset_saved1 != 0 && offset1 != 0 { + offset_saved1 + } else { + offset_saved2 + }; + unsafe { + *reps = if offset1 != 0 { offset1 } else { offset_saved1 }; + *reps.add(1) = if offset2 != 0 { offset2 } else { offset_saved2 }; + } + unsafe { iend.offset_from(anchor) as usize } +} + +/// Rust implementation called by the C ABI wrapper for external dictionaries. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_compressBlock_fast_extDict( + hash_table: *mut u32, + base: *const u8, + dict_base: *const u8, + dict_limit: u32, + low_limit: u32, + loaded_dict_end: u32, + hash_log: u32, + min_match: u32, + target_length: u32, + window_log: u32, + seq_store: *mut c_void, + reps: *mut u32, + src: *const c_void, + src_size: usize, +) -> usize { + unsafe { + compress_block_fast_ext_dict( + hash_table, + base, + dict_base, + dict_limit, + low_limit, + loaded_dict_end, + hash_log, + target_length, + window_log, + seq_store.cast::(), + reps, + src.cast::(), + src_size, + fast_mls(min_match), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::mem::{align_of, offset_of, size_of}; + + #[test] + fn seq_store_layout_matches_the_c_leaf_abi() { + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), align_of::()); + assert_eq!(offset_of!(SeqStore_t, sequencesStart), 0); + assert_eq!( + offset_of!(SeqStore_t, longLengthPos), + 9 * size_of::() + 4 + ); + assert_eq!(size_of::(), 9 * size_of::() + 8); + } + + #[test] + fn tagged_index_keeps_index_and_tag_in_c_format() { + let mut table = [0u32; 4]; + write_tagged_index( + table.as_mut_ptr(), + (2 << SHORT_CACHE_TAG_BITS) | 0x5a, + 0x123456, + ); + assert_eq!(table[2], (0x123456 << SHORT_CACHE_TAG_BITS) | 0x5a); + assert!(packed_tags_match( + table[2], + (1 << SHORT_CACHE_TAG_BITS) | 0x5a + )); + assert!(!packed_tags_match( + table[2], + (1 << SHORT_CACHE_TAG_BITS) | 0x5b + )); + } + + #[test] + fn store_seq_preserves_literal_and_long_match_contracts() { + let mut sequences = [SeqDef { + offBase: 0, + litLength: 0, + mlBase: 0, + }; 2]; + let mut literals = [0u8; 16]; + let source = *b"literals"; + let mut store = SeqStore_t { + sequencesStart: sequences.as_mut_ptr(), + sequences: sequences.as_mut_ptr(), + litStart: literals.as_mut_ptr(), + lit: literals.as_mut_ptr(), + llCode: ptr::null_mut(), + mlCode: ptr::null_mut(), + ofCode: ptr::null_mut(), + maxNbSeq: sequences.len(), + maxNbLit: literals.len(), + longLengthType: 0, + longLengthPos: 0, + }; + + unsafe { + store_seq( + &mut store, + source.len(), + source.as_ptr(), + source.as_ptr().add(source.len()), + 13, + 3 + 0x1_0000, + ); + } + assert_eq!(&literals[..source.len()], &source); + assert_eq!(sequences[0].offBase, 13); + assert_eq!(sequences[0].litLength, source.len() as u16); + assert_eq!(sequences[0].mlBase, 0); + assert_eq!(store.longLengthType, 2); + assert_eq!(store.longLengthPos, 0); + assert_eq!( + unsafe { store.sequences.offset_from(store.sequencesStart) }, + 1 + ); + } +}