ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(rust): port fast block matching

Browse Source 
Move fast hash-table filling and no-dictionary, external-dictionary, and
attached-CDict match loops into Rust. The C shim keeps `ZSTD_MatchState_t`
opaque, extracts only its needed fields, and asserts the mirrored SeqStore
leaf layout so existing compression dispatch remains ABI-compatible.

Test Plan:
- cargo clippy
- cargo clippy --benches
- cargo clippy --tests
- cargo +nightly fmt
- cargo test zstd_fast::tests -- --nocapture
- cargo test --target i686-unknown-linux-gnu zstd_fast::tests -- --nocapture
- byte-identical C-control frames for dictionary and streaming variants
- fuzzer -s2066 -i5 --no-big-tests
- invalidDictionaries and zstreamtest -i1

Refs: Rust superblock port fde1d70c
This commit is contained in:
ddidderr
2026-07-10 22:46:15 +02:00
parent fde1d70cc5
commit 74140c318f
3 changed files with 1657 additions and 946 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_fast.c
+56 -946
View File
@@ -8,978 +8,88 @@
* You may select, at your option, one of the above-listed licenses.
*/
#include "zstd_compress_internal.h" /* ZSTD_hashPtr, ZSTD_count, ZSTD_storeSeq */
/* The matching algorithms live in rust/src/zstd_fast.rs. Keep match-state
* extraction here: it is a large private structure whose layout should not be
* duplicated in Rust. */
#include "zstd_compress_internal.h"
#include "zstd_fast.h"
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_fillHashTableForCDict(ZSTD_MatchState_t* ms,
const void* const end,
ZSTD_dictTableLoadMethod_e dtlm)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashTable = ms->hashTable;
U32 const hBits = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
U32 const mls = cParams->minMatch;
const BYTE* const base = ms->window.base;
const BYTE* ip = base + ms->nextToUpdate;
const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
const U32 fastHashFillStep = 3;
typedef char ZSTD_rust_fast_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
typedef char ZSTD_rust_fast_seqstore_long_length_pos[
(offsetof(SeqStore_t, longLengthPos) == 9 * sizeof(size_t) + 4) ? 1 : -1];
typedef char ZSTD_rust_fast_seqstore_layout[
(sizeof(SeqStore_t) == 9 * sizeof(size_t) + 8) ? 1 : -1];
typedef char ZSTD_rust_fast_rep_count[(ZSTD_REP_NUM == 3) ? 1 : -1];
/* Currently, we always use ZSTD_dtlm_full for filling CDict tables.
* Feel free to remove this assert if there's a good reason! */
assert(dtlm == ZSTD_dtlm_full);
void ZSTD_rust_fillHashTable(U32* hashTable, const BYTE* base, U32 nextToUpdate,
const void* end, U32 hashLog, U32 minMatch,
int fullTableLoad, int forCDict);
/* Always insert every fastHashFillStep position into the hash table.
* Insert the other positions if their hash entry is empty.
*/
for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) {
U32 const curr = (U32)(ip - base);
{ size_t const hashAndTag = ZSTD_hashPtr(ip, hBits, mls);
ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr); }
size_t ZSTD_rust_compressBlock_fast(
U32* hashTable, const BYTE* base, U32 dictLimit, U32 loadedDictEnd,
U32 hashLog, U32 minMatch, U32 targetLength, U32 windowLog,
void* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize);
if (dtlm == ZSTD_dtlm_fast) continue;
/* Only load extra positions for ZSTD_dtlm_full */
{ U32 p;
for (p = 1; p < fastHashFillStep; ++p) {
size_t const hashAndTag = ZSTD_hashPtr(ip + p, hBits, mls);
if (hashTable[hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { /* not yet filled */
ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr + p);
} } } }
}
size_t ZSTD_rust_compressBlock_fast_dictMatchState(
U32* hashTable, const BYTE* base, U32 prefixStartIndex,
U32 hashLog, U32 minMatch, U32 targetLength,
void* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize,
const U32* dictHashTable, const BYTE* dictBase, U32 dictStartIndex,
const BYTE* dictEnd, U32 dictHashLog, int prefetchCDictTables);
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_fillHashTableForCCtx(ZSTD_MatchState_t* ms,
const void* const end,
ZSTD_dictTableLoadMethod_e dtlm)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashTable = ms->hashTable;
U32 const hBits = cParams->hashLog;
U32 const mls = cParams->minMatch;
const BYTE* const base = ms->window.base;
const BYTE* ip = base + ms->nextToUpdate;
const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
const U32 fastHashFillStep = 3;
/* Currently, we always use ZSTD_dtlm_fast for filling CCtx tables.
* Feel free to remove this assert if there's a good reason! */
assert(dtlm == ZSTD_dtlm_fast);
/* Always insert every fastHashFillStep position into the hash table.
* Insert the other positions if their hash entry is empty.
*/
for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) {
U32 const curr = (U32)(ip - base);
size_t const hash0 = ZSTD_hashPtr(ip, hBits, mls);
hashTable[hash0] = curr;
if (dtlm == ZSTD_dtlm_fast) continue;
/* Only load extra positions for ZSTD_dtlm_full */
{ U32 p;
for (p = 1; p < fastHashFillStep; ++p) {
size_t const hash = ZSTD_hashPtr(ip + p, hBits, mls);
if (hashTable[hash] == 0) { /* not yet filled */
hashTable[hash] = curr + p;
} } } }
}
size_t ZSTD_rust_compressBlock_fast_extDict(
U32* hashTable, const BYTE* base, const BYTE* dictBase,
U32 dictLimit, U32 lowLimit, U32 loadedDictEnd,
U32 hashLog, U32 minMatch, U32 targetLength, U32 windowLog,
void* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize);
void ZSTD_fillHashTable(ZSTD_MatchState_t* ms,
const void* const end,
const void* end,
ZSTD_dictTableLoadMethod_e dtlm,
ZSTD_tableFillPurpose_e tfp)
{
if (tfp == ZSTD_tfp_forCDict) {
ZSTD_fillHashTableForCDict(ms, end, dtlm);
} else {
ZSTD_fillHashTableForCCtx(ms, end, dtlm);
}
assert((tfp == ZSTD_tfp_forCDict && dtlm == ZSTD_dtlm_full)
|| (tfp != ZSTD_tfp_forCDict && dtlm == ZSTD_dtlm_fast));
ZSTD_rust_fillHashTable(ms->hashTable, ms->window.base, ms->nextToUpdate,
end, ms->cParams.hashLog, ms->cParams.minMatch,
dtlm == ZSTD_dtlm_full, tfp == ZSTD_tfp_forCDict);
}
typedef int (*ZSTD_match4Found) (const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit);
static int
ZSTD_match4Found_cmov(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit)
{
/* Array of ~random data, should have low probability of matching data.
* Load from here if the index is invalid.
* Used to avoid unpredictable branches. */
static const BYTE dummy[] = {0x12,0x34,0x56,0x78};
/* currentIdx >= lowLimit is a (somewhat) unpredictable branch.
* However expression below compiles into conditional move.
*/
const BYTE* mvalAddr = ZSTD_selectAddr(matchIdx, idxLowLimit, matchAddress, dummy);
/* Note: this used to be written as : return test1 && test2;
* Unfortunately, once inlined, these tests become branches,
* in which case it becomes critical that they are executed in the right order (test1 then test2).
* So we have to write these tests in a specific manner to ensure their ordering.
*/
if (MEM_read32(currentPtr) != MEM_read32(mvalAddr)) return 0;
/* force ordering of these tests, which matters once the function is inlined, as they become branches */
#if defined(__GNUC__)
__asm__("");
#endif
return matchIdx >= idxLowLimit;
}
static int
ZSTD_match4Found_branch(const BYTE* currentPtr, const BYTE* matchAddress, U32 matchIdx, U32 idxLowLimit)
{
/* using a branch instead of a cmov,
* because it's faster in scenarios where matchIdx >= idxLowLimit is generally true,
* aka almost all candidates are within range */
U32 mval;
if (matchIdx >= idxLowLimit) {
mval = MEM_read32(matchAddress);
} else {
mval = MEM_read32(currentPtr) ^ 1; /* guaranteed to not match. */
}
return (MEM_read32(currentPtr) == mval);
}
/**
* If you squint hard enough (and ignore repcodes), the search operation at any
* given position is broken into 4 stages:
*
* 1. Hash (map position to hash value via input read)
* 2. Lookup (map hash val to index via hashtable read)
* 3. Load (map index to value at that position via input read)
* 4. Compare
*
* Each of these steps involves a memory read at an address which is computed
* from the previous step. This means these steps must be sequenced and their
* latencies are cumulative.
*
* Rather than do 1->2->3->4 sequentially for a single position before moving
* onto the next, this implementation interleaves these operations across the
* next few positions:
*
* R = Repcode Read & Compare
* H = Hash
* T = Table Lookup
* M = Match Read & Compare
*
* Pos | Time -->
* ----+-------------------
* N | ... M
* N+1 | ... TM
* N+2 | R H T M
* N+3 | H TM
* N+4 | R H T M
* N+5 | H ...
* N+6 | R ...
*
* This is very much analogous to the pipelining of execution in a CPU. And just
* like a CPU, we have to dump the pipeline when we find a match (i.e., take a
* branch).
*
* When this happens, we throw away our current state, and do the following prep
* to re-enter the loop:
*
* Pos | Time -->
* ----+-------------------
* N | H T
* N+1 | H
*
* This is also the work we do at the beginning to enter the loop initially.
*/
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_fast_noDict_generic(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize,
U32 const mls, int useCmov)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashTable = ms->hashTable;
U32 const hlog = cParams->hashLog;
size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1; /* min 2 */
const BYTE* const base = ms->window.base;
const BYTE* const istart = (const BYTE*)src;
const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
const U32 prefixStartIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog);
const BYTE* const prefixStart = base + prefixStartIndex;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - HASH_READ_SIZE;
const BYTE* anchor = istart;
const BYTE* ip0 = istart;
const BYTE* ip1;
const BYTE* ip2;
const BYTE* ip3;
U32 current0;
U32 rep_offset1 = rep[0];
U32 rep_offset2 = rep[1];
U32 offsetSaved1 = 0, offsetSaved2 = 0;
size_t hash0; /* hash for ip0 */
size_t hash1; /* hash for ip1 */
U32 matchIdx; /* match idx for ip0 */
U32 offcode;
const BYTE* match0;
size_t mLength;
/* ip0 and ip1 are always adjacent. The targetLength skipping and
* uncompressibility acceleration is applied to every other position,
* matching the behavior of #1562. step therefore represents the gap
* between pairs of positions, from ip0 to ip2 or ip1 to ip3. */
size_t step;
const BYTE* nextStep;
const size_t kStepIncr = (1 << (kSearchStrength - 1));
const ZSTD_match4Found matchFound = useCmov ? ZSTD_match4Found_cmov : ZSTD_match4Found_branch;
DEBUGLOG(5, "ZSTD_compressBlock_fast_generic");
ip0 += (ip0 == prefixStart);
{ U32 const curr = (U32)(ip0 - base);
U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog);
U32 const maxRep = curr - windowLow;
if (rep_offset2 > maxRep) offsetSaved2 = rep_offset2, rep_offset2 = 0;
if (rep_offset1 > maxRep) offsetSaved1 = rep_offset1, rep_offset1 = 0;
}
/* start each op */
_start: /* Requires: ip0 */
step = stepSize;
nextStep = ip0 + kStepIncr;
/* calculate positions, ip0 - anchor == 0, so we skip step calc */
ip1 = ip0 + 1;
ip2 = ip0 + step;
ip3 = ip2 + 1;
if (ip3 >= ilimit) {
goto _cleanup;
}
hash0 = ZSTD_hashPtr(ip0, hlog, mls);
hash1 = ZSTD_hashPtr(ip1, hlog, mls);
matchIdx = hashTable[hash0];
do {
/* load repcode match for ip[2]*/
const U32 rval = MEM_read32(ip2 - rep_offset1);
/* write back hash table entry */
current0 = (U32)(ip0 - base);
hashTable[hash0] = current0;
/* check repcode at ip[2] */
if ((MEM_read32(ip2) == rval) & (rep_offset1 > 0)) {
ip0 = ip2;
match0 = ip0 - rep_offset1;
mLength = ip0[-1] == match0[-1];
ip0 -= mLength;
match0 -= mLength;
offcode = REPCODE1_TO_OFFBASE;
mLength += 4;
/* Write next hash table entry: it's already calculated.
* This write is known to be safe because ip1 is before the
* repcode (ip2). */
hashTable[hash1] = (U32)(ip1 - base);
goto _match;
}
if (matchFound(ip0, base + matchIdx, matchIdx, prefixStartIndex)) {
/* Write next hash table entry (it's already calculated).
* This write is known to be safe because the ip1 == ip0 + 1,
* so searching will resume after ip1 */
hashTable[hash1] = (U32)(ip1 - base);
goto _offset;
}
/* lookup ip[1] */
matchIdx = hashTable[hash1];
/* hash ip[2] */
hash0 = hash1;
hash1 = ZSTD_hashPtr(ip2, hlog, mls);
/* advance to next positions */
ip0 = ip1;
ip1 = ip2;
ip2 = ip3;
/* write back hash table entry */
current0 = (U32)(ip0 - base);
hashTable[hash0] = current0;
if (matchFound(ip0, base + matchIdx, matchIdx, prefixStartIndex)) {
/* Write next hash table entry, since it's already calculated */
if (step <= 4) {
/* Avoid writing an index if it's >= position where search will resume.
* The minimum possible match has length 4, so search can resume at ip0 + 4.
*/
hashTable[hash1] = (U32)(ip1 - base);
}
goto _offset;
}
/* lookup ip[1] */
matchIdx = hashTable[hash1];
/* hash ip[2] */
hash0 = hash1;
hash1 = ZSTD_hashPtr(ip2, hlog, mls);
/* advance to next positions */
ip0 = ip1;
ip1 = ip2;
ip2 = ip0 + step;
ip3 = ip1 + step;
/* calculate step */
if (ip2 >= nextStep) {
step++;
PREFETCH_L1(ip1 + 64);
PREFETCH_L1(ip1 + 128);
nextStep += kStepIncr;
}
} while (ip3 < ilimit);
_cleanup:
/* Note that there are probably still a couple positions one could search.
* However, it seems to be a meaningful performance hit to try to search
* them. So let's not. */
/* When the repcodes are outside of the prefix, we set them to zero before the loop.
* When the offsets are still zero, we need to restore them after the block to have a correct
* repcode history. If only one offset was invalid, it is easy. The tricky case is when both
* offsets were invalid. We need to figure out which offset to refill with.
* - If both offsets are zero they are in the same order.
* - If both offsets are non-zero, we won't restore the offsets from `offsetSaved[12]`.
* - If only one is zero, we need to decide which offset to restore.
* - If rep_offset1 is non-zero, then rep_offset2 must be offsetSaved1.
* - It is impossible for rep_offset2 to be non-zero.
*
* So if rep_offset1 started invalid (offsetSaved1 != 0) and became valid (rep_offset1 != 0), then
* set rep[0] = rep_offset1 and rep[1] = offsetSaved1.
*/
offsetSaved2 = ((offsetSaved1 != 0) && (rep_offset1 != 0)) ? offsetSaved1 : offsetSaved2;
/* save reps for next block */
rep[0] = rep_offset1 ? rep_offset1 : offsetSaved1;
rep[1] = rep_offset2 ? rep_offset2 : offsetSaved2;
/* Return the last literals size */
return (size_t)(iend - anchor);
_offset: /* Requires: ip0, idx */
/* Compute the offset code. */
match0 = base + matchIdx;
rep_offset2 = rep_offset1;
rep_offset1 = (U32)(ip0-match0);
offcode = OFFSET_TO_OFFBASE(rep_offset1);
mLength = 4;
/* Count the backwards match length. */
while (((ip0>anchor) & (match0>prefixStart)) && (ip0[-1] == match0[-1])) {
ip0--;
match0--;
mLength++;
}
_match: /* Requires: ip0, match0, offcode */
/* Count the forward length. */
mLength += ZSTD_count(ip0 + mLength, match0 + mLength, iend);
ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength);
ip0 += mLength;
anchor = ip0;
/* Fill table and check for immediate repcode. */
if (ip0 <= ilimit) {
/* Fill Table */
assert(base+current0+2 > istart); /* check base overflow */
hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */
hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base);
if (rep_offset2 > 0) { /* rep_offset2==0 means rep_offset2 is invalidated */
while ( (ip0 <= ilimit) && (MEM_read32(ip0) == MEM_read32(ip0 - rep_offset2)) ) {
/* store sequence */
size_t const rLength = ZSTD_count(ip0+4, ip0+4-rep_offset2, iend) + 4;
{ U32 const tmpOff = rep_offset2; rep_offset2 = rep_offset1; rep_offset1 = tmpOff; } /* swap rep_offset2 <=> rep_offset1 */
hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base);
ip0 += rLength;
ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, REPCODE1_TO_OFFBASE, rLength);
anchor = ip0;
continue; /* faster when present (confirmed on gcc-8) ... (?) */
} } }
goto _start;
}
#define ZSTD_GEN_FAST_FN(dictMode, mml, cmov) \
static size_t ZSTD_compressBlock_fast_##dictMode##_##mml##_##cmov( \
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
void const* src, size_t srcSize) \
{ \
return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mml, cmov); \
}
ZSTD_GEN_FAST_FN(noDict, 4, 1)
ZSTD_GEN_FAST_FN(noDict, 5, 1)
ZSTD_GEN_FAST_FN(noDict, 6, 1)
ZSTD_GEN_FAST_FN(noDict, 7, 1)
ZSTD_GEN_FAST_FN(noDict, 4, 0)
ZSTD_GEN_FAST_FN(noDict, 5, 0)
ZSTD_GEN_FAST_FN(noDict, 6, 0)
ZSTD_GEN_FAST_FN(noDict, 7, 0)
size_t ZSTD_compressBlock_fast(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
const void* src, size_t srcSize)
{
U32 const mml = ms->cParams.minMatch;
/* use cmov when "candidate in range" branch is likely unpredictable */
int const useCmov = ms->cParams.windowLog < 19;
assert(ms->dictMatchState == NULL);
if (useCmov) {
switch(mml)
{
default: /* includes case 3 */
case 4 :
return ZSTD_compressBlock_fast_noDict_4_1(ms, seqStore, rep, src, srcSize);
case 5 :
return ZSTD_compressBlock_fast_noDict_5_1(ms, seqStore, rep, src, srcSize);
case 6 :
return ZSTD_compressBlock_fast_noDict_6_1(ms, seqStore, rep, src, srcSize);
case 7 :
return ZSTD_compressBlock_fast_noDict_7_1(ms, seqStore, rep, src, srcSize);
}
} else {
/* use a branch instead */
switch(mml)
{
default: /* includes case 3 */
case 4 :
return ZSTD_compressBlock_fast_noDict_4_0(ms, seqStore, rep, src, srcSize);
case 5 :
return ZSTD_compressBlock_fast_noDict_5_0(ms, seqStore, rep, src, srcSize);
case 6 :
return ZSTD_compressBlock_fast_noDict_6_0(ms, seqStore, rep, src, srcSize);
case 7 :
return ZSTD_compressBlock_fast_noDict_7_0(ms, seqStore, rep, src, srcSize);
}
}
return ZSTD_rust_compressBlock_fast(
ms->hashTable, ms->window.base, ms->window.dictLimit, ms->loadedDictEnd,
ms->cParams.hashLog, ms->cParams.minMatch,
ms->cParams.targetLength, ms->cParams.windowLog,
seqStore, rep, src, srcSize);
}
FORCE_INLINE_TEMPLATE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_fast_dictMatchState_generic(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize, U32 const mls, U32 const hasStep)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashTable = ms->hashTable;
U32 const hlog = cParams->hashLog;
/* support stepSize of 0 */
U32 const stepSize = cParams->targetLength + !(cParams->targetLength);
const BYTE* const base = ms->window.base;
const BYTE* const istart = (const BYTE*)src;
const BYTE* ip0 = istart;
const BYTE* ip1 = ip0 + stepSize; /* we assert below that stepSize >= 1 */
const BYTE* anchor = istart;
const U32 prefixStartIndex = ms->window.dictLimit;
const BYTE* const prefixStart = base + prefixStartIndex;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - HASH_READ_SIZE;
U32 offset_1=rep[0], offset_2=rep[1];
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
const ZSTD_compressionParameters* const dictCParams = &dms->cParams ;
const U32* const dictHashTable = dms->hashTable;
const U32 dictStartIndex = dms->window.dictLimit;
const BYTE* const dictBase = dms->window.base;
const BYTE* const dictStart = dictBase + dictStartIndex;
const BYTE* const dictEnd = dms->window.nextSrc;
const U32 dictIndexDelta = prefixStartIndex - (U32)(dictEnd - dictBase);
const U32 dictAndPrefixLength = (U32)(istart - prefixStart + dictEnd - dictStart);
const U32 dictHBits = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
/* if a dictionary is still attached, it necessarily means that
* it is within window size. So we just check it. */
const U32 maxDistance = 1U << cParams->windowLog;
const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
assert(endIndex - prefixStartIndex <= maxDistance);
(void)maxDistance; (void)endIndex; /* these variables are not used when assert() is disabled */
(void)hasStep; /* not currently specialized on whether it's accelerated */
/* ensure there will be no underflow
* when translating a dict index into a local index */
assert(prefixStartIndex >= (U32)(dictEnd - dictBase));
if (ms->prefetchCDictTables) {
size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32);
PREFETCH_AREA(dictHashTable, hashTableBytes);
}
/* init */
DEBUGLOG(5, "ZSTD_compressBlock_fast_dictMatchState_generic");
ip0 += (dictAndPrefixLength == 0);
/* dictMatchState repCode checks don't currently handle repCode == 0
* disabling. */
assert(offset_1 <= dictAndPrefixLength);
assert(offset_2 <= dictAndPrefixLength);
/* Outer search loop */
assert(stepSize >= 1);
while (ip1 <= ilimit) { /* repcode check at (ip0 + 1) is safe because ip0 < ip1 */
size_t mLength;
size_t hash0 = ZSTD_hashPtr(ip0, hlog, mls);
size_t const dictHashAndTag0 = ZSTD_hashPtr(ip0, dictHBits, mls);
U32 dictMatchIndexAndTag = dictHashTable[dictHashAndTag0 >> ZSTD_SHORT_CACHE_TAG_BITS];
int dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag0);
U32 matchIndex = hashTable[hash0];
U32 curr = (U32)(ip0 - base);
size_t step = stepSize;
const size_t kStepIncr = 1 << kSearchStrength;
const BYTE* nextStep = ip0 + kStepIncr;
/* Inner search loop */
while (1) {
const BYTE* match = base + matchIndex;
const U32 repIndex = curr + 1 - offset_1;
const BYTE* repMatch = (repIndex < prefixStartIndex) ?
dictBase + (repIndex - dictIndexDelta) :
base + repIndex;
const size_t hash1 = ZSTD_hashPtr(ip1, hlog, mls);
size_t const dictHashAndTag1 = ZSTD_hashPtr(ip1, dictHBits, mls);
hashTable[hash0] = curr; /* update hash table */
if ((ZSTD_index_overlap_check(prefixStartIndex, repIndex))
&& (MEM_read32(repMatch) == MEM_read32(ip0 + 1))) {
const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip0 + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixStart) + 4;
ip0++;
ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
break;
}
if (dictTagsMatch) {
/* Found a possible dict match */
const U32 dictMatchIndex = dictMatchIndexAndTag >> ZSTD_SHORT_CACHE_TAG_BITS;
const BYTE* dictMatch = dictBase + dictMatchIndex;
if (dictMatchIndex > dictStartIndex &&
MEM_read32(dictMatch) == MEM_read32(ip0)) {
/* To replicate extDict parse behavior, we only use dict matches when the normal matchIndex is invalid */
if (matchIndex <= prefixStartIndex) {
U32 const offset = (U32) (curr - dictMatchIndex - dictIndexDelta);
mLength = ZSTD_count_2segments(ip0 + 4, dictMatch + 4, iend, dictEnd, prefixStart) + 4;
while (((ip0 > anchor) & (dictMatch > dictStart))
&& (ip0[-1] == dictMatch[-1])) {
ip0--;
dictMatch--;
mLength++;
} /* catch up */
offset_2 = offset_1;
offset_1 = offset;
ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
break;
}
}
}
if (ZSTD_match4Found_cmov(ip0, match, matchIndex, prefixStartIndex)) {
/* found a regular match of size >= 4 */
U32 const offset = (U32) (ip0 - match);
mLength = ZSTD_count(ip0 + 4, match + 4, iend) + 4;
while (((ip0 > anchor) & (match > prefixStart))
&& (ip0[-1] == match[-1])) {
ip0--;
match--;
mLength++;
} /* catch up */
offset_2 = offset_1;
offset_1 = offset;
ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
break;
}
/* Prepare for next iteration */
dictMatchIndexAndTag = dictHashTable[dictHashAndTag1 >> ZSTD_SHORT_CACHE_TAG_BITS];
dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag1);
matchIndex = hashTable[hash1];
if (ip1 >= nextStep) {
step++;
nextStep += kStepIncr;
}
ip0 = ip1;
ip1 = ip1 + step;
if (ip1 > ilimit) goto _cleanup;
curr = (U32)(ip0 - base);
hash0 = hash1;
} /* end inner search loop */
/* match found */
assert(mLength);
ip0 += mLength;
anchor = ip0;
if (ip0 <= ilimit) {
/* Fill Table */
assert(base+curr+2 > istart); /* check base overflow */
hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; /* here because curr+2 could be > iend-8 */
hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base);
/* check immediate repcode */
while (ip0 <= ilimit) {
U32 const current2 = (U32)(ip0-base);
U32 const repIndex2 = current2 - offset_2;
const BYTE* repMatch2 = repIndex2 < prefixStartIndex ?
dictBase - dictIndexDelta + repIndex2 :
base + repIndex2;
if ( (ZSTD_index_overlap_check(prefixStartIndex, repIndex2))
&& (MEM_read32(repMatch2) == MEM_read32(ip0))) {
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = current2;
ip0 += repLength2;
anchor = ip0;
continue;
}
break;
}
}
/* Prepare for next iteration */
assert(ip0 == anchor);
ip1 = ip0 + stepSize;
}
_cleanup:
/* save reps for next block */
rep[0] = offset_1;
rep[1] = offset_2;
/* Return the last literals size */
return (size_t)(iend - anchor);
}
ZSTD_GEN_FAST_FN(dictMatchState, 4, 0)
ZSTD_GEN_FAST_FN(dictMatchState, 5, 0)
ZSTD_GEN_FAST_FN(dictMatchState, 6, 0)
ZSTD_GEN_FAST_FN(dictMatchState, 7, 0)
size_t ZSTD_compressBlock_fast_dictMatchState(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
const void* src, size_t srcSize)
{
U32 const mls = ms->cParams.minMatch;
assert(ms->dictMatchState != NULL);
switch(mls)
{
default: /* includes case 3 */
case 4 :
return ZSTD_compressBlock_fast_dictMatchState_4_0(ms, seqStore, rep, src, srcSize);
case 5 :
return ZSTD_compressBlock_fast_dictMatchState_5_0(ms, seqStore, rep, src, srcSize);
case 6 :
return ZSTD_compressBlock_fast_dictMatchState_6_0(ms, seqStore, rep, src, srcSize);
case 7 :
return ZSTD_compressBlock_fast_dictMatchState_7_0(ms, seqStore, rep, src, srcSize);
}
const ZSTD_MatchState_t* const dms = ms->dictMatchState;
assert(dms != NULL);
return ZSTD_rust_compressBlock_fast_dictMatchState(
ms->hashTable, ms->window.base, ms->window.dictLimit,
ms->cParams.hashLog, ms->cParams.minMatch, ms->cParams.targetLength,
seqStore, rep, src, srcSize,
dms->hashTable, dms->window.base, dms->window.dictLimit,
dms->window.nextSrc, dms->cParams.hashLog, ms->prefetchCDictTables);
}
static
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_compressBlock_fast_extDict_generic(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize, U32 const mls, U32 const hasStep)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashTable = ms->hashTable;
U32 const hlog = cParams->hashLog;
/* support stepSize of 0 */
size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1;
const BYTE* const base = ms->window.base;
const BYTE* const dictBase = ms->window.dictBase;
const BYTE* const istart = (const BYTE*)src;
const BYTE* anchor = istart;
const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog);
const U32 dictStartIndex = lowLimit;
const BYTE* const dictStart = dictBase + dictStartIndex;
const U32 dictLimit = ms->window.dictLimit;
const U32 prefixStartIndex = dictLimit < lowLimit ? lowLimit : dictLimit;
const BYTE* const prefixStart = base + prefixStartIndex;
const BYTE* const dictEnd = dictBase + prefixStartIndex;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
U32 offset_1=rep[0], offset_2=rep[1];
U32 offsetSaved1 = 0, offsetSaved2 = 0;
const BYTE* ip0 = istart;
const BYTE* ip1;
const BYTE* ip2;
const BYTE* ip3;
U32 current0;
size_t hash0; /* hash for ip0 */
size_t hash1; /* hash for ip1 */
U32 idx; /* match idx for ip0 */
const BYTE* idxBase; /* base pointer for idx */
U32 offcode;
const BYTE* match0;
size_t mLength;
const BYTE* matchEnd = 0; /* initialize to avoid warning, assert != 0 later */
size_t step;
const BYTE* nextStep;
const size_t kStepIncr = (1 << (kSearchStrength - 1));
(void)hasStep; /* not currently specialized on whether it's accelerated */
DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic (offset_1=%u)", offset_1);
/* switch to "regular" variant if extDict is invalidated due to maxDistance */
if (prefixStartIndex == dictStartIndex)
return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize);
{ U32 const curr = (U32)(ip0 - base);
U32 const maxRep = curr - dictStartIndex;
if (offset_2 >= maxRep) offsetSaved2 = offset_2, offset_2 = 0;
if (offset_1 >= maxRep) offsetSaved1 = offset_1, offset_1 = 0;
}
/* start each op */
_start: /* Requires: ip0 */
step = stepSize;
nextStep = ip0 + kStepIncr;
/* calculate positions, ip0 - anchor == 0, so we skip step calc */
ip1 = ip0 + 1;
ip2 = ip0 + step;
ip3 = ip2 + 1;
if (ip3 >= ilimit) {
goto _cleanup;
}
hash0 = ZSTD_hashPtr(ip0, hlog, mls);
hash1 = ZSTD_hashPtr(ip1, hlog, mls);
idx = hashTable[hash0];
idxBase = idx < prefixStartIndex ? dictBase : base;
do {
{ /* load repcode match for ip[2] */
U32 const current2 = (U32)(ip2 - base);
U32 const repIndex = current2 - offset_1;
const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base;
U32 rval;
if ( ((U32)(prefixStartIndex - repIndex) >= 4) /* intentional underflow */
& (offset_1 > 0) ) {
rval = MEM_read32(repBase + repIndex);
} else {
rval = MEM_read32(ip2) ^ 1; /* guaranteed to not match. */
}
/* write back hash table entry */
current0 = (U32)(ip0 - base);
hashTable[hash0] = current0;
/* check repcode at ip[2] */
if (MEM_read32(ip2) == rval) {
ip0 = ip2;
match0 = repBase + repIndex;
matchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
assert((match0 != prefixStart) & (match0 != dictStart));
mLength = ip0[-1] == match0[-1];
ip0 -= mLength;
match0 -= mLength;
offcode = REPCODE1_TO_OFFBASE;
mLength += 4;
goto _match;
} }
{ /* load match for ip[0] */
U32 const mval = idx >= dictStartIndex ?
MEM_read32(idxBase + idx) :
MEM_read32(ip0) ^ 1; /* guaranteed not to match */
/* check match at ip[0] */
if (MEM_read32(ip0) == mval) {
/* found a match! */
goto _offset;
} }
/* lookup ip[1] */
idx = hashTable[hash1];
idxBase = idx < prefixStartIndex ? dictBase : base;
/* hash ip[2] */
hash0 = hash1;
hash1 = ZSTD_hashPtr(ip2, hlog, mls);
/* advance to next positions */
ip0 = ip1;
ip1 = ip2;
ip2 = ip3;
/* write back hash table entry */
current0 = (U32)(ip0 - base);
hashTable[hash0] = current0;
{ /* load match for ip[0] */
U32 const mval = idx >= dictStartIndex ?
MEM_read32(idxBase + idx) :
MEM_read32(ip0) ^ 1; /* guaranteed not to match */
/* check match at ip[0] */
if (MEM_read32(ip0) == mval) {
/* found a match! */
goto _offset;
} }
/* lookup ip[1] */
idx = hashTable[hash1];
idxBase = idx < prefixStartIndex ? dictBase : base;
/* hash ip[2] */
hash0 = hash1;
hash1 = ZSTD_hashPtr(ip2, hlog, mls);
/* advance to next positions */
ip0 = ip1;
ip1 = ip2;
ip2 = ip0 + step;
ip3 = ip1 + step;
/* calculate step */
if (ip2 >= nextStep) {
step++;
PREFETCH_L1(ip1 + 64);
PREFETCH_L1(ip1 + 128);
nextStep += kStepIncr;
}
} while (ip3 < ilimit);
_cleanup:
/* Note that there are probably still a couple positions we could search.
* However, it seems to be a meaningful performance hit to try to search
* them. So let's not. */
/* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0),
* rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */
offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2;
/* save reps for next block */
rep[0] = offset_1 ? offset_1 : offsetSaved1;
rep[1] = offset_2 ? offset_2 : offsetSaved2;
/* Return the last literals size */
return (size_t)(iend - anchor);
_offset: /* Requires: ip0, idx, idxBase */
/* Compute the offset code. */
{ U32 const offset = current0 - idx;
const BYTE* const lowMatchPtr = idx < prefixStartIndex ? dictStart : prefixStart;
matchEnd = idx < prefixStartIndex ? dictEnd : iend;
match0 = idxBase + idx;
offset_2 = offset_1;
offset_1 = offset;
offcode = OFFSET_TO_OFFBASE(offset);
mLength = 4;
/* Count the backwards match length. */
while (((ip0>anchor) & (match0>lowMatchPtr)) && (ip0[-1] == match0[-1])) {
ip0--;
match0--;
mLength++;
} }
_match: /* Requires: ip0, match0, offcode, matchEnd */
/* Count the forward length. */
assert(matchEnd != 0);
mLength += ZSTD_count_2segments(ip0 + mLength, match0 + mLength, iend, matchEnd, prefixStart);
ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength);
ip0 += mLength;
anchor = ip0;
/* write next hash table entry */
if (ip1 < ip0) {
hashTable[hash1] = (U32)(ip1 - base);
}
/* Fill table and check for immediate repcode. */
if (ip0 <= ilimit) {
/* Fill Table */
assert(base+current0+2 > istart); /* check base overflow */
hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */
hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base);
while (ip0 <= ilimit) {
U32 const repIndex2 = (U32)(ip0-base) - offset_2;
const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
if ( ((ZSTD_index_overlap_check(prefixStartIndex, repIndex2)) & (offset_2 > 0))
&& (MEM_read32(repMatch2) == MEM_read32(ip0)) ) {
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
{ U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base);
ip0 += repLength2;
anchor = ip0;
continue;
}
break;
} }
goto _start;
}
ZSTD_GEN_FAST_FN(extDict, 4, 0)
ZSTD_GEN_FAST_FN(extDict, 5, 0)
ZSTD_GEN_FAST_FN(extDict, 6, 0)
ZSTD_GEN_FAST_FN(extDict, 7, 0)
size_t ZSTD_compressBlock_fast_extDict(
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
const void* src, size_t srcSize)
{
U32 const mls = ms->cParams.minMatch;
assert(ms->dictMatchState == NULL);
switch(mls)
{
default: /* includes case 3 */
case 4 :
return ZSTD_compressBlock_fast_extDict_4_0(ms, seqStore, rep, src, srcSize);
case 5 :
return ZSTD_compressBlock_fast_extDict_5_0(ms, seqStore, rep, src, srcSize);
case 6 :
return ZSTD_compressBlock_fast_extDict_6_0(ms, seqStore, rep, src, srcSize);
case 7 :
return ZSTD_compressBlock_fast_extDict_7_0(ms, seqStore, rep, src, srcSize);
}
return ZSTD_rust_compressBlock_fast_extDict(
ms->hashTable, ms->window.base, ms->window.dictBase,
ms->window.dictLimit, ms->window.lowLimit, ms->loadedDictEnd,
ms->cParams.hashLog, ms->cParams.minMatch,
ms->cParams.targetLength, ms->cParams.windowLog,
seqStore, rep, src, srcSize);
}
rust/src/lib.rs
+2
View File
@@ -30,4 +30,6 @@ pub mod zstd_compress_superblock;
#[cfg(feature = "decompression")]
pub mod zstd_ddict;
#[cfg(feature = "compression")]
pub mod zstd_fast;
#[cfg(feature = "compression")]
pub mod zstd_presplit;
rust/src/zstd_fast.rs
+1599
View File
@@ -0,0 +1,1599 @@
#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::too_many_arguments)]
//! Fast block match finder.
//!
//! The C translation unit keeps the public/internal ABI entry points and
//! extracts the fields it needs from `ZSTD_MatchState_t`. This module owns
//! the hash-table search and sequence-generation algorithms themselves. That
//! boundary keeps the large, evolving match-state opaque while retaining the
//! hot matching loops in Rust.
use crate::mem::{
MEM_64bits, MEM_isLittleEndian, MEM_read16, MEM_read32, MEM_readLE32, MEM_readLE64, MEM_readST,
};
use std::ffi::c_void;
use std::mem::size_of;
use std::os::raw::c_int;
use std::ptr;
const ZSTD_REP_NUM: usize = 3;
const MINMATCH: usize = 3;
const HASH_READ_SIZE: usize = 8;
const SHORT_CACHE_TAG_BITS: u32 = 8;
const SHORT_CACHE_TAG_MASK: u32 = (1 << SHORT_CACHE_TAG_BITS) - 1;
const K_SEARCH_STRENGTH: usize = 8;
const REPCODE1_TO_OFFBASE: u32 = 1;
#[repr(C)]
#[derive(Clone, Copy)]
struct SeqDef {
offBase: u32,
litLength: u16,
mlBase: u16,
}
/// The only leaf layout written by the matcher. The C shim verifies this
/// shape at compile time; `ZSTD_MatchState_t` itself never crosses the FFI.
#[repr(C)]
struct SeqStore_t {
sequencesStart: *mut SeqDef,
sequences: *mut SeqDef,
litStart: *mut u8,
lit: *mut u8,
llCode: *mut u8,
mlCode: *mut u8,
ofCode: *mut u8,
maxNbSeq: usize,
maxNbLit: usize,
longLengthType: c_int,
longLengthPos: u32,
}
#[inline]
fn ptr_lt(left: *const u8, right: *const u8) -> bool {
(left as usize) < (right as usize)
}
#[inline]
fn ptr_le(left: *const u8, right: *const u8) -> bool {
(left as usize) <= (right as usize)
}
#[inline]
fn ptr_ge(left: *const u8, right: *const u8) -> bool {
(left as usize) >= (right as usize)
}
#[inline]
fn ptr_gt(left: *const u8, right: *const u8) -> bool {
(left as usize) > (right as usize)
}
#[inline]
unsafe fn index_from(base: *const u8, ptr: *const u8) -> u32 {
unsafe { ptr.offset_from(base) as u32 }
}
#[inline]
unsafe fn read32(ptr: *const u8) -> u32 {
unsafe { MEM_read32(ptr.cast::<c_void>()) }
}
#[inline]
unsafe fn table_get(table: *const u32, index: usize) -> u32 {
unsafe { *table.add(index) }
}
#[inline]
unsafe fn table_set(table: *mut u32, index: usize, value: u32) {
unsafe { *table.add(index) = value };
}
#[inline]
fn hash_shift32(value: u32, hbits: u32) -> usize {
if hbits == 0 {
0
} else {
(value >> (32 - hbits)) as usize
}
}
#[inline]
fn hash_shift64(value: u64, hbits: u32) -> usize {
if hbits == 0 {
0
} else {
(value >> (64 - hbits)) as usize
}
}
#[inline]
unsafe fn hash_ptr(ptr: *const u8, hbits: u32, mls: u32) -> usize {
match mls {
5 => {
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
hash_shift64(value.wrapping_shl(24).wrapping_mul(889_523_592_379), hbits)
}
6 => {
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
hash_shift64(
value.wrapping_shl(16).wrapping_mul(227_718_039_650_203),
hbits,
)
}
7 => {
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
hash_shift64(
value.wrapping_shl(8).wrapping_mul(58_295_818_150_454_627),
hbits,
)
}
8 => {
let value = unsafe { MEM_readLE64(ptr.cast::<c_void>()) };
hash_shift64(value.wrapping_mul(0xCF1B_BCDC_B7A5_6463), hbits)
}
_ => {
let value = unsafe { MEM_readLE32(ptr.cast::<c_void>()) };
hash_shift32(value.wrapping_mul(2_654_435_761), hbits)
}
}
}
#[inline]
fn common_bytes(word: usize) -> usize {
let zeros = if MEM_isLittleEndian() {
word.trailing_zeros()
} else {
word.leading_zeros()
};
(zeros / 8) as usize
}
/// Equivalent to C's `ZSTD_count()`, including its word-at-a-time fast path.
unsafe fn count(mut input: *const u8, mut matched: *const u8, input_limit: *const u8) -> usize {
let input_start = input;
let word_size = size_of::<usize>();
while unsafe { input_limit.offset_from(input) as usize } >= word_size {
let diff =
unsafe { MEM_readST(matched.cast::<c_void>()) ^ MEM_readST(input.cast::<c_void>()) };
if diff != 0 {
return unsafe { input.offset_from(input_start) as usize } + common_bytes(diff);
}
input = input.wrapping_add(word_size);
matched = matched.wrapping_add(word_size);
}
if MEM_64bits()
&& unsafe { input_limit.offset_from(input) as usize } >= 4
&& unsafe { MEM_read32(matched.cast::<c_void>()) == MEM_read32(input.cast::<c_void>()) }
{
input = input.wrapping_add(4);
matched = matched.wrapping_add(4);
}
if unsafe { input_limit.offset_from(input) as usize } >= 2
&& unsafe { MEM_read16(matched.cast::<c_void>()) == MEM_read16(input.cast::<c_void>()) }
{
input = input.wrapping_add(2);
matched = matched.wrapping_add(2);
}
if ptr_lt(input, input_limit) && unsafe { *matched == *input } {
input = input.wrapping_add(1);
}
unsafe { input.offset_from(input_start) as usize }
}
unsafe fn count_2segments(
input: *const u8,
matched: *const u8,
input_end: *const u8,
match_end: *const u8,
input_start: *const u8,
) -> usize {
let match_remaining = unsafe { match_end.offset_from(matched) as usize };
let input_remaining = unsafe { input_end.offset_from(input) as usize };
let first_end = input.wrapping_add(match_remaining.min(input_remaining));
let first_count = unsafe { count(input, matched, first_end) };
if matched.wrapping_add(first_count) != match_end {
return first_count;
}
first_count + unsafe { count(input.wrapping_add(first_count), input_start, input_end) }
}
#[inline]
fn lowest_prefix_index(dict_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 {
let max_distance = 1u32.wrapping_shl(window_log);
let within_window = if curr.wrapping_sub(dict_limit) > max_distance {
curr.wrapping_sub(max_distance)
} else {
dict_limit
};
if loaded_dict_end != 0 {
dict_limit
} else {
within_window
}
}
#[inline]
fn lowest_match_index(low_limit: u32, loaded_dict_end: u32, curr: u32, window_log: u32) -> u32 {
let max_distance = 1u32.wrapping_shl(window_log);
let within_window = if curr.wrapping_sub(low_limit) > max_distance {
curr.wrapping_sub(max_distance)
} else {
low_limit
};
if loaded_dict_end != 0 {
low_limit
} else {
within_window
}
}
#[inline]
fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool {
prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3
}
#[inline]
fn write_tagged_index(table: *mut u32, hash_and_tag: usize, index: u32) {
let hash = hash_and_tag >> SHORT_CACHE_TAG_BITS;
let tag = (hash_and_tag as u32) & SHORT_CACHE_TAG_MASK;
unsafe { table_set(table, hash, (index << SHORT_CACHE_TAG_BITS) | tag) };
}
#[inline]
fn packed_tags_match(first: u32, second: usize) -> bool {
(first & SHORT_CACHE_TAG_MASK) == ((second as u32) & SHORT_CACHE_TAG_MASK)
}
/// Stores a sequence using the C `SeqStore_t` ABI. Copying exactly the
/// literal range is equivalent to C's over-copying wildcopy path for all
/// observable sequence-store bytes and avoids speculative reads in Rust.
unsafe fn store_seq(
seq_store: *mut SeqStore_t,
lit_length: usize,
literals: *const u8,
_lit_limit: *const u8,
off_base: u32,
match_length: usize,
) {
let seq_store = unsafe { &mut *seq_store };
let sequence = seq_store.sequences;
debug_assert!(
unsafe { sequence.offset_from(seq_store.sequencesStart) as usize } < seq_store.maxNbSeq
);
debug_assert!(lit_length <= seq_store.maxNbLit);
debug_assert!(match_length >= MINMATCH);
if lit_length != 0 {
unsafe { ptr::copy_nonoverlapping(literals, seq_store.lit, lit_length) };
}
seq_store.lit = seq_store.lit.wrapping_add(lit_length);
let sequence_index = unsafe { sequence.offset_from(seq_store.sequencesStart) as u32 };
if lit_length > u16::MAX as usize {
debug_assert_eq!(seq_store.longLengthType, 0);
seq_store.longLengthType = 1;
seq_store.longLengthPos = sequence_index;
}
unsafe { (*sequence).litLength = lit_length as u16 };
unsafe { (*sequence).offBase = off_base };
let match_base = match_length - MINMATCH;
if match_base > u16::MAX as usize {
debug_assert_eq!(seq_store.longLengthType, 0);
seq_store.longLengthType = 2;
seq_store.longLengthPos = sequence_index;
}
unsafe { (*sequence).mlBase = match_base as u16 };
seq_store.sequences = sequence.wrapping_add(1);
}
#[inline]
unsafe fn match4_found(
current: *const u8,
match_address: *const u8,
match_index: u32,
low_limit: u32,
) -> bool {
match_index >= low_limit && unsafe { read32(current) == read32(match_address) }
}
unsafe fn finish_no_dict_match(
hash_table: *mut u32,
base: *const u8,
hash_log: u32,
mls: u32,
seq_store: *mut SeqStore_t,
rep_offset1: &mut u32,
rep_offset2: &mut u32,
mut ip0: *const u8,
match0: *const u8,
anchor: *const u8,
iend: *const u8,
ilimit: *const u8,
current0: u32,
offcode: u32,
mut match_length: usize,
) -> (*const u8, *const u8) {
match_length += unsafe {
count(
ip0.wrapping_add(match_length),
match0.wrapping_add(match_length),
iend,
)
};
unsafe {
store_seq(
seq_store,
ip0.offset_from(anchor) as usize,
anchor,
iend,
offcode,
match_length,
)
};
ip0 = ip0.wrapping_add(match_length);
let mut new_anchor = ip0;
if ptr_le(ip0, ilimit) {
let table_hash =
unsafe { hash_ptr(base.wrapping_add(current0 as usize + 2), hash_log, mls) };
unsafe { table_set(hash_table, table_hash, current0.wrapping_add(2)) };
let table_hash = unsafe { hash_ptr(ip0.wrapping_sub(2), hash_log, mls) };
unsafe {
table_set(
hash_table,
table_hash,
index_from(base, ip0.wrapping_sub(2)),
)
};
if *rep_offset2 > 0 {
while ptr_le(ip0, ilimit)
&& unsafe { read32(ip0) == read32(ip0.wrapping_sub(*rep_offset2 as usize)) }
{
let repeat_length = unsafe {
count(
ip0.wrapping_add(4),
ip0.wrapping_add(4).wrapping_sub(*rep_offset2 as usize),
iend,
) + 4
};
std::mem::swap(rep_offset1, rep_offset2);
let table_hash = unsafe { hash_ptr(ip0, hash_log, mls) };
unsafe { table_set(hash_table, table_hash, index_from(base, ip0)) };
unsafe {
store_seq(
seq_store,
0,
new_anchor,
iend,
REPCODE1_TO_OFFBASE,
repeat_length,
)
};
ip0 = ip0.wrapping_add(repeat_length);
new_anchor = ip0;
}
}
}
(ip0, new_anchor)
}
unsafe fn compress_block_fast_no_dict(
hash_table: *mut u32,
base: *const u8,
dict_limit: u32,
loaded_dict_end: u32,
hash_log: u32,
target_length: u32,
window_log: u32,
seq_store: *mut SeqStore_t,
reps: *mut u32,
src: *const u8,
src_size: usize,
mls: u32,
_use_cmov: bool,
) -> usize {
if src_size < HASH_READ_SIZE {
return src_size;
}
let istart = src;
let iend = istart.wrapping_add(src_size);
let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
let end_index = unsafe { index_from(base, istart) }.wrapping_add(src_size as u32);
let prefix_start_index =
lowest_prefix_index(dict_limit, loaded_dict_end, end_index, window_log);
let prefix_start = base.wrapping_add(prefix_start_index as usize);
let step_size = target_length as usize + usize::from(target_length == 0) + 1;
let mut ip0 = istart;
let mut anchor = istart;
let mut rep_offset1 = unsafe { *reps };
let mut rep_offset2 = unsafe { *reps.add(1) };
let mut offset_saved1 = 0u32;
let mut offset_saved2 = 0u32;
if ip0 == prefix_start {
ip0 = ip0.wrapping_add(1);
}
let current = unsafe { index_from(base, ip0) };
let window_low = lowest_prefix_index(dict_limit, loaded_dict_end, current, window_log);
let max_rep = current.wrapping_sub(window_low);
if rep_offset2 > max_rep {
offset_saved2 = rep_offset2;
rep_offset2 = 0;
}
if rep_offset1 > max_rep {
offset_saved1 = rep_offset1;
rep_offset1 = 0;
}
'start: loop {
let mut step = step_size;
let mut next_step = ip0.wrapping_add(1 << (K_SEARCH_STRENGTH - 1));
let mut ip1 = ip0.wrapping_add(1);
let mut ip2 = ip0.wrapping_add(step);
let mut ip3 = ip2.wrapping_add(1);
if ptr_ge(ip3, ilimit) {
break;
}
let mut hash0 = unsafe { hash_ptr(ip0, hash_log, mls) };
let mut hash1 = unsafe { hash_ptr(ip1, hash_log, mls) };
let mut match_index = unsafe { table_get(hash_table, hash0) };
loop {
let rval = if rep_offset1 > 0 {
unsafe { read32(ip2.wrapping_sub(rep_offset1 as usize)) }
} else {
unsafe { read32(ip2) }
};
let mut current0 = unsafe { index_from(base, ip0) };
unsafe { table_set(hash_table, hash0, current0) };
if rep_offset1 > 0 && unsafe { read32(ip2) == rval } {
ip0 = ip2;
let mut match0 = ip0.wrapping_sub(rep_offset1 as usize);
let match_length =
usize::from(unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) });
ip0 = ip0.wrapping_sub(match_length);
match0 = match0.wrapping_sub(match_length);
unsafe { table_set(hash_table, hash1, index_from(base, ip1)) };
let (new_ip0, new_anchor) = unsafe {
finish_no_dict_match(
hash_table,
base,
hash_log,
mls,
seq_store,
&mut rep_offset1,
&mut rep_offset2,
ip0,
match0,
anchor,
iend,
ilimit,
current0,
REPCODE1_TO_OFFBASE,
match_length + 4,
)
};
ip0 = new_ip0;
anchor = new_anchor;
continue 'start;
}
if unsafe {
match4_found(
ip0,
base.wrapping_add(match_index as usize),
match_index,
prefix_start_index,
)
} {
unsafe { table_set(hash_table, hash1, index_from(base, ip1)) };
let mut match0 = base.wrapping_add(match_index as usize);
rep_offset2 = rep_offset1;
rep_offset1 = unsafe { index_from(match0, ip0) };
let mut match_length = 4usize;
while ptr_gt(ip0, anchor)
&& ptr_gt(match0, prefix_start)
&& unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) }
{
ip0 = ip0.wrapping_sub(1);
match0 = match0.wrapping_sub(1);
match_length += 1;
}
let offcode = rep_offset1.wrapping_add(ZSTD_REP_NUM as u32);
let (new_ip0, new_anchor) = unsafe {
finish_no_dict_match(
hash_table,
base,
hash_log,
mls,
seq_store,
&mut rep_offset1,
&mut rep_offset2,
ip0,
match0,
anchor,
iend,
ilimit,
current0,
offcode,
match_length,
)
};
ip0 = new_ip0;
anchor = new_anchor;
continue 'start;
}
match_index = unsafe { table_get(hash_table, hash1) };
hash0 = hash1;
hash1 = unsafe { hash_ptr(ip2, hash_log, mls) };
ip0 = ip1;
ip1 = ip2;
ip2 = ip3;
current0 = unsafe { index_from(base, ip0) };
unsafe { table_set(hash_table, hash0, current0) };
if unsafe {
match4_found(
ip0,
base.wrapping_add(match_index as usize),
match_index,
prefix_start_index,
)
} {
if step <= 4 {
unsafe { table_set(hash_table, hash1, index_from(base, ip1)) };
}
let mut match0 = base.wrapping_add(match_index as usize);
rep_offset2 = rep_offset1;
rep_offset1 = unsafe { index_from(match0, ip0) };
let mut match_length = 4usize;
while ptr_gt(ip0, anchor)
&& ptr_gt(match0, prefix_start)
&& unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) }
{
ip0 = ip0.wrapping_sub(1);
match0 = match0.wrapping_sub(1);
match_length += 1;
}
let offcode = rep_offset1.wrapping_add(ZSTD_REP_NUM as u32);
let (new_ip0, new_anchor) = unsafe {
finish_no_dict_match(
hash_table,
base,
hash_log,
mls,
seq_store,
&mut rep_offset1,
&mut rep_offset2,
ip0,
match0,
anchor,
iend,
ilimit,
current0,
offcode,
match_length,
)
};
ip0 = new_ip0;
anchor = new_anchor;
continue 'start;
}
match_index = unsafe { table_get(hash_table, hash1) };
hash0 = hash1;
hash1 = unsafe { hash_ptr(ip2, hash_log, mls) };
ip0 = ip1;
ip1 = ip2;
ip2 = ip0.wrapping_add(step);
ip3 = ip1.wrapping_add(step);
if ptr_ge(ip2, next_step) {
step += 1;
next_step = next_step.wrapping_add(1 << (K_SEARCH_STRENGTH - 1));
}
if ptr_ge(ip3, ilimit) {
break 'start;
}
}
}
offset_saved2 = if offset_saved1 != 0 && rep_offset1 != 0 {
offset_saved1
} else {
offset_saved2
};
unsafe {
*reps = if rep_offset1 != 0 {
rep_offset1
} else {
offset_saved1
};
*reps.add(1) = if rep_offset2 != 0 {
rep_offset2
} else {
offset_saved2
};
}
unsafe { iend.offset_from(anchor) as usize }
}
unsafe fn fill_hash_table(
hash_table: *mut u32,
base: *const u8,
next_to_update: u32,
end: *const u8,
hash_log: u32,
min_match: u32,
full_table_load: bool,
tagged_indices: bool,
) {
if unsafe { end.offset_from(base) } < HASH_READ_SIZE as isize {
return;
}
let hbits = hash_log
+ if tagged_indices {
SHORT_CACHE_TAG_BITS
} else {
0
};
let mut input = base.wrapping_add(next_to_update as usize);
let input_end = end.wrapping_sub(HASH_READ_SIZE);
while ptr_lt(input.wrapping_add(3), input_end.wrapping_add(2)) {
let current = unsafe { index_from(base, input) };
let hash_and_tag = unsafe { hash_ptr(input, hbits, min_match) };
if tagged_indices {
write_tagged_index(hash_table, hash_and_tag, current);
} else {
unsafe { table_set(hash_table, hash_and_tag, current) };
}
if full_table_load {
for position in 1..3usize {
let hash_and_tag =
unsafe { hash_ptr(input.wrapping_add(position), hbits, min_match) };
let table_index = if tagged_indices {
hash_and_tag >> SHORT_CACHE_TAG_BITS
} else {
hash_and_tag
};
if unsafe { table_get(hash_table, table_index) } == 0 {
if tagged_indices {
write_tagged_index(
hash_table,
hash_and_tag,
current.wrapping_add(position as u32),
);
} else {
unsafe {
table_set(
hash_table,
table_index,
current.wrapping_add(position as u32),
)
};
}
}
}
}
input = input.wrapping_add(3);
}
}
#[inline]
fn fast_mls(min_match: u32) -> u32 {
match min_match {
5..=7 => min_match,
_ => 4,
}
}
/// Rust implementation called by the C ABI wrapper for `ZSTD_fillHashTable`.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_fillHashTable(
hash_table: *mut u32,
base: *const u8,
next_to_update: u32,
end: *const c_void,
hash_log: u32,
min_match: u32,
full_table_load: c_int,
for_cdict: c_int,
) {
unsafe {
fill_hash_table(
hash_table,
base,
next_to_update,
end.cast::<u8>(),
hash_log,
min_match,
full_table_load != 0,
for_cdict != 0,
)
};
}
/// Rust implementation called by the C ABI wrapper for `ZSTD_compressBlock_fast`.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBlock_fast(
hash_table: *mut u32,
base: *const u8,
dict_limit: u32,
loaded_dict_end: u32,
hash_log: u32,
min_match: u32,
target_length: u32,
window_log: u32,
seq_store: *mut c_void,
reps: *mut u32,
src: *const c_void,
src_size: usize,
) -> usize {
unsafe {
compress_block_fast_no_dict(
hash_table,
base,
dict_limit,
loaded_dict_end,
hash_log,
target_length,
window_log,
seq_store.cast::<SeqStore_t>(),
reps,
src.cast::<u8>(),
src_size,
fast_mls(min_match),
window_log < 19,
)
}
}
unsafe fn compress_block_fast_dict_match_state(
hash_table: *mut u32,
base: *const u8,
prefix_start_index: u32,
hash_log: u32,
target_length: u32,
seq_store: *mut SeqStore_t,
reps: *mut u32,
src: *const u8,
src_size: usize,
mls: u32,
dict_hash_table: *const u32,
dict_base: *const u8,
dict_start_index: u32,
dict_end: *const u8,
dict_hash_log: u32,
_prefetch_cdict_tables: bool,
) -> usize {
if src_size < HASH_READ_SIZE {
return src_size;
}
let istart = src;
let iend = istart.wrapping_add(src_size);
let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
let prefix_start = base.wrapping_add(prefix_start_index as usize);
let dict_start = dict_base.wrapping_add(dict_start_index as usize);
let dict_index_delta =
prefix_start_index.wrapping_sub(unsafe { index_from(dict_base, dict_end) });
let dict_and_prefix_length = unsafe {
istart.offset_from(prefix_start) as u32 + dict_end.offset_from(dict_start) as u32
};
let dict_hbits = dict_hash_log + SHORT_CACHE_TAG_BITS;
let step_size = target_length + u32::from(target_length == 0);
let mut ip0 = istart;
let mut ip1 = ip0.wrapping_add(step_size as usize);
let mut anchor = istart;
let mut offset1 = unsafe { *reps };
let mut offset2 = unsafe { *reps.add(1) };
if dict_and_prefix_length == 0 {
ip0 = ip0.wrapping_add(1);
}
'outer: while ptr_le(ip1, ilimit) {
let mut match_length: usize;
let mut hash0 = unsafe { hash_ptr(ip0, hash_log, mls) };
let dict_hash_and_tag0 = unsafe { hash_ptr(ip0, dict_hbits, mls) };
let mut dict_match_index_and_tag =
unsafe { table_get(dict_hash_table, dict_hash_and_tag0 >> SHORT_CACHE_TAG_BITS) };
let mut dict_tags_match = packed_tags_match(dict_match_index_and_tag, dict_hash_and_tag0);
let mut match_index = unsafe { table_get(hash_table, hash0) };
let mut current = unsafe { index_from(base, ip0) };
let mut step = step_size as usize;
let mut next_step = ip0.wrapping_add(1 << K_SEARCH_STRENGTH);
loop {
let match_ptr = base.wrapping_add(match_index as usize);
let rep_index = current.wrapping_add(1).wrapping_sub(offset1);
let rep_match = if rep_index < prefix_start_index {
dict_base.wrapping_add(rep_index.wrapping_sub(dict_index_delta) as usize)
} else {
base.wrapping_add(rep_index as usize)
};
let hash1 = unsafe { hash_ptr(ip1, hash_log, mls) };
let dict_hash_and_tag1 = unsafe { hash_ptr(ip1, dict_hbits, mls) };
unsafe { table_set(hash_table, hash0, current) };
if index_overlap_check(prefix_start_index, rep_index)
&& unsafe { read32(rep_match) == read32(ip0.wrapping_add(1)) }
{
let rep_match_end = if rep_index < prefix_start_index {
dict_end
} else {
iend
};
match_length = unsafe {
count_2segments(
ip0.wrapping_add(5),
rep_match.wrapping_add(4),
iend,
rep_match_end,
prefix_start,
) + 4
};
ip0 = ip0.wrapping_add(1);
unsafe {
store_seq(
seq_store,
ip0.offset_from(anchor) as usize,
anchor,
iend,
REPCODE1_TO_OFFBASE,
match_length,
)
};
break;
}
if dict_tags_match {
let dict_match_index = dict_match_index_and_tag >> SHORT_CACHE_TAG_BITS;
let mut dict_match = dict_base.wrapping_add(dict_match_index as usize);
if dict_match_index > dict_start_index
&& unsafe { read32(dict_match) == read32(ip0) }
&& match_index <= prefix_start_index
{
let offset = current
.wrapping_sub(dict_match_index)
.wrapping_sub(dict_index_delta);
match_length = unsafe {
count_2segments(
ip0.wrapping_add(4),
dict_match.wrapping_add(4),
iend,
dict_end,
prefix_start,
) + 4
};
while ptr_gt(ip0, anchor)
&& ptr_gt(dict_match, dict_start)
&& unsafe { *ip0.wrapping_sub(1) == *dict_match.wrapping_sub(1) }
{
ip0 = ip0.wrapping_sub(1);
dict_match = dict_match.wrapping_sub(1);
match_length += 1;
}
offset2 = offset1;
offset1 = offset;
unsafe {
store_seq(
seq_store,
ip0.offset_from(anchor) as usize,
anchor,
iend,
offset.wrapping_add(ZSTD_REP_NUM as u32),
match_length,
)
};
break;
}
}
if unsafe { match4_found(ip0, match_ptr, match_index, prefix_start_index) } {
let offset = unsafe { index_from(match_ptr, ip0) };
match_length =
unsafe { count(ip0.wrapping_add(4), match_ptr.wrapping_add(4), iend) + 4 };
let mut matched = match_ptr;
while ptr_gt(ip0, anchor)
&& ptr_gt(matched, prefix_start)
&& unsafe { *ip0.wrapping_sub(1) == *matched.wrapping_sub(1) }
{
ip0 = ip0.wrapping_sub(1);
matched = matched.wrapping_sub(1);
match_length += 1;
}
offset2 = offset1;
offset1 = offset;
unsafe {
store_seq(
seq_store,
ip0.offset_from(anchor) as usize,
anchor,
iend,
offset.wrapping_add(ZSTD_REP_NUM as u32),
match_length,
)
};
break;
}
dict_match_index_and_tag =
unsafe { table_get(dict_hash_table, dict_hash_and_tag1 >> SHORT_CACHE_TAG_BITS) };
dict_tags_match = packed_tags_match(dict_match_index_and_tag, dict_hash_and_tag1);
match_index = unsafe { table_get(hash_table, hash1) };
if ptr_ge(ip1, next_step) {
step += 1;
next_step = next_step.wrapping_add(1 << K_SEARCH_STRENGTH);
}
ip0 = ip1;
ip1 = ip1.wrapping_add(step);
if ptr_gt(ip1, ilimit) {
break 'outer;
}
current = unsafe { index_from(base, ip0) };
hash0 = hash1;
}
ip0 = ip0.wrapping_add(match_length);
anchor = ip0;
if ptr_le(ip0, ilimit) {
let table_hash =
unsafe { hash_ptr(base.wrapping_add(current as usize + 2), hash_log, mls) };
unsafe { table_set(hash_table, table_hash, current.wrapping_add(2)) };
let table_hash = unsafe { hash_ptr(ip0.wrapping_sub(2), hash_log, mls) };
unsafe {
table_set(
hash_table,
table_hash,
index_from(base, ip0.wrapping_sub(2)),
)
};
while ptr_le(ip0, ilimit) {
let current2 = unsafe { index_from(base, ip0) };
let rep_index2 = current2.wrapping_sub(offset2);
let rep_match2 = if rep_index2 < prefix_start_index {
dict_base
.wrapping_sub(dict_index_delta as usize)
.wrapping_add(rep_index2 as usize)
} else {
base.wrapping_add(rep_index2 as usize)
};
if index_overlap_check(prefix_start_index, rep_index2)
&& unsafe { read32(rep_match2) == read32(ip0) }
{
let rep_end2 = if rep_index2 < prefix_start_index {
dict_end
} else {
iend
};
let repeat_length = unsafe {
count_2segments(
ip0.wrapping_add(4),
rep_match2.wrapping_add(4),
iend,
rep_end2,
prefix_start,
) + 4
};
std::mem::swap(&mut offset1, &mut offset2);
unsafe {
store_seq(
seq_store,
0,
anchor,
iend,
REPCODE1_TO_OFFBASE,
repeat_length,
)
};
let table_hash = unsafe { hash_ptr(ip0, hash_log, mls) };
unsafe { table_set(hash_table, table_hash, current2) };
ip0 = ip0.wrapping_add(repeat_length);
anchor = ip0;
continue;
}
break;
}
}
ip1 = ip0.wrapping_add(step_size as usize);
}
unsafe {
*reps = offset1;
*reps.add(1) = offset2;
}
unsafe { iend.offset_from(anchor) as usize }
}
/// Rust implementation called by the C ABI wrapper for attached dictionaries.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBlock_fast_dictMatchState(
hash_table: *mut u32,
base: *const u8,
prefix_start_index: u32,
hash_log: u32,
min_match: u32,
target_length: u32,
seq_store: *mut c_void,
reps: *mut u32,
src: *const c_void,
src_size: usize,
dict_hash_table: *const u32,
dict_base: *const u8,
dict_start_index: u32,
dict_end: *const u8,
dict_hash_log: u32,
prefetch_cdict_tables: c_int,
) -> usize {
unsafe {
compress_block_fast_dict_match_state(
hash_table,
base,
prefix_start_index,
hash_log,
target_length,
seq_store.cast::<SeqStore_t>(),
reps,
src.cast::<u8>(),
src_size,
fast_mls(min_match),
dict_hash_table,
dict_base,
dict_start_index,
dict_end,
dict_hash_log,
prefetch_cdict_tables != 0,
)
}
}
unsafe fn finish_ext_dict_match(
hash_table: *mut u32,
base: *const u8,
dict_base: *const u8,
prefix_start_index: u32,
dict_end: *const u8,
prefix_start: *const u8,
hash_log: u32,
mls: u32,
seq_store: *mut SeqStore_t,
offset1: &mut u32,
offset2: &mut u32,
mut ip0: *const u8,
match0: *const u8,
match_end: *const u8,
anchor: *const u8,
iend: *const u8,
ilimit: *const u8,
current0: u32,
hash1: usize,
ip1: *const u8,
offcode: u32,
mut match_length: usize,
) -> (*const u8, *const u8) {
match_length += unsafe {
count_2segments(
ip0.wrapping_add(match_length),
match0.wrapping_add(match_length),
iend,
match_end,
prefix_start,
)
};
unsafe {
store_seq(
seq_store,
ip0.offset_from(anchor) as usize,
anchor,
iend,
offcode,
match_length,
)
};
ip0 = ip0.wrapping_add(match_length);
let mut new_anchor = ip0;
if ptr_lt(ip1, ip0) {
let table_hash = hash1;
unsafe { table_set(hash_table, table_hash, index_from(base, ip1)) };
}
if ptr_le(ip0, ilimit) {
let table_hash =
unsafe { hash_ptr(base.wrapping_add(current0 as usize + 2), hash_log, mls) };
unsafe { table_set(hash_table, table_hash, current0.wrapping_add(2)) };
let table_hash = unsafe { hash_ptr(ip0.wrapping_sub(2), hash_log, mls) };
unsafe {
table_set(
hash_table,
table_hash,
index_from(base, ip0.wrapping_sub(2)),
)
};
while ptr_le(ip0, ilimit) {
let rep_index2 = unsafe { index_from(base, ip0) }.wrapping_sub(*offset2);
let rep_match2 = if rep_index2 < prefix_start_index {
dict_base.wrapping_add(rep_index2 as usize)
} else {
base.wrapping_add(rep_index2 as usize)
};
if *offset2 > 0
&& index_overlap_check(prefix_start_index, rep_index2)
&& unsafe { read32(rep_match2) == read32(ip0) }
{
let rep_end2 = if rep_index2 < prefix_start_index {
dict_end
} else {
iend
};
let repeat_length = unsafe {
count_2segments(
ip0.wrapping_add(4),
rep_match2.wrapping_add(4),
iend,
rep_end2,
prefix_start,
) + 4
};
std::mem::swap(offset1, offset2);
unsafe {
store_seq(
seq_store,
0,
new_anchor,
iend,
REPCODE1_TO_OFFBASE,
repeat_length,
)
};
let table_hash = unsafe { hash_ptr(ip0, hash_log, mls) };
unsafe { table_set(hash_table, table_hash, index_from(base, ip0)) };
ip0 = ip0.wrapping_add(repeat_length);
new_anchor = ip0;
continue;
}
break;
}
}
(ip0, new_anchor)
}
unsafe fn compress_block_fast_ext_dict(
hash_table: *mut u32,
base: *const u8,
dict_base: *const u8,
dict_limit: u32,
low_limit: u32,
loaded_dict_end: u32,
hash_log: u32,
target_length: u32,
window_log: u32,
seq_store: *mut SeqStore_t,
reps: *mut u32,
src: *const u8,
src_size: usize,
mls: u32,
) -> usize {
if src_size < HASH_READ_SIZE {
return src_size;
}
let istart = src;
let iend = istart.wrapping_add(src_size);
let ilimit = iend.wrapping_sub(HASH_READ_SIZE);
let end_index = unsafe { index_from(base, istart) }.wrapping_add(src_size as u32);
let dict_start_index = lowest_match_index(low_limit, loaded_dict_end, end_index, window_log);
let dict_start = dict_base.wrapping_add(dict_start_index as usize);
let prefix_start_index = dict_limit.max(dict_start_index);
let prefix_start = base.wrapping_add(prefix_start_index as usize);
let dict_end = dict_base.wrapping_add(prefix_start_index as usize);
if prefix_start_index == dict_start_index {
return unsafe {
compress_block_fast_no_dict(
hash_table,
base,
dict_limit,
loaded_dict_end,
hash_log,
target_length,
window_log,
seq_store,
reps,
src,
src_size,
mls,
window_log < 19,
)
};
}
let step_size = target_length as usize + usize::from(target_length == 0) + 1;
let mut ip0 = istart;
let mut anchor = istart;
let mut offset1 = unsafe { *reps };
let mut offset2 = unsafe { *reps.add(1) };
let mut offset_saved1 = 0u32;
let mut offset_saved2 = 0u32;
let current = unsafe { index_from(base, ip0) };
let max_rep = current.wrapping_sub(dict_start_index);
if offset2 >= max_rep {
offset_saved2 = offset2;
offset2 = 0;
}
if offset1 >= max_rep {
offset_saved1 = offset1;
offset1 = 0;
}
'start: loop {
let mut step = step_size;
let mut next_step = ip0.wrapping_add(1 << (K_SEARCH_STRENGTH - 1));
let mut ip1 = ip0.wrapping_add(1);
let mut ip2 = ip0.wrapping_add(step);
let mut ip3 = ip2.wrapping_add(1);
if ptr_ge(ip3, ilimit) {
break;
}
let mut hash0 = unsafe { hash_ptr(ip0, hash_log, mls) };
let mut hash1 = unsafe { hash_ptr(ip1, hash_log, mls) };
let mut index = unsafe { table_get(hash_table, hash0) };
let mut index_base = if index < prefix_start_index {
dict_base
} else {
base
};
loop {
let current2 = unsafe { index_from(base, ip2) };
let rep_index = current2.wrapping_sub(offset1);
let rep_base = if rep_index < prefix_start_index {
dict_base
} else {
base
};
let rep_value = if offset1 > 0 && prefix_start_index.wrapping_sub(rep_index) >= 4 {
unsafe { read32(rep_base.wrapping_add(rep_index as usize)) }
} else {
unsafe { read32(ip2) ^ 1 }
};
let mut current0 = unsafe { index_from(base, ip0) };
unsafe { table_set(hash_table, hash0, current0) };
if unsafe { read32(ip2) == rep_value } {
ip0 = ip2;
let mut match0 = rep_base.wrapping_add(rep_index as usize);
let match_end = if rep_index < prefix_start_index {
dict_end
} else {
iend
};
let match_length =
usize::from(unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) });
ip0 = ip0.wrapping_sub(match_length);
match0 = match0.wrapping_sub(match_length);
let (new_ip0, new_anchor) = unsafe {
finish_ext_dict_match(
hash_table,
base,
dict_base,
prefix_start_index,
dict_end,
prefix_start,
hash_log,
mls,
seq_store,
&mut offset1,
&mut offset2,
ip0,
match0,
match_end,
anchor,
iend,
ilimit,
current0,
hash1,
ip1,
REPCODE1_TO_OFFBASE,
match_length + 4,
)
};
ip0 = new_ip0;
anchor = new_anchor;
continue 'start;
}
if index >= dict_start_index
&& unsafe { read32(index_base.wrapping_add(index as usize)) == read32(ip0) }
{
let offset = current0.wrapping_sub(index);
let low_match_ptr = if index < prefix_start_index {
dict_start
} else {
prefix_start
};
let match_end = if index < prefix_start_index {
dict_end
} else {
iend
};
let mut match0 = index_base.wrapping_add(index as usize);
offset2 = offset1;
offset1 = offset;
let mut match_length = 4usize;
while ptr_gt(ip0, anchor)
&& ptr_gt(match0, low_match_ptr)
&& unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) }
{
ip0 = ip0.wrapping_sub(1);
match0 = match0.wrapping_sub(1);
match_length += 1;
}
let (new_ip0, new_anchor) = unsafe {
finish_ext_dict_match(
hash_table,
base,
dict_base,
prefix_start_index,
dict_end,
prefix_start,
hash_log,
mls,
seq_store,
&mut offset1,
&mut offset2,
ip0,
match0,
match_end,
anchor,
iend,
ilimit,
current0,
hash1,
ip1,
offset.wrapping_add(ZSTD_REP_NUM as u32),
match_length,
)
};
ip0 = new_ip0;
anchor = new_anchor;
continue 'start;
}
index = unsafe { table_get(hash_table, hash1) };
index_base = if index < prefix_start_index {
dict_base
} else {
base
};
hash0 = hash1;
hash1 = unsafe { hash_ptr(ip2, hash_log, mls) };
ip0 = ip1;
ip1 = ip2;
ip2 = ip3;
current0 = unsafe { index_from(base, ip0) };
unsafe { table_set(hash_table, hash0, current0) };
if index >= dict_start_index
&& unsafe { read32(index_base.wrapping_add(index as usize)) == read32(ip0) }
{
let offset = current0.wrapping_sub(index);
let low_match_ptr = if index < prefix_start_index {
dict_start
} else {
prefix_start
};
let match_end = if index < prefix_start_index {
dict_end
} else {
iend
};
let mut match0 = index_base.wrapping_add(index as usize);
offset2 = offset1;
offset1 = offset;
let mut match_length = 4usize;
while ptr_gt(ip0, anchor)
&& ptr_gt(match0, low_match_ptr)
&& unsafe { *ip0.wrapping_sub(1) == *match0.wrapping_sub(1) }
{
ip0 = ip0.wrapping_sub(1);
match0 = match0.wrapping_sub(1);
match_length += 1;
}
let (new_ip0, new_anchor) = unsafe {
finish_ext_dict_match(
hash_table,
base,
dict_base,
prefix_start_index,
dict_end,
prefix_start,
hash_log,
mls,
seq_store,
&mut offset1,
&mut offset2,
ip0,
match0,
match_end,
anchor,
iend,
ilimit,
current0,
hash1,
ip1,
offset.wrapping_add(ZSTD_REP_NUM as u32),
match_length,
)
};
ip0 = new_ip0;
anchor = new_anchor;
continue 'start;
}
index = unsafe { table_get(hash_table, hash1) };
index_base = if index < prefix_start_index {
dict_base
} else {
base
};
hash0 = hash1;
hash1 = unsafe { hash_ptr(ip2, hash_log, mls) };
ip0 = ip1;
ip1 = ip2;
ip2 = ip0.wrapping_add(step);
ip3 = ip1.wrapping_add(step);
if ptr_ge(ip2, next_step) {
step += 1;
next_step = next_step.wrapping_add(1 << (K_SEARCH_STRENGTH - 1));
}
if ptr_ge(ip3, ilimit) {
break 'start;
}
}
}
offset_saved2 = if offset_saved1 != 0 && offset1 != 0 {
offset_saved1
} else {
offset_saved2
};
unsafe {
*reps = if offset1 != 0 { offset1 } else { offset_saved1 };
*reps.add(1) = if offset2 != 0 { offset2 } else { offset_saved2 };
}
unsafe { iend.offset_from(anchor) as usize }
}
/// Rust implementation called by the C ABI wrapper for external dictionaries.
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_compressBlock_fast_extDict(
hash_table: *mut u32,
base: *const u8,
dict_base: *const u8,
dict_limit: u32,
low_limit: u32,
loaded_dict_end: u32,
hash_log: u32,
min_match: u32,
target_length: u32,
window_log: u32,
seq_store: *mut c_void,
reps: *mut u32,
src: *const c_void,
src_size: usize,
) -> usize {
unsafe {
compress_block_fast_ext_dict(
hash_table,
base,
dict_base,
dict_limit,
low_limit,
loaded_dict_end,
hash_log,
target_length,
window_log,
seq_store.cast::<SeqStore_t>(),
reps,
src.cast::<u8>(),
src_size,
fast_mls(min_match),
)
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::mem::{align_of, offset_of, size_of};
#[test]
fn seq_store_layout_matches_the_c_leaf_abi() {
assert_eq!(size_of::<SeqDef>(), 8);
assert_eq!(align_of::<SeqDef>(), align_of::<u32>());
assert_eq!(offset_of!(SeqStore_t, sequencesStart), 0);
assert_eq!(
offset_of!(SeqStore_t, longLengthPos),
9 * size_of::<usize>() + 4
);
assert_eq!(size_of::<SeqStore_t>(), 9 * size_of::<usize>() + 8);
}
#[test]
fn tagged_index_keeps_index_and_tag_in_c_format() {
let mut table = [0u32; 4];
write_tagged_index(
table.as_mut_ptr(),
(2 << SHORT_CACHE_TAG_BITS) | 0x5a,
0x123456,
);
assert_eq!(table[2], (0x123456 << SHORT_CACHE_TAG_BITS) | 0x5a);
assert!(packed_tags_match(
table[2],
(1 << SHORT_CACHE_TAG_BITS) | 0x5a
));
assert!(!packed_tags_match(
table[2],
(1 << SHORT_CACHE_TAG_BITS) | 0x5b
));
}
#[test]
fn store_seq_preserves_literal_and_long_match_contracts() {
let mut sequences = [SeqDef {
offBase: 0,
litLength: 0,
mlBase: 0,
}; 2];
let mut literals = [0u8; 16];
let source = *b"literals";
let mut store = SeqStore_t {
sequencesStart: sequences.as_mut_ptr(),
sequences: sequences.as_mut_ptr(),
litStart: literals.as_mut_ptr(),
lit: literals.as_mut_ptr(),
llCode: ptr::null_mut(),
mlCode: ptr::null_mut(),
ofCode: ptr::null_mut(),
maxNbSeq: sequences.len(),
maxNbLit: literals.len(),
longLengthType: 0,
longLengthPos: 0,
};
unsafe {
store_seq(
&mut store,
source.len(),
source.as_ptr(),
source.as_ptr().add(source.len()),
13,
3 + 0x1_0000,
);
}
assert_eq!(&literals[..source.len()], &source);
assert_eq!(sequences[0].offBase, 13);
assert_eq!(sequences[0].litLength, source.len() as u16);
assert_eq!(sequences[0].mlBase, 0);
assert_eq!(store.longLengthType, 2);
assert_eq!(store.longLengthPos, 0);
assert_eq!(
unsafe { store.sequences.offset_from(store.sequencesStart) },
1
);
}
}