ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add 64 row entry support for lazy

Browse Source
This commit is contained in:
senhuang42
2021-08-04 16:19:12 -04:00
parent 31820e032c
commit e411040ea1
4 changed files with 507 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+14 -14
View File
@@ -1785,8 +1785,8 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
if (ms->tagTable) ZSTD_memset(ms->tagTable, 0, tagTableSize);
}
{ /* Switch to 32-entry rows if searchLog is 5 (or more) */
U32 const rowLog = cParams->searchLog < 5 ? 4 : 5;
assert(cParams->hashLog > rowLog);
U32 const rowLog = MAX(MIN(cParams->searchLog, 6), 4);
assert(cParams->hashLog >= rowLog);
ms->rowHashLog = cParams->hashLog - rowLog;
}
}
@@ -6213,12 +6213,12 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */
{ 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */
{ 18, 16, 17, 3, 5, 2, ZSTD_greedy }, /* level 4.*/
{ 18, 18, 18, 4, 5, 2, ZSTD_greedy }, /* level 5.*/
{ 18, 18, 19, 5, 5, 4, ZSTD_greedy }, /* level 6.*/
{ 18, 18, 19, 3, 4, 4, ZSTD_lazy }, /* level 7 */
{ 18, 18, 19, 4, 4, 8, ZSTD_lazy }, /* level 8 */
{ 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 9 */
{ 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 10 */
{ 18, 17, 18, 5, 5, 2, ZSTD_greedy }, /* level 5.*/
{ 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/
{ 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */
{ 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */
{ 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */
{ 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */
{ 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/
{ 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/
{ 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */
@@ -6239,12 +6239,12 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */
{ 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */
{ 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */
{ 17, 16, 17, 4, 4, 2, ZSTD_greedy }, /* level 5 */
{ 17, 16, 17, 5, 4, 4, ZSTD_greedy }, /* level 6 */
{ 17, 16, 17, 4, 4, 8, ZSTD_lazy }, /* level 7 */
{ 17, 16, 17, 5, 4, 8, ZSTD_lazy }, /* level 8 */
{ 17, 16, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 9 */
{ 17, 16, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 10 */
{ 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */
{ 17, 16, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */
{ 17, 16, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */
{ 17, 16, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */
{ 17, 16, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */
{ 17, 16, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */
{ 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */
{ 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */
{ 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/
lib/compress/zstd_lazy.c
+81 -30
View File
@@ -865,39 +865,48 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS (
* (SIMD) Row-based matchfinder
***********************************/
/* Constants for row-based hash */
#define ZSTD_ROW_HASH_TAG_OFFSET 16 /* byte offset of hashes in the match state's tagTable from the beginning of a row */
#define ZSTD_ROW_HASH_TAG_BITS 8 /* nb bits to use for the tag */
#define ZSTD_ROW_HASH_TAG_OFFSET 16 /* byte offset of hashes in the match state's tagTable from the beginning of a row */
#define ZSTD_ROW_HASH_TAG_BITS 8 /* nb bits to use for the tag */
#define ZSTD_ROW_HASH_TAG_MASK ((1u << ZSTD_ROW_HASH_TAG_BITS) - 1)
#define ZSTD_ROW_HASH_MAX_ENTRIES 64 /* absolute maximum number of entries per row, for all configurations */
#define ZSTD_ROW_HASH_CACHE_MASK (ZSTD_ROW_HASH_CACHE_SIZE - 1)
typedef U32 ZSTD_VecMask; /* Clarifies when we are interacting with a U32 representing a mask of matches */
typedef U64 ZSTD_VecMask; /* Clarifies when we are interacting with a U64 representing a mask of matches */
/* ZSTD_VecMask_next():
* Starting from the LSB, returns the idx of the next non-zero bit.
* Basically counting the nb of trailing zeroes.
*/
static U32 ZSTD_VecMask_next(ZSTD_VecMask val) {
# if defined(_MSC_VER) /* Visual */
assert(val != 0);
# if defined(_MSC_VER) && defined(_WIN64)
unsigned long r=0;
return _BitScanForward(&r, val) ? (U32)r : 0;
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (U32)__builtin_ctz(val);
return _BitScanForward64(&r, val) ? (U64)r : 0; /* _BitScanForward64 not defined outside of x86/64 */
# elif (defined(__GNUC__) && ((__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))
return (U32)__builtin_ctzll(val);
# else
/* Software ctz version: http://graphics.stanford.edu/~seander/bithacks.html#ZerosOnRightMultLookup */
static const U32 multiplyDeBruijnBitPosition[32] =
{
0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9
};
return multiplyDeBruijnBitPosition[((U32)((val & -(int)val) * 0x077CB531U)) >> 27];
/* Software ctz version: http://aggregate.org/MAGIC/#Trailing%20Zero%20Count
* and: https://stackoverflow.com/questions/2709430/count-number-of-bits-in-a-64-bit-long-big-integer
*/
val = ~val & (val - 1ULL); /* Lowest set bit mask */
val = val - ((val >> 1) & 0x5555555555555555);
val = (val & 0x3333333333333333ULL) + ((val >> 2) & 0x3333333333333333ULL);
return (U32)((((val + (val >> 4)) & 0xF0F0F0F0F0F0F0FULL) * 0x101010101010101ULL) >> 56);
# endif
}
/* ZSTD_rotateRight_U32():
/* ZSTD_rotateRight_*():
* Rotates a bitfield to the right by "count" bits.
* https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts
*/
FORCE_INLINE_TEMPLATE
U64 ZSTD_rotateRight_U64(U64 const value, U32 count) {
assert(count < 64);
count &= 0x3F; /* for fickle pattern recognition */
return (value >> count) | (U64)(value << ((0U - count) & 0x3F));
}
FORCE_INLINE_TEMPLATE
U32 ZSTD_rotateRight_U32(U32 const value, U32 count) {
assert(count < 32);
@@ -935,13 +944,17 @@ MEM_STATIC int ZSTD_isAligned(void const* ptr, size_t align) {
*/
FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, U16 const* tagTable, U32 const relRow, U32 const rowLog) {
PREFETCH_L1(hashTable + relRow);
if (rowLog == 5) {
if (rowLog >= 5) {
PREFETCH_L1(hashTable + relRow + 16);
/* Note: prefetching more of the hash table does not appear to be beneficial for 128-entry rows */
}
PREFETCH_L1(tagTable + relRow);
assert(rowLog == 4 || rowLog == 5);
if (rowLog == 6) {
PREFETCH_L1(tagTable + relRow + 32);
}
assert(rowLog == 4 || rowLog == 5 || rowLog == 6);
assert(ZSTD_isAligned(hashTable + relRow, 64)); /* prefetched hash row always 64-byte aligned */
assert(ZSTD_isAligned(tagTable + relRow, (size_t)1 << rowLog)); /* prefetched tagRow sits on a multiple of 32 or 64 bytes */
assert(ZSTD_isAligned(tagTable + relRow, (size_t)1 << rowLog)); /* prefetched tagRow sits on correct multiple of bytes (32,64,128) */
}
/* ZSTD_row_fillHashCache():
@@ -1025,7 +1038,7 @@ FORCE_INLINE_TEMPLATE void ZSTD_row_update_internal(ZSTD_matchState_t* ms, const
* processing.
*/
void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip) {
const U32 rowLog = ms->cParams.searchLog < 5 ? 4 : 5;
const U32 rowLog = MAX(MIN(ms->cParams.searchLog, 6), 4);
const U32 rowMask = (1u << rowLog) - 1;
const U32 mls = MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */);
@@ -1040,14 +1053,15 @@ void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip) {
FORCE_INLINE_TEMPLATE
ZSTD_VecMask ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 head, const U32 rowEntries) {
const BYTE* const src = tagRow + ZSTD_ROW_HASH_TAG_OFFSET;
assert((rowEntries == 16) || (rowEntries == 32));
assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64);
assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES);
#if defined(ZSTD_ARCH_X86_SSE2)
if (rowEntries == 16) {
const __m128i chunk = _mm_loadu_si128((const __m128i*)(const void*)src);
const __m128i equalMask = _mm_cmpeq_epi8(chunk, _mm_set1_epi8(tag));
const U16 matches = (U16)_mm_movemask_epi8(equalMask);
return ZSTD_rotateRight_U16(matches, head);
} else { /* rowEntries == 32 */
} else if (rowEntries == 32) {
const __m128i chunk0 = _mm_loadu_si128((const __m128i*)(const void*)&src[0]);
const __m128i chunk1 = _mm_loadu_si128((const __m128i*)(const void*)&src[16]);
const __m128i equalMask0 = _mm_cmpeq_epi8(chunk0, _mm_set1_epi8(tag));
@@ -1055,6 +1069,21 @@ ZSTD_VecMask ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, con
const U32 lo = (U32)_mm_movemask_epi8(equalMask0);
const U32 hi = (U32)_mm_movemask_epi8(equalMask1);
return ZSTD_rotateRight_U32((hi << 16) | lo, head);
} else { /* rowEntries == 64 */
const __m128i chunk0 = _mm_loadu_si128((const __m128i*)(const void*)&src[0]);
const __m128i chunk1 = _mm_loadu_si128((const __m128i*)(const void*)&src[16]);
const __m128i chunk2 = _mm_loadu_si128((const __m128i*)(const void*)&src[32]);
const __m128i chunk3 = _mm_loadu_si128((const __m128i*)(const void*)&src[48]);
const __m128i comparisonMask = _mm_set1_epi8(tag);
const __m128i equalMask0 = _mm_cmpeq_epi8(chunk0, comparisonMask);
const __m128i equalMask1 = _mm_cmpeq_epi8(chunk1, comparisonMask);
const __m128i equalMask2 = _mm_cmpeq_epi8(chunk2, comparisonMask);
const __m128i equalMask3 = _mm_cmpeq_epi8(chunk3, comparisonMask);
const U64 mask0 = (U64)_mm_movemask_epi8(equalMask0);
const U64 mask1 = (U64)_mm_movemask_epi8(equalMask1);
const U64 mask2 = (U64)_mm_movemask_epi8(equalMask2);
const U64 mask3 = (U64)_mm_movemask_epi8(equalMask3);
return ZSTD_rotateRight_U64((mask3 << 48) | (mask2 << 32) | (mask1 << 16) | mask0, head);
}
#else
# if defined(ZSTD_ARCH_ARM_NEON)
@@ -1069,7 +1098,7 @@ ZSTD_VecMask ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, con
const U16 hi = (U16)vgetq_lane_u8(t3, 8);
const U16 lo = (U16)vgetq_lane_u8(t3, 0);
return ZSTD_rotateRight_U16((hi << 8) | lo, head);
} else { /* rowEntries == 32 */
} else if (rowEntries == 32) {
const uint16x8x2_t chunk = vld2q_u16((const U16*)(const void*)src);
const uint8x16_t chunk0 = vreinterpretq_u8_u16(chunk.val[0]);
const uint8x16_t chunk1 = vreinterpretq_u8_u16(chunk.val[1]);
@@ -1084,6 +1113,21 @@ ZSTD_VecMask ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, con
const uint8x8_t t4 = vsri_n_u8(t3.val[1], t3.val[0], 4);
const U32 matches = vget_lane_u32(vreinterpret_u32_u8(t4), 0);
return ZSTD_rotateRight_U32(matches, head);
} else { /* rowEntries == 64 */
const uint8x16x4_t chunk = vld4q_u8(src);
const uint8x16_t dup = vdupq_n_u8(tag);
const uint8x16_t cmp0 = vceqq_u8(chunk.val[0], dup);
const uint8x16_t cmp1 = vceqq_u8(chunk.val[1], dup);
const uint8x16_t cmp2 = vceqq_u8(chunk.val[2], dup);
const uint8x16_t cmp3 = vceqq_u8(chunk.val[3], dup);
const uint8x16_t t0 = vsriq_n_u8(cmp1, cmp0, 1);
const uint8x16_t t1 = vsriq_n_u8(cmp3, cmp2, 1);
const uint8x16_t t2 = vsriq_n_u8(t1, t0, 2);
const uint8x16_t t3 = vsriq_n_u8(t2, t2, 4);
const uint8x8_t t4 = vshrn_n_u16(vreinterpretq_u16_u8(t3), 4);
const U64 matches = vget_lane_u64(vreinterpret_u64_u8(t4), 0);
return ZSTD_rotateRight_U64(matches, head);
}
}
# endif
@@ -1094,7 +1138,7 @@ ZSTD_VecMask ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, con
const size_t x01 = xFF / 0xFF;
const size_t x80 = x01 << 7;
const size_t splatChar = tag * x01;
size_t matches = 0;
ZSTD_VecMask matches = 0;
int i = rowEntries - chunkSize;
assert((sizeof(size_t) == 4) || (sizeof(size_t) == 8));
if (MEM_isLittleEndian()) { /* runtime check so have two loops */
@@ -1122,8 +1166,10 @@ ZSTD_VecMask ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, con
matches = ~matches;
if (rowEntries == 16) {
return ZSTD_rotateRight_U16((U16)matches, head);
} else { /* rowEntries == 32 */
} else if (rowEntries == 32) {
return ZSTD_rotateRight_U32((U32)matches, head);
} else {
return ZSTD_rotateRight_U64((U64)matches, head);
}
}
#endif
@@ -1176,6 +1222,7 @@ size_t ZSTD_RowFindBestMatch_generic (
/* DMS/DDS variables that may be referenced laster */
const ZSTD_matchState_t* const dms = ms->dictMatchState;
/* Initialize the following variables to satisfy static analyzer */
size_t ddsIdx = 0;
U32 ddsExtraAttempts = 0; /* cctx hash tables are limited in searches, but allow extra searches into DDS */
@@ -1213,7 +1260,7 @@ size_t ZSTD_RowFindBestMatch_generic (
U32* const row = hashTable + relRow;
BYTE* tagRow = (BYTE*)(tagTable + relRow);
U32 const head = *tagRow & rowMask;
U32 matchBuffer[32 /* maximum nb entries per row */];
U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES];
size_t numMatches = 0;
size_t currMatch = 0;
ZSTD_VecMask matches = ZSTD_row_getMatchMask(tagRow, (BYTE)tag, head, rowEntries);
@@ -1281,7 +1328,7 @@ size_t ZSTD_RowFindBestMatch_generic (
const U32 dmsIndexDelta = dictLimit - dmsSize;
{ U32 const head = *dmsTagRow & rowMask;
U32 matchBuffer[32 /* maximum nb row entries */];
U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES];
size_t numMatches = 0;
size_t currMatch = 0;
ZSTD_VecMask matches = ZSTD_row_getMatchMask(dmsTagRow, (BYTE)dmsTag, head, rowEntries);
@@ -1340,12 +1387,13 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_RowFindBestMatch_selectRowLog (
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
const U32 cappedSearchLog = MIN(ms->cParams.searchLog, 5);
const U32 cappedSearchLog = MIN(ms->cParams.searchLog, 6);
switch(cappedSearchLog)
{
default :
case 4 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_noDict, offsetPtr, 4);
case 5 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_noDict, offsetPtr, 5);
case 6 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_noDict, offsetPtr, 6);
}
}
@@ -1354,12 +1402,13 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_RowFindBestMatch_dictMatchState_selectRowLog(
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
const U32 cappedSearchLog = MIN(ms->cParams.searchLog, 5);
const U32 cappedSearchLog = MIN(ms->cParams.searchLog, 6);
switch(cappedSearchLog)
{
default :
case 4 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_dictMatchState, offsetPtr, 4);
case 5 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_dictMatchState, offsetPtr, 5);
case 6 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_dictMatchState, offsetPtr, 6);
}
}
@@ -1368,12 +1417,13 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_RowFindBestMatch_dedicatedDictSearch_selectRow
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
const U32 cappedSearchLog = MIN(ms->cParams.searchLog, 5);
const U32 cappedSearchLog = MIN(ms->cParams.searchLog, 6);
switch(cappedSearchLog)
{
default :
case 4 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_dedicatedDictSearch, offsetPtr, 4);
case 5 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_dedicatedDictSearch, offsetPtr, 5);
case 6 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_dedicatedDictSearch, offsetPtr, 6);
}
}
@@ -1382,12 +1432,13 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_RowFindBestMatch_extDict_selectRowLog (
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
const U32 cappedSearchLog = MIN(ms->cParams.searchLog, 5);
const U32 cappedSearchLog = MIN(ms->cParams.searchLog, 6);
switch(cappedSearchLog)
{
default :
case 4 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_extDict, offsetPtr, 4);
case 5 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_extDict, offsetPtr, 5);
case 6 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_extDict, offsetPtr, 6);
}
}