ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2689 from danlark1/dev

Browse Source 
Optimize zstd decompression by another x%
This commit is contained in:
Nick Terrell
2021-06-29 11:34:36 -07:00
committed by GitHub
parent 88f107b5b9 2c2c9e7dfd
commit 094b26081f
2 changed files with 103 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/common/bitstream.h
+9
View File
@@ -332,7 +332,16 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 c
U32 const regMask = sizeof(bitContainer)*8 - 1; U32 const regMask = sizeof(bitContainer)*8 - 1;
/* if start > regMask, bitstream is corrupted, and result is undefined */ /* if start > regMask, bitstream is corrupted, and result is undefined */
assert(nbBits < BIT_MASK_SIZE); assert(nbBits < BIT_MASK_SIZE);
/* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better
* than accessing memory. When bmi2 instruction is not present, we consider
* such cpus old (pre-Haswell, 2013) and their performance is not of that
* importance.
*/
#if defined(__x86_64__) || defined(_M_X86)
return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1);
#else
return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
#endif
} }
MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
lib/decompress/zstd_decompress_block.c
+94 -106
View File
@@ -905,20 +905,10 @@ ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqS
} }
FORCE_INLINE_TEMPLATE void FORCE_INLINE_TEMPLATE void
ZSTD_updateFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD) ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits)
{ {
ZSTD_seqSymbol const DInfo = DStatePtr->table[DStatePtr->state];
U32 const nbBits = DInfo.nbBits;
size_t const lowBits = BIT_readBits(bitD, nbBits); size_t const lowBits = BIT_readBits(bitD, nbBits);
DStatePtr->state = DInfo.nextState + lowBits; DStatePtr->state = nextState + lowBits;
}
FORCE_INLINE_TEMPLATE void
ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, ZSTD_seqSymbol const DInfo)
{
U32 const nbBits = DInfo.nbBits;
size_t const lowBits = BIT_readBits(bitD, nbBits);
DStatePtr->state = DInfo.nextState + lowBits;
} }
/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum /* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum
@@ -937,102 +927,100 @@ FORCE_INLINE_TEMPLATE seq_t
ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
{ {
seq_t seq; seq_t seq;
ZSTD_seqSymbol const llDInfo = seqState->stateLL.table[seqState->stateLL.state]; const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state;
ZSTD_seqSymbol const mlDInfo = seqState->stateML.table[seqState->stateML.state]; const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state;
ZSTD_seqSymbol const ofDInfo = seqState->stateOffb.table[seqState->stateOffb.state]; const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state;
U32 const llBase = llDInfo.baseValue; seq.matchLength = mlDInfo->baseValue;
U32 const mlBase = mlDInfo.baseValue; seq.litLength = llDInfo->baseValue;
U32 const ofBase = ofDInfo.baseValue; { U32 const ofBase = ofDInfo->baseValue;
BYTE const llBits = llDInfo.nbAdditionalBits; BYTE const llBits = llDInfo->nbAdditionalBits;
BYTE const mlBits = mlDInfo.nbAdditionalBits; BYTE const mlBits = mlDInfo->nbAdditionalBits;
BYTE const ofBits = ofDInfo.nbAdditionalBits; BYTE const ofBits = ofDInfo->nbAdditionalBits;
BYTE const totalBits = llBits+mlBits+ofBits; BYTE const totalBits = llBits+mlBits+ofBits;
/* sequence */ U16 const llNext = llDInfo->nextState;
{ size_t offset; U16 const mlNext = mlDInfo->nextState;
if (ofBits > 1) { U16 const ofNext = ofDInfo->nextState;
ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); U32 const llnbBits = llDInfo->nbBits;
ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); U32 const mlnbBits = mlDInfo->nbBits;
assert(ofBits <= MaxOff); U32 const ofnbBits = ofDInfo->nbBits;
if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { /*
U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed); * As gcc has better branch and block analyzers, sometimes it is only
offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); * valuable to mark likelyness for clang, it gives around 3-4% of
BIT_reloadDStream(&seqState->DStream); * performance.
if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); */
assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32); /* to avoid another reload */
} else { /* sequence */
offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ { size_t offset;
if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); #if defined(__clang__)
} if (LIKELY(ofBits > 1)) {
seqState->prevOffset[2] = seqState->prevOffset[1]; #else
seqState->prevOffset[1] = seqState->prevOffset[0]; if (ofBits > 1) {
seqState->prevOffset[0] = offset; #endif
} else { ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
U32 const ll0 = (llBase == 0); ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
if (LIKELY((ofBits == 0))) { assert(ofBits <= MaxOff);
if (LIKELY(!ll0)) if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) {
offset = seqState->prevOffset[0]; U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed);
else { offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
offset = seqState->prevOffset[1]; BIT_reloadDStream(&seqState->DStream);
seqState->prevOffset[1] = seqState->prevOffset[0]; if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits);
seqState->prevOffset[0] = offset; assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32); /* to avoid another reload */
} else {
offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
} }
seqState->prevOffset[2] = seqState->prevOffset[1];
seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[0] = offset;
} else { } else {
offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); U32 const ll0 = (llDInfo->baseValue == 0);
{ size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; if (LIKELY((ofBits == 0))) {
temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ offset = seqState->prevOffset[ll0];
if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; seqState->prevOffset[1] = seqState->prevOffset[!ll0];
seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[0] = offset;
seqState->prevOffset[0] = offset = temp; } else {
} } } offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1);
seq.offset = offset; { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
} temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
seq.matchLength = mlBase; seqState->prevOffset[1] = seqState->prevOffset[0];
if (mlBits > 0) seqState->prevOffset[0] = offset = temp;
seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); } } }
seq.offset = offset;
if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32))
BIT_reloadDStream(&seqState->DStream);
if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog)))
BIT_reloadDStream(&seqState->DStream);
/* Ensure there are enough bits to read the rest of data in 64-bit mode. */
ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64);
seq.litLength = llBase;
if (llBits > 0)
seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/);
if (MEM_32bits())
BIT_reloadDStream(&seqState->DStream);
DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u",
(U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
/* ANS state update
* gcc-9.0.0 does 2.5% worse with ZSTD_updateFseStateWithDInfo().
* clang-9.2.0 does 7% worse with ZSTD_updateFseState().
* Naturally it seems like ZSTD_updateFseStateWithDInfo() should be the
* better option, so it is the default for other compilers. But, if you
* measure that it is worse, please put up a pull request.
*/
{
#if defined(__GNUC__) && !defined(__clang__)
const int kUseUpdateFseState = 1;
#else
const int kUseUpdateFseState = 0;
#endif
if (kUseUpdateFseState) {
ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */
ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */
if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */
} else {
ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llDInfo); /* <= 9 bits */
ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlDInfo); /* <= 9 bits */
if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofDInfo); /* <= 8 bits */
} }
#if defined(__clang__)
if (UNLIKELY(mlBits > 0))
#else
if (mlBits > 0)
#endif
seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/);
if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32))
BIT_reloadDStream(&seqState->DStream);
if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog)))
BIT_reloadDStream(&seqState->DStream);
/* Ensure there are enough bits to read the rest of data in 64-bit mode. */
ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64);
#if defined(__clang__)
if (UNLIKELY(llBits > 0))
#else
if (llBits > 0)
#endif
seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/);
if (MEM_32bits())
BIT_reloadDStream(&seqState->DStream);
DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u",
(U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */
ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */
if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */
} }
return seq; return seq;
@@ -1166,8 +1154,8 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx,
__asm__("nop"); __asm__("nop");
__asm__(".p2align 5"); __asm__(".p2align 5");
__asm__("nop"); __asm__("nop");
# if __GNUC__ >= 9 # if __GNUC__ >= 9 && __GNUC__ < 11
/* better for gcc-9 and gcc-10, worse for clang and gcc-8 */ /* better for gcc-9 and gcc-10, worse for clang and gcc-8, gcc-11 */
__asm__(".p2align 3"); __asm__(".p2align 3");
# else # else
__asm__(".p2align 4"); __asm__(".p2align 4");