ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #4413 from arpadpanyik-arm/huf_decode2x

Browse Source 
AArch64: Enhance struct access in Huffman decode 2X
This commit is contained in:
Yann Collet
2025-07-23 15:03:37 -08:00
committed by GitHub
parent 6f1cb87ade bd38fc2c5f
commit 34f3a0ab11
1 changed files with 49 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/decompress/huf_decompress.c
+49 -41
View File
@@ -785,19 +785,19 @@ void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs*
} }
#endif #endif
#define HUF_4X1_DECODE_SYMBOL(_stream, _symbol) \ #define HUF_4X1_DECODE_SYMBOL(_stream, _symbol) \
do { \ do { \
int const index = (int)(bits[(_stream)] >> 53); \ U64 const index = bits[(_stream)] >> 53; \
int const entry = (int)dtable[index]; \ U16 const entry = dtable[index]; \
bits[(_stream)] <<= (entry & 0x3F); \ bits[(_stream)] <<= entry & 0x3F; \
op[(_stream)][(_symbol)] = (BYTE)((entry >> 8) & 0xFF); \ op[(_stream)][(_symbol)] = (BYTE)(entry >> 8); \
} while (0) } while (0)
#define HUF_4X1_RELOAD_STREAM(_stream) \ #define HUF_5X1_RELOAD_STREAM(_stream) \
do { \ do { \
int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \ U64 const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \
int const nbBits = ctz & 7; \ U64 const nbBits = ctz & 7; \
int const nbBytes = ctz >> 3; \ U64 const nbBytes = ctz >> 3; \
op[(_stream)] += 5; \ op[(_stream)] += 5; \
ip[(_stream)] -= nbBytes; \ ip[(_stream)] -= nbBytes; \
bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \ bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \
@@ -816,11 +816,11 @@ void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs*
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 4); HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 4);
/* Reload each of the 4 the bitstreams */ /* Reload each of the 4 the bitstreams */
HUF_4X_FOR_EACH_STREAM(HUF_4X1_RELOAD_STREAM); HUF_4X_FOR_EACH_STREAM(HUF_5X1_RELOAD_STREAM);
} while (op[3] < olimit); } while (op[3] < olimit);
#undef HUF_4X1_DECODE_SYMBOL #undef HUF_4X1_DECODE_SYMBOL
#undef HUF_4X1_RELOAD_STREAM #undef HUF_5X1_RELOAD_STREAM
} }
_out: _out:
@@ -1603,57 +1603,65 @@ void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs*
} }
#endif #endif
#define HUF_4X2_DECODE_SYMBOL(_stream, _decode3) \ #define HUF_4X2_DECODE_SYMBOL(_stream, _decode3) \
do { \ do { \
if ((_decode3) || (_stream) != 3) { \ if ((_decode3) || (_stream) != 3) { \
int const index = (int)(bits[(_stream)] >> 53); \ U64 const index = bits[(_stream)] >> 53; \
HUF_DEltX2 const entry = dtable[index]; \ size_t const entry = MEM_readLE32(&dtable[index]); \
MEM_write16(op[(_stream)], entry.sequence); \ MEM_write16(op[(_stream)], (U16)entry); \
bits[(_stream)] <<= (entry.nbBits) & 0x3F; \ bits[(_stream)] <<= (entry >> 16) & 0x3F; \
op[(_stream)] += (entry.length); \ op[(_stream)] += entry >> 24; \
} \ } \
} while (0) } while (0)
#define HUF_4X2_RELOAD_STREAM(_stream) \ #define HUF_5X2_RELOAD_STREAM(_stream, _decode3) \
do { \ do { \
HUF_4X2_DECODE_SYMBOL(3, 1); \ if (_decode3) HUF_4X2_DECODE_SYMBOL(3, 1); \
{ \ { \
int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \ U64 const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \
int const nbBits = ctz & 7; \ U64 const nbBits = ctz & 7; \
int const nbBytes = ctz >> 3; \ U64 const nbBytes = ctz >> 3; \
ip[(_stream)] -= nbBytes; \ ip[(_stream)] -= nbBytes; \
bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \ bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \
bits[(_stream)] <<= nbBits; \ bits[(_stream)] <<= nbBits; \
} \ } \
} while (0) } while (0)
#if defined(__aarch64__)
# define HUF_4X2_4WAY 1
#else
# define HUF_4X2_4WAY 0
#endif
#define HUF_4X2_3WAY !HUF_4X2_4WAY
/* Manually unroll the loop because compilers don't consistently /* Manually unroll the loop because compilers don't consistently
* unroll the inner loops, which destroys performance. * unroll the inner loops, which destroys performance.
*/ */
do { do {
/* Decode 5 symbols from each of the first 3 streams. /* Decode 5 symbols from each of the first 3 or 4 streams.
* The final stream will be decoded during the reload phase * In the 3-way case the final stream will be decoded during
* to reduce register pressure. * the reload phase to reduce register pressure.
*/ */
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, HUF_4X2_4WAY);
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, HUF_4X2_4WAY);
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, HUF_4X2_4WAY);
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, HUF_4X2_4WAY);
HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, HUF_4X2_4WAY);
/* Decode one symbol from the final stream */ /* In the 3-way case decode one symbol from the final stream. */
HUF_4X2_DECODE_SYMBOL(3, 1); HUF_4X2_DECODE_SYMBOL(3, HUF_4X2_3WAY);
/* Decode 4 symbols from the final stream & reload bitstreams. /* In the 3-way case decode 4 symbols from the final stream &
* The final stream is reloaded last, meaning that all 5 symbols * reload bitstreams. The final stream is reloaded last, meaning
* are decoded from the final stream before it is reloaded. * that all 5 symbols are decoded from the final stream before
* it is reloaded.
*/ */
HUF_4X_FOR_EACH_STREAM(HUF_4X2_RELOAD_STREAM); HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_5X2_RELOAD_STREAM, HUF_4X2_3WAY);
} while (op[3] < olimit); } while (op[3] < olimit);
} }
#undef HUF_4X2_DECODE_SYMBOL #undef HUF_4X2_DECODE_SYMBOL
#undef HUF_4X2_RELOAD_STREAM #undef HUF_5X2_RELOAD_STREAM
_out: _out: