From 64ee73269434634393362c09dc6885da149f826d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 9 Feb 2018 12:33:28 -0800 Subject: [PATCH 01/10] decide long-offset mode based on offcode statistics threshold vaguely estimated --- lib/decompress/zstd_decompress.c | 61 ++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index afae80d1c..3d1f36404 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1082,7 +1082,7 @@ size_t ZSTD_execSequence(BYTE* op, static size_t ZSTD_decompressSequences( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, + const void* seqStart, size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) { const BYTE* ip = (const BYTE*)seqStart; @@ -1095,16 +1095,8 @@ static size_t ZSTD_decompressSequences( const BYTE* const base = (const BYTE*) (dctx->base); const BYTE* const vBase = (const BYTE*) (dctx->vBase); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - int nbSeq; - DEBUGLOG(5, "ZSTD_decompressSequences"); - /* Build Decoding Tables */ - { size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); - DEBUGLOG(5, "ZSTD_decodeSeqHeaders: size=%u, nbSeq=%i", - (U32)seqHSize, nbSeq); - if (ZSTD_isError(seqHSize)) return seqHSize; - ip += seqHSize; - } + DEBUGLOG(5, "ZSTD_decompressSequences"); /* Regen sequences */ if (nbSeq) { @@ -1328,7 +1320,7 @@ size_t ZSTD_execSequenceLong(BYTE* op, static size_t ZSTD_decompressSequencesLong( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, + const void* seqStart, size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) { const BYTE* ip = (const BYTE*)seqStart; @@ -1341,13 +1333,6 @@ static size_t ZSTD_decompressSequencesLong( const BYTE* const prefixStart = (const BYTE*) (dctx->base); const BYTE* const dictStart = (const BYTE*) (dctx->vBase); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - int nbSeq; - - /* Build Decoding Tables */ - { size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); - if (ZSTD_isError(seqHSize)) return seqHSize; - ip += seqHSize; - } /* Regen sequences */ if (nbSeq) { @@ -1408,6 +1393,24 @@ static size_t ZSTD_decompressSequencesLong( } +static unsigned +ZSTD_shareLongOffsets(const FSE_DTable* offTable) +{ + U32 const tableLog = ((const FSE_DTableHeader*)offTable)[0].tableLog; + const FSE_decode_t* table = (const FSE_decode_t*)(offTable + 1); + U32 const max = 1 << tableLog; + U32 u, total = 0; + + assert(tableLog <= OffFSELog); + for (u=0; u 23) total += 1; + + total <<= (OffFSELog - tableLog); + + return total; +} + + static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const int frame) @@ -1430,11 +1433,23 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, ip += litCSize; srcSize -= litCSize; } - if ( frame /* windowSize exists */ - && (dctx->fParams.windowSize > (1<<24)) - && MEM_64bits() /* x86 benefits less from long mode than x64 */ ) - return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, isLongOffset); - return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, isLongOffset); + + /* Build Decoding Tables */ + { int nbSeq; + size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); + if (ZSTD_isError(seqHSize)) return seqHSize; + ip += seqHSize; + srcSize -= seqHSize; + + if (dctx->fParams.windowSize > (1<<24)) { + U32 const shareLongOffsets = ZSTD_shareLongOffsets(dctx->entropy.OFTable); + U32 const minShare = MEM_64bits() ? 12 : 20; + if (shareLongOffsets >= minShare) + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); + } + + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); + } } From 426944c3e31c13e0eecce63983d543f14b5caeb0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 9 Feb 2018 13:10:32 -0800 Subject: [PATCH 02/10] fixed strict aliasing issue tuned threshold --- lib/decompress/zstd_decompress.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 3d1f36404..63985933a 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1394,10 +1394,11 @@ static size_t ZSTD_decompressSequencesLong( static unsigned -ZSTD_shareLongOffsets(const FSE_DTable* offTable) +ZSTD_getLongOffsetsShare(const FSE_DTable* offTable) { - U32 const tableLog = ((const FSE_DTableHeader*)offTable)[0].tableLog; - const FSE_decode_t* table = (const FSE_decode_t*)(offTable + 1); + const void* ptr = offTable; + U32 const tableLog = ((const FSE_DTableHeader*)ptr)[0].tableLog; + const FSE_decode_t* table = ((const FSE_decode_t*)ptr) + 1; U32 const max = 1 << tableLog; U32 u, total = 0; @@ -1442,8 +1443,8 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, srcSize -= seqHSize; if (dctx->fParams.windowSize > (1<<24)) { - U32 const shareLongOffsets = ZSTD_shareLongOffsets(dctx->entropy.OFTable); - U32 const minShare = MEM_64bits() ? 12 : 20; + U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->entropy.OFTable); + U32 const minShare = MEM_64bits() ? 5 : 13; if (shareLongOffsets >= minShare) return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); } From af48f0b62baf343115547c0a24e9dc5b49d2ea7f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 9 Feb 2018 15:15:46 -0800 Subject: [PATCH 03/10] fix : offset table pointer when using default table --- lib/decompress/zstd_decompress.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 63985933a..c72fc71d8 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -748,7 +748,6 @@ static size_t ZSTD_buildSeqTable(FSE_DTable* DTableSpace, const FSE_DTable** DTa case set_repeat: if (!flagRepeatTable) return ERROR(corruption_detected); return 0; - default : /* impossible */ case set_compressed : { U32 tableLog; S16 norm[MaxSeq+1]; @@ -758,7 +757,10 @@ static size_t ZSTD_buildSeqTable(FSE_DTable* DTableSpace, const FSE_DTable** DTa FSE_buildDTable(DTableSpace, norm, max, tableLog); *DTablePtr = DTableSpace; return headerSize; - } } + } + default : /* impossible */ + assert(0); + } } size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, @@ -1443,7 +1445,7 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, srcSize -= seqHSize; if (dctx->fParams.windowSize > (1<<24)) { - U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->entropy.OFTable); + U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); U32 const minShare = MEM_64bits() ? 5 : 13; if (shareLongOffsets >= minShare) return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); From 04a3f85ce7ea2738e246cacfdb9c8827efcd9d90 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 9 Feb 2018 15:34:59 -0800 Subject: [PATCH 04/10] fixed gcc warning on a switch code path --- lib/decompress/zstd_decompress.c | 1 + programs/bench.c | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index c72fc71d8..17139d8ce 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -760,6 +760,7 @@ static size_t ZSTD_buildSeqTable(FSE_DTable* DTableSpace, const FSE_DTable** DTa } default : /* impossible */ assert(0); + return ERROR(GENERIC); } } diff --git a/programs/bench.c b/programs/bench.c index 843920c89..939890064 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -461,8 +461,8 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, } /* for (testNb = 1; testNb <= (g_nbSeconds + !g_nbSeconds); testNb++) */ if (g_displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */ - double cSpeed = (double)srcSize / fastestC; - double dSpeed = (double)srcSize / fastestD; + double const cSpeed = (double)srcSize / fastestC; + double const dSpeed = (double)srcSize / fastestD; if (g_additionalParam) DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam); else From cb789d2df8ac2c10d7cb46d7a4dea09a831b5c46 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 5 Mar 2018 13:08:59 -0800 Subject: [PATCH 05/10] re-inserted offset evaluation --- lib/decompress/zstd_decompress.c | 52 ++++++++++++++++----------- lib/decompress/zstd_decompress_impl.h | 20 ++--------- 2 files changed, 34 insertions(+), 38 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 5fa4e3056..470aac286 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1248,47 +1248,49 @@ typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset #endif typedef size_t (*ZSTD_decompressSequences_t)( - ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, const void *seqStart, - size_t seqSize, const ZSTD_longOffset_e isLongOffset); + ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, + const void *seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset); static size_t ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, + const void* seqStart, size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) { #if DYNAMIC_BMI2 if (dctx->bmi2) { - return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, isLongOffset); + return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif - return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, isLongOffset); + return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } -static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, - const ZSTD_longOffset_e isLongOffset) +static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) { #if DYNAMIC_BMI2 if (dctx->bmi2) { - return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, isLongOffset); + return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } #endif - return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, isLongOffset); + return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } static unsigned -ZSTD_getLongOffsetsShare(const FSE_DTable* offTable) +ZSTD_getLongOffsetsShare(const ZSTD_seqSymbol* offTable) { const void* ptr = offTable; U32 const tableLog = ((const FSE_DTableHeader*)ptr)[0].tableLog; - const FSE_decode_t* table = ((const FSE_decode_t*)ptr) + 1; + const ZSTD_seqSymbol* table = offTable + 1; U32 const max = 1 << tableLog; U32 u, total = 0; - assert(tableLog <= OffFSELog); for (u=0; u 23) total += 1; + if (table[u].nbAdditionalBits > 23) total += 1; - total <<= (OffFSELog - tableLog); + assert(tableLog <= OffFSELog); + total <<= (OffFSELog - tableLog); /* scale to OffFSELog */ return total; } @@ -1317,12 +1319,22 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, srcSize -= litCSize; } - if ( frame /* windowSize exists */ - && (dctx->fParams.windowSize > (1<<24)) - && MEM_64bits() /* x86 benefits less from long mode than x64 */ ) - return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, isLongOffset); + /* Build Decoding Tables */ + { int nbSeq; + size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); + if (ZSTD_isError(seqHSize)) return seqHSize; + ip += seqHSize; + srcSize -= seqHSize; - return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, isLongOffset); + if (dctx->fParams.windowSize > (1<<24)) { + U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); + U32 const minShare = MEM_64bits() ? 5 : 13; + if (shareLongOffsets >= minShare) + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); + } + + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); + } } diff --git a/lib/decompress/zstd_decompress_impl.h b/lib/decompress/zstd_decompress_impl.h index 6454a7429..18b3a296e 100644 --- a/lib/decompress/zstd_decompress_impl.h +++ b/lib/decompress/zstd_decompress_impl.h @@ -209,7 +209,7 @@ static TARGET size_t FUNCTION(ZSTD_decompressSequences)( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, + const void* seqStart, size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) { const BYTE* ip = (const BYTE*)seqStart; @@ -222,17 +222,8 @@ size_t FUNCTION(ZSTD_decompressSequences)( const BYTE* const base = (const BYTE*) (dctx->base); const BYTE* const vBase = (const BYTE*) (dctx->vBase); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - int nbSeq; DEBUGLOG(5, "ZSTD_decompressSequences"); - /* Build Decoding Tables */ - { size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); - DEBUGLOG(5, "ZSTD_decodeSeqHeaders: size=%u, nbSeq=%i", - (U32)seqHSize, nbSeq); - if (ZSTD_isError(seqHSize)) return seqHSize; - ip += seqHSize; - } - /* Regen sequences */ if (nbSeq) { seqState_t seqState; @@ -273,7 +264,7 @@ static TARGET size_t FUNCTION(ZSTD_decompressSequencesLong)( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, + const void* seqStart, size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) { const BYTE* ip = (const BYTE*)seqStart; @@ -286,13 +277,6 @@ size_t FUNCTION(ZSTD_decompressSequencesLong)( const BYTE* const prefixStart = (const BYTE*) (dctx->base); const BYTE* const dictStart = (const BYTE*) (dctx->vBase); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - int nbSeq; - - /* Build Decoding Tables */ - { size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); - if (ZSTD_isError(seqHSize)) return seqHSize; - ip += seqHSize; - } /* Regen sequences */ if (nbSeq) { From c0393a538f8a73160f5818afd276f4775804c642 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 5 Mar 2018 15:12:10 -0800 Subject: [PATCH 06/10] fixed counting long distance weights --- lib/decompress/zstd_decompress.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 470aac286..b28bcae63 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -94,7 +94,7 @@ typedef struct { U32 baseValue; } ZSTD_seqSymbol; -#define SEQSYMBOL_TABLE_SIZE(log) (1 + (1<bmi2) { return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); @@ -1269,6 +1270,7 @@ static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, const void* seqStart, size_t seqSize, int nbSeq, const ZSTD_longOffset_e isLongOffset) { + DEBUGLOG(5, "ZSTD_decompressSequencesLong"); #if DYNAMIC_BMI2 if (dctx->bmi2) { return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); @@ -1281,13 +1283,16 @@ static unsigned ZSTD_getLongOffsetsShare(const ZSTD_seqSymbol* offTable) { const void* ptr = offTable; - U32 const tableLog = ((const FSE_DTableHeader*)ptr)[0].tableLog; + U32 const tableLog = ((const ZSTD_seqSymbol_header*)ptr)[0].tableLog; const ZSTD_seqSymbol* table = offTable + 1; U32 const max = 1 << tableLog; U32 u, total = 0; + DEBUGLOG(5, "ZSTD_getLongOffsetsShare: (tableLog=%u)", tableLog); - for (u=0; u 23) total += 1; + assert(max <= (1 << OffFSELog)); /* max not too large */ + for (u=0; u 22) total += 1; + } assert(tableLog <= OffFSELog); total <<= (OffFSELog - tableLog); /* scale to OffFSELog */ From 7bd7a3ad432b2102dbb33b88c12a9e05d693df0e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 5 Mar 2018 16:16:49 -0800 Subject: [PATCH 07/10] long offset mode : new default threshold for 64-bits mode --- lib/decompress/zstd_decompress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index b28bcae63..ef6e16d3f 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1333,7 +1333,7 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, if (dctx->fParams.windowSize > (1<<24)) { U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); - U32 const minShare = MEM_64bits() ? 5 : 13; + U32 const minShare = MEM_64bits() ? 7 : 13; if (shareLongOffsets >= minShare) return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); } From 9a91afe6efc8ac24a8b3f7f3b98df42d44abe07f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 5 Mar 2018 16:41:08 -0800 Subject: [PATCH 08/10] long offset mode : new default threshold for 32-bit --- lib/decompress/zstd_decompress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index ef6e16d3f..12c46e572 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1333,7 +1333,7 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, if (dctx->fParams.windowSize > (1<<24)) { U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); - U32 const minShare = MEM_64bits() ? 7 : 13; + U32 const minShare = MEM_64bits() ? 7 : 20; if (shareLongOffsets >= minShare) return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); } From 06ca9c7d7ce2c9883551ab1e4c535f6a456b5e5d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 6 Mar 2018 01:50:19 -0800 Subject: [PATCH 09/10] fixed 0-seq blocks in block-decompression mode --- lib/decompress/zstd_decompress.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 12c46e572..53f555337 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1331,7 +1331,8 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, ip += seqHSize; srcSize -= seqHSize; - if (dctx->fParams.windowSize > (1<<24)) { + if ( (dctx->fParams.windowSize > (1<<24) || !frame) + && (nbSeq>0) ) { /* could probably use a larger nbSeq limit */ U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); U32 const minShare = MEM_64bits() ? 7 : 20; if (shareLongOffsets >= minShare) From db147ea620151f30b6825ad7c2fb29161cade94d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 6 Mar 2018 18:15:26 -0800 Subject: [PATCH 10/10] improved comments following @terrelln suggestions --- lib/decompress/zstd_decompress.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 53f555337..8aa21db7e 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1279,6 +1279,10 @@ static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); } +/* ZSTD_getLongOffsetsShare() : + * condition : offTable must be valid + * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) + * compared to maximum possible of (1<fParams.windowSize > (1<<24) || !frame) + if ( (!frame || dctx->fParams.windowSize > (1<<24)) && (nbSeq>0) ) { /* could probably use a larger nbSeq limit */ U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); - U32 const minShare = MEM_64bits() ? 7 : 20; + U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ if (shareLongOffsets >= minShare) return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); }