From ff773bfcdebb75730eb0546c425030396168b6fb Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 26 Jun 2018 17:24:41 -0700 Subject: [PATCH 1/2] zeroise freq table with memset() improves decoding speed by ~5% in github_users sample set --- lib/common/entropy_common.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c index 33fd04bd6..b12944e1d 100644 --- a/lib/common/entropy_common.c +++ b/lib/common/entropy_common.c @@ -85,6 +85,8 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t } } assert(hbSize >= 4); + /* init */ + memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */ bitStream = MEM_readLE32(ip); nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); @@ -156,11 +158,6 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t } } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ if (remaining != 1) return ERROR(corruption_detected); if (bitCount > 32) return ERROR(corruption_detected); - /* zeroise the rest */ - { unsigned symbNb = charnum; - for (symbNb=charnum; symbNb <= *maxSVPtr; symbNb++) - normalizedCounter[symbNb] = 0; - } *maxSVPtr = charnum-1; ip += (bitCount+7)>>3; From 4489daec09f5fe058f107ed4828f4fff0c563004 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 26 Jun 2018 20:10:45 -0700 Subject: [PATCH 2/2] slightly adjusted default-distribution threshold depending on strategy. fast favors faster compression and decompression speeds. --- doc/zstd_manual.html | 22 ++++++++++++++++++++++ lib/compress/zstd_compress.c | 5 ++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 6bf731dd9..bd792008b 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -872,6 +872,28 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize, * even when referencing into Dictionary content (default:0) */ + ZSTD_p_forceAttachDict, /* ZSTD supports usage of a CDict in-place + * (avoiding having to copy the compression tables + * from the CDict into the working context). Using + * a CDict in this way saves an initial setup step, + * but comes at the cost of more work per byte of + * input. ZSTD has a simple internal heuristic that + * guesses which strategy will be faster. You can + * use this flag to override that guess. + * + * Note that the by-reference, in-place strategy is + * only used when reusing a compression context + * with compatible compression parameters. (If + * incompatible / uninitialized, the working + * context needs to be cleared anyways, which is + * about as expensive as overwriting it with the + * dictionary context, so there's no savings in + * using the CDict by-ref.) + * + * Values greater than 0 force attaching the dict. + * Values less than 0 force copying the dict. + * 0 selects the default heuristic-guided behavior. + */ } ZSTD_cParameter;
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 96ffe244a..c66862524 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1819,8 +1819,11 @@ ZSTD_selectEncodingType( if (strategy < ZSTD_lazy) { if (isDefaultAllowed) { size_t const staticFse_nbSeq_max = 1000; - size_t const dynamicFse_nbSeq_min = (size_t)1 << defaultNormLog; /* 32 for offset, 64 for lengths */ + size_t const mult = 10 - strategy; + size_t const baseLog = 3; + size_t const dynamicFse_nbSeq_min = (((size_t)1 << defaultNormLog) * mult) >> baseLog; /* 28-36 for offset, 56-72 for lengths */ assert(defaultNormLog >= 5 && defaultNormLog <= 6); /* xx_DEFAULTNORMLOG */ + assert(mult <= 9 && mult >= 7); if ( (*repeatMode == FSE_repeat_valid) && (nbSeq < staticFse_nbSeq_max) ) { DEBUGLOG(5, "Selected set_repeat");