From c7190c69ccb83b68b3b4e1a34d7e57470d7d6258 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 18 Jan 2018 11:15:23 -0800 Subject: [PATCH 1/3] fixes for @terrelln comments --- lib/compress/zstd_compress_internal.h | 6 +++--- lib/compress/zstdmt_compress.c | 2 +- lib/zstd.h | 6 ++++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h index 899c5b08b..64f03a0d2 100644 --- a/lib/compress/zstd_compress_internal.h +++ b/lib/compress/zstd_compress_internal.h @@ -170,9 +170,9 @@ struct ZSTD_CCtx_s { void* workSpace; size_t workSpaceSize; size_t blockSize; - U64 pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */ - U64 consumedSrcSize; - U64 producedCSize; + unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */ + unsigned long long consumedSrcSize; + unsigned long long producedCSize; XXH64_state_t xxhState; ZSTD_customMem customMem; size_t staticSize; diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 5624b5e3d..f9a25270a 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -662,7 +662,7 @@ unsigned ZSTDMT_getNbThreads(const ZSTDMT_CCtx* mtctx) /* ZSTDMT_getFrameProgression(): * tells how much data has been consumed (input) and produced (output) for current frame. * able to count progression inside worker threads. - * Note : mutex will be triggered during statistics collection. */ + * Note : mutex will be acquired during statistics collection. */ ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx) { ZSTD_frameProgression fs; diff --git a/lib/zstd.h b/lib/zstd.h index a34d37f0b..aab6be3b0 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -723,8 +723,10 @@ typedef struct { } ZSTD_frameProgression; /* ZSTD_getFrameProgression(): - * tells how much data has been consumed (input) and produced (output) for current frame. - * able to count progression inside worker threads (non-blocking mode). + * tells how much data has been ingested (read from input) + * consumed (input actually compressed) and produced (output) for current frame. + * Therefore, (ingested - consumed) is amount of input data buffered internally, not yet compressed. + * Can report progression inside worker threads (multi-threading and non-blocking mode). */ ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); From 4d08ba8b779e4e653a45102667b04611b0213529 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 18 Jan 2018 11:27:13 -0800 Subject: [PATCH 2/3] fileio: READY_FOR_UPDATE() is now a function-like macro as suggested by @terrelln --- programs/fileio.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 18552aa3b..7045a5323 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -84,10 +84,10 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } static const U64 g_refreshRate = SEC_TO_MICRO / 6; static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; -#define READY_FOR_UPDATE (UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) +#define READY_FOR_UPDATE() (UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) #define DISPLAYUPDATE(l, ...) { \ if (g_displayLevel>=l) { \ - if (READY_FOR_UPDATE || (g_displayLevel>=4)) { \ + if (READY_FOR_UPDATE() || (g_displayLevel>=4)) { \ g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ if (g_displayLevel>=4) fflush(stderr); \ } } } @@ -813,7 +813,7 @@ static int FIO_compressFilename_internal(cRess_t ress, } } #if 1 - if (READY_FOR_UPDATE) { + if (READY_FOR_UPDATE()) { ZSTD_frameProgression const zfp = ZSTD_getFrameProgression(ress.cctx); DISPLAYUPDATE(2, "\rRead :%6u MB - Consumed :%6u MB - Compressed :%6u MB => %.2f%%", (U32)(zfp.ingested >> 20), From 9d96761520e75d9902bc94cf319e8c18d4ce9e1a Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 18 Jan 2018 13:28:30 -0800 Subject: [PATCH 3/3] Set repcodes for empty ZSTD_CDict When the dictionary is <= 8 bytes, no data is loaded from the dictionary. In this case the repcodes weren't set, because they were inserted after the size check. Fix this problem in general by first setting the cdict state to a clean state of an empty dictionary, then filling the state from there. --- lib/compress/zstd_compress.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index bbe9146bd..9b927367f 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2540,6 +2540,8 @@ static size_t ZSTD_initCDict_internal( } cdict->dictContentSize = dictSize; + /* Reset the state to no dictionary */ + ZSTD_reset_compressedBlockState(&cdict->cBlockState); { void* const end = ZSTD_reset_matchState( &cdict->matchState, @@ -2548,6 +2550,9 @@ static size_t ZSTD_initCDict_internal( assert(end == (char*)cdict->workspace + cdict->workspaceSize); (void)end; } + /* (Maybe) load the dictionary + * Skips loading the dictionary if it is <= 8 bytes. + */ { ZSTD_CCtx_params params; memset(¶ms, 0, sizeof(params));