From 4a9b7d136fad5cd663cd86db2499d6175693d0bb Mon Sep 17 00:00:00 2001 From: Carl Woffenden Date: Mon, 22 Jun 2020 10:31:36 +0200 Subject: [PATCH 1/2] Initial implementation (files added, macros fixed) Hashing functions still to fix. --- contrib/single_file_libs/zstd-in.c | 5 +++++ lib/dictBuilder/cover.c | 17 +++++++++++++---- lib/dictBuilder/fastcover.c | 17 +++++++++++++---- lib/dictBuilder/zdict.c | 3 +++ 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/contrib/single_file_libs/zstd-in.c b/contrib/single_file_libs/zstd-in.c index cac805549..d18db202b 100644 --- a/contrib/single_file_libs/zstd-in.c +++ b/contrib/single_file_libs/zstd-in.c @@ -74,3 +74,8 @@ #include "decompress/zstd_ddict.c" #include "decompress/zstd_decompress.c" #include "decompress/zstd_decompress_block.c" + +#include "dictBuilder/cover.c" +#include "dictBuilder/divsufsort.c" +#include "dictBuilder/fastcover.c" +#include "dictBuilder/zdict.c" diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index da54ef15f..fb453f8df 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -40,23 +40,33 @@ * Constants ***************************************/ #define COVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB)) -#define DEFAULT_SPLITPOINT 1.0 +#define COVER_DEFAULT_SPLITPOINT 1.0 /*-************************************* * Console display ***************************************/ +#ifndef LOCALDISPLAYLEVEL static int g_displayLevel = 2; +#endif +#undef DISPLAY #define DISPLAY(...) \ { \ fprintf(stderr, __VA_ARGS__); \ fflush(stderr); \ } +#undef LOCALDISPLAYLEVEL #define LOCALDISPLAYLEVEL(displayLevel, l, ...) \ if (displayLevel >= l) { \ DISPLAY(__VA_ARGS__); \ } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ +#undef DISPLAYLEVEL #define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__) +#ifndef LOCALDISPLAYUPDATE +static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; +#endif +#undef LOCALDISPLAYUPDATE #define LOCALDISPLAYUPDATE(displayLevel, l, ...) \ if (displayLevel >= l) { \ if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \ @@ -64,9 +74,8 @@ static int g_displayLevel = 2; DISPLAY(__VA_ARGS__); \ } \ } +#undef DISPLAYUPDATE #define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__) -static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; -static clock_t g_time = 0; /*-************************************* * Hash table @@ -1106,7 +1115,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( /* constants */ const unsigned nbThreads = parameters->nbThreads; const double splitPoint = - parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; + parameters->splitPoint <= 0.0 ? COVER_DEFAULT_SPLITPOINT : parameters->splitPoint; const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c index 485c333b5..a3be4c266 100644 --- a/lib/dictBuilder/fastcover.c +++ b/lib/dictBuilder/fastcover.c @@ -33,7 +33,7 @@ #define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((unsigned)-1) : ((unsigned)1 GB)) #define FASTCOVER_MAX_F 31 #define FASTCOVER_MAX_ACCEL 10 -#define DEFAULT_SPLITPOINT 0.75 +#define FASTCOVER_DEFAULT_SPLITPOINT 0.75 #define DEFAULT_F 20 #define DEFAULT_ACCEL 1 @@ -41,18 +41,28 @@ /*-************************************* * Console display ***************************************/ +#ifndef LOCALDISPLAYLEVEL static int g_displayLevel = 2; +#endif +#undef DISPLAY #define DISPLAY(...) \ { \ fprintf(stderr, __VA_ARGS__); \ fflush(stderr); \ } +#undef LOCALDISPLAYLEVEL #define LOCALDISPLAYLEVEL(displayLevel, l, ...) \ if (displayLevel >= l) { \ DISPLAY(__VA_ARGS__); \ } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ +#undef DISPLAYLEVEL #define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__) +#ifndef LOCALDISPLAYUPDATE +static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; +#endif +#undef LOCALDISPLAYUPDATE #define LOCALDISPLAYUPDATE(displayLevel, l, ...) \ if (displayLevel >= l) { \ if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \ @@ -60,9 +70,8 @@ static int g_displayLevel = 2; DISPLAY(__VA_ARGS__); \ } \ } +#undef DISPLAYUPDATE #define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__) -static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; -static clock_t g_time = 0; /*-************************************* @@ -617,7 +626,7 @@ ZDICT_optimizeTrainFromBuffer_fastCover( /* constants */ const unsigned nbThreads = parameters->nbThreads; const double splitPoint = - parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; + parameters->splitPoint <= 0.0 ? FASTCOVER_DEFAULT_SPLITPOINT : parameters->splitPoint; const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 6f13392a0..98c6c4135 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -69,7 +69,9 @@ static const U32 g_selectivity_default = 9; /*-************************************* * Console display ***************************************/ +#undef DISPLAY #define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); } +#undef DISPLAYLEVEL #define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; } @@ -529,6 +531,7 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize, clock_t displayClock = 0; clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10; +# undef DISPLAYUPDATE # define DISPLAYUPDATE(l, ...) if (notificationLevel>=l) { \ if (ZDICT_clockSpan(displayClock) > refreshRate) \ { displayClock = clock(); DISPLAY(__VA_ARGS__); \ From 38cdb6a072b59db037a7b8ac8a145ccb00769fce Mon Sep 17 00:00:00 2001 From: Carl Woffenden Date: Mon, 22 Jun 2020 11:54:24 +0200 Subject: [PATCH 2/2] Renamed cover and fast cover hash functions/vars --- lib/dictBuilder/cover.c | 4 ++-- lib/dictBuilder/fastcover.c | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index fb453f8df..50217c13d 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -129,9 +129,9 @@ static int COVER_map_init(COVER_map_t *map, U32 size) { /** * Internal hash function */ -static const U32 prime4bytes = 2654435761U; +static const U32 COVER_prime4bytes = 2654435761U; static U32 COVER_map_hash(COVER_map_t *map, U32 key) { - return (key * prime4bytes) >> (32 - map->sizeLog); + return (key * COVER_prime4bytes) >> (32 - map->sizeLog); } /** diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c index a3be4c266..520ca1929 100644 --- a/lib/dictBuilder/fastcover.c +++ b/lib/dictBuilder/fastcover.c @@ -75,15 +75,15 @@ static clock_t g_time = 0; /*-************************************* -* Hash Functions +* Hash Functions (matching zstd_compress_internal.h) ***************************************/ -static const U64 prime6bytes = 227718039650203ULL; -static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; } -static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); } +static const U64 FASTCOVER_prime6bytes = 227718039650203ULL; +static size_t FASTCOVER_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * FASTCOVER_prime6bytes) >> (64-h)) ; } +static size_t FASTCOVER_hash6Ptr(const void* p, U32 h) { return FASTCOVER_hash6(MEM_readLE64(p), h); } -static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; -static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } -static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } +static const U64 FASTCOVER_prime8bytes = 0xCF1BBCDCB7A56463ULL; +static size_t FASTCOVER_hash8(U64 u, U32 h) { return (size_t)(((u) * FASTCOVER_prime8bytes) >> (64-h)) ; } +static size_t FASTCOVER_hash8Ptr(const void* p, U32 h) { return FASTCOVER_hash8(MEM_readLE64(p), h); } /** @@ -91,9 +91,9 @@ static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE */ static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 h, unsigned d) { if (d == 6) { - return ZSTD_hash6Ptr(p, h) & ((1 << h) - 1); + return FASTCOVER_hash6Ptr(p, h) & ((1 << h) - 1); } - return ZSTD_hash8Ptr(p, h) & ((1 << h) - 1); + return FASTCOVER_hash8Ptr(p, h) & ((1 << h) - 1); }