From ed1a42987301ada4968c6acd86dafe56e8d10605 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 27 Jun 2018 16:57:28 -0700 Subject: [PATCH 01/14] test multi-lines travis yaml file --- .travis.yml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index b7099c24f..20c16fa90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,28 +10,32 @@ addons: matrix: include: # Ubuntu 14.04 - - env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd' + - env: Cmd='make gcc6install && CC=gcc-6 make -j all \ + && make clean && CC=gcc-6 make clean uasan-test-zstd' - env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test-zstd32' - env: Cmd='make gcc7install && CC=gcc-7 make clean uasan-test-zstd' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd' - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest' - - env: Cmd='make gcc6install libc6install && CC=gcc-6 CFLAGS=-m32 make clean uasan-fuzztest' + - env: Cmd='make gcc6install libc6install \ + && make clean && CC=gcc-6 CFLAGS=-m32 make uasan-fuzztest' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest' - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream' - - env: Cmd='make -C tests test-fuzzer-stackmode' - - - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest' - - env: Cmd='make arminstall && make armfuzz' - env: Cmd='make arminstall && make aarch64fuzz' - - env: Cmd='make ppcinstall && make ppcfuzz' - env: Cmd='make ppcinstall && make ppc64fuzz' - - env: Cmd='make -j uasanregressiontest && make clean && make -j msanregressiontest' - - env: Cmd='make lz4install && make -C tests test-lz4 test-pool && make clean && bash tests/libzstd_partial_builds.sh' + - env: Cmd='make -j uasanregressiontest \ + && make clean && make -j msanregressiontest' + + - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest \ + && make clean && make -C tests test-fuzzer-stackmode' + + - env: Cmd='make lz4install && make -C tests test-lz4 \ + && make clean && make -C tests test-pool \ + && make clean && bash tests/libzstd_partial_builds.sh' # tag-specific test - if: tag =~ ^v[0-9]\.[0-9] @@ -44,6 +48,7 @@ branches: only: - dev - master + - travisTest script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') From ab1ebd6578f9b1528382eaa948374b9eed490034 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 27 Jun 2018 19:29:18 -0700 Subject: [PATCH 02/14] removed \ at end of line --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 20c16fa90..566f3aa4e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,14 +10,14 @@ addons: matrix: include: # Ubuntu 14.04 - - env: Cmd='make gcc6install && CC=gcc-6 make -j all \ + - env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd' - env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test-zstd32' - env: Cmd='make gcc7install && CC=gcc-7 make clean uasan-test-zstd' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd' - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest' - - env: Cmd='make gcc6install libc6install \ + - env: Cmd='make gcc6install libc6install && make clean && CC=gcc-6 CFLAGS=-m32 make uasan-fuzztest' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest' - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream' @@ -27,14 +27,14 @@ matrix: - env: Cmd='make ppcinstall && make ppcfuzz' - env: Cmd='make ppcinstall && make ppc64fuzz' - - env: Cmd='make -j uasanregressiontest \ + - env: Cmd='make -j uasanregressiontest && make clean && make -j msanregressiontest' - - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest \ + - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest && make clean && make -C tests test-fuzzer-stackmode' - - env: Cmd='make lz4install && make -C tests test-lz4 \ - && make clean && make -C tests test-pool \ + - env: Cmd='make lz4install && make -C tests test-lz4 + && make clean && make -C tests test-pool && make clean && bash tests/libzstd_partial_builds.sh' # tag-specific test From b8a81a988c5d81f5efdc4af9f28417f3fe71cfe8 Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Tue, 17 Jul 2018 19:02:17 +0200 Subject: [PATCH 03/14] added a test to be played on travis to check a make variable construction --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 80406064d..1fd7e6aad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ addons: matrix: include: # Ubuntu 14.04 + - env: Cmd='make test' - env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd Date: Tue, 17 Jul 2018 19:19:48 +0200 Subject: [PATCH 04/14] fix make test on Linux MOREFLAGS+= doesn't work on Linux --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0f2fcc958..89c5a418d 100644 --- a/Makefile +++ b/Makefile @@ -63,8 +63,9 @@ zlibwrapper: $(MAKE) -C $(ZWRAPDIR) test .PHONY: test +test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror test: - MOREFLAGS+="-g -DDEBUGLEVEL=1 -Werror" $(MAKE) -C $(PRGDIR) allVariants + MOREFLAGS="$(MOREFLAGS)" $(MAKE) -C $(PRGDIR) allVariants $(MAKE) -C $(TESTDIR) $@ .PHONY: shortest From 9889bca530a2b52615eb1cd06260e9cd0c29e001 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 27 Jul 2018 17:30:03 -0700 Subject: [PATCH 05/14] [FSE] Fix division by zero When the primary normalization method fails, and `(1 << tableLog) == (maxSymbolValue + 1)`, and every symbol gets assigned normalized weight 1 or -1 in the first loop, then the next division can raise `SIGFPE`. --- lib/compress/fse_compress.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index 07b3ab89b..95e7c1c7e 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -394,6 +394,9 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, } ToDistribute = (1 << tableLog) - distributed; + if (ToDistribute == 0) + return 0; + if ((total / ToDistribute) > lowOne) { /* risk of rounding to zero */ lowOne = (U32)((total * 3) / (ToDistribute * 2)); From 31229e527bfa3ae9f0a5e81ed7e00a56dfd494bf Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Mon, 30 Jul 2018 12:54:22 -0700 Subject: [PATCH 06/14] Increment frequency for every dmer occurence within same sample instead of at most once per sample --- .../benchmarkDictBuilder/README.md | 200 +++++++++--------- .../fastCover/fastCover.c | 10 +- 2 files changed, 101 insertions(+), 109 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index 20fbde954..1fdd323c2 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -18,109 +18,109 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" - Fourth column is chosen d and fifth column is chosen k github: -NODICT 0.000005 2.999642 -RANDOM 0.141553 8.786957 -LEGACY 0.904340 8.989482 -COVER 53.621302 10.641263 8 1298 -COVER 4.085037 10.641263 8 1298 -FAST15 17.636211 10.586461 8 1778 -FAST15 0.221236 10.586461 8 1778 -FAST16 18.716259 10.492503 6 1778 -FAST16 0.251522 10.492503 6 1778 -FAST17 17.614391 10.611737 8 1778 -FAST17 0.241011 10.611737 8 1778 -FAST18 19.926270 10.621586 8 1778 -FAST18 0.287195 10.621586 8 1778 -FAST19 19.626808 10.629626 8 1778 -FAST19 0.340191 10.629626 8 1778 -FAST20 18.918657 10.610308 8 1778 -FAST20 0.463307 10.610308 8 1778 -FAST21 20.502362 10.625733 8 1778 -FAST21 0.638202 10.625733 8 1778 -FAST22 22.702695 10.625281 8 1778 -FAST22 1.353399 10.625281 8 1778 -FAST23 28.041990 10.602342 8 1778 -FAST23 3.029502 10.602342 8 1778 -FAST24 35.662961 10.603379 8 1778 -FAST24 6.524258 10.603379 8 1778 +NODICT 0.000004 2.999642 +RANDOM 0.161907 8.786957 +LEGACY 0.960128 8.989482 +COVER 69.031037 10.641263 8 1298 +COVER 7.017782 10.641263 8 1298 +FAST15 24.710713 10.547583 8 1874 +FAST15 0.271657 10.547583 8 1874 +FAST16 23.906902 10.690723 8 1106 +FAST16 0.315039 10.690723 8 1106 +FAST17 25.384572 10.642322 8 1106 +FAST17 0.319237 10.642322 8 1106 +FAST18 21.935494 10.491283 8 1826 +FAST18 0.255488 10.491283 8 1826 +FAST19 21.349385 10.522182 8 1826 +FAST19 0.311369 10.522182 8 1826 +FAST20 23.124955 10.487431 8 1826 +FAST20 0.317411 10.487431 8 1826 +FAST21 27.311387 10.491047 8 1778 +FAST21 0.398483 10.491047 8 1778 +FAST22 23.993620 10.502191 8 1826 +FAST22 0.329767 10.502191 8 1826 +FAST23 27.793381 10.502191 8 1826 +FAST23 0.359659 10.502191 8 1826 +FAST24 29.281399 10.509461 8 1826 +FAST24 0.398369 10.509461 8 1826 hg-commands: -NODICT 0.000005 2.425291 -RANDOM 0.080469 3.489515 -LEGACY 0.794417 3.911896 -COVER 54.198788 4.131136 8 386 -COVER 2.191729 4.131136 8 386 -FAST15 11.852793 3.903719 6 1106 -FAST15 0.175406 3.903719 6 1106 -FAST16 12.863315 4.005077 8 530 -FAST16 0.158410 4.005077 8 530 -FAST17 11.977917 4.097811 8 818 -FAST17 0.162381 4.097811 8 818 -FAST18 11.749304 4.136081 8 770 -FAST18 0.173242 4.136081 8 770 -FAST19 11.905785 4.166021 8 530 -FAST19 0.186403 4.166021 8 530 -FAST20 13.293999 4.163740 8 482 -FAST20 0.241508 4.163740 8 482 -FAST21 16.623177 4.157057 8 434 -FAST21 0.372647 4.157057 8 434 -FAST22 20.918409 4.158195 8 290 -FAST22 0.570431 4.158195 8 290 -FAST23 21.762805 4.161450 8 434 -FAST23 1.162206 4.161450 8 434 -FAST24 29.133745 4.159658 8 338 -FAST24 3.054376 4.159658 8 338 +NODICT 0.000007 2.425291 +RANDOM 0.083477 3.489515 +LEGACY 0.941867 3.911896 +COVER 67.314295 4.131136 8 386 +COVER 2.757895 4.131136 8 386 +FAST15 13.466983 3.920128 6 1106 +FAST15 0.162656 3.920128 6 1106 +FAST16 12.618110 4.032422 8 674 +FAST16 0.159073 4.032422 8 674 +FAST17 12.883772 4.063581 8 1490 +FAST17 0.183131 4.063581 8 1490 +FAST18 13.904432 4.085034 8 290 +FAST18 0.161078 4.085034 8 290 +FAST19 13.762269 4.097054 8 578 +FAST19 0.179906 4.097054 8 578 +FAST20 15.303927 4.101575 8 434 +FAST20 0.213146 4.101575 8 434 +FAST21 19.619482 4.104879 8 530 +FAST21 0.289158 4.104879 8 530 +FAST22 23.187937 4.102448 8 530 +FAST22 0.335220 4.102448 8 530 +FAST23 24.946655 4.095162 8 914 +FAST23 0.396927 4.095162 8 914 +FAST24 27.634065 4.114624 8 722 +FAST24 0.434278 4.114624 8 722 hg-changelog: -NODICT 0.000006 1.377613 -RANDOM 0.601346 2.096785 -LEGACY 2.544973 2.058273 -COVER 222.639708 2.188654 8 98 -COVER 6.072892 2.188654 8 98 -FAST15 70.394523 2.127194 8 866 -FAST15 0.899766 2.127194 8 866 -FAST16 69.845529 2.145401 8 338 -FAST16 0.881569 2.145401 8 338 -FAST17 69.382431 2.157544 8 194 -FAST17 0.943291 2.157544 8 194 -FAST18 71.348283 2.173127 8 98 -FAST18 1.034765 2.173127 8 98 -FAST19 71.380923 2.179527 8 98 -FAST19 1.254700 2.179527 8 98 -FAST20 72.802714 2.183233 6 98 -FAST20 1.368704 2.183233 6 98 -FAST21 82.042339 2.180920 8 98 -FAST21 2.213864 2.180920 8 98 -FAST22 90.666200 2.184297 8 98 -FAST22 3.590399 2.184297 8 98 -FAST23 108.926377 2.187666 6 98 -FAST23 8.723759 2.187666 6 98 -FAST24 134.296232 2.189889 6 98 -FAST24 19.396532 2.189889 6 98 +NODICT 0.000027 1.377613 +RANDOM 0.676272 2.096785 +LEGACY 2.871887 2.058273 +COVER 226.371004 2.188654 8 98 +COVER 5.359820 2.188654 8 98 +FAST15 66.776425 2.130548 6 386 +FAST15 0.796836 2.130548 6 386 +FAST16 64.405113 2.144136 8 194 +FAST16 0.778969 2.144136 8 194 +FAST17 65.062292 2.155745 8 98 +FAST17 0.822089 2.155745 8 98 +FAST18 65.819104 2.172062 6 98 +FAST18 0.804247 2.172062 6 98 +FAST19 66.184016 2.179446 6 98 +FAST19 0.883526 2.179446 6 98 +FAST20 72.900924 2.187017 6 98 +FAST20 0.908220 2.187017 6 98 +FAST21 77.869945 2.183583 6 146 +FAST21 0.932666 2.183583 6 146 +FAST22 84.041413 2.182030 6 98 +FAST22 1.092310 2.182030 6 98 +FAST23 89.539265 2.185291 8 98 +FAST23 1.294779 2.185291 8 98 +FAST24 97.193482 2.184939 6 98 +FAST24 1.270493 2.184939 6 98 hg-manifest: -NODICT 0.000005 1.866385 -RANDOM 0.982192 2.309485 -LEGACY 9.507729 2.506775 -COVER 922.742066 2.582597 8 434 -COVER 36.500276 2.582597 8 434 -FAST15 163.886717 2.377689 8 1682 -FAST15 2.107328 2.377689 8 1682 -FAST16 152.684592 2.464814 8 1538 -FAST16 2.157789 2.464814 8 1538 -FAST17 154.463459 2.539834 6 1826 -FAST17 2.282455 2.539834 6 1826 -FAST18 155.540044 2.576924 8 1922 -FAST18 2.101807 2.576924 8 1922 -FAST19 152.650343 2.592479 6 290 -FAST19 2.359461 2.592479 6 290 -FAST20 174.623634 2.594551 8 194 -FAST20 2.870022 2.594551 8 194 -FAST21 219.876653 2.597128 6 194 -FAST21 4.386269 2.597128 6 194 -FAST22 247.986803 2.596971 6 386 -FAST22 6.201144 2.596971 6 386 -FAST23 276.051806 2.601416 8 194 -FAST23 11.613477 2.601416 8 194 -FAST24 328.234024 2.602830 6 194 -FAST24 26.710364 2.602830 6 194 +NODICT 0.000004 1.866385 +RANDOM 0.969045 2.309485 +LEGACY 8.849052 2.506775 +COVER 905.855524 2.582597 8 434 +COVER 34.951973 2.582597 8 434 +FAST15 154.816926 2.391764 6 1826 +FAST15 1.932845 2.391764 6 1826 +FAST16 142.197120 2.480738 6 1922 +FAST16 1.759330 2.480738 6 1922 +FAST17 147.276099 2.548313 6 1682 +FAST17 1.819175 2.548313 6 1682 +FAST18 164.543366 2.567448 6 386 +FAST18 2.728845 2.567448 6 386 +FAST19 195.670852 2.581170 8 338 +FAST19 2.439487 2.581170 8 338 +FAST20 195.716408 2.587062 6 194 +FAST20 2.056303 2.587062 6 194 +FAST21 211.483191 2.590136 6 242 +FAST21 2.983587 2.590136 6 242 +FAST22 239.562966 2.591033 6 194 +FAST22 3.355746 2.591033 6 194 +FAST23 264.547195 2.590403 8 434 +FAST23 3.667851 2.590403 8 434 +FAST24 296.258379 2.591723 6 290 +FAST24 3.858688 2.591723 6 290 diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c index 3c1aa951c..cf71075ab 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.c +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -266,25 +266,17 @@ static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t *ctx) { * Calculate for frequency of hash value of each dmer in ctx->samples */ static void FASTCOVER_computeFrequency(U32 *freqs, unsigned f, FASTCOVER_ctx_t *ctx){ - /* inCurrSample keeps track of this hash value has already be seen in previous dmers in the same sample*/ - BYTE* inCurrSample = (BYTE *)malloc((1 << f) * sizeof(BYTE)); size_t start; /* start of current dmer */ for (unsigned i = 0; i < ctx->nbTrainSamples; i++) { - memset(inCurrSample, 0, (1 << f) * sizeof(*inCurrSample)); /* Reset inCurrSample for each sample */ size_t currSampleStart = ctx->offsets[i]; size_t currSampleEnd = ctx->offsets[i+1]; start = currSampleStart; while (start + ctx->d <= currSampleEnd) { const size_t dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, ctx->d); - /* if no dmer with same hash value has been seen in current sample */ - if (inCurrSample[dmerIndex] == 0) { - inCurrSample[dmerIndex]++; - freqs[dmerIndex]++; - } + freqs[dmerIndex]++; start++; } } - free(inCurrSample); } /** From b9faaa1dc3b32b0e6ed248e860db54b4052f0aab Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 30 Jul 2018 12:57:11 -0700 Subject: [PATCH 07/14] [FSE] Add division by zero test --- tests/fuzzer.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index f7bacd5ca..8856a504a 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -27,6 +27,7 @@ #include /* strcmp */ #include #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ +#include "fse.h" #include "zstd.h" /* ZSTD_VERSION_STRING */ #include "zstd_errors.h" /* ZSTD_getErrorCode */ #include "zstdmt_compress.h" @@ -1423,6 +1424,24 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++); + { + short norm[32]; + unsigned count[32]; + unsigned const tableLog = 5; + size_t const nbSeq = 32; + unsigned const maxSymbolValue = 31; + size_t i; + + for (i = 0; i < 32; ++i) + count[i] = 1; + /* Calling FSE_normalizeCount() on a uniform distribution should not + * cause a division by zero. + */ + FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue); + } + DISPLAYLEVEL(3, "OK \n"); + _end: free(CNBuffer); free(compressedBuffer); From 4e29bc24699f5e6beb7cff70bc7ffc388bd481e1 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Tue, 31 Jul 2018 10:36:45 -0700 Subject: [PATCH 08/14] Use CDict instead of CCtx in analyzeEntropy --- .../benchmarkDictBuilder/README.md | 200 +++++++++--------- lib/dictBuilder/zdict.c | 32 ++- 2 files changed, 114 insertions(+), 118 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index 1fdd323c2..a18311973 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -18,109 +18,109 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" - Fourth column is chosen d and fifth column is chosen k github: -NODICT 0.000004 2.999642 -RANDOM 0.161907 8.786957 -LEGACY 0.960128 8.989482 -COVER 69.031037 10.641263 8 1298 -COVER 7.017782 10.641263 8 1298 -FAST15 24.710713 10.547583 8 1874 -FAST15 0.271657 10.547583 8 1874 -FAST16 23.906902 10.690723 8 1106 -FAST16 0.315039 10.690723 8 1106 -FAST17 25.384572 10.642322 8 1106 -FAST17 0.319237 10.642322 8 1106 -FAST18 21.935494 10.491283 8 1826 -FAST18 0.255488 10.491283 8 1826 -FAST19 21.349385 10.522182 8 1826 -FAST19 0.311369 10.522182 8 1826 -FAST20 23.124955 10.487431 8 1826 -FAST20 0.317411 10.487431 8 1826 -FAST21 27.311387 10.491047 8 1778 -FAST21 0.398483 10.491047 8 1778 -FAST22 23.993620 10.502191 8 1826 -FAST22 0.329767 10.502191 8 1826 -FAST23 27.793381 10.502191 8 1826 -FAST23 0.359659 10.502191 8 1826 -FAST24 29.281399 10.509461 8 1826 -FAST24 0.398369 10.509461 8 1826 +NODICT 0.000005 2.999642 +RANDOM 0.036114 8.791189 +LEGACY 1.111024 8.173529 +COVER 57.856477 10.652243 8 1298 +COVER 5.769965 10.652243 8 1298 +FAST15 9.965877 10.555630 8 1874 +FAST15 0.140285 10.555630 8 1874 +FAST16 10.337194 10.701698 8 1106 +FAST16 0.114887 10.701698 8 1106 +FAST17 10.207121 10.650652 8 1106 +FAST17 0.135424 10.650652 8 1106 +FAST18 11.463120 10.499142 8 1826 +FAST18 0.154287 10.499142 8 1826 +FAST19 12.143020 10.527140 8 1826 +FAST19 0.158889 10.527140 8 1826 +FAST20 12.510857 10.494710 8 1826 +FAST20 0.171334 10.494710 8 1826 +FAST21 13.201432 10.503488 8 1778 +FAST21 0.192867 10.503488 8 1778 +FAST22 13.754560 10.509284 8 1826 +FAST22 0.206276 10.509284 8 1826 +FAST23 14.708633 10.509284 8 1826 +FAST23 0.221751 10.509284 8 1826 +FAST24 15.134848 10.512369 8 1826 +FAST24 0.234242 10.512369 8 1826 hg-commands: -NODICT 0.000007 2.425291 -RANDOM 0.083477 3.489515 -LEGACY 0.941867 3.911896 -COVER 67.314295 4.131136 8 386 -COVER 2.757895 4.131136 8 386 -FAST15 13.466983 3.920128 6 1106 -FAST15 0.162656 3.920128 6 1106 -FAST16 12.618110 4.032422 8 674 -FAST16 0.159073 4.032422 8 674 -FAST17 12.883772 4.063581 8 1490 -FAST17 0.183131 4.063581 8 1490 -FAST18 13.904432 4.085034 8 290 -FAST18 0.161078 4.085034 8 290 -FAST19 13.762269 4.097054 8 578 -FAST19 0.179906 4.097054 8 578 -FAST20 15.303927 4.101575 8 434 -FAST20 0.213146 4.101575 8 434 -FAST21 19.619482 4.104879 8 530 -FAST21 0.289158 4.104879 8 530 -FAST22 23.187937 4.102448 8 530 -FAST22 0.335220 4.102448 8 530 -FAST23 24.946655 4.095162 8 914 -FAST23 0.396927 4.095162 8 914 -FAST24 27.634065 4.114624 8 722 -FAST24 0.434278 4.114624 8 722 +NODICT 0.000004 2.425291 +RANDOM 0.055073 3.490331 +LEGACY 0.927414 3.911682 +COVER 72.749028 4.132653 8 386 +COVER 3.391066 4.132653 8 386 +FAST15 10.910989 3.920720 6 1106 +FAST15 0.130480 3.920720 6 1106 +FAST16 10.565224 4.033306 8 674 +FAST16 0.146228 4.033306 8 674 +FAST17 11.394137 4.064132 8 1490 +FAST17 0.175567 4.064132 8 1490 +FAST18 11.040248 4.086714 8 290 +FAST18 0.132692 4.086714 8 290 +FAST19 11.335856 4.097947 8 578 +FAST19 0.181441 4.097947 8 578 +FAST20 14.166272 4.102851 8 434 +FAST20 0.203632 4.102851 8 434 +FAST21 15.848896 4.105350 8 530 +FAST21 0.269518 4.105350 8 530 +FAST22 15.570995 4.104100 8 530 +FAST22 0.238512 4.104100 8 530 +FAST23 17.437566 4.098110 8 914 +FAST23 0.270788 4.098110 8 914 +FAST24 18.836604 4.117367 8 722 +FAST24 0.323618 4.117367 8 722 hg-changelog: -NODICT 0.000027 1.377613 -RANDOM 0.676272 2.096785 -LEGACY 2.871887 2.058273 -COVER 226.371004 2.188654 8 98 -COVER 5.359820 2.188654 8 98 -FAST15 66.776425 2.130548 6 386 -FAST15 0.796836 2.130548 6 386 -FAST16 64.405113 2.144136 8 194 -FAST16 0.778969 2.144136 8 194 -FAST17 65.062292 2.155745 8 98 -FAST17 0.822089 2.155745 8 98 -FAST18 65.819104 2.172062 6 98 -FAST18 0.804247 2.172062 6 98 -FAST19 66.184016 2.179446 6 98 -FAST19 0.883526 2.179446 6 98 -FAST20 72.900924 2.187017 6 98 -FAST20 0.908220 2.187017 6 98 -FAST21 77.869945 2.183583 6 146 -FAST21 0.932666 2.183583 6 146 -FAST22 84.041413 2.182030 6 98 -FAST22 1.092310 2.182030 6 98 -FAST23 89.539265 2.185291 8 98 -FAST23 1.294779 2.185291 8 98 -FAST24 97.193482 2.184939 6 98 -FAST24 1.270493 2.184939 6 98 +NODICT 0.000006 1.377613 +RANDOM 0.253393 2.097487 +LEGACY 2.410568 2.058907 +COVER 203.550681 2.189685 8 98 +COVER 7.381697 2.189685 8 98 +FAST15 45.960609 2.130794 6 386 +FAST15 0.512057 2.130794 6 386 +FAST16 44.594817 2.144845 8 194 +FAST16 0.601258 2.144845 8 194 +FAST17 45.852992 2.156099 8 242 +FAST17 0.500844 2.156099 8 242 +FAST18 46.624930 2.172439 6 98 +FAST18 0.680501 2.172439 6 98 +FAST19 47.754905 2.180321 6 98 +FAST19 0.606180 2.180321 6 98 +FAST20 56.733632 2.187431 6 98 +FAST20 0.710149 2.187431 6 98 +FAST21 59.723173 2.184185 6 146 +FAST21 0.875562 2.184185 6 146 +FAST22 66.570788 2.182830 6 98 +FAST22 1.061013 2.182830 6 98 +FAST23 73.817645 2.186399 8 98 +FAST23 0.838496 2.186399 8 98 +FAST24 78.059933 2.185608 6 98 +FAST24 0.843158 2.185608 6 98 hg-manifest: -NODICT 0.000004 1.866385 -RANDOM 0.969045 2.309485 -LEGACY 8.849052 2.506775 -COVER 905.855524 2.582597 8 434 -COVER 34.951973 2.582597 8 434 -FAST15 154.816926 2.391764 6 1826 -FAST15 1.932845 2.391764 6 1826 -FAST16 142.197120 2.480738 6 1922 -FAST16 1.759330 2.480738 6 1922 -FAST17 147.276099 2.548313 6 1682 -FAST17 1.819175 2.548313 6 1682 -FAST18 164.543366 2.567448 6 386 -FAST18 2.728845 2.567448 6 386 -FAST19 195.670852 2.581170 8 338 -FAST19 2.439487 2.581170 8 338 -FAST20 195.716408 2.587062 6 194 -FAST20 2.056303 2.587062 6 194 -FAST21 211.483191 2.590136 6 242 -FAST21 2.983587 2.590136 6 242 -FAST22 239.562966 2.591033 6 194 -FAST22 3.355746 2.591033 6 194 -FAST23 264.547195 2.590403 8 434 -FAST23 3.667851 2.590403 8 434 -FAST24 296.258379 2.591723 6 290 -FAST24 3.858688 2.591723 6 290 +NODICT 0.000005 1.866385 +RANDOM 0.735840 2.309436 +LEGACY 9.322081 2.506977 +COVER 885.961515 2.582528 8 434 +COVER 32.678552 2.582528 8 434 +FAST15 114.414413 2.392920 6 1826 +FAST15 1.412690 2.392920 6 1826 +FAST16 113.869718 2.480762 6 1922 +FAST16 1.539424 2.480762 6 1922 +FAST17 113.333636 2.548285 6 1682 +FAST17 1.473196 2.548285 6 1682 +FAST18 111.717871 2.567634 6 386 +FAST18 1.421200 2.567634 6 386 +FAST19 112.428344 2.581653 8 338 +FAST19 1.412185 2.581653 8 338 +FAST20 128.897480 2.586881 8 194 +FAST20 1.586570 2.586881 8 194 +FAST21 168.465684 2.590051 6 242 +FAST21 2.190732 2.590051 6 242 +FAST22 202.320435 2.591376 6 194 +FAST22 2.667877 2.591376 6 194 +FAST23 228.952201 2.591131 8 434 +FAST23 3.315501 2.591131 8 434 +FAST24 327.320020 2.591548 6 290 +FAST24 5.048348 2.591548 6 290 diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 2024e0bbb..09b558fea 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -581,7 +581,7 @@ static void ZDICT_fillNoise(void* buffer, size_t length) typedef struct { - ZSTD_CCtx* ref; /* contains reference to dictionary */ + ZSTD_CDict* ref; /* contains reference to dictionary */ ZSTD_CCtx* zc; /* working context */ void* workPlace; /* must be ZSTD_BLOCKSIZE_MAX allocated */ } EStats_ress_t; @@ -597,8 +597,9 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params, size_t cSize; if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */ - { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref, 0); - if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; } + { size_t const errorCode = ZSTD_compressBegin_usingCDict(esr.zc, esr.ref); + if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_compressBegin_usingCDict failed \n"); return; } + } cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize); if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (U32)srcSize); return; } @@ -708,14 +709,6 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, /* init */ DEBUGLOG(4, "ZDICT_analyzeEntropy"); - esr.ref = ZSTD_createCCtx(); - esr.zc = ZSTD_createCCtx(); - esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX); - if (!esr.ref || !esr.zc || !esr.workPlace) { - eSize = ERROR(memory_allocation); - DISPLAYLEVEL(1, "Not enough memory \n"); - goto _cleanup; - } if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionaryCreation_failed); goto _cleanup; } /* too large dictionary */ for (u=0; u<256; u++) countLit[u] = 1; /* any character must be described */ for (u=0; u<=offcodeMax; u++) offcodeCount[u] = 1; @@ -726,12 +719,15 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, memset(bestRepOffset, 0, sizeof(bestRepOffset)); if (compressionLevel==0) compressionLevel = g_compressionLevel_default; params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize); - { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0); - if (ZSTD_isError(beginResult)) { - DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced() failed : %s \n", ZSTD_getErrorName(beginResult)); - eSize = ERROR(GENERIC); - goto _cleanup; - } } + + esr.ref = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, params.cParams, ZSTD_defaultCMem); + esr.zc = ZSTD_createCCtx(); + esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX); + if (!esr.ref || !esr.zc || !esr.workPlace) { + eSize = ERROR(memory_allocation); + DISPLAYLEVEL(1, "Not enough memory \n"); + goto _cleanup; + } /* collect stats on all samples */ for (u=0; u Date: Tue, 31 Jul 2018 13:58:54 -0700 Subject: [PATCH 09/14] Refactoring --- lib/dictBuilder/zdict.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 09b558fea..b8a51789a 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -293,7 +293,7 @@ static dictItem ZDICT_analyzePos( refinedEnd = refinedStart + selectedCount; } - /* evaluate gain based on new ref */ + /* evaluate gain based on new dict */ start = refinedStart; pos = suffix[refinedStart]; end = start; @@ -341,7 +341,7 @@ static dictItem ZDICT_analyzePos( for (i=MINMATCHLENGTH; i<=(int)maxLength; i++) savings[i] = savings[i-1] + (lengthList[i] * (i-3)); - DISPLAYLEVEL(4, "Selected ref at position %u, of length %u : saves %u (ratio: %.2f) \n", + DISPLAYLEVEL(4, "Selected dict at position %u, of length %u : saves %u (ratio: %.2f) \n", (U32)pos, (U32)maxLength, savings[maxLength], (double)savings[maxLength] / maxLength); solution.pos = (U32)pos; @@ -581,7 +581,7 @@ static void ZDICT_fillNoise(void* buffer, size_t length) typedef struct { - ZSTD_CDict* ref; /* contains reference to dictionary */ + ZSTD_CDict* dict; /* dictionary */ ZSTD_CCtx* zc; /* working context */ void* workPlace; /* must be ZSTD_BLOCKSIZE_MAX allocated */ } EStats_ress_t; @@ -597,7 +597,7 @@ static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params, size_t cSize; if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */ - { size_t const errorCode = ZSTD_compressBegin_usingCDict(esr.zc, esr.ref); + { size_t const errorCode = ZSTD_compressBegin_usingCDict(esr.zc, esr.dict); if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_compressBegin_usingCDict failed \n"); return; } } @@ -720,10 +720,10 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, if (compressionLevel==0) compressionLevel = g_compressionLevel_default; params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize); - esr.ref = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, params.cParams, ZSTD_defaultCMem); + esr.dict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, params.cParams, ZSTD_defaultCMem); esr.zc = ZSTD_createCCtx(); esr.workPlace = malloc(ZSTD_BLOCKSIZE_MAX); - if (!esr.ref || !esr.zc || !esr.workPlace) { + if (!esr.dict || !esr.zc || !esr.workPlace) { eSize = ERROR(memory_allocation); DISPLAYLEVEL(1, "Not enough memory \n"); goto _cleanup; @@ -852,7 +852,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, eSize += 12; _cleanup: - ZSTD_freeCDict(esr.ref); + ZSTD_freeCDict(esr.dict); ZSTD_freeCCtx(esr.zc); free(esr.workPlace); From 0acb0abd1e470b8e36ebde1498d79fc6a1990cc5 Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Wed, 1 Aug 2018 11:06:16 -0700 Subject: [PATCH 10/14] Add non-optimize FASTCOVER (#1260) * Add non-optimize FASTCOVER * Minor fix * Pass param as value instead of pointer --- .../benchmarkDictBuilder/README.md | 200 +++++++++--------- .../benchmarkDictBuilder/benchmark.c | 22 +- .../fastCover/README.md | 4 +- .../fastCover/fastCover.c | 55 ++++- .../fastCover/fastCover.h | 24 ++- .../fastCover/main.c | 14 +- .../fastCover/test.sh | 8 +- 7 files changed, 201 insertions(+), 126 deletions(-) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index a18311973..559776e2b 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -18,109 +18,109 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" - Fourth column is chosen d and fifth column is chosen k github: -NODICT 0.000005 2.999642 -RANDOM 0.036114 8.791189 -LEGACY 1.111024 8.173529 -COVER 57.856477 10.652243 8 1298 -COVER 5.769965 10.652243 8 1298 -FAST15 9.965877 10.555630 8 1874 -FAST15 0.140285 10.555630 8 1874 -FAST16 10.337194 10.701698 8 1106 -FAST16 0.114887 10.701698 8 1106 -FAST17 10.207121 10.650652 8 1106 -FAST17 0.135424 10.650652 8 1106 -FAST18 11.463120 10.499142 8 1826 -FAST18 0.154287 10.499142 8 1826 -FAST19 12.143020 10.527140 8 1826 -FAST19 0.158889 10.527140 8 1826 -FAST20 12.510857 10.494710 8 1826 -FAST20 0.171334 10.494710 8 1826 -FAST21 13.201432 10.503488 8 1778 -FAST21 0.192867 10.503488 8 1778 -FAST22 13.754560 10.509284 8 1826 -FAST22 0.206276 10.509284 8 1826 -FAST23 14.708633 10.509284 8 1826 -FAST23 0.221751 10.509284 8 1826 -FAST24 15.134848 10.512369 8 1826 -FAST24 0.234242 10.512369 8 1826 +NODICT 0.000025 2.999642 +RANDOM 0.030101 8.791189 +LEGACY 0.913108 8.173529 +COVER 59.234160 10.652243 8 1298 +COVER 6.258459 10.652243 8 1298 +FAST15 9.959246 10.555630 8 1874 +FAST15 0.077719 10.555630 8 1874 +FAST16 10.028343 10.701698 8 1106 +FAST16 0.078117 10.701698 8 1106 +FAST17 10.567355 10.650652 8 1106 +FAST17 0.124833 10.650652 8 1106 +FAST18 11.795287 10.499142 8 1826 +FAST18 0.086992 10.499142 8 1826 +FAST19 13.132451 10.527140 8 1826 +FAST19 0.134716 10.527140 8 1826 +FAST20 14.366314 10.494710 8 1826 +FAST20 0.128844 10.494710 8 1826 +FAST21 14.941238 10.503488 8 1778 +FAST21 0.134975 10.503488 8 1778 +FAST22 15.146226 10.509284 8 1826 +FAST22 0.146918 10.509284 8 1826 +FAST23 16.260552 10.509284 8 1826 +FAST23 0.158494 10.509284 8 1826 +FAST24 16.806037 10.512369 8 1826 +FAST24 0.190464 10.512369 8 1826 hg-commands: -NODICT 0.000004 2.425291 -RANDOM 0.055073 3.490331 -LEGACY 0.927414 3.911682 -COVER 72.749028 4.132653 8 386 -COVER 3.391066 4.132653 8 386 -FAST15 10.910989 3.920720 6 1106 -FAST15 0.130480 3.920720 6 1106 -FAST16 10.565224 4.033306 8 674 -FAST16 0.146228 4.033306 8 674 -FAST17 11.394137 4.064132 8 1490 -FAST17 0.175567 4.064132 8 1490 -FAST18 11.040248 4.086714 8 290 -FAST18 0.132692 4.086714 8 290 -FAST19 11.335856 4.097947 8 578 -FAST19 0.181441 4.097947 8 578 -FAST20 14.166272 4.102851 8 434 -FAST20 0.203632 4.102851 8 434 -FAST21 15.848896 4.105350 8 530 -FAST21 0.269518 4.105350 8 530 -FAST22 15.570995 4.104100 8 530 -FAST22 0.238512 4.104100 8 530 -FAST23 17.437566 4.098110 8 914 -FAST23 0.270788 4.098110 8 914 -FAST24 18.836604 4.117367 8 722 -FAST24 0.323618 4.117367 8 722 +NODICT 0.000026 2.425291 +RANDOM 0.046270 3.490331 +LEGACY 0.847904 3.911682 +COVER 71.691804 4.132653 8 386 +COVER 3.187085 4.132653 8 386 +FAST15 11.593687 3.920720 6 1106 +FAST15 0.082431 3.920720 6 1106 +FAST16 11.775958 4.033306 8 674 +FAST16 0.092587 4.033306 8 674 +FAST17 11.965064 4.064132 8 1490 +FAST17 0.106382 4.064132 8 1490 +FAST18 11.438197 4.086714 8 290 +FAST18 0.097293 4.086714 8 290 +FAST19 12.292512 4.097947 8 578 +FAST19 0.104406 4.097947 8 578 +FAST20 13.857857 4.102851 8 434 +FAST20 0.139467 4.102851 8 434 +FAST21 14.599613 4.105350 8 530 +FAST21 0.189416 4.105350 8 530 +FAST22 15.966109 4.104100 8 530 +FAST22 0.183817 4.104100 8 530 +FAST23 18.033645 4.098110 8 914 +FAST23 0.246641 4.098110 8 914 +FAST24 22.992891 4.117367 8 722 +FAST24 0.285994 4.117367 8 722 hg-changelog: -NODICT 0.000006 1.377613 -RANDOM 0.253393 2.097487 -LEGACY 2.410568 2.058907 -COVER 203.550681 2.189685 8 98 -COVER 7.381697 2.189685 8 98 -FAST15 45.960609 2.130794 6 386 -FAST15 0.512057 2.130794 6 386 -FAST16 44.594817 2.144845 8 194 -FAST16 0.601258 2.144845 8 194 -FAST17 45.852992 2.156099 8 242 -FAST17 0.500844 2.156099 8 242 -FAST18 46.624930 2.172439 6 98 -FAST18 0.680501 2.172439 6 98 -FAST19 47.754905 2.180321 6 98 -FAST19 0.606180 2.180321 6 98 -FAST20 56.733632 2.187431 6 98 -FAST20 0.710149 2.187431 6 98 -FAST21 59.723173 2.184185 6 146 -FAST21 0.875562 2.184185 6 146 -FAST22 66.570788 2.182830 6 98 -FAST22 1.061013 2.182830 6 98 -FAST23 73.817645 2.186399 8 98 -FAST23 0.838496 2.186399 8 98 -FAST24 78.059933 2.185608 6 98 -FAST24 0.843158 2.185608 6 98 +NODICT 0.000007 1.377613 +RANDOM 0.297345 2.097487 +LEGACY 2.633992 2.058907 +COVER 219.179786 2.189685 8 98 +COVER 6.620852 2.189685 8 98 +FAST15 47.635082 2.130794 6 386 +FAST15 0.321297 2.130794 6 386 +FAST16 43.837676 2.144845 8 194 +FAST16 0.312640 2.144845 8 194 +FAST17 49.349017 2.156099 8 242 +FAST17 0.348459 2.156099 8 242 +FAST18 51.153784 2.172439 6 98 +FAST18 0.353106 2.172439 6 98 +FAST19 52.627045 2.180321 6 98 +FAST19 0.390612 2.180321 6 98 +FAST20 63.748782 2.187431 6 98 +FAST20 0.489544 2.187431 6 98 +FAST21 68.709198 2.184185 6 146 +FAST21 0.530852 2.184185 6 146 +FAST22 68.491639 2.182830 6 98 +FAST22 0.645699 2.182830 6 98 +FAST23 72.558688 2.186399 8 98 +FAST23 0.593539 2.186399 8 98 +FAST24 76.137195 2.185608 6 98 +FAST24 0.680132 2.185608 6 98 hg-manifest: -NODICT 0.000005 1.866385 -RANDOM 0.735840 2.309436 -LEGACY 9.322081 2.506977 -COVER 885.961515 2.582528 8 434 -COVER 32.678552 2.582528 8 434 -FAST15 114.414413 2.392920 6 1826 -FAST15 1.412690 2.392920 6 1826 -FAST16 113.869718 2.480762 6 1922 -FAST16 1.539424 2.480762 6 1922 -FAST17 113.333636 2.548285 6 1682 -FAST17 1.473196 2.548285 6 1682 -FAST18 111.717871 2.567634 6 386 -FAST18 1.421200 2.567634 6 386 -FAST19 112.428344 2.581653 8 338 -FAST19 1.412185 2.581653 8 338 -FAST20 128.897480 2.586881 8 194 -FAST20 1.586570 2.586881 8 194 -FAST21 168.465684 2.590051 6 242 -FAST21 2.190732 2.590051 6 242 -FAST22 202.320435 2.591376 6 194 -FAST22 2.667877 2.591376 6 194 -FAST23 228.952201 2.591131 8 434 -FAST23 3.315501 2.591131 8 434 -FAST24 327.320020 2.591548 6 290 -FAST24 5.048348 2.591548 6 290 +NODICT 0.000026 1.866385 +RANDOM 0.784554 2.309436 +LEGACY 10.193714 2.506977 +COVER 988.206583 2.582528 8 434 +COVER 39.726199 2.582528 8 434 +FAST15 168.388819 2.392920 6 1826 +FAST15 1.272178 2.392920 6 1826 +FAST16 161.822607 2.480762 6 1922 +FAST16 1.164908 2.480762 6 1922 +FAST17 157.688544 2.548285 6 1682 +FAST17 1.222439 2.548285 6 1682 +FAST18 154.529585 2.567634 6 386 +FAST18 1.217596 2.567634 6 386 +FAST19 160.244979 2.581653 8 338 +FAST19 1.282450 2.581653 8 338 +FAST20 191.503297 2.586881 8 194 +FAST20 2.009748 2.586881 8 194 +FAST21 226.389709 2.590051 6 242 +FAST21 2.494543 2.590051 6 242 +FAST22 217.859055 2.591376 6 194 +FAST22 2.295693 2.591376 6 194 +FAST23 236.819791 2.591131 8 434 +FAST23 2.744711 2.591131 8 434 +FAST24 269.187800 2.591548 6 290 +FAST24 2.923671 2.591548 6 290 diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c index 75008a087..d92e8d5cb 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c @@ -91,14 +91,26 @@ dictInfo* createDictFromFiles(sampleInfo *info, unsigned maxDictSize, dictSize = ZDICT_trainFromBuffer_random(dictBuffer, maxDictSize, info->srcBuffer, info->samplesSizes, info->nbSamples, *randomParams); }else if(coverParams) { - dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize, info->srcBuffer, - info->samplesSizes, info->nbSamples, coverParams); + /* Run the optimize version if either k or d is not provided */ + if (!coverParams->d || !coverParams->k){ + dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, coverParams); + } else { + dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, *coverParams); + } } else if(legacyParams) { dictSize = ZDICT_trainFromBuffer_legacy(dictBuffer, maxDictSize, info->srcBuffer, info->samplesSizes, info->nbSamples, *legacyParams); } else if(fastParams) { - dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer, - info->samplesSizes, info->nbSamples, fastParams); + /* Run the optimize version if either k or d is not provided */ + if (!fastParams->d || !fastParams->k) { + dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, fastParams); + } else { + dictSize = ZDICT_trainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, *fastParams); + } } else { dictSize = 0; } @@ -403,7 +415,6 @@ int main(int argCount, const char* argv[]) goto _cleanup; } - /* for fastCover (with k and d provided) */ const int fastResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); @@ -411,7 +422,6 @@ int main(int argCount, const char* argv[]) result = 1; goto _cleanup; } - } diff --git a/contrib/experimental_dict_builders/fastCover/README.md b/contrib/experimental_dict_builders/fastCover/README.md index 66e00ee04..ad377743f 100644 --- a/contrib/experimental_dict_builders/fastCover/README.md +++ b/contrib/experimental_dict_builders/fastCover/README.md @@ -16,8 +16,8 @@ make test ###Usage: -To build a random dictionary with the provided arguments: make ARG= followed by arguments - +To build a FASTCOVER dictionary with the provided arguments: make ARG= followed by arguments +If k or d is not provided, the optimize version of FASTCOVER is run. ### Examples: make ARG="in=../../../lib/dictBuilder out=dict100 dictID=520" diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c index cf71075ab..84d841b10 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.c +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -629,6 +629,55 @@ _cleanup: } } +ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover( + void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t parameters) { + BYTE* const dict = (BYTE*)dictBuffer; + FASTCOVER_ctx_t ctx; + parameters.splitPoint = 1.0; + /* Initialize global data */ + g_displayLevel = parameters.zParams.notificationLevel; + /* Checks */ + if (!FASTCOVER_checkParameters(parameters, dictBufferCapacity)) { + DISPLAYLEVEL(1, "FASTCOVER parameters incorrect\n"); + return ERROR(GENERIC); + } + if (nbSamples == 0) { + DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n"); + return ERROR(GENERIC); + } + if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { + DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n", + ZDICT_DICTSIZE_MIN); + return ERROR(dstSize_tooSmall); + } + /* Initialize context */ + if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, + parameters.d, parameters.splitPoint, parameters.f)) { + DISPLAYLEVEL(1, "Failed to initialize context\n"); + return ERROR(GENERIC); + } + /* Build the dictionary */ + DISPLAYLEVEL(2, "Building dictionary\n"); + { + const size_t tail = FASTCOVER_buildDictionary(&ctx, ctx.freqs, dictBuffer, + dictBufferCapacity, parameters); + + const size_t dictionarySize = ZDICT_finalizeDictionary( + dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail, + samplesBuffer, samplesSizes, (unsigned)ctx.nbTrainSamples, + parameters.zParams); + if (!ZSTD_isError(dictionarySize)) { + DISPLAYLEVEL(2, "Constructed dictionary of size %u\n", + (U32)dictionarySize); + } + FASTCOVER_ctx_destroy(&ctx); + return dictionarySize; + } +} + + + ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, @@ -657,15 +706,15 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( /* Checks */ if (splitPoint <= 0 || splitPoint > 1) { - LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect splitPoint\n"); return ERROR(GENERIC); } if (kMinK < kMaxD || kMaxK < kMinK) { - LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect parameters\n"); + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect k\n"); return ERROR(GENERIC); } if (nbSamples == 0) { - DISPLAYLEVEL(1, "fast must have at least one input file\n"); + DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n"); return ERROR(GENERIC); } if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.h b/contrib/experimental_dict_builders/fastCover/fastCover.h index eca04baab..958e9f423 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.h +++ b/contrib/experimental_dict_builders/fastCover/fastCover.h @@ -12,9 +12,6 @@ #include "zdict.h" - - - typedef struct { unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ @@ -26,7 +23,6 @@ typedef struct { } ZDICT_fastCover_params_t; - /*! ZDICT_optimizeTrainFromBuffer_fastCover(): * Train a dictionary from an array of samples using a modified version of the COVER algorithm. * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, @@ -41,7 +37,21 @@ typedef struct { * or an error code, which can be tested with ZDICT_isError(). * On success `*parameters` contains the parameters selected. */ -ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( + ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( + void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t *parameters); + + +/*! ZDICT_trainFromBuffer_fastCover(): + * Train a dictionary from an array of samples using a modified version of the COVER algorithm. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * d, k, and f are required. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + */ +ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover( void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, - const size_t *samplesSizes, unsigned nbSamples, - ZDICT_fastCover_params_t *parameters); + const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t parameters); diff --git a/contrib/experimental_dict_builders/fastCover/main.c b/contrib/experimental_dict_builders/fastCover/main.c index f286b0506..df7d91812 100644 --- a/contrib/experimental_dict_builders/fastCover/main.c +++ b/contrib/experimental_dict_builders/fastCover/main.c @@ -64,8 +64,14 @@ int FASTCOVER_trainFromFiles(const char* dictFileName, sampleInfo *info, EXM_THROW(12, "not enough memory for trainFromFiles"); /* should not happen */ { size_t dictSize; - dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer, - info->samplesSizes, info->nbSamples, params); + /* Run the optimize version if either k or d is not provided */ + if (!params->d || !params->k) { + dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, params); + } else { + dictSize = ZDICT_trainFromBuffer_fastCover(dictBuffer, maxDictSize, info->srcBuffer, + info->samplesSizes, info->nbSamples, *params); + } DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", params->k, params->d, params->f, params->steps, (unsigned)(params->splitPoint*100)); if (ZDICT_isError(dictSize)) { DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ @@ -92,8 +98,8 @@ int main(int argCount, const char* argv[]) int operationResult = 0; /* Initialize arguments to default values */ - unsigned k = 200; - unsigned d = 8; + unsigned k = 0; + unsigned d = 0; unsigned f = 23; unsigned steps = 32; unsigned nbThreads = 1; diff --git a/contrib/experimental_dict_builders/fastCover/test.sh b/contrib/experimental_dict_builders/fastCover/test.sh index 91d4f4923..f86915b59 100644 --- a/contrib/experimental_dict_builders/fastCover/test.sh +++ b/contrib/experimental_dict_builders/fastCover/test.sh @@ -1,8 +1,8 @@ -echo "Building fastCover dictionary with in=../../lib/common k=200 f=20 out=dict1" -./main in=../../../lib/common k=200 f=20 out=dict1 +echo "Building fastCover dictionary with in=../../lib/common f=20 out=dict1" +./main in=../../../lib/common f=20 out=dict1 zstd -be3 -D dict1 -r ../../../lib/common -q -echo "Building fastCover dictionary with in=../../lib/common k=500 f=24 out=dict2 dictID=100 maxdict=140000" -./main in=../../../lib/common k=500 f=24 out=dict2 dictID=100 maxdict=140000 +echo "Building fastCover dictionary with in=../../lib/common k=500 d=6 f=24 out=dict2 dictID=100 maxdict=140000" +./main in=../../../lib/common k=500 d=6 f=24 out=dict2 dictID=100 maxdict=140000 zstd -be3 -D dict2 -r ../../../lib/common -q echo "Building fastCover dictionary with 2 sample sources" ./main in=../../../lib/common in=../../../lib/compress out=dict3 From dc5a67cb7b84a86bb3729ae4a85ef61e37797ac2 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 2 Aug 2018 11:12:17 -0700 Subject: [PATCH 11/14] Disallow tableLog == srcLog --- lib/compress/fse_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index 95e7c1c7e..70daae3bc 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -331,7 +331,7 @@ void FSE_freeCTable (FSE_CTable* ct) { free(ct); } /* provides the minimum logSize to safely represent a distribution */ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) { - U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1; + U32 minBitsSrc = BIT_highbit32((U32)(srcSize)) + 1; U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; assert(srcSize > 1); /* Not supported, RLE should be used instead */ From 5203f01774397f4e6265d0ab3300805f8f4bc6fb Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Aug 2018 07:54:29 -0700 Subject: [PATCH 12/14] fix : zstd cli can be built with build macro ZSTD_NOBENCH which disables bench.c module --- Makefile | 2 +- programs/zstdcli.c | 32 +++++++++++++++++--------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 89c5a418d..98c35a346 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ zlibwrapper: .PHONY: test test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror test: - MOREFLAGS="$(MOREFLAGS)" $(MAKE) -C $(PRGDIR) allVariants + MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants $(MAKE) -C $(TESTDIR) $@ .PHONY: shortest diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 36ba21156..34f73302a 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -406,7 +406,6 @@ int main(int argCount, const char* argv[]) singleThread = 0, ultra=0; double compressibility = 0.5; - BMK_advancedParams_t adv = BMK_initAdvancedParams(); unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ size_t blockSize = 0; zstd_operation_mode operation = zom_compress; @@ -434,6 +433,9 @@ int main(int argCount, const char* argv[]) ZDICT_cover_params_t coverParams = defaultCoverParams(); int cover = 1; #endif +#ifndef ZSTD_NOBENCH + BMK_advancedParams_t benchParams = BMK_initAdvancedParams(); +#endif /* init */ @@ -620,7 +622,7 @@ int main(int argCount, const char* argv[]) /* Decoding */ case 'd': #ifndef ZSTD_NOBENCH - adv.mode = BMK_decodeOnly; + benchParams.mode = BMK_decodeOnly; if (operation==zom_bench) { argument++; break; } /* benchmark decode (hidden option) */ #endif operation=zom_decompress; argument++; break; @@ -713,7 +715,7 @@ int main(int argCount, const char* argv[]) case 'p': argument++; #ifndef ZSTD_NOBENCH if ((*argument>='0') && (*argument<='9')) { - adv.additionalParam = (int)readU32FromChar(&argument); + benchParams.additionalParam = (int)readU32FromChar(&argument); } else #endif main_pause=1; @@ -826,18 +828,18 @@ int main(int argCount, const char* argv[]) /* Check if benchmark is selected */ if (operation==zom_bench) { #ifndef ZSTD_NOBENCH - adv.blockSize = blockSize; - adv.nbWorkers = nbWorkers; - adv.realTime = setRealTimePrio; - adv.nbSeconds = bench_nbSeconds; - adv.ldmFlag = ldmFlag; - adv.ldmMinMatch = g_ldmMinMatch; - adv.ldmHashLog = g_ldmHashLog; + benchParams.blockSize = blockSize; + benchParams.nbWorkers = nbWorkers; + benchParams.realTime = setRealTimePrio; + benchParams.nbSeconds = bench_nbSeconds; + benchParams.ldmFlag = ldmFlag; + benchParams.ldmMinMatch = g_ldmMinMatch; + benchParams.ldmHashLog = g_ldmHashLog; if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) { - adv.ldmBucketSizeLog = g_ldmBucketSizeLog; + benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog; } if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) { - adv.ldmHashEveryLog = g_ldmHashEveryLog; + benchParams.ldmHashEveryLog = g_ldmHashEveryLog; } if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); @@ -852,17 +854,17 @@ int main(int argCount, const char* argv[]) int c; DISPLAYLEVEL(2, "Benchmarking %s \n", filenameTable[i]); for(c = cLevel; c <= cLevelLast; c++) { - BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &adv); + BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams); } } } else { for(; cLevel <= cLevelLast; cLevel++) { - BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &adv); + BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams); } } } else { for(; cLevel <= cLevelLast; cLevel++) { - BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &adv); + BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams); } } From ca785c4b2093dcfaccf739d769eecf93fbb01232 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Aug 2018 07:59:33 -0700 Subject: [PATCH 13/14] fix .travis.yml --- .travis.yml | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index d41a6774d..ea71818e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,13 +10,10 @@ addons: matrix: include: # Ubuntu 14.04 -<<<<<<< HEAD - - env: Cmd='make gcc6install && CC=gcc-6 make -j all - && make clean && CC=gcc-6 make clean uasan-test-zstd' -======= - env: Cmd='make test' - - env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd >>>>>> 0840d02ecf74eae656e0df0d900000d9b0154cde + + - env: Cmd='make gcc6install && CC=gcc-6 make -j all + && make clean && CC=gcc-6 make clean uasan-test-zstd collect2: error: ld terminated with signal 11 [Segmentation fault], core dumped -# to be re-enabled in a few commit, as it's possible that a random code change circumvent the ld bug -# - env: Cmd='make arminstall && make aarch64fuzz' - ->>>>>>> dev - env: Cmd='make ppcinstall && make ppcfuzz' - env: Cmd='make ppcinstall && make ppc64fuzz' From 2fdab1629be5f46c778dcf9a4c0175ffff47e9ea Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Aug 2018 08:30:01 -0700 Subject: [PATCH 14/14] fix unused variable warning --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 34f73302a..d5a2216d6 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -869,7 +869,7 @@ int main(int argCount, const char* argv[]) } #else - (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; + (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility; #endif goto _end; }