From ed1a42987301ada4968c6acd86dafe56e8d10605 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 27 Jun 2018 16:57:28 -0700 Subject: [PATCH 01/92] test multi-lines travis yaml file --- .travis.yml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index b7099c24f..20c16fa90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,28 +10,32 @@ addons: matrix: include: # Ubuntu 14.04 - - env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd' + - env: Cmd='make gcc6install && CC=gcc-6 make -j all \ + && make clean && CC=gcc-6 make clean uasan-test-zstd' - env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test-zstd32' - env: Cmd='make gcc7install && CC=gcc-7 make clean uasan-test-zstd' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd' - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest' - - env: Cmd='make gcc6install libc6install && CC=gcc-6 CFLAGS=-m32 make clean uasan-fuzztest' + - env: Cmd='make gcc6install libc6install \ + && make clean && CC=gcc-6 CFLAGS=-m32 make uasan-fuzztest' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest' - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream' - - env: Cmd='make -C tests test-fuzzer-stackmode' - - - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest' - - env: Cmd='make arminstall && make armfuzz' - env: Cmd='make arminstall && make aarch64fuzz' - - env: Cmd='make ppcinstall && make ppcfuzz' - env: Cmd='make ppcinstall && make ppc64fuzz' - - env: Cmd='make -j uasanregressiontest && make clean && make -j msanregressiontest' - - env: Cmd='make lz4install && make -C tests test-lz4 test-pool && make clean && bash tests/libzstd_partial_builds.sh' + - env: Cmd='make -j uasanregressiontest \ + && make clean && make -j msanregressiontest' + + - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest \ + && make clean && make -C tests test-fuzzer-stackmode' + + - env: Cmd='make lz4install && make -C tests test-lz4 \ + && make clean && make -C tests test-pool \ + && make clean && bash tests/libzstd_partial_builds.sh' # tag-specific test - if: tag =~ ^v[0-9]\.[0-9] @@ -44,6 +48,7 @@ branches: only: - dev - master + - travisTest script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') From ab1ebd6578f9b1528382eaa948374b9eed490034 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 27 Jun 2018 19:29:18 -0700 Subject: [PATCH 02/92] removed \ at end of line --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 20c16fa90..566f3aa4e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,14 +10,14 @@ addons: matrix: include: # Ubuntu 14.04 - - env: Cmd='make gcc6install && CC=gcc-6 make -j all \ + - env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd' - env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test-zstd32' - env: Cmd='make gcc7install && CC=gcc-7 make clean uasan-test-zstd' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd' - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest' - - env: Cmd='make gcc6install libc6install \ + - env: Cmd='make gcc6install libc6install && make clean && CC=gcc-6 CFLAGS=-m32 make uasan-fuzztest' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest' - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream' @@ -27,14 +27,14 @@ matrix: - env: Cmd='make ppcinstall && make ppcfuzz' - env: Cmd='make ppcinstall && make ppc64fuzz' - - env: Cmd='make -j uasanregressiontest \ + - env: Cmd='make -j uasanregressiontest && make clean && make -j msanregressiontest' - - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest \ + - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest && make clean && make -C tests test-fuzzer-stackmode' - - env: Cmd='make lz4install && make -C tests test-lz4 \ - && make clean && make -C tests test-pool \ + - env: Cmd='make lz4install && make -C tests test-lz4 + && make clean && make -C tests test-pool && make clean && bash tests/libzstd_partial_builds.sh' # tag-specific test From 9597b438e931dcdd0cc8497bc330e9bcc495992a Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Tue, 17 Jul 2018 18:52:57 +0200 Subject: [PATCH 03/92] fix #1241 Ensure that first input position is valid for a match even during first usage of context by starting reference at 1 (avoiding the problematic 0). --- lib/compress/zstd_compress.c | 3 +++ tests/fuzzer.c | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index c66862524..8f63eb953 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1032,6 +1032,9 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, ms->hashLog3 = hashLog3; memset(&ms->window, 0, sizeof(ms->window)); + ms->window.dictLimit = 1; /* start from 1, so that 1st position is valid */ + ms->window.lowLimit = 1; /* it ensures first and later CCtx usages compress the same */ + ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */ ZSTD_invalidateMatchState(ms); /* opt parser space */ diff --git a/tests/fuzzer.c b/tests/fuzzer.c index f7bacd5ca..b9d319ea2 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -411,6 +411,26 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++); + { int i; + for (i=0; i<20; i++) + ((char*)CNBuffer)[i] = i; /* ensure no match during initial section */ + memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */ + for (i=1; i<=19; i++) { + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t size1, size2; + DISPLAYLEVEL(5, "l%i ", i); + size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 30, i); + CHECK_Z(size1); + size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 30, i); + CHECK_Z(size2); + CHECK_EQ(size1, size2); + + ZSTD_freeCCtx(cctx); + } + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++); { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); ZSTD_outBuffer out = {NULL, 0, 0}; From b8a81a988c5d81f5efdc4af9f28417f3fe71cfe8 Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Tue, 17 Jul 2018 19:02:17 +0200 Subject: [PATCH 04/92] added a test to be played on travis to check a make variable construction --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 80406064d..1fd7e6aad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ addons: matrix: include: # Ubuntu 14.04 + - env: Cmd='make test' - env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd Date: Tue, 17 Jul 2018 19:19:48 +0200 Subject: [PATCH 05/92] fix make test on Linux MOREFLAGS+= doesn't work on Linux --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0f2fcc958..89c5a418d 100644 --- a/Makefile +++ b/Makefile @@ -63,8 +63,9 @@ zlibwrapper: $(MAKE) -C $(ZWRAPDIR) test .PHONY: test +test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror test: - MOREFLAGS+="-g -DDEBUGLEVEL=1 -Werror" $(MAKE) -C $(PRGDIR) allVariants + MOREFLAGS="$(MOREFLAGS)" $(MAKE) -C $(PRGDIR) allVariants $(MAKE) -C $(TESTDIR) $@ .PHONY: shortest From 7d1bc9cc8c79f7412c44835b910945792b2238c7 Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Wed, 18 Jul 2018 16:10:23 +0200 Subject: [PATCH 06/92] fix minor conversion warning --- tests/fuzzer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index b9d319ea2..c411d4177 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -414,7 +414,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++); { int i; for (i=0; i<20; i++) - ((char*)CNBuffer)[i] = i; /* ensure no match during initial section */ + ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */ memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */ for (i=1; i<=19; i++) { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); From 9889bca530a2b52615eb1cd06260e9cd0c29e001 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Fri, 27 Jul 2018 17:30:03 -0700 Subject: [PATCH 07/92] [FSE] Fix division by zero When the primary normalization method fails, and `(1 << tableLog) == (maxSymbolValue + 1)`, and every symbol gets assigned normalized weight 1 or -1 in the first loop, then the next division can raise `SIGFPE`. --- lib/compress/fse_compress.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index 07b3ab89b..95e7c1c7e 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -394,6 +394,9 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, } ToDistribute = (1 << tableLog) - distributed; + if (ToDistribute == 0) + return 0; + if ((total / ToDistribute) > lowOne) { /* risk of rounding to zero */ lowOne = (U32)((total * 3) / (ToDistribute * 2)); From 3d4b09a5aa7592b86df77a7956ea51b70799d135 Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Mon, 30 Jul 2018 16:29:20 +0200 Subject: [PATCH 08/92] support %zu under mingw --- tests/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/Makefile b/tests/Makefile index 813380cc2..0955272ef 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -27,6 +27,9 @@ DEBUGLEVEL ?= 1 DEBUGFLAGS = -g -DDEBUGLEVEL=$(DEBUGLEVEL) CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) +ifeq ($(OS),Windows_NT) # MinGW assumed +CPPFLAGS += -D__USE_MINGW_ANSI_STDIO +endif CFLAGS ?= -O3 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ From c738a2c795380c2a75810c48f320df93fd8bd2ea Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Mon, 30 Jul 2018 16:44:20 +0200 Subject: [PATCH 09/92] ensure appveyor test fails due to formatting error to catch %zu incompatibility --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 742f61206..42eb71d64 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -181,7 +181,7 @@ - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x64" - SCRIPT: "make allzstd" + SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd" - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x86" From 3f535007e40f1e43ea1513c2ff9356114f1be024 Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Mon, 30 Jul 2018 16:56:18 +0200 Subject: [PATCH 10/92] fix %zu support under minGW and relevant test on Appveyor --- appveyor.yml | 2 +- lib/Makefile | 7 +++++-- programs/Makefile | 5 ++++- tests/Makefile | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 42eb71d64..c9f88dce6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -181,7 +181,7 @@ - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x64" - SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd" + SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2" - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x86" diff --git a/lib/Makefile b/lib/Makefile index 9cedd53b7..01689c6d5 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -19,6 +19,9 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT)) VERSION?= $(LIBVER) CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_ +ifeq ($(OS),Windows_NT) # MinGW assumed +CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting +endif CFLAGS ?= -O3 DEBUGFLAGS = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ @@ -52,11 +55,11 @@ ifeq ($(ZSTD_LIB_DECOMPRESSION), 0) endif ifneq ($(ZSTD_LIB_COMPRESSION), 0) - ZSTD_FILES += $(ZSTDCOMP_FILES) + ZSTD_FILES += $(ZSTDCOMP_FILES) endif ifneq ($(ZSTD_LIB_DECOMPRESSION), 0) - ZSTD_FILES += $(ZSTDDECOMP_FILES) + ZSTD_FILES += $(ZSTDDECOMP_FILES) endif ifneq ($(ZSTD_LIB_DEPRECATED), 0) diff --git a/programs/Makefile b/programs/Makefile index 4202764c2..912f9eff0 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -38,6 +38,9 @@ endif CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder \ -DXXH_NAMESPACE=ZSTD_ +ifeq ($(OS),Windows_NT) # MinGW assumed +CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting +endif CFLAGS ?= -O3 DEBUGFLAGS+=-Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ @@ -158,7 +161,7 @@ zstd-release: DEBUGFLAGS := zstd-release: zstd zstd32 : CPPFLAGS += $(THREAD_CPP) -zstd32 : LDFLAGS += $(THREAD_LD) +zstd32 : LDFLAGS += $(THREAD_LD) zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) zstd32 : $(ZSTDLIB_FILES) zstdcli.c fileio.c bench.c datagen.c dibio.c ifneq (,$(filter Windows%,$(OS))) diff --git a/tests/Makefile b/tests/Makefile index 0955272ef..81e685780 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -28,7 +28,7 @@ DEBUGFLAGS = -g -DDEBUGLEVEL=$(DEBUGLEVEL) CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) ifeq ($(OS),Windows_NT) # MinGW assumed -CPPFLAGS += -D__USE_MINGW_ANSI_STDIO +CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting endif CFLAGS ?= -O3 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ From e85b4c56b288a3480e10117b9e37d6744b665fe9 Mon Sep 17 00:00:00 2001 From: cyan4973 Date: Mon, 30 Jul 2018 17:08:40 +0200 Subject: [PATCH 11/92] speed up appveyor tests --- appveyor.yml | 4 ++-- build/cmake/lib/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index c9f88dce6..2b674ce3c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -185,11 +185,11 @@ - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x86" - SCRIPT: "make allzstd" + SCRIPT: "CFLAGS=-Werror make -j allzstd" - COMPILER: "clang" HOST: "mingw" PLATFORM: "x64" - SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allzstd" + SCRIPT: "CFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make -j allzstd" - COMPILER: "visual" HOST: "visual" diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index c4c2f81e6..e84e06301 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -14,7 +14,7 @@ OPTION(ZSTD_BUILD_STATIC "BUILD STATIC LIBRARIES" ON) OPTION(ZSTD_BUILD_SHARED "BUILD SHARED LIBRARIES" ON) IF(NOT ZSTD_BUILD_SHARED AND NOT ZSTD_BUILD_STATIC) - MESSAGE(SEND_ERROR "You need to build at least one flavor of libstd") + MESSAGE(SEND_ERROR "You need to build at least one flavor of libzstd") ENDIF() # Define library directory, where sources and header files are located From b9faaa1dc3b32b0e6ed248e860db54b4052f0aab Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 30 Jul 2018 12:57:11 -0700 Subject: [PATCH 12/92] [FSE] Add division by zero test --- tests/fuzzer.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index f7bacd5ca..8856a504a 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -27,6 +27,7 @@ #include /* strcmp */ #include #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ +#include "fse.h" #include "zstd.h" /* ZSTD_VERSION_STRING */ #include "zstd_errors.h" /* ZSTD_getErrorCode */ #include "zstdmt_compress.h" @@ -1423,6 +1424,24 @@ static int basicUnitTests(U32 seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++); + { + short norm[32]; + unsigned count[32]; + unsigned const tableLog = 5; + size_t const nbSeq = 32; + unsigned const maxSymbolValue = 31; + size_t i; + + for (i = 0; i < 32; ++i) + count[i] = 1; + /* Calling FSE_normalizeCount() on a uniform distribution should not + * cause a division by zero. + */ + FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue); + } + DISPLAYLEVEL(3, "OK \n"); + _end: free(CNBuffer); free(compressedBuffer); From dc5a67cb7b84a86bb3729ae4a85ef61e37797ac2 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 2 Aug 2018 11:12:17 -0700 Subject: [PATCH 13/92] Disallow tableLog == srcLog --- lib/compress/fse_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index 95e7c1c7e..70daae3bc 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -331,7 +331,7 @@ void FSE_freeCTable (FSE_CTable* ct) { free(ct); } /* provides the minimum logSize to safely represent a distribution */ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) { - U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1; + U32 minBitsSrc = BIT_highbit32((U32)(srcSize)) + 1; U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; assert(srcSize > 1); /* Not supported, RLE should be used instead */ From 5203f01774397f4e6265d0ab3300805f8f4bc6fb Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Aug 2018 07:54:29 -0700 Subject: [PATCH 14/92] fix : zstd cli can be built with build macro ZSTD_NOBENCH which disables bench.c module --- Makefile | 2 +- programs/zstdcli.c | 32 +++++++++++++++++--------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 89c5a418d..98c35a346 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ zlibwrapper: .PHONY: test test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror test: - MOREFLAGS="$(MOREFLAGS)" $(MAKE) -C $(PRGDIR) allVariants + MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants $(MAKE) -C $(TESTDIR) $@ .PHONY: shortest diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 36ba21156..34f73302a 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -406,7 +406,6 @@ int main(int argCount, const char* argv[]) singleThread = 0, ultra=0; double compressibility = 0.5; - BMK_advancedParams_t adv = BMK_initAdvancedParams(); unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ size_t blockSize = 0; zstd_operation_mode operation = zom_compress; @@ -434,6 +433,9 @@ int main(int argCount, const char* argv[]) ZDICT_cover_params_t coverParams = defaultCoverParams(); int cover = 1; #endif +#ifndef ZSTD_NOBENCH + BMK_advancedParams_t benchParams = BMK_initAdvancedParams(); +#endif /* init */ @@ -620,7 +622,7 @@ int main(int argCount, const char* argv[]) /* Decoding */ case 'd': #ifndef ZSTD_NOBENCH - adv.mode = BMK_decodeOnly; + benchParams.mode = BMK_decodeOnly; if (operation==zom_bench) { argument++; break; } /* benchmark decode (hidden option) */ #endif operation=zom_decompress; argument++; break; @@ -713,7 +715,7 @@ int main(int argCount, const char* argv[]) case 'p': argument++; #ifndef ZSTD_NOBENCH if ((*argument>='0') && (*argument<='9')) { - adv.additionalParam = (int)readU32FromChar(&argument); + benchParams.additionalParam = (int)readU32FromChar(&argument); } else #endif main_pause=1; @@ -826,18 +828,18 @@ int main(int argCount, const char* argv[]) /* Check if benchmark is selected */ if (operation==zom_bench) { #ifndef ZSTD_NOBENCH - adv.blockSize = blockSize; - adv.nbWorkers = nbWorkers; - adv.realTime = setRealTimePrio; - adv.nbSeconds = bench_nbSeconds; - adv.ldmFlag = ldmFlag; - adv.ldmMinMatch = g_ldmMinMatch; - adv.ldmHashLog = g_ldmHashLog; + benchParams.blockSize = blockSize; + benchParams.nbWorkers = nbWorkers; + benchParams.realTime = setRealTimePrio; + benchParams.nbSeconds = bench_nbSeconds; + benchParams.ldmFlag = ldmFlag; + benchParams.ldmMinMatch = g_ldmMinMatch; + benchParams.ldmHashLog = g_ldmHashLog; if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) { - adv.ldmBucketSizeLog = g_ldmBucketSizeLog; + benchParams.ldmBucketSizeLog = g_ldmBucketSizeLog; } if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) { - adv.ldmHashEveryLog = g_ldmHashEveryLog; + benchParams.ldmHashEveryLog = g_ldmHashEveryLog; } if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); @@ -852,17 +854,17 @@ int main(int argCount, const char* argv[]) int c; DISPLAYLEVEL(2, "Benchmarking %s \n", filenameTable[i]); for(c = cLevel; c <= cLevelLast; c++) { - BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &adv); + BMK_benchFilesAdvanced(&filenameTable[i], 1, dictFileName, c, &compressionParams, g_displayLevel, &benchParams); } } } else { for(; cLevel <= cLevelLast; cLevel++) { - BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &adv); + BMK_benchFilesAdvanced(filenameTable, filenameIdx, dictFileName, cLevel, &compressionParams, g_displayLevel, &benchParams); } } } else { for(; cLevel <= cLevelLast; cLevel++) { - BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &adv); + BMK_syntheticTest(cLevel, compressibility, &compressionParams, g_displayLevel, &benchParams); } } From ca785c4b2093dcfaccf739d769eecf93fbb01232 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Aug 2018 07:59:33 -0700 Subject: [PATCH 15/92] fix .travis.yml --- .travis.yml | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index d41a6774d..ea71818e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,13 +10,10 @@ addons: matrix: include: # Ubuntu 14.04 -<<<<<<< HEAD - - env: Cmd='make gcc6install && CC=gcc-6 make -j all - && make clean && CC=gcc-6 make clean uasan-test-zstd' -======= - env: Cmd='make test' - - env: Cmd='make gcc6install && CC=gcc-6 make -j all && make clean && CC=gcc-6 make clean uasan-test-zstd >>>>>> 0840d02ecf74eae656e0df0d900000d9b0154cde + + - env: Cmd='make gcc6install && CC=gcc-6 make -j all + && make clean && CC=gcc-6 make clean uasan-test-zstd collect2: error: ld terminated with signal 11 [Segmentation fault], core dumped -# to be re-enabled in a few commit, as it's possible that a random code change circumvent the ld bug -# - env: Cmd='make arminstall && make aarch64fuzz' - ->>>>>>> dev - env: Cmd='make ppcinstall && make ppcfuzz' - env: Cmd='make ppcinstall && make ppc64fuzz' From 2fdab1629be5f46c778dcf9a4c0175ffff47e9ea Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 3 Aug 2018 08:30:01 -0700 Subject: [PATCH 16/92] fix unused variable warning --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 34f73302a..d5a2216d6 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -869,7 +869,7 @@ int main(int argCount, const char* argv[]) } #else - (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; + (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles; (void)compressibility; #endif goto _end; } From 2ca7c69167e17006f867ab550408b6e0487a81e7 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Tue, 7 Aug 2018 17:05:05 -0700 Subject: [PATCH 17/92] Fix CDict Attachment to Handle CDicts with Non-Zero Starts CDicts were previously guaranteed to be generated with `lowLimit=dictLimit=0`. This is no longer true, and so the old length and index calculations are no longer valid. This diff fixes them to handle non-zero start indices in CDicts. --- lib/compress/zstd_compress.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 3ebfd019f..ed3aab871 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1284,8 +1284,9 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, } if (attachDict) { - const U32 cdictLen = (U32)( cdict->matchState.window.nextSrc + const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc - cdict->matchState.window.base); + const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; if (cdictLen == 0) { /* don't even attach dictionaries with no contents */ DEBUGLOG(4, "skipping attaching empty dictionary"); @@ -1295,9 +1296,9 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, /* prep working match state so dict matches never have negative indices * when they are translated to the working context's index space. */ - if (cctx->blockState.matchState.window.dictLimit < cdictLen) { + if (cctx->blockState.matchState.window.dictLimit < cdictEnd) { cctx->blockState.matchState.window.nextSrc = - cctx->blockState.matchState.window.base + cdictLen; + cctx->blockState.matchState.window.base + cdictEnd; ZSTD_window_clear(&cctx->blockState.matchState.window); } cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit; From 93750a54a9df528c9692380b80b4ed4755f4f05a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 8 Aug 2018 10:22:19 -0700 Subject: [PATCH 18/92] try to improve some travis test speed --- .travis.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index ea71818e9..a3b514c8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,10 +12,11 @@ matrix: # Ubuntu 14.04 - env: Cmd='make test' - - env: Cmd='make gcc6install && CC=gcc-6 make -j all - && make clean && CC=gcc-6 make clean uasan-test-zstd Date: Wed, 8 Aug 2018 10:26:54 -0700 Subject: [PATCH 19/92] =?UTF-8?q?try=20adding=20gcc-8=C2=A0compilation=20t?= =?UTF-8?q?ests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 1 + Makefile | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index a3b514c8e..257de66e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ matrix: - env: Cmd='make gcc6install libc6install && make clean && CC=gcc-6 make -j uasan-test-zstd32' - env: Cmd='make gcc7install && make clean && CC=gcc-7 make -j uasan-test-zstd' + - env: Cmd='make gcc8install && CC=gcc-8 CFLAGS=-Werror make -j all' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd' - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest' diff --git a/Makefile b/Makefile index 98c35a346..f67791275 100644 --- a/Makefile +++ b/Makefile @@ -287,6 +287,9 @@ gcc6install: apt-add-repo gcc7install: apt-add-repo APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-7 gcc-7-multilib" $(MAKE) apt-install +gcc8install: apt-add-repo + APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-8 gcc-8-multilib" $(MAKE) apt-install + gpp6install: apt-add-repo APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install From 9deef2c41bc092fa83c8c15d6bc9402b67325127 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 8 Aug 2018 10:28:22 -0700 Subject: [PATCH 20/92] Add an Auto-Gen'ed CircleCI 2.0 Config Built via the cci-config-generator.sh script. --- .circleci/config.yml | 105 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..4780c3d68 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,105 @@ +# This configuration was automatically generated from a CircleCI 1.0 config. +# It should include any build commands you had along with commands that CircleCI +# inferred from your project structure. We strongly recommend you read all the +# comments in this file to understand the structure of CircleCI 2.0, as the idiom +# for configuration has changed substantially in 2.0 to allow arbitrary jobs rather +# than the prescribed lifecycle of 1.0. In general, we recommend using this generated +# configuration as a reference rather than using it in production, though in most +# cases it should duplicate the execution of your original 1.0 config. +version: 2 +jobs: + build: + working_directory: ~/felixhandte/zstd + parallelism: 1 + shell: /bin/bash --login + # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did. + # If any of these refer to each other, rewrite them so that they don't or see https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables . + environment: + CIRCLE_ARTIFACTS: /tmp/circleci-artifacts + CIRCLE_TEST_REPORTS: /tmp/circleci-test-results + # In CircleCI 1.0 we used a pre-configured image with a large number of languages and other packages. + # In CircleCI 2.0 you can now specify your own image, or use one of our pre-configured images. + # The following configuration line tells CircleCI to use the specified docker image as the runtime environment for you job. + # We have selected a pre-built image that mirrors the build environment we use on + # the 1.0 platform, but we recommend you choose an image more tailored to the needs + # of each job. For more information on choosing an image (or alternatively using a + # VM instead of a container) see https://circleci.com/docs/2.0/executor-types/ + # To see the list of pre-built images that CircleCI provides for most common languages see + # https://circleci.com/docs/2.0/circleci-images/ + docker: + - image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37 + command: /sbin/init + steps: + # Machine Setup + # If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each + # The following `checkout` command checks out your code to your working directory. In 1.0 we did this implicitly. In 2.0 you can choose where in the course of a job your code should be checked out. + - checkout + # Prepare for artifact and test results collection equivalent to how it was done on 1.0. + # In many cases you can simplify this from what is generated here. + # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/' + - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS + # Dependencies + # This would typically go in either a build or a build-and-test job when using workflows + # Restore the dependency cache + - restore_cache: + keys: + # This branch if available + - v1-dep-{{ .Branch }}- + # Default branch if not + - v1-dep-dev- + # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly + - v1-dep- + # This is based on your 1.0 configuration file or project settings + - run: sudo dpkg --add-architecture i386 + - run: sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update + - run: sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross + # Save dependency cache + - save_cache: + key: v1-dep-{{ .Branch }}-{{ epoch }} + paths: + # This is a broad list of cache paths to include many possible development environments + # You can probably delete some of these entries + - vendor/bundle + - ~/virtualenvs + - ~/.m2 + - ~/.ivy2 + - ~/.bundle + - ~/.go_workspace + - ~/.gradle + - ~/.cache/bower + # Test + # This would typically be a build job when using workflows, possibly combined with build + # This is based on your 1.0 configuration file or project settings + - run: | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then cc -v; CFLAGS="-O0 -Werror" make all && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build && make clean; fi + - run: | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c99build && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu99build && make clean; fi + - run: | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c11build && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppc64build && make clean; fi + - run: | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make aarch64build && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild && make clean; fi + - run: | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -j regressiontest && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild && make clean; fi + - run: | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make shortest && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi + - run: | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make cxxtest && make clean; fi && + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib libzstd-nomt && make clean; fi + # This is based on your 1.0 configuration file or project settings + - run: echo Circle CI tests finished + # Teardown + # If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each + # Save test results + - store_test_results: + path: /tmp/circleci-test-results + # Save artifacts + - store_artifacts: + path: /tmp/circleci-artifacts + - store_artifacts: + path: /tmp/circleci-test-results From 4f5e4fe9ed27dd7cee7cf46cebf2b15e8014378b Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 8 Aug 2018 12:32:55 -0700 Subject: [PATCH 21/92] Fix Path --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4780c3d68..389fb11ae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ version: 2 jobs: build: - working_directory: ~/felixhandte/zstd + working_directory: ~/facebook/zstd parallelism: 1 shell: /bin/bash --login # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did. From 0ede1735b015ed993ece4a43778334a1a72ba78a Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 8 Aug 2018 12:31:37 -0700 Subject: [PATCH 22/92] Remove Old CircleCI Config --- circle.yml | 63 ------------------------------------------------------ 1 file changed, 63 deletions(-) delete mode 100644 circle.yml diff --git a/circle.yml b/circle.yml deleted file mode 100644 index ed50d59e5..000000000 --- a/circle.yml +++ /dev/null @@ -1,63 +0,0 @@ -dependencies: - override: - - sudo dpkg --add-architecture i386 - - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update - - sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross - -test: - override: - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then cc -v; CFLAGS="-O0 -Werror" make all && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c99build && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu99build && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make c11build && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppc64build && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make aarch64build && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make -j regressiontest && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make shortest && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi - : - parallel: true - - ? | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make cxxtest && make clean; fi && - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib libzstd-nomt && make clean; fi - : - parallel: true - - post: - - echo Circle CI tests finished - - # Longer tests - #- make -C tests test-zstd-nolegacy && make clean - #- pyenv global 3.4.4; make -C tests versionsTest && make clean - #- make zlibwrapper && make clean - #- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean - #- make uasan && make clean - #- make asan32 && make clean - #- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu" - # Valgrind tests - #- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean - #- make -C tests valgrindTest && make clean - # ARM, AArch64, PowerPC, PowerPC64 tests - #- make ppctest && make clean - #- make ppc64test && make clean - #- make armtest && make clean - #- make aarch64test && make clean From d1afd48e44404200e3ec1ee003b0e9bfcc8d7ee8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 8 Aug 2018 12:37:08 -0700 Subject: [PATCH 23/92] some errors may only happen when optimizations are enabled --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 257de66e0..36df67052 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ matrix: - env: Cmd='make gcc6install libc6install && make clean && CC=gcc-6 make -j uasan-test-zstd32' - env: Cmd='make gcc7install && make clean && CC=gcc-7 make -j uasan-test-zstd' - - env: Cmd='make gcc8install && CC=gcc-8 CFLAGS=-Werror make -j all' + - env: Cmd='make gcc8install && CC=gcc-8 CFLAGS="-Werror -O3" make -j all' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd' - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest' From 6518745024bf9e3d804d998c14f41144e4c16d91 Mon Sep 17 00:00:00 2001 From: "W. Felix Handte" Date: Wed, 8 Aug 2018 14:09:12 -0700 Subject: [PATCH 24/92] Preserve Commented-Out Longer Tests --- .circleci/config.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 389fb11ae..7af8b41ed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -103,3 +103,20 @@ jobs: path: /tmp/circleci-artifacts - store_artifacts: path: /tmp/circleci-test-results + + # Longer tests + #- make -C tests test-zstd-nolegacy && make clean + #- pyenv global 3.4.4; make -C tests versionsTest && make clean + #- make zlibwrapper && make clean + #- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean + #- make uasan && make clean + #- make asan32 && make clean + #- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu" + # Valgrind tests + #- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean + #- make -C tests valgrindTest && make clean + # ARM, AArch64, PowerPC, PowerPC64 tests + #- make ppctest && make clean + #- make ppc64test && make clean + #- make armtest && make clean + #- make aarch64test && make clean From cffb6da339970c7a5334e52abdf96493764561f7 Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 6 Jun 2018 16:19:09 -0700 Subject: [PATCH 25/92] Parses additional parameters Additional constraint checking Minor fixes more param parsing Add Memory Change paramVariation work on feasibility reformat bench Changed Paramgrill to use bench.c benchmarking customlevel macro Printing Flag Minor changes Explicit casting Makefile fix casting, type fix Printing Flag Minor Changes comments, helper fn's --- programs/bench.c | 2 + programs/bench.h | 1 + tests/paramgrill.c | 369 ++++++++++++++++++++++++++++++++++++++------- 3 files changed, 320 insertions(+), 52 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 76d1ff6dd..2fa9262c9 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -692,6 +692,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ + results.result.cMem = ZSTD_sizeof_CCtx(ctx); return results; } @@ -731,6 +732,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv); } + /* clean up */ BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); diff --git a/programs/bench.h b/programs/bench.h index 87cf56380..f1ac255f8 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -34,6 +34,7 @@ typedef struct { size_t cSize; double cSpeed; /* bytes / sec */ double dSpeed; + size_t cMem; } BMK_result_t; ERROR_STRUCT(BMK_result_t, BMK_return_t); diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 025bc6aad..81c6dffc2 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -63,6 +63,16 @@ static const int g_maxNbVariations = 64; #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) #define CUSTOM_LEVEL 99 +/* indices for each of the variables */ +#define WLOG_IND 0 +#define CLOG_IND 1 +#define HLOG_IND 2 +#define SLOG_IND 3 +#define SLEN_IND 4 +#define TLEN_IND 5 +#define STRT_IND 6 +#define NUM_PARAMS 7 + /*-************************************ * Benchmark Parameters **************************************/ @@ -140,6 +150,13 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) return result; } + +typedef struct { + U32 cSpeed; /* bytes / sec */ + U32 dSpeed; + U32 Mem; /* bytes */ +} constraint_t; + /*-******************************************************* * Bench functions *********************************************************/ @@ -340,7 +357,6 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para return better; } - /* nullified useless params, to ensure count stats */ static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters params) { @@ -354,8 +370,114 @@ static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters par return &g_params; } +/* res should be NUM_PARAMS size */ +static int variableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { + int j = 0; + if(!paramConstraints.windowLog) { + res[j] = WLOG_IND; + j++; + } + if(!paramConstraints.chainLog) { + res[j] = CLOG_IND; + j++; + } + if(!paramConstraints.hashLog) { + res[j] = HLOG_IND; + j++; + } + if(!paramConstraints.searchLog) { + res[j] = SLOG_IND; + j++; + } + if(!paramConstraints.searchLength) { + res[j] = SLEN_IND; + j++; + } + if(!paramConstraints.targetLength) { + res[j] = TLEN_IND; + j++; + } + if(!(U32)paramConstraints.strategy) { + res[j] = STRT_IND; + j++; + } + return j; +} -static void paramVariation(ZSTD_compressionParameters* ptr) +/* computes inverse of above array, returns same number, -1 = unused ind */ +static int inverseVariableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { + int j = 0; + if(!paramConstraints.windowLog) { + res[WLOG_IND] = j; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.chainLog) { + res[j] = CLOG_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.hashLog) { + res[j] = HLOG_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.searchLog) { + res[j] = SLOG_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.searchLength) { + res[j] = SLEN_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!paramConstraints.targetLength) { + res[j] = TLEN_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + if(!(U32)paramConstraints.strategy) { + res[j] = STRT_IND; + j++; + } else { + res[WLOG_IND] = -1; + } + return j; +} + +/* amt will probably always be \pm 1? */ +/* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ +static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* ptr) { + switch(paramIndex) + { + case WLOG_IND: ptr->windowLog += amt; break; + case CLOG_IND: ptr->chainLog += amt; break; + case HLOG_IND: ptr->hashLog += amt; break; + case SLOG_IND: ptr->searchLog += amt; break; + case SLEN_IND: ptr->searchLength += amt; break; + case TLEN_IND: + if(amt > 0) { + ptr->targetLength <<= amt; + } else { + ptr->targetLength >>= -amt; + } + break; + case STRT_IND: ptr->strategy += amt; break; + default: break; + } +} + +//Don't fuzz fixed variables. +//turn pcs to pcs array with macro for params. +//pass in variation array from variableParams +static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) { ZSTD_compressionParameters p; U32 validated = 0; @@ -363,40 +485,13 @@ static void paramVariation(ZSTD_compressionParameters* ptr) U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1; p = *ptr; for ( ; nbChanges ; nbChanges--) { - const U32 changeID = FUZ_rand(&g_rand) % 14; - switch(changeID) - { - case 0: - p.chainLog++; break; - case 1: - p.chainLog--; break; - case 2: - p.hashLog++; break; - case 3: - p.hashLog--; break; - case 4: - p.searchLog++; break; - case 5: - p.searchLog--; break; - case 6: - p.windowLog++; break; - case 7: - p.windowLog--; break; - case 8: - p.searchLength++; break; - case 9: - p.searchLength--; break; - case 10: - p.strategy = (ZSTD_strategy)(((U32)p.strategy)+1); break; - case 11: - p.strategy = (ZSTD_strategy)(((U32)p.strategy)-1); break; - case 12: - p.targetLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break; - case 13: - p.targetLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break; - } + const U32 changeID = FUZ_rand(&g_rand) % (2 * varyLen); + paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } validated = !ZSTD_isError(ZSTD_checkCParams(p)); + + //Make sure memory is at least close to feasible? + //ZSTD_estimateCCtxSize thing. } *ptr = p; } @@ -418,12 +513,14 @@ static void playAround(FILE* f, winnerInfo_t* winners, { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); + const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; + while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { ZSTD_compressionParameters p = params; if (nbVariations++ > g_maxNbVariations) break; - paramVariation(&p); + paramVariation(&p, unconstrained, 7); /* exclude faster if already played params */ if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1)) @@ -459,6 +556,24 @@ static ZSTD_compressionParameters randomParams(void) return p; } +static ZSTD_compressionParameters randomConstrainedParams(ZSTD_compressionParameters pc) +{ + ZSTD_compressionParameters p; + U32 validated = 0; + while (!validated) { + /* totally random entry */ + if(!pc.chainLog) p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; + if(!pc.chainLog) p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; + if(!pc.chainLog) p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; + if(!pc.chainLog) p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; + if(!pc.chainLog) p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; + if(!pc.chainLog) p.targetLength=(FUZ_rand(&g_rand) % (512)) + 1; //ZSTD_TARGETLENGTH_MIN; //change to 2^[0,10?] + if(!pc.chainLog) p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + } + return p; +} + static void BMK_selectRandomStart( FILE* f, winnerInfo_t* winners, const void* srcBuffer, size_t srcSize, @@ -638,14 +753,86 @@ static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); } +//Results currently don't capture memory usage or anything. +//parameter feasibility is not checked, should just be restricted from use. +static int feasible(BMK_result_t results, constraint_t target) { + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.Mem || !target.Mem); +} + +/* returns 1 if result2 is strictly 'better' than result1 */ +static int objective_lt(BMK_result_t result1, BMK_result_t result2) { + return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed); +} + +/* res gives array dimensions, should be size NUM_PARAMS */ +static size_t computeStateSize(const ZSTD_compressionParameters paramConstraints, U32* res) { + int ind = 0; + size_t base = 1; + if(!paramConstraints.windowLog) { res[ind] = ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.chainLog) { res[ind] = ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.hashLog) { res[ind] = ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.searchLog) { res[ind] = ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.searchLength) { res[ind] = ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1; base *= res[ind]; ind++; } + if(!paramConstraints.targetLength) { res[ind] = 11; base *= res[ind]; ind++; } //restricting from 2^[0,10], no such macros + if(!(U32)paramConstraints.strategy) { res[ind] = 8; base *= 8; } //not strictly true, maybe would want to case on this. + + return base; +} + + +static unsigned calcViolation(BMK_result_t results, constraint_t target) { + int diffcSpeed = MAX(target.cSpeed - results.cSpeed, 0); + int diffdSpeed = MAX(target.dSpeed - results.dSpeed, 0); + int diffcMem = MAX(results.cMem - target.Mem, 0); + return diffcSpeed + diffdSpeed + diffcMem; +} + +/* finds some set of parameters which fulfills req's + * Prioritize highest / try to locally minimize sum? + * Is it ever useful to go out of the param constraints? ? + * random / perturb when revisit? + * momentum? + */ +static ZSTD_compressionParameters findFeasible(constraint_t target, ZSTD_compressionParameters paramTarget) { + unsigned violation; + + ZSTD_compressionParameters winner = randomConstrainedParams(paramTarget); + //just use g_alreadyTested and xxhash? + BYTE* memotable; + do { + //prioritize memory + if(diffcMem >= diffcSpeed && diffcMem >= diffdSpeed) { + + //prioritize compression Speed + } else if (diffcSpeeed >= diffdSpeed && diffcSpeed >= diffcMem) { + + //prioritize decompressionSpeed + } else { + + } + violation = calcViolation(result, target); + } while(objective); + if(validate) { + DISPLAY("Feasible Point Found\n"); + return winner; + } else { + DISPLAY("No solution found\n"); + ZSTD_compressionParameters ret = { 0, 0, 0, 0, 0, 0, 0 }; + return ret; + } +} + /* optimizeForSize(): - * targetSpeed : expressed in MB/s */ -int optimizeForSize(const char* inFileName, U32 targetSpeed) + * targetSpeed : expressed in B/s */ +/* if state space is small (from paramTarget) */ +int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { FILE* const inFile = fopen( inFileName, "rb" ); U64 const inFileSize = UTIL_getFileSize(inFileName); size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; void* origBuff; + U32 paramVarArray [NUM_PARAMS]; + int paramCount = variableParams(paramTarget, paramVarArray); /* Init */ if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } if (inFileSize == UTIL_FILESIZE_UNKNOWN) { @@ -682,8 +869,11 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) /* bench */ DISPLAY("\r%79s\r", ""); - DISPLAY("optimizing for %s - limit speed %u MB/s \n", inFileName, targetSpeed); - targetSpeed *= 1000000; + DISPLAY("optimizing for %s", inFileName); + if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } + if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } + if(target.Mem != 0) { DISPLAY(" - limit memory %u MB", target.Mem / 1000000); } + DISPLAY("\n"); { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); winnerInfo_t winner; @@ -696,23 +886,22 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) winner.result.cSize = (size_t)(-1); /* find best solution from default params */ + //Can't do this iteration normally w/ cparameter constraints { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); int i; for (i=1; i<=maxSeeds; i++) { ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams); - if (candidate.cSpeed < (double)targetSpeed) { + if (!feasible(candidate, target) ) { break; } - if ( (candidate.cSize < winner.result.cSize) - | ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) + if (feasible(candidate,target) && objective_lt(winner.result, candidate)) { winner.params = CParams; winner.result = candidate; BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); } } } - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); BMK_translateAdvancedParams(winner.params); @@ -721,7 +910,7 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) { time_t const grillStart = time(NULL); do { ZSTD_compressionParameters params = winner.params; - paramVariation(¶ms); + paramVariation(¶ms, paramVarArray, paramCount); if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */ params = ZSTD_adjustCParams(params, blockSize, 0); @@ -733,9 +922,7 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, params); /* improvement found => new winner */ - if ( (candidate.cSpeed > targetSpeed) - & ( (candidate.cSize < winner.result.cSize) - | ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) ) + if (feasible(candidate,target) && objective_lt(winner.result, candidate)) { winner.params = params; winner.result = candidate; @@ -744,8 +931,13 @@ int optimizeForSize(const char* inFileName, U32 targetSpeed) } } while (BMK_timeSpan(grillStart) < g_grillDuration_s); } - /* end summary */ + /* no solution found */ + if(winner.result.cSize == (size_t)-1) { + DISPLAY("No feasible solution found\n"); + return 1; + } + /* end summary */ BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); BMK_translateAdvancedParams(winner.params); DISPLAY("grillParams size - optimizer completed \n"); @@ -834,7 +1026,9 @@ int main(int argc, const char** argv) const char* input_filename=0; U32 optimizer = 0; U32 main_pause = 0; - U32 targetSpeed = 0; + + constraint_t target = { 0 , 0, 0 }; //0 for anything unset + ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; assert(argc>=1); /* for exename */ @@ -847,8 +1041,31 @@ int main(int argc, const char** argv) if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; } + if (longCommandWArg(&argument, "--optimize=")) { + optimizer = 1; + for ( ; ;) { + if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { paramTarget.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { paramTarget.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { paramTarget.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { paramTarget.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { paramTarget.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { paramTarget.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { paramTarget.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.Mem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + /* in MB or MB/s */ + DISPLAY("invalid optimization parameter \n"); + return 1; + } + + if (argument[0] != 0) { + DISPLAY("invalid --optimize= format\n"); + return 1; /* check the end of string */ + } + continue; + } else if (longCommandWArg(&argument, "--zstd=")) { /* Decode command (note : aggregated commands are allowed) */ - if (longCommandWArg(&argument, "--zstd=")) { g_singleRun = 1; g_params = ZSTD_getCParams(2, g_blockSize, 0); for ( ; ;) { @@ -868,6 +1085,7 @@ int main(int argc, const char** argv) DISPLAY("invalid --zstd= format\n"); return 1; /* check the end of string */ } + continue; /* if not return, success */ } else if (argument[0]=='-') { argument++; @@ -900,7 +1118,54 @@ int main(int argc, const char** argv) case 'O': argument++; optimizer = 1; - targetSpeed = readU32FromChar(&argument); + for ( ; ; ) { + switch(*argument) + { + /* Inputs in MB or MB/s */ + case 'C': + argument++; + target.cSpeed = readU32FromChar(&argument) * 1000000; + continue; + case 'D': + argument++; + target.dSpeed = readU32FromChar(&argument) * 1000000; + continue; + case 'M': + argument++; + target.Mem = readU32FromChar(&argument) * 1000000; + continue; + case 'w': + argument++; + paramTarget.windowLog = readU32FromChar(&argument); + continue; + case 'c': + argument++; + paramTarget.chainLog = readU32FromChar(&argument); + continue; + case 'h': + argument++; + paramTarget.hashLog = readU32FromChar(&argument); + continue; + case 's': + argument++; + paramTarget.searchLog = readU32FromChar(&argument); + continue; + case 'l': /* search length */ + argument++; + paramTarget.searchLength = readU32FromChar(&argument); + continue; + case 't': /* target length */ + argument++; + paramTarget.targetLength = readU32FromChar(&argument); + continue; + case 'S': /* strategy */ + argument++; + paramTarget.strategy = (ZSTD_strategy)readU32FromChar(&argument); + continue; + default : ; + } + break; + } break; /* Run Single conf */ @@ -990,7 +1255,7 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize(input_filename, targetSpeed); + result = optimizeForSize(input_filename, target, paramTarget); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From 5f49034520940af38deb5e13d9d624afc7ffe534 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 18 Jun 2018 11:59:45 -0700 Subject: [PATCH 26/92] Working V1 --- programs/bench.c | 95 ++-- programs/bench.h | 3 + tests/Makefile | 2 +- tests/paramgrill.c | 1256 ++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 1213 insertions(+), 143 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 2fa9262c9..0d35f3eb3 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -85,7 +85,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; * Exceptions ***************************************/ #ifndef DEBUG -# define DEBUG 0 +# define DEBUG 1 #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } @@ -281,12 +281,14 @@ static size_t local_defaultDecompress( } +volatile char g_touched; + /* initFn will be measured once, bench fn will be measured x times */ /* benchFn should return error value or out Size */ /* takes # of blocks and list of size & stuff for each. */ /* only does looping */ /* note time per loop could be zero if interval too short */ -BMK_customReturn_t BMK_benchFunction( +BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, @@ -299,16 +301,6 @@ BMK_customReturn_t BMK_benchFunction( BMK_customReturn_t retval; UTIL_time_t clockStart; - { - unsigned i; - for(i = 0; i < blockCount; i++) { - memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ - } - - UTIL_sleepMilli(5); /* give processor time to other processes */ - UTIL_waitForNextTick(); - } - if(!nbLoops) { EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n"); } @@ -317,6 +309,22 @@ BMK_customReturn_t BMK_benchFunction( srcSize += srcBlockSizes[ind]; } + { + unsigned i, j; + for(i = 0; i < blockCount; i++) { + for(j = 0; j < srcBlockSizes[i]; j++) { + g_touched = ((const char*)srcBlockBuffers[i])[j]; /* touch */ + } + } + for(i = 0; i < blockCount; i++) { + memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ + //this is written at end proc? where did compressed data get overwritten ny this? + } + + UTIL_sleepMilli(5); /* give processor time to other processes */ + UTIL_waitForNextTick(); + } + { unsigned i, j, firstIter = 1; clockStart = UTIL_getTime(); @@ -327,9 +335,9 @@ BMK_customReturn_t BMK_benchFunction( if(ZSTD_isError(res)) { EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); - } else if(firstIter) { + } else if(firstIter) { dstSize += res; - } + } } firstIter = 0; } @@ -393,7 +401,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( { U64 const loopDuration = r.result.result.nanoSecPerRun * cont->nbLoops; r.completed = (cont->timeRemaining <= loopDuration); cont->timeRemaining -= loopDuration; - if (loopDuration > 0) { + if (loopDuration > (TIMELOOP_NANOSEC / 100)) { fastest = MIN(fastest, r.result.result.nanoSecPerRun); if(loopDuration >= MINUSABLETIME) { r.result.result.nanoSecPerRun = fastest; @@ -420,7 +428,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( const void ** const srcPtrs, size_t* const srcSizes, void** const cPtrs, size_t* const cSizes, void** const resPtrs, size_t* const resSizes, - void* resultBuffer, void* compressedBuffer, + void** resultBufferPtr, void* compressedBuffer, const size_t maxCompressedSize, BMK_timedFnState_t* timeStateCompress, BMK_timedFnState_t* timeStateDecompress, @@ -432,7 +440,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) { size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ - BMK_return_t results; + BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; size_t const loadedCompressedSize = srcSize; size_t cSize = 0; double ratio = 0.; @@ -454,13 +462,14 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( srcPtr += fileSizes[fileNb]; } { size_t const decodedSize = (size_t)totalDSize64; - free(resultBuffer); - resultBuffer = malloc(decodedSize); - if (!resultBuffer) { + free(*resultBufferPtr); + //TODO: decodedSize corrupted? + *resultBufferPtr = malloc(decodedSize); + if (!(*resultBufferPtr)) { EXM_THROW(33, BMK_return_t, "not enough memory"); } if (totalDSize64 > decodedSize) { - free(resultBuffer); + free(*resultBufferPtr); EXM_THROW(32, BMK_return_t, "original size is too large"); /* size_t overflow */ } cSize = srcSize; @@ -472,7 +481,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( /* Init data blocks */ { const char* srcPtr = (const char*)srcBuffer; char* cPtr = (char*)compressedBuffer; - char* resPtr = (char*)resultBuffer; + char* resPtr = (char*)(*resultBufferPtr); U32 fileNb; for (nbBlocks=0, fileNb=0; fileNbmode != BMK_decodeOnly) { + BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds); + nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds); if(compressionResults.error) { results.error = compressionResults.error; return results; @@ -642,7 +652,8 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } /* CRC Checking */ - { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); + { void* resultBuffer = *resultBufferPtr; + U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); /* adv->mode == 0 -> compress + decompress */ if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) { size_t u; @@ -692,11 +703,13 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ - results.result.cMem = ZSTD_sizeof_CCtx(ctx); + results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); + results.error = 0; return results; } BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstCapacity, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, @@ -717,26 +730,41 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - const size_t maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ - void* compressedBuffer = malloc(maxCompressedSize); - void* resultBuffer = malloc(srcSize); BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds); BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds); + void* compressedBuffer; + const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); + void* resultBuffer = malloc(srcSize); + + BMK_return_t results; - int allocationincomplete = !compressedBuffer || !resultBuffer || + int allocationincomplete; + + if(!dstCapacity) { + compressedBuffer = malloc(maxCompressedSize); + } else { + compressedBuffer = dstBuffer; + } + + allocationincomplete = !compressedBuffer || !resultBuffer || !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes; + if (!allocationincomplete) { results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cSizes, - resPtrs, resSizes, resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress, + resPtrs, resSizes, &resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv); } + + /* clean up */ BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); - free(compressedBuffer); + if(!dstCapacity) { /* only free if not given */ + free(compressedBuffer); + } free(resultBuffer); free((void*)srcPtrs); @@ -749,7 +777,6 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, if(allocationincomplete) { EXM_THROW(31, BMK_return_t, "allocation error : not enough memory"); } - results.error = 0; return results; } @@ -762,6 +789,7 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, const BMK_advancedParams_t adv = BMK_initAdvancedParams(); return BMK_benchMemAdvanced(srcBuffer, srcSize, + NULL, 0, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, @@ -783,6 +811,7 @@ static BMK_return_t BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize, EXM_THROW(12, BMK_return_t, "not enough memory for contexts"); } res = BMK_benchMemAdvanced(srcBuffer, srcSize, + NULL, 0, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, diff --git a/programs/bench.h b/programs/bench.h index f1ac255f8..625b65757 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -122,6 +122,8 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, * (cLevel, comprParams + adv in advanced Mode) */ /* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode * srcSize - size of data in srcBuffer + * dstBuffer - destination buffer to write compressed output in, optional (NULL) + * dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL * cLevel - compression level * comprParams - basic compression parameters * dictBuffer - a dictionary if used, null otherwise @@ -144,6 +146,7 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, /* See benchMem for normal parameter uses and return, see advancedParams_t for adv */ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstCapacity, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, diff --git a/tests/Makefile b/tests/Makefile index 81e685780..ecf7fe2a7 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -200,7 +200,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) -paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements +#paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 81c6dffc2..44bcedef8 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -41,6 +41,8 @@ #define MB *(1<<20) #define GB *(1ULL<<30) +#define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */ + #define NBLOOPS 2 #define TIMELOOP (2 * SEC_TO_MICRO) #define NB_LEVELS_TRACKED 22 /* ensured being >= ZSTD_maxCLevel() in BMK_init_level_constraints() */ @@ -70,13 +72,39 @@ static const int g_maxNbVariations = 64; #define SLOG_IND 3 #define SLEN_IND 4 #define TLEN_IND 5 -#define STRT_IND 6 -#define NUM_PARAMS 7 +//#define STRT_IND 6 +//#define NUM_PARAMS 7 +#define NUM_PARAMS 6 +//just don't use strategy as a param. +#undef ZSTD_WINDOWLOG_MAX +#define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. + +//make 2^[0,10] w/ 999 +#define ZSTD_TARGETLENGTH_MIN 0 //actually targeLengthlog min +#define ZSTD_TARGETLENGTH_MAX 10 + +//#define ZSTD_TARGETLENGTH_MAX 1024 +#define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1) +#define CLOG_RANGE (ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1) +#define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1) +#define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1) +#define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) +#define TLEN_RANGE 11 +//hard coded since we only use powers of 2 (and 999 ~ 1024) + +static const int mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN }; +static const int maxtable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX }; +static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; + +//use grid-search or something when space is small enough? +#define SMALL_SEARCH_SPACE 1000 /*-************************************ * Benchmark Parameters **************************************/ +typedef BYTE U8; + static double g_grillDuration_s = 99999; /* about 27 hours */ static U32 g_nbIterations = NBLOOPS; static double g_compressibility = COMPRESSIBILITY_DEFAULT; @@ -150,13 +178,74 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) return result; } +//assume that clock can at least measure .01 second intervals? +//make this a settable global initialized with fn? +//#define CLOCK_GRANULARITY 100000000ULL +static U64 g_clockGranularity = 100000000ULL; + +static void findClockGranularity(void) { + UTIL_time_t clockStart = UTIL_getTime(); + U64 el1 = 0, el2 = 0; + int i = 0; + do { + el1 = el2; + el2 = UTIL_clockSpanNano(clockStart); + if(el1 < el2) { + U64 iv = el2 - el1; + if(g_clockGranularity > iv) { + g_clockGranularity = iv; + i = 0; + } else { + i++; + } + } + } while(i < 10); + DISPLAY("Granularity: %llu\n", (unsigned long long)g_clockGranularity); +} typedef struct { U32 cSpeed; /* bytes / sec */ U32 dSpeed; - U32 Mem; /* bytes */ + U32 cMem; /* bytes */ } constraint_t; +#define CLAMPCHECK(val,min,max) { \ + if (val && (((val)<(min)) | ((val)>(max)))) { \ + DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \ + return 0; \ +} } + + +/* Like ZSTD_checkCParams() but allows 0's */ +/* no check on targetLen? */ +static int cParamValid(ZSTD_compressionParameters paramTarget) { + CLAMPCHECK(paramTarget.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + CLAMPCHECK(paramTarget.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); + CLAMPCHECK(paramTarget.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); + CLAMPCHECK(paramTarget.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); + CLAMPCHECK(paramTarget.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); + if(paramTarget.strategy > ZSTD_btultra) { + DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); + return 0; + } + return 1; +} + +static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) { + paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN; + paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN; + paramTarget->chainLog = paramTarget->chainLog ? paramTarget->chainLog : ZSTD_CHAINLOG_MIN; + paramTarget->hashLog = paramTarget->hashLog ? paramTarget->hashLog : ZSTD_HASHLOG_MIN; + paramTarget->searchLength = paramTarget->searchLength ? paramTarget->searchLength : ZSTD_SEARCHLENGTH_MIN; + paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 1; +} + +static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) +{ + DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); +} + /*-******************************************************* * Bench functions *********************************************************/ @@ -357,20 +446,45 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para return better; } +/* bounds check in sanitize too? */ +#define CLAMP(var, lo, hi) { \ + var = MAX(MIN(var, hi), lo); \ +} + /* nullified useless params, to ensure count stats */ -static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters params) +/* no point in windowLog < chainLog (no point 2x chainLog for bt) */ +/* now with built in bounds-checking */ +/* no longer does anything with sanitizeVarArray + clampcheck */ +static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters params) { - g_params = params; if (params.strategy == ZSTD_fast) g_params.chainLog = 0, g_params.searchLog = 0; if (params.strategy == ZSTD_dfast) g_params.searchLog = 0; - if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra) + if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra && params.strategy != ZSTD_fast) g_params.targetLength = 0; - return &g_params; + + return params; +} + +/* new length */ +/* keep old array, will need if iter over strategy. */ +static int sanitizeVarArray(int varLength, U32* varArray, U32* varNew, ZSTD_strategy strat) { + int i, j = 0; + for(i = 0; i < varLength; i++) { + if( !((varArray[i] == CLOG_IND && strat == ZSTD_fast) + || (varArray[i] == SLOG_IND && strat == ZSTD_dfast) + || (varArray[i] == TLEN_IND && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { + varNew[j] = varArray[i]; + j++; + } + } + return j; + } /* res should be NUM_PARAMS size */ +/* constructs varArray from ZSTD_compressionParameters style parameter */ static int variableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { int j = 0; if(!paramConstraints.windowLog) { @@ -397,10 +511,6 @@ static int variableParams(const ZSTD_compressionParameters paramConstraints, U32 res[j] = TLEN_IND; j++; } - if(!(U32)paramConstraints.strategy) { - res[j] = STRT_IND; - j++; - } return j; } @@ -417,43 +527,39 @@ static int inverseVariableParams(const ZSTD_compressionParameters paramConstrain res[j] = CLOG_IND; j++; } else { - res[WLOG_IND] = -1; + res[CLOG_IND] = -1; } if(!paramConstraints.hashLog) { res[j] = HLOG_IND; j++; } else { - res[WLOG_IND] = -1; + res[HLOG_IND] = -1; } if(!paramConstraints.searchLog) { res[j] = SLOG_IND; j++; } else { - res[WLOG_IND] = -1; + res[SLOG_IND] = -1; } if(!paramConstraints.searchLength) { res[j] = SLEN_IND; j++; } else { - res[WLOG_IND] = -1; + res[SLEN_IND] = -1; } if(!paramConstraints.targetLength) { res[j] = TLEN_IND; j++; } else { - res[WLOG_IND] = -1; - } - if(!(U32)paramConstraints.strategy) { - res[j] = STRT_IND; - j++; - } else { - res[WLOG_IND] = -1; + res[TLEN_IND] = -1; } + return j; } /* amt will probably always be \pm 1? */ /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ +/* take max/min bounds into account as well? */ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* ptr) { switch(paramIndex) { @@ -463,13 +569,16 @@ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* p case SLOG_IND: ptr->searchLog += amt; break; case SLEN_IND: ptr->searchLength += amt; break; case TLEN_IND: - if(amt > 0) { + if(amt >= 0) { ptr->targetLength <<= amt; + ptr->targetLength = MIN(ptr->targetLength, 999); } else { + if(ptr->targetLength == 999) { + ptr->targetLength = 1024; + } ptr->targetLength >>= -amt; } break; - case STRT_IND: ptr->strategy += amt; break; default: break; } } @@ -477,34 +586,143 @@ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* p //Don't fuzz fixed variables. //turn pcs to pcs array with macro for params. //pass in variation array from variableParams -static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) +//take nbChanges as argument? +static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, U32 nbChanges) { ZSTD_compressionParameters p; U32 validated = 0; while (!validated) { - U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1; + U32 i; p = *ptr; - for ( ; nbChanges ; nbChanges--) { - const U32 changeID = FUZ_rand(&g_rand) % (2 * varyLen); + for (i = 0 ; i < nbChanges ; i++) { + const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } - validated = !ZSTD_isError(ZSTD_checkCParams(p)); - + //validated = !ZSTD_isError(ZSTD_checkCParams(p)); + validated = cParamValid(p); + //Make sure memory is at least close to feasible? //ZSTD_estimateCCtxSize thing. } - *ptr = p; + *ptr = sanitizeParams(p); } +//varyParams gives us table size? +//1 per strategy +//varyParams should always be sorted smallest to largest +//take arrayLen to allocate memotable +//should be ~10^7 unconstrained. +static size_t memoTableLen(const U32* varyParams, const int varyLen) { + size_t arrayLen = 1; + int i; + for(i = 0; i < varyLen; i++) { + arrayLen *= rangetable[varyParams[i]]; + } + return arrayLen; +} + +//sort of ~lg2 (replace 1024 w/ 999) for memoTableInd Tlen +static unsigned lg2(unsigned x) { + unsigned j = 0; + if(x == 999) { + return 10; + } + while(x >>= 1) { + j++; + } + return j; +} + +//indexes compressionParameters into memotable +//of form +static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) { + int i; + unsigned ind = 0; + for(i = 0; i < varyLen; i++) { + switch(varyParams[i]) { + case WLOG_IND: ind *= WLOG_RANGE; ind += ptr->windowLog - ZSTD_WINDOWLOG_MIN ; break; + case CLOG_IND: ind *= CLOG_RANGE; ind += ptr->chainLog - ZSTD_CHAINLOG_MIN ; break; + case HLOG_IND: ind *= HLOG_RANGE; ind += ptr->hashLog - ZSTD_HASHLOG_MIN ; break; + case SLOG_IND: ind *= SLOG_RANGE; ind += ptr->searchLog - ZSTD_SEARCHLOG_MIN ; break; + case SLEN_IND: ind *= SLEN_RANGE; ind += ptr->searchLength - ZSTD_SEARCHLENGTH_MIN; break; + case TLEN_IND: ind *= TLEN_RANGE; ind += lg2(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; + } + } + return ind; +} + +/* presumably, the unfilled parameters are already at their correct value */ +/* inverse above function for varyParams */ +static void memoTableIndInv(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, size_t ind) { + int i; + for(i = varyLen - 1; i >= 0; i--) { + switch(varyParams[i]) { + case WLOG_IND: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; + case CLOG_IND: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; + case HLOG_IND: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; + case SLOG_IND: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; + case SLEN_IND: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; + case TLEN_IND: ptr->targetLength = MIN(1 << (ind % TLEN_RANGE), 999); ind /= TLEN_RANGE; break; + } + } +} + +//initializing memoTable +/* */ +static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen) { + size_t i; + size_t arrayLen = memoTableLen(varyParams, varyLen); + int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; + int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog; + int j = 0; + memset(memoTable, 0, arrayLen); + + + for(i = 0; i < arrayLen; i++) { + memoTableIndInv(¶mConstraints, varyParams, varyLen, i); + BMK_translateAdvancedParams(paramConstraints); + if(ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1 << paramConstraints.windowLog) > target.cMem) { + //infeasible; + memoTable[i] = 255; + j++; + } + /* nil out parameter sets equivalent to others. */ + if(cwFixed/* at most least 1 param fixed. */) { + if(paramConstraints.strategy == ZSTD_btlazy2 || paramConstraints.strategy == ZSTD_btopt || paramConstraints.strategy == ZSTD_btultra) { + if(paramConstraints.chainLog > paramConstraints.windowLog + 1) { + if(memoTable[i] != 255) { j++; } + memoTable[i] = 255; + } + } else { + if(paramConstraints.chainLog > paramConstraints.windowLog) { + if(memoTable[i] != 255) { j++; } + memoTable[i] = 255; + } + } + } + if(scFixed) { + if(paramConstraints.searchLog > paramConstraints.chainLog) { + if(memoTable[i] != 255) { j++; } + memoTable[i] = 255; + } + } + } + DISPLAY("%d / %d Invalid\n", j, (int)i); +} #define PARAMTABLELOG 25 #define PARAMTABLESIZE (1<> 3) & PARAMTABLEMASK] + g_alreadyTested[(XXH64(((void*)&sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK] */ +static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { + ZSTD_compressionParameters p2 = sanitizeParams(p); + return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK]; +} static void playAround(FILE* f, winnerInfo_t* winners, ZSTD_compressionParameters params, @@ -513,21 +731,23 @@ static void playAround(FILE* f, winnerInfo_t* winners, { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); - const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; + const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5 }; while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { ZSTD_compressionParameters p = params; + BYTE* b; if (nbVariations++ > g_maxNbVariations) break; - paramVariation(&p, unconstrained, 7); + paramVariation(&p, unconstrained, 7, 4); /* exclude faster if already played params */ - if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1)) + if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) continue; /* test */ - NB_TESTS_PLAYED(p)++; + b = NB_TESTS_PLAYED(p); + (*b)++; if (!BMK_seed(winners, p, srcBuffer, srcSize, ctx, dctx)) continue; /* improvement found => search more */ @@ -544,34 +764,37 @@ static ZSTD_compressionParameters randomParams(void) U32 validated = 0; while (!validated) { /* totally random entry */ - p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; - p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; - p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; - p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; + p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; + p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; + p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; + p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; p.targetLength=(FUZ_rand(&g_rand) % (512)); p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); - validated = !ZSTD_isError(ZSTD_checkCParams(p)); + //validated = !ZSTD_isError(ZSTD_checkCParams(p)); + validated = cParamValid(p); } return p; } -static ZSTD_compressionParameters randomConstrainedParams(ZSTD_compressionParameters pc) +//destructively modifies pc. +//Maybe if memoTable[ind] > 0 too often, count zeroes and explicitly choose from free stuff? +//^ maybe this doesn't matter, with |mt| size it has \approx 1-(1/e) of finding even single free spot in |mt| tries, not too bad. +//TODO: maybe memoTable pc before sanitization too so no repeats? +static void randomConstrainedParams(ZSTD_compressionParameters* pc, U32* varArray, int varLen, U8* memoTable) { - ZSTD_compressionParameters p; - U32 validated = 0; - while (!validated) { - /* totally random entry */ - if(!pc.chainLog) p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; - if(!pc.chainLog) p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; - if(!pc.chainLog) p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; - if(!pc.chainLog) p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; - if(!pc.chainLog) p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; - if(!pc.chainLog) p.targetLength=(FUZ_rand(&g_rand) % (512)) + 1; //ZSTD_TARGETLENGTH_MIN; //change to 2^[0,10?] - if(!pc.chainLog) p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); - validated = !ZSTD_isError(ZSTD_checkCParams(p)); - } - return p; + int tries = memoTableLen(varArray, varLen); //configurable, + const size_t maxSize = memoTableLen(varArray, varLen); + size_t ind; + do { + ind = (FUZ_rand(&g_rand)) % maxSize; + tries--; + } while(memoTable[ind] > 0 && tries > 0); + //&& FUZ_rand(&g_rand) % 256 > memoTable[ind]); get nd choosing? (helpful w/ distance) /* maybe > infeasible bound? */ + + /* memoTable[ind] == 0 -> unexplored */ + memoTableIndInv(pc, varArray, varLen, (unsigned)ind); + *pc = sanitizeParams(*pc); } static void BMK_selectRandomStart( @@ -746,22 +969,80 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } - -static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) -{ - DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); -} - -//Results currently don't capture memory usage or anything. //parameter feasibility is not checked, should just be restricted from use. static int feasible(BMK_result_t results, constraint_t target) { - return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.Mem || !target.Mem); + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); +} + +#define EPSILON 0.01 +static int epsilonEqual(double c1, double c2) { + return MAX(c1/c2,c2/c1) < 1 + EPSILON; +} + +//so the compiler stops warning +static int eqZero(double c1) { + return (U64)c1 == (U64)0.0 || (U64)c1 == (U64)-0.0; } /* returns 1 if result2 is strictly 'better' than result1 */ +/* strict comparison / cutoff based */ static int objective_lt(BMK_result_t result1, BMK_result_t result2) { - return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed); + return (result1.cSize > result2.cSize) || (epsilonEqual(result1.cSize, result2.cSize) && result2.cSpeed > result1.cSpeed) + || (epsilonEqual(result1.cSize,result2.cSize) && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); +} + +//will probably be some linear combinartion of comp speed, decompSpeed, & ratio (maybe size), and memory? +//pretty arbitrary right now +//maybe better - higher coefficient when below threshold, lower when above +//need to normalize speed? or just use ratio speed / target? +//Maybe don't use ratio at all when looking for feasibility? + +/* maybe dynamically vary the coefficients for this around based on what's already been discovered. (maybe make a reversed ratio cutoff?) concave to pheaily penalize below ratio? */ +static double resultScore(BMK_result_t res, size_t srcSize, constraint_t target) { + double cs = 0., ds = 0., rt, cm = 0.; + const double r1 = 1, r2 = 0.1, rtr = 0.5; + double ret; + if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; } + if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; } + if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; } + rt = ((double)srcSize / res.cSize); + + //(void)rt; + //(void)rtr; + + ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + + (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; + //DISPLAY("resultScore: %f\n", ret); + return ret; +} + +/* + double W_ratio = (double)srcSize / testResult.cSize; + double O_ratio = (double)srcSize / winners[cLevel].result.cSize; + double W_ratioNote = log (W_ratio); + double O_ratioNote = log (O_ratio); + size_t W_DMemUsed = (1 << params.windowLog) + (16 KB); + size_t O_DMemUsed = (1 << winners[cLevel].params.windowLog) + (16 KB); + double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); + double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); + + size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(params); + size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(winners[cLevel].params); + double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); + double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); + + double W_CSpeed_note = W_ratioNote * ( 30 + 10*cLevel) + log(testResult.cSpeed); + double O_CSpeed_note = O_ratioNote * ( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed); + + double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log(testResult.dSpeed); + double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed); + +*/ +//ratio tradeoffs, may be useful in guiding + +/* objective_lt, but based on scoring function */ +static int objective_lt2(BMK_result_t result1, BMK_result_t result2, size_t srcSize, constraint_t target) { + return resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target); } /* res gives array dimensions, should be size NUM_PARAMS */ @@ -783,48 +1064,800 @@ static size_t computeStateSize(const ZSTD_compressionParameters paramConstraints static unsigned calcViolation(BMK_result_t results, constraint_t target) { int diffcSpeed = MAX(target.cSpeed - results.cSpeed, 0); int diffdSpeed = MAX(target.dSpeed - results.dSpeed, 0); - int diffcMem = MAX(results.cMem - target.Mem, 0); + int diffcMem = MAX(results.cMem - target.cMem, 0); return diffcSpeed + diffdSpeed + diffcMem; } -/* finds some set of parameters which fulfills req's - * Prioritize highest / try to locally minimize sum? - * Is it ever useful to go out of the param constraints? ? - * random / perturb when revisit? - * momentum? - */ -static ZSTD_compressionParameters findFeasible(constraint_t target, ZSTD_compressionParameters paramTarget) { - unsigned violation; - - ZSTD_compressionParameters winner = randomConstrainedParams(paramTarget); - //just use g_alreadyTested and xxhash? - BYTE* memotable; - do { - //prioritize memory - if(diffcMem >= diffcSpeed && diffcMem >= diffdSpeed) { - - //prioritize compression Speed - } else if (diffcSpeeed >= diffdSpeed && diffcSpeed >= diffcMem) { - - //prioritize decompressionSpeed - } else { - - } - violation = calcViolation(result, target); - } while(objective); - if(validate) { - DISPLAY("Feasible Point Found\n"); - return winner; +/* +uncertaintyConstant >= 1 +returns -1 = 'certainly' infeasible + 0 = unceratin + 1 = 'certainly' feasible +*/ +//paramTarget misnamed, should just be target +static int uncertainFeasibility(double const uncertaintyConstantC, double const uncertaintyConstantD, const constraint_t paramTarget, const BMK_result_t* const results) { + if((paramTarget.cSpeed != 0 && results->cSpeed * uncertaintyConstantC < paramTarget.cSpeed) || + (paramTarget.dSpeed != 0 && results->dSpeed * uncertaintyConstantD < paramTarget.dSpeed) || + (paramTarget.cMem != 0 && results->cMem > paramTarget.cMem)) { + return -1; + } else if((paramTarget.cSpeed == 0 || results->cSpeed / uncertaintyConstantC > paramTarget.cSpeed) && + (paramTarget.dSpeed == 0 || results->dSpeed / uncertaintyConstantD > paramTarget.dSpeed) && + (paramTarget.cMem == 0 || results->cMem <= paramTarget.cMem)) { + return 1; } else { - DISPLAY("No solution found\n"); - ZSTD_compressionParameters ret = { 0, 0, 0, 0, 0, 0, 0 }; - return ret; + return 0; } } +/* 1 - better than prev best + 0 - uncertain + -1 - worse + assume prev_best status is run fully? + but then we'd have to rerun any winners anyway */ +//presumably memory has already been compared, mostly worried about mem, cspeed, dspeed +//uncertainty only applies to speed. +//if using objective fn, this could be much easier since we could just scale that. +//difficult to make judgements about later parameters in prioritization type when there's +//uncertainty on the first. +static int uncertainComparison(double const uncertaintyConstantC, double const uncertaintyConstantD, BMK_result_t* candidate, BMK_result_t* prevBest) { + (void)uncertaintyConstantD; //unused for now + if(candidate->cSpeed > prevBest->cSpeed * uncertaintyConstantC) { + return 1; + } else if (candidate->cSpeed * uncertaintyConstantC < prevBest->cSpeed) { + return -1; + } else { + return 0; + } +} + +/* speed in b, srcSize in b/s loopDuration in ns */ +//TODO: simplify code in feasibleBench with this instead of writing it all out. +//only applicable for single loop +static double calcUncertainty(double speed, size_t srcSize) { + U64 loopDuration; + if(eqZero(speed)) { return 2; } + loopDuration = ((srcSize * TIMELOOP_NANOSEC) / speed); + return MIN((loopDuration + (double)2 * g_clockGranularity) / loopDuration, 2); +} + +//benchmarks and tests feasibility together +//1 = true = better +//0 = false = not better +//if true then resultPtr will give results. +//2+ on error? +//alt: error = 0 / infeasible as well; +//maybe use compress_only mode for ratio-finding benchmark? +//prioritize ratio > cSpeed > dSpeed > cMem +//Misnamed - should be worse, better, error +//alternative (to make work for feasible-pt searching as well) - only compare to winner, not to target +//but then we need to judge what better means in this context, which shouldn't be the same (strict ratio improvement) +#define INFEASIBLE_RESULT 0 +#define FEASIBLE_RESULT 1 +#define ERROR_RESULT 2 +static int feasibleBench(BMK_result_t* resultPtr, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult) { + BMK_advancedParams_t adv = BMK_initAdvancedParams(); + BMK_return_t benchres; + U64 loopDurationC = 0, loopDurationD = 0; + double uncertaintyConstantC, uncertaintyConstantD; + adv.loopMode = BMK_iterMode; + adv.nbSeconds = 1; //get ratio and 2x approx speed? + + //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), + //maybe iter this until 2x measurable for better guarantee? + DISPLAY("Feas:\n"); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres.error) { + DISPLAY("ERROR %d !!\n", benchres.error); + } + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + + if(!benchres.error) { + *resultPtr = benchres.result; + /* if speed is 0 (only happens when time = 0) */ + if(eqZero(benchres.result.cSpeed)) { + loopDurationC = 0; + uncertaintyConstantC = 2; + } else { + loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? + //possibly has to do with initCCtx? or system stuff? + //asymmetric +/- constant needed? + uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC), 2); //.02 seconds + } + if(eqZero(benchres.result.dSpeed)) { + loopDurationD = 0; + uncertaintyConstantD = 2; + } else { + loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? + //possibly has to do with initCCtx? or system stuff? + //asymmetric +/- constant needed? + uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD), 2); //.02 seconds + } + + + if(benchres.result.cSize < winnerResult->cSize) { //better compression ratio, just needs to be feasible + //optimistic assume speed + //incoporate some sort of tradeoff comparison with the winner's results? + int feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); + if(feas == 0) { // uncertain feasibility + adv.loopMode = BMK_timeMode; + if(loopDurationC < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.cSpeed = benchres2.result.cSpeed; + } + } + if(loopDurationD < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + *resultPtr = benchres.result; + return feasible(benchres.result, target); + } else { //feas = 1 or -1 map to 1, 0 respectively + return (feas + 1) >> 1; //relies on INFEASIBLE_RESULT == 0, FEASIBLE_RESULT == 1 + } + } else if (benchres.result.cSize == winnerResult->cSize) { //equal ratio, needs to be better than winner in cSpeed/ dSpeed / cMem + int feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); + if(feas == 0) { // uncertain feasibility + adv.loopMode = BMK_timeMode; + if(loopDurationC < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.cSpeed = benchres2.result.cSpeed; + } + } + if(loopDurationD < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + + *resultPtr = benchres.result; + return feasible(benchres.result, target) && objective_lt(*winnerResult, benchres.result); + } else if (feas == 1) { //no need to check feasibility compares (maybe only it is chosen as a winner) + int btw = uncertainComparison(uncertaintyConstantC, uncertaintyConstantD, &(benchres.result), winnerResult); + if(btw == -1) { + return INFEASIBLE_RESULT; + } else { //possibly better, benchmark and find out + adv.loopMode = BMK_timeMode; + benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + *resultPtr = benchres.result; + return objective_lt(*winnerResult, benchres.result); + } + } else { //feas == -1 + return INFEASIBLE_RESULT; //infeasible + } + } else { + return INFEASIBLE_RESULT; //infeasible + } + } else { + return ERROR_RESULT; //BMK error + } + +} +//sameas before, but +/-? +//alternative, just return comparison result, leave caller to worry about feasibility. +//have version of benchMemAdvanced which takes in dstBuffer/cap as well? +//(motivation: repeat tests (maybe just on decompress) don't need further compress runs) +static int infeasibleBench(BMK_result_t* resultPtr, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult) { + BMK_advancedParams_t adv = BMK_initAdvancedParams(); + BMK_return_t benchres; + BMK_result_t resultMin, resultMax; + UTIL_time_t startTime; + U64 loopDurationC = 0, loopDurationD = 0; + double uncertaintyConstantC, uncertaintyConstantD; + double winnerRS = resultScore(*winnerResult, srcSize, target); + adv.loopMode = BMK_iterMode; //can only use this for ratio measurement then, super inaccurate timing + adv.nbSeconds = 1; //get ratio and 2x approx speed? //maybe run until twice MIN(minloopinterval * clockDuration) + + (void)startTime; //TODO: actually use this to adjust timing + DISPLAY("WinnerScore: %f\n ", winnerRS); + /* + adv.loopMode = BMK_timeMode; + adv.nbSeconds = 1; */ + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + + if(!benchres.error) { + *resultPtr = benchres.result; + if(eqZero(benchres.result.cSpeed)) { + loopDurationC = 0; + uncertaintyConstantC = 2; + } else { + loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? + //possibly has to do with initCCtx? or system stuff? + uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC), 2); //.02 seconds + } + + if(eqZero(benchres.result.dSpeed)) { + loopDurationD = 0; + uncertaintyConstantD = 2; + } else { + loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? + //possibly has to do with initCCtx? or system stuff? + uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD), 2); //.02 seconds + } + + /* benchres's certainty range. */ + resultMax = benchres.result; + resultMin = benchres.result; + resultMax.cSpeed *= uncertaintyConstantC; + resultMax.dSpeed *= uncertaintyConstantD; + resultMin.cSpeed /= uncertaintyConstantC; + resultMin.dSpeed /= uncertaintyConstantD; + (void)resultMin; + //TODO: consider if resultMin is actually needed. + if (winnerRS > resultScore(resultMax, srcSize, target)) { + return INFEASIBLE_RESULT; + } else { + //do this w/o copying / stuff + adv.loopMode = BMK_timeMode; + if(loopDurationC < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.cSpeed = benchres2.result.cSpeed; + } + } + if(loopDurationD < TIMELOOP_NANOSEC) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + //TODO: dstBuffer corrupted sometime between top and now + //probably occuring in feasible bench too. + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + *resultPtr = benchres.result; + return (resultScore(benchres.result, srcSize, target) > winnerRS); + } + + *resultPtr = benchres.result; + } else { + return ERROR_RESULT; //BMK error + } + +} + +/* wrap feasibleBench w/ memotable */ +//TODO: void sanitized and unsanitized ver's so input doesn't double-choose +#define INFEASIBLE_THRESHOLD 200 +static int feasibleBenchMemo(BMK_result_t* resultPtr, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult, U8* memoTable, + U32* varyParams, const int varyLen) { + + size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); + if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { + return INFEASIBLE_RESULT; //probably pick a different code for already tested? + //maybe remove this if we incorporate nonrandom location picking? + //what is the intended behavior in this case? + //ignore? stop iterating completely? other? + } else { + int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, + cParams, target, winnerResult); + memoTable[memind] = 255; //tested are all infeasible (other possible values for opti) + return res; + } +} + +//should infeasible stage searching also be memo-marked in the same way? +//don't actually memoize unless result is feasible/error? +static int infeasibleBenchMemo(BMK_result_t* resultPtr, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult, U8* memoTable, + U32* varyParams, const int varyLen) { + size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); + if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { + return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns + } else { + int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, + cParams, target, winnerResult); + if(res == FEASIBLE_RESULT) { + memoTable[memind] = 255; //infeasible resultscores could still be normal feasible. + } + return res; + } +} + +/* specifically feasibleBenchMemo and infeasibleBenchMemo */ +//maybe not necessary +typedef int (*BMK_benchMemo_t)(BMK_result_t*, const void*, size_t, void*, size_t, ZSTD_CCtx*, ZSTD_DCtx*, + const ZSTD_compressionParameters, const constraint_t, BMK_result_t*, U8*, U32*, const int); + +//varArray should be sanitized when this is called. +//TODO: transition to simpler greedy method if evaluation time is too long? +//would it be better to start at best feasible via feasible or infeasible metric? both? +//possibility climb is infeasible, responsibility of caller to check that. but if something feasible is evaluated, it will be returned +// *actually if it performs too +//sanitize all params here. +//all generation after random should be sanitized. (maybe sanitize random) +//TODO: paramTarget uneeded at this point w/ varArray and init; +static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varLen, U8* memoTable, + const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { + //pick later initializations non-randomly? high dist from explored nodes. + //how to do this efficiently? (might not be too much of a problem, happens rarely, running time probably dominated by benchmarking) + //distance maximizing selection? + //cparam - currently considered center + //candidate - params to benchmark/results + //winner - best option found so far. + ZSTD_compressionParameters cparam = init; + winnerInfo_t candidateInfo, winnerInfo; + int better = 1; + + winnerInfo.params = init; + winnerInfo.result.cSpeed = 0; + winnerInfo.result.dSpeed = 0; + winnerInfo.result.cMem = (size_t)-1; + winnerInfo.result.cSize = (size_t)-1; + + /* ineasible -> (hopefully) feasible */ + /* when nothing is found, this garbages part 2. */ + { + //TODO: initialize these values! + winnerInfo_t bestFeasible1; /* uses feasibleBench Metric */ + winnerInfo_t bestFeasible2; /* uses resultScore Metric */ + + //init these params + bestFeasible1.params = cparam; + bestFeasible2.params = cparam; + bestFeasible1.result.cSpeed = 0; + bestFeasible1.result.dSpeed = 0; + bestFeasible1.result.cMem = (size_t)-1; + bestFeasible1.result.cSize = (size_t)-1; + bestFeasible2.result.cSpeed = 0; + bestFeasible2.result.dSpeed = 0; + bestFeasible2.result.cMem = (size_t)-1; + bestFeasible2.result.cSize = (size_t)-1; + DISPLAY("Climb Part 1\n"); + while(better) { + + //UTIL_time_t timestart = UTIL_getTime(); TODO: adjust sampling based on time + int i, d; + better = 0; + DISPLAY("Start\n"); + cparam = winnerInfo.params; + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + candidateInfo.params = cparam; + //all dist-1 targets + for(i = 0; i < varLen; i++) { + paramVaryOnce(varArray[i], 1, &candidateInfo.params); /* +1 */ + candidateInfo.params = sanitizeParams(candidateInfo.params); + //evaluate + //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + if(cParamValid(candidateInfo.params)) { + int res = infeasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { /* synonymous with better when called w/ infeasibleBM */ + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + if(feasible(candidateInfo.result, target)) { + bestFeasible2 = winnerInfo; + if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { + bestFeasible1 = bestFeasible2; /* using feasibleBench metric */ + } + } + } + } + candidateInfo.params = cparam; + paramVaryOnce(varArray[i], -1, &candidateInfo.params); /* -1 */ + candidateInfo.params = sanitizeParams(candidateInfo.params); + //evaluate + //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + if(cParamValid(candidateInfo.params)) { + int res = infeasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + if(feasible(candidateInfo.result, target)) { + bestFeasible2 = winnerInfo; + if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { + bestFeasible1 = bestFeasible2; + } + } + } + } + } + + if(better) { + continue; + } + //if 'better' enough, skip further parameter search, center there? + //possible improvement - guide direction here w/ knowledge rather than completely random variation. + for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ + for(i = 0; i < 5; i++) { //make ? relative to # of free dimensions. + int res; + candidateInfo.params = cparam; + /* param error checking already done here */ + paramVariation(&candidateInfo.params, varArray, varLen, d); + res = infeasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { /* synonymous with better in this case*/ + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + if(feasible(candidateInfo.result, target)) { + bestFeasible2 = winnerInfo; + if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { + bestFeasible1 = bestFeasible2; + } + } + } + + } + if(better) { + continue; + } + } + //bias to test previous delta? + //change cparam -> candidate before restart + } + //TODO:Consider if this is best config. idea: explore from obj best keep rbest + cparam = bestFeasible2.params; + candidateInfo = bestFeasible2; + winnerInfo = bestFeasible1; + } + + //is it better to break here instead of bumbling about? + if(winnerInfo.result.cMem == (U32)-1) { + DISPLAY("No Feasible Found\n"); + return winnerInfo; + } + DISPLAY("Climb Part 2\n"); + + better = 1; + /* feasible -> best feasible (hopefully) */ + { + while(better) { + + //UTIL_time_t timestart = UTIL_getTime(); //TODO: if benchmarking is taking too long, be more greedy. + int i, d; + better = 0; + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + //all dist-1 targets + cparam = winnerInfo.params; //TODO: this messes the taking bestFeasible1, bestFeasible2 + candidateInfo.params = cparam; + for(i = 0; i < varLen; i++) { + paramVaryOnce(varArray[i], 1, &candidateInfo.params); + candidateInfo.params = sanitizeParams(candidateInfo.params); + + //evaluate + //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + if(cParamValid(candidateInfo.params)) { + int res = feasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + } + } + candidateInfo.params = cparam; + paramVaryOnce(varArray[i], -1, &candidateInfo.params); + candidateInfo.params = sanitizeParams(candidateInfo.params); + //evaluate + //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + if(cParamValid(candidateInfo.params)) { + int res = feasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + } + } + } + //if 'better' enough, skip further parameter search, center there? + //possible improvement - guide direction here w/ knowledge rather than completely random variation. + for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ + for(i = 0; i < 5; i++) { //TODO: make ? relative to # of free dimensions. + int res; + candidateInfo.params = cparam; + /* param error checking already done here */ + paramVariation(&candidateInfo.params, varArray, varLen, d); //info candidateInfo.params is garbage, this is too. + res = feasibleBenchMemo(&candidateInfo.result, + srcBuffer, srcSize, + dstBuffer, dstSize, + ctx, dctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen); + if(res == FEASIBLE_RESULT) { + winnerInfo = candidateInfo; + //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + better = 1; + } + } + if(better) { + continue; + } + } + //bias to test previous delta? + //change cparam -> candidate before restart + } + } + return winnerInfo; +} + +//optimizeForSize but with fixed strategy +//place to configure/filter out strategy specific parameters. +//need args for all buffers and parameter stuff +//sanitization here. + +//flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) +//weight more on visit for bad results, less on good results/more on later results / ones with more failures. +//allocate memoTable here. +//only real use for paramTarget is to get the fixed values, right? +static winnerInfo_t optimizeFixedStrategy( + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, size_t dstSize, + constraint_t target, ZSTD_compressionParameters paramTarget, + ZSTD_strategy strat, U32* varArray, int varLen) { + int i = 0; //TODO: Temp fix 10 iters, check effects of changing this? + U32* varNew = malloc(sizeof(U32) * varLen); + int varLenNew = sanitizeVarArray(varLen, varArray, varNew, strat); + size_t memoLen = memoTableLen(varNew, varLenNew); + U8* memoTable = malloc(sizeof(U8) * memoLen); + ZSTD_compressionParameters init; + ZSTD_CCtx* ctx = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + winnerInfo_t winnerInfo, candidateInfo; + winnerInfo.result.cSpeed = 0; + winnerInfo.result.dSpeed = 0; + winnerInfo.result.cMem = (size_t)(-1); + winnerInfo.result.cSize = (size_t)(-1); + /* so climb is given the right fixed strategy */ + paramTarget.strategy = strat; + /* to pass ZSTD_checkCParams */ + cParamZeroMin(¶mTarget); + memoTableInit(memoTable, paramTarget, target, varNew, varLenNew); + + + init = paramTarget; + + + if(!ctx || !dctx || !memoTable || !varNew) { + DISPLAY("NOT ENOUGH MEMORY ! ! ! \n"); + goto _cleanUp; + } + + while(i < 10) { + DISPLAY("Restart\n"); + randomConstrainedParams(&init, varNew, varLenNew, memoTable); + candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, init); + if(objective_lt(winnerInfo.result, candidateInfo.result)) { + winnerInfo = candidateInfo; + DISPLAY("Climb Winner: "); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + } + i++; + } + +_cleanUp: + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + free(memoTable); + free(varNew); + return winnerInfo; +} + +// bigger and (hopefully) better* than optimizeForSize +// TODO: Change level bm'ing to respect constraints. +static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) +{ + FILE* const inFile = fopen( inFileName, "rb" ); + U64 const inFileSize = UTIL_getFileSize(inFileName); + size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; + void* origBuff; + U32 varArray [NUM_PARAMS]; + int varLen = variableParams(paramTarget, varArray); + /* Init */ + + + if(!cParamValid(paramTarget)) { + return 10; + } + + if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } + if (inFileSize == UTIL_FILESIZE_UNKNOWN) { + DISPLAY("Pb evaluatin size of %s \n", inFileName); + fclose(inFile); + return 11; + } + + /* Memory allocation & restrictions */ + if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; + if (benchedSize < inFileSize) { + DISPLAY("Not enough memory for '%s' \n", inFileName); + fclose(inFile); + return 11; + } + + /* Alloc */ + origBuff = malloc(benchedSize); + if(!origBuff) { + DISPLAY("\nError: not enough memory!\n"); + fclose(inFile); + return 12; + } + + /* Fill input buffer */ + DISPLAY("Loading %s... \r", inFileName); + { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); + fclose(inFile); + if(readSize != benchedSize) { + DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + free(origBuff); + return 13; + } } + + /* bench */ + DISPLAY("\r%79s\r", ""); + DISPLAY("optimizing for %s", inFileName); + if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } + if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } + if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } + DISPLAY("\n"); + findClockGranularity(); + + { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + winnerInfo_t winner; + //BMK_result_t candidate; + const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; + U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1; + const size_t maxCompressedSize = ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024); + void* compressedBuffer = malloc(maxCompressedSize); + + /* init */ + if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} + if(compressedBuffer==NULL) { DISPLAY("\n Allocation Error \n"); free(origBuff); free(ctx); return 15; } + + memset(&winner, 0, sizeof(winner)); + winner.result.cSize = (size_t)(-1); + + + /* find best solution from default params */ + //Can't do this w/ cparameter constraints + //still useful though? + /* + { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + int i; + for (i=1; i<=maxSeeds; i++) { + ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); + BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams); + if (!feasible(candidate, target) ) { + break; + } + if (feasible(candidate,target) && objective_lt(winner.result, candidate)) + { + winner.params = CParams; + winner.result = candidate; + BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); + } } + }*/ + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); + + BMK_translateAdvancedParams(winner.params); + + /* start real tests */ + { + if(paramTarget.strategy == 0) { + int st; + for(st = 1; st <= 8; st++) { + winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, + target, paramTarget, st, varArray, varLen); + DISPLAY("StratNum %d\n", st); + if(objective_lt(winner.result, wc.result)) { + winner = wc; + } + } + } else { + winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, + target, paramTarget, paramTarget.strategy, varArray, varLen); + } + + } + + /* no solution found */ + if(winner.result.cSize == (size_t)-1) { + DISPLAY("No feasible solution found\n"); + return 1; + } + /* end summary */ + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); + BMK_translateAdvancedParams(winner.params); + DISPLAY("grillParams size - optimizer completed \n"); + + /* clean up*/ + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + } + + free(origBuff); + return 0; +} + + /* optimizeForSize(): * targetSpeed : expressed in B/s */ -/* if state space is small (from paramTarget) */ +/* expresses targeted compression, decompression speeds and memory requirements */ +/* if state space is small (from paramTarget), exhaustive search? */ +//things to consider : if doing strategy-separate approach, what cutoffs to evaluate each strategy +//or do all? can't be absolute, should be relative after some sort of calibration +//(synthetic? test levels (we don't care about data specifics rn, scale?) ? int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { FILE* const inFile = fopen( inFileName, "rb" ); @@ -872,7 +1905,7 @@ int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressio DISPLAY("optimizing for %s", inFileName); if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } - if(target.Mem != 0) { DISPLAY(" - limit memory %u MB", target.Mem / 1000000); } + if(target.cMem != 0) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } DISPLAY("\n"); { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); @@ -881,12 +1914,13 @@ int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressio const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; /* init */ - if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} + if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14; } + memset(&winner, 0, sizeof(winner)); winner.result.cSize = (size_t)(-1); /* find best solution from default params */ - //Can't do this iteration normally w/ cparameter constraints + //Can't do this w/ cparameter constraints { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); int i; for (i=1; i<=maxSeeds; i++) { @@ -910,15 +1944,17 @@ int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressio { time_t const grillStart = time(NULL); do { ZSTD_compressionParameters params = winner.params; - paramVariation(¶ms, paramVarArray, paramCount); + BYTE* b; + paramVariation(¶ms, paramVarArray, paramCount, 4); if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */ params = ZSTD_adjustCParams(params, blockSize, 0); /* exclude faster if already played set of params */ - if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue; + if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(params))-1)) continue; /* test */ - NB_TESTS_PLAYED(params)++; + b = NB_TESTS_PLAYED(params); + (*b)++; BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, params); /* improvement found => new winner */ @@ -1027,7 +2063,8 @@ int main(int argc, const char** argv) U32 optimizer = 0; U32 main_pause = 0; - constraint_t target = { 0 , 0, 0 }; //0 for anything unset + + constraint_t target = { 0, 0, (U32)-1 }; //0 for anything unset ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; assert(argc>=1); /* for exename */ @@ -1053,7 +2090,7 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { paramTarget.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.Mem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.cMem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } /* in MB or MB/s */ DISPLAY("invalid optimization parameter \n"); return 1; @@ -1132,7 +2169,7 @@ int main(int argc, const char** argv) continue; case 'M': argument++; - target.Mem = readU32FromChar(&argument) * 1000000; + target.cMem = readU32FromChar(&argument) * 1000000; continue; case 'w': argument++; @@ -1255,7 +2292,8 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize(input_filename, target, paramTarget); + result = optimizeForSize2(input_filename, target, paramTarget); + //optimizeForSize(input_filename, target, paramTarget); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From eb21b7f48224b35ceb91b31a2fe1bb764545724e Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 9 Jul 2018 13:44:01 -0700 Subject: [PATCH 27/92] Not crashing --- programs/bench.c | 10 ++++---- tests/paramgrill.c | 59 +++++++++++++++++++++++++++------------------- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 0d35f3eb3..5d5b47aba 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -288,7 +288,7 @@ volatile char g_touched; /* takes # of blocks and list of size & stuff for each. */ /* only does looping */ /* note time per loop could be zero if interval too short */ -BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( +BMK_customReturn_t BMK_benchFunction( BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, @@ -310,6 +310,7 @@ BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( } { + unsigned i, j; for(i = 0; i < blockCount; i++) { for(j = 0; j < srcBlockSizes[i]; j++) { @@ -318,11 +319,11 @@ BMK_customReturn_t __attribute__((optimize("O0"))) BMK_benchFunction( } for(i = 0; i < blockCount; i++) { memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ - //this is written at end proc? where did compressed data get overwritten ny this? } - UTIL_sleepMilli(5); /* give processor time to other processes */ - UTIL_waitForNextTick(); + //UTIL_sleepMilli(5); /* give processor time to other processes */ + //UTIL_waitForNextTick(); + } { @@ -463,7 +464,6 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } { size_t const decodedSize = (size_t)totalDSize64; free(*resultBufferPtr); - //TODO: decodedSize corrupted? *resultBufferPtr = malloc(decodedSize); if (!(*resultBufferPtr)) { EXM_THROW(33, BMK_return_t, "not enough memory"); diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 44bcedef8..77e9f2250 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -598,8 +598,8 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParam const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } - //validated = !ZSTD_isError(ZSTD_checkCParams(p)); - validated = cParamValid(p); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + //validated = cParamValid(p); //Make sure memory is at least close to feasible? //ZSTD_estimateCCtxSize thing. @@ -669,23 +669,27 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const U32* varyPara //initializing memoTable /* */ -static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen) { +static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog; + int wFixed = !paramConstraints.windowLog; int j = 0; memset(memoTable, 0, arrayLen); - + cParamZeroMin(¶mConstraints); for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - BMK_translateAdvancedParams(paramConstraints); if(ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1 << paramConstraints.windowLog) > target.cMem) { //infeasible; memoTable[i] = 255; j++; } + //TODO: remove any where memoTable wlog is mark any where windowlog is too big for data. + if(wFixed && (1 << paramConstraints.windowLog) > (srcSize << 1)) { + memoTable[i] = 255; + } /* nil out parameter sets equivalent to others. */ if(cwFixed/* at most least 1 param fixed. */) { if(paramConstraints.strategy == ZSTD_btlazy2 || paramConstraints.strategy == ZSTD_btopt || paramConstraints.strategy == ZSTD_btultra) { @@ -771,8 +775,8 @@ static ZSTD_compressionParameters randomParams(void) p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; p.targetLength=(FUZ_rand(&g_rand) % (512)); p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); - //validated = !ZSTD_isError(ZSTD_checkCParams(p)); - validated = cParamValid(p); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + //validated = cParamValid(p); } return p; } @@ -1201,7 +1205,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1230,7 +1234,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1275,14 +1279,12 @@ static int infeasibleBench(BMK_result_t* resultPtr, BMK_advancedParams_t adv = BMK_initAdvancedParams(); BMK_return_t benchres; BMK_result_t resultMin, resultMax; - UTIL_time_t startTime; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; double winnerRS = resultScore(*winnerResult, srcSize, target); adv.loopMode = BMK_iterMode; //can only use this for ratio measurement then, super inaccurate timing adv.nbSeconds = 1; //get ratio and 2x approx speed? //maybe run until twice MIN(minloopinterval * clockDuration) - (void)startTime; //TODO: actually use this to adjust timing DISPLAY("WinnerScore: %f\n ", winnerRS); /* adv.loopMode = BMK_timeMode; @@ -1290,6 +1292,11 @@ static int infeasibleBench(BMK_result_t* resultPtr, benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + adv.loopMode = BMK_timeMode; + adv.nbSeconds = 1; + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + if(!benchres.error) { *resultPtr = benchres.result; if(eqZero(benchres.result.cSpeed)) { @@ -1372,6 +1379,7 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, U32* varyParams, const int varyLen) { size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //probably pick a different code for already tested? @@ -1397,6 +1405,7 @@ static int infeasibleBenchMemo(BMK_result_t* resultPtr, BMK_result_t* winnerResult, U8* memoTable, U32* varyParams, const int varyLen) { size_t memind = memoTableInd(&cParams, varyParams, varyLen); + //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns @@ -1422,7 +1431,6 @@ typedef int (*BMK_benchMemo_t)(BMK_result_t*, const void*, size_t, void*, size_t // *actually if it performs too //sanitize all params here. //all generation after random should be sanitized. (maybe sanitize random) -//TODO: paramTarget uneeded at this point w/ varArray and init; static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varLen, U8* memoTable, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { //pick later initializations non-randomly? high dist from explored nodes. @@ -1470,12 +1478,13 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); candidateInfo.params = cparam; //all dist-1 targets + //if we early end this, we should also randomize the order these are picked. for(i = 0; i < varLen; i++) { paramVaryOnce(varArray[i], 1, &candidateInfo.params); /* +1 */ candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1498,8 +1507,8 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL paramVaryOnce(varArray[i], -1, &candidateInfo.params); /* -1 */ candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1587,8 +1596,8 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1605,8 +1614,8 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL paramVaryOnce(varArray[i], -1, &candidateInfo.params); candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate - //if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - if(cParamValid(candidateInfo.params)) { + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + //if(cParamValid(candidateInfo.params)) { int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, @@ -1681,9 +1690,11 @@ static winnerInfo_t optimizeFixedStrategy( /* so climb is given the right fixed strategy */ paramTarget.strategy = strat; /* to pass ZSTD_checkCParams */ - cParamZeroMin(¶mTarget); - memoTableInit(memoTable, paramTarget, target, varNew, varLenNew); + memoTableInit(memoTable, paramTarget, target, varNew, varLenNew, srcSize); + + //needs to happen after memoTableInit as that assumes 0 = undefined. + cParamZeroMin(¶mTarget); init = paramTarget; @@ -1699,7 +1710,7 @@ static winnerInfo_t optimizeFixedStrategy( candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, init); if(objective_lt(winnerInfo.result, candidateInfo.result)) { winnerInfo = candidateInfo; - DISPLAY("Climb Winner: "); + DISPLAY("New Winner: "); BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); } i++; @@ -1714,7 +1725,7 @@ _cleanUp: } // bigger and (hopefully) better* than optimizeForSize -// TODO: Change level bm'ing to respect constraints. +// TODO: allow accept multiple files like benchFiles or bench.c fn's static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { FILE* const inFile = fopen( inFileName, "rb" ); From fab44388014a8fbb3cf4ba9f46cfdc241807299d Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 9 Jul 2018 18:37:54 -0700 Subject: [PATCH 28/92] Dictionary + Multiple file Loading --- tests/paramgrill.c | 209 ++++++++++++++++++++++++++++++++------------- 1 file changed, 151 insertions(+), 58 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 77e9f2250..2ad8c5752 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -114,6 +114,7 @@ static U32 g_singleRun = 0; static U32 g_target = 0; static U32 g_noSeed = 0; static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy }; +static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ void BMK_SetNbIterations(int nbLoops) { @@ -1141,6 +1142,7 @@ static double calcUncertainty(double speed, size_t srcSize) { static int feasibleBench(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1155,7 +1157,7 @@ static int feasibleBench(BMK_result_t* resultPtr, //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), //maybe iter this until 2x measurable for better guarantee? DISPLAY("Feas:\n"); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres.error) { DISPLAY("ERROR %d !!\n", benchres.error); } @@ -1195,7 +1197,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1205,7 +1207,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1224,7 +1226,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1234,7 +1236,7 @@ static int feasibleBench(BMK_result_t* resultPtr, if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1250,7 +1252,7 @@ static int feasibleBench(BMK_result_t* resultPtr, return INFEASIBLE_RESULT; } else { //possibly better, benchmark and find out adv.loopMode = BMK_timeMode; - benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); *resultPtr = benchres.result; return objective_lt(*winnerResult, benchres.result); } @@ -1272,6 +1274,7 @@ static int feasibleBench(BMK_result_t* resultPtr, static int infeasibleBench(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1289,12 +1292,14 @@ static int infeasibleBench(BMK_result_t* resultPtr, /* adv.loopMode = BMK_timeMode; adv.nbSeconds = 1; */ - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); adv.loopMode = BMK_timeMode; adv.nbSeconds = 1; - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); if(!benchres.error) { @@ -1336,7 +1341,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1348,7 +1353,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, adv.mode = BMK_decodeOnly; //TODO: dstBuffer corrupted sometime between top and now //probably occuring in feasible bench too. - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1372,6 +1377,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, static int feasibleBenchMemo(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1387,7 +1393,7 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, //what is the intended behavior in this case? //ignore? stop iterating completely? other? } else { - int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, + int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, cParams, target, winnerResult); memoTable[memind] = 255; //tested are all infeasible (other possible values for opti) return res; @@ -1399,6 +1405,7 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, static int infeasibleBenchMemo(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1410,7 +1417,7 @@ static int infeasibleBenchMemo(BMK_result_t* resultPtr, if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns } else { - int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, + int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, cParams, target, winnerResult); if(res == FEASIBLE_RESULT) { memoTable[memind] = 255; //infeasible resultscores could still be normal feasible. @@ -1432,7 +1439,7 @@ typedef int (*BMK_benchMemo_t)(BMK_result_t*, const void*, size_t, void*, size_t //sanitize all params here. //all generation after random should be sanitized. (maybe sanitize random) static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varLen, U8* memoTable, - const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { + const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { //pick later initializations non-randomly? high dist from explored nodes. //how to do this efficiently? (might not be too much of a problem, happens rarely, running time probably dominated by benchmarking) //distance maximizing selection? @@ -1488,6 +1495,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1512,6 +1520,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL int res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1519,7 +1528,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL winnerInfo = candidateInfo; //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; - if(feasible(candidateInfo.result, target)) { + if(feasible(candidateInfo.result, target)) { //TODO: maybe just break here and move on to part 2? bestFeasible2 = winnerInfo; if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { bestFeasible1 = bestFeasible2; @@ -1535,14 +1544,15 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL //if 'better' enough, skip further parameter search, center there? //possible improvement - guide direction here w/ knowledge rather than completely random variation. for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ - for(i = 0; i < 5; i++) { //make ? relative to # of free dimensions. + for(i = 0; i < 2 * varLen + 2; i++) { int res; candidateInfo.params = cparam; /* param error checking already done here */ paramVariation(&candidateInfo.params, varArray, varLen, d); res = infeasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, - dstBuffer, dstSize, + dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1601,6 +1611,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1619,6 +1630,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1632,7 +1644,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL //if 'better' enough, skip further parameter search, center there? //possible improvement - guide direction here w/ knowledge rather than completely random variation. for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ - for(i = 0; i < 5; i++) { //TODO: make ? relative to # of free dimensions. + for(i = 0; i < 2 * varLen + 2; i++) { int res; candidateInfo.params = cparam; /* param error checking already done here */ @@ -1640,6 +1652,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, + dictBuffer, dictSize, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); @@ -1672,6 +1685,7 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL static winnerInfo_t optimizeFixedStrategy( const void* srcBuffer, const size_t srcSize, void* dstBuffer, size_t dstSize, + void* dictBuffer, size_t dictSize, constraint_t target, ZSTD_compressionParameters paramTarget, ZSTD_strategy strat, U32* varArray, int varLen) { int i = 0; //TODO: Temp fix 10 iters, check effects of changing this? @@ -1706,12 +1720,14 @@ static winnerInfo_t optimizeFixedStrategy( while(i < 10) { DISPLAY("Restart\n"); + //TODO: look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); - candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, ctx, dctx, init); + candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, init); if(objective_lt(winnerInfo.result, candidateInfo.result)) { winnerInfo = candidateInfo; DISPLAY("New Winner: "); BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + i = 0; } i++; } @@ -1724,59 +1740,125 @@ _cleanUp: return winnerInfo; } +static int BMK_loadFiles(void* buffer, size_t bufferSize, + size_t* fileSizes, const char* const * const fileNamesTable, + unsigned nbFiles) +{ + size_t pos = 0, totalSize = 0; + unsigned n; + for (n=0; n bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ + { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + if (readSize != (size_t)fileSize) { + DISPLAY("could not read %s", fileNamesTable[n]); + return 11; + } + pos += readSize; } + fileSizes[n] = (size_t)fileSize; + totalSize += (size_t)fileSize; + fclose(f); + } + + if (totalSize == 0) { DISPLAY("no data to bench\n"); return 12; } + return 0; +} + // bigger and (hopefully) better* than optimizeForSize // TODO: allow accept multiple files like benchFiles or bench.c fn's -static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) +static int optimizeForSize2(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { - FILE* const inFile = fopen( inFileName, "rb" ); - U64 const inFileSize = UTIL_getFileSize(inFileName); - size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; + size_t benchedSize; void* origBuff; + void* dictBuffer; + size_t dictBufferSize; U32 varArray [NUM_PARAMS]; int varLen = variableParams(paramTarget, varArray); /* Init */ - - if(!cParamValid(paramTarget)) { return 10; } - if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } - if (inFileSize == UTIL_FILESIZE_UNKNOWN) { - DISPLAY("Pb evaluatin size of %s \n", inFileName); - fclose(inFile); - return 11; - } - - /* Memory allocation & restrictions */ - if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) { - DISPLAY("Not enough memory for '%s' \n", inFileName); - fclose(inFile); - return 11; - } - - /* Alloc */ - origBuff = malloc(benchedSize); - if(!origBuff) { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - return 12; + /* load dictionary*/ + if (dictFileName != NULL) { + U64 const dictFileSize = UTIL_getFileSize(dictFileName); + if (dictFileSize > 64 MB) { + DISPLAY("dictionary file %s too large", dictFileName); + return 10; + } + dictBufferSize = (size_t)dictFileSize; + dictBuffer = malloc(dictBufferSize); + if (dictBuffer==NULL) { + DISPLAY("not enough memory for dictionary (%u bytes)", + (U32)dictBufferSize); + return 11; + } + { + int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, &dictBufferSize, &dictFileName, 1); + if(errorCode) { + free(dictBuffer); + return errorCode; + } + } } /* Fill input buffer */ - DISPLAY("Loading %s... \r", inFileName); - { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - if(readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + if(nbFiles == 1) { + DISPLAY("Loading %s... \r", fileNamesTable[0]); + } else { + DISPLAY("Loading %zd Files... \r", nbFiles); + } + + { + U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); + int ec; + size_t* fileSizes = calloc(sizeof(size_t),nbFiles); + benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; + origBuff = malloc(benchedSize); + if(!origBuff || !fileSizes) { + DISPLAY("Not enough memory for stuff\n"); free(origBuff); - return 13; - } } + free(fileSizes); + free(dictBuffer); + return 1; + } + ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, nbFiles); + if(ec) { + DISPLAY("Error Loading Files"); + free(origBuff); + free(fileSizes); + free(dictBuffer); + return ec; + } + free(fileSizes); + } + + /* bench */ DISPLAY("\r%79s\r", ""); - DISPLAY("optimizing for %s", inFileName); + if(nbFiles == 1) { + DISPLAY("optimizing for %s", fileNamesTable[0]); + } else { + DISPLAY("optimizing for %zd Files", nbFiles); + } if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } @@ -1828,7 +1910,7 @@ static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_co if(paramTarget.strategy == 0) { int st; for(st = 1; st <= 8; st++) { - winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, + winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, target, paramTarget, st, varArray, varLen); DISPLAY("StratNum %d\n", st); if(objective_lt(winner.result, wc.result)) { @@ -1836,7 +1918,7 @@ static int optimizeForSize2(const char* inFileName, constraint_t target, ZSTD_co } } } else { - winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, + winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, target, paramTarget, paramTarget.strategy, varArray, varLen); } @@ -2070,7 +2152,8 @@ int main(int argc, const char** argv) filenamesStart=0, result; const char* exename=argv[0]; - const char* input_filename=0; + const char* input_filename = 0; + const char* dictFileName = 0; U32 optimizer = 0; U32 main_pause = 0; @@ -2283,6 +2366,17 @@ int main(int argc, const char** argv) g_grillDuration_s = (double)readU32FromChar(&argument); break; + /* load dictionary file (only applicable for optimizer rn) */ + case 'D': + if(i == argc - 1) { //last argument, return error. + DISPLAY("Dictionary file expected but not given\n"); + return 1; + } else { + i++; + dictFileName = argv[i]; + } + break; + /* Unknown command */ default : return badusage(exename); } @@ -2303,8 +2397,7 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize2(input_filename, target, paramTarget); - //optimizeForSize(input_filename, target, paramTarget); + result = optimizeForSize2(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From 3adc217ea47f93f11e32ca6d3bf2f103ff3dbbd0 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 12 Jul 2018 17:30:39 -0700 Subject: [PATCH 29/92] Total Changes: Add different constraint types (decompression speed, compression memory, parameter constraints) Separate search space by strategy + strategy selection Memoize results Real random restarts Support multiple files Support Dictionary inputs Debug Macro for extra printing --- programs/bench.c | 14 +- programs/bench.h | 4 +- tests/Makefile | 2 +- tests/fullbench.c | 4 +- tests/paramgrill.c | 1090 ++++++++++++++++++++++---------------------- 5 files changed, 548 insertions(+), 566 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 5d5b47aba..f5184fa8f 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -85,7 +85,7 @@ static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; * Exceptions ***************************************/ #ifndef DEBUG -# define DEBUG 1 +# define DEBUG 0 #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } @@ -188,7 +188,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy , comprParams->strategy); ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); } @@ -293,7 +293,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, + void** const dstBlockBuffers, size_t* dstBlockCapacities, unsigned nbLoops) { size_t srcSize = 0, dstSize = 0, ind = 0; U64 totalTime; @@ -338,6 +338,11 @@ BMK_customReturn_t BMK_benchFunction( j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); } else if(firstIter) { dstSize += res; + //Make compressed blocks continuous + if(j != blockCount - 1) { + dstBlockBuffers[j+1] = (void*)((char*)dstBlockBuffers[j] + res); + dstBlockCapacities[j] = res; + } } } firstIter = 0; @@ -370,13 +375,14 @@ void BMK_freeTimeState(BMK_timedFnState_t* state) { free(state); } +/* make option for dstBlocks to be */ BMK_customTimedReturn_t BMK_benchFunctionTimed( BMK_timedFnState_t* cont, BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const* const srcBlockBuffers, const size_t* srcBlockSizes, - void* const* const dstBlockBuffers, const size_t* dstBlockCapacities) + void** const dstBlockBuffers, size_t* dstBlockCapacities) { U64 fastest = cont->fastestTime; int completed = 0; diff --git a/programs/bench.h b/programs/bench.h index 625b65757..1a298cc48 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -191,7 +191,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, - void* const * const dstBuffers, const size_t* dstCapacities, + void** const dstBuffers, size_t* dstCapacities, unsigned nbLoops); @@ -220,7 +220,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities); + void** const dstBlockBuffers, size_t* dstBlockCapacities); #endif /* BENCH_H_121279284357 */ diff --git a/tests/Makefile b/tests/Makefile index ecf7fe2a7..81e685780 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -200,7 +200,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) -#paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements +paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) diff --git a/tests/fullbench.c b/tests/fullbench.c index 7859745a3..9e7639f92 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -336,7 +336,7 @@ size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, void* dst, static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, ZSTD_compressionParameters* cparams) { BYTE* dstBuff; - size_t const dstBuffSize = ZSTD_compressBound(srcSize); + size_t dstBuffSize = ZSTD_compressBound(srcSize); void* buff2, *buff1; const char* benchName; BMK_benchFn_t benchFunction; @@ -516,7 +516,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, { r = BMK_benchFunction(benchFunction, buff2, NULL, NULL, 1, &src, &srcSize, - (void * const * const)&dstBuff, &dstBuffSize, g_nbIterations); + (void **)&dstBuff, &dstBuffSize, g_nbIterations); if(r.error) { DISPLAY("ERROR %d ! ! \n", r.error); errorcode = r.error; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 2ad8c5752..9af60a740 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -58,6 +58,11 @@ static const int g_maxNbVariations = 64; * Macros **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define TIMED 0 +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } #undef MIN #undef MAX @@ -81,8 +86,8 @@ static const int g_maxNbVariations = 64; #define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. //make 2^[0,10] w/ 999 -#define ZSTD_TARGETLENGTH_MIN 0 //actually targeLengthlog min -#define ZSTD_TARGETLENGTH_MAX 10 +#define ZSTD_TARGETLENGTH_MIN 0 +#define ZSTD_TARGETLENGTH_MAX 999 //#define ZSTD_TARGETLENGTH_MAX 1024 #define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1) @@ -90,15 +95,14 @@ static const int g_maxNbVariations = 64; #define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1) #define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1) #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) -#define TLEN_RANGE 11 +#define TLEN_RANGE 12 +//TLEN_RANGE = 0, 2^0 to 2^10; //hard coded since we only use powers of 2 (and 999 ~ 1024) -static const int mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN }; -static const int maxtable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX }; +//static const int mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN }; +//static const int maxtable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX }; static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; -//use grid-search or something when space is small enough? -#define SMALL_SEARCH_SPACE 1000 /*-************************************ * Benchmark Parameters **************************************/ @@ -201,7 +205,7 @@ static void findClockGranularity(void) { } } } while(i < 10); - DISPLAY("Granularity: %llu\n", (unsigned long long)g_clockGranularity); + DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity); } typedef struct { @@ -225,6 +229,10 @@ static int cParamValid(ZSTD_compressionParameters paramTarget) { CLAMPCHECK(paramTarget.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); CLAMPCHECK(paramTarget.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); CLAMPCHECK(paramTarget.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); + if(paramTarget.targetLength > ZSTD_TARGETLENGTH_MAX) { + DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); + return 0; + } if(paramTarget.strategy > ZSTD_btultra) { DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); return 0; @@ -232,21 +240,68 @@ static int cParamValid(ZSTD_compressionParameters paramTarget) { return 1; } +//TODO: let targetLength = 0; static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) { paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN; paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN; paramTarget->chainLog = paramTarget->chainLog ? paramTarget->chainLog : ZSTD_CHAINLOG_MIN; paramTarget->hashLog = paramTarget->hashLog ? paramTarget->hashLog : ZSTD_HASHLOG_MIN; paramTarget->searchLength = paramTarget->searchLength ? paramTarget->searchLength : ZSTD_SEARCHLENGTH_MIN; - paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 1; + paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 0; } -static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) +static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params) { DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); } +/* checks results are feasible */ +static int feasible(const BMK_result_t results, const constraint_t target) { + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); +} + +#define EPSILON 0.01 +static int epsilonEqual(const double c1, const double c2) { + return MAX(c1/c2,c2/c1) < 1 + EPSILON; +} + +/* checks exact equivalence to 0, to stop compiler complaining fpeq */ +static int eqZero(const double c1) { + return (U64)c1 == (U64)0.0 || (U64)c1 == (U64)-0.0; +} + +/* returns 1 if result2 is strictly 'better' than result1 */ +/* strict comparison / cutoff based */ +static int objective_lt(const BMK_result_t result1, const BMK_result_t result2) { + return (result1.cSize > result2.cSize) || (epsilonEqual(result1.cSize, result2.cSize) && result2.cSpeed > result1.cSpeed) + || (epsilonEqual(result1.cSize,result2.cSize) && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); +} + +/* hill climbing value for part 1 */ +static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) { + double cs = 0., ds = 0., rt, cm = 0.; + const double r1 = 1, r2 = 0.1, rtr = 0.5; + double ret; + if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; } + if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; } + if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; } + rt = ((double)srcSize / res.cSize); + + ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + + (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; + //DISPLAY("resultScore: %f\n", ret); + return ret; +} + +/* factor sort of arbitrary */ +static constraint_t relaxTarget(constraint_t target) { + target.cMem = (U32)-1; + target.cSpeed *= 0.9; + target.dSpeed *= 0.9; + return target; +} + /*-******************************************************* * Bench functions *********************************************************/ @@ -268,9 +323,21 @@ const char* g_stratName[ZSTD_btultra+1] = { "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -/* TODO: support additional parameters (more files, fileSizes) */ static size_t BMK_benchParam(BMK_result_t* resultPtr, + const void* srcBuffer, const size_t srcSize, + const size_t* fileSizes, const unsigned nbFiles, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams) { + + BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, fileSizes, nbFiles, 0, &cParams, NULL, 0, ctx, dctx, 0, "File"); + *resultPtr = res.result; + return res.error; +} + +/* benchParam but only takes in one file. */ +static size_t +BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams) { @@ -280,27 +347,54 @@ BMK_benchParam(BMK_result_t* resultPtr, return res.error; } -static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize) -{ - char lvlstr[15] = "Custom Level"; - DISPLAY("\r%79s\r", ""); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, - params.targetLength, g_stratName[(U32)(params.strategy)]); - if(cLevel != CUSTOM_LEVEL) { - snprintf(lvlstr, 15, " Level %2u ", cLevel); - } - fprintf(f, - "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); -} - - typedef struct { BMK_result_t result; ZSTD_compressionParameters params; } winnerInfo_t; +/* global winner used for display. */ +//Should be totally 0 initialized? +static winnerInfo_t g_winner; //TODO: ratio is infinite at initialization, instead of 0 +static constraint_t g_targetConstraints; + +static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) +{ + if(DEBUG || (objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints))) { + char lvlstr[15] = "Custom Level"; + const U64 time = UTIL_clockSpanNano(g_time); + const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); + if(DEBUG && (objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints))) { + DISPLAY("New Winner: \n"); + } + + DISPLAY("\r%79s\r", ""); + + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, + params.targetLength, g_stratName[(U32)(params.strategy)]); + if(cLevel != CUSTOM_LEVEL) { + snprintf(lvlstr, 15, " Level %2u ", cLevel); + } + fprintf(f, + "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", + lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); + + if(TIMED) { fprintf(f, " - %lu:%lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } + fprintf(f, "\n"); + if(objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints)) { + BMK_translateAdvancedParams(params); + g_winner.result = result; + g_winner.params = params; + } + } + //else { + // DISPLAY("G_WINNER: "); + // DISPLAY("/* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",(double)srcSize / g_winner.result.cSize , g_winner.result.cSpeed / 1000000 , g_winner.result.dSpeed / 1000000); + // DISPLAY("LOSER : "); + // DISPLAY("/* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",(double)srcSize / result.cSize, result.cSpeed / 1000000 , result.dSpeed / 1000000); + //} +} + static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) { int cLevel; @@ -358,7 +452,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para int better = 0; int cLevel; - BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, dctx, params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, ctx, dctx, params); for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { @@ -470,7 +564,7 @@ static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters para /* new length */ /* keep old array, will need if iter over strategy. */ -static int sanitizeVarArray(int varLength, U32* varArray, U32* varNew, ZSTD_strategy strat) { +static int sanitizeVarArray(const int varLength, const U32* varArray, U32* varNew, const ZSTD_strategy strat) { int i, j = 0; for(i = 0; i < varLength; i++) { if( !((varArray[i] == CLOG_IND && strat == ZSTD_fast) @@ -515,53 +609,10 @@ static int variableParams(const ZSTD_compressionParameters paramConstraints, U32 return j; } -/* computes inverse of above array, returns same number, -1 = unused ind */ -static int inverseVariableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { - int j = 0; - if(!paramConstraints.windowLog) { - res[WLOG_IND] = j; - j++; - } else { - res[WLOG_IND] = -1; - } - if(!paramConstraints.chainLog) { - res[j] = CLOG_IND; - j++; - } else { - res[CLOG_IND] = -1; - } - if(!paramConstraints.hashLog) { - res[j] = HLOG_IND; - j++; - } else { - res[HLOG_IND] = -1; - } - if(!paramConstraints.searchLog) { - res[j] = SLOG_IND; - j++; - } else { - res[SLOG_IND] = -1; - } - if(!paramConstraints.searchLength) { - res[j] = SLEN_IND; - j++; - } else { - res[SLEN_IND] = -1; - } - if(!paramConstraints.targetLength) { - res[j] = TLEN_IND; - j++; - } else { - res[TLEN_IND] = -1; - } - - return j; -} - /* amt will probably always be \pm 1? */ /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ /* take max/min bounds into account as well? */ -static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* ptr) { +static void paramVaryOnce(const U32 paramIndex, const int amt, ZSTD_compressionParameters* ptr) { switch(paramIndex) { case WLOG_IND: ptr->windowLog += amt; break; @@ -571,8 +622,14 @@ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* p case SLEN_IND: ptr->searchLength += amt; break; case TLEN_IND: if(amt >= 0) { - ptr->targetLength <<= amt; - ptr->targetLength = MIN(ptr->targetLength, 999); + if(ptr->targetLength == 0) { + if(amt > 0) { + ptr->targetLength = MIN(1 << (amt - 1), 999); + } + } else { + ptr->targetLength <<= amt; + ptr->targetLength = MIN(ptr->targetLength, 999); + } } else { if(ptr->targetLength == 999) { ptr->targetLength = 1024; @@ -584,11 +641,8 @@ static void paramVaryOnce(U32 paramIndex, int amt, ZSTD_compressionParameters* p } } -//Don't fuzz fixed variables. -//turn pcs to pcs array with macro for params. -//pass in variation array from variableParams -//take nbChanges as argument? -static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, U32 nbChanges) +/* varies ptr by nbChanges respecting varyParams*/ +static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, const U32 nbChanges) { ZSTD_compressionParameters p; U32 validated = 0; @@ -600,19 +654,11 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParam paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } validated = !ZSTD_isError(ZSTD_checkCParams(p)); - //validated = cParamValid(p); - - //Make sure memory is at least close to feasible? - //ZSTD_estimateCCtxSize thing. } - *ptr = sanitizeParams(p); + *ptr = p;//sanitizeParams(p); } -//varyParams gives us table size? -//1 per strategy -//varyParams should always be sorted smallest to largest -//take arrayLen to allocate memotable -//should be ~10^7 unconstrained. +/* length of memo table given free variables */ static size_t memoTableLen(const U32* varyParams, const int varyLen) { size_t arrayLen = 1; int i; @@ -622,11 +668,14 @@ static size_t memoTableLen(const U32* varyParams, const int varyLen) { return arrayLen; } -//sort of ~lg2 (replace 1024 w/ 999) for memoTableInd Tlen +//sort of ~lg2 (replace 1024 w/ 999, and add 0 at lower end of range) for memoTableInd Tlen static unsigned lg2(unsigned x) { - unsigned j = 0; + unsigned j = 1; if(x == 999) { - return 10; + return 11; + } + if(!x) { + return 0; } while(x >>= 1) { j++; @@ -634,8 +683,7 @@ static unsigned lg2(unsigned x) { return j; } -//indexes compressionParameters into memotable -//of form +/* returns unique index of compression parameters */ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) { int i; unsigned ind = 0; @@ -652,25 +700,24 @@ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const U32* v return ind; } -/* presumably, the unfilled parameters are already at their correct value */ -/* inverse above function for varyParams */ +/* inverse of above function (from index to parameters) */ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, size_t ind) { int i; for(i = varyLen - 1; i >= 0; i--) { switch(varyParams[i]) { - case WLOG_IND: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; - case CLOG_IND: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; - case HLOG_IND: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; - case SLOG_IND: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; - case SLEN_IND: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; - case TLEN_IND: ptr->targetLength = MIN(1 << (ind % TLEN_RANGE), 999); ind /= TLEN_RANGE; break; + case WLOG_IND: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; + case CLOG_IND: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; + case HLOG_IND: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; + case SLOG_IND: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; + case SLEN_IND: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; + case TLEN_IND: ptr->targetLength = (ind % TLEN_RANGE) ? MIN(1 << ((ind % TLEN_RANGE) - 1), 999) : 0; ind /= TLEN_RANGE; break; } } } -//initializing memoTable -/* */ -static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { + +/* Initialize memotable, immediately mark redundant / obviously infeasible params as */ +static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; @@ -682,13 +729,11 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - if(ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1 << paramConstraints.windowLog) > target.cMem) { - //infeasible; + if((ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1ULL << paramConstraints.windowLog)) > (size_t)target.cMem + (size_t)(target.cMem / 10)) { memoTable[i] = 255; j++; } - //TODO: remove any where memoTable wlog is mark any where windowlog is too big for data. - if(wFixed && (1 << paramConstraints.windowLog) > (srcSize << 1)) { + if(wFixed && (1ULL << paramConstraints.windowLog) > (srcSize << 1)) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ @@ -712,7 +757,42 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra } } } - DISPLAY("%d / %d Invalid\n", j, (int)i); + DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i); + if((int)i == j) { + DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.strategy) + } +} + +/* inits memotables for all (including mallocs), all strategies */ +/* takes unsanitized varyParams */ + +//TODO: check for errors/nulls +static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { + U32 varNew[NUM_PARAMS]; + int varLenNew; + U8** mtAll = malloc(sizeof(U8*) * (ZSTD_btultra + 1)); + int i; + if(mtAll == NULL) { + return NULL; + } + for(i = 1; i <= (int)ZSTD_btultra; i++) { + varLenNew = sanitizeVarArray(varyLen, varyParams, varNew, i); + mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + if(mtAll[i] == NULL) { + return NULL; + } + memoTableInit(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); + } + return mtAll; +} + +static void memoTableFreeAll(U8** mtAll) { + int i; + if(mtAll == NULL) { return; } + for(i = 1; i <= (int)ZSTD_btultra; i++) { + free(mtAll[i]); + } + free(mtAll); } #define PARAMTABLELOG 25 @@ -782,10 +862,7 @@ static ZSTD_compressionParameters randomParams(void) return p; } -//destructively modifies pc. -//Maybe if memoTable[ind] > 0 too often, count zeroes and explicitly choose from free stuff? -//^ maybe this doesn't matter, with |mt| size it has \approx 1-(1/e) of finding even single free spot in |mt| tries, not too bad. -//TODO: maybe memoTable pc before sanitization too so no repeats? +/* Sets pc to random unmeasured set of parameters */ static void randomConstrainedParams(ZSTD_compressionParameters* pc, U32* varArray, int varLen, U8* memoTable) { int tries = memoTableLen(varArray, varLen); //configurable, @@ -795,9 +872,7 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, U32* varArra ind = (FUZ_rand(&g_rand)) % maxSize; tries--; } while(memoTable[ind] > 0 && tries > 0); - //&& FUZ_rand(&g_rand) % 256 > memoTable[ind]); get nd choosing? (helpful w/ distance) /* maybe > infeasible bound? */ - /* memoTable[ind] == 0 -> unexplored */ memoTableIndInv(pc, varArray, varLen, (unsigned)ind); *pc = sanitizeParams(*pc); } @@ -822,7 +897,7 @@ static void BMK_benchOnce(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffe { BMK_result_t testResult; g_params = ZSTD_adjustCParams(g_params, srcSize, 0); - BMK_benchParam(&testResult, srcBuffer, srcSize, cctx, dctx, g_params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, g_params); DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, testResult.cSpeed / 1000000, testResult.dSpeed / 1000000); return; @@ -847,7 +922,7 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); BMK_result_t testResult; - BMK_benchParam(&testResult, srcBuffer, srcSize, cctx, dctx, l1params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, l1params); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); } @@ -974,112 +1049,14 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } -//parameter feasibility is not checked, should just be restricted from use. -static int feasible(BMK_result_t results, constraint_t target) { - return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); -} -#define EPSILON 0.01 -static int epsilonEqual(double c1, double c2) { - return MAX(c1/c2,c2/c1) < 1 + EPSILON; -} - -//so the compiler stops warning -static int eqZero(double c1) { - return (U64)c1 == (U64)0.0 || (U64)c1 == (U64)-0.0; -} - -/* returns 1 if result2 is strictly 'better' than result1 */ -/* strict comparison / cutoff based */ -static int objective_lt(BMK_result_t result1, BMK_result_t result2) { - return (result1.cSize > result2.cSize) || (epsilonEqual(result1.cSize, result2.cSize) && result2.cSpeed > result1.cSpeed) - || (epsilonEqual(result1.cSize,result2.cSize) && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); -} - -//will probably be some linear combinartion of comp speed, decompSpeed, & ratio (maybe size), and memory? -//pretty arbitrary right now -//maybe better - higher coefficient when below threshold, lower when above -//need to normalize speed? or just use ratio speed / target? -//Maybe don't use ratio at all when looking for feasibility? - -/* maybe dynamically vary the coefficients for this around based on what's already been discovered. (maybe make a reversed ratio cutoff?) concave to pheaily penalize below ratio? */ -static double resultScore(BMK_result_t res, size_t srcSize, constraint_t target) { - double cs = 0., ds = 0., rt, cm = 0.; - const double r1 = 1, r2 = 0.1, rtr = 0.5; - double ret; - if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; } - if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; } - if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; } - rt = ((double)srcSize / res.cSize); - - //(void)rt; - //(void)rtr; - - ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + - (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; - //DISPLAY("resultScore: %f\n", ret); - return ret; -} - -/* - double W_ratio = (double)srcSize / testResult.cSize; - double O_ratio = (double)srcSize / winners[cLevel].result.cSize; - double W_ratioNote = log (W_ratio); - double O_ratioNote = log (O_ratio); - size_t W_DMemUsed = (1 << params.windowLog) + (16 KB); - size_t O_DMemUsed = (1 << winners[cLevel].params.windowLog) + (16 KB); - double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); - double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); - - size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(params); - size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(winners[cLevel].params); - double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); - double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); - - double W_CSpeed_note = W_ratioNote * ( 30 + 10*cLevel) + log(testResult.cSpeed); - double O_CSpeed_note = O_ratioNote * ( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed); - - double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log(testResult.dSpeed); - double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed); - -*/ -//ratio tradeoffs, may be useful in guiding - -/* objective_lt, but based on scoring function */ -static int objective_lt2(BMK_result_t result1, BMK_result_t result2, size_t srcSize, constraint_t target) { - return resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target); -} - -/* res gives array dimensions, should be size NUM_PARAMS */ -static size_t computeStateSize(const ZSTD_compressionParameters paramConstraints, U32* res) { - int ind = 0; - size_t base = 1; - if(!paramConstraints.windowLog) { res[ind] = ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.chainLog) { res[ind] = ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.hashLog) { res[ind] = ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.searchLog) { res[ind] = ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.searchLength) { res[ind] = ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1; base *= res[ind]; ind++; } - if(!paramConstraints.targetLength) { res[ind] = 11; base *= res[ind]; ind++; } //restricting from 2^[0,10], no such macros - if(!(U32)paramConstraints.strategy) { res[ind] = 8; base *= 8; } //not strictly true, maybe would want to case on this. - - return base; -} - - -static unsigned calcViolation(BMK_result_t results, constraint_t target) { - int diffcSpeed = MAX(target.cSpeed - results.cSpeed, 0); - int diffdSpeed = MAX(target.dSpeed - results.dSpeed, 0); - int diffcMem = MAX(results.cMem - target.cMem, 0); - return diffcSpeed + diffdSpeed + diffcMem; -} /* -uncertaintyConstant >= 1 -returns -1 = 'certainly' infeasible - 0 = unceratin - 1 = 'certainly' feasible +checks feasibility with uncertainty. +-1 : certainly infeasible + 0 : uncertain + 1 : certainly feasible */ -//paramTarget misnamed, should just be target static int uncertainFeasibility(double const uncertaintyConstantC, double const uncertaintyConstantD, const constraint_t paramTarget, const BMK_result_t* const results) { if((paramTarget.cSpeed != 0 && results->cSpeed * uncertaintyConstantC < paramTarget.cSpeed) || (paramTarget.dSpeed != 0 && results->dSpeed * uncertaintyConstantD < paramTarget.dSpeed) || @@ -1099,12 +1076,8 @@ static int uncertainFeasibility(double const uncertaintyConstantC, double const -1 - worse assume prev_best status is run fully? but then we'd have to rerun any winners anyway */ -//presumably memory has already been compared, mostly worried about mem, cspeed, dspeed -//uncertainty only applies to speed. -//if using objective fn, this could be much easier since we could just scale that. -//difficult to make judgements about later parameters in prioritization type when there's -//uncertainty on the first. -static int uncertainComparison(double const uncertaintyConstantC, double const uncertaintyConstantD, BMK_result_t* candidate, BMK_result_t* prevBest) { +/* not as useful as initially believed */ +static int uncertainComparison(double const uncertaintyConstantC, double const uncertaintyConstantD, const BMK_result_t* candidate, const BMK_result_t* prevBest) { (void)uncertaintyConstantD; //unused for now if(candidate->cSpeed > prevBest->cSpeed * uncertaintyConstantC) { return 1; @@ -1115,34 +1088,21 @@ static int uncertainComparison(double const uncertaintyConstantC, double const u } } -/* speed in b, srcSize in b/s loopDuration in ns */ -//TODO: simplify code in feasibleBench with this instead of writing it all out. -//only applicable for single loop -static double calcUncertainty(double speed, size_t srcSize) { - U64 loopDuration; - if(eqZero(speed)) { return 2; } - loopDuration = ((srcSize * TIMELOOP_NANOSEC) / speed); - return MIN((loopDuration + (double)2 * g_clockGranularity) / loopDuration, 2); -} +/*benchmarks and tests feasibility together + 1 = true = better + 0 = false = not better + if true then resultPtr will give results. + 2+ on error? */ -//benchmarks and tests feasibility together -//1 = true = better -//0 = false = not better -//if true then resultPtr will give results. -//2+ on error? -//alt: error = 0 / infeasible as well; -//maybe use compress_only mode for ratio-finding benchmark? -//prioritize ratio > cSpeed > dSpeed > cMem -//Misnamed - should be worse, better, error -//alternative (to make work for feasible-pt searching as well) - only compare to winner, not to target -//but then we need to judge what better means in this context, which shouldn't be the same (strict ratio improvement) +//Maybe use compress_only for benchmark #define INFEASIBLE_RESULT 0 #define FEASIBLE_RESULT 1 #define ERROR_RESULT 2 static int feasibleBench(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1156,8 +1116,8 @@ static int feasibleBench(BMK_result_t* resultPtr, //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), //maybe iter this until 2x measurable for better guarantee? - DISPLAY("Feas:\n"); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + DEBUGOUTPUT("Feas:\n"); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres.error) { DISPLAY("ERROR %d !!\n", benchres.error); } @@ -1174,7 +1134,7 @@ static int feasibleBench(BMK_result_t* resultPtr, //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? //possibly has to do with initCCtx? or system stuff? //asymmetric +/- constant needed? - uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC), 2); //.02 seconds + uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC) * 1.1, 3); //.02 seconds } if(eqZero(benchres.result.dSpeed)) { loopDurationD = 0; @@ -1184,20 +1144,41 @@ static int feasibleBench(BMK_result_t* resultPtr, //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? //possibly has to do with initCCtx? or system stuff? //asymmetric +/- constant needed? - uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD), 2); //.02 seconds + uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1, 3); //.02 seconds } if(benchres.result.cSize < winnerResult->cSize) { //better compression ratio, just needs to be feasible - //optimistic assume speed - //incoporate some sort of tradeoff comparison with the winner's results? - int feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); + int feas; + if(loopDurationC < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres = benchres2; + } + } + if(loopDurationD < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + *resultPtr = benchres.result; + + feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); if(feas == 0) { // uncertain feasibility adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1220,13 +1201,34 @@ static int feasibleBench(BMK_result_t* resultPtr, return (feas + 1) >> 1; //relies on INFEASIBLE_RESULT == 0, FEASIBLE_RESULT == 1 } } else if (benchres.result.cSize == winnerResult->cSize) { //equal ratio, needs to be better than winner in cSpeed/ dSpeed / cMem - int feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); + int feas; + if(loopDurationC < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres = benchres2; + } + } + if(loopDurationD < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); if(feas == 0) { // uncertain feasibility adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1252,7 +1254,7 @@ static int feasibleBench(BMK_result_t* resultPtr, return INFEASIBLE_RESULT; } else { //possibly better, benchmark and find out adv.loopMode = BMK_timeMode; - benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); *resultPtr = benchres.result; return objective_lt(*winnerResult, benchres.result); } @@ -1265,16 +1267,17 @@ static int feasibleBench(BMK_result_t* resultPtr, } else { return ERROR_RESULT; //BMK error } - } -//sameas before, but +/-? + +//same as before, but +/-? //alternative, just return comparison result, leave caller to worry about feasibility. //have version of benchMemAdvanced which takes in dstBuffer/cap as well? //(motivation: repeat tests (maybe just on decompress) don't need further compress runs) static int infeasibleBench(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, @@ -1288,19 +1291,10 @@ static int infeasibleBench(BMK_result_t* resultPtr, adv.loopMode = BMK_iterMode; //can only use this for ratio measurement then, super inaccurate timing adv.nbSeconds = 1; //get ratio and 2x approx speed? //maybe run until twice MIN(minloopinterval * clockDuration) - DISPLAY("WinnerScore: %f\n ", winnerRS); - /* - adv.loopMode = BMK_timeMode; - adv.nbSeconds = 1; */ - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); - adv.loopMode = BMK_timeMode; - adv.nbSeconds = 1; - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); if(!benchres.error) { *resultPtr = benchres.result; @@ -1309,9 +1303,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, uncertaintyConstantC = 2; } else { loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? - //possibly has to do with initCCtx? or system stuff? - uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC), 2); //.02 seconds + uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC * 1.1), 3); //.02 seconds } if(eqZero(benchres.result.dSpeed)) { @@ -1319,11 +1311,32 @@ static int infeasibleBench(BMK_result_t* resultPtr, uncertaintyConstantD = 2; } else { loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? - //possibly has to do with initCCtx? or system stuff? - uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD), 2); //.02 seconds + uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1 , 3); //.02 seconds } + + if(loopDurationC < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_compressOnly; + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres = benchres2; + } + } + if(loopDurationD < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2; + adv.mode = BMK_decodeOnly; + benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + if(benchres2.error) { + return ERROR_RESULT; + } else { + benchres.result.dSpeed = benchres2.result.dSpeed; + } + } + *resultPtr = benchres.result; + /* benchres's certainty range. */ resultMax = benchres.result; resultMin = benchres.result; @@ -1331,8 +1344,6 @@ static int infeasibleBench(BMK_result_t* resultPtr, resultMax.dSpeed *= uncertaintyConstantD; resultMin.cSpeed /= uncertaintyConstantC; resultMin.dSpeed /= uncertaintyConstantD; - (void)resultMin; - //TODO: consider if resultMin is actually needed. if (winnerRS > resultScore(resultMax, srcSize, target)) { return INFEASIBLE_RESULT; } else { @@ -1341,7 +1352,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC) { BMK_return_t benchres2; adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, &srcSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1350,9 +1361,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, } if(loopDurationD < TIMELOOP_NANOSEC) { BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - //TODO: dstBuffer corrupted sometime between top and now - //probably occuring in feasible bench too. + adv.mode = BMK_decodeOnly; benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); if(benchres2.error) { return ERROR_RESULT; @@ -1372,28 +1381,27 @@ static int infeasibleBench(BMK_result_t* resultPtr, } /* wrap feasibleBench w/ memotable */ -//TODO: void sanitized and unsanitized ver's so input doesn't double-choose #define INFEASIBLE_THRESHOLD 200 static int feasibleBenchMemo(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, BMK_result_t* winnerResult, U8* memoTable, - U32* varyParams, const int varyLen) { + const U32* varyParams, const int varyLen) { - size_t memind = memoTableInd(&cParams, varyParams, varyLen); + const size_t memind = memoTableInd(&cParams, varyParams, varyLen); - //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //probably pick a different code for already tested? //maybe remove this if we incorporate nonrandom location picking? //what is the intended behavior in this case? //ignore? stop iterating completely? other? } else { - int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, + int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, cParams, target, winnerResult); memoTable[memind] = 255; //tested are all infeasible (other possible values for opti) return res; @@ -1403,21 +1411,21 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, //should infeasible stage searching also be memo-marked in the same way? //don't actually memoize unless result is feasible/error? static int infeasibleBenchMemo(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, + const void* srcBuffer, const size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, BMK_result_t* winnerResult, U8* memoTable, - U32* varyParams, const int varyLen) { + const U32* varyParams, const int varyLen) { size_t memind = memoTableInd(&cParams, varyParams, varyLen); - //BMK_translateAdvancedParams(cParams); if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns } else { - int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, + int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, cParams, target, winnerResult); if(res == FEASIBLE_RESULT) { memoTable[memind] = 255; //infeasible resultscores could still be normal feasible. @@ -1432,14 +1440,18 @@ typedef int (*BMK_benchMemo_t)(BMK_result_t*, const void*, size_t, void*, size_t const ZSTD_compressionParameters, const constraint_t, BMK_result_t*, U8*, U32*, const int); //varArray should be sanitized when this is called. -//TODO: transition to simpler greedy method if evaluation time is too long? -//would it be better to start at best feasible via feasible or infeasible metric? both? //possibility climb is infeasible, responsibility of caller to check that. but if something feasible is evaluated, it will be returned // *actually if it performs too //sanitize all params here. //all generation after random should be sanitized. (maybe sanitize random) -static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varLen, U8* memoTable, - const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstSize, void* dictBuffer, size_t dictSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, ZSTD_compressionParameters init) { +static winnerInfo_t climbOnce(const constraint_t target, + const U32* varArray, const int varLen, U8* memoTable, + const void* srcBuffer, size_t srcSize, + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters init) { //pick later initializations non-randomly? high dist from explored nodes. //how to do this efficiently? (might not be too much of a problem, happens rarely, running time probably dominated by benchmarking) //distance maximizing selection? @@ -1459,28 +1471,20 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL /* ineasible -> (hopefully) feasible */ /* when nothing is found, this garbages part 2. */ { - //TODO: initialize these values! winnerInfo_t bestFeasible1; /* uses feasibleBench Metric */ - winnerInfo_t bestFeasible2; /* uses resultScore Metric */ //init these params bestFeasible1.params = cparam; - bestFeasible2.params = cparam; bestFeasible1.result.cSpeed = 0; bestFeasible1.result.dSpeed = 0; bestFeasible1.result.cMem = (size_t)-1; bestFeasible1.result.cSize = (size_t)-1; - bestFeasible2.result.cSpeed = 0; - bestFeasible2.result.dSpeed = 0; - bestFeasible2.result.cMem = (size_t)-1; - bestFeasible2.result.cSize = (size_t)-1; DISPLAY("Climb Part 1\n"); while(better) { - //UTIL_time_t timestart = UTIL_getTime(); TODO: adjust sampling based on time int i, d; better = 0; - DISPLAY("Start\n"); + DEBUGOUTPUT("Start\n"); cparam = winnerInfo.params; BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); candidateInfo.params = cparam; @@ -1496,18 +1500,16 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { /* synonymous with better when called w/ infeasibleBM */ winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; - if(feasible(candidateInfo.result, target)) { - bestFeasible2 = winnerInfo; - if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { - bestFeasible1 = bestFeasible2; /* using feasibleBench metric */ - } + if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { + bestFeasible1 = winnerInfo; } } } @@ -1521,18 +1523,16 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; - if(feasible(candidateInfo.result, target)) { //TODO: maybe just break here and move on to part 2? - bestFeasible2 = winnerInfo; - if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { - bestFeasible1 = bestFeasible2; - } + if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { + bestFeasible1 = winnerInfo; } } } @@ -1553,18 +1553,16 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; - if(feasible(candidateInfo.result, target)) { - bestFeasible2 = winnerInfo; - if(objective_lt(bestFeasible1.result, bestFeasible2.result)) { - bestFeasible1 = bestFeasible2; - } + if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { + bestFeasible1 = winnerInfo; } } @@ -1576,15 +1574,12 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL //bias to test previous delta? //change cparam -> candidate before restart } - //TODO:Consider if this is best config. idea: explore from obj best keep rbest - cparam = bestFeasible2.params; - candidateInfo = bestFeasible2; winnerInfo = bestFeasible1; } - //is it better to break here instead of bumbling about? + //break out if no feasible. if(winnerInfo.result.cMem == (U32)-1) { - DISPLAY("No Feasible Found\n"); + DEBUGOUTPUT("No Feasible Found\n"); return winnerInfo; } DISPLAY("Climb Part 2\n"); @@ -1592,14 +1587,12 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL better = 1; /* feasible -> best feasible (hopefully) */ { - while(better) { - - //UTIL_time_t timestart = UTIL_getTime(); //TODO: if benchmarking is taking too long, be more greedy. + while(better) { int i, d; better = 0; BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); //all dist-1 targets - cparam = winnerInfo.params; //TODO: this messes the taking bestFeasible1, bestFeasible2 + cparam = winnerInfo.params; candidateInfo.params = cparam; for(i = 0; i < varLen; i++) { paramVaryOnce(varArray[i], 1, &candidateInfo.params); @@ -1612,12 +1605,13 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; } } @@ -1626,17 +1620,17 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL candidateInfo.params = sanitizeParams(candidateInfo.params); //evaluate if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - //if(cParamValid(candidateInfo.params)) { int res = feasibleBenchMemo(&candidateInfo.result, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; } } @@ -1653,12 +1647,13 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, + fileSizes, nbFiles, ctx, dctx, candidateInfo.params, target, &winnerInfo.result, memoTable, varArray, varLen); if(res == FEASIBLE_RESULT) { winnerInfo = candidateInfo; - //BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); better = 1; } } @@ -1684,29 +1679,26 @@ static winnerInfo_t climbOnce(constraint_t target, U32* varArray, const int varL //only real use for paramTarget is to get the fixed values, right? static winnerInfo_t optimizeFixedStrategy( const void* srcBuffer, const size_t srcSize, - void* dstBuffer, size_t dstSize, - void* dictBuffer, size_t dictSize, - constraint_t target, ZSTD_compressionParameters paramTarget, - ZSTD_strategy strat, U32* varArray, int varLen) { - int i = 0; //TODO: Temp fix 10 iters, check effects of changing this? + void* dstBuffer, const size_t dstSize, + void* dictBuffer, const size_t dictSize, + const size_t* fileSizes, const size_t nbFiles, + const constraint_t target, ZSTD_compressionParameters paramTarget, + const ZSTD_strategy strat, const U32* varArray, const int varLen, U8* memoTable) { + int i = 0; U32* varNew = malloc(sizeof(U32) * varLen); int varLenNew = sanitizeVarArray(varLen, varArray, varNew, strat); - size_t memoLen = memoTableLen(varNew, varLenNew); - U8* memoTable = malloc(sizeof(U8) * memoLen); ZSTD_compressionParameters init; ZSTD_CCtx* ctx = ZSTD_createCCtx(); ZSTD_DCtx* dctx = ZSTD_createDCtx(); winnerInfo_t winnerInfo, candidateInfo; winnerInfo.result.cSpeed = 0; winnerInfo.result.dSpeed = 0; - winnerInfo.result.cMem = (size_t)(-1); - winnerInfo.result.cSize = (size_t)(-1); + winnerInfo.result.cMem = (size_t)(-1LL); + winnerInfo.result.cSize = (size_t)(-1LL); /* so climb is given the right fixed strategy */ paramTarget.strategy = strat; /* to pass ZSTD_checkCParams */ - memoTableInit(memoTable, paramTarget, target, varNew, varLenNew, srcSize); - //needs to happen after memoTableInit as that assumes 0 = undefined. cParamZeroMin(¶mTarget); @@ -1718,11 +1710,11 @@ static winnerInfo_t optimizeFixedStrategy( goto _cleanUp; } - while(i < 10) { - DISPLAY("Restart\n"); - //TODO: look into improving this to maximize distance from searched infeasible stuff / towards promising regions? + while(i < 10) { //make i adjustable (user input?) depending on how much time they have. + DISPLAY("Restart\n"); //TODO: make better printing across restarts + //look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); - candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, ctx, dctx, init); + candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, init); if(objective_lt(winnerInfo.result, candidateInfo.result)) { winnerInfo = candidateInfo; DISPLAY("New Winner: "); @@ -1735,7 +1727,6 @@ static winnerInfo_t optimizeFixedStrategy( _cleanUp: ZSTD_freeCCtx(ctx); ZSTD_freeDCtx(dctx); - free(memoTable); free(varNew); return winnerInfo; } @@ -1777,20 +1768,54 @@ static int BMK_loadFiles(void* buffer, size_t bufferSize, fclose(f); } - if (totalSize == 0) { DISPLAY("no data to bench\n"); return 12; } + if (totalSize == 0) { DISPLAY("\nno data to bench\n"); return 12; } return 0; } -// bigger and (hopefully) better* than optimizeForSize -// TODO: allow accept multiple files like benchFiles or bench.c fn's -static int optimizeForSize2(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget) +//goes best, best-1, best+1, best-2, ... +//return 0 if nothing remaining +static int nextStrategy(const int currentStrategy, const int bestStrategy) { + if(bestStrategy <= currentStrategy) { + int candidate = 2 * bestStrategy - currentStrategy - 1; + if(candidate < 1) { + candidate = currentStrategy + 1; + if(candidate > (int)ZSTD_btultra) { + return 0; + } else { + return candidate; + } + } else { + return candidate; + } + } else { /* bestStrategy >= currentStrategy */ + int candidate = 2 * bestStrategy - currentStrategy; + if(candidate > (int)ZSTD_btultra) { + candidate = currentStrategy - 1; + if(candidate < 1) { + return 0; + } else { + return candidate; + } + } else { + return candidate; + } + } +} + +//optimize fixed strategy. +static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget) { size_t benchedSize; - void* origBuff; - void* dictBuffer; - size_t dictBufferSize; + void* origBuff = NULL; + void* dictBuffer = NULL; + size_t dictBufferSize = 0; U32 varArray [NUM_PARAMS]; - int varLen = variableParams(paramTarget, varArray); + int ret = 0; + size_t* fileSizes = calloc(sizeof(size_t),nbFiles); + const int varLen = variableParams(paramTarget, varArray); + U8** allMT = NULL; + g_targetConstraints = target; + g_winner.result.cSize = (size_t)-1; /* Init */ if(!cParamValid(paramTarget)) { return 10; @@ -1801,20 +1826,24 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size U64 const dictFileSize = UTIL_getFileSize(dictFileName); if (dictFileSize > 64 MB) { DISPLAY("dictionary file %s too large", dictFileName); - return 10; + ret = 10; + goto _cleanUp; } dictBufferSize = (size_t)dictFileSize; dictBuffer = malloc(dictBufferSize); if (dictBuffer==NULL) { DISPLAY("not enough memory for dictionary (%u bytes)", (U32)dictBufferSize); - return 11; + ret = 11; + goto _cleanUp; + } + { int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, &dictBufferSize, &dictFileName, 1); if(errorCode) { - free(dictBuffer); - return errorCode; + ret = errorCode; + goto _cleanUp; } } } @@ -1823,41 +1852,46 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size if(nbFiles == 1) { DISPLAY("Loading %s... \r", fileNamesTable[0]); } else { - DISPLAY("Loading %zd Files... \r", nbFiles); + DISPLAY("Loading %lu Files... \r", (unsigned long)nbFiles); } { U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); int ec; - size_t* fileSizes = calloc(sizeof(size_t),nbFiles); + unsigned i; benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; + origBuff = malloc(benchedSize); if(!origBuff || !fileSizes) { DISPLAY("Not enough memory for stuff\n"); - free(origBuff); - free(fileSizes); - free(dictBuffer); - return 1; + ret = 1; + goto _cleanUp; } ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, nbFiles); if(ec) { DISPLAY("Error Loading Files"); - free(origBuff); - free(fileSizes); - free(dictBuffer); - return ec; + ret = ec; + goto _cleanUp; } - free(fileSizes); + benchedSize = 0; + for(i = 0; i < nbFiles; i++) { + benchedSize += fileSizes[i]; + } + origBuff = realloc(origBuff, benchedSize); } - + allMT = memoTableInitAll(paramTarget, target, varArray, varLen, benchedSize); + if(!allMT) { + ret = 2; + goto _cleanUp; + } /* bench */ DISPLAY("\r%79s\r", ""); if(nbFiles == 1) { DISPLAY("optimizing for %s", fileNamesTable[0]); } else { - DISPLAY("optimizing for %zd Files", nbFiles); + DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles); } if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } @@ -1868,7 +1902,7 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); winnerInfo_t winner; - //BMK_result_t candidate; + U32 varNew[NUM_PARAMS]; const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1; const size_t maxCompressedSize = ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024); @@ -1877,49 +1911,120 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size /* init */ if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} if(compressedBuffer==NULL) { DISPLAY("\n Allocation Error \n"); free(origBuff); free(ctx); return 15; } - memset(&winner, 0, sizeof(winner)); winner.result.cSize = (size_t)(-1); /* find best solution from default params */ - //Can't do this w/ cparameter constraints - //still useful though? - /* - { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); - int i; - for (i=1; i<=maxSeeds; i++) { - ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); - BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams); - if (!feasible(candidate, target) ) { - break; + { + /* strategy selection */ + //TODO: don't factor memory into strategy selection in constraint_t + const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + DEBUGOUTPUT("Strategy Selection\n"); + if(varLen == NUM_PARAMS && paramTarget.strategy == 0) { /* no variable based constraints */ + BMK_result_t candidate; + int feas = 0, i; + for (i=1; i<=maxSeeds; i++) { + ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); + int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams); + BMK_printWinner(stdout, i, candidate, CParams, benchedSize); + + if(!ec) { + if(feas) { + if(feasible(candidate, relaxTarget(target)) && objective_lt(winner.result, candidate)) { + winner.result = candidate; + winner.params = CParams; + } + } else { + if(feasible(candidate, relaxTarget(target))) { + feas = 1; + winner.result = candidate; + winner.params = CParams; + + } else { + if(resultScore(candidate, benchedSize, target) > resultScore(winner.result, benchedSize, target)) { + winner.result = candidate; + winner.params = CParams; + } + } + } + } + } //best, -1, +1, ..., + + } else if (paramTarget.strategy == 0) { //constrained + int feas = 0, i, j; + for(j = 1; j < 10; j++) { + for(i = 1; i <= maxSeeds; i++) { + int varLenNew = sanitizeVarArray(varLen, varArray, varNew, i); + ZSTD_compressionParameters candidateParams = paramTarget; + BMK_result_t candidate; + int ec; + randomConstrainedParams(&candidateParams, varNew, varLenNew, allMT[i]); + cParamZeroMin(&candidateParams); + candidateParams = sanitizeParams(candidateParams); + ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, candidateParams); + + if(!ec) { + if(feas) { + if(feasible(candidate, relaxTarget(target)) && objective_lt(winner.result, candidate)) { + winner.result = candidate; + winner.params = candidateParams; + BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); + } + } else { + if(feasible(candidate, relaxTarget(target))) { + feas = 1; + winner.result = candidate; + winner.params = candidateParams; + BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); + + } else { + if(resultScore(candidate, benchedSize, target) > resultScore(winner.result, benchedSize, target)) { + winner.result = candidate; + winner.params = candidateParams; + BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); + } + } + } + } + + } } - if (feasible(candidate,target) && objective_lt(winner.result, candidate)) - { - winner.params = CParams; - winner.result = candidate; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } } - }*/ + } + } + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); - BMK_translateAdvancedParams(winner.params); - - /* start real tests */ + DEBUGOUTPUT("Real Opt\n"); + /* start 'real' tests */ { + int bestStrategy = (int)winner.params.strategy; if(paramTarget.strategy == 0) { - int st; - for(st = 1; st <= 8; st++) { - winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, - target, paramTarget, st, varArray, varLen); - DISPLAY("StratNum %d\n", st); + int st = (int)winner.params.strategy; + + { + int varLenNew = sanitizeVarArray(varLen, varArray, varNew, st); + winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], + origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, + fileSizes, nbFiles, ctx, dctx, winner.params); + if(objective_lt(winner.result, w1.result)) { + winner = w1; + } + } + + while(st) { + winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, fileSizes, nbFiles, + target, paramTarget, st, varArray, varLen, allMT[st]); + DEBUGOUTPUT("StratNum %d\n", st); if(objective_lt(winner.result, wc.result)) { winner = wc; } + //TODO: we could double back to increase search of 'better' strategies + st = nextStrategy(st, bestStrategy); } } else { - winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, - target, paramTarget, paramTarget.strategy, varArray, varLen); + winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, fileSizes, nbFiles, + target, paramTarget, paramTarget.strategy, varArray, varLen, allMT[paramTarget.strategy]); } } @@ -1934,150 +2039,18 @@ static int optimizeForSize2(const char* const * const fileNamesTable, const size BMK_translateAdvancedParams(winner.params); DISPLAY("grillParams size - optimizer completed \n"); - /* clean up*/ - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); - } - - free(origBuff); - return 0; -} - - -/* optimizeForSize(): - * targetSpeed : expressed in B/s */ -/* expresses targeted compression, decompression speeds and memory requirements */ -/* if state space is small (from paramTarget), exhaustive search? */ -//things to consider : if doing strategy-separate approach, what cutoffs to evaluate each strategy -//or do all? can't be absolute, should be relative after some sort of calibration -//(synthetic? test levels (we don't care about data specifics rn, scale?) ? -int optimizeForSize(const char* inFileName, constraint_t target, ZSTD_compressionParameters paramTarget) -{ - FILE* const inFile = fopen( inFileName, "rb" ); - U64 const inFileSize = UTIL_getFileSize(inFileName); - size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; - void* origBuff; - U32 paramVarArray [NUM_PARAMS]; - int paramCount = variableParams(paramTarget, paramVarArray); - /* Init */ - if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } - if (inFileSize == UTIL_FILESIZE_UNKNOWN) { - DISPLAY("Pb evaluatin size of %s \n", inFileName); - fclose(inFile); - return 11; - } - - /* Memory allocation & restrictions */ - if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) { - DISPLAY("Not enough memory for '%s' \n", inFileName); - fclose(inFile); - return 11; - } - - /* Alloc */ - origBuff = malloc(benchedSize); - if(!origBuff) { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - return 12; - } - - /* Fill input buffer */ - DISPLAY("Loading %s... \r", inFileName); - { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - if(readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(origBuff); - return 13; - } } - - /* bench */ - DISPLAY("\r%79s\r", ""); - DISPLAY("optimizing for %s", inFileName); - if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } - if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } - if(target.cMem != 0) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } - DISPLAY("\n"); - { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - winnerInfo_t winner; - BMK_result_t candidate; - const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; - - /* init */ - if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14; } - - memset(&winner, 0, sizeof(winner)); - winner.result.cSize = (size_t)(-1); - - /* find best solution from default params */ - //Can't do this w/ cparameter constraints - { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); - int i; - for (i=1; i<=maxSeeds; i++) { - ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); - BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, CParams); - if (!feasible(candidate, target) ) { - break; - } - if (feasible(candidate,target) && objective_lt(winner.result, candidate)) - { - winner.params = CParams; - winner.result = candidate; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } } - } - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); - - BMK_translateAdvancedParams(winner.params); - - /* start tests */ - { time_t const grillStart = time(NULL); - do { - ZSTD_compressionParameters params = winner.params; - BYTE* b; - paramVariation(¶ms, paramVarArray, paramCount, 4); - if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */ - params = ZSTD_adjustCParams(params, blockSize, 0); - - /* exclude faster if already played set of params */ - if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(params))-1)) continue; - - /* test */ - b = NB_TESTS_PLAYED(params); - (*b)++; - BMK_benchParam(&candidate, origBuff, benchedSize, ctx, dctx, params); - - /* improvement found => new winner */ - if (feasible(candidate,target) && objective_lt(winner.result, candidate)) - { - winner.params = params; - winner.result = candidate; - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); - BMK_translateAdvancedParams(winner.params); - } - } while (BMK_timeSpan(grillStart) < g_grillDuration_s); - } - - /* no solution found */ - if(winner.result.cSize == (size_t)-1) { - DISPLAY("No feasible solution found\n"); - return 1; - } - /* end summary */ - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); - BMK_translateAdvancedParams(winner.params); - DISPLAY("grillParams size - optimizer completed \n"); /* clean up*/ ZSTD_freeCCtx(ctx); ZSTD_freeDCtx(dctx); - } + } +_cleanUp: + free(fileSizes); + free(dictBuffer); + memoTableFreeAll(allMT); free(origBuff); - return 0; + return ret; } static void errorOut(const char* msg) @@ -2136,6 +2109,7 @@ static int usage_advanced(void) DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); DISPLAY( " -v : Prints Benchmarking output\n"); + DISPLAY( " -D : Next argument dictionary file\n"); return 0; } @@ -2163,6 +2137,8 @@ int main(int argc, const char** argv) assert(argc>=1); /* for exename */ + g_time = UTIL_getTime(); + /* Welcome message */ DISPLAY(WELCOME_MESSAGE); @@ -2397,7 +2373,7 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize2(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget); + result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From 7b5b3d7ae383f4efe53a95d68a4a1f1760f78650 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 16 Jul 2018 16:16:31 -0700 Subject: [PATCH 30/92] BenchMem with block compressed sizes passed back up --- programs/bench.c | 31 +-- programs/bench.h | 13 +- tests/paramgrill.c | 625 +++++++++++++++++++++++++++++++++++++-------- 3 files changed, 540 insertions(+), 129 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index f5184fa8f..8e57db4c9 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -188,7 +188,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy , comprParams->strategy); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); } @@ -281,8 +281,6 @@ static size_t local_defaultDecompress( } -volatile char g_touched; - /* initFn will be measured once, bench fn will be measured x times */ /* benchFn should return error value or out Size */ /* takes # of blocks and list of size & stuff for each. */ @@ -293,7 +291,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void** const dstBlockBuffers, size_t* dstBlockCapacities, + void* const * const dstBlockBuffers, size_t* dstBlockCapacitiesToSizes, unsigned nbLoops) { size_t srcSize = 0, dstSize = 0, ind = 0; U64 totalTime; @@ -310,20 +308,13 @@ BMK_customReturn_t BMK_benchFunction( } { - - unsigned i, j; + size_t i; for(i = 0; i < blockCount; i++) { - for(j = 0; j < srcBlockSizes[i]; j++) { - g_touched = ((const char*)srcBlockBuffers[i])[j]; /* touch */ - } - } - for(i = 0; i < blockCount; i++) { - memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ + memset(dstBlockBuffers[i], 0xE5, dstBlockCapacitiesToSizes[i]); /* warm up and erase result buffer */ } //UTIL_sleepMilli(5); /* give processor time to other processes */ //UTIL_waitForNextTick(); - } { @@ -332,17 +323,13 @@ BMK_customReturn_t BMK_benchFunction( if(initFn != NULL) { initFn(initPayload); } for(i = 0; i < nbLoops; i++) { for(j = 0; j < blockCount; j++) { - size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload); + size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacitiesToSizes[j], benchPayload); if(ZSTD_isError(res)) { EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", - j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); + j, (U32)dstBlockCapacitiesToSizes[j], ZSTD_getErrorName(res)); } else if(firstIter) { dstSize += res; - //Make compressed blocks continuous - if(j != blockCount - 1) { - dstBlockBuffers[j+1] = (void*)((char*)dstBlockBuffers[j] + res); - dstBlockCapacities[j] = res; - } + dstBlockCapacitiesToSizes[j] = res; } } firstIter = 0; @@ -382,7 +369,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const* const srcBlockBuffers, const size_t* srcBlockSizes, - void** const dstBlockBuffers, size_t* dstBlockCapacities) + void * const * const dstBlockBuffers, size_t * dstBlockCapacitiesToSizes) { U64 fastest = cont->fastestTime; int completed = 0; @@ -399,7 +386,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( } r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload, - blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, cont->nbLoops); + blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacitiesToSizes, cont->nbLoops); if(r.result.error) { /* completed w/ error */ r.completed = 1; return r; diff --git a/programs/bench.h b/programs/bench.h index 1a298cc48..25d9f24a7 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -122,8 +122,6 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, * (cLevel, comprParams + adv in advanced Mode) */ /* srcBuffer - data source, expected to be valid compressed data if in Decode Only Mode * srcSize - size of data in srcBuffer - * dstBuffer - destination buffer to write compressed output in, optional (NULL) - * dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL * cLevel - compression level * comprParams - basic compression parameters * dictBuffer - a dictionary if used, null otherwise @@ -144,7 +142,10 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName); -/* See benchMem for normal parameter uses and return, see advancedParams_t for adv */ +/* See benchMem for normal parameter uses and return, see advancedParams_t for adv + * dstBuffer - destination buffer to write compressed output in, NULL if none provided. + * dstCapacity - capacity of destination buffer, give 0 if dstBuffer = NULL + */ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, void* dstBuffer, size_t dstCapacity, const size_t* fileSizes, unsigned nbFiles, @@ -174,7 +175,7 @@ typedef size_t (*BMK_initFn_t)(void*); * srcBuffers - an array of buffers to be operated on by benchFn * srcSizes - an array of the sizes of above buffers * dstBuffers - an array of buffers to be written into by benchFn - * dstCapacities - an array of the capacities of above buffers. + * dstCapacitiesToSizes - an array of the capacities of above buffers. Output modified to compressed sizes of those blocks. * nbLoops - defines number of times benchFn is run. * return * .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return @@ -191,7 +192,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, - void** const dstBuffers, size_t* dstCapacities, + void * const * const dstBuffers, size_t* dstCapacitiesToSizes, unsigned nbLoops); @@ -220,7 +221,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void** const dstBlockBuffers, size_t* dstBlockCapacities); + void* const * const dstBlockBuffers, size_t* dstBlockCapacitiesToSizes); #endif /* BENCH_H_121279284357 */ diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 9af60a740..e4d468846 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -27,7 +27,8 @@ #include "xxhash.h" #include "util.h" #include "bench.h" - +#include "zstd_errors.h" +#include "zstd_internal.h" /*-************************************ * Constants @@ -36,11 +37,6 @@ #define AUTHOR "Yann Collet" #define WELCOME_MESSAGE "*** %s %s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR - -#define KB *(1<<10) -#define MB *(1<<20) -#define GB *(1ULL<<30) - #define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */ #define NBLOOPS 2 @@ -240,7 +236,6 @@ static int cParamValid(ZSTD_compressionParameters paramTarget) { return 1; } -//TODO: let targetLength = 0; static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) { paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN; paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN; @@ -261,7 +256,7 @@ static int feasible(const BMK_result_t results, const constraint_t target) { return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); } -#define EPSILON 0.01 +#define EPSILON 0.001 static int epsilonEqual(const double c1, const double c2) { return MAX(c1/c2,c2/c1) < 1 + EPSILON; } @@ -274,8 +269,8 @@ static int eqZero(const double c1) { /* returns 1 if result2 is strictly 'better' than result1 */ /* strict comparison / cutoff based */ static int objective_lt(const BMK_result_t result1, const BMK_result_t result2) { - return (result1.cSize > result2.cSize) || (epsilonEqual(result1.cSize, result2.cSize) && result2.cSpeed > result1.cSpeed) - || (epsilonEqual(result1.cSize,result2.cSize) && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); + return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) + || (result1.cSize == result2.cSize && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); } /* hill climbing value for part 1 */ @@ -352,9 +347,362 @@ typedef struct { ZSTD_compressionParameters params; } winnerInfo_t; +/*-******************************************************* +* From Paramgrill +*********************************************************/ + +static void BMK_initCCtx(ZSTD_CCtx* ctx, + const void* dictBuffer, size_t dictBufferSize, int cLevel, + const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { + if (adv->nbWorkers==1) { + ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0); + } else { + ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, adv->nbWorkers); + } + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); + ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); + ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); +} + + +static void BMK_initDCtx(ZSTD_DCtx* dctx, + const void* dictBuffer, size_t dictBufferSize) { + ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); +} + +typedef struct { + ZSTD_CCtx* ctx; + const void* dictBuffer; + size_t dictBufferSize; + int cLevel; + const ZSTD_compressionParameters* comprParams; + const BMK_advancedParams_t* adv; +} BMK_initCCtxArgs; + +static size_t local_initCCtx(void* payload) { + BMK_initCCtxArgs* ag = (BMK_initCCtxArgs*)payload; + BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv); + return 0; +} + +typedef struct { + ZSTD_DCtx* dctx; + const void* dictBuffer; + size_t dictBufferSize; +} BMK_initDCtxArgs; + +static size_t local_initDCtx(void* payload) { + BMK_initDCtxArgs* ag = (BMK_initDCtxArgs*)payload; + BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize); + return 0; +} + +/* additional argument is just the context */ +static size_t local_defaultCompress( + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + void* addArgs) { + size_t moreToFlush = 1; + ZSTD_CCtx* ctx = (ZSTD_CCtx*)addArgs; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + in.src = srcBuffer; + in.size = srcSize; + in.pos = 0; + out.dst = dstBuffer; + out.size = dstSize; + out.pos = 0; + while (moreToFlush) { + if(out.pos == out.size) { + return (size_t)-ZSTD_error_dstSize_tooSmall; + } + moreToFlush = ZSTD_compress_generic(ctx, &out, &in, ZSTD_e_end); + if (ZSTD_isError(moreToFlush)) { + return moreToFlush; + } + } + return out.pos; +} + +/* additional argument is just the context */ +static size_t local_defaultDecompress( + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + void* addArgs) { + size_t moreToFlush = 1; + ZSTD_DCtx* dctx = (ZSTD_DCtx*)addArgs; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + in.src = srcBuffer; + in.size = srcSize; + in.pos = 0; + out.dst = dstBuffer; + out.size = dstSize; + out.pos = 0; + while (moreToFlush) { + if(out.pos == out.size) { + return (size_t)-ZSTD_error_dstSize_tooSmall; + } + moreToFlush = ZSTD_decompress_generic(dctx, + &out, &in); + if (ZSTD_isError(moreToFlush)) { + return moreToFlush; + } + } + return out.pos; + +} + +/*-******************************************************* +* From Paramgrill End +*********************************************************/ + +/* Replicate function of benchMemAdvanced, but with pre-split src / dst buffers, with relevant info to invert it (compressedSizes) passed out. */ +/*BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ +/* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */ + +/* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ +/* dictionary nullable, nothing else though. */ +static BMK_return_t BMK_benchMemInvertible(const void * const * const srcPtrs, size_t const * const srcSizes, + void** dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, + const int cLevel, const ZSTD_compressionParameters* comprParams, + const void* dictBuffer, const size_t dictBufferSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) { + U32 i; + BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; + size_t srcSize = 0; + void** const resPtrs = malloc(sizeof(void*) * nbBlocks); /* only really needed in both mode. */ + size_t* const resSizes = malloc(sizeof(size_t) * nbBlocks); + int freeDST = 0; + + BMK_advancedParams_t adv = BMK_initAdvancedParams(); + adv.mode = mode; + adv.loopMode = loopMode; + adv.nbSeconds = nbSeconds; + + /* resSizes == srcSizes, but modifiable */ + memcpy(resSizes, srcSizes, sizeof(size_t) * nbBlocks); + + for(i = 0; i < nbBlocks; i++) { + srcSize += srcSizes[i]; + } + + if(!ctx || !dctx || !srcPtrs || ! srcSizes) + { + results.error = 31; + DISPLAY("error: passed in null argument\n"); + free(resPtrs); + free(resSizes); + return results; + } + if(!resPtrs || !resSizes) { + results.error = 32; + DISPLAY("error: allocation failed\n"); + free(resPtrs); + free(resSizes); + return results; + } + + /* so resPtr is continuous */ + resPtrs[0] = malloc(srcSize); + + if(!(resPtrs[0])) { + results.error = 32; + DISPLAY("error: allocation failed\n"); + free(resPtrs); + free(resSizes); + return results; + } + + for(i = 1; i < nbBlocks; i++) { + resPtrs[i] = (void*)(((char*)resPtrs[i-1]) + srcSizes[i-1]); + } + + /* allocate own dst if NULL */ + if(dstPtrs == NULL) { + freeDST = 1; + dstPtrs = malloc(nbBlocks * sizeof(void*)); + dstCapacityToSizes = malloc(nbBlocks * sizeof(size_t)); + if(dstPtrs == NULL) { + results.error = 33; + DISPLAY("error: allocation failed\n"); + free(resPtrs); + free(resSizes); + return results; + } + + if(mode == BMK_decodeOnly) { //dst is src + size_t dstSize = 0; + for(i = 0; i < nbBlocks; i++) { + dstCapacityToSizes[i] = ZSTD_getDecompressedSize(srcPtrs[i], srcSizes[i]); + dstSize += dstCapacityToSizes[i]; + } + dstPtrs[0] = malloc(dstSize); + if(dstPtrs[0] == NULL) { + results.error = 34; + DISPLAY("error: allocation failed\n"); + goto _cleanUp; + } + for(i = 1; i < nbBlocks; i++) { + dstPtrs[i] = (void*)(((char*)dstPtrs[i-1]) + ZSTD_getDecompressedSize(srcPtrs[i-1], srcSizes[i-1])); + } + } else { + dstPtrs[0] = malloc(ZSTD_compressBound(srcSize) + (nbBlocks * 1024)); + if(dstPtrs[0] == NULL) { + results.error = 35; + DISPLAY("error: allocation failed\n"); + goto _cleanUp; + } + dstCapacityToSizes[0] = ZSTD_compressBound(srcSizes[0]); + for(i = 1; i < nbBlocks; i++) { + dstPtrs[i] = (void*)(((char*)dstPtrs[i-1]) + dstCapacityToSizes[i-1]); + dstCapacityToSizes[i] = ZSTD_compressBound(srcSizes[i]); + } + } + } + + /* warmimg up memory */ + for(i = 0; i < nbBlocks; i++) { + RDG_genBuffer(dstPtrs[i], dstCapacityToSizes[i], 0.10, 0.50, 1); + } + + /* Bench */ + { + { + BMK_initCCtxArgs cctxprep; + BMK_initDCtxArgs dctxprep; + cctxprep.ctx = ctx; + cctxprep.dictBuffer = dictBuffer; + cctxprep.dictBufferSize = dictBufferSize; + cctxprep.cLevel = cLevel; + cctxprep.comprParams = comprParams; + cctxprep.adv = &adv; + dctxprep.dctx = dctx; + dctxprep.dictBuffer = dictBuffer; + dctxprep.dictBufferSize = dictBufferSize; + if(loopMode == BMK_timeMode) { + BMK_customTimedReturn_t intermediateResultCompress; + BMK_customTimedReturn_t intermediateResultDecompress; + BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(nbSeconds); + BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(nbSeconds); + if(mode == BMK_compressOnly) { + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 1; + } else if (mode == BMK_decodeOnly) { + intermediateResultCompress.completed = 1; + intermediateResultDecompress.completed = 0; + } else { /* both */ + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 0; + } + while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) { + if(!intermediateResultCompress.completed) { + intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes); + if(intermediateResultCompress.result.error) { + results.error = intermediateResultCompress.result.error; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + goto _cleanUp; + } + results.result.cSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000000000; + results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; + } + + if(!intermediateResultDecompress.completed) { + if(mode == BMK_decodeOnly) { + intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, + nbBlocks, (const void* const*)srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes); + } else { /* both, decompressed result already written to dstPtr */ + intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, + nbBlocks, (const void* const*)dstPtrs, dstCapacityToSizes, resPtrs, resSizes); + } + + if(intermediateResultDecompress.result.error) { + results.error = intermediateResultDecompress.result.error; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + goto _cleanUp; + } + results.result.dSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000000000; + } + } + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + } else { //iterMode; + if(mode != BMK_decodeOnly) { + + BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbSeconds); + if(compressionResults.error) { + results.error = compressionResults.error; + goto _cleanUp; + } + if(compressionResults.result.nanoSecPerRun == 0) { + results.result.cSpeed = 0; + } else { + results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + } + results.result.cSize = compressionResults.result.sumOfReturn; + } + if(mode != BMK_compressOnly) { + BMK_customReturn_t decompressionResults; + if(mode == BMK_decodeOnly) { + decompressionResults = BMK_benchFunction( + &local_defaultDecompress, (void*)(dctx), + &local_initDCtx, (void*)&dctxprep, nbBlocks, + (const void* const*)srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, + nbSeconds); + } else { + decompressionResults = BMK_benchFunction( + &local_defaultDecompress, (void*)(dctx), + &local_initDCtx, (void*)&dctxprep, nbBlocks, + (const void* const*)dstPtrs, dstCapacityToSizes, resPtrs, resSizes, + nbSeconds); + } + + if(decompressionResults.error) { + results.error = decompressionResults.error; + goto _cleanUp; + } + + if(decompressionResults.result.nanoSecPerRun == 0) { + results.result.dSpeed = 0; + } else { + results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + } + } + } + } + } /* Bench */ + results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); + +_cleanUp: + free(resPtrs[0]); + free(resPtrs); + free(resSizes); + if(freeDST) { + free(dstPtrs[0]); + free(dstPtrs); + } + return results; +} + /* global winner used for display. */ //Should be totally 0 initialized? -static winnerInfo_t g_winner; //TODO: ratio is infinite at initialization, instead of 0 +static winnerInfo_t g_winner; static constraint_t g_targetConstraints; static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) @@ -379,7 +727,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); - if(TIMED) { fprintf(f, " - %lu:%lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } + if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } fprintf(f, "\n"); if(objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints)) { BMK_translateAdvancedParams(params); @@ -670,17 +1018,10 @@ static size_t memoTableLen(const U32* varyParams, const int varyLen) { //sort of ~lg2 (replace 1024 w/ 999, and add 0 at lower end of range) for memoTableInd Tlen static unsigned lg2(unsigned x) { - unsigned j = 1; if(x == 999) { return 11; } - if(!x) { - return 0; - } - while(x >>= 1) { - j++; - } - return j; + return x ? ZSTD_highbit32(x) + 1 : 0; } /* returns unique index of compression parameters */ @@ -729,7 +1070,7 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - if((ZSTD_estimateCCtxSize_usingCParams(paramConstraints) + (1ULL << paramConstraints.windowLog)) > (size_t)target.cMem + (size_t)(target.cMem / 10)) { + if(ZSTD_estimateCStreamSize_usingCParams(paramConstraints) > (size_t)target.cMem) { memoTable[i] = 255; j++; } @@ -765,8 +1106,6 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ - -//TODO: check for errors/nulls static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { U32 varNew[NUM_PARAMS]; int varLenNew; @@ -1099,28 +1438,29 @@ static int uncertainComparison(double const uncertaintyConstantC, double const u #define FEASIBLE_RESULT 1 #define ERROR_RESULT 2 static int feasibleBench(BMK_result_t* resultPtr, - const void* srcBuffer, const size_t srcSize, - void* dstBuffer, const size_t dstSize, - void* dictBuffer, const size_t dictSize, - const size_t* fileSizes, const size_t nbFiles, + const void* const * const srcPtrs, size_t const * const srcSizes, + void** const dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, + void* dictBuffer, const size_t dictBufferSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams, const constraint_t target, BMK_result_t* winnerResult) { - BMK_advancedParams_t adv = BMK_initAdvancedParams(); BMK_return_t benchres; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; - adv.loopMode = BMK_iterMode; - adv.nbSeconds = 1; //get ratio and 2x approx speed? - + size_t srcSize = 0; + U32 i; //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), //maybe iter this until 2x measurable for better guarantee? DEBUGOUTPUT("Feas:\n"); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_both, BMK_iterMode, 1); if(benchres.error) { DISPLAY("ERROR %d !!\n", benchres.error); } + for(i = 0; i < nbBlocks; i++) { + srcSize += srcSizes[i]; + } BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); if(!benchres.error) { @@ -1151,9 +1491,8 @@ static int feasibleBench(BMK_result_t* resultPtr, if(benchres.result.cSize < winnerResult->cSize) { //better compression ratio, just needs to be feasible int feas; if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1161,9 +1500,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1174,11 +1512,9 @@ static int feasibleBench(BMK_result_t* resultPtr, feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); if(feas == 0) { // uncertain feasibility - adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1186,9 +1522,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1203,9 +1538,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } else if (benchres.result.cSize == winnerResult->cSize) { //equal ratio, needs to be better than winner in cSpeed/ dSpeed / cMem int feas; if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1213,9 +1547,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1224,11 +1557,9 @@ static int feasibleBench(BMK_result_t* resultPtr, } feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); if(feas == 0) { // uncertain feasibility - adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1236,9 +1567,8 @@ static int feasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer,dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1253,8 +1583,8 @@ static int feasibleBench(BMK_result_t* resultPtr, if(btw == -1) { return INFEASIBLE_RESULT; } else { //possibly better, benchmark and find out - adv.loopMode = BMK_timeMode; - benchres = BMK_benchMemAdvanced(srcBuffer, srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_both, BMK_timeMode, 1); *resultPtr = benchres.result; return objective_lt(*winnerResult, benchres.result); } @@ -1274,28 +1604,31 @@ static int feasibleBench(BMK_result_t* resultPtr, //have version of benchMemAdvanced which takes in dstBuffer/cap as well? //(motivation: repeat tests (maybe just on decompress) don't need further compress runs) static int infeasibleBench(BMK_result_t* resultPtr, - const void* srcBuffer, const size_t srcSize, - void* dstBuffer, const size_t dstSize, - void* dictBuffer, const size_t dictSize, - const size_t* fileSizes, const size_t nbFiles, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - const ZSTD_compressionParameters cParams, - const constraint_t target, - BMK_result_t* winnerResult) { - BMK_advancedParams_t adv = BMK_initAdvancedParams(); + const void* const * const srcPtrs, size_t const * const srcSizes, + void** const dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, + void* dictBuffer, const size_t dictBufferSize, + ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, + const ZSTD_compressionParameters cParams, + const constraint_t target, + BMK_result_t* winnerResult) { BMK_return_t benchres; BMK_result_t resultMin, resultMax; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; - double winnerRS = resultScore(*winnerResult, srcSize, target); - adv.loopMode = BMK_iterMode; //can only use this for ratio measurement then, super inaccurate timing - adv.nbSeconds = 1; //get ratio and 2x approx speed? //maybe run until twice MIN(minloopinterval * clockDuration) + double winnerRS; + size_t srcSize = 0; + U32 i; + + benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_both, BMK_iterMode, 1); + for(i = 0; i < nbBlocks; i++) { + srcSize += srcSizes[i]; + } + BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); + winnerRS = resultScore(*winnerResult, srcSize, target); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); - benchres = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); - if(!benchres.error) { *resultPtr = benchres.result; if(eqZero(benchres.result.cSpeed)) { @@ -1316,9 +1649,8 @@ static int infeasibleBench(BMK_result_t* resultPtr, if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1326,9 +1658,8 @@ static int infeasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1347,12 +1678,9 @@ static int infeasibleBench(BMK_result_t* resultPtr, if (winnerRS > resultScore(resultMax, srcSize, target)) { return INFEASIBLE_RESULT; } else { - //do this w/o copying / stuff - adv.loopMode = BMK_timeMode; if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_compressOnly; - benchres2 = BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1360,9 +1688,8 @@ static int infeasibleBench(BMK_result_t* resultPtr, } } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2; - adv.mode = BMK_decodeOnly; - benchres2 = BMK_benchMemAdvanced(dstBuffer, dstSize, NULL, 0, &benchres.result.cSize, 1, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); + BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, + BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } else { @@ -1394,16 +1721,62 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, const U32* varyParams, const int varyLen) { const size_t memind = memoTableInd(&cParams, varyParams, varyLen); - if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { - return INFEASIBLE_RESULT; //probably pick a different code for already tested? - //maybe remove this if we incorporate nonrandom location picking? - //what is the intended behavior in this case? - //ignore? stop iterating completely? other? + return INFEASIBLE_RESULT; } else { - int res = feasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, + const size_t blockSize = g_blockSize ? g_blockSize : srcSize; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const dstCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + U32 nbBlocks; + int res; + + if(!srcPtrs || !srcSizes || !dstPtrs || !dstCapacities) { + free(srcPtrs); + free(srcSizes); + free(dstPtrs); + free(dstCapacities); + DISPLAY("Allocation Error\n"); + return ERROR_RESULT; + } + + { + const char* srcPtr = (const char*)srcBuffer; + char* dstPtr = (char*)dstBuffer; + size_t dstSizeRemaining = dstSize; + U32 fileNb; + for (nbBlocks=0, fileNb=0; fileNb dstSizeRemaining) { + DEBUGOUTPUT("Warning: dstSize too small to benchmark completely \n"); + remaining = dstSizeRemaining; + dstSizeRemaining = 0; + } else { + dstSizeRemaining -= remaining; + } + for ( ; nbBlocks= INFEASIBLE_THRESHOLD) { return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns } else { - int res = infeasibleBench(resultPtr, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, + const size_t blockSize = g_blockSize ? g_blockSize : srcSize; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); + size_t* const dstCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + U32 nbBlocks; + int res; + + if(!srcPtrs || !srcSizes || !dstPtrs || !dstCapacities) { + free(srcPtrs); + free(srcSizes); + free(dstPtrs); + free(dstCapacities); + DISPLAY("Allocation Error\n"); + return ERROR_RESULT; + } + + { + const char* srcPtr = (const char*)srcBuffer; + char* dstPtr = (char*)dstBuffer; + size_t dstSizeRemaining = dstSize; + U32 fileNb; + for (nbBlocks=0, fileNb=0; fileNb dstSizeRemaining) { + DEBUGOUTPUT("Warning: dstSize too small to benchmark completely \n"); + remaining = dstSizeRemaining; + dstSizeRemaining = 0; + } else { + dstSizeRemaining -= remaining; + } + for ( ; nbBlocks candidate before restart } } + return winnerInfo; } @@ -1711,7 +2135,7 @@ static winnerInfo_t optimizeFixedStrategy( } while(i < 10) { //make i adjustable (user input?) depending on how much time they have. - DISPLAY("Restart\n"); //TODO: make better printing across restarts + DEBUGOUTPUT("Restart\n"); //look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, init); @@ -1918,7 +2342,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* find best solution from default params */ { /* strategy selection */ - //TODO: don't factor memory into strategy selection in constraint_t const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); DEBUGOUTPUT("Strategy Selection\n"); if(varLen == NUM_PARAMS && paramTarget.strategy == 0) { /* no variable based constraints */ @@ -2019,7 +2442,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(objective_lt(winner.result, wc.result)) { winner = wc; } - //TODO: we could double back to increase search of 'better' strategies + //We could double back to increase search of 'better' strategies st = nextStrategy(st, bestStrategy); } } else { From 0f91b039ff934c4fd42d292a90ea62b52cb2d696 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 16 Jul 2018 18:04:57 -0700 Subject: [PATCH 31/92] Add Levels --- tests/paramgrill.c | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) mode change 100644 => 100755 tests/paramgrill.c diff --git a/tests/paramgrill.c b/tests/paramgrill.c old mode 100644 new mode 100755 index e4d468846..086931b3d --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -56,7 +56,7 @@ static const int g_maxNbVariations = 64; #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define TIMED 0 #ifndef DEBUG -# define DEBUG 0 +# define DEBUG 1 #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } @@ -2227,7 +2227,7 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { } //optimize fixed strategy. -static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget) +static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { size_t benchedSize; void* origBuff = NULL; @@ -2238,7 +2238,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ size_t* fileSizes = calloc(sizeof(size_t),nbFiles); const int varLen = variableParams(paramTarget, varArray); U8** allMT = NULL; - g_targetConstraints = target; g_winner.result.cSize = (size_t)-1; /* Init */ if(!cParamValid(paramTarget)) { @@ -2310,6 +2309,30 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } + //TODO: cLevel Stuff. + if(cLevel) { + BMK_result_t candidate; + const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; + ZSTD_CCtx* const ctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + ZSTD_compressionParameters const CParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); + if(BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams)) { + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + ret = 3; + goto _cleanUp; + } + + target.cSpeed = candidate.cSpeed; //TODO: maybe have a small bit of slack here, like x.99? + target.dSpeed = candidate.dSpeed; + BMK_printWinner(stdout, cLevel, candidate, CParams, benchedSize); + + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + } + + g_targetConstraints = target; + /* bench */ DISPLAY("\r%79s\r", ""); if(nbFiles == 1) { @@ -2348,7 +2371,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ BMK_result_t candidate; int feas = 0, i; for (i=1; i<=maxSeeds; i++) { - ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); + ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, dictBufferSize); int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams); BMK_printWinner(stdout, i, candidate, CParams, benchedSize); @@ -2553,6 +2576,7 @@ int main(int argc, const char** argv) const char* dictFileName = 0; U32 optimizer = 0; U32 main_pause = 0; + int optimizerCLevel = 0; constraint_t target = { 0, 0, (U32)-1 }; //0 for anything unset @@ -2584,6 +2608,7 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.cMem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } + //TODO: add Level; /* in MB or MB/s */ DISPLAY("invalid optimization parameter \n"); return 1; @@ -2692,6 +2717,10 @@ int main(int argc, const char** argv) argument++; paramTarget.strategy = (ZSTD_strategy)readU32FromChar(&argument); continue; + case 'L': /* level centers around a level */ + argument++; + optimizerCLevel = (int)readU32FromChar(&argument); + continue; default : ; } break; @@ -2796,7 +2825,7 @@ int main(int argc, const char** argv) } } else { if (optimizer) { - result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget); + result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, optimizerCLevel); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart); } } From df026e159f7e5ff57a0caa5817176b00c059bc49 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 16 Jul 2018 18:22:04 -0700 Subject: [PATCH 32/92] Fix windows implicit casting bugs --- programs/bench.c | 4 ++-- tests/paramgrill.c | 40 ++++++++++++++++++++-------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 8e57db4c9..a34b74cf1 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -696,7 +696,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } DISPLAYLEVEL(2, "%2i#\n", cLevel); } /* Bench */ - results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); + results.result.cMem = (1ULL << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); results.error = 0; return results; } @@ -731,7 +731,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, void* resultBuffer = malloc(srcSize); - BMK_return_t results; + BMK_return_t results = { { 0, 0, 0, 0 }, 0 }; int allocationincomplete; if(!dstCapacity) { diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 086931b3d..02811b9c1 100755 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -263,7 +263,9 @@ static int epsilonEqual(const double c1, const double c2) { /* checks exact equivalence to 0, to stop compiler complaining fpeq */ static int eqZero(const double c1) { - return (U64)c1 == (U64)0.0 || (U64)c1 == (U64)-0.0; + const double z1 = 0.0; + const double z2 = -0.0; + return !(memcmp(&c1, &z1, sizeof(double))) || !(memcmp(&c1, &z2, sizeof(double))); } /* returns 1 if result2 is strictly 'better' than result1 */ @@ -318,7 +320,7 @@ const char* g_stratName[ZSTD_btultra+1] = { "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -static size_t +static int BMK_benchParam(BMK_result_t* resultPtr, const void* srcBuffer, const size_t srcSize, const size_t* fileSizes, const unsigned nbFiles, @@ -331,7 +333,7 @@ BMK_benchParam(BMK_result_t* resultPtr, } /* benchParam but only takes in one file. */ -static size_t +static int BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, @@ -1204,7 +1206,7 @@ static ZSTD_compressionParameters randomParams(void) /* Sets pc to random unmeasured set of parameters */ static void randomConstrainedParams(ZSTD_compressionParameters* pc, U32* varArray, int varLen, U8* memoTable) { - int tries = memoTableLen(varArray, varLen); //configurable, + size_t tries = memoTableLen(varArray, varLen); //configurable, const size_t maxSize = memoTableLen(varArray, varLen); size_t ind; do { @@ -1470,7 +1472,7 @@ static int feasibleBench(BMK_result_t* resultPtr, loopDurationC = 0; uncertaintyConstantC = 2; } else { - loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + loopDurationC = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? //possibly has to do with initCCtx? or system stuff? //asymmetric +/- constant needed? @@ -1480,7 +1482,7 @@ static int feasibleBench(BMK_result_t* resultPtr, loopDurationD = 0; uncertaintyConstantD = 2; } else { - loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + loopDurationD = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? //possibly has to do with initCCtx? or system stuff? //asymmetric +/- constant needed? @@ -1635,7 +1637,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, loopDurationC = 0; uncertaintyConstantC = 2; } else { - loopDurationC = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + loopDurationC = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC * 1.1), 3); //.02 seconds } @@ -1643,7 +1645,7 @@ static int infeasibleBench(BMK_result_t* resultPtr, loopDurationD = 0; uncertaintyConstantD = 2; } else { - loopDurationD = ((srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + loopDurationD = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1 , 3); //.02 seconds } @@ -1725,7 +1727,7 @@ static int feasibleBenchMemo(BMK_result_t* resultPtr, return INFEASIBLE_RESULT; } else { const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); @@ -1798,7 +1800,7 @@ static int infeasibleBenchMemo(BMK_result_t* resultPtr, return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns } else { const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); @@ -2279,7 +2281,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } { - U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); + U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); int ec; unsigned i; benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; @@ -2290,7 +2292,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ ret = 1; goto _cleanUp; } - ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, nbFiles); + ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, (U32)nbFiles); if(ec) { DISPLAY("Error Loading Files"); ret = ec; @@ -2308,23 +2310,21 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ ret = 2; goto _cleanUp; } - - //TODO: cLevel Stuff. + if(cLevel) { BMK_result_t candidate; const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); ZSTD_compressionParameters const CParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); - if(BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams)) { + if(BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, CParams)) { ZSTD_freeCCtx(ctx); ZSTD_freeDCtx(dctx); ret = 3; goto _cleanUp; } - target.cSpeed = candidate.cSpeed; //TODO: maybe have a small bit of slack here, like x.99? - target.dSpeed = candidate.dSpeed; + target.cSpeed = (U32)candidate.cSpeed; //Maybe have a small bit of slack here, like x.99? BMK_printWinner(stdout, cLevel, candidate, CParams, benchedSize); ZSTD_freeCCtx(ctx); @@ -2372,7 +2372,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int feas = 0, i; for (i=1; i<=maxSeeds; i++) { ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, dictBufferSize); - int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, CParams); + int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, CParams); BMK_printWinner(stdout, i, candidate, CParams, benchedSize); if(!ec) { @@ -2408,7 +2408,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ randomConstrainedParams(&candidateParams, varNew, varLenNew, allMT[i]); cParamZeroMin(&candidateParams); candidateParams = sanitizeParams(candidateParams); - ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, nbFiles, ctx, dctx, candidateParams); + ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, candidateParams); if(!ec) { if(feas) { @@ -2608,7 +2608,7 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.cMem = readU32FromChar(&argument) * 1000000; if (argument[0]==',') { argument++; continue; } else break; } - //TODO: add Level; + if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { optimizerCLevel = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } /* in MB or MB/s */ DISPLAY("invalid optimization parameter \n"); return 1; From e148db366ea424bbc2dc7269c566adb97cc11ec6 Mon Sep 17 00:00:00 2001 From: George Lu Date: Fri, 20 Jul 2018 14:35:09 -0700 Subject: [PATCH 33/92] Separate capacity vs size Also: Make suggested fixes -varInds_t -reorder some arguments -remove code duplication -update README / -h -Fix memory leaks --- programs/bench.c | 63 +- programs/bench.h | 8 +- tests/README.md | 32 + tests/fullbench.c | 2 +- tests/paramgrill.c | 1963 ++++++++++++++++---------------------------- 5 files changed, 779 insertions(+), 1289 deletions(-) mode change 100755 => 100644 tests/paramgrill.c diff --git a/programs/bench.c b/programs/bench.c index a34b74cf1..177dbe0e3 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -171,6 +171,8 @@ struct BMK_timeState_t{ static void BMK_initCCtx(ZSTD_CCtx* ctx, const void* dictBuffer, size_t dictBufferSize, int cLevel, const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { + ZSTD_CCtx_reset(ctx); + ZSTD_CCtx_resetParameters(ctx); if (adv->nbWorkers==1) { ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0); } else { @@ -195,6 +197,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, static void BMK_initDCtx(ZSTD_DCtx* dctx, const void* dictBuffer, size_t dictBufferSize) { + ZSTD_DCtx_reset(dctx); ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); } @@ -291,9 +294,9 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, size_t* dstBlockCapacitiesToSizes, + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* cSizes, unsigned nbLoops) { - size_t srcSize = 0, dstSize = 0, ind = 0; + size_t dstSize = 0; U64 totalTime; BMK_customReturn_t retval; @@ -303,36 +306,37 @@ BMK_customReturn_t BMK_benchFunction( EXM_THROW_ND(1, BMK_customReturn_t, "nbLoops must be nonzero \n"); } - for(ind = 0; ind < blockCount; ind++) { - srcSize += srcBlockSizes[ind]; - } - { size_t i; for(i = 0; i < blockCount; i++) { - memset(dstBlockBuffers[i], 0xE5, dstBlockCapacitiesToSizes[i]); /* warm up and erase result buffer */ + memset(dstBlockBuffers[i], 0xE5, dstBlockCapacities[i]); /* warm up and erase result buffer */ } - - //UTIL_sleepMilli(5); /* give processor time to other processes */ - //UTIL_waitForNextTick(); +#if 0 + /* based on testing these seem to lower accuracy of multiple calls of 1 nbLoops vs 1 call of multiple nbLoops + * (Makes former slower) + */ + UTIL_sleepMilli(5); /* give processor time to other processes */ + UTIL_waitForNextTick(); +#endif } { - unsigned i, j, firstIter = 1; + unsigned i, j; clockStart = UTIL_getTime(); if(initFn != NULL) { initFn(initPayload); } for(i = 0; i < nbLoops; i++) { for(j = 0; j < blockCount; j++) { - size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacitiesToSizes[j], benchPayload); + size_t res = benchFn(srcBlockBuffers[j], srcBlockSizes[j], dstBlockBuffers[j], dstBlockCapacities[j], benchPayload); if(ZSTD_isError(res)) { EXM_THROW_ND(2, BMK_customReturn_t, "Function benchmarking failed on block %u of size %u : %s \n", - j, (U32)dstBlockCapacitiesToSizes[j], ZSTD_getErrorName(res)); - } else if(firstIter) { + j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); + } else if(i == nbLoops - 1) { dstSize += res; - dstBlockCapacitiesToSizes[j] = res; + if(cSizes != NULL) { + cSizes[j] = res; + } } } - firstIter = 0; } totalTime = UTIL_clockSpanNano(clockStart); } @@ -369,7 +373,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const* const srcBlockBuffers, const size_t* srcBlockSizes, - void * const * const dstBlockBuffers, size_t * dstBlockCapacitiesToSizes) + void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* dstSizes) { U64 fastest = cont->fastestTime; int completed = 0; @@ -384,9 +388,9 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( UTIL_sleep(COOLPERIOD_SEC); cont->coolTime = UTIL_getTime(); } - + /* reinitialize capacity */ r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload, - blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacitiesToSizes, cont->nbLoops); + blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, dstSizes, cont->nbLoops); if(r.result.error) { /* completed w/ error */ r.completed = 1; return r; @@ -420,7 +424,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( /* benchMem with no allocation */ static BMK_return_t BMK_benchMemAdvancedNoAlloc( const void ** const srcPtrs, size_t* const srcSizes, - void** const cPtrs, size_t* const cSizes, + void** const cPtrs, size_t* const cCapacities, size_t* const cSizes, void** const resPtrs, size_t* const resSizes, void** resultBufferPtr, void* compressedBuffer, const size_t maxCompressedSize, @@ -485,11 +489,11 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( srcPtrs[nbBlocks] = (const void*)srcPtr; srcSizes[nbBlocks] = thisBlockSize; cPtrs[nbBlocks] = (void*)cPtr; - cSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize); + cCapacities[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize); resPtrs[nbBlocks] = (void*)resPtr; resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize; srcPtr += thisBlockSize; - cPtr += cSizes[nbBlocks]; + cPtr += cCapacities[nbBlocks]; resPtr += thisBlockSize; remaining -= thisBlockSize; } @@ -540,7 +544,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) { if(!intermediateResultCompress.completed) { intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes); + nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes); if(intermediateResultCompress.result.error) { results.error = intermediateResultCompress.result.error; return results; @@ -564,7 +568,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if(!intermediateResultDecompress.completed) { intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, - nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes); + nbBlocks, (const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL); if(intermediateResultDecompress.result.error) { results.error = intermediateResultDecompress.result.error; return results; @@ -590,7 +594,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if(adv->mode != BMK_decodeOnly) { BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, cPtrs, cSizes, adv->nbSeconds); + nbBlocks, srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, adv->nbSeconds); if(compressionResults.error) { results.error = compressionResults.error; return results; @@ -617,7 +621,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( BMK_customReturn_t decompressionResults = BMK_benchFunction( &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, nbBlocks, - (const void* const*)cPtrs, cSizes, resPtrs, resSizes, + (const void* const*)cPtrs, cSizes, resPtrs, resSizes, NULL, adv->nbSeconds); if(decompressionResults.error) { results.error = decompressionResults.error; @@ -717,8 +721,10 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const cPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const cSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + size_t* const cCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); @@ -744,13 +750,11 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes; if (!allocationincomplete) { - results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cSizes, + results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, resPtrs, resSizes, &resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, ctx, dctx, displayLevel, displayName, adv); } - - /* clean up */ BMK_freeTimeState(timeStateCompress); @@ -764,6 +768,7 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, free(srcSizes); free(cPtrs); free(cSizes); + free(cCapacities); free(resPtrs); free(resSizes); diff --git a/programs/bench.h b/programs/bench.h index 25d9f24a7..2a9945ac8 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -177,6 +177,7 @@ typedef size_t (*BMK_initFn_t)(void*); * dstBuffers - an array of buffers to be written into by benchFn * dstCapacitiesToSizes - an array of the capacities of above buffers. Output modified to compressed sizes of those blocks. * nbLoops - defines number of times benchFn is run. + * assumed array of size blockCount, will have compressed size of each block written to it. * return * .error will give a nonzero value if ZSTD_isError() is nonzero for any of the return * of the calls to initFn and benchFn, or if benchFunction errors internally @@ -187,12 +188,11 @@ typedef size_t (*BMK_initFn_t)(void*); * into dstBuffer, hence this value will be the total amount of bytes written to * dstBuffer. */ -BMK_customReturn_t BMK_benchFunction( - BMK_benchFn_t benchFn, void* benchPayload, +BMK_customReturn_t BMK_benchFunction(BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, - void * const * const dstBuffers, size_t* dstCapacitiesToSizes, + void * const * const dstBuffers, const size_t* dstCapacities, size_t* cSizes, unsigned nbLoops); @@ -221,7 +221,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, size_t* dstBlockCapacitiesToSizes); + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* cSizes); #endif /* BENCH_H_121279284357 */ diff --git a/tests/README.md b/tests/README.md index 24a28ab7b..8bedd0a3c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -88,3 +88,35 @@ as well as the 10,000 original files for more detailed comparison of decompressi will choose a random seed, and for 1 minute, generate random test frames and ensure that the zstd library correctly decompresses them in both simple and streaming modes. + +#### `paramgrill` - tool for generating compression table parameters and optimizing parameters on file given constraints + +Full list of arguments +``` + -T# : set level 1 speed objective + -B# : cut input into blocks of size # (default : single block) + -i# : iteration loops + -S : benchmarks a single run (example command: -Sl3w10h12) + w# - windowLog + h# - hashLog + c# - chainLog + s# - searchLog + l# - searchLength + t# - targetLength + S# - strategy + L# - level + --zstd= : Single run, parameter selection syntax same as zstdcli + --optimize= : find parameters to maximize compression ratio given parameters + Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints + cSpeed= - Minimum compression speed + dSpeed= - Minimum decompression speed + cMem= - compression memory + lvl= - Automatically sets compression speed constraint to the speed of that level + --optimize= : same as -O with more verbose syntax + -P# : generated sample compressibility + -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) + -v : Prints Benchmarking output + -D : Next argument dictionary file + +``` + Any inputs afterwards are treated as files to benchmark. diff --git a/tests/fullbench.c b/tests/fullbench.c index 9e7639f92..270cac86a 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -516,7 +516,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, { r = BMK_benchFunction(benchFunction, buff2, NULL, NULL, 1, &src, &srcSize, - (void **)&dstBuff, &dstBuffSize, g_nbIterations); + (void **)&dstBuff, &dstBuffSize, NULL, g_nbIterations); if(r.error) { DISPLAY("ERROR %d ! ! \n", r.error); errorcode = r.error; diff --git a/tests/paramgrill.c b/tests/paramgrill.c old mode 100755 new mode 100644 index 02811b9c1..099be3688 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -56,7 +56,7 @@ static const int g_maxNbVariations = 64; #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define TIMED 0 #ifndef DEBUG -# define DEBUG 1 +# define DEBUG 0 #endif #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } @@ -67,14 +67,15 @@ static const int g_maxNbVariations = 64; #define CUSTOM_LEVEL 99 /* indices for each of the variables */ -#define WLOG_IND 0 -#define CLOG_IND 1 -#define HLOG_IND 2 -#define SLOG_IND 3 -#define SLEN_IND 4 -#define TLEN_IND 5 -//#define STRT_IND 6 -//#define NUM_PARAMS 7 +typedef enum { + wlog_ind = 0, + clog_ind = 1, + hlog_ind = 2, + slog_ind = 3, + slen_ind = 4, + tlen_ind = 5 +} varInds_t; + #define NUM_PARAMS 6 //just don't use strategy as a param. @@ -85,20 +86,16 @@ static const int g_maxNbVariations = 64; #define ZSTD_TARGETLENGTH_MIN 0 #define ZSTD_TARGETLENGTH_MAX 999 -//#define ZSTD_TARGETLENGTH_MAX 1024 #define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1) #define CLOG_RANGE (ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1) #define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1) #define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1) #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) -#define TLEN_RANGE 12 -//TLEN_RANGE = 0, 2^0 to 2^10; -//hard coded since we only use powers of 2 (and 999 ~ 1024) +#define TLEN_RANGE 17 +/* TLEN_RANGE picked manually */ -//static const int mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN }; -//static const int maxtable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX }; static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; - +static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; /*-************************************ * Benchmark Parameters **************************************/ @@ -179,9 +176,6 @@ static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) return result; } -//assume that clock can at least measure .01 second intervals? -//make this a settable global initialized with fn? -//#define CLOCK_GRANULARITY 100000000ULL static U64 g_clockGranularity = 100000000ULL; static void findClockGranularity(void) { @@ -253,7 +247,7 @@ static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params) /* checks results are feasible */ static int feasible(const BMK_result_t results, const constraint_t target) { - return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem || !target.cMem); + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem); } #define EPSILON 0.001 @@ -268,13 +262,6 @@ static int eqZero(const double c1) { return !(memcmp(&c1, &z1, sizeof(double))) || !(memcmp(&c1, &z2, sizeof(double))); } -/* returns 1 if result2 is strictly 'better' than result1 */ -/* strict comparison / cutoff based */ -static int objective_lt(const BMK_result_t result1, const BMK_result_t result2) { - return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) - || (result1.cSize == result2.cSize && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); -} - /* hill climbing value for part 1 */ static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) { double cs = 0., ds = 0., rt, cm = 0.; @@ -291,6 +278,16 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co return ret; } +/* return true if r2 strictly better than r1 */ +static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) { + if(feasible(result1, target) && feasible(result2, target)) { + return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) + || (result1.cSize == result2.cSize && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); + } + return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); + +} + /* factor sort of arbitrary */ static constraint_t relaxTarget(constraint_t target) { target.cMem = (U32)-1; @@ -303,35 +300,11 @@ static constraint_t relaxTarget(constraint_t target) { * Bench functions *********************************************************/ -typedef struct -{ - const char* srcPtr; - size_t srcSize; - char* cPtr; - size_t cRoom; - size_t cSize; - char* resPtr; - size_t resSize; -} blockParam_t; - - const char* g_stratName[ZSTD_btultra+1] = { "(none) ", "ZSTD_fast ", "ZSTD_dfast ", "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -static int -BMK_benchParam(BMK_result_t* resultPtr, - const void* srcBuffer, const size_t srcSize, - const size_t* fileSizes, const unsigned nbFiles, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - const ZSTD_compressionParameters cParams) { - - BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, fileSizes, nbFiles, 0, &cParams, NULL, 0, ctx, dctx, 0, "File"); - *resultPtr = res.result; - return res.error; -} - /* benchParam but only takes in one file. */ static int BMK_benchParam1(BMK_result_t* resultPtr, @@ -349,6 +322,49 @@ typedef struct { ZSTD_compressionParameters params; } winnerInfo_t; +static ZSTD_compressionParameters emptyParams(void) { + ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 }; + return p; +} + +static winnerInfo_t initWinnerInfo(ZSTD_compressionParameters p) { + winnerInfo_t w1; + w1.result.cSpeed = 0.; + w1.result.dSpeed = 0.; + w1.result.cMem = (size_t)-1; + w1.result.cSize = (size_t)-1; + w1.params = p; + return w1; +} + +typedef struct { + size_t srcSize; + void** srcPtrs; + size_t* srcSizes; + void** dstPtrs; + size_t* dstCapacities; + size_t* dstSizes; + void** resPtrs; + size_t* resSizes; + size_t nbBlocks; +} buffers_t; + +typedef struct { + size_t dictSize; + void* dictBuffer; + ZSTD_CCtx* cctx; + ZSTD_DCtx* dctx; +} contexts_t; + +static int +BMK_benchParam(BMK_result_t* resultPtr, + buffers_t buf, contexts_t ctx, + const ZSTD_compressionParameters cParams) { + BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, ctx.cctx, ctx.dctx, 0, "Files"); + *resultPtr = res.result; + return res.error; +} + /*-******************************************************* * From Paramgrill *********************************************************/ @@ -356,6 +372,8 @@ typedef struct { static void BMK_initCCtx(ZSTD_CCtx* ctx, const void* dictBuffer, size_t dictBufferSize, int cLevel, const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { + ZSTD_CCtx_reset(ctx); + ZSTD_CCtx_resetParameters(ctx); if (adv->nbWorkers==1) { ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0); } else { @@ -380,6 +398,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, static void BMK_initDCtx(ZSTD_DCtx* dctx, const void* dictBuffer, size_t dictBufferSize) { + ZSTD_DCtx_reset(dctx); ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); } @@ -425,6 +444,7 @@ static size_t local_defaultCompress( out.dst = dstBuffer; out.size = dstSize; out.pos = 0; + assert(dstSize == ZSTD_compressBound(srcSize)); /* specific to this version, which is only used in paramgrill */ while (moreToFlush) { if(out.pos == out.size) { return (size_t)-ZSTD_error_dstSize_tooSmall; @@ -471,249 +491,164 @@ static size_t local_defaultDecompress( *********************************************************/ /* Replicate function of benchMemAdvanced, but with pre-split src / dst buffers, with relevant info to invert it (compressedSizes) passed out. */ -/*BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ +/* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ /* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */ /* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ /* dictionary nullable, nothing else though. */ -static BMK_return_t BMK_benchMemInvertible(const void * const * const srcPtrs, size_t const * const srcSizes, - void** dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, +static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, const int cLevel, const ZSTD_compressionParameters* comprParams, - const void* dictBuffer, const size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) { + U32 i; BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; - size_t srcSize = 0; - void** const resPtrs = malloc(sizeof(void*) * nbBlocks); /* only really needed in both mode. */ - size_t* const resSizes = malloc(sizeof(size_t) * nbBlocks); - int freeDST = 0; + const void *const *const srcPtrs = (const void *const *const)buf.srcPtrs; + size_t const *const srcSizes = buf.srcSizes; + void** dstPtrs = buf.dstPtrs; + size_t* dstCapacities = buf.dstCapacities; + size_t* dstSizes = buf.dstSizes; + void** resPtrs = buf.resPtrs; + size_t* resSizes = buf.resSizes; + const void* dictBuffer = ctx.dictBuffer; + const size_t dictBufferSize = ctx.dictSize; + const size_t nbBlocks = buf.nbBlocks; + const size_t srcSize = buf.srcSize; + ZSTD_CCtx* cctx = ctx.cctx; + ZSTD_DCtx* dctx = ctx.dctx; BMK_advancedParams_t adv = BMK_initAdvancedParams(); adv.mode = mode; adv.loopMode = loopMode; adv.nbSeconds = nbSeconds; - /* resSizes == srcSizes, but modifiable */ - memcpy(resSizes, srcSizes, sizeof(size_t) * nbBlocks); - - for(i = 0; i < nbBlocks; i++) { - srcSize += srcSizes[i]; - } - - if(!ctx || !dctx || !srcPtrs || ! srcSizes) - { - results.error = 31; - DISPLAY("error: passed in null argument\n"); - free(resPtrs); - free(resSizes); - return results; - } - if(!resPtrs || !resSizes) { - results.error = 32; - DISPLAY("error: allocation failed\n"); - free(resPtrs); - free(resSizes); - return results; - } - - /* so resPtr is continuous */ - resPtrs[0] = malloc(srcSize); - - if(!(resPtrs[0])) { - results.error = 32; - DISPLAY("error: allocation failed\n"); - free(resPtrs); - free(resSizes); - return results; - } - - for(i = 1; i < nbBlocks; i++) { - resPtrs[i] = (void*)(((char*)resPtrs[i-1]) + srcSizes[i-1]); - } - - /* allocate own dst if NULL */ - if(dstPtrs == NULL) { - freeDST = 1; - dstPtrs = malloc(nbBlocks * sizeof(void*)); - dstCapacityToSizes = malloc(nbBlocks * sizeof(size_t)); - if(dstPtrs == NULL) { - results.error = 33; - DISPLAY("error: allocation failed\n"); - free(resPtrs); - free(resSizes); - return results; - } - - if(mode == BMK_decodeOnly) { //dst is src - size_t dstSize = 0; - for(i = 0; i < nbBlocks; i++) { - dstCapacityToSizes[i] = ZSTD_getDecompressedSize(srcPtrs[i], srcSizes[i]); - dstSize += dstCapacityToSizes[i]; - } - dstPtrs[0] = malloc(dstSize); - if(dstPtrs[0] == NULL) { - results.error = 34; - DISPLAY("error: allocation failed\n"); - goto _cleanUp; - } - for(i = 1; i < nbBlocks; i++) { - dstPtrs[i] = (void*)(((char*)dstPtrs[i-1]) + ZSTD_getDecompressedSize(srcPtrs[i-1], srcSizes[i-1])); - } - } else { - dstPtrs[0] = malloc(ZSTD_compressBound(srcSize) + (nbBlocks * 1024)); - if(dstPtrs[0] == NULL) { - results.error = 35; - DISPLAY("error: allocation failed\n"); - goto _cleanUp; - } - dstCapacityToSizes[0] = ZSTD_compressBound(srcSizes[0]); - for(i = 1; i < nbBlocks; i++) { - dstPtrs[i] = (void*)(((char*)dstPtrs[i-1]) + dstCapacityToSizes[i-1]); - dstCapacityToSizes[i] = ZSTD_compressBound(srcSizes[i]); - } - } - } - /* warmimg up memory */ - for(i = 0; i < nbBlocks; i++) { - RDG_genBuffer(dstPtrs[i], dstCapacityToSizes[i], 0.10, 0.50, 1); + /* can't do this if decode only */ + for(i = 0; i < buf.nbBlocks; i++) { + if(mode != BMK_decodeOnly) { + RDG_genBuffer(dstPtrs[i], dstCapacities[i], 0.10, 0.50, 1); + } else { + RDG_genBuffer(resPtrs[i], resSizes[i], 0.10, 0.50, 1); + } } /* Bench */ - { - { - BMK_initCCtxArgs cctxprep; - BMK_initDCtxArgs dctxprep; - cctxprep.ctx = ctx; - cctxprep.dictBuffer = dictBuffer; - cctxprep.dictBufferSize = dictBufferSize; - cctxprep.cLevel = cLevel; - cctxprep.comprParams = comprParams; - cctxprep.adv = &adv; - dctxprep.dctx = dctx; - dctxprep.dictBuffer = dictBuffer; - dctxprep.dictBufferSize = dictBufferSize; - if(loopMode == BMK_timeMode) { - BMK_customTimedReturn_t intermediateResultCompress; - BMK_customTimedReturn_t intermediateResultDecompress; - BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(nbSeconds); - BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(nbSeconds); - if(mode == BMK_compressOnly) { - intermediateResultCompress.completed = 0; - intermediateResultDecompress.completed = 1; - } else if (mode == BMK_decodeOnly) { - intermediateResultCompress.completed = 1; - intermediateResultDecompress.completed = 0; - } else { /* both */ - intermediateResultCompress.completed = 0; - intermediateResultDecompress.completed = 0; + + { + /* init args */ + BMK_initCCtxArgs cctxprep; + BMK_initDCtxArgs dctxprep; + cctxprep.ctx = cctx; + cctxprep.dictBuffer = dictBuffer; + cctxprep.dictBufferSize = dictBufferSize; + cctxprep.cLevel = cLevel; + cctxprep.comprParams = comprParams; + cctxprep.adv = &adv; + dctxprep.dctx = dctx; + dctxprep.dictBuffer = dictBuffer; + dctxprep.dictBufferSize = dictBufferSize; + + if(loopMode == BMK_timeMode) { + BMK_customTimedReturn_t intermediateResultCompress; + BMK_customTimedReturn_t intermediateResultDecompress; + BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(nbSeconds); + BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(nbSeconds); + if(mode == BMK_compressOnly) { + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 1; + } else if (mode == BMK_decodeOnly) { + intermediateResultCompress.completed = 1; + intermediateResultDecompress.completed = 0; + } else { /* both */ + intermediateResultCompress.completed = 0; + intermediateResultDecompress.completed = 0; + } + + while(!intermediateResultCompress.completed) { + intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes); + + if(intermediateResultCompress.result.error) { + results.error = intermediateResultCompress.result.error; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + return results; } - while(!(intermediateResultCompress.completed && intermediateResultDecompress.completed)) { - if(!intermediateResultCompress.completed) { - intermediateResultCompress = BMK_benchFunctionTimed(timeStateCompress, &local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes); - if(intermediateResultCompress.result.error) { - results.error = intermediateResultCompress.result.error; - BMK_freeTimeState(timeStateCompress); - BMK_freeTimeState(timeStateDecompress); - goto _cleanUp; - } - results.result.cSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000000000; - results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; - } + results.result.cSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * TIMELOOP_NANOSEC; + results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; + } - if(!intermediateResultDecompress.completed) { - if(mode == BMK_decodeOnly) { - intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, - nbBlocks, (const void* const*)srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes); - } else { /* both, decompressed result already written to dstPtr */ - intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, - nbBlocks, (const void* const*)dstPtrs, dstCapacityToSizes, resPtrs, resSizes); - } + while(!intermediateResultDecompress.completed) { + intermediateResultDecompress = BMK_benchFunctionTimed(timeStateDecompress, &local_defaultDecompress, (void*)(dctx), &local_initDCtx, (void*)&dctxprep, + nbBlocks, (const void* const*)dstPtrs, dstSizes, resPtrs, resSizes, NULL); - if(intermediateResultDecompress.result.error) { - results.error = intermediateResultDecompress.result.error; - BMK_freeTimeState(timeStateCompress); - BMK_freeTimeState(timeStateDecompress); - goto _cleanUp; - } - results.result.dSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000000000; - } + if(intermediateResultDecompress.result.error) { + results.error = intermediateResultDecompress.result.error; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + return results; } - BMK_freeTimeState(timeStateCompress); - BMK_freeTimeState(timeStateDecompress); - } else { //iterMode; - if(mode != BMK_decodeOnly) { + results.result.dSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * TIMELOOP_NANOSEC; + } - BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)ctx, &local_initCCtx, (void*)&cctxprep, - nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbSeconds); - if(compressionResults.error) { - results.error = compressionResults.error; - goto _cleanUp; - } - if(compressionResults.result.nanoSecPerRun == 0) { - results.result.cSpeed = 0; - } else { - results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; - } - results.result.cSize = compressionResults.result.sumOfReturn; + BMK_freeTimeState(timeStateCompress); + BMK_freeTimeState(timeStateDecompress); + + } else { //iterMode; + if(mode != BMK_decodeOnly) { + + BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep, + nbBlocks, srcPtrs, srcSizes, dstPtrs, dstCapacities, dstSizes, nbSeconds); + if(compressionResults.error) { + results.error = compressionResults.error; + return results; } - if(mode != BMK_compressOnly) { - BMK_customReturn_t decompressionResults; - if(mode == BMK_decodeOnly) { - decompressionResults = BMK_benchFunction( - &local_defaultDecompress, (void*)(dctx), - &local_initDCtx, (void*)&dctxprep, nbBlocks, - (const void* const*)srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, - nbSeconds); - } else { - decompressionResults = BMK_benchFunction( - &local_defaultDecompress, (void*)(dctx), - &local_initDCtx, (void*)&dctxprep, nbBlocks, - (const void* const*)dstPtrs, dstCapacityToSizes, resPtrs, resSizes, - nbSeconds); - } + if(compressionResults.result.nanoSecPerRun == 0) { + results.result.cSpeed = 0; + } else { + results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + } + results.result.cSize = compressionResults.result.sumOfReturn; + } - if(decompressionResults.error) { - results.error = decompressionResults.error; - goto _cleanUp; - } + if(mode != BMK_compressOnly) { + BMK_customReturn_t decompressionResults; + decompressionResults = BMK_benchFunction( + &local_defaultDecompress, (void*)(dctx), + &local_initDCtx, (void*)&dctxprep, nbBlocks, + (const void* const*)dstPtrs, dstSizes, resPtrs, resSizes, NULL, + nbSeconds); - if(decompressionResults.result.nanoSecPerRun == 0) { - results.result.dSpeed = 0; - } else { - results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; - } + if(decompressionResults.error) { + results.error = decompressionResults.error; + return results; + } + + if(decompressionResults.result.nanoSecPerRun == 0) { + results.result.dSpeed = 0; + } else { + results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; } } } - } /* Bench */ - results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(ctx); - -_cleanUp: - free(resPtrs[0]); - free(resPtrs); - free(resSizes); - if(freeDST) { - free(dstPtrs[0]); - free(dstPtrs); } + /* Bench */ + results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx); return results; } /* global winner used for display. */ //Should be totally 0 initialized? -static winnerInfo_t g_winner; +static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; static constraint_t g_targetConstraints; static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) { - if(DEBUG || (objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints))) { + if(DEBUG || compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { char lvlstr[15] = "Custom Level"; const U64 time = UTIL_clockSpanNano(g_time); const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); - if(DEBUG && (objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints))) { + + if(DEBUG && compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); } @@ -722,27 +657,24 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, g_stratName[(U32)(params.strategy)]); + if(cLevel != CUSTOM_LEVEL) { snprintf(lvlstr, 15, " Level %2u ", cLevel); } + fprintf(f, "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", - lvlstr, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); + lvlstr, (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20)); if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } fprintf(f, "\n"); - if(objective_lt(g_winner.result, result) && feasible(result, g_targetConstraints)) { + + if(compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { BMK_translateAdvancedParams(params); g_winner.result = result; g_winner.params = params; } } - //else { - // DISPLAY("G_WINNER: "); - // DISPLAY("/* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",(double)srcSize / g_winner.result.cSize , g_winner.result.cSpeed / 1000000 , g_winner.result.dSpeed / 1000000); - // DISPLAY("LOSER : "); - // DISPLAY("/* R:%5.3f at %5.1f MB/s - %5.1f MB/s */ \n",(double)srcSize / result.cSize, result.cSpeed / 1000000 , result.dSpeed / 1000000); - //} } static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) @@ -903,23 +835,23 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters params) { if (params.strategy == ZSTD_fast) - g_params.chainLog = 0, g_params.searchLog = 0; + params.chainLog = 0, params.searchLog = 0; if (params.strategy == ZSTD_dfast) - g_params.searchLog = 0; + params.searchLog = 0; if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra && params.strategy != ZSTD_fast) - g_params.targetLength = 0; + params.targetLength = 0; return params; } /* new length */ /* keep old array, will need if iter over strategy. */ -static int sanitizeVarArray(const int varLength, const U32* varArray, U32* varNew, const ZSTD_strategy strat) { +static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInds_t* varArray, const ZSTD_strategy strat) { int i, j = 0; for(i = 0; i < varLength; i++) { - if( !((varArray[i] == CLOG_IND && strat == ZSTD_fast) - || (varArray[i] == SLOG_IND && strat == ZSTD_dfast) - || (varArray[i] == TLEN_IND && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { + if( !((varArray[i] == clog_ind && strat == ZSTD_fast) + || (varArray[i] == slog_ind && strat == ZSTD_dfast) + || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { varNew[j] = varArray[i]; j++; } @@ -930,69 +862,72 @@ static int sanitizeVarArray(const int varLength, const U32* varArray, U32* varNe /* res should be NUM_PARAMS size */ /* constructs varArray from ZSTD_compressionParameters style parameter */ -static int variableParams(const ZSTD_compressionParameters paramConstraints, U32* res) { +static int variableParams(const ZSTD_compressionParameters paramConstraints, varInds_t* res) { int j = 0; if(!paramConstraints.windowLog) { - res[j] = WLOG_IND; + res[j] = wlog_ind; j++; } if(!paramConstraints.chainLog) { - res[j] = CLOG_IND; + res[j] = clog_ind; j++; } if(!paramConstraints.hashLog) { - res[j] = HLOG_IND; + res[j] = hlog_ind; j++; } if(!paramConstraints.searchLog) { - res[j] = SLOG_IND; + res[j] = slog_ind; j++; } if(!paramConstraints.searchLength) { - res[j] = SLEN_IND; + res[j] = slen_ind; j++; } if(!paramConstraints.targetLength) { - res[j] = TLEN_IND; + res[j] = tlen_ind; j++; } return j; } +/* bin-search on tlen_table for correct index. */ +static int tlen_inv(U32 x) { + int lo = 0; + int hi = TLEN_RANGE; + while(lo < hi) { + int mid = (lo + hi) / 2; + if(tlen_table[mid] < x) { + lo = mid + 1; + } if(tlen_table[mid] == x) { + return mid; + } else { + hi = mid; + } + } + return lo; +} + /* amt will probably always be \pm 1? */ /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ /* take max/min bounds into account as well? */ -static void paramVaryOnce(const U32 paramIndex, const int amt, ZSTD_compressionParameters* ptr) { +static void paramVaryOnce(const varInds_t paramIndex, const int amt, ZSTD_compressionParameters* ptr) { switch(paramIndex) { - case WLOG_IND: ptr->windowLog += amt; break; - case CLOG_IND: ptr->chainLog += amt; break; - case HLOG_IND: ptr->hashLog += amt; break; - case SLOG_IND: ptr->searchLog += amt; break; - case SLEN_IND: ptr->searchLength += amt; break; - case TLEN_IND: - if(amt >= 0) { - if(ptr->targetLength == 0) { - if(amt > 0) { - ptr->targetLength = MIN(1 << (amt - 1), 999); - } - } else { - ptr->targetLength <<= amt; - ptr->targetLength = MIN(ptr->targetLength, 999); - } - } else { - if(ptr->targetLength == 999) { - ptr->targetLength = 1024; - } - ptr->targetLength >>= -amt; - } + case wlog_ind: ptr->windowLog += amt; break; + case clog_ind: ptr->chainLog += amt; break; + case hlog_ind: ptr->hashLog += amt; break; + case slog_ind: ptr->searchLog += amt; break; + case slen_ind: ptr->searchLength += amt; break; + case tlen_ind: + ptr->targetLength = tlen_table[MAX(0, MIN(TLEN_RANGE - 1, tlen_inv(ptr->targetLength) + amt))]; break; default: break; } } /* varies ptr by nbChanges respecting varyParams*/ -static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, const U32 nbChanges) +static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) { ZSTD_compressionParameters p; U32 validated = 0; @@ -1009,7 +944,7 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const U32* varyParam } /* length of memo table given free variables */ -static size_t memoTableLen(const U32* varyParams, const int varyLen) { +static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { size_t arrayLen = 1; int i; for(i = 0; i < varyLen; i++) { @@ -1018,53 +953,45 @@ static size_t memoTableLen(const U32* varyParams, const int varyLen) { return arrayLen; } -//sort of ~lg2 (replace 1024 w/ 999, and add 0 at lower end of range) for memoTableInd Tlen -static unsigned lg2(unsigned x) { - if(x == 999) { - return 11; - } - return x ? ZSTD_highbit32(x) + 1 : 0; -} - /* returns unique index of compression parameters */ -static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen) { +static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen) { int i; unsigned ind = 0; for(i = 0; i < varyLen; i++) { switch(varyParams[i]) { - case WLOG_IND: ind *= WLOG_RANGE; ind += ptr->windowLog - ZSTD_WINDOWLOG_MIN ; break; - case CLOG_IND: ind *= CLOG_RANGE; ind += ptr->chainLog - ZSTD_CHAINLOG_MIN ; break; - case HLOG_IND: ind *= HLOG_RANGE; ind += ptr->hashLog - ZSTD_HASHLOG_MIN ; break; - case SLOG_IND: ind *= SLOG_RANGE; ind += ptr->searchLog - ZSTD_SEARCHLOG_MIN ; break; - case SLEN_IND: ind *= SLEN_RANGE; ind += ptr->searchLength - ZSTD_SEARCHLENGTH_MIN; break; - case TLEN_IND: ind *= TLEN_RANGE; ind += lg2(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; + case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog - ZSTD_WINDOWLOG_MIN ; break; + case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog - ZSTD_CHAINLOG_MIN ; break; + case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog - ZSTD_HASHLOG_MIN ; break; + case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog - ZSTD_SEARCHLOG_MIN ; break; + case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength - ZSTD_SEARCHLENGTH_MIN; break; + case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; } } return ind; } /* inverse of above function (from index to parameters) */ -static void memoTableIndInv(ZSTD_compressionParameters* ptr, const U32* varyParams, const int varyLen, size_t ind) { +static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) { int i; for(i = varyLen - 1; i >= 0; i--) { switch(varyParams[i]) { - case WLOG_IND: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; - case CLOG_IND: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; - case HLOG_IND: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; - case SLOG_IND: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; - case SLEN_IND: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; - case TLEN_IND: ptr->targetLength = (ind % TLEN_RANGE) ? MIN(1 << ((ind % TLEN_RANGE) - 1), 999) : 0; ind /= TLEN_RANGE; break; + case wlog_ind: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; + case clog_ind: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; + case hlog_ind: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; + case slog_ind: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; + case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; + case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; ind /= TLEN_RANGE; break; } } } - /* Initialize memotable, immediately mark redundant / obviously infeasible params as */ -static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { +static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog; + int whFixed = !paramConstraints.windowLog || !paramConstraints.hashLog; int wFixed = !paramConstraints.windowLog; int j = 0; memset(memoTable, 0, arrayLen); @@ -1076,7 +1003,7 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra memoTable[i] = 255; j++; } - if(wFixed && (1ULL << paramConstraints.windowLog) > (srcSize << 1)) { + if(wFixed && (1ULL << paramConstraints.windowLog) > (srcSize << 2)) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ @@ -1093,12 +1020,20 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra } } } + if(scFixed) { if(paramConstraints.searchLog > paramConstraints.chainLog) { if(memoTable[i] != 255) { j++; } memoTable[i] = 255; } } + + if(whFixed) { + if(paramConstraints.hashLog > paramConstraints.windowLog + 1) { + if(memoTable[i] != 255) { j++; } + memoTable[i] = 255; + } + } } DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i); if((int)i == j) { @@ -1106,27 +1041,7 @@ static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstra } } -/* inits memotables for all (including mallocs), all strategies */ -/* takes unsanitized varyParams */ -static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const U32* varyParams, const int varyLen, const size_t srcSize) { - U32 varNew[NUM_PARAMS]; - int varLenNew; - U8** mtAll = malloc(sizeof(U8*) * (ZSTD_btultra + 1)); - int i; - if(mtAll == NULL) { - return NULL; - } - for(i = 1; i <= (int)ZSTD_btultra; i++) { - varLenNew = sanitizeVarArray(varyLen, varyParams, varNew, i); - mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); - if(mtAll[i] == NULL) { - return NULL; - } - memoTableInit(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); - } - return mtAll; -} - +/* frees all allocated memotables */ static void memoTableFreeAll(U8** mtAll) { int i; if(mtAll == NULL) { return; } @@ -1136,6 +1051,28 @@ static void memoTableFreeAll(U8** mtAll) { free(mtAll); } +/* inits memotables for all (including mallocs), all strategies */ +/* takes unsanitized varyParams */ +static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { + varInds_t varNew[NUM_PARAMS]; + int varLenNew; + U8** mtAll = calloc(sizeof(U8*),(ZSTD_btultra + 1)); + int i; + if(mtAll == NULL) { + return NULL; + } + for(i = 1; i <= (int)ZSTD_btultra; i++) { + varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); + mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + if(mtAll[i] == NULL) { + memoTableFreeAll(mtAll); + return NULL; + } + memoTableInit(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); + } + return mtAll; +} + #define PARAMTABLELOG 25 #define PARAMTABLESIZE (1< 0 && tries > 0); memoTableIndInv(pc, varArray, varLen, (unsigned)ind); - *pc = sanitizeParams(*pc); + //*pc = sanitizeParams(*pc); } static void BMK_selectRandomStart( @@ -1250,7 +1187,7 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src winnerInfo_t winners[NB_LEVELS_TRACKED+1]; const char* const rfName = "grillResults.txt"; FILE* const f = fopen(rfName, "w"); - const size_t blockSize = g_blockSize ? g_blockSize : srcSize; /* cut by block or not ? */ + const size_t blockSize = g_blockSize ? g_blockSize : ZSTD_BLOCKSIZE_MAX; /* cut by block or not ? */ /* init */ assert(g_singleRun==0); @@ -1390,574 +1327,204 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } - - -/* -checks feasibility with uncertainty. --1 : certainly infeasible - 0 : uncertain - 1 : certainly feasible -*/ -static int uncertainFeasibility(double const uncertaintyConstantC, double const uncertaintyConstantD, const constraint_t paramTarget, const BMK_result_t* const results) { - if((paramTarget.cSpeed != 0 && results->cSpeed * uncertaintyConstantC < paramTarget.cSpeed) || - (paramTarget.dSpeed != 0 && results->dSpeed * uncertaintyConstantD < paramTarget.dSpeed) || - (paramTarget.cMem != 0 && results->cMem > paramTarget.cMem)) { - return -1; - } else if((paramTarget.cSpeed == 0 || results->cSpeed / uncertaintyConstantC > paramTarget.cSpeed) && - (paramTarget.dSpeed == 0 || results->dSpeed / uncertaintyConstantD > paramTarget.dSpeed) && - (paramTarget.cMem == 0 || results->cMem <= paramTarget.cMem)) { - return 1; - } else { - return 0; - } -} - -/* 1 - better than prev best - 0 - uncertain - -1 - worse - assume prev_best status is run fully? - but then we'd have to rerun any winners anyway */ -/* not as useful as initially believed */ -static int uncertainComparison(double const uncertaintyConstantC, double const uncertaintyConstantD, const BMK_result_t* candidate, const BMK_result_t* prevBest) { - (void)uncertaintyConstantD; //unused for now - if(candidate->cSpeed > prevBest->cSpeed * uncertaintyConstantC) { - return 1; - } else if (candidate->cSpeed * uncertaintyConstantC < prevBest->cSpeed) { - return -1; - } else { - return 0; - } -} - /*benchmarks and tests feasibility together 1 = true = better 0 = false = not better if true then resultPtr will give results. 2+ on error? */ -//Maybe use compress_only for benchmark -#define INFEASIBLE_RESULT 0 -#define FEASIBLE_RESULT 1 +//Maybe use compress_only for benchmark first run? +#define WORSE_RESULT 0 +#define BETTER_RESULT 1 #define ERROR_RESULT 2 -static int feasibleBench(BMK_result_t* resultPtr, - const void* const * const srcPtrs, size_t const * const srcSizes, - void** const dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, - void* dictBuffer, const size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - const ZSTD_compressionParameters cParams, - const constraint_t target, - BMK_result_t* winnerResult) { - BMK_return_t benchres; - U64 loopDurationC = 0, loopDurationD = 0; - double uncertaintyConstantC, uncertaintyConstantD; - size_t srcSize = 0; - U32 i; - //alternative - test 1 iter for ratio, (possibility of error 3 which is fine), - //maybe iter this until 2x measurable for better guarantee? - DEBUGOUTPUT("Feas:\n"); - benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_both, BMK_iterMode, 1); - if(benchres.error) { - DISPLAY("ERROR %d !!\n", benchres.error); - } - for(i = 0; i < nbBlocks; i++) { - srcSize += srcSizes[i]; - } - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); +//add worse result complete for worse results of length > 1 sec? - if(!benchres.error) { - *resultPtr = benchres.result; - /* if speed is 0 (only happens when time = 0) */ - if(eqZero(benchres.result.cSpeed)) { - loopDurationC = 0; - uncertaintyConstantC = 2; - } else { - loopDurationC = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? - //possibly has to do with initCCtx? or system stuff? - //asymmetric +/- constant needed? - uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC) * 1.1, 3); //.02 seconds - } - if(eqZero(benchres.result.dSpeed)) { - loopDurationD = 0; - uncertaintyConstantD = 2; - } else { - loopDurationD = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - //problem - tested in fullbench, saw speed vary 3x between iters, maybe raise uncertaintyConstraint up? - //possibly has to do with initCCtx? or system stuff? - //asymmetric +/- constant needed? - uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1, 3); //.02 seconds - } - - - if(benchres.result.cSize < winnerResult->cSize) { //better compression ratio, just needs to be feasible - int feas; - if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres = benchres2; - } - } - if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - *resultPtr = benchres.result; - - feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); - if(feas == 0) { // uncertain feasibility - if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.cSpeed = benchres2.result.cSpeed; - } - } - if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - *resultPtr = benchres.result; - return feasible(benchres.result, target); - } else { //feas = 1 or -1 map to 1, 0 respectively - return (feas + 1) >> 1; //relies on INFEASIBLE_RESULT == 0, FEASIBLE_RESULT == 1 - } - } else if (benchres.result.cSize == winnerResult->cSize) { //equal ratio, needs to be better than winner in cSpeed/ dSpeed / cMem - int feas; - if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres = benchres2; - } - } - if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - feas = uncertainFeasibility(uncertaintyConstantC, uncertaintyConstantD, target, &(benchres.result)); - if(feas == 0) { // uncertain feasibility - if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.cSpeed = benchres2.result.cSpeed; - } - } - if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - - *resultPtr = benchres.result; - return feasible(benchres.result, target) && objective_lt(*winnerResult, benchres.result); - } else if (feas == 1) { //no need to check feasibility compares (maybe only it is chosen as a winner) - int btw = uncertainComparison(uncertaintyConstantC, uncertaintyConstantD, &(benchres.result), winnerResult); - if(btw == -1) { - return INFEASIBLE_RESULT; - } else { //possibly better, benchmark and find out - benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_both, BMK_timeMode, 1); - *resultPtr = benchres.result; - return objective_lt(*winnerResult, benchres.result); - } - } else { //feas == -1 - return INFEASIBLE_RESULT; //infeasible - } - } else { - return INFEASIBLE_RESULT; //infeasible - } - } else { - return ERROR_RESULT; //BMK error - } -} - -//same as before, but +/-? -//alternative, just return comparison result, leave caller to worry about feasibility. -//have version of benchMemAdvanced which takes in dstBuffer/cap as well? -//(motivation: repeat tests (maybe just on decompress) don't need further compress runs) -static int infeasibleBench(BMK_result_t* resultPtr, - const void* const * const srcPtrs, size_t const * const srcSizes, - void** const dstPtrs, size_t* dstCapacityToSizes, U32 const nbBlocks, - void* dictBuffer, const size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, +/* variation between 2nd run and full second bmk */ +#define VARIANCE 1.1 +static int allBench(BMK_result_t* resultPtr, + buffers_t buf, contexts_t ctx, const ZSTD_compressionParameters cParams, const constraint_t target, - BMK_result_t* winnerResult) { + BMK_result_t* winnerResult, int feas) { BMK_return_t benchres; - BMK_result_t resultMin, resultMax; + BMK_result_t resultMax; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; double winnerRS; - size_t srcSize = 0; - U32 i; - benchres = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_both, BMK_iterMode, 1); - for(i = 0; i < nbBlocks; i++) { - srcSize += srcSizes[i]; - } - BMK_printWinner(stdout, CUSTOM_LEVEL, benchres.result, cParams, srcSize); - winnerRS = resultScore(*winnerResult, srcSize, target); + /* initial benchmarking, gives exact ratio and memory, warms up future runs */ + benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1); + winnerRS = resultScore(*winnerResult, buf.srcSize, target); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); - if(!benchres.error) { - *resultPtr = benchres.result; - if(eqZero(benchres.result.cSpeed)) { - loopDurationC = 0; - uncertaintyConstantC = 2; - } else { - loopDurationC = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - uncertaintyConstantC = MIN((loopDurationC + (double)(2 * g_clockGranularity)/loopDurationC * 1.1), 3); //.02 seconds - } + if(benchres.error) { + DEBUGOUTPUT("Benchmarking failed\n"); + return ERROR_RESULT; + } + *resultPtr = benchres.result; - if(eqZero(benchres.result.dSpeed)) { - loopDurationD = 0; - uncertaintyConstantD = 2; - } else { - loopDurationD = (U64)((double)(srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - uncertaintyConstantD = MIN((loopDurationD + (double)(2 * g_clockGranularity)/loopDurationD) * 1.1 , 3); //.02 seconds - } + /* calculate uncertainty in compression / decompression runs */ + if(eqZero(benchres.result.cSpeed)) { + loopDurationC = 0; + uncertaintyConstantC = 3; + } else { + loopDurationC = (U64)((double)(buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC) * VARIANCE; + } + if(eqZero(benchres.result.dSpeed)) { + loopDurationD = 0; + uncertaintyConstantD = 3; + } else { + loopDurationD = (U64)((double)(buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD) * VARIANCE; + } - if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres = benchres2; - } - } - if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - *resultPtr = benchres.result; + /* anything with worse ratio in feas is definitely worse, discard */ + if(feas && benchres.result.cSize < winnerResult->cSize) { + return WORSE_RESULT; + } - /* benchres's certainty range. */ - resultMax = benchres.result; - resultMin = benchres.result; - resultMax.cSpeed *= uncertaintyConstantC; - resultMax.dSpeed *= uncertaintyConstantD; - resultMin.cSpeed /= uncertaintyConstantC; - resultMin.dSpeed /= uncertaintyConstantD; - if (winnerRS > resultScore(resultMax, srcSize, target)) { - return INFEASIBLE_RESULT; - } else { - if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(srcPtrs, srcSizes, dstPtrs, dstCapacityToSizes, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_compressOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.cSpeed = benchres2.result.cSpeed; - } - } - if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible((const void* const*)dstPtrs, dstCapacityToSizes, NULL, NULL, nbBlocks, 0, &cParams, dictBuffer, dictBufferSize, ctx, dctx, - BMK_decodeOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } else { - benchres.result.dSpeed = benchres2.result.dSpeed; - } - } - *resultPtr = benchres.result; - return (resultScore(benchres.result, srcSize, target) > winnerRS); + /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ + if(loopDurationC < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_iterMode, 1); + if(benchres2.error) { + return ERROR_RESULT; } + benchres = benchres2; + } + if(loopDurationD < TIMELOOP_NANOSEC / 10) { + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_iterMode, 1); + if(benchres2.error) { + return ERROR_RESULT; + } + benchres.result.dSpeed = benchres2.result.dSpeed; + } *resultPtr = benchres.result; - } else { - return ERROR_RESULT; //BMK error + + /* optimistic assumption of benchres.result */ + resultMax = benchres.result; + resultMax.cSpeed *= uncertaintyConstantC; + resultMax.dSpeed *= uncertaintyConstantD; + + /* disregard infeasible results in feas mode */ + /* disregard if resultMax < winner in infeas mode */ + if((feas && !feasible(resultMax, target)) || + (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) { + return WORSE_RESULT; + } + + /* Final full run if estimates are unclear */ + if(loopDurationC < TIMELOOP_NANOSEC) { + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_timeMode, 1); + if(benchres2.error) { + return ERROR_RESULT; + } + benchres.result.cSpeed = benchres2.result.cSpeed; + } + + if(loopDurationD < TIMELOOP_NANOSEC) { + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_timeMode, 1); + if(benchres2.error) { + return ERROR_RESULT; + } + benchres.result.dSpeed = benchres2.result.dSpeed; + } + + *resultPtr = benchres.result; + + /* compare by resultScore when in infeas */ + /* compare by compareResultLT when in feas */ + if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || + (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) ) { + return BETTER_RESULT; + } else { + return WORSE_RESULT; } } /* wrap feasibleBench w/ memotable */ #define INFEASIBLE_THRESHOLD 200 -static int feasibleBenchMemo(BMK_result_t* resultPtr, - const void* srcBuffer, const size_t srcSize, - void* dstBuffer, const size_t dstSize, - void* dictBuffer, const size_t dictSize, - const size_t* fileSizes, const size_t nbFiles, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, - const ZSTD_compressionParameters cParams, - const constraint_t target, - BMK_result_t* winnerResult, U8* memoTable, - const U32* varyParams, const int varyLen) { - - const size_t memind = memoTableInd(&cParams, varyParams, varyLen); - if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { - return INFEASIBLE_RESULT; - } else { - const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; - const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const dstCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - U32 nbBlocks; - int res; - - if(!srcPtrs || !srcSizes || !dstPtrs || !dstCapacities) { - free(srcPtrs); - free(srcSizes); - free(dstPtrs); - free(dstCapacities); - DISPLAY("Allocation Error\n"); - return ERROR_RESULT; - } - - { - const char* srcPtr = (const char*)srcBuffer; - char* dstPtr = (char*)dstBuffer; - size_t dstSizeRemaining = dstSize; - U32 fileNb; - for (nbBlocks=0, fileNb=0; fileNb dstSizeRemaining) { - DEBUGOUTPUT("Warning: dstSize too small to benchmark completely \n"); - remaining = dstSizeRemaining; - dstSizeRemaining = 0; - } else { - dstSizeRemaining -= remaining; - } - for ( ; nbBlocks= INFEASIBLE_THRESHOLD) { - return INFEASIBLE_RESULT; //see feasibleBenchMemo for concerns - } else { - const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; - const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - void ** const dstPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const dstCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - U32 nbBlocks; - int res; + int res; - if(!srcPtrs || !srcSizes || !dstPtrs || !dstCapacities) { - free(srcPtrs); - free(srcSizes); - free(dstPtrs); - free(dstCapacities); - DISPLAY("Allocation Error\n"); - return ERROR_RESULT; - } + if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return WORSE_RESULT; } - { - const char* srcPtr = (const char*)srcBuffer; - char* dstPtr = (char*)dstBuffer; - size_t dstSizeRemaining = dstSize; - U32 fileNb; - for (nbBlocks=0, fileNb=0; fileNb dstSizeRemaining) { - DEBUGOUTPUT("Warning: dstSize too small to benchmark completely \n"); - remaining = dstSizeRemaining; - dstSizeRemaining = 0; - } else { - dstSizeRemaining -= remaining; - } - for ( ; nbBlocks (hopefully) feasible */ - /* when nothing is found, this garbages part 2. */ { - winnerInfo_t bestFeasible1; /* uses feasibleBench Metric */ - - //init these params - bestFeasible1.params = cparam; - bestFeasible1.result.cSpeed = 0; - bestFeasible1.result.dSpeed = 0; - bestFeasible1.result.cMem = (size_t)-1; - bestFeasible1.result.cSize = (size_t)-1; + winnerInfo_t bestFeasible1 = initWinnerInfo(cparam); DISPLAY("Climb Part 1\n"); while(better) { - int i, d; + int i, dist, offset; better = 0; DEBUGOUTPUT("Start\n"); cparam = winnerInfo.params; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); candidateInfo.params = cparam; //all dist-1 targets //if we early end this, we should also randomize the order these are picked. for(i = 0; i < varLen; i++) { - paramVaryOnce(varArray[i], 1, &candidateInfo.params); /* +1 */ - candidateInfo.params = sanitizeParams(candidateInfo.params); - //evaluate - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - //if(cParamValid(candidateInfo.params)) { - int res = infeasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { /* synonymous with better when called w/ infeasibleBM */ - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { - bestFeasible1 = winnerInfo; - } - } - } - candidateInfo.params = cparam; - paramVaryOnce(varArray[i], -1, &candidateInfo.params); /* -1 */ - candidateInfo.params = sanitizeParams(candidateInfo.params); - //evaluate - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - //if(cParamValid(candidateInfo.params)) { - int res = infeasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { - bestFeasible1 = winnerInfo; + for(offset = -1; offset <= 1; offset += 2) { + candidateInfo.params = cparam; + paramVaryOnce(varArray[i], offset, &candidateInfo.params); /* +1 */ + candidateInfo.params = sanitizeParams(candidateInfo.params); + //evaluate + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { + int res = benchMemo(&candidateInfo.result, + buf, ctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen, feas); + if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ + winnerInfo = candidateInfo; + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + better = 1; + if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { + bestFeasible1 = winnerInfo; + } } } } @@ -1966,235 +1533,255 @@ static winnerInfo_t climbOnce(const constraint_t target, if(better) { continue; } - //if 'better' enough, skip further parameter search, center there? - //possible improvement - guide direction here w/ knowledge rather than completely random variation. - for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ + + for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */ for(i = 0; i < 2 * varLen + 2; i++) { int res; candidateInfo.params = cparam; /* param error checking already done here */ - paramVariation(&candidateInfo.params, varArray, varLen, d); - res = infeasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { /* synonymous with better in this case*/ + paramVariation(&candidateInfo.params, varArray, varLen, dist); + res = benchMemo(&candidateInfo.result, + buf, ctx, + candidateInfo.params, target, &winnerInfo.result, memoTable, + varArray, varLen, feas); + if(res == BETTER_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); better = 1; - if(feasible(candidateInfo.result, target) && objective_lt(bestFeasible1.result, winnerInfo.result)) { + if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; } } } if(better) { - continue; + break; } } - //bias to test previous delta? - //change cparam -> candidate before restart + + if(!better) { //infeas -> feas -> stop. + if(feas) { return winnerInfo; } + + feas = 1; + better = 1; + winnerInfo = bestFeasible1; /* note with change, bestFeasible may not necessarily be feasible, but if one has been benchmarked, it will be. */ + DISPLAY("Climb Part 2\n"); + } } winnerInfo = bestFeasible1; } - //break out if no feasible. - if(winnerInfo.result.cMem == (U32)-1) { - DEBUGOUTPUT("No Feasible Found\n"); - return winnerInfo; - } - DISPLAY("Climb Part 2\n"); - - better = 1; - /* feasible -> best feasible (hopefully) */ - { - while(better) { - int i, d; - better = 0; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - //all dist-1 targets - cparam = winnerInfo.params; - candidateInfo.params = cparam; - for(i = 0; i < varLen; i++) { - paramVaryOnce(varArray[i], 1, &candidateInfo.params); - candidateInfo.params = sanitizeParams(candidateInfo.params); - - //evaluate - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - //if(cParamValid(candidateInfo.params)) { - int res = feasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - } - } - candidateInfo.params = cparam; - paramVaryOnce(varArray[i], -1, &candidateInfo.params); - candidateInfo.params = sanitizeParams(candidateInfo.params); - //evaluate - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - int res = feasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - } - } - } - //if 'better' enough, skip further parameter search, center there? - //possible improvement - guide direction here w/ knowledge rather than completely random variation. - for(d = 2; d < varLen + 2; d++) { /* varLen is # dimensions */ - for(i = 0; i < 2 * varLen + 2; i++) { - int res; - candidateInfo.params = cparam; - /* param error checking already done here */ - paramVariation(&candidateInfo.params, varArray, varLen, d); //info candidateInfo.params is garbage, this is too. - res = feasibleBenchMemo(&candidateInfo.result, - srcBuffer, srcSize, - dstBuffer, dstSize, - dictBuffer, dictSize, - fileSizes, nbFiles, - ctx, dctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen); - if(res == FEASIBLE_RESULT) { - winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); - better = 1; - } - } - if(better) { - continue; - } - } - //bias to test previous delta? - //change cparam -> candidate before restart - } - } - return winnerInfo; } //optimizeForSize but with fixed strategy //place to configure/filter out strategy specific parameters. -//need args for all buffers and parameter stuff -//sanitization here. //flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) //weight more on visit for bad results, less on good results/more on later results / ones with more failures. //allocate memoTable here. //only real use for paramTarget is to get the fixed values, right? +//maybe allow giving it a first init? static winnerInfo_t optimizeFixedStrategy( - const void* srcBuffer, const size_t srcSize, - void* dstBuffer, const size_t dstSize, - void* dictBuffer, const size_t dictSize, - const size_t* fileSizes, const size_t nbFiles, - const constraint_t target, ZSTD_compressionParameters paramTarget, - const ZSTD_strategy strat, const U32* varArray, const int varLen, U8* memoTable) { + buffers_t buf, contexts_t ctx, + const constraint_t target, ZSTD_compressionParameters paramTarget, + const ZSTD_strategy strat, + const varInds_t* varArray, const int varLen, + U8* memoTable, const int tries) { int i = 0; - U32* varNew = malloc(sizeof(U32) * varLen); - int varLenNew = sanitizeVarArray(varLen, varArray, varNew, strat); + varInds_t varNew[NUM_PARAMS]; + int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); ZSTD_compressionParameters init; - ZSTD_CCtx* ctx = ZSTD_createCCtx(); - ZSTD_DCtx* dctx = ZSTD_createDCtx(); winnerInfo_t winnerInfo, candidateInfo; - winnerInfo.result.cSpeed = 0; - winnerInfo.result.dSpeed = 0; - winnerInfo.result.cMem = (size_t)(-1LL); - winnerInfo.result.cSize = (size_t)(-1LL); + winnerInfo = initWinnerInfo(emptyParams()); /* so climb is given the right fixed strategy */ paramTarget.strategy = strat; /* to pass ZSTD_checkCParams */ - //needs to happen after memoTableInit as that assumes 0 = undefined. cParamZeroMin(¶mTarget); init = paramTarget; - - if(!ctx || !dctx || !memoTable || !varNew) { - DISPLAY("NOT ENOUGH MEMORY ! ! ! \n"); - goto _cleanUp; - } - - while(i < 10) { //make i adjustable (user input?) depending on how much time they have. + while(i < tries) { //make i adjustable (user input?) depending on how much time they have. DEBUGOUTPUT("Restart\n"); //look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); - candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, srcBuffer, srcSize, dstBuffer, dstSize, dictBuffer, dictSize, fileSizes, nbFiles, ctx, dctx, init); - if(objective_lt(winnerInfo.result, candidateInfo.result)) { + candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init); + if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { winnerInfo = candidateInfo; - DISPLAY("New Winner: "); - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, srcSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); i = 0; } i++; } - -_cleanUp: - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); - free(varNew); return winnerInfo; } -static int BMK_loadFiles(void* buffer, size_t bufferSize, - size_t* fileSizes, const char* const * const fileNamesTable, - unsigned nbFiles) +static void freeBuffers(buffers_t b) { + if(b.srcPtrs != NULL) { + free(b.srcPtrs[0]); + } + free(b.srcPtrs); + free(b.srcSizes); + + if(b.dstPtrs != NULL) { + free(b.dstPtrs[0]); + } + free(b.dstPtrs); + free(b.dstCapacities); + free(b.dstSizes); + + if(b.resPtrs != NULL) { + free(b.resPtrs[0]); + } + free(b.resPtrs); +} + +/* allocates buffer's arguments. returns success / failuere */ +static int initBuffers(buffers_t* buff, const char* const * const fileNamesTable, + size_t nbFiles) { - size_t pos = 0, totalSize = 0; - unsigned n; - for (n=0; nsrcPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstCapacities || !buff->resPtrs || !buff->resSizes) { + DISPLAY("alloc error\n"); + freeBuffers(*buff); + return 1; + } + + buff->srcPtrs[0] = malloc(benchedSize); + buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024)); + buff->resPtrs[0] = malloc(benchedSize); + + if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) { + DISPLAY("alloc error\n"); + freeBuffers(*buff); + return 1; + } + + for(n = 0; n < nbFiles; n++) { FILE* f; U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); if (UTIL_isDirectory(fileNamesTable[n])) { DISPLAY("Ignoring %s directory... \n", fileNamesTable[n]); - fileSizes[n] = 0; continue; } if (fileSize == UTIL_FILESIZE_UNKNOWN) { DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]); - fileSizes[n] = 0; continue; } f = fopen(fileNamesTable[n], "rb"); if (f==NULL) { - DISPLAY("impossible to open file %s", fileNamesTable[n]); + DISPLAY("impossible to open file %s\n", fileNamesTable[n]); + freeBuffers(*buff); + fclose(f); return 10; } + DISPLAY("Loading %s... \r", fileNamesTable[n]); - if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ - { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + + if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ + { + char* buffer = (char*)(buff->srcPtrs[0]); + size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); + size_t blocked = 0; + while(blocked < readSize) { + buff->srcPtrs[blockNb] = (buffer) + (pos + blocked); + buff->srcSizes[blockNb] = blockSize; + blocked += blockSize; + blockNb++; + } + if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; } + if (readSize != (size_t)fileSize) { DISPLAY("could not read %s", fileNamesTable[n]); - return 11; + freeBuffers(*buff); + fclose(f); + return 1; } - pos += readSize; } - fileSizes[n] = (size_t)fileSize; - totalSize += (size_t)fileSize; + + pos += readSize; + + } fclose(f); } - if (totalSize == 0) { DISPLAY("\nno data to bench\n"); return 12; } + buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); + buff->dstSizes[0] = buff->dstCapacities[0]; + buff->resSizes[0] = buff->srcSizes[0]; + + for(n = 1; n < blockNb; n++) { + buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1]; + buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1]; + buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]); + buff->dstSizes[n] = buff->dstCapacities[n]; + buff->resSizes[n] = buff->srcSizes[n]; + } + buff->srcSize = pos; + buff->nbBlocks = blockNb; + + if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; } + + return 0; +} + +static void freeContexts(contexts_t ctx) { + free(ctx.dictBuffer); + ZSTD_freeCCtx(ctx.cctx); + ZSTD_freeDCtx(ctx.dctx); +} + +static int initContexts(contexts_t* ctx, const char* dictFileName) { + FILE* f; + size_t readSize; + ctx->cctx = ZSTD_createCCtx(); + ctx->dctx = ZSTD_createDCtx(); + if(dictFileName == NULL) { + ctx->dictSize = 0; + ctx->dictBuffer = NULL; + return 0; + } + ctx->dictSize = UTIL_getFileSize(dictFileName); + ctx->dictBuffer = malloc(ctx->dictSize); + + f = fopen(dictFileName, "rb"); + + if(!f) { + DISPLAY("unable to open file\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } + + if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) { + DISPLAY("dictionary too large\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } + readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f); + if(readSize != ctx->dictSize) { + DISPLAY("unable to read file\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } return 0; } @@ -2228,107 +1815,98 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { } } +static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { + base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog; + base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog; + base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog; + base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog; + base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength; + base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength; + base.strategy = mask.strategy ? mask.strategy : base.strategy; + return base; +} + +#define MAX_TRIES 8 //optimize fixed strategy. static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { - size_t benchedSize; - void* origBuff = NULL; - void* dictBuffer = NULL; - size_t dictBufferSize = 0; - U32 varArray [NUM_PARAMS]; + varInds_t varArray [NUM_PARAMS]; int ret = 0; - size_t* fileSizes = calloc(sizeof(size_t),nbFiles); const int varLen = variableParams(paramTarget, varArray); + winnerInfo_t winner = initWinnerInfo(emptyParams()); U8** allMT = NULL; - g_winner.result.cSize = (size_t)-1; + size_t k; + size_t maxBlockSize = 0; + contexts_t ctx; + buffers_t buf; + /* Init */ if(!cParamValid(paramTarget)) { - return 10; + return 1; } /* load dictionary*/ - if (dictFileName != NULL) { - U64 const dictFileSize = UTIL_getFileSize(dictFileName); - if (dictFileSize > 64 MB) { - DISPLAY("dictionary file %s too large", dictFileName); - ret = 10; - goto _cleanUp; - } - dictBufferSize = (size_t)dictFileSize; - dictBuffer = malloc(dictBufferSize); - if (dictBuffer==NULL) { - DISPLAY("not enough memory for dictionary (%u bytes)", - (U32)dictBufferSize); - ret = 11; - goto _cleanUp; + if(initBuffers(&buf, fileNamesTable, nbFiles)) { + DISPLAY("unable to load files\n"); + return 1; + } - } - - { - int errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, &dictBufferSize, &dictFileName, 1); - if(errorCode) { - ret = errorCode; - goto _cleanUp; - } - } + if(initContexts(&ctx, dictFileName)) { + DISPLAY("unable to load dictionary\n"); + freeBuffers(buf); + return 2; } - /* Fill input buffer */ if(nbFiles == 1) { DISPLAY("Loading %s... \r", fileNamesTable[0]); } else { DISPLAY("Loading %lu Files... \r", (unsigned long)nbFiles); } - { - U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); - int ec; - unsigned i; - benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; - origBuff = malloc(benchedSize); - if(!origBuff || !fileSizes) { - DISPLAY("Not enough memory for stuff\n"); - ret = 1; - goto _cleanUp; - } - ec = BMK_loadFiles(origBuff, benchedSize, fileSizes, fileNamesTable, (U32)nbFiles); - if(ec) { - DISPLAY("Error Loading Files"); - ret = ec; - goto _cleanUp; - } - benchedSize = 0; - for(i = 0; i < nbFiles; i++) { - benchedSize += fileSizes[i]; - } - origBuff = realloc(origBuff, benchedSize); + for(k = 0; k < buf.nbBlocks; k++) { + maxBlockSize = MAX(buf.srcSizes[k], maxBlockSize); } - allMT = memoTableInitAll(paramTarget, target, varArray, varLen, benchedSize); + /* if strategy is fixed, only init that part of memotable */ + if(paramTarget.strategy) { + varInds_t varNew[NUM_PARAMS]; + int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.strategy); + allMT = calloc(sizeof(U8), (ZSTD_btultra + 1)); + if(allMT == NULL) { + ret = 57; + goto _cleanUp; + } + + allMT[paramTarget.strategy] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + + if(allMT[paramTarget.strategy] == NULL) { + ret = 58; + goto _cleanUp; + } + + memoTableInit(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); + } else { + allMT = memoTableInitAll(paramTarget, target, varArray, varLen, maxBlockSize); + } + + if(!allMT) { + DISPLAY("MemoTable Init Error\n"); ret = 2; goto _cleanUp; } if(cLevel) { - BMK_result_t candidate; - const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; - ZSTD_CCtx* const ctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - ZSTD_compressionParameters const CParams = ZSTD_getCParams(cLevel, blockSize, dictBufferSize); - if(BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, CParams)) { - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); + winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); + if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { ret = 3; goto _cleanUp; } - target.cSpeed = (U32)candidate.cSpeed; //Maybe have a small bit of slack here, like x.99? - BMK_printWinner(stdout, cLevel, candidate, CParams, benchedSize); - - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); + target.cSpeed = (U32)winner.result.cSpeed; //Maybe have a small bit of slack here, like x.99? + g_targetConstraints = target; + BMK_printWinner(stdout, cLevel, winner.result, winner.params, buf.srcSize); } g_targetConstraints = target; @@ -2340,106 +1918,40 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } else { DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles); } - if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed / 1000000); } - if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed / 1000000); } - if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem / 1000000); } + if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed >> 20); } + if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed >> 20); } + if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem >> 20); } + DISPLAY("\n"); findClockGranularity(); - { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - winnerInfo_t winner; - U32 varNew[NUM_PARAMS]; - const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; - U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1; - const size_t maxCompressedSize = ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024); - void* compressedBuffer = malloc(maxCompressedSize); - - /* init */ - if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} - if(compressedBuffer==NULL) { DISPLAY("\n Allocation Error \n"); free(origBuff); free(ctx); return 15; } - memset(&winner, 0, sizeof(winner)); - winner.result.cSize = (size_t)(-1); - + { + varInds_t varNew[NUM_PARAMS]; /* find best solution from default params */ { /* strategy selection */ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); DEBUGOUTPUT("Strategy Selection\n"); - if(varLen == NUM_PARAMS && paramTarget.strategy == 0) { /* no variable based constraints */ + if(paramTarget.strategy == 0) { /* no variable based constraints */ BMK_result_t candidate; - int feas = 0, i; + int i; for (i=1; i<=maxSeeds; i++) { - ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, dictBufferSize); - int ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, CParams); - BMK_printWinner(stdout, i, candidate, CParams, benchedSize); - - if(!ec) { - if(feas) { - if(feasible(candidate, relaxTarget(target)) && objective_lt(winner.result, candidate)) { - winner.result = candidate; - winner.params = CParams; - } - } else { - if(feasible(candidate, relaxTarget(target))) { - feas = 1; - winner.result = candidate; - winner.params = CParams; - - } else { - if(resultScore(candidate, benchedSize, target) > resultScore(winner.result, benchedSize, target)) { - winner.result = candidate; - winner.params = CParams; - } - } - } - } - } //best, -1, +1, ..., - - } else if (paramTarget.strategy == 0) { //constrained - int feas = 0, i, j; - for(j = 1; j < 10; j++) { - for(i = 1; i <= maxSeeds; i++) { - int varLenNew = sanitizeVarArray(varLen, varArray, varNew, i); - ZSTD_compressionParameters candidateParams = paramTarget; - BMK_result_t candidate; - int ec; - randomConstrainedParams(&candidateParams, varNew, varLenNew, allMT[i]); - cParamZeroMin(&candidateParams); - candidateParams = sanitizeParams(candidateParams); - ec = BMK_benchParam(&candidate, origBuff, benchedSize, fileSizes, (U32)nbFiles, ctx, dctx, candidateParams); - - if(!ec) { - if(feas) { - if(feasible(candidate, relaxTarget(target)) && objective_lt(winner.result, candidate)) { - winner.result = candidate; - winner.params = candidateParams; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } - } else { - if(feasible(candidate, relaxTarget(target))) { - feas = 1; - winner.result = candidate; - winner.params = candidateParams; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - - } else { - if(resultScore(candidate, benchedSize, target) > resultScore(winner.result, benchedSize, target)) { - winner.result = candidate; - winner.params = candidateParams; - BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } - } - } - } + int ec; + ZSTD_compressionParameters CParams = ZSTD_getCParams(i, maxBlockSize, ctx.dictSize); + CParams = maskParams(CParams, paramTarget); + ec = BMK_benchParam(&candidate, buf, ctx, CParams); + BMK_printWinner(stdout, i, candidate, CParams, buf.srcSize); + if(!ec && compareResultLT(winner.result, candidate, relaxTarget(target), buf.srcSize)) { + winner.result = candidate; + winner.params = CParams; } } } } - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, buf.srcSize); BMK_translateAdvancedParams(winner.params); DEBUGOUTPUT("Real Opt\n"); /* start 'real' tests */ @@ -2447,55 +1959,51 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int bestStrategy = (int)winner.params.strategy; if(paramTarget.strategy == 0) { int st = (int)winner.params.strategy; + int tries = MAX_TRIES; { - int varLenNew = sanitizeVarArray(varLen, varArray, varNew, st); + int varLenNew = sanitizeVarArray(varNew, varLen, varArray, st); winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], - origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, - fileSizes, nbFiles, ctx, dctx, winner.params); - if(objective_lt(winner.result, w1.result)) { + buf, ctx, winner.params); + if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) { winner = w1; } } - while(st) { - winnerInfo_t wc = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, fileSizes, nbFiles, - target, paramTarget, st, varArray, varLen, allMT[st]); + while(st && tries) { + winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, + st, varArray, varLen, allMT[st], tries); DEBUGOUTPUT("StratNum %d\n", st); - if(objective_lt(winner.result, wc.result)) { + if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; } //We could double back to increase search of 'better' strategies st = nextStrategy(st, bestStrategy); + tries--; } } else { - winner = optimizeFixedStrategy(origBuff, benchedSize, compressedBuffer, maxCompressedSize, dictBuffer, dictBufferSize, fileSizes, nbFiles, - target, paramTarget, paramTarget.strategy, varArray, varLen, allMT[paramTarget.strategy]); + winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, + varArray, varLen, allMT[paramTarget.strategy], 10); } } /* no solution found */ if(winner.result.cSize == (size_t)-1) { + ret = 1; DISPLAY("No feasible solution found\n"); - return 1; + goto _cleanUp; } /* end summary */ - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, benchedSize); + BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, buf.srcSize); BMK_translateAdvancedParams(winner.params); DISPLAY("grillParams size - optimizer completed \n"); - - /* clean up*/ - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); - } _cleanUp: - free(fileSizes); - free(dictBuffer); + freeContexts(ctx); + freeBuffers(buf); memoTableFreeAll(allMT); - free(origBuff); return ret; } @@ -2546,16 +2054,16 @@ static int usage(const char* exename) static int usage_advanced(void) { DISPLAY( "\nAdvanced options :\n"); - DISPLAY( " -T# : set level 1 speed objective \n"); - DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n"); - DISPLAY( " -i# : iteration loops [1-9](default : %i) \n", NBLOOPS); - DISPLAY( " -O# : find Optimized parameters for # MB/s compression speed (default : 0) \n"); - DISPLAY( " -S : Single run \n"); - DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n"); - DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); - DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); - DISPLAY( " -v : Prints Benchmarking output\n"); - DISPLAY( " -D : Next argument dictionary file\n"); + DISPLAY( " -T# : set level 1 speed objective \n"); + DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n"); + DISPLAY( " -i# : iteration loops (default : %i) \n", NBLOOPS); + DISPLAY( " --optimize= : same as -O with more verbose syntax (see README.md)\n"); + DISPLAY( " -S : Single run \n"); + DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n"); + DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); + DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); + DISPLAY( " -v : Prints Benchmarking output\n"); + DISPLAY( " -D : Next argument dictionary file\n"); return 0; } @@ -2572,8 +2080,8 @@ int main(int argc, const char** argv) filenamesStart=0, result; const char* exename=argv[0]; - const char* input_filename = 0; - const char* dictFileName = 0; + const char* input_filename = NULL; + const char* dictFileName = NULL; U32 optimizer = 0; U32 main_pause = 0; int optimizerCLevel = 0; @@ -2591,6 +2099,9 @@ int main(int argc, const char** argv) for(i=1; i Date: Fri, 27 Jul 2018 08:49:25 -0700 Subject: [PATCH 34/92] Renaming / Style fixes --- programs/bench.c | 56 +++++++++++++++++++++++++--------------------- programs/bench.h | 7 +++--- tests/README.md | 1 - tests/paramgrill.c | 27 +++++++++++----------- 4 files changed, 48 insertions(+), 43 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 177dbe0e3..b496caf27 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -294,7 +294,7 @@ BMK_customReturn_t BMK_benchFunction( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* cSizes, + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResult, unsigned nbLoops) { size_t dstSize = 0; U64 totalTime; @@ -332,8 +332,8 @@ BMK_customReturn_t BMK_benchFunction( j, (U32)dstBlockCapacities[j], ZSTD_getErrorName(res)); } else if(i == nbLoops - 1) { dstSize += res; - if(cSizes != NULL) { - cSizes[j] = res; + if(blockResult != NULL) { + blockResult[j] = res; } } } @@ -358,6 +358,9 @@ void BMK_resetTimeState(BMK_timedFnState_t* r, unsigned nbSeconds) { BMK_timedFnState_t* BMK_createTimeState(unsigned nbSeconds) { BMK_timedFnState_t* r = (BMK_timedFnState_t*)malloc(sizeof(struct BMK_timeState_t)); + if(r == NULL) { + return r; + } BMK_resetTimeState(r, nbSeconds); return r; } @@ -373,7 +376,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const* const srcBlockBuffers, const size_t* srcBlockSizes, - void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* dstSizes) + void * const * const dstBlockBuffers, const size_t * dstBlockCapacities, size_t* blockResults) { U64 fastest = cont->fastestTime; int completed = 0; @@ -390,7 +393,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed( } /* reinitialize capacity */ r.result = BMK_benchFunction(benchFn, benchPayload, initFn, initPayload, - blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, dstSizes, cont->nbLoops); + blockCount, srcBlockBuffers, srcBlockSizes, dstBlockBuffers, dstBlockCapacities, blockResults, cont->nbLoops); if(r.result.error) { /* completed w/ error */ r.completed = 1; return r; @@ -718,38 +721,36 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; /* these are the blockTable parameters, just split up */ - const void ** const srcPtrs = (const void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const srcSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + const void ** const srcPtrs = (const void**)malloc(maxNbBlocks * sizeof(void*)); + size_t* const srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - void ** const cPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const cSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); - size_t* const cCapacities = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const cPtrs = (void**)malloc(maxNbBlocks * sizeof(void*)); + size_t* const cSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + size_t* const cCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - void ** const resPtrs = (void** const)malloc(maxNbBlocks * sizeof(void*)); - size_t* const resSizes = (size_t* const)malloc(maxNbBlocks * sizeof(size_t)); + void ** const resPtrs = (void**)malloc(maxNbBlocks * sizeof(void*)); + size_t* const resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds); BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds); - void* compressedBuffer; const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); + + void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize); + void* const compressedBuffer = dstBuffer ? dstBuffer : internalDstBuffer; + void* resultBuffer = malloc(srcSize); - BMK_return_t results = { { 0, 0, 0, 0 }, 0 }; - int allocationincomplete; + int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs || + !cSizes || !cCapacities || !resPtrs || !resSizes || + !timeStateCompress || !timeStateDecompress || !compressedBuffer || !resultBuffer; - if(!dstCapacity) { - compressedBuffer = malloc(maxCompressedSize); - } else { - compressedBuffer = dstBuffer; - } + int parametersConflict = !dstBuffer ^ !dstCapacity; - allocationincomplete = !compressedBuffer || !resultBuffer || - !srcPtrs || !srcSizes || !cPtrs || !cSizes || !resPtrs || !resSizes; - if (!allocationincomplete) { + if (!allocationincomplete && !parametersConflict) { results = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, cPtrs, cCapacities, cSizes, resPtrs, resSizes, &resultBuffer, compressedBuffer, maxCompressedSize, timeStateCompress, timeStateDecompress, srcBuffer, srcSize, fileSizes, nbFiles, cLevel, comprParams, @@ -759,9 +760,8 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, /* clean up */ BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); - if(!dstCapacity) { /* only free if not given */ - free(compressedBuffer); - } + + free(internalDstBuffer); free(resultBuffer); free((void*)srcPtrs); @@ -775,6 +775,10 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, if(allocationincomplete) { EXM_THROW(31, BMK_return_t, "allocation error : not enough memory"); } + + if(parametersConflict) { + EXM_THROW(32, BMK_return_t, "Conflicting input results"); + } return results; } diff --git a/programs/bench.h b/programs/bench.h index 2a9945ac8..8baf33a0a 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -175,7 +175,8 @@ typedef size_t (*BMK_initFn_t)(void*); * srcBuffers - an array of buffers to be operated on by benchFn * srcSizes - an array of the sizes of above buffers * dstBuffers - an array of buffers to be written into by benchFn - * dstCapacitiesToSizes - an array of the capacities of above buffers. Output modified to compressed sizes of those blocks. + * dstCapacities - an array of the capacities of above buffers + * blockResults - the return value of benchFn called on each block. * nbLoops - defines number of times benchFn is run. * assumed array of size blockCount, will have compressed size of each block written to it. * return @@ -192,7 +193,7 @@ BMK_customReturn_t BMK_benchFunction(BMK_benchFn_t benchFn, void* benchPayload, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBuffers, const size_t* srcSizes, - void * const * const dstBuffers, const size_t* dstCapacities, size_t* cSizes, + void * const * const dstBuffers, const size_t* dstCapacities, size_t* blockResults, unsigned nbLoops); @@ -221,7 +222,7 @@ BMK_customTimedReturn_t BMK_benchFunctionTimed(BMK_timedFnState_t* cont, BMK_initFn_t initFn, void* initPayload, size_t blockCount, const void* const * const srcBlockBuffers, const size_t* srcBlockSizes, - void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* cSizes); + void* const * const dstBlockBuffers, const size_t* dstBlockCapacities, size_t* blockResults); #endif /* BENCH_H_121279284357 */ diff --git a/tests/README.md b/tests/README.md index 8bedd0a3c..2f0026fda 100644 --- a/tests/README.md +++ b/tests/README.md @@ -112,7 +112,6 @@ Full list of arguments dSpeed= - Minimum decompression speed cMem= - compression memory lvl= - Automatically sets compression speed constraint to the speed of that level - --optimize= : same as -O with more verbose syntax -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 099be3688..a4f4a5f15 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -986,7 +986,7 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va } /* Initialize memotable, immediately mark redundant / obviously infeasible params as */ -static void memoTableInit(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; @@ -1053,23 +1053,24 @@ static void memoTableFreeAll(U8** mtAll) { /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ -static U8** memoTableInitAll(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { varInds_t varNew[NUM_PARAMS]; - int varLenNew; U8** mtAll = calloc(sizeof(U8*),(ZSTD_btultra + 1)); int i; if(mtAll == NULL) { return NULL; } + for(i = 1; i <= (int)ZSTD_btultra; i++) { - varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); + const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); if(mtAll[i] == NULL) { memoTableFreeAll(mtAll); return NULL; } - memoTableInit(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); + createMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); } + return mtAll; } @@ -1187,7 +1188,7 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src winnerInfo_t winners[NB_LEVELS_TRACKED+1]; const char* const rfName = "grillResults.txt"; FILE* const f = fopen(rfName, "w"); - const size_t blockSize = g_blockSize ? g_blockSize : ZSTD_BLOCKSIZE_MAX; /* cut by block or not ? */ + const size_t blockSize = g_blockSize ? g_blockSize : srcSize; /* cut by block or not ? */ /* init */ assert(g_singleRun==0); @@ -1638,7 +1639,7 @@ static void freeBuffers(buffers_t b) { } /* allocates buffer's arguments. returns success / failuere */ -static int initBuffers(buffers_t* buff, const char* const * const fileNamesTable, +static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, size_t nbFiles) { size_t pos = 0; @@ -1659,7 +1660,7 @@ static int initBuffers(buffers_t* buff, const char* const * const fileNamesTable buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstCapacities || !buff->resPtrs || !buff->resSizes) { + if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) { DISPLAY("alloc error\n"); freeBuffers(*buff); return 1; @@ -1747,7 +1748,7 @@ static void freeContexts(contexts_t ctx) { ZSTD_freeDCtx(ctx.dctx); } -static int initContexts(contexts_t* ctx, const char* dictFileName) { +static int createContexts(contexts_t* ctx, const char* dictFileName) { FILE* f; size_t readSize; ctx->cctx = ZSTD_createCCtx(); @@ -1846,12 +1847,12 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* load dictionary*/ - if(initBuffers(&buf, fileNamesTable, nbFiles)) { + if(createBuffers(&buf, fileNamesTable, nbFiles)) { DISPLAY("unable to load files\n"); return 1; } - if(initContexts(&ctx, dictFileName)) { + if(createContexts(&ctx, dictFileName)) { DISPLAY("unable to load dictionary\n"); freeBuffers(buf); return 2; @@ -1885,9 +1886,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } - memoTableInit(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); + createMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); } else { - allMT = memoTableInitAll(paramTarget, target, varArray, varLen, maxBlockSize); + allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize); } From 8faeb41679a5ef379cfb7893825e6ed98e571eb0 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 30 Jul 2018 11:30:38 -0700 Subject: [PATCH 35/92] Update Documentation Change comment // to /* */ Add more description of what functions do Remove outdated comments --- tests/paramgrill.c | 117 ++++++++++++++++++++++++--------------------- 1 file changed, 63 insertions(+), 54 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index a4f4a5f15..b8396c72e 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -77,12 +77,11 @@ typedef enum { } varInds_t; #define NUM_PARAMS 6 -//just don't use strategy as a param. +/* just don't use strategy as a param. */ #undef ZSTD_WINDOWLOG_MAX #define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. -//make 2^[0,10] w/ 999 #define ZSTD_TARGETLENGTH_MIN 0 #define ZSTD_TARGETLENGTH_MAX 999 @@ -274,7 +273,6 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; - //DISPLAY("resultScore: %f\n", ret); return ret; } @@ -593,7 +591,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); - } else { //iterMode; + } else { /* iterMode; */ if(mode != BMK_decodeOnly) { BMK_customReturn_t compressionResults = BMK_benchFunction(&local_defaultCompress, (void*)cctx, &local_initCCtx, (void*)&cctxprep, @@ -636,8 +634,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, return results; } -/* global winner used for display. */ -//Should be totally 0 initialized? +/* global winner used for display. */ static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; static constraint_t g_targetConstraints; @@ -940,7 +937,7 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var } validated = !ZSTD_isError(ZSTD_checkCParams(p)); } - *ptr = p;//sanitizeParams(p); + *ptr = p; } /* length of memo table given free variables */ @@ -953,7 +950,7 @@ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { return arrayLen; } -/* returns unique index of compression parameters */ +/* returns unique index in memotable of compression parameters */ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen) { int i; unsigned ind = 0; @@ -985,7 +982,7 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va } } -/* Initialize memotable, immediately mark redundant / obviously infeasible params as */ +/* Initialize memotable, immediately mark redundant / obviously infeasible params as such */ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); @@ -1003,7 +1000,7 @@ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConst memoTable[i] = 255; j++; } - if(wFixed && (1ULL << paramConstraints.windowLog) > (srcSize << 2)) { + if(wFixed && (1ULL << paramConstraints.windowLog) > srcSize) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ @@ -1121,7 +1118,7 @@ static void playAround(FILE* f, winnerInfo_t* winners, } - +/* Completely random parameter selection */ static ZSTD_compressionParameters randomParams(void) { ZSTD_compressionParameters p; @@ -1136,7 +1133,6 @@ static ZSTD_compressionParameters randomParams(void) p.targetLength=(FUZ_rand(&g_rand) % (512)); p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); validated = !ZSTD_isError(ZSTD_checkCParams(p)); - //validated = cParamValid(p); } return p; } @@ -1144,7 +1140,7 @@ static ZSTD_compressionParameters randomParams(void) /* Sets pc to random unmeasured set of parameters */ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* varArray, int varLen, U8* memoTable) { - size_t tries = memoTableLen(varArray, varLen); //configurable, + size_t tries = memoTableLen(varArray, varLen); const size_t maxSize = memoTableLen(varArray, varLen); size_t ind; do { @@ -1153,7 +1149,6 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v } while(memoTable[ind] > 0 && tries > 0); memoTableIndInv(pc, varArray, varLen, (unsigned)ind); - //*pc = sanitizeParams(*pc); } static void BMK_selectRandomStart( @@ -1328,19 +1323,13 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } -/*benchmarks and tests feasibility together - 1 = true = better - 0 = false = not better - if true then resultPtr will give results. - 2+ on error? */ -//Maybe use compress_only for benchmark first run? + #define WORSE_RESULT 0 #define BETTER_RESULT 1 #define ERROR_RESULT 2 -//add worse result complete for worse results of length > 1 sec? -/* variation between 2nd run and full second bmk */ +/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ #define VARIANCE 1.1 static int allBench(BMK_result_t* resultPtr, buffers_t buf, contexts_t ctx, @@ -1447,8 +1436,9 @@ static int allBench(BMK_result_t* resultPtr, } -/* wrap feasibleBench w/ memotable */ #define INFEASIBLE_THRESHOLD 200 + +/* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ static int benchMemo(BMK_result_t* resultPtr, buffers_t buf, contexts_t ctx, const ZSTD_compressionParameters cParams, @@ -1475,18 +1465,28 @@ static int benchMemo(BMK_result_t* resultPtr, return res; } - -//sanitize all params here. -//all generation after random should be sanitized. (maybe sanitize random) +/* One iteration of hill climbing. Specifically, it first tries all + * valid parameter configurations w/ manhattan distance 1 and picks the best one + * failing that, it progressively tries candidates further and further away (up to #dim + 2) + * if it finds a candidate exceeding winnerInfo, it will repeat. Otherwise, it will stop the + * current stage of hill climbing. + * Each iteration of hill climbing proceeds in 2 'phases'. Phase 1 climbs according to + * the resultScore function, which is effectively a linear increase in reward until it reaches + * the constraint-satisfying value, it which point any excess results in only logarithmic reward. + * This aims to find some constraint-satisfying point. + * Phase 2 optimizes in accordance with what the original function sets out to maximize, with + * all feasible solutions valued over all infeasible solutions. + */ static winnerInfo_t climbOnce(const constraint_t target, const varInds_t* varArray, const int varLen, U8* memoTable, buffers_t buf, contexts_t ctx, const ZSTD_compressionParameters init) { - //distance maximizing selection? - //cparam - currently considered 'center' - //candidate - params to benchmark/results - //winner - best option found so far. + /* + * cparam - currently considered 'center' + * candidate - params to benchmark/results + * winner - best option found so far. + */ ZSTD_compressionParameters cparam = init; winnerInfo_t candidateInfo, winnerInfo; int better = 1; @@ -1506,14 +1506,12 @@ static winnerInfo_t climbOnce(const constraint_t target, cparam = winnerInfo.params; BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); candidateInfo.params = cparam; - //all dist-1 targets - //if we early end this, we should also randomize the order these are picked. + /* all dist-1 candidates */ for(i = 0; i < varLen; i++) { for(offset = -1; offset <= 1; offset += 2) { candidateInfo.params = cparam; - paramVaryOnce(varArray[i], offset, &candidateInfo.params); /* +1 */ + paramVaryOnce(varArray[i], offset, &candidateInfo.params); candidateInfo.params = sanitizeParams(candidateInfo.params); - //evaluate if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { int res = benchMemo(&candidateInfo.result, buf, ctx, @@ -1560,7 +1558,7 @@ static winnerInfo_t climbOnce(const constraint_t target, } } - if(!better) { //infeas -> feas -> stop. + if(!better) { /* infeas -> feas -> stop */ if(feas) { return winnerInfo; } feas = 1; @@ -1575,14 +1573,14 @@ static winnerInfo_t climbOnce(const constraint_t target, return winnerInfo; } -//optimizeForSize but with fixed strategy -//place to configure/filter out strategy specific parameters. +/* Optimizes for a fixed strategy */ -//flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) -//weight more on visit for bad results, less on good results/more on later results / ones with more failures. -//allocate memoTable here. -//only real use for paramTarget is to get the fixed values, right? -//maybe allow giving it a first init? +/* flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) + weight more on visit for bad results, less on good results/more on later results / ones with more failures. + allocate memoTable here. + only real use for paramTarget is to get the fixed values, right? + maybe allow giving it a first init? + */ static winnerInfo_t optimizeFixedStrategy( buffers_t buf, contexts_t ctx, const constraint_t target, ZSTD_compressionParameters paramTarget, @@ -1603,9 +1601,8 @@ static winnerInfo_t optimizeFixedStrategy( init = paramTarget; - while(i < tries) { //make i adjustable (user input?) depending on how much time they have. + while(i < tries) { DEBUGOUTPUT("Restart\n"); - //look into improving this to maximize distance from searched infeasible stuff / towards promising regions? randomConstrainedParams(&init, varNew, varLenNew, memoTable); candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init); if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { @@ -1638,7 +1635,7 @@ static void freeBuffers(buffers_t b) { free(b.resPtrs); } -/* allocates buffer's arguments. returns success / failuere */ +/* allocates buffer's arguments. returns 0 = success / 1 = failuere */ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, size_t nbFiles) { @@ -1748,16 +1745,25 @@ static void freeContexts(contexts_t ctx) { ZSTD_freeDCtx(ctx.dctx); } +/* Creates struct holding contexts and dictionary buffers. returns 0 on success, 1 on failure. */ static int createContexts(contexts_t* ctx, const char* dictFileName) { FILE* f; size_t readSize; ctx->cctx = ZSTD_createCCtx(); ctx->dctx = ZSTD_createDCtx(); + ctx->dictSize = 0; + ctx->dictBuffer = NULL; + + if(!ctx->cctx || !ctx->dctx) { + DISPLAY("context allocation error\n"); + freeContexts(*ctx); + return 1; + } + if(dictFileName == NULL) { - ctx->dictSize = 0; - ctx->dictBuffer = NULL; return 0; } + ctx->dictSize = UTIL_getFileSize(dictFileName); ctx->dictBuffer = malloc(ctx->dictSize); @@ -1786,8 +1792,8 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) { return 0; } -//goes best, best-1, best+1, best-2, ... -//return 0 if nothing remaining +/* goes best, best-1, best+1, best-2, ... */ +/* return 0 if nothing remaining */ static int nextStrategy(const int currentStrategy, const int bestStrategy) { if(bestStrategy <= currentStrategy) { int candidate = 2 * bestStrategy - currentStrategy - 1; @@ -1828,7 +1834,10 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS } #define MAX_TRIES 8 -//optimize fixed strategy. +/* main fn called when using --optimize */ +/* Does strategy selection by benchmarking default compression levels + * then optimizes by strategy, starting with the best one and moving + * progressively moving further away by number */ static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { varInds_t varArray [NUM_PARAMS]; @@ -1905,7 +1914,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } - target.cSpeed = (U32)winner.result.cSpeed; //Maybe have a small bit of slack here, like x.99? + target.cSpeed = (U32)winner.result.cSpeed; g_targetConstraints = target; BMK_printWinner(stdout, cLevel, winner.result, winner.params, buf.srcSize); } @@ -1978,7 +1987,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; } - //We could double back to increase search of 'better' strategies + st = nextStrategy(st, bestStrategy); tries--; } @@ -2088,7 +2097,7 @@ int main(int argc, const char** argv) int optimizerCLevel = 0; - constraint_t target = { 0, 0, (U32)-1 }; //0 for anything unset + constraint_t target = { 0, 0, (U32)-1 }; ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; assert(argc>=1); /* for exename */ @@ -2250,7 +2259,7 @@ int main(int argc, const char** argv) /* load dictionary file (only applicable for optimizer rn) */ case 'D': - if(i == argc - 1) { //last argument, return error. + if(i == argc - 1) { /* last argument, return error. */ DISPLAY("Dictionary file expected but not given : %d\n", i); return 1; } else { From 3d230db85391aa5810abc6301035e9741c986765 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 6 Aug 2018 17:13:36 -0700 Subject: [PATCH 36/92] Change speed representation from floating point to integral --- programs/bench.c | 43 +++++++++++++++++++------------------------ programs/bench.h | 4 ++-- tests/paramgrill.c | 46 +++++++++++++++++----------------------------- 3 files changed, 38 insertions(+), 55 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index b496caf27..49b317870 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -555,9 +555,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( ratio = (double)(srcSize / intermediateResultCompress.result.result.sumOfReturn); { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * 1000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - results.result.cSpeed = compressionSpeed * 1000000; + results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / intermediateResultCompress.result.result.nanoSecPerRun); cSize = intermediateResultCompress.result.result.sumOfReturn; results.result.cSize = cSize; ratio = (double)srcSize / results.result.cSize; @@ -565,7 +563,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed); + results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB)); } } @@ -579,16 +577,13 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = results.result.cSpeed / 1000000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - double const decompressionSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * 1000; - results.result.dSpeed = decompressionSpeed * 1000000; + results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ intermediateResultDecompress.result.result.nanoSecPerRun); markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed, - decompressionSpeed); + results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB), + (double)results.result.dSpeed / (1 MB)); } } } @@ -605,19 +600,20 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if(compressionResults.result.nanoSecPerRun == 0) { results.result.cSpeed = 0; } else { - results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun; } results.result.cSize = compressionResults.result.sumOfReturn; { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = results.result.cSpeed / 1000000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; + results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun); + cSize = compressionResults.result.sumOfReturn; + results.result.cSize = cSize; ratio = (double)srcSize / results.result.cSize; markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed); + results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB)); } } if(adv->mode != BMK_compressOnly) { @@ -633,19 +629,18 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( if(decompressionResults.result.nanoSecPerRun == 0) { results.result.dSpeed = 0; } else { - results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun; } - { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - double const compressionSpeed = results.result.cSpeed / 1000000; - int const cSpeedAccuracy = (compressionSpeed < 10.) ? 2 : 1; - double const decompressionSpeed = ((double)srcSize / decompressionResults.result.nanoSecPerRun) * 1000; - results.result.dSpeed = decompressionSpeed * 1000000; + + { + int const ratioAccuracy = (ratio < 10.) ? 3 : 2; + results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ decompressionResults.result.nanoSecPerRun); markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, ratioAccuracy, ratio, - cSpeedAccuracy, compressionSpeed, - decompressionSpeed); + results.result.cSpeed < (10 MB) ? 2 : 1, (double)results.result.cSpeed / (1 MB), + (double)results.result.dSpeed / (1 MB)); } } } @@ -693,8 +688,8 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( } /* CRC Checking */ if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */ - double const cSpeed = results.result.cSpeed / 1000000; - double const dSpeed = results.result.dSpeed / 1000000; + double const cSpeed = (double)results.result.cSpeed / (1 MB); + double const dSpeed = (double)results.result.dSpeed / (1 MB); if (adv->additionalParam) { DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam); } else { diff --git a/programs/bench.h b/programs/bench.h index 8baf33a0a..6247fa596 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -32,8 +32,8 @@ extern "C" { typedef struct { size_t cSize; - double cSpeed; /* bytes / sec */ - double dSpeed; + U64 cSpeed; /* bytes / sec */ + U64 dSpeed; size_t cMem; } BMK_result_t; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index b8396c72e..5af3d88fa 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -249,18 +249,6 @@ static int feasible(const BMK_result_t results, const constraint_t target) { return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem); } -#define EPSILON 0.001 -static int epsilonEqual(const double c1, const double c2) { - return MAX(c1/c2,c2/c1) < 1 + EPSILON; -} - -/* checks exact equivalence to 0, to stop compiler complaining fpeq */ -static int eqZero(const double c1) { - const double z1 = 0.0; - const double z2 = -0.0; - return !(memcmp(&c1, &z1, sizeof(double))) || !(memcmp(&c1, &z2, sizeof(double))); -} - /* hill climbing value for part 1 */ static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) { double cs = 0., ds = 0., rt, cm = 0.; @@ -280,7 +268,7 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) { if(feasible(result1, target) && feasible(result2, target)) { return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) - || (result1.cSize == result2.cSize && epsilonEqual(result2.cSpeed, result1.cSpeed) && result2.dSpeed > result1.dSpeed); + || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed); } return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); @@ -661,7 +649,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result fprintf(f, "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", - lvlstr, (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20)); + lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20)); if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } fprintf(f, "\n"); @@ -696,8 +684,8 @@ static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSiz typedef struct { - double cSpeed_min; - double dSpeed_min; + U64 cSpeed_min; + U64 dSpeed_min; U32 windowLog_max; ZSTD_strategy strategy_max; } level_constraints_t; @@ -794,16 +782,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* too large compression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, testResult.cSpeed / 1000000, - O_ratio, winners[cLevel].result.cSpeed / 1000000., cLevel); + W_ratio, (double)testResult.cSpeed / 1000000, + O_ratio, (double)winners[cLevel].result.cSpeed / 1000000., cLevel); continue; } if (W_DSpeed_note < O_DSpeed_note ) { /* too large decompression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, testResult.dSpeed / 1000000., - O_ratio, winners[cLevel].result.dSpeed / 1000000., cLevel); + W_ratio, (double)testResult.dSpeed / 1000000., + O_ratio, (double)winners[cLevel].result.dSpeed / 1000000., cLevel); continue; } @@ -1173,7 +1161,7 @@ static void BMK_benchOnce(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffe g_params = ZSTD_adjustCParams(g_params, srcSize, 0); BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, g_params); DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, - testResult.cSpeed / 1000000, testResult.dSpeed / 1000000); + (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000); return; } @@ -1355,20 +1343,20 @@ static int allBench(BMK_result_t* resultPtr, *resultPtr = benchres.result; /* calculate uncertainty in compression / decompression runs */ - if(eqZero(benchres.result.cSpeed)) { + if(benchres.result.cSpeed) { + loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC) * VARIANCE; + } else { loopDurationC = 0; uncertaintyConstantC = 3; - } else { - loopDurationC = (U64)((double)(buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC) * VARIANCE; } - if(eqZero(benchres.result.dSpeed)) { + if(benchres.result.dSpeed) { + loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD) * VARIANCE; + } else { loopDurationD = 0; uncertaintyConstantD = 3; - } else { - loopDurationD = (U64)((double)(buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD) * VARIANCE; } /* anything with worse ratio in feas is definitely worse, discard */ From 8278a49cb608af483b70f301742899f5b0b8ab7d Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 6 Aug 2018 18:00:36 -0700 Subject: [PATCH 37/92] const srcPtrs --- programs/bench.c | 2 +- tests/paramgrill.c | 23 +++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 49b317870..874710c02 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -441,7 +441,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) { size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ - BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; + BMK_return_t results = { { 0, 0, 0, 0 }, 0 } ; size_t const loadedCompressedSize = srcSize; size_t cSize = 0; double ratio = 0.; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 5af3d88fa..5d1a73081 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -250,6 +250,11 @@ static int feasible(const BMK_result_t results, const constraint_t target) { } /* hill climbing value for part 1 */ +/* Scoring here is a linear reward for all set constraints normalized between 0 to 1 + * (with 0 at 0 and 1 being fully fulfilling the constraint), summed with a logarithmic + * bonus to exceeding the constraint value. We also give linear ratio for compression ratio. + * The constant factors are experimental. + */ static double resultScore(const BMK_result_t res, const size_t srcSize, const constraint_t target) { double cs = 0., ds = 0., rt, cm = 0.; const double r1 = 1, r2 = 0.1, rtr = 0.5; @@ -291,7 +296,7 @@ const char* g_stratName[ZSTD_btultra+1] = { "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -/* benchParam but only takes in one file. */ +/* benchParam but only takes in one input buffer. */ static int BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, @@ -324,8 +329,9 @@ static winnerInfo_t initWinnerInfo(ZSTD_compressionParameters p) { } typedef struct { + void* srcBuffer; size_t srcSize; - void** srcPtrs; + const void** srcPtrs; size_t* srcSizes; void** dstPtrs; size_t* dstCapacities; @@ -1605,7 +1611,7 @@ static winnerInfo_t optimizeFixedStrategy( static void freeBuffers(buffers_t b) { if(b.srcPtrs != NULL) { - free(b.srcPtrs[0]); + free(b.srcBuffer); } free(b.srcPtrs); free(b.srcSizes); @@ -1635,7 +1641,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles; U32 blockNb = 0; - buff->srcPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); @@ -1651,7 +1657,8 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab return 1; } - buff->srcPtrs[0] = malloc(benchedSize); + buff->srcBuffer = malloc(benchedSize); + buff->srcPtrs[0] = (const void*)buff->srcBuffer; buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024)); buff->resPtrs[0] = malloc(benchedSize); @@ -1684,11 +1691,11 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ { - char* buffer = (char*)(buff->srcPtrs[0]); - size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); + char* buffer = (char*)(buff->srcBuffer); + size_t const readSize = fread(((buffer)+pos), 1, (size_t)fileSize, f); size_t blocked = 0; while(blocked < readSize) { - buff->srcPtrs[blockNb] = (buffer) + (pos + blocked); + buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked)); buff->srcSizes[blockNb] = blockSize; blocked += blockSize; blockNb++; From ad16a69408139d87cf44eb830b6749bf6066bee1 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 6 Aug 2018 18:37:55 -0700 Subject: [PATCH 38/92] Readability improvements, renaming --- tests/paramgrill.c | 131 +++++++++++++++++++++++++-------------------- 1 file changed, 72 insertions(+), 59 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 5d1a73081..c0679e88e 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -362,7 +362,7 @@ BMK_benchParam(BMK_result_t* resultPtr, *********************************************************/ static void BMK_initCCtx(ZSTD_CCtx* ctx, - const void* dictBuffer, size_t dictBufferSize, int cLevel, + const void* dictBuffer, const size_t dictBufferSize, const int cLevel, const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { ZSTD_CCtx_reset(ctx); ZSTD_CCtx_resetParameters(ctx); @@ -389,7 +389,7 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, static void BMK_initDCtx(ZSTD_DCtx* dctx, - const void* dictBuffer, size_t dictBufferSize) { + const void* dictBuffer, const size_t dictBufferSize) { ZSTD_DCtx_reset(dctx); ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); } @@ -404,7 +404,7 @@ typedef struct { } BMK_initCCtxArgs; static size_t local_initCCtx(void* payload) { - BMK_initCCtxArgs* ag = (BMK_initCCtxArgs*)payload; + const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload; BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv); return 0; } @@ -416,7 +416,7 @@ typedef struct { } BMK_initDCtxArgs; static size_t local_initDCtx(void* payload) { - BMK_initDCtxArgs* ag = (BMK_initDCtxArgs*)payload; + const BMK_initDCtxArgs* ag = (const BMK_initDCtxArgs*)payload; BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize); return 0; } @@ -565,7 +565,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, BMK_freeTimeState(timeStateDecompress); return results; } - results.result.cSpeed = ((double)srcSize / intermediateResultCompress.result.result.nanoSecPerRun) * TIMELOOP_NANOSEC; + results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultCompress.result.result.nanoSecPerRun; results.result.cSize = intermediateResultCompress.result.result.sumOfReturn; } @@ -579,7 +579,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, BMK_freeTimeState(timeStateDecompress); return results; } - results.result.dSpeed = ((double)srcSize / intermediateResultDecompress.result.result.nanoSecPerRun) * TIMELOOP_NANOSEC; + results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC) / intermediateResultDecompress.result.result.nanoSecPerRun; } BMK_freeTimeState(timeStateCompress); @@ -597,7 +597,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, if(compressionResults.result.nanoSecPerRun == 0) { results.result.cSpeed = 0; } else { - results.result.cSpeed = (double)srcSize / compressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun; } results.result.cSize = compressionResults.result.sumOfReturn; } @@ -618,7 +618,7 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, if(decompressionResults.result.nanoSecPerRun == 0) { results.result.dSpeed = 0; } else { - results.result.dSpeed = (double)srcSize / decompressionResults.result.nanoSecPerRun * TIMELOOP_NANOSEC; + results.result.dSpeed = srcSize * TIMELOOP_NANOSEC / decompressionResults.result.nanoSecPerRun; } } } @@ -628,39 +628,44 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, return results; } -/* global winner used for display. */ -static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; -static constraint_t g_targetConstraints; static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) { - if(DEBUG || compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { - char lvlstr[15] = "Custom Level"; - const U64 time = UTIL_clockSpanNano(g_time); - const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); + char lvlstr[15] = "Custom Level"; + const U64 time = UTIL_clockSpanNano(g_time); + const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); - if(DEBUG && compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { + DISPLAY("\r%79s\r", ""); + + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, + params.targetLength, g_stratName[(U32)(params.strategy)]); + + if(cLevel != CUSTOM_LEVEL) { + snprintf(lvlstr, 15, " Level %2u ", cLevel); + } + + fprintf(f, + "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", + lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20)); + + if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } + fprintf(f, "\n"); +} + +static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const constraint_t targetConstraints, const size_t srcSize) +{ + /* global winner used for constraints */ + static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; + + if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { + if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); } - DISPLAY("\r%79s\r", ""); + BMK_printWinner(f, cLevel, result, params, srcSize); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, - params.targetLength, g_stratName[(U32)(params.strategy)]); - - if(cLevel != CUSTOM_LEVEL) { - snprintf(lvlstr, 15, " Level %2u ", cLevel); - } - - fprintf(f, - "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", - lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20)); - - if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } - fprintf(f, "\n"); - - if(compareResultLT(g_winner.result, result, g_targetConstraints, srcSize)) { + if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { BMK_translateAdvancedParams(params); g_winner.result = result; g_winner.params = params; @@ -950,12 +955,18 @@ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInd unsigned ind = 0; for(i = 0; i < varyLen; i++) { switch(varyParams[i]) { - case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog - ZSTD_WINDOWLOG_MIN ; break; - case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog - ZSTD_CHAINLOG_MIN ; break; - case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog - ZSTD_HASHLOG_MIN ; break; - case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog - ZSTD_SEARCHLOG_MIN ; break; - case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength - ZSTD_SEARCHLENGTH_MIN; break; - case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; + case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog + - ZSTD_WINDOWLOG_MIN ; break; + case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog + - ZSTD_CHAINLOG_MIN ; break; + case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog + - ZSTD_HASHLOG_MIN ; break; + case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog + - ZSTD_SEARCHLOG_MIN ; break; + case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength + - ZSTD_SEARCHLENGTH_MIN; break; + case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) + - ZSTD_TARGETLENGTH_MIN; break; } } return ind; @@ -976,8 +987,12 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va } } -/* Initialize memotable, immediately mark redundant / obviously infeasible params as such */ -static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +/* Initialize memoization table, which tracks and prevents repeated benchmarking + * of the same set of parameters. In addition, it is also used to immediately mark + * redundant / obviously non-optimal parameter configurations (e.g. wlog - 1 larger) + * than srcSize, clog > wlog, ... + */ +static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; @@ -985,6 +1000,7 @@ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConst int whFixed = !paramConstraints.windowLog || !paramConstraints.hashLog; int wFixed = !paramConstraints.windowLog; int j = 0; + assert(memoTable != NULL); memset(memoTable, 0, arrayLen); cParamZeroMin(¶mConstraints); @@ -994,7 +1010,7 @@ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConst memoTable[i] = 255; j++; } - if(wFixed && (1ULL << paramConstraints.windowLog) > srcSize) { + if(wFixed && (1ULL << (paramConstraints.windowLog - 1)) > srcSize) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ @@ -1033,7 +1049,7 @@ static void createMemoTable(U8* memoTable, ZSTD_compressionParameters paramConst } /* frees all allocated memotables */ -static void memoTableFreeAll(U8** mtAll) { +static void freeMemoTableArray(U8** mtAll) { int i; if(mtAll == NULL) { return; } for(i = 1; i <= (int)ZSTD_btultra; i++) { @@ -1056,10 +1072,10 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); if(mtAll[i] == NULL) { - memoTableFreeAll(mtAll); + freeMemoTableArray(mtAll); return NULL; } - createMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); + initMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); } return mtAll; @@ -1451,7 +1467,7 @@ static int benchMemo(BMK_result_t* resultPtr, DISPLAY("Count: %d\n", bmcount); bmcount++; } - BMK_printWinner(stdout, CUSTOM_LEVEL, *resultPtr, cParams, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize); if(res == BETTER_RESULT || feas) { memoTable[memind] = 255; @@ -1498,7 +1514,7 @@ static winnerInfo_t climbOnce(const constraint_t target, better = 0; DEBUGOUTPUT("Start\n"); cparam = winnerInfo.params; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); candidateInfo.params = cparam; /* all dist-1 candidates */ for(i = 0; i < varLen; i++) { @@ -1513,7 +1529,7 @@ static winnerInfo_t climbOnce(const constraint_t target, varArray, varLen, feas); if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; @@ -1539,7 +1555,7 @@ static winnerInfo_t climbOnce(const constraint_t target, varArray, varLen, feas); if(res == BETTER_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; @@ -1601,7 +1617,7 @@ static winnerInfo_t optimizeFixedStrategy( candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init); if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { winnerInfo = candidateInfo; - BMK_printWinner(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); i = 0; } i++; @@ -1890,7 +1906,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } - createMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); + initMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); } else { allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize); } @@ -1910,12 +1926,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } target.cSpeed = (U32)winner.result.cSpeed; - g_targetConstraints = target; - BMK_printWinner(stdout, cLevel, winner.result, winner.params, buf.srcSize); + BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } - g_targetConstraints = target; - /* bench */ DISPLAY("\r%79s\r", ""); if(nbFiles == 1) { @@ -1946,7 +1959,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ ZSTD_compressionParameters CParams = ZSTD_getCParams(i, maxBlockSize, ctx.dictSize); CParams = maskParams(CParams, paramTarget); ec = BMK_benchParam(&candidate, buf, ctx, CParams); - BMK_printWinner(stdout, i, candidate, CParams, buf.srcSize); + BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize); if(!ec && compareResultLT(winner.result, candidate, relaxTarget(target), buf.srcSize)) { winner.result = candidate; @@ -1956,7 +1969,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } } - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); BMK_translateAdvancedParams(winner.params); DEBUGOUTPUT("Real Opt\n"); /* start 'real' tests */ @@ -2000,7 +2013,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } /* end summary */ - BMK_printWinner(stdout, CUSTOM_LEVEL, winner.result, winner.params, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); BMK_translateAdvancedParams(winner.params); DISPLAY("grillParams size - optimizer completed \n"); @@ -2008,7 +2021,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ _cleanUp: freeContexts(ctx); freeBuffers(buf); - memoTableFreeAll(allMT); + freeMemoTableArray(allMT); return ret; } From 6f480927af465d34ac9914789291094477c619bb Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 7 Aug 2018 11:56:14 -0700 Subject: [PATCH 39/92] argument parsing cleanup + clarifying comment --- tests/paramgrill.c | 95 +++++++++++++++++++++++++++++++--------------- 1 file changed, 64 insertions(+), 31 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index c0679e88e..a1d409d67 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -977,12 +977,18 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va int i; for(i = varyLen - 1; i >= 0; i--) { switch(varyParams[i]) { - case wlog_ind: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; ind /= WLOG_RANGE; break; - case clog_ind: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; ind /= CLOG_RANGE; break; - case hlog_ind: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; ind /= HLOG_RANGE; break; - case slog_ind: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; ind /= SLOG_RANGE; break; - case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; ind /= SLEN_RANGE; break; - case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; ind /= TLEN_RANGE; break; + case wlog_ind: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; + ind /= WLOG_RANGE; break; + case clog_ind: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; + ind /= CLOG_RANGE; break; + case hlog_ind: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; + ind /= HLOG_RANGE; break; + case slog_ind: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; + ind /= SLOG_RANGE; break; + case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; + ind /= SLEN_RANGE; break; + case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; + ind /= TLEN_RANGE; break; } } } @@ -1135,13 +1141,20 @@ static ZSTD_compressionParameters randomParams(void) U32 validated = 0; while (!validated) { /* totally random entry */ - p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + ZSTD_CHAINLOG_MIN; - p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + ZSTD_HASHLOG_MIN; - p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + ZSTD_SEARCHLOG_MIN; - p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + ZSTD_WINDOWLOG_MIN; - p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + ZSTD_SEARCHLENGTH_MIN; + p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) + + ZSTD_CHAINLOG_MIN; + p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) + + ZSTD_HASHLOG_MIN; + p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) + + ZSTD_SEARCHLOG_MIN; + p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) + + ZSTD_WINDOWLOG_MIN; + p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) + + ZSTD_SEARCHLENGTH_MIN; p.targetLength=(FUZ_rand(&g_rand) % (512)); + p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); } return p; @@ -1657,6 +1670,8 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles; U32 blockNb = 0; + memset(buff, 0, sizeof(buffers_t)); + buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); @@ -1760,6 +1775,7 @@ static void freeContexts(contexts_t ctx) { static int createContexts(contexts_t* ctx, const char* dictFileName) { FILE* f; size_t readSize; + U64 dictSize; ctx->cctx = ZSTD_createCCtx(); ctx->dctx = ZSTD_createDCtx(); ctx->dictSize = 0; @@ -1775,7 +1791,16 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) { return 0; } - ctx->dictSize = UTIL_getFileSize(dictFileName); + dictSize = UTIL_getFileSize(dictFileName); + + if(dictSize == UTIL_FILESIZE_UNKNOWN) { + DISPLAY("Unable to get dictionary size\n"); + freeContexts(*ctx); + return 1; + } else { + ctx->dictSize = (size_t)dictSize; + } + ctx->dictBuffer = malloc(ctx->dictSize); f = fopen(dictFileName, "rb"); @@ -1848,7 +1873,15 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS /* main fn called when using --optimize */ /* Does strategy selection by benchmarking default compression levels * then optimizes by strategy, starting with the best one and moving - * progressively moving further away by number */ + * progressively moving further away by number + * args: + * fileNamesTable - list of files to benchmark + * nbFiles - length of fileNamesTable + * dictFileName - name of dictionary file if one, else NULL + * target - performance constraints (cSpeed, dSpeed, cMem) + * paramTarget - parameter constraints (i.e. restriction search space to where strategy = ZSTD_fast) + * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) + */ static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { varInds_t varArray [NUM_PARAMS]; @@ -2092,6 +2125,18 @@ static int badusage(const char* exename) return 1; } +#define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } } +#define PARSE_CPARAMS(variable) \ +{ \ + PARSE_SUB_ARGS("windowLog=", "wlog=", variable.windowLog); \ + PARSE_SUB_ARGS("chainLog=" , "clog=", variable.chainLog); \ + PARSE_SUB_ARGS("hashLog=", "hlog=", variable.hashLog); \ + PARSE_SUB_ARGS("searchLog=" , "slog=", variable.searchLog); \ + PARSE_SUB_ARGS("searchLength=", "slen=", variable.searchLength); \ + PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.targetLength); \ + PARSE_SUB_ARGS("strategy=", "strat=", variable.strategy); \ +} + int main(int argc, const char** argv) { int i, @@ -2127,17 +2172,11 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "--optimize=")) { optimizer = 1; for ( ; ;) { - if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { paramTarget.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { paramTarget.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { paramTarget.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { paramTarget.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { paramTarget.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { paramTarget.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { paramTarget.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "compressionSpeed=") || longCommandWArg(&argument, "cSpeed=")) { target.cSpeed = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "decompressionSpeed=") || longCommandWArg(&argument, "dSpeed=")) { target.dSpeed = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "compressionMemory=") || longCommandWArg(&argument, "cMem=")) { target.cMem = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { optimizerCLevel = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + PARSE_CPARAMS(paramTarget); + PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed); + PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); + PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); + PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel); DISPLAY("invalid optimization parameter \n"); return 1; } @@ -2152,13 +2191,7 @@ int main(int argc, const char** argv) g_singleRun = 1; g_params = ZSTD_getCParams(2, g_blockSize, 0); for ( ; ;) { - if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { g_params.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { g_params.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { g_params.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { g_params.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "searchLength=") || longCommandWArg(&argument, "slen=")) { g_params.searchLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { g_params.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } - if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { g_params.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } + PARSE_CPARAMS(g_params) if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { g_params = ZSTD_getCParams(readU32FromChar(&argument), g_blockSize, 0); if (argument[0]==',') { argument++; continue; } else break; } DISPLAY("invalid compression parameter \n"); return 1; From 0ece2e5cdc33d9e1499125820a85cae684c6deaa Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 9 Aug 2018 11:38:09 -0700 Subject: [PATCH 40/92] Add consts + fix gcc-8 warnings --- tests/fullbench.c | 3 ++- tests/paramgrill.c | 48 ++++++++++++++++++++++++---------------------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/tests/fullbench.c b/tests/fullbench.c index 270cac86a..12c1e1ae4 100644 --- a/tests/fullbench.c +++ b/tests/fullbench.c @@ -514,9 +514,10 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb, int cLevel, /* benchmark loop */ { + void* dstBuffv = (void*)dstBuff; r = BMK_benchFunction(benchFunction, buff2, NULL, NULL, 1, &src, &srcSize, - (void **)&dstBuff, &dstBuffSize, NULL, g_nbIterations); + &dstBuffv, &dstBuffSize, NULL, g_nbIterations); if(r.error) { DISPLAY("ERROR %d ! ! \n", r.error); errorcode = r.error; diff --git a/tests/paramgrill.c b/tests/paramgrill.c index a1d409d67..eeafc8302 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -350,7 +350,7 @@ typedef struct { static int BMK_benchParam(BMK_result_t* resultPtr, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams) { BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, ctx.cctx, ctx.dctx, 0, "Files"); *resultPtr = res.result; @@ -482,13 +482,14 @@ static size_t local_defaultDecompress( * From Paramgrill End *********************************************************/ -/* Replicate function of benchMemAdvanced, but with pre-split src / dst buffers, with relevant info to invert it (compressedSizes) passed out. */ +/* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */ +/* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */ /* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ /* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */ /* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ /* dictionary nullable, nothing else though. */ -static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, +static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx, const int cLevel, const ZSTD_compressionParameters* comprParams, const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) { @@ -496,11 +497,11 @@ static BMK_return_t BMK_benchMemInvertible(buffers_t buf, contexts_t ctx, BMK_return_t results = { { 0, 0., 0., 0 }, 0 } ; const void *const *const srcPtrs = (const void *const *const)buf.srcPtrs; size_t const *const srcSizes = buf.srcSizes; - void** dstPtrs = buf.dstPtrs; - size_t* dstCapacities = buf.dstCapacities; - size_t* dstSizes = buf.dstSizes; - void** resPtrs = buf.resPtrs; - size_t* resSizes = buf.resSizes; + void** const dstPtrs = buf.dstPtrs; + size_t const *const dstCapacities = buf.dstCapacities; + size_t* const dstSizes = buf.dstSizes; + void** const resPtrs = buf.resPtrs; + size_t const *const resSizes = buf.resSizes; const void* dictBuffer = ctx.dictBuffer; const size_t dictBufferSize = ctx.dictSize; const size_t nbBlocks = buf.nbBlocks; @@ -1355,7 +1356,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles) /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ #define VARIANCE 1.1 static int allBench(BMK_result_t* resultPtr, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams, const constraint_t target, BMK_result_t* winnerResult, int feas) { @@ -1463,11 +1464,11 @@ static int allBench(BMK_result_t* resultPtr, /* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ static int benchMemo(BMK_result_t* resultPtr, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams, const constraint_t target, - BMK_result_t* winnerResult, U8* memoTable, - const varInds_t* varyParams, const int varyLen, int feas) { + BMK_result_t* winnerResult, U8* const memoTable, + const varInds_t* varyParams, const int varyLen, const int feas) { static int bmcount = 0; size_t memind = memoTableInd(&cParams, varyParams, varyLen); int res; @@ -1502,8 +1503,8 @@ static int benchMemo(BMK_result_t* resultPtr, */ static winnerInfo_t climbOnce(const constraint_t target, const varInds_t* varArray, const int varLen, - U8* memoTable, - buffers_t buf, contexts_t ctx, + U8* const memoTable, + const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters init) { /* * cparam - currently considered 'center' @@ -1605,11 +1606,11 @@ static winnerInfo_t climbOnce(const constraint_t target, maybe allow giving it a first init? */ static winnerInfo_t optimizeFixedStrategy( - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const constraint_t target, ZSTD_compressionParameters paramTarget, const ZSTD_strategy strat, const varInds_t* varArray, const int varLen, - U8* memoTable, const int tries) { + U8* const memoTable, const int tries) { int i = 0; varInds_t varNew[NUM_PARAMS]; int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); @@ -1638,7 +1639,7 @@ static winnerInfo_t optimizeFixedStrategy( return winnerInfo; } -static void freeBuffers(buffers_t b) { +static void freeBuffers(const buffers_t b) { if(b.srcPtrs != NULL) { free(b.srcBuffer); } @@ -1659,8 +1660,8 @@ static void freeBuffers(buffers_t b) { } /* allocates buffer's arguments. returns 0 = success / 1 = failuere */ -static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, - size_t nbFiles) +static int createBuffers(buffers_t* const buff, const char* const * const fileNamesTable, + const size_t nbFiles) { size_t pos = 0; size_t n; @@ -1720,7 +1721,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab DISPLAY("Loading %s... \r", fileNamesTable[n]); - if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ + if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n = nbFiles; /* buffer too small - stop after this file */ { char* buffer = (char*)(buff->srcBuffer); size_t const readSize = fread(((buffer)+pos), 1, (size_t)fileSize, f); @@ -1765,14 +1766,14 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab return 0; } -static void freeContexts(contexts_t ctx) { +static void freeContexts(const contexts_t ctx) { free(ctx.dictBuffer); ZSTD_freeCCtx(ctx.cctx); ZSTD_freeDCtx(ctx.dctx); } /* Creates struct holding contexts and dictionary buffers. returns 0 on success, 1 on failure. */ -static int createContexts(contexts_t* ctx, const char* dictFileName) { +static int createContexts(contexts_t* const ctx, const char* dictFileName) { FILE* f; size_t readSize; U64 dictSize; @@ -2268,7 +2269,8 @@ int main(int argc, const char** argv) g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument); continue; case 'L': - { int const cLevel = readU32FromChar(&argument); + { argument++; + int const cLevel = readU32FromChar(&argument); g_params = ZSTD_getCParams(cLevel, g_blockSize, 0); continue; } From bfe8392e23cbb561963451c154bde2a368fde926 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 9 Aug 2018 12:07:57 -0700 Subject: [PATCH 41/92] Remove ctx from benchMem --- programs/bench.c | 46 ++++++++++---------------------------- programs/bench.h | 4 ---- tests/paramgrill.c | 55 ++++++++++++++++++---------------------------- 3 files changed, 33 insertions(+), 72 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 874710c02..7662678ab 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -708,7 +708,6 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) { @@ -730,6 +729,9 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, BMK_timedFnState_t* timeStateCompress = BMK_createTimeState(adv->nbSeconds); BMK_timedFnState_t* timeStateDecompress = BMK_createTimeState(adv->nbSeconds); + ZSTD_CCtx* ctx = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize); @@ -756,6 +758,9 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, BMK_freeTimeState(timeStateCompress); BMK_freeTimeState(timeStateDecompress); + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + free(internalDstBuffer); free(resultBuffer); @@ -781,7 +786,6 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName) { const BMK_advancedParams_t adv = BMK_initAdvancedParams(); @@ -790,35 +794,9 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, fileSizes, nbFiles, cLevel, comprParams, dictBuffer, dictBufferSize, - ctx, dctx, displayLevel, displayName, &adv); } -static BMK_return_t BMK_benchMemCtxless(const void* srcBuffer, size_t srcSize, - const size_t* fileSizes, unsigned nbFiles, - int cLevel, const ZSTD_compressionParameters* const comprParams, - const void* dictBuffer, size_t dictBufferSize, - int displayLevel, const char* displayName, - const BMK_advancedParams_t* const adv) -{ - BMK_return_t res; - ZSTD_CCtx* ctx = ZSTD_createCCtx(); - ZSTD_DCtx* dctx = ZSTD_createDCtx(); - if(ctx == NULL || dctx == NULL) { - EXM_THROW(12, BMK_return_t, "not enough memory for contexts"); - } - res = BMK_benchMemAdvanced(srcBuffer, srcSize, - NULL, 0, - fileSizes, nbFiles, - cLevel, comprParams, - dictBuffer, dictBufferSize, - ctx, dctx, - displayLevel, displayName, adv); - ZSTD_freeCCtx(ctx); - ZSTD_freeDCtx(dctx); - return res; -} - static size_t BMK_findMaxMem(U64 requiredMem) { size_t const step = 64 MB; @@ -859,12 +837,12 @@ static BMK_return_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize, if (displayLevel == 1 && !adv->additionalParam) DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, adv->nbSeconds, (U32)(adv->blockSize>>10)); - res = BMK_benchMemCtxless(srcBuffer, benchedSize, - fileSizes, nbFiles, - cLevel, comprParams, - dictBuffer, dictBufferSize, - displayLevel, displayName, - adv); + res = BMK_benchMemAdvanced(srcBuffer, benchedSize, + NULL, 0, + fileSizes, nbFiles, + cLevel, comprParams, + dictBuffer, dictBufferSize, + displayLevel, displayName, adv); return res; } diff --git a/programs/bench.h b/programs/bench.h index 6247fa596..ad4ede1f0 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -126,8 +126,6 @@ BMK_return_t BMK_syntheticTest(int cLevel, double compressibility, * comprParams - basic compression parameters * dictBuffer - a dictionary if used, null otherwise * dictBufferSize - size of dictBuffer, 0 otherwise - * ctx - Compression Context (must be provided) - * dctx - Decompression Context (must be provided) * diplayLevel - see BMK_benchFiles * displayName - name used by display * return @@ -139,7 +137,6 @@ BMK_return_t BMK_benchMem(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName); /* See benchMem for normal parameter uses and return, see advancedParams_t for adv @@ -151,7 +148,6 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, const size_t* fileSizes, unsigned nbFiles, const int cLevel, const ZSTD_compressionParameters* comprParams, const void* dictBuffer, size_t dictBufferSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, int displayLevel, const char* displayName, const BMK_advancedParams_t* adv); diff --git a/tests/paramgrill.c b/tests/paramgrill.c index eeafc8302..e530af735 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -300,10 +300,9 @@ const char* g_stratName[ZSTD_btultra+1] = { static int BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx, const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, ctx, dctx, 0, "File"); + BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File"); *resultPtr = res.result; return res.error; } @@ -352,7 +351,7 @@ static int BMK_benchParam(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, ctx.cctx, ctx.dctx, 0, "Files"); + BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, 0, "Files"); *resultPtr = res.result; return res.error; } @@ -724,14 +723,13 @@ static void BMK_init_level_constraints(int bytePerSec_level1) } static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, - const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx) + const void* srcBuffer, size_t srcSize) { BMK_result_t testResult; int better = 0; int cLevel; - BMK_benchParam1(&testResult, srcBuffer, srcSize, ctx, dctx, params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, params); for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { @@ -1104,8 +1102,7 @@ static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { static void playAround(FILE* f, winnerInfo_t* winners, ZSTD_compressionParameters params, - const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx) + const void* srcBuffer, size_t srcSize) { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); @@ -1126,11 +1123,11 @@ static void playAround(FILE* f, winnerInfo_t* winners, /* test */ b = NB_TESTS_PLAYED(p); (*b)++; - if (!BMK_seed(winners, p, srcBuffer, srcSize, ctx, dctx)) continue; + if (!BMK_seed(winners, p, srcBuffer, srcSize)) continue; /* improvement found => search more */ BMK_printWinners(f, winners, srcSize); - playAround(f, winners, p, srcBuffer, srcSize, ctx, dctx); + playAround(f, winners, p, srcBuffer, srcSize); } } @@ -1177,31 +1174,30 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v static void BMK_selectRandomStart( FILE* f, winnerInfo_t* winners, - const void* srcBuffer, size_t srcSize, - ZSTD_CCtx* ctx, ZSTD_DCtx* dctx) + const void* srcBuffer, size_t srcSize) { U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); if ((id==0) || (winners[id].params.windowLog==0)) { /* use some random entry */ ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0); - playAround(f, winners, p, srcBuffer, srcSize, ctx, dctx); + playAround(f, winners, p, srcBuffer, srcSize); } else { - playAround(f, winners, winners[id].params, srcBuffer, srcSize, ctx, dctx); + playAround(f, winners, winners[id].params, srcBuffer, srcSize); } } -static void BMK_benchOnce(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffer, size_t srcSize) +static void BMK_benchOnce(const void* srcBuffer, size_t srcSize) { BMK_result_t testResult; g_params = ZSTD_adjustCParams(g_params, srcSize, 0); - BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, g_params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params); DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000); return; } -static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* srcBuffer, size_t srcSize) +static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) { ZSTD_compressionParameters params; winnerInfo_t winners[NB_LEVELS_TRACKED+1]; @@ -1220,7 +1216,7 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); BMK_result_t testResult; - BMK_benchParam1(&testResult, srcBuffer, srcSize, cctx, dctx, l1params); + BMK_benchParam1(&testResult, srcBuffer, srcSize, l1params); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); } @@ -1229,14 +1225,14 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src int i; for (i=0; i<=maxSeeds; i++) { params = ZSTD_getCParams(i, blockSize, 0); - BMK_seed(winners, params, srcBuffer, srcSize, cctx, dctx); + BMK_seed(winners, params, srcBuffer, srcSize); } } BMK_printWinners(f, winners, srcSize); /* start tests */ { const time_t grillStart = time(NULL); do { - BMK_selectRandomStart(f, winners, srcBuffer, srcSize, cctx, dctx); + BMK_selectRandomStart(f, winners, srcBuffer, srcSize); } while (BMK_timeSpan(grillStart) < g_grillDuration_s); } @@ -1248,21 +1244,12 @@ static void BMK_benchFullTable(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, const void* src fclose(f); } -static void BMK_benchMem_usingCCtx(ZSTD_CCtx* const cctx, ZSTD_DCtx* const dctx, const void* srcBuffer, size_t srcSize) +static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize) { if (g_singleRun) - return BMK_benchOnce(cctx, dctx, srcBuffer, srcSize); + return BMK_benchOnce(srcBuffer, srcSize); else - return BMK_benchFullTable(cctx, dctx, srcBuffer, srcSize); -} - -static void BMK_benchMemCCtxInit(const void* srcBuffer, size_t srcSize) -{ - ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); - if (cctx==NULL || dctx==NULL) { DISPLAY("Context Creation failed \n"); exit(1); } - BMK_benchMem_usingCCtx(cctx, dctx, srcBuffer, srcSize); - ZSTD_freeCCtx(cctx); + return BMK_benchFullTable(srcBuffer, srcSize); } @@ -1280,7 +1267,7 @@ static int benchSample(void) /* bench */ DISPLAY("\r%79s\r", ""); DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100)); - BMK_benchMemCCtxInit(origBuff, benchedSize); + BMK_benchMemInit(origBuff, benchedSize); free(origBuff); return 0; @@ -1338,7 +1325,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles) /* bench */ DISPLAY("\r%79s\r", ""); DISPLAY("using %s : \n", inFileName); - BMK_benchMemCCtxInit(origBuff, benchedSize); + BMK_benchMemInit(origBuff, benchedSize); /* clean */ free(origBuff); From 51e71a5ec71091d6cb2e0e23b82d15eac9d1833f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 9 Aug 2018 12:28:25 -0700 Subject: [PATCH 42/92] added zstdgrep documentation presenting `zstdgrep` limit regarding dictionary compression with workaround recommended by @tobwen (#1268) --- programs/README.md | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/programs/README.md b/programs/README.md index 2833875e5..22a00409c 100644 --- a/programs/README.md +++ b/programs/README.md @@ -185,7 +185,7 @@ version is less than `128 MiB`). Compression Speed vs Ratio | Decompression Speed ---------------------------|--------------------- -![Compression Speed vs Ratio](../doc/images/ldmCspeed.png "Compression Speed vs Ratio") | ![Decompression Speed](../doc/images/ldmDspeed.png "Decompression Speed") +![Compression Speed vs Ratio](https://raw.githubusercontent.com/facebook/zstd/v1.3.3/doc/images/ldmCspeed.png "Compression Speed vs Ratio") | ![Decompression Speed](https://raw.githubusercontent.com/facebook/zstd/v1.3.3/doc/images/ldmDspeed.png "Decompression Speed") | Method | Compression ratio | Compression speed | Decompression speed | |:-------|------------------:|-------------------------:|---------------------------:| @@ -208,10 +208,24 @@ The below table illustrates this on the [Silesia compression corpus]. [Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia | Method | Compression ratio | Compression speed | Decompression speed | -|:-------|------------------:|-------------------------:|---------------------------:| -| `zstd -1` | `2.878` | `231.7 MB/s` | `594.4 MB/s` | -| `zstd -1 --long` | `2.929` | `106.5 MB/s` | `517.9 MB/s` | -| `zstd -5` | `3.274` | `77.1 MB/s` | `464.2 MB/s` | -| `zstd -5 --long` | `3.319` | `51.7 MB/s` | `371.9 MB/s` | -| `zstd -10` | `3.523` | `16.4 MB/s` | `489.2 MB/s` | -| `zstd -10 --long`| `3.566` | `16.2 MB/s` | `415.7 MB/s` | +|:-------|------------------:|------------------:|---------------------:| +| `zstd -1` | `2.878` | `231.7 MB/s` | `594.4 MB/s` | +| `zstd -1 --long` | `2.929` | `106.5 MB/s` | `517.9 MB/s` | +| `zstd -5` | `3.274` | `77.1 MB/s` | `464.2 MB/s` | +| `zstd -5 --long` | `3.319` | `51.7 MB/s` | `371.9 MB/s` | +| `zstd -10` | `3.523` | `16.4 MB/s` | `489.2 MB/s` | +| `zstd -10 --long`| `3.566` | `16.2 MB/s` | `415.7 MB/s` | + + +#### zstdgrep + +`zstdgrep` is a utility which makes it possible to `grep` directly a `.zst` compressed file. +It's used the same way as normal `grep`, for example : +`zstdgrep pattern file.zst` + +`zstdgrep` is _not_ compatible with dictionary compression. + +To search into a file compressed with a dictionary, +it's necessary to decompress it using `zstd` or `zstdcat`, +and then pipe the result to `grep`. For example : +`zstdcat -D dictionary -qc -- file.zst | grep pattern` From 79a35ac20d073271c46d14cf76d719f95a2a8f9c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 9 Aug 2018 15:16:31 -0700 Subject: [PATCH 43/92] minor code comments improvements --- lib/compress/zstdmt_compress.c | 9 ++++++--- programs/fileio.c | 11 ++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index d5193d52a..74f9dc29c 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -1080,9 +1080,9 @@ ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx) { ZSTD_frameProgression fps; DEBUGLOG(6, "ZSTDMT_getFrameProgression"); + fps.ingested = mtctx->consumed + mtctx->inBuff.filled; fps.consumed = mtctx->consumed; fps.produced = mtctx->produced; - fps.ingested = mtctx->consumed + mtctx->inBuff.filled; { unsigned jobNb; unsigned lastJobNb = mtctx->nextJobID + mtctx->jobReady; assert(mtctx->jobReady <= 1); DEBUGLOG(6, "ZSTDMT_getFrameProgression: jobs: from %u to <%u (jobReady:%u)", @@ -1092,8 +1092,8 @@ ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx) ZSTD_pthread_mutex_lock(&mtctx->jobs[wJobID].job_mutex); { size_t const cResult = mtctx->jobs[wJobID].cSize; size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; - fps.consumed += mtctx->jobs[wJobID].consumed; fps.ingested += mtctx->jobs[wJobID].src.size; + fps.consumed += mtctx->jobs[wJobID].consumed; fps.produced += produced; } ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); @@ -1545,6 +1545,8 @@ static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZS /*! ZSTDMT_flushProduced() : + * flush whatever data has been produced but not yet flushed in current job. + * move to next job if current one is fully flushed. * `output` : `pos` will be updated with amount of data flushed . * `blockToFlush` : if >0, the function will block and wait if there is no data available to flush . * @return : amount of data remaining within internal buffer, 0 if no more, 1 if unknown but > 0, or an error code */ @@ -1593,6 +1595,7 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u mtctx->jobs[wJobID].cSize += 4; /* can write this shared value, as worker is no longer active */ mtctx->jobs[wJobID].frameChecksumNeeded = 0; } + if (cSize > 0) { /* compression is ongoing or completed */ size_t const toFlush = MIN(cSize - mtctx->jobs[wJobID].dstFlushed, output->size - output->pos); DEBUGLOG(5, "ZSTDMT_flushProduced: Flushing %u bytes from job %u (completion:%u/%u, generated:%u)", @@ -1606,7 +1609,7 @@ static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, u output->pos += toFlush; mtctx->jobs[wJobID].dstFlushed += toFlush; /* can write : this value is only used by mtctx */ - if ( (srcConsumed == srcSize) /* job completed */ + if ( (srcConsumed == srcSize) /* job is completed */ && (mtctx->jobs[wJobID].dstFlushed == cSize) ) { /* output buffer fully flushed => free this job position */ DEBUGLOG(5, "Job %u completed (%u bytes), moving to next one", mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed); diff --git a/programs/fileio.c b/programs/fileio.c index 85367fdfc..c1587f8f8 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -727,11 +727,6 @@ static unsigned long long FIO_compressLz4Frame(cRess_t* ress, #endif -/*! FIO_compressFilename_internal() : - * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. - * @return : 0 : compression completed correctly, - * 1 : missing or pb opening srcFileName - */ static unsigned long long FIO_compressZstdFrame(const cRess_t* ressPtr, const char* srcFileName, U64 fileSize, @@ -763,7 +758,8 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, directive = ZSTD_e_end; result = 1; - while (inBuff.pos != inBuff.size || (directive == ZSTD_e_end && result != 0)) { + while ((inBuff.pos != inBuff.size) /* input buffer must be entirely ingested */ + || (directive == ZSTD_e_end && result != 0) ) { ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 }; CHECK_V(result, ZSTD_compress_generic(ress.cctx, &outBuff, &inBuff, directive)); @@ -786,7 +782,8 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, (U32)(zfp.consumed >> 20), (U32)(zfp.produced >> 20), cShare ); - } else { /* g_displayLevel == 2 */ + } else { + assert(g_displayLevel == 2); DISPLAYLEVEL(2, "\rRead : %u ", (U32)(zfp.consumed >> 20)); if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAYLEVEL(2, "/ %u ", (U32)(fileSize >> 20)); From 754942cb799fc81a3c590733a1b77958bf74d818 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 9 Aug 2018 15:57:19 -0700 Subject: [PATCH 44/92] fixed assert() condition --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index c1587f8f8..39b2c741c 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -783,7 +783,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr, (U32)(zfp.produced >> 20), cShare ); } else { - assert(g_displayLevel == 2); + /* g_displayLevel <= 2; only display notifications if == 2; */ DISPLAYLEVEL(2, "\rRead : %u ", (U32)(zfp.consumed >> 20)); if (fileSize != UTIL_FILESIZE_UNKNOWN) DISPLAYLEVEL(2, "/ %u ", (U32)(fileSize >> 20)); From 3ac2c22485ab5508f47e3eab642b787af0e68b5f Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 9 Aug 2018 16:38:32 -0700 Subject: [PATCH 45/92] Reorder declaration --- tests/paramgrill.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index e530af735..bcb6bcae3 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -2256,8 +2256,9 @@ int main(int argc, const char** argv) g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument); continue; case 'L': - { argument++; - int const cLevel = readU32FromChar(&argument); + { int cLevel; + argument++; + cLevel = readU32FromChar(&argument); g_params = ZSTD_getCParams(cLevel, g_blockSize, 0); continue; } From 0cc75d6ee02efe9da7901848ce588a7f9f61b4f9 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 13 Aug 2018 13:56:18 -0700 Subject: [PATCH 46/92] Default lvl 1 MB to 2^20 --- tests/paramgrill.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index bcb6bcae3..d7165021d 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -65,6 +65,7 @@ static const int g_maxNbVariations = 64; #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) #define CUSTOM_LEVEL 99 +#define BASE_CLEVEL 1 /* indices for each of the variables */ typedef enum { @@ -302,7 +303,7 @@ BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File"); + BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, BASE_CLEVEL, &cParams, NULL, 0, 0, "File"); *resultPtr = res.result; return res.error; } @@ -792,16 +793,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* too large compression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, (double)testResult.cSpeed / 1000000, - O_ratio, (double)winners[cLevel].result.cSpeed / 1000000., cLevel); + W_ratio, (double)testResult.cSpeed / (1 MB), + O_ratio, (double)winners[cLevel].result.cSpeed / (1 MB), cLevel); continue; } if (W_DSpeed_note < O_DSpeed_note ) { /* too large decompression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, (double)testResult.dSpeed / 1000000., - O_ratio, (double)winners[cLevel].result.dSpeed / 1000000., cLevel); + W_ratio, (double)testResult.dSpeed / (1 MB), + O_ratio, (double)winners[cLevel].result.dSpeed / (1 MB), cLevel); continue; } @@ -1193,7 +1194,7 @@ static void BMK_benchOnce(const void* srcBuffer, size_t srcSize) g_params = ZSTD_adjustCParams(g_params, srcSize, 0); BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params); DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, - (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000); + (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB)); return; } @@ -1211,7 +1212,7 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } if (g_target) { - BMK_init_level_constraints(g_target*1000000); + BMK_init_level_constraints(g_target*(1 MB)); } else { /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); @@ -1256,7 +1257,7 @@ static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize) static int benchSample(void) { const char* const name = "Sample 10MB"; - size_t const benchedSize = 10000000; + size_t const benchedSize = (10 MB); void* origBuff = malloc(benchedSize); if (!origBuff) { perror("not enough memory"); return 12; } @@ -1354,7 +1355,7 @@ static int allBench(BMK_result_t* resultPtr, double winnerRS; /* initial benchmarking, gives exact ratio and memory, warms up future runs */ - benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1); + benchres = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_iterMode, 1); winnerRS = resultScore(*winnerResult, buf.srcSize, target); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); @@ -1389,14 +1390,14 @@ static int allBench(BMK_result_t* resultPtr, /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_iterMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } benchres = benchres2; } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_iterMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } @@ -1419,7 +1420,7 @@ static int allBench(BMK_result_t* resultPtr, /* Final full run if estimates are unclear */ if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_timeMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } @@ -1427,7 +1428,7 @@ static int allBench(BMK_result_t* resultPtr, } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_timeMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } From 486e586eed5d30f6c4ec079bb403d6c15004bb9d Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 13 Aug 2018 16:13:46 -0700 Subject: [PATCH 47/92] Revert "Default lvl 1" This reverts commit 0cc75d6ee02efe9da7901848ce588a7f9f61b4f9. --- tests/paramgrill.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index d7165021d..bcb6bcae3 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -65,7 +65,6 @@ static const int g_maxNbVariations = 64; #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) #define CUSTOM_LEVEL 99 -#define BASE_CLEVEL 1 /* indices for each of the variables */ typedef enum { @@ -303,7 +302,7 @@ BMK_benchParam1(BMK_result_t* resultPtr, const void* srcBuffer, size_t srcSize, const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, BASE_CLEVEL, &cParams, NULL, 0, 0, "File"); + BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File"); *resultPtr = res.result; return res.error; } @@ -793,16 +792,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* too large compression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, (double)testResult.cSpeed / (1 MB), - O_ratio, (double)winners[cLevel].result.cSpeed / (1 MB), cLevel); + W_ratio, (double)testResult.cSpeed / 1000000, + O_ratio, (double)winners[cLevel].result.cSpeed / 1000000., cLevel); continue; } if (W_DSpeed_note < O_DSpeed_note ) { /* too large decompression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, (double)testResult.dSpeed / (1 MB), - O_ratio, (double)winners[cLevel].result.dSpeed / (1 MB), cLevel); + W_ratio, (double)testResult.dSpeed / 1000000., + O_ratio, (double)winners[cLevel].result.dSpeed / 1000000., cLevel); continue; } @@ -1194,7 +1193,7 @@ static void BMK_benchOnce(const void* srcBuffer, size_t srcSize) g_params = ZSTD_adjustCParams(g_params, srcSize, 0); BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params); DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, - (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB)); + (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000); return; } @@ -1212,7 +1211,7 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } if (g_target) { - BMK_init_level_constraints(g_target*(1 MB)); + BMK_init_level_constraints(g_target*1000000); } else { /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); @@ -1257,7 +1256,7 @@ static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize) static int benchSample(void) { const char* const name = "Sample 10MB"; - size_t const benchedSize = (10 MB); + size_t const benchedSize = 10000000; void* origBuff = malloc(benchedSize); if (!origBuff) { perror("not enough memory"); return 12; } @@ -1355,7 +1354,7 @@ static int allBench(BMK_result_t* resultPtr, double winnerRS; /* initial benchmarking, gives exact ratio and memory, warms up future runs */ - benchres = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_iterMode, 1); + benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1); winnerRS = resultScore(*winnerResult, buf.srcSize, target); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); @@ -1390,14 +1389,14 @@ static int allBench(BMK_result_t* resultPtr, /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_iterMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } benchres = benchres2; } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_iterMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } @@ -1420,7 +1419,7 @@ static int allBench(BMK_result_t* resultPtr, /* Final full run if estimates are unclear */ if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_timeMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } @@ -1428,7 +1427,7 @@ static int allBench(BMK_result_t* resultPtr, } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_timeMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } From 0cea75402478ded21d2300544bcb538f4d3e3786 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 13 Aug 2018 16:15:34 -0700 Subject: [PATCH 48/92] Revert "Reorder declaration" This reverts commit 3ac2c22485ab5508f47e3eab642b787af0e68b5f. --- tests/paramgrill.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index bcb6bcae3..e530af735 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -2256,9 +2256,8 @@ int main(int argc, const char** argv) g_params.strategy = (ZSTD_strategy)readU32FromChar(&argument); continue; case 'L': - { int cLevel; - argument++; - cLevel = readU32FromChar(&argument); + { argument++; + int const cLevel = readU32FromChar(&argument); g_params = ZSTD_getCParams(cLevel, g_blockSize, 0); continue; } From 13611249a5d1426e4902fe32b1cf913228f246fa Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 24 Jul 2018 17:26:21 -0700 Subject: [PATCH 49/92] Table Compiling +Euclidean Metric --- tests/paramgrill.c | 186 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 179 insertions(+), 7 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index e530af735..5c654d993 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -279,6 +279,29 @@ static int compareResultLT(const BMK_result_t result1, const BMK_result_t result } +/* calculates normalized euclidean distance of result1 if it is in the first quadrant relative to lvlRes */ +static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRes) { + double normalizedCSpeedGain1 = result1.cSpeed / lvlRes.cSpeed - 1; + double normalizedRatioGain1 = lvlRes.cSize / result1.cSize - 1; + if(normalizedRatioGain1 < 0 || normalizedRatioGain1 < 0) { + return 0.0; + } + return normalizedRatioGain1 * normalizedRatioGain1 + normalizedCSpeedGain1 * normalizedCSpeedGain1; +} + +static int lvlFeasible(const BMK_result_t result, const BMK_result_t lvlRes) { + return lvlRes.cSpeed < result.cSpeed && lvlRes.cSize > result.cSize; +} + +/* redefines feasibility for lvl mode */ +static int compareResultLT2(const BMK_result_t result1, const BMK_result_t result2, const BMK_result_t lvltarget, size_t srcSize) { + constraint_t target = { (U32)lvltarget.cSpeed, 0, (U32)-1 }; + if(lvlFeasible(result1, lvltarget) && lvlFeasible(result2, lvltarget)) { + return resultDistLvl(result1, lvltarget) < resultDistLvl(result2, lvltarget); + } + return lvlFeasible(result2, lvltarget) || (!lvlFeasible(result1, lvltarget) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); +} + /* factor sort of arbitrary */ static constraint_t relaxTarget(constraint_t target) { target.cMem = (U32)-1; @@ -629,6 +652,132 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t } +typedef struct ll_node ll_node; +struct ll_node { + winnerInfo_t res; + ll_node* next; +}; + +static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ +static BMK_result_t g_lvltarget; + +/* comparison function: */ +/* strictly better, strictly worse, equal, speed-side adv, size-side adv */ +//Maybe use compress_only for benchmark first run? +#define WORSE_RESULT 0 +#define BETTER_RESULT 1 +#define ERROR_RESULT 2 + +#define SPEED_RESULT 4 +#define SIZE_RESULT 5 +static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) { + if(r1.cSpeed > r2.cSpeed) { + if(r1.cSize <= r2.cSize) { + return WORSE_RESULT; + } + return SIZE_RESULT; /* r2 is smaller but not faster. */ + } else { + if(r1.cSize >= r2.cSize) { + return BETTER_RESULT; + } + return SPEED_RESULT; /* r2 is faster but not smaller */ + } +} +/* assumes candidate is already strictly better than old winner. */ +/* 0 for success, 1 for no insert */ +/* indicate whether inserted as well? */ +/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */ +static int insertWinner(winnerInfo_t w) { + BMK_result_t r = w.result; + ll_node* cur_node = g_winners; + /* first node to insert */ + if(!lvlFeasible(r, g_lvltarget)) { + return 1; + } + + if(g_winners == NULL) { + ll_node* first_node = malloc(sizeof(ll_node)); + if(first_node == NULL) { + return 1; + } + first_node->next = NULL; + first_node->res = w; + g_winners = first_node; + return 0; + } + + while(cur_node->next != NULL) { + switch(speedSizeCompare(r, cur_node->res.result)) { + case BETTER_RESULT: + { + return 1; /* never insert if better */ + } + case WORSE_RESULT: + { + ll_node* tmp; + cur_node->res = cur_node->next->res; + tmp = cur_node->next; + cur_node->next = cur_node->next->next; + free(tmp); + break; + } + case SPEED_RESULT: + cur_node = cur_node->next; + case SIZE_RESULT: /* insert after first size result, then return */ + { + ll_node* newnode = malloc(sizeof(ll_node)); + if(newnode == NULL) { + return 1; + } + newnode->res = cur_node->res; + cur_node->res = w; + newnode->next = cur_node->next; + cur_node->next = newnode; + return 0; + } + } + + } + + //assert(cur_node->next == NULL) + switch(speedSizeCompare(r, cur_node->res.result)) { + case BETTER_RESULT: + { + return 1; /* never insert if better */ + } + case WORSE_RESULT: + { + cur_node->res = w; + return 0; + } + case SPEED_RESULT: + { + ll_node* newnode = malloc(sizeof(ll_node)); + if(newnode == NULL) { + return 1; + } + newnode->res = w; + newnode->next = NULL; + cur_node->next = newnode; + return 0; + } + case SIZE_RESULT: /* insert before first size result, then return */ + { + ll_node* newnode = malloc(sizeof(ll_node)); + if(newnode == NULL) { + return 1; + } + newnode->res = cur_node->res; + cur_node->res = w; + newnode->next = cur_node->next; + cur_node->next = newnode; + return 0; + } + default: + return 1; + } +} + static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) { char lvlstr[15] = "Custom Level"; @@ -658,6 +807,29 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res /* global winner used for constraints */ static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; + /* print lvl if optmode */ + if(g_lvltarget.cSize != 0) { + winnerInfo_t w; + ll_node* n; + int i; + w.result = result; + w.params = params; + i = insertWinner(w); + if(i) return; + + fprintf(f, "\033c"); + for(n = g_winners; n != NULL; n = n->next) { + DISPLAY("\r%79s\r", ""); + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, + params.targetLength, g_stratName[(U32)(params.strategy)]); + fprintf(f, + " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20)); + } + return; + } + if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); @@ -1334,12 +1506,6 @@ int benchFiles(const char** fileNamesTable, int nbFiles) return 0; } - - -#define WORSE_RESULT 0 -#define BETTER_RESULT 1 -#define ERROR_RESULT 2 - /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ #define VARIANCE 1.1 static int allBench(BMK_result_t* resultPtr, @@ -1939,6 +2105,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } + /* use level'ing mode instead of normal target mode */ if(cLevel) { winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { @@ -1947,6 +2114,11 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } target.cSpeed = (U32)winner.result.cSpeed; + + g_targetConstraints = target; + + g_lvltarget = winner.result; + BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } @@ -2137,8 +2309,8 @@ int main(int argc, const char** argv) U32 main_pause = 0; int optimizerCLevel = 0; - constraint_t target = { 0, 0, (U32)-1 }; + ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; assert(argc>=1); /* for exename */ From 5f4502fc0775f10175e759a7987289a80d9dbd92 Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 24 Jul 2018 17:55:17 -0700 Subject: [PATCH 50/92] New climb feas part 2 uses euclidean metric --- tests/paramgrill.c | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 5c654d993..ffb5c1f7d 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -1518,6 +1518,7 @@ static int allBench(BMK_result_t* resultPtr, U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; double winnerRS; + int lvlmode = g_lvltarget.cSize != 0; /* initial benchmarking, gives exact ratio and memory, warms up future runs */ benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1); @@ -1548,9 +1549,11 @@ static int allBench(BMK_result_t* resultPtr, uncertaintyConstantD = 3; } + if(!lvlmode) { /* anything with worse ratio in feas is definitely worse, discard */ - if(feas && benchres.result.cSize < winnerResult->cSize) { - return WORSE_RESULT; + if(feas && benchres.result.cSize < winnerResult->cSize) { + return WORSE_RESULT; + } } /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ @@ -1578,8 +1581,9 @@ static int allBench(BMK_result_t* resultPtr, /* disregard infeasible results in feas mode */ /* disregard if resultMax < winner in infeas mode */ - if((feas && !feasible(resultMax, target)) || - (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) { + if((feas && (!lvlmode && !feasible(resultMax, target))) || + (!feas && ((!lvlmode && winnerRS > resultScore(resultMax, buf.srcSize, target)) || + (lvlmode && resultDistLvl(*winnerResult, g_lvltarget) > resultDistLvl(resultMax, g_lvltarget))))) { return WORSE_RESULT; } @@ -1604,11 +1608,20 @@ static int allBench(BMK_result_t* resultPtr, /* compare by resultScore when in infeas */ /* compare by compareResultLT when in feas */ - if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || - (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) ) { - return BETTER_RESULT; - } else { - return WORSE_RESULT; + if(!lvlmode) { + if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || + (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) ) { + return BETTER_RESULT; + } else { + return WORSE_RESULT; + } + } else { + if((feas && (compareResultLT2(*winnerResult, benchres.result, g_lvltarget, buf.srcSize))) || + (!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target)))) { + return BETTER_RESULT; + } else { + return WORSE_RESULT; + } } } From f67d040c39af10f68d7b6c61b06e72e1273551ff Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 25 Jul 2018 11:37:20 -0700 Subject: [PATCH 51/92] Bugfixes, style changes Complete euclidean distance climb --- tests/paramgrill.c | 191 +++++++++++++++++++++++++-------------------- 1 file changed, 108 insertions(+), 83 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index ffb5c1f7d..0dbe66534 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -112,6 +112,36 @@ static U32 g_noSeed = 0; static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy }; static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ + +typedef struct { + BMK_result_t result; + ZSTD_compressionParameters params; +} winnerInfo_t; + +/* global winner used for display. */ +//Should be totally 0 initialized? +static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; + +typedef struct { + U32 cSpeed; /* bytes / sec */ + U32 dSpeed; + U32 cMem; /* bytes */ +} constraint_t; + +static constraint_t g_targetConstraints; + +typedef struct ll_node ll_node; +struct ll_node { + winnerInfo_t res; + ll_node* next; +}; + +static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ +static BMK_result_t g_lvltarget; + +/* range 0 - 99 */ +static U32 g_strictness = 99; + void BMK_SetNbIterations(int nbLoops) { g_nbIterations = nbLoops; @@ -197,12 +227,6 @@ static void findClockGranularity(void) { DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity); } -typedef struct { - U32 cSpeed; /* bytes / sec */ - U32 dSpeed; - U32 cMem; /* bytes */ -} constraint_t; - #define CLAMPCHECK(val,min,max) { \ if (val && (((val)<(min)) | ((val)>(max)))) { \ DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \ @@ -246,7 +270,7 @@ static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params) /* checks results are feasible */ static int feasible(const BMK_result_t results, const constraint_t target) { - return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem); + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_lvltarget.cSize || results.cSize <= g_lvltarget.cSize); } /* hill climbing value for part 1 */ @@ -269,44 +293,33 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co return ret; } -/* return true if r2 strictly better than r1 */ -static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) { - if(feasible(result1, target) && feasible(result2, target)) { - return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) - || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed); - } - return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); - -} - /* calculates normalized euclidean distance of result1 if it is in the first quadrant relative to lvlRes */ static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRes) { - double normalizedCSpeedGain1 = result1.cSpeed / lvlRes.cSpeed - 1; - double normalizedRatioGain1 = lvlRes.cSize / result1.cSize - 1; - if(normalizedRatioGain1 < 0 || normalizedRatioGain1 < 0) { + double normalizedCSpeedGain1 = (result1.cSpeed / lvlRes.cSpeed) - 1; + double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1; + if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) { return 0.0; } return normalizedRatioGain1 * normalizedRatioGain1 + normalizedCSpeedGain1 * normalizedCSpeedGain1; } -static int lvlFeasible(const BMK_result_t result, const BMK_result_t lvlRes) { - return lvlRes.cSpeed < result.cSpeed && lvlRes.cSize > result.cSize; -} - -/* redefines feasibility for lvl mode */ -static int compareResultLT2(const BMK_result_t result1, const BMK_result_t result2, const BMK_result_t lvltarget, size_t srcSize) { - constraint_t target = { (U32)lvltarget.cSpeed, 0, (U32)-1 }; - if(lvlFeasible(result1, lvltarget) && lvlFeasible(result2, lvltarget)) { - return resultDistLvl(result1, lvltarget) < resultDistLvl(result2, lvltarget); +/* return true if r2 strictly better than r1 */ +static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) { + if(feasible(result1, target) && feasible(result2, target)) { + if(g_lvltarget.cSize == 0) { + return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) + || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed); + } else { + return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget); + } } - return lvlFeasible(result2, lvltarget) || (!lvlFeasible(result1, lvltarget) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); + return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); } -/* factor sort of arbitrary */ static constraint_t relaxTarget(constraint_t target) { target.cMem = (U32)-1; - target.cSpeed *= 0.9; - target.dSpeed *= 0.9; + target.cSpeed *= ((double)(g_strictness + 1) / 100); + target.dSpeed *= ((double)(g_strictness + 1) / 100); return target; } @@ -330,11 +343,6 @@ BMK_benchParam1(BMK_result_t* resultPtr, return res.error; } -typedef struct { - BMK_result_t result; - ZSTD_compressionParameters params; -} winnerInfo_t; - static ZSTD_compressionParameters emptyParams(void) { ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 }; return p; @@ -651,16 +659,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t return results; } - -typedef struct ll_node ll_node; -struct ll_node { - winnerInfo_t res; - ll_node* next; -}; - -static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ -static BMK_result_t g_lvltarget; - /* comparison function: */ /* strictly better, strictly worse, equal, speed-side adv, size-side adv */ //Maybe use compress_only for benchmark first run? @@ -691,7 +689,7 @@ static int insertWinner(winnerInfo_t w) { BMK_result_t r = w.result; ll_node* cur_node = g_winners; /* first node to insert */ - if(!lvlFeasible(r, g_lvltarget)) { + if(!feasible(r, g_targetConstraints)) { return 1; } @@ -722,7 +720,10 @@ static int insertWinner(winnerInfo_t w) { break; } case SPEED_RESULT: + { cur_node = cur_node->next; + break; + } case SIZE_RESULT: /* insert after first size result, then return */ { ll_node* newnode = malloc(sizeof(ll_node)); @@ -843,6 +844,34 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res g_winner.params = params; } } + + //prints out tradeoff table if using lvl + if(g_lvltarget.cSize != 0) { + winnerInfo_t w; + ll_node* n; + int i; + w.result = result; + w.params = params; + i = insertWinner(w); + if(i) return; + + if(!DEBUG) { fprintf(f, "\033c"); } + fprintf(f, "\n"); + + /* the table */ + fprintf(f, "================================\n"); + for(n = g_winners; n != NULL; n = n->next) { + DISPLAY("\r%79s\r", ""); + + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", + n->res.params.windowLog, n->res.params.chainLog, n->res.params.hashLog, n->res.params.searchLog, n->res.params.searchLength, + n->res.params.targetLength, g_stratName[(U32)(n->res.params.strategy)]); + fprintf(f, + " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + (double)srcSize / n->res.result.cSize, n->res.result.cSpeed / (1 << 20), n->res.result.dSpeed / (1 << 20)); + } + fprintf(f, "================================\n"); + } } static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) @@ -1518,7 +1547,6 @@ static int allBench(BMK_result_t* resultPtr, U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC, uncertaintyConstantD; double winnerRS; - int lvlmode = g_lvltarget.cSize != 0; /* initial benchmarking, gives exact ratio and memory, warms up future runs */ benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1); @@ -1549,11 +1577,9 @@ static int allBench(BMK_result_t* resultPtr, uncertaintyConstantD = 3; } - if(!lvlmode) { /* anything with worse ratio in feas is definitely worse, discard */ - if(feas && benchres.result.cSize < winnerResult->cSize) { - return WORSE_RESULT; - } + if(feas && benchres.result.cSize < winnerResult->cSize && g_lvltarget.cSize == 0) { + return WORSE_RESULT; } /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ @@ -1581,9 +1607,8 @@ static int allBench(BMK_result_t* resultPtr, /* disregard infeasible results in feas mode */ /* disregard if resultMax < winner in infeas mode */ - if((feas && (!lvlmode && !feasible(resultMax, target))) || - (!feas && ((!lvlmode && winnerRS > resultScore(resultMax, buf.srcSize, target)) || - (lvlmode && resultDistLvl(*winnerResult, g_lvltarget) > resultDistLvl(resultMax, g_lvltarget))))) { + if((feas && !feasible(resultMax, target)) || + (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) { return WORSE_RESULT; } @@ -1608,22 +1633,12 @@ static int allBench(BMK_result_t* resultPtr, /* compare by resultScore when in infeas */ /* compare by compareResultLT when in feas */ - if(!lvlmode) { - if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || - (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) ) { - return BETTER_RESULT; - } else { - return WORSE_RESULT; - } - } else { - if((feas && (compareResultLT2(*winnerResult, benchres.result, g_lvltarget, buf.srcSize))) || - (!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target)))) { - return BETTER_RESULT; - } else { - return WORSE_RESULT; - } + if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || + (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) ) { + return BETTER_RESULT; + } else { + return WORSE_RESULT; } - } #define INFEASIBLE_THRESHOLD 200 @@ -1655,6 +1670,7 @@ static int benchMemo(BMK_result_t* resultPtr, return res; } + /* One iteration of hill climbing. Specifically, it first tries all * valid parameter configurations w/ manhattan distance 1 and picks the best one * failing that, it progressively tries candidates further and further away (up to #dim + 2) @@ -1667,6 +1683,10 @@ static int benchMemo(BMK_result_t* resultPtr, * Phase 2 optimizes in accordance with what the original function sets out to maximize, with * all feasible solutions valued over all infeasible solutions. */ + +/* sanitize all params here. + * all generation after random should be sanitized. (maybe sanitize random) + */ static winnerInfo_t climbOnce(const constraint_t target, const varInds_t* varArray, const int varLen, U8* const memoTable, @@ -1800,6 +1820,7 @@ static winnerInfo_t optimizeFixedStrategy( BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); i = 0; } + i++; } return winnerInfo; @@ -2129,8 +2150,10 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ target.cSpeed = (U32)winner.result.cSpeed; g_targetConstraints = target; - - g_lvltarget = winner.result; + + g_lvltarget = winner.result; + g_lvltarget.cSpeed *= ((double)(g_strictness + 1) / 100); + g_lvltarget.cSize /= ((double)(g_strictness + 1) / 100); BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } @@ -2299,15 +2322,16 @@ static int badusage(const char* exename) } #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } } -#define PARSE_CPARAMS(variable) \ -{ \ - PARSE_SUB_ARGS("windowLog=", "wlog=", variable.windowLog); \ - PARSE_SUB_ARGS("chainLog=" , "clog=", variable.chainLog); \ - PARSE_SUB_ARGS("hashLog=", "hlog=", variable.hashLog); \ - PARSE_SUB_ARGS("searchLog=" , "slog=", variable.searchLog); \ - PARSE_SUB_ARGS("searchLength=", "slen=", variable.searchLength); \ - PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.targetLength); \ - PARSE_SUB_ARGS("strategy=", "strat=", variable.strategy); \ +#define PARSE_CPARAMS(variable) \ +{ \ + PARSE_SUB_ARGS("windowLog=", "wlog=", variable.vals[wlog_ind]); \ + PARSE_SUB_ARGS("chainLog=" , "clog=", variable.vals[clog_ind]); \ + PARSE_SUB_ARGS("hashLog=", "hlog=", variable.vals[hlog_ind]); \ + PARSE_SUB_ARGS("searchLog=" , "slog=", variable.vals[slog_ind]); \ + PARSE_SUB_ARGS("searchLength=", "slen=", variable.vals[slen_ind]); \ + PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.vals[tlen_ind]); \ + PARSE_SUB_ARGS("strategy=", "strat=", variable.vals[strt_ind]); \ + PARSE_SUB_ARGS("forceAttachDict=", "fad=" , variable.vals[strt_ind]); \ } int main(int argc, const char** argv) @@ -2350,6 +2374,7 @@ int main(int argc, const char** argv) PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel); + PARSE_SUB_ARGS("strict=", "stc=", g_strictness); DISPLAY("invalid optimization parameter \n"); return 1; } From 2bdfe6ca71d2171c02468214759080bc1d3e7e3a Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 25 Jul 2018 11:55:09 -0700 Subject: [PATCH 52/92] Better Display --- tests/paramgrill.c | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 0dbe66534..fa5a5390a 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -293,7 +293,7 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co return ret; } -/* calculates normalized euclidean distance of result1 if it is in the first quadrant relative to lvlRes */ +/* calculates normalized squared euclidean distance of result1 if it is in the first quadrant relative to lvlRes */ static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRes) { double normalizedCSpeedGain1 = (result1.cSpeed / lvlRes.cSpeed) - 1; double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1; @@ -668,6 +668,7 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t #define SPEED_RESULT 4 #define SIZE_RESULT 5 +/* maybe have epsilon-eq to limit table size? */ static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) { if(r1.cSpeed > r2.cSpeed) { if(r1.cSize <= r2.cSize) { @@ -853,11 +854,11 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res w.result = result; w.params = params; i = insertWinner(w); - if(i) return; + //if(i) return; if(!DEBUG) { fprintf(f, "\033c"); } fprintf(f, "\n"); - + /* the table */ fprintf(f, "================================\n"); for(n = g_winners; n != NULL; n = n->next) { @@ -871,6 +872,27 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res (double)srcSize / n->res.result.cSize, n->res.result.cSpeed / (1 << 20), n->res.result.dSpeed / (1 << 20)); } fprintf(f, "================================\n"); + fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n", + (double)srcSize / g_lvltarget.cSize, g_lvltarget.cSpeed / (1 << 20)); + + + fprintf(f, "Overall Winner: \n"); + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", + g_winner.params.windowLog, g_winner.params.chainLog, g_winner.params.hashLog, g_winner.params.searchLog, g_winner.params.searchLength, + g_winner.params.targetLength, g_stratName[(U32)(g_winner.params.strategy)]); + fprintf(f, + " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + (double)srcSize / g_winner.result.cSize, g_winner.result.cSpeed / (1 << 20), g_winner.result.dSpeed / (1 << 20)); + + + fprintf(f, "Latest BMK: \n"); + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, + params.targetLength, g_stratName[(U32)(params.strategy)]); + fprintf(f, + " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20)); + } } @@ -1707,7 +1729,7 @@ static winnerInfo_t climbOnce(const constraint_t target, { winnerInfo_t bestFeasible1 = initWinnerInfo(cparam); - DISPLAY("Climb Part 1\n"); + DEBUGOUTPUT("Climb Part 1\n"); while(better) { int i, dist, offset; @@ -1774,7 +1796,7 @@ static winnerInfo_t climbOnce(const constraint_t target, feas = 1; better = 1; winnerInfo = bestFeasible1; /* note with change, bestFeasible may not necessarily be feasible, but if one has been benchmarked, it will be. */ - DISPLAY("Climb Part 2\n"); + DEBUGOUTPUT("Climb Part 2\n"); } } winnerInfo = bestFeasible1; @@ -2218,9 +2240,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } while(st && tries) { + DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, st, varArray, varLen, allMT[st], tries); - DEBUGOUTPUT("StratNum %d\n", st); if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; } From 3a2e95eba45a28afa57afae00dac15dce29959e9 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 26 Jul 2018 16:45:00 -0700 Subject: [PATCH 53/92] Perf improvements try decay strategy selection skipping --- tests/paramgrill.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index fa5a5390a..0db8510e1 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -2079,7 +2079,9 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS return base; } +/* experiment with playing with this and decay value */ #define MAX_TRIES 8 +#define TRY_DECAY 3 /* main fn called when using --optimize */ /* Does strategy selection by benchmarking default compression levels * then optimizes by strategy, starting with the best one and moving @@ -2092,6 +2094,7 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS * paramTarget - parameter constraints (i.e. restriction search space to where strategy = ZSTD_fast) * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) */ + static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { varInds_t varArray [NUM_PARAMS]; @@ -2202,7 +2205,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* strategy selection */ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); DEBUGOUTPUT("Strategy Selection\n"); - if(paramTarget.strategy == 0) { /* no variable based constraints */ + if(paramTarget.strategy == 0) { BMK_result_t candidate; int i; for (i=1; i<=maxSeeds; i++) { @@ -2216,6 +2219,11 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ winner.result = candidate; winner.params = CParams; } + + /* if the current params are too slow, just stop. */ + if(target.cSpeed != 0 && target.cSpeed > winner.result.cSpeed / 2) { + break; + } } } } @@ -2248,7 +2256,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } st = nextStrategy(st, bestStrategy); - tries--; + tries -= TRY_DECAY; } } else { winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, From 8ff0de15e4484b656dc199e118b063b3665cb2dd Mon Sep 17 00:00:00 2001 From: George Lu Date: Fri, 27 Jul 2018 08:20:31 -0700 Subject: [PATCH 54/92] Generalize, macro magic numbers --- tests/README.md | 3 +++ tests/paramgrill.c | 54 +++++++++++++++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/tests/README.md b/tests/README.md index 2f0026fda..04eb5094e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -112,6 +112,9 @@ Full list of arguments dSpeed= - Minimum decompression speed cMem= - compression memory lvl= - Automatically sets compression speed constraint to the speed of that level + stc= - In lvl mode, represents slack in ratio/cSpeed allowed for a solution to be considered + - In normal operation, represents slack in strategy selection in choosing the default parameters + --optimize= : same as -O with more verbose syntax -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 0db8510e1..fca6b2002 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -139,8 +139,9 @@ struct ll_node { static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ static BMK_result_t g_lvltarget; -/* range 0 - 99 */ -static U32 g_strictness = 99; +/* range 0 - 99, measure of how strict */ +#define DEFAULT_STRICTNESS 99999 +static U32 g_strictness = DEFAULT_STRICTNESS; void BMK_SetNbIterations(int nbLoops) { @@ -318,8 +319,8 @@ static int compareResultLT(const BMK_result_t result1, const BMK_result_t result static constraint_t relaxTarget(constraint_t target) { target.cMem = (U32)-1; - target.cSpeed *= ((double)(g_strictness + 1) / 100); - target.dSpeed *= ((double)(g_strictness + 1) / 100); + target.cSpeed *= ((double)(g_strictness) / 100); + target.dSpeed *= ((double)(g_strictness) / 100); return target; } @@ -2080,8 +2081,7 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS } /* experiment with playing with this and decay value */ -#define MAX_TRIES 8 -#define TRY_DECAY 3 + /* main fn called when using --optimize */ /* Does strategy selection by benchmarking default compression levels * then optimizes by strategy, starting with the best one and moving @@ -2095,6 +2095,9 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) */ +#define MAX_TRIES 3 +#define TRY_DECAY 1 + static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) { varInds_t varArray [NUM_PARAMS]; @@ -2164,6 +2167,21 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } + /* default strictness = Maximum for */ + if(g_strictness == DEFAULT_STRICTNESS) { + if(cLevel) { + g_strictness = 99; + } else { + g_strictness = 90; + } + } else { + if(0 >= g_strictness || g_strictness > 100) { + DISPLAY("Strictness Outside of Bounds\n"); + ret = 4; + goto _cleanUp; + } + } + /* use level'ing mode instead of normal target mode */ if(cLevel) { winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); @@ -2177,8 +2195,8 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ g_targetConstraints = target; g_lvltarget = winner.result; - g_lvltarget.cSpeed *= ((double)(g_strictness + 1) / 100); - g_lvltarget.cSize /= ((double)(g_strictness + 1) / 100); + g_lvltarget.cSpeed *= ((double)(g_strictness) / 100); + g_lvltarget.cSize /= ((double)(g_strictness) / 100); BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } @@ -2199,6 +2217,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ { varInds_t varNew[NUM_PARAMS]; + ZSTD_compressionParameters CParams; /* find best solution from default params */ { @@ -2210,8 +2229,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int i; for (i=1; i<=maxSeeds; i++) { int ec; - ZSTD_compressionParameters CParams = ZSTD_getCParams(i, maxBlockSize, ctx.dictSize); - CParams = maskParams(CParams, paramTarget); + CParams = maskParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget); ec = BMK_benchParam(&candidate, buf, ctx, CParams); BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize); @@ -2221,9 +2239,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* if the current params are too slow, just stop. */ - if(target.cSpeed != 0 && target.cSpeed > winner.result.cSpeed / 2) { - break; - } + if(target.cSpeed > candidate.cSpeed * 2) { break; } } } } @@ -2239,6 +2255,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int tries = MAX_TRIES; { + /* one iterations of hill climbing with the level-defined parameters. */ int varLenNew = sanitizeVarArray(varNew, varLen, varArray, st); winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], buf, ctx, winner.params); @@ -2247,16 +2264,19 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } } - while(st && tries) { + while(st && tries > 0) { DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, st, varArray, varLen, allMT[st], tries); + if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; + tries = MAX_TRIES; + bestStrategy = st; + } else { + st = nextStrategy(st, bestStrategy); + tries -= TRY_DECAY; } - - st = nextStrategy(st, bestStrategy); - tries -= TRY_DECAY; } } else { winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, From b3544217b77bcff66b7bf754a4f7b745a7d0493a Mon Sep 17 00:00:00 2001 From: George Lu Date: Fri, 27 Jul 2018 11:47:14 -0700 Subject: [PATCH 55/92] Cleanup --- tests/README.md | 6 ++++-- tests/paramgrill.c | 30 ++++++++++++++++++++---------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/tests/README.md b/tests/README.md index 04eb5094e..1410ca979 100644 --- a/tests/README.md +++ b/tests/README.md @@ -111,9 +111,11 @@ Full list of arguments cSpeed= - Minimum compression speed dSpeed= - Minimum decompression speed cMem= - compression memory - lvl= - Automatically sets compression speed constraint to the speed of that level - stc= - In lvl mode, represents slack in ratio/cSpeed allowed for a solution to be considered + lvl= - Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, + stc= - When invoked with lvl=, represents slack in ratio/cSpeed allowed for a solution to be considered - In normal operation, represents slack in strategy selection in choosing the default parameters + prefer[Speed/Ratio]= - Only affects lvl= invocations. Defines value placed on compression speed or ratio + when determining overall winner (default 1 for both). --optimize= : same as -O with more verbose syntax -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index fca6b2002..188b04935 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -123,7 +123,7 @@ typedef struct { static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; typedef struct { - U32 cSpeed; /* bytes / sec */ + U32 cSpeed; /* bytes / sec */ U32 dSpeed; U32 cMem; /* bytes */ } constraint_t; @@ -138,6 +138,12 @@ struct ll_node { static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ static BMK_result_t g_lvltarget; +static int g_optmode = 0; + +static U32 g_speedMultiplier = 1; +static U32 g_ratioMultiplier = 1; + +/* g_mode? */ /* range 0 - 99, measure of how strict */ #define DEFAULT_STRICTNESS 99999 @@ -271,7 +277,7 @@ static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params) /* checks results are feasible */ static int feasible(const BMK_result_t results, const constraint_t target) { - return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_lvltarget.cSize || results.cSize <= g_lvltarget.cSize); + return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_optmode || results.cSize <= g_lvltarget.cSize); } /* hill climbing value for part 1 */ @@ -291,6 +297,7 @@ static double resultScore(const BMK_result_t res, const size_t srcSize, const co ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; + return ret; } @@ -301,17 +308,17 @@ static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRe if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) { return 0.0; } - return normalizedRatioGain1 * normalizedRatioGain1 + normalizedCSpeedGain1 * normalizedCSpeedGain1; + return normalizedRatioGain1 * g_ratioMultiplier + normalizedCSpeedGain1 * g_speedMultiplier; } /* return true if r2 strictly better than r1 */ static int compareResultLT(const BMK_result_t result1, const BMK_result_t result2, const constraint_t target, size_t srcSize) { if(feasible(result1, target) && feasible(result2, target)) { - if(g_lvltarget.cSize == 0) { + if(g_optmode) { + return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget); + } else { return (result1.cSize > result2.cSize) || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed); - } else { - return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget); } } return feasible(result2, target) || (!feasible(result1, target) && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); @@ -848,7 +855,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res } //prints out tradeoff table if using lvl - if(g_lvltarget.cSize != 0) { + if(g_optmode) { winnerInfo_t w; ll_node* n; int i; @@ -1601,7 +1608,7 @@ static int allBench(BMK_result_t* resultPtr, } /* anything with worse ratio in feas is definitely worse, discard */ - if(feas && benchres.result.cSize < winnerResult->cSize && g_lvltarget.cSize == 0) { + if(feas && benchres.result.cSize < winnerResult->cSize && !g_optmode) { return WORSE_RESULT; } @@ -2169,7 +2176,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* default strictness = Maximum for */ if(g_strictness == DEFAULT_STRICTNESS) { - if(cLevel) { + if(g_optmode) { g_strictness = 99; } else { g_strictness = 90; @@ -2183,7 +2190,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* use level'ing mode instead of normal target mode */ - if(cLevel) { + if(g_optmode) { winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { ret = 3; @@ -2425,6 +2432,9 @@ int main(int argc, const char** argv) PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel); PARSE_SUB_ARGS("strict=", "stc=", g_strictness); + PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier); + PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier); + DISPLAY("invalid optimization parameter \n"); return 1; } From a884b76bc2fedd086c36b16e52b0dc43d075ca28 Mon Sep 17 00:00:00 2001 From: George Lu Date: Fri, 27 Jul 2018 14:19:55 -0700 Subject: [PATCH 56/92] Style Changes Add single run dictionaries Change MB to be consistent 1 << 20 rather than 1,000,000 --- tests/paramgrill.c | 481 ++++++++++++++++++++++----------------------- 1 file changed, 238 insertions(+), 243 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 188b04935..8df72eb01 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -107,6 +107,7 @@ static double g_compressibility = COMPRESSIBILITY_DEFAULT; static U32 g_blockSize = 0; static U32 g_rand = 1; static U32 g_singleRun = 0; +static U32 g_optimizer = 0; static U32 g_target = 0; static U32 g_noSeed = 0; static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy }; @@ -118,10 +119,6 @@ typedef struct { ZSTD_compressionParameters params; } winnerInfo_t; -/* global winner used for display. */ -//Should be totally 0 initialized? -static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; - typedef struct { U32 cSpeed; /* bytes / sec */ U32 dSpeed; @@ -386,8 +383,7 @@ typedef struct { ZSTD_DCtx* dctx; } contexts_t; -static int -BMK_benchParam(BMK_result_t* resultPtr, +static int BMK_benchParam(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams) { BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, 0, "Files"); @@ -520,6 +516,175 @@ static size_t local_defaultDecompress( * From Paramgrill End *********************************************************/ +static void freeBuffers(const buffers_t b) { + if(b.srcPtrs != NULL) { + free(b.srcBuffer); + } + free(b.srcPtrs); + free(b.srcSizes); + + if(b.dstPtrs != NULL) { + free(b.dstPtrs[0]); + } + free(b.dstPtrs); + free(b.dstCapacities); + free(b.dstSizes); + + if(b.resPtrs != NULL) { + free(b.resPtrs[0]); + } + free(b.resPtrs); +} + +/* allocates buffer's arguments. returns success / failuere */ +static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, + const size_t nbFiles) +{ + size_t pos = 0; + size_t n; + U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); + size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); + const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; //(largest fileSize or total fileSize) + U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles; + U32 blockNb = 0; + + buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); + buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) { + DISPLAY("alloc error\n"); + freeBuffers(*buff); + return 1; + } + + buff->srcBuffer = malloc(benchedSize); + buff->srcPtrs[0] = (const void*)buff->srcBuffer; + buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024)); + buff->resPtrs[0] = malloc(benchedSize); + + if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) { + DISPLAY("alloc error\n"); + freeBuffers(*buff); + return 1; + } + + for(n = 0; n < nbFiles; n++) { + FILE* f; + U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); + if (UTIL_isDirectory(fileNamesTable[n])) { + DISPLAY("Ignoring %s directory... \n", fileNamesTable[n]); + continue; + } + if (fileSize == UTIL_FILESIZE_UNKNOWN) { + DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]); + continue; + } + f = fopen(fileNamesTable[n], "rb"); + if (f==NULL) { + DISPLAY("impossible to open file %s\n", fileNamesTable[n]); + freeBuffers(*buff); + fclose(f); + return 10; + } + + DISPLAY("Loading %s... \r", fileNamesTable[n]); + + if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n=nbFiles; /* buffer too small - stop after this file */ + { + char* buffer = (char*)(buff->srcBuffer); + size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); + size_t blocked = 0; + while(blocked < readSize) { + buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked)); + buff->srcSizes[blockNb] = blockSize; + blocked += blockSize; + blockNb++; + } + if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; } + + if (readSize != (size_t)fileSize) { + DISPLAY("could not read %s", fileNamesTable[n]); + freeBuffers(*buff); + fclose(f); + return 1; + } + + pos += readSize; + + } + fclose(f); + } + + buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); + buff->dstSizes[0] = buff->dstCapacities[0]; + buff->resSizes[0] = buff->srcSizes[0]; + + for(n = 1; n < blockNb; n++) { + buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1]; + buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1]; + buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]); + buff->dstSizes[n] = buff->dstCapacities[n]; + buff->resSizes[n] = buff->srcSizes[n]; + } + buff->srcSize = pos; + buff->nbBlocks = blockNb; + + if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; } + + return 0; +} + +static void freeContexts(const contexts_t ctx) { + free(ctx.dictBuffer); + ZSTD_freeCCtx(ctx.cctx); + ZSTD_freeDCtx(ctx.dctx); +} + +static int createContexts(contexts_t* ctx, const char* dictFileName) { + FILE* f; + size_t readSize; + ctx->cctx = ZSTD_createCCtx(); + ctx->dctx = ZSTD_createDCtx(); + if(dictFileName == NULL) { + ctx->dictSize = 0; + ctx->dictBuffer = NULL; + return 0; + } + ctx->dictSize = UTIL_getFileSize(dictFileName); + ctx->dictBuffer = malloc(ctx->dictSize); + + f = fopen(dictFileName, "rb"); + + if(!f) { + DISPLAY("unable to open file\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } + + if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) { + DISPLAY("dictionary too large\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } + readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f); + if(readSize != ctx->dictSize) { + DISPLAY("unable to read file\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } + return 0; +} + /* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */ /* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */ /* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ @@ -788,6 +953,8 @@ static int insertWinner(winnerInfo_t w) { } } +/* Writes to f the results of a parameter benchmark */ +/* when used with --optimize, will only print results better than previously discovered */ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) { char lvlstr[15] = "Custom Level"; @@ -806,7 +973,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result fprintf(f, "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", - lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 << 20), (double)result.dSpeed / (1 << 20)); + lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } fprintf(f, "\n"); @@ -816,29 +983,6 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res { /* global winner used for constraints */ static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; - - /* print lvl if optmode */ - if(g_lvltarget.cSize != 0) { - winnerInfo_t w; - ll_node* n; - int i; - w.result = result; - w.params = params; - i = insertWinner(w); - if(i) return; - - fprintf(f, "\033c"); - for(n = g_winners; n != NULL; n = n->next) { - DISPLAY("\r%79s\r", ""); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, - params.targetLength, g_stratName[(U32)(params.strategy)]); - fprintf(f, - " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20)); - } - return; - } if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { @@ -855,14 +999,12 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res } //prints out tradeoff table if using lvl - if(g_optmode) { + if(g_optmode && g_optimizer) { winnerInfo_t w; ll_node* n; - int i; w.result = result; w.params = params; - i = insertWinner(w); - //if(i) return; + insertWinner(w); if(!DEBUG) { fprintf(f, "\033c"); } fprintf(f, "\n"); @@ -877,11 +1019,11 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res n->res.params.targetLength, g_stratName[(U32)(n->res.params.strategy)]); fprintf(f, " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / n->res.result.cSize, n->res.result.cSpeed / (1 << 20), n->res.result.dSpeed / (1 << 20)); + (double)srcSize / n->res.result.cSize, (double)n->res.result.cSpeed / (1 MB), (double)n->res.result.dSpeed / (1 MB)); } fprintf(f, "================================\n"); fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n", - (double)srcSize / g_lvltarget.cSize, g_lvltarget.cSpeed / (1 << 20)); + (double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / (1 MB)); fprintf(f, "Overall Winner: \n"); @@ -890,8 +1032,9 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res g_winner.params.targetLength, g_stratName[(U32)(g_winner.params.strategy)]); fprintf(f, " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / g_winner.result.cSize, g_winner.result.cSpeed / (1 << 20), g_winner.result.dSpeed / (1 << 20)); + (double)srcSize / g_winner.result.cSize, (double)g_winner.result.cSpeed / (1 MB), (double)g_winner.result.dSpeed / (1 MB)); + BMK_translateAdvancedParams(g_winner.params); fprintf(f, "Latest BMK: \n"); fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", @@ -899,7 +1042,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res params.targetLength, g_stratName[(U32)(params.strategy)]); fprintf(f, " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / result.cSize, result.cSpeed / (1 << 20), result.dSpeed / (1 << 20)); + (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); } } @@ -1023,16 +1166,16 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* too large compression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, (double)testResult.cSpeed / 1000000, - O_ratio, (double)winners[cLevel].result.cSpeed / 1000000., cLevel); + W_ratio, (double)testResult.cSpeed / (1 MB), + O_ratio, (double)winners[cLevel].result.cSpeed / (1 MB), cLevel); continue; } if (W_DSpeed_note < O_DSpeed_note ) { /* too large decompression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", - W_ratio, (double)testResult.dSpeed / 1000000., - O_ratio, (double)winners[cLevel].result.dSpeed / 1000000., cLevel); + W_ratio, (double)testResult.dSpeed / (1 MB), + O_ratio, (double)winners[cLevel].result.dSpeed / (1 MB), cLevel); continue; } @@ -1417,7 +1560,6 @@ static void BMK_selectRandomStart( } } - static void BMK_benchOnce(const void* srcBuffer, size_t srcSize) { BMK_result_t testResult; @@ -1442,7 +1584,7 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } if (g_target) { - BMK_init_level_constraints(g_target*1000000); + BMK_init_level_constraints(g_target * (1 MB)); } else { /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); @@ -1487,7 +1629,7 @@ static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize) static int benchSample(void) { const char* const name = "Sample 10MB"; - size_t const benchedSize = 10000000; + size_t const benchedSize = 10 MB; void* origBuff = malloc(benchedSize); if (!origBuff) { perror("not enough memory"); return 12; } @@ -1505,13 +1647,56 @@ static int benchSample(void) } +static int benchOnce(const char** fileNamesTable, int nbFiles, const char* dictFileName) { + buffers_t buf; + contexts_t ctx; + BMK_result_t testResult; + size_t maxBlockSize = 0, i; + + if(createBuffers(&buf, fileNamesTable, nbFiles)) { + DISPLAY("unable to load files\n"); + return 1; + } + + if(createContexts(&ctx, dictFileName)) { + DISPLAY("unable to load dictionary\n"); + freeBuffers(buf); + return 2; + } + + for(i = 0; i < buf.nbBlocks; i++) { + maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]); + } + + g_params = ZSTD_adjustCParams(g_params, maxBlockSize, 0); + + if(BMK_benchParam(&testResult, buf, ctx, g_params)) { + DISPLAY("Error during benchmarking\n"); + freeBuffers(buf); + freeContexts(ctx); + return 3; + } + + DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, + (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB)); + + freeBuffers(buf); + freeContexts(ctx); + return 0; +} + /* benchFiles() : * note: while this function takes a table of filenames, * in practice, only the first filename will be used */ -int benchFiles(const char** fileNamesTable, int nbFiles) +//TODO: dictionaries still not supported in fullTable mode +int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName) { int fileIdx=0; + if(g_singleRun) { + return benchOnce(fileNamesTable, nbFiles, dictFileName); + } + /* Loop for each file */ while (fileIdxsrcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); - buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - - buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); - buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - - buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); - buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - - if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) { - DISPLAY("alloc error\n"); - freeBuffers(*buff); - return 1; - } - - buff->srcBuffer = malloc(benchedSize); - buff->srcPtrs[0] = (const void*)buff->srcBuffer; - buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024)); - buff->resPtrs[0] = malloc(benchedSize); - - if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) { - DISPLAY("alloc error\n"); - freeBuffers(*buff); - return 1; - } - - for(n = 0; n < nbFiles; n++) { - FILE* f; - U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); - if (UTIL_isDirectory(fileNamesTable[n])) { - DISPLAY("Ignoring %s directory... \n", fileNamesTable[n]); - continue; - } - if (fileSize == UTIL_FILESIZE_UNKNOWN) { - DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]); - continue; - } - f = fopen(fileNamesTable[n], "rb"); - if (f==NULL) { - DISPLAY("impossible to open file %s\n", fileNamesTable[n]); - freeBuffers(*buff); - fclose(f); - return 10; - } - - DISPLAY("Loading %s... \r", fileNamesTable[n]); - - if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n = nbFiles; /* buffer too small - stop after this file */ - { - char* buffer = (char*)(buff->srcBuffer); - size_t const readSize = fread(((buffer)+pos), 1, (size_t)fileSize, f); - size_t blocked = 0; - while(blocked < readSize) { - buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked)); - buff->srcSizes[blockNb] = blockSize; - blocked += blockSize; - blockNb++; - } - if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; } - - if (readSize != (size_t)fileSize) { - DISPLAY("could not read %s", fileNamesTable[n]); - freeBuffers(*buff); - fclose(f); - return 1; - } - - pos += readSize; - - } - fclose(f); - } - - buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); - buff->dstSizes[0] = buff->dstCapacities[0]; - buff->resSizes[0] = buff->srcSizes[0]; - - for(n = 1; n < blockNb; n++) { - buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1]; - buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1]; - buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]); - buff->dstSizes[n] = buff->dstCapacities[n]; - buff->resSizes[n] = buff->srcSizes[n]; - } - buff->srcSize = pos; - buff->nbBlocks = blockNb; - - if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; } - - return 0; -} - -static void freeContexts(const contexts_t ctx) { - free(ctx.dictBuffer); - ZSTD_freeCCtx(ctx.cctx); - ZSTD_freeDCtx(ctx.dctx); -} - -/* Creates struct holding contexts and dictionary buffers. returns 0 on success, 1 on failure. */ -static int createContexts(contexts_t* const ctx, const char* dictFileName) { - FILE* f; - size_t readSize; - U64 dictSize; - ctx->cctx = ZSTD_createCCtx(); - ctx->dctx = ZSTD_createDCtx(); - ctx->dictSize = 0; - ctx->dictBuffer = NULL; - - if(!ctx->cctx || !ctx->dctx) { - DISPLAY("context allocation error\n"); - freeContexts(*ctx); - return 1; - } - - if(dictFileName == NULL) { - return 0; - } - - dictSize = UTIL_getFileSize(dictFileName); - - if(dictSize == UTIL_FILESIZE_UNKNOWN) { - DISPLAY("Unable to get dictionary size\n"); - freeContexts(*ctx); - return 1; - } else { - ctx->dictSize = (size_t)dictSize; - } - - ctx->dictBuffer = malloc(ctx->dictSize); - - f = fopen(dictFileName, "rb"); - - if(!f) { - DISPLAY("unable to open file\n"); - fclose(f); - freeContexts(*ctx); - return 1; - } - - if(ctx->dictSize > 64 MB || !(ctx->dictBuffer)) { - DISPLAY("dictionary too large\n"); - fclose(f); - freeContexts(*ctx); - return 1; - } - readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f); - if(readSize != ctx->dictSize) { - DISPLAY("unable to read file\n"); - fclose(f); - freeContexts(*ctx); - return 1; - } - return 0; -} - /* goes best, best-1, best+1, best-2, ... */ /* return 0 if nothing remaining */ static int nextStrategy(const int currentStrategy, const int bestStrategy) { @@ -2102,7 +2097,7 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) */ -#define MAX_TRIES 3 +#define MAX_TRIES 5 #define TRY_DECAY 1 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) @@ -2272,8 +2267,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } while(st && tries > 0) { + winnerInfo_t wc; DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); - winnerInfo_t wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, + wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, st, varArray, varLen, allMT[st], tries); if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { @@ -2399,7 +2395,6 @@ int main(int argc, const char** argv) const char* exename=argv[0]; const char* input_filename = NULL; const char* dictFileName = NULL; - U32 optimizer = 0; U32 main_pause = 0; int optimizerCLevel = 0; @@ -2424,7 +2419,7 @@ int main(int argc, const char** argv) if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; } if (longCommandWArg(&argument, "--optimize=")) { - optimizer = 1; + g_optimizer = 1; for ( ; ;) { PARSE_CPARAMS(paramTarget); PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed); @@ -2580,17 +2575,17 @@ int main(int argc, const char** argv) if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } } if (filenamesStart==0) { - if (optimizer) { + if (g_optimizer) { DISPLAY("Optimizer Expects File\n"); return 1; } else { result = benchSample(); } } else { - if (optimizer) { + if (g_optimizer) { result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, optimizerCLevel); } else { - result = benchFiles(argv+filenamesStart, argc-filenamesStart); + result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName); } } if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; } From 43b4971ca8910ccedebb6c23a6daa89069df6a17 Mon Sep 17 00:00:00 2001 From: George Lu Date: Fri, 27 Jul 2018 16:49:33 -0700 Subject: [PATCH 57/92] Renames, Documentation Updates --- tests/README.md | 22 +++++++----- tests/paramgrill.c | 84 +++++++++++++++++++++++++--------------------- 2 files changed, 58 insertions(+), 48 deletions(-) diff --git a/tests/README.md b/tests/README.md index 1410ca979..946f890b1 100644 --- a/tests/README.md +++ b/tests/README.md @@ -107,15 +107,19 @@ Full list of arguments L# - level --zstd= : Single run, parameter selection syntax same as zstdcli --optimize= : find parameters to maximize compression ratio given parameters - Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints - cSpeed= - Minimum compression speed - dSpeed= - Minimum decompression speed - cMem= - compression memory - lvl= - Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, - stc= - When invoked with lvl=, represents slack in ratio/cSpeed allowed for a solution to be considered - - In normal operation, represents slack in strategy selection in choosing the default parameters - prefer[Speed/Ratio]= - Only affects lvl= invocations. Defines value placed on compression speed or ratio - when determining overall winner (default 1 for both). + Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints + cSpeed= : Minimum compression speed + dSpeed= : Minimum decompression speed + cMem= : Maximum compression memory + lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, + stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 99%) + : In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters + (Lower value will begin with stronger strategies) (Default 90%) + preferSpeed= / preferRatio= + : Only affects lvl = invocations. Defines value placed on compression speed or ratio + when determining overall winner (default 1 for both, higher = more valued). + tries= : Maximum number of random restarts on a single strategy before switching (Default 5) + Higher values will make optimizer run longer, more chances to find better solution. --optimize= : same as -O with more verbose syntax -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 8df72eb01..7e86c2e9e 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -125,15 +125,13 @@ typedef struct { U32 cMem; /* bytes */ } constraint_t; -static constraint_t g_targetConstraints; - -typedef struct ll_node ll_node; -struct ll_node { +typedef struct winner_ll_node winner_ll_node; +struct winner_ll_node { winnerInfo_t res; - ll_node* next; + winner_ll_node* next; }; -static ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ +static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ static BMK_result_t g_lvltarget; static int g_optmode = 0; @@ -323,8 +321,8 @@ static int compareResultLT(const BMK_result_t result1, const BMK_result_t result static constraint_t relaxTarget(constraint_t target) { target.cMem = (U32)-1; - target.cSpeed *= ((double)(g_strictness) / 100); - target.dSpeed *= ((double)(g_strictness) / 100); + target.cSpeed *= ((double)g_strictness) / 100; + target.dSpeed *= ((double)g_strictness) / 100; return target; } @@ -855,20 +853,19 @@ static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) { return SPEED_RESULT; /* r2 is faster but not smaller */ } } -/* assumes candidate is already strictly better than old winner. */ -/* 0 for success, 1 for no insert */ -/* indicate whether inserted as well? */ + +/* 0 for insertion, 1 for no insert */ /* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */ -static int insertWinner(winnerInfo_t w) { +static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { BMK_result_t r = w.result; - ll_node* cur_node = g_winners; + winner_ll_node* cur_node = g_winners; /* first node to insert */ - if(!feasible(r, g_targetConstraints)) { + if(!feasible(r, targetConstraints)) { return 1; } if(g_winners == NULL) { - ll_node* first_node = malloc(sizeof(ll_node)); + winner_ll_node* first_node = malloc(sizeof(winner_ll_node)); if(first_node == NULL) { return 1; } @@ -886,7 +883,7 @@ static int insertWinner(winnerInfo_t w) { } case WORSE_RESULT: { - ll_node* tmp; + winner_ll_node* tmp; cur_node->res = cur_node->next->res; tmp = cur_node->next; cur_node->next = cur_node->next->next; @@ -900,7 +897,7 @@ static int insertWinner(winnerInfo_t w) { } case SIZE_RESULT: /* insert after first size result, then return */ { - ll_node* newnode = malloc(sizeof(ll_node)); + winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); if(newnode == NULL) { return 1; } @@ -927,7 +924,7 @@ static int insertWinner(winnerInfo_t w) { } case SPEED_RESULT: { - ll_node* newnode = malloc(sizeof(ll_node)); + winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); if(newnode == NULL) { return 1; } @@ -938,7 +935,7 @@ static int insertWinner(winnerInfo_t w) { } case SIZE_RESULT: /* insert before first size result, then return */ { - ll_node* newnode = malloc(sizeof(ll_node)); + winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); if(newnode == NULL) { return 1; } @@ -961,7 +958,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result const U64 time = UTIL_clockSpanNano(g_time); const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); - DISPLAY("\r%79s\r", ""); + fprintf(f, "\r%79s\r", ""); fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, @@ -1001,10 +998,10 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res //prints out tradeoff table if using lvl if(g_optmode && g_optimizer) { winnerInfo_t w; - ll_node* n; + winner_ll_node* n; w.result = result; w.params = params; - insertWinner(w); + insertWinner(w, targetConstraints); if(!DEBUG) { fprintf(f, "\033c"); } fprintf(f, "\n"); @@ -1012,7 +1009,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res /* the table */ fprintf(f, "================================\n"); for(n = g_winners; n != NULL; n = n->next) { - DISPLAY("\r%79s\r", ""); + fprintf(f, "\r%79s\r", ""); fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", n->res.params.windowLog, n->res.params.chainLog, n->res.params.hashLog, n->res.params.searchLog, n->res.params.searchLength, @@ -1045,6 +1042,12 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); } + +#if 0 + if(BMK_timeSpan(g_time) > g_grillDuration_s) { + exit(0); + } +#endif } static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) @@ -2097,7 +2100,7 @@ static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZS * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) */ -#define MAX_TRIES 5 +static int g_maxTries = 5; #define TRY_DECAY 1 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) @@ -2185,20 +2188,21 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* use level'ing mode instead of normal target mode */ + /* Should lvl be parameter-masked here? */ if(g_optmode) { winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { ret = 3; goto _cleanUp; } - - target.cSpeed = (U32)winner.result.cSpeed; - - g_targetConstraints = target; - + g_lvltarget = winner.result; - g_lvltarget.cSpeed *= ((double)(g_strictness) / 100); - g_lvltarget.cSize /= ((double)(g_strictness) / 100); + g_lvltarget.cSpeed *= ((double)g_strictness) / 100; + g_lvltarget.dSpeed *= ((double)g_strictness) / 100; + g_lvltarget.cSize /= ((double)g_strictness) / 100; + + target.cSpeed = (U32)g_lvltarget.cSpeed; + target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is worth BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } @@ -2241,7 +2245,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* if the current params are too slow, just stop. */ - if(target.cSpeed > candidate.cSpeed * 2) { break; } + if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; } } } } @@ -2254,7 +2258,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int bestStrategy = (int)winner.params.strategy; if(paramTarget.strategy == 0) { int st = (int)winner.params.strategy; - int tries = MAX_TRIES; + int tries = g_maxTries; { /* one iterations of hill climbing with the level-defined parameters. */ @@ -2269,12 +2273,13 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ while(st && tries > 0) { winnerInfo_t wc; DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); + wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, st, varArray, varLen, allMT[st], tries); if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; - tries = MAX_TRIES; + tries = g_maxTries; bestStrategy = st; } else { st = nextStrategy(st, bestStrategy); @@ -2283,7 +2288,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } } else { winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, - varArray, varLen, allMT[paramTarget.strategy], 10); + varArray, varLen, allMT[paramTarget.strategy], g_maxTries); } } @@ -2402,17 +2407,17 @@ int main(int argc, const char** argv) ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; - assert(argc>=1); /* for exename */ - g_time = UTIL_getTime(); + assert(argc>=1); /* for exename */ + /* Welcome message */ DISPLAY(WELCOME_MESSAGE); for(i=1; i Date: Mon, 30 Jul 2018 17:42:46 -0700 Subject: [PATCH 58/92] Update fulltable to use same interface Add seperateFiles flag --- tests/README.md | 1 + tests/paramgrill.c | 389 ++++++++++++++++++++++++--------------------- 2 files changed, 207 insertions(+), 183 deletions(-) diff --git a/tests/README.md b/tests/README.md index 946f890b1..736916936 100644 --- a/tests/README.md +++ b/tests/README.md @@ -125,6 +125,7 @@ Full list of arguments -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output -D : Next argument dictionary file + -s : Benchmark all files separately ``` Any inputs afterwards are treated as files to benchmark. diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 7e86c2e9e..cba97ebbb 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -96,7 +96,7 @@ typedef enum { static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; /*-************************************ -* Benchmark Parameters +* Benchmark Parameters/Global Variables **************************************/ typedef BYTE U8; @@ -110,7 +110,7 @@ static U32 g_singleRun = 0; static U32 g_optimizer = 0; static U32 g_target = 0; static U32 g_noSeed = 0; -static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy }; +static ZSTD_compressionParameters g_params; static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ @@ -150,6 +150,14 @@ void BMK_SetNbIterations(int nbLoops) DISPLAY("- %u iterations -\n", g_nbIterations); } +/* + * Additional Global Variables (Defined Above Use) + * g_stratName + * g_level_constraint + * g_alreadyTested + * g_maxTries + */ + /*-******************************************************* * Private functions *********************************************************/ @@ -335,17 +343,6 @@ const char* g_stratName[ZSTD_btultra+1] = { "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; -/* benchParam but only takes in one input buffer. */ -static int -BMK_benchParam1(BMK_result_t* resultPtr, - const void* srcBuffer, size_t srcSize, - const ZSTD_compressionParameters cParams) { - - BMK_return_t res = BMK_benchMem(srcBuffer,srcSize, &srcSize, 1, 0, &cParams, NULL, 0, 0, "File"); - *resultPtr = res.result; - return res.error; -} - static ZSTD_compressionParameters emptyParams(void) { ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 }; return p; @@ -381,14 +378,6 @@ typedef struct { ZSTD_DCtx* dctx; } contexts_t; -static int BMK_benchParam(BMK_result_t* resultPtr, - const buffers_t buf, const contexts_t ctx, - const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMem(buf.srcPtrs[0], buf.srcSize, buf.srcSizes, (unsigned)buf.nbBlocks, 0, &cParams, ctx.dictBuffer, ctx.dictSize, 0, "Files"); - *resultPtr = res.result; - return res.error; -} - /*-******************************************************* * From Paramgrill *********************************************************/ @@ -542,7 +531,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab size_t n; U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); - const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; //(largest fileSize or total fileSize) + const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles; U32 blockNb = 0; @@ -830,6 +819,14 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t return results; } +static int BMK_benchParam(BMK_result_t* resultPtr, + buffers_t buf, contexts_t ctx, + const ZSTD_compressionParameters cParams) { + BMK_return_t res = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_timeMode, 3); + *resultPtr = res.result; + return res.error; +} + /* comparison function: */ /* strictly better, strictly worse, equal, speed-side adv, size-side adv */ //Maybe use compress_only for benchmark first run? @@ -911,7 +908,7 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { } - //assert(cur_node->next == NULL) + assert(cur_node->next == NULL); switch(speedSizeCompare(r, cur_node->res.result)) { case BETTER_RESULT: { @@ -995,7 +992,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res } } - //prints out tradeoff table if using lvl + //prints out tradeoff table if using lvloptimize if(g_optmode && g_optimizer) { winnerInfo_t w; winner_ll_node* n; @@ -1042,12 +1039,6 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); } - -#if 0 - if(BMK_timeSpan(g_time) > g_grillDuration_s) { - exit(0); - } -#endif } static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) @@ -1099,14 +1090,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1) } } } -static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, - const void* srcBuffer, size_t srcSize) +static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, + buffers_t buf, contexts_t ctx) { BMK_result_t testResult; int better = 0; int cLevel; - BMK_benchParam1(&testResult, srcBuffer, srcSize, params); + BMK_benchParam(&testResult, buf, ctx, params); for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { @@ -1122,15 +1113,15 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* first solution for this cLevel */ winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, params, srcSize); + BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); better = 1; continue; } if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) { /* Validate solution is "good enough" */ - double W_ratio = (double)srcSize / testResult.cSize; - double O_ratio = (double)srcSize / winners[cLevel].result.cSize; + double W_ratio = (double)buf.srcSize / testResult.cSize; + double O_ratio = (double)buf.srcSize / winners[cLevel].result.cSize; double W_ratioNote = log (W_ratio); double O_ratioNote = log (O_ratio); size_t W_DMemUsed = (1 << params.windowLog) + (16 KB); @@ -1187,7 +1178,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, params, srcSize); + BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); better = 1; } } @@ -1315,6 +1306,26 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var *ptr = p; } +/* maybe put strategy back in */ +static void paramVariationWithStrategy(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) +{ + ZSTD_compressionParameters p; + U32 validated = 0; + while (!validated) { + U32 i; + p = *ptr; + for (i = 0 ; i < nbChanges ; i++) { + const U32 changeID = FUZ_rand(&g_rand) % ((varyLen + 1) << 1); + if(changeID < (U32)(varyLen << 1)) { + paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); + } else { + p.strategy += ((FUZ_rand(&g_rand) % 2) << 1) - 1; /* +/- 1 */ + } + } + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + } + *ptr = p; +} /* length of memo table given free variables */ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { size_t arrayLen = 1; @@ -1463,6 +1474,17 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co return mtAll; } +static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { + base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog; + base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog; + base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog; + base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog; + base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength; + base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength; + base.strategy = mask.strategy ? mask.strategy : base.strategy; + return base; +} + #define PARAMTABLELOG 25 #define PARAMTABLESIZE (1< g_maxNbVariations) break; - paramVariation(&p, unconstrained, 7, 4); + paramVariationWithStrategy(&p, unconstrained, NUM_PARAMS, 4); /* exclude faster if already played params */ if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) @@ -1500,11 +1522,11 @@ static void playAround(FILE* f, winnerInfo_t* winners, /* test */ b = NB_TESTS_PLAYED(p); (*b)++; - if (!BMK_seed(winners, p, srcBuffer, srcSize)) continue; + if (!BMK_seed(winners, p, buf, ctx)) continue; /* improvement found => search more */ - BMK_printWinners(f, winners, srcSize); - playAround(f, winners, p, srcBuffer, srcSize); + BMK_printWinners(f, winners, buf.srcSize); + playAround(f, winners, p, buf, ctx); } } @@ -1551,35 +1573,24 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v static void BMK_selectRandomStart( FILE* f, winnerInfo_t* winners, - const void* srcBuffer, size_t srcSize) + buffers_t buf, contexts_t ctx) { U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); if ((id==0) || (winners[id].params.windowLog==0)) { /* use some random entry */ - ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0); - playAround(f, winners, p, srcBuffer, srcSize); + ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), buf.srcSize, 0); + playAround(f, winners, p, buf, ctx); } else { - playAround(f, winners, winners[id].params, srcBuffer, srcSize); + playAround(f, winners, winners[id].params, buf, ctx); } } -static void BMK_benchOnce(const void* srcBuffer, size_t srcSize) -{ - BMK_result_t testResult; - g_params = ZSTD_adjustCParams(g_params, srcSize, 0); - BMK_benchParam1(&testResult, srcBuffer, srcSize, g_params); - DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)srcSize / testResult.cSize, - (double)testResult.cSpeed / 1000000, (double)testResult.dSpeed / 1000000); - return; -} - -static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) +static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize) { ZSTD_compressionParameters params; winnerInfo_t winners[NB_LEVELS_TRACKED+1]; const char* const rfName = "grillResults.txt"; FILE* const f = fopen(rfName, "w"); - const size_t blockSize = g_blockSize ? g_blockSize : srcSize; /* cut by block or not ? */ /* init */ assert(g_singleRun==0); @@ -1590,9 +1601,9 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) BMK_init_level_constraints(g_target * (1 MB)); } else { /* baseline config for level 1 */ - ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, blockSize, 0); + ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); //is dictionary ever even useful here? BMK_result_t testResult; - BMK_benchParam1(&testResult, srcBuffer, srcSize, l1params); + BMK_benchParam(&testResult, buf, ctx, l1params); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); } @@ -1600,61 +1611,124 @@ static void BMK_benchFullTable(const void* srcBuffer, size_t srcSize) { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); int i; for (i=0; i<=maxSeeds; i++) { - params = ZSTD_getCParams(i, blockSize, 0); - BMK_seed(winners, params, srcBuffer, srcSize); + params = ZSTD_getCParams(i, maxBlockSize, 0); + BMK_seed(winners, params, buf, ctx); } } - BMK_printWinners(f, winners, srcSize); + BMK_printWinners(f, winners, buf.srcSize); /* start tests */ { const time_t grillStart = time(NULL); do { - BMK_selectRandomStart(f, winners, srcBuffer, srcSize); + BMK_selectRandomStart(f, winners, buf, ctx); } while (BMK_timeSpan(grillStart) < g_grillDuration_s); } /* end summary */ - BMK_printWinners(f, winners, srcSize); + BMK_printWinners(f, winners, buf.srcSize); DISPLAY("grillParams operations completed \n"); /* clean up*/ fclose(f); } -static void BMK_benchMemInit(const void* srcBuffer, size_t srcSize) -{ - if (g_singleRun) - return BMK_benchOnce(srcBuffer, srcSize); - else - return BMK_benchFullTable(srcBuffer, srcSize); -} - - static int benchSample(void) { const char* const name = "Sample 10MB"; size_t const benchedSize = 10 MB; + U32 blockSize = g_blockSize ? g_blockSize : benchedSize; + U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1; + size_t splitSize = 0; - void* origBuff = malloc(benchedSize); - if (!origBuff) { perror("not enough memory"); return 12; } + buffers_t buf; + contexts_t ctx; - /* Fill buffer */ - RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0); + buf.srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); + buf.dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buf.resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buf.srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buf.dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buf.dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buf.resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buf.srcSize = benchedSize; + + if(!buf.srcPtrs || !buf.dstPtrs || !buf.resPtrs || !buf.srcSizes || !buf.dstSizes || !buf.dstCapacities || !buf.resSizes) { + DISPLAY("Allocation Error\n"); + freeBuffers(buf); + return 1; + } + + buf.srcBuffer = malloc(benchedSize); + buf.srcPtrs[0] = (const void*)buf.srcBuffer; + buf.dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + 1024 * maxNbBlocks); + buf.resPtrs[0] = malloc(benchedSize); + + if(!buf.srcPtrs[0] || !buf.dstPtrs[0] || !buf.resPtrs[0]) { + DISPLAY("Allocation Error\n"); + freeBuffers(buf); + return 1; + } + + + splitSize = MIN(benchedSize, blockSize); + buf.srcSizes[0] = splitSize; + buf.dstCapacities[0] = ZSTD_compressBound(splitSize); + buf.resSizes[0] = splitSize; + + for(buf.nbBlocks = 1; splitSize < benchedSize; buf.nbBlocks++) { + const size_t i = buf.nbBlocks; + const size_t nextBlockSize = MIN(benchedSize - splitSize, blockSize); + buf.srcSizes[i] = nextBlockSize; + buf.dstCapacities[i] = ZSTD_compressBound(nextBlockSize); + buf.resSizes[i] = nextBlockSize; + buf.srcPtrs[i] = (const void*)(((const char*)buf.srcPtrs[i-1]) + buf.srcSizes[i-1]); + buf.dstPtrs[i] = (void*)(((char*)buf.dstPtrs[i-1]) + buf.dstSizes[i-1]); + buf.resPtrs[i] = (void*)(((char*)buf.resPtrs[i-1]) + buf.resSizes[i-1]); + splitSize += nextBlockSize; + } + + if(createContexts(&ctx, NULL)) { + DISPLAY("Context Creation Error\n"); + freeBuffers(buf); + return 1; + } + + RDG_genBuffer(buf.srcBuffer, benchedSize, g_compressibility, 0.0, 0); /* bench */ DISPLAY("\r%79s\r", ""); DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100)); - BMK_benchMemInit(origBuff, benchedSize); - free(origBuff); + BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize)); + + freeBuffers(buf); + freeContexts(ctx); + return 0; } -static int benchOnce(const char** fileNamesTable, int nbFiles, const char* dictFileName) { +static int benchOnce(buffers_t buf, contexts_t ctx) { + BMK_result_t testResult; + + if(BMK_benchParam(&testResult, buf, ctx, g_params)) { + DISPLAY("Error during benchmarking\n"); + return 1; + } + + DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, + (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB)); + return 0; +} + +/* benchFiles() : + * note: while this function takes a table of filenames, + * in practice, only the first filename will be used */ +int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName, int cLevel) +{ buffers_t buf; contexts_t ctx; - BMK_result_t testResult; size_t maxBlockSize = 0, i; + int ret = 0; if(createBuffers(&buf, fileNamesTable, nbFiles)) { DISPLAY("unable to load files\n"); @@ -1671,86 +1745,24 @@ static int benchOnce(const char** fileNamesTable, int nbFiles, const char* dictF maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]); } - g_params = ZSTD_adjustCParams(g_params, maxBlockSize, 0); - - if(BMK_benchParam(&testResult, buf, ctx, g_params)) { - DISPLAY("Error during benchmarking\n"); - freeBuffers(buf); - freeContexts(ctx); - return 3; + DISPLAY("\r%79s\r", ""); + if(nbFiles == 1) { + DISPLAY("using %s : \n", fileNamesTable[0]); + } else { + DISPLAY("using %d Files : \n", nbFiles); } - DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, - (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB)); + g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); + + if(g_singleRun) { + ret = benchOnce(buf, ctx); + } else { + BMK_benchFullTable(buf, ctx, maxBlockSize); + } freeBuffers(buf); freeContexts(ctx); - return 0; -} - -/* benchFiles() : - * note: while this function takes a table of filenames, - * in practice, only the first filename will be used */ -//TODO: dictionaries still not supported in fullTable mode -int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName) -{ - int fileIdx=0; - - if(g_singleRun) { - return benchOnce(fileNamesTable, nbFiles, dictFileName); - } - - /* Loop for each file */ - while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) - DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); - origBuff = malloc(benchedSize); - if (origBuff==NULL) { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - return 12; - } - - /* Fill input buffer */ - DISPLAY("Loading %s... \r", inFileName); - { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); - fclose(inFile); - if(readSize != benchedSize) { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(origBuff); - return 13; - } } - - /* bench */ - DISPLAY("\r%79s\r", ""); - DISPLAY("using %s : \n", inFileName); - BMK_benchMemInit(origBuff, benchedSize); - - /* clean */ - free(origBuff); - } - - return 0; + return ret; } /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ @@ -2074,17 +2086,6 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { } } -static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { - base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog; - base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog; - base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog; - base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog; - base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength; - base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength; - base.strategy = mask.strategy ? mask.strategy : base.strategy; - return base; -} - /* experiment with playing with this and decay value */ /* main fn called when using --optimize */ @@ -2115,6 +2116,8 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ contexts_t ctx; buffers_t buf; + g_time = UTIL_getTime(); + /* Init */ if(!cParamValid(paramTarget)) { return 1; @@ -2202,7 +2205,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ g_lvltarget.cSize /= ((double)g_strictness) / 100; target.cSpeed = (U32)g_lvltarget.cSpeed; - target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is worth + target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is reasonable. BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } @@ -2214,6 +2217,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } else { DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles); } + if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed >> 20); } if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed >> 20); } if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem >> 20); } @@ -2369,6 +2373,7 @@ static int usage_advanced(void) DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); DISPLAY( " -v : Prints Benchmarking output\n"); DISPLAY( " -D : Next argument dictionary file\n"); + DISPLAY( " -s : Seperate Files\n"); return 0; } @@ -2401,13 +2406,13 @@ int main(int argc, const char** argv) const char* input_filename = NULL; const char* dictFileName = NULL; U32 main_pause = 0; - int optimizerCLevel = 0; + int cLevel = 0; + int seperateFiles = 0; constraint_t target = { 0, 0, (U32)-1 }; - ZSTD_compressionParameters paramTarget = { 0, 0, 0, 0, 0, 0, 0 }; - - g_time = UTIL_getTime(); + ZSTD_compressionParameters paramTarget = emptyParams(); + g_params = emptyParams(); assert(argc>=1); /* for exename */ @@ -2430,11 +2435,11 @@ int main(int argc, const char** argv) PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed); PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); - PARSE_SUB_ARGS("level=", "lvl=", optimizerCLevel); + PARSE_SUB_ARGS("level=", "lvl=", cLevel); PARSE_SUB_ARGS("strict=", "stc=", g_strictness); PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier); PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier); - PARSE_SUB_ARGS("maxTries", "tries", g_maxTries); + PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries); DISPLAY("invalid optimization parameter \n"); return 1; @@ -2448,10 +2453,11 @@ int main(int argc, const char** argv) } else if (longCommandWArg(&argument, "--zstd=")) { /* Decode command (note : aggregated commands are allowed) */ g_singleRun = 1; - g_params = ZSTD_getCParams(2, g_blockSize, 0); + cLevel = 2; for ( ; ;) { PARSE_CPARAMS(g_params) - if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { g_params = ZSTD_getCParams(readU32FromChar(&argument), g_blockSize, 0); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; } + DISPLAY("invalid compression parameter \n"); return 1; } @@ -2528,8 +2534,8 @@ int main(int argc, const char** argv) continue; case 'L': { argument++; - int const cLevel = readU32FromChar(&argument); - g_params = ZSTD_getCParams(cLevel, g_blockSize, 0); + cLevel = readU32FromChar(&argument); + g_params = emptyParams(); continue; } default : ; @@ -2558,6 +2564,10 @@ int main(int argc, const char** argv) g_grillDuration_s = (double)readU32FromChar(&argument); break; + case 's': + seperateFiles = 1; + break; + /* load dictionary file (only applicable for optimizer rn) */ case 'D': if(i == argc - 1) { /* last argument, return error. */ @@ -2588,11 +2598,24 @@ int main(int argc, const char** argv) result = benchSample(); } } else { - if (g_optimizer) { - result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, optimizerCLevel); + if(seperateFiles) { + for(i = 0; i < argc - filenamesStart; i++) { + if (g_optimizer) { + result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevel); + if(result) { DISPLAY("Error on File %d", i); return result; } + } else { + result = benchFiles(argv+filenamesStart + i, 1, dictFileName, cLevel); + if(result) { DISPLAY("Error on File %d", i); return result; } + } + } } else { - result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName); - } } + if (g_optimizer) { + result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevel); + } else { + result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevel); + } + } + } if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; } From 3b36fe5c68952dd03556ab04ff50dcbe0f1c9ec4 Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 31 Jul 2018 11:13:44 -0700 Subject: [PATCH 59/92] strategy switching --- tests/paramgrill.c | 124 +++++++++++++++++++++++---------------------- 1 file changed, 64 insertions(+), 60 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index cba97ebbb..955032e84 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -73,10 +73,11 @@ typedef enum { hlog_ind = 2, slog_ind = 3, slen_ind = 4, - tlen_ind = 5 + tlen_ind = 5, + strt_ind = 6 } varInds_t; -#define NUM_PARAMS 6 +#define NUM_PARAMS 7 /* just don't use strategy as a param. */ #undef ZSTD_WINDOWLOG_MAX @@ -91,9 +92,10 @@ typedef enum { #define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1) #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) #define TLEN_RANGE 17 +#define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1) /* TLEN_RANGE picked manually */ -static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE }; +static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE }; static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; /*-************************************ * Benchmark Parameters/Global Variables @@ -1213,8 +1215,10 @@ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInd int i, j = 0; for(i = 0; i < varLength; i++) { if( !((varArray[i] == clog_ind && strat == ZSTD_fast) + || (varArray[i] == slog_ind && strat == ZSTD_fast) || (varArray[i] == slog_ind && strat == ZSTD_dfast) - || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { + || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast) + /* || varArray[i] == strt_ind */ )) { varNew[j] = varArray[i]; j++; } @@ -1251,6 +1255,10 @@ static int variableParams(const ZSTD_compressionParameters paramConstraints, var res[j] = tlen_ind; j++; } + if(!paramConstraints.strategy) { + res[j] = strt_ind; + j++; + } return j; } @@ -1285,6 +1293,7 @@ static void paramVaryOnce(const varInds_t paramIndex, const int amt, ZSTD_compre case tlen_ind: ptr->targetLength = tlen_table[MAX(0, MIN(TLEN_RANGE - 1, tlen_inv(ptr->targetLength) + amt))]; break; + case strt_ind: ptr->strategy += amt; break; default: break; } } @@ -1301,37 +1310,19 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } - validated = !ZSTD_isError(ZSTD_checkCParams(p)); + validated = !ZSTD_isError(ZSTD_checkCParams(p)) && p.strategy > 0; } *ptr = p; } -/* maybe put strategy back in */ -static void paramVariationWithStrategy(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) -{ - ZSTD_compressionParameters p; - U32 validated = 0; - while (!validated) { - U32 i; - p = *ptr; - for (i = 0 ; i < nbChanges ; i++) { - const U32 changeID = FUZ_rand(&g_rand) % ((varyLen + 1) << 1); - if(changeID < (U32)(varyLen << 1)) { - paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); - } else { - p.strategy += ((FUZ_rand(&g_rand) % 2) << 1) - 1; /* +/- 1 */ - } - } - validated = !ZSTD_isError(ZSTD_checkCParams(p)); - } - *ptr = p; -} /* length of memo table given free variables */ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { size_t arrayLen = 1; int i; for(i = 0; i < varyLen; i++) { - arrayLen *= rangetable[varyParams[i]]; + if(varyParams[i] != strt_ind) { + arrayLen *= rangetable[varyParams[i]]; + } } return arrayLen; } @@ -1354,6 +1345,7 @@ static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInd - ZSTD_SEARCHLENGTH_MIN; break; case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) - ZSTD_TARGETLENGTH_MIN; break; + case strt_ind: break; } } return ind; @@ -1376,6 +1368,7 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va ind /= SLEN_RANGE; break; case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; ind /= TLEN_RANGE; break; + case strt_ind: break; } } } @@ -1505,7 +1498,7 @@ static void playAround(FILE* f, winnerInfo_t* winners, { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); - const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5 }; + const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { @@ -1513,7 +1506,7 @@ static void playAround(FILE* f, winnerInfo_t* winners, BYTE* b; if (nbVariations++ > g_maxNbVariations) break; - paramVariationWithStrategy(&p, unconstrained, NUM_PARAMS, 4); + paramVariation(&p, unconstrained, NUM_PARAMS, 4); /* exclude faster if already played params */ if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) @@ -1715,8 +1708,7 @@ static int benchOnce(buffers_t buf, contexts_t ctx) { return 1; } - DISPLAY("Compression Ratio: %.3f Compress Speed: %.1f MB/s Decompress Speed: %.1f MB/s\n", (double)buf.srcSize / testResult.cSize, - (double)testResult.cSpeed / (1 MB), (double)testResult.dSpeed / (1 MB)); + BMK_printWinner(stdout, CUSTOM_LEVEL, testResult, g_params, buf.srcSize); return 0; } @@ -1766,7 +1758,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam } /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ -#define VARIANCE 1.1 +#define VARIANCE 1.2 static int allBench(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams, @@ -1918,9 +1910,9 @@ static int benchMemo(BMK_result_t* resultPtr, * all generation after random should be sanitized. (maybe sanitize random) */ static winnerInfo_t climbOnce(const constraint_t target, - const varInds_t* varArray, const int varLen, - U8* const memoTable, - const buffers_t buf, const contexts_t ctx, + const varInds_t* varArray, const int varLen, ZSTD_strategy strat, + U8** memoTableArray, + buffers_t buf, contexts_t ctx, const ZSTD_compressionParameters init) { /* * cparam - currently considered 'center' @@ -1931,6 +1923,8 @@ static winnerInfo_t climbOnce(const constraint_t target, winnerInfo_t candidateInfo, winnerInfo; int better = 1; int feas = 0; + varInds_t varNew[NUM_PARAMS]; + int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); winnerInfo = initWinnerInfo(init); candidateInfo = winnerInfo; @@ -1951,15 +1945,20 @@ static winnerInfo_t climbOnce(const constraint_t target, for(offset = -1; offset <= 1; offset += 2) { candidateInfo.params = cparam; paramVaryOnce(varArray[i], offset, &candidateInfo.params); - candidateInfo.params = sanitizeParams(candidateInfo.params); - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params))) { - int res = benchMemo(&candidateInfo.result, + + if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params)) && candidateInfo.params.strategy > 0) { + int res; + if(strat != candidateInfo.params.strategy) { /* maybe only try strategy switching after exhausting non-switching solutions? */ + strat = candidateInfo.params.strategy; + varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); + } + res = benchMemo(&candidateInfo.result, buf, ctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen, feas); + sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat], + varNew, varLenNew, feas); if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ winnerInfo = candidateInfo; - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; @@ -1974,22 +1973,29 @@ static winnerInfo_t climbOnce(const constraint_t target, } for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */ - for(i = 0; i < 2 * varLen + 2; i++) { + for(i = 0; i < (1 << varLen) / varLen + 2; i++) { int res; candidateInfo.params = cparam; /* param error checking already done here */ paramVariation(&candidateInfo.params, varArray, varLen, dist); + + if(strat != candidateInfo.params.strategy) { + strat = candidateInfo.params.strategy; + varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); + } + res = benchMemo(&candidateInfo.result, buf, ctx, - candidateInfo.params, target, &winnerInfo.result, memoTable, - varArray, varLen, feas); + sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat], + varNew, varLenNew, feas); if(res == BETTER_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; } + break; } } @@ -2026,10 +2032,11 @@ static winnerInfo_t optimizeFixedStrategy( const constraint_t target, ZSTD_compressionParameters paramTarget, const ZSTD_strategy strat, const varInds_t* varArray, const int varLen, - U8* const memoTable, const int tries) { + U8** memoTableArray, const int tries) { int i = 0; varInds_t varNew[NUM_PARAMS]; int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); + ZSTD_compressionParameters init; winnerInfo_t winnerInfo, candidateInfo; winnerInfo = initWinnerInfo(emptyParams()); @@ -2043,8 +2050,8 @@ static winnerInfo_t optimizeFixedStrategy( while(i < tries) { DEBUGOUTPUT("Restart\n"); - randomConstrainedParams(&init, varNew, varLenNew, memoTable); - candidateInfo = climbOnce(target, varNew, varLenNew, memoTable, buf, ctx, init); + randomConstrainedParams(&init, varNew, varLenNew, memoTableArray[strat]); + candidateInfo = climbOnce(target, varArray, varLen, strat, memoTableArray, buf, ctx, init); if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { winnerInfo = candidateInfo; BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); @@ -2226,7 +2233,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ findClockGranularity(); { - varInds_t varNew[NUM_PARAMS]; ZSTD_compressionParameters CParams; /* find best solution from default params */ @@ -2266,8 +2272,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ { /* one iterations of hill climbing with the level-defined parameters. */ - int varLenNew = sanitizeVarArray(varNew, varLen, varArray, st); - winnerInfo_t w1 = climbOnce(target, varNew, varLenNew, allMT[st], + winnerInfo_t w1 = climbOnce(target, varArray, varLen, st, allMT, buf, ctx, winner.params); if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) { winner = w1; @@ -2279,7 +2284,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, - st, varArray, varLen, allMT[st], tries); + st, varArray, varLen, allMT, tries); if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; @@ -2292,7 +2297,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } } else { winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, - varArray, varLen, allMT[paramTarget.strategy], g_maxTries); + varArray, varLen, allMT, g_maxTries); } } @@ -2387,14 +2392,13 @@ static int badusage(const char* exename) #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } } #define PARSE_CPARAMS(variable) \ { \ - PARSE_SUB_ARGS("windowLog=", "wlog=", variable.vals[wlog_ind]); \ - PARSE_SUB_ARGS("chainLog=" , "clog=", variable.vals[clog_ind]); \ - PARSE_SUB_ARGS("hashLog=", "hlog=", variable.vals[hlog_ind]); \ - PARSE_SUB_ARGS("searchLog=" , "slog=", variable.vals[slog_ind]); \ - PARSE_SUB_ARGS("searchLength=", "slen=", variable.vals[slen_ind]); \ - PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.vals[tlen_ind]); \ - PARSE_SUB_ARGS("strategy=", "strat=", variable.vals[strt_ind]); \ - PARSE_SUB_ARGS("forceAttachDict=", "fad=" , variable.vals[strt_ind]); \ + PARSE_SUB_ARGS("windowLog=", "wlog=", variable.windowLog); \ + PARSE_SUB_ARGS("chainLog=" , "clog=", variable.chainLog); \ + PARSE_SUB_ARGS("hashLog=", "hlog=", variable.hashLog); \ + PARSE_SUB_ARGS("searchLog=" , "slog=", variable.searchLog); \ + PARSE_SUB_ARGS("searchLength=", "slen=", variable.searchLength); \ + PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.targetLength); \ + PARSE_SUB_ARGS("strategy=", "strat=", variable.strategy); \ } int main(int argc, const char** argv) From a6df961497b766546db31b7876aef5b3718c17a4 Mon Sep 17 00:00:00 2001 From: Eden Zik Date: Mon, 13 Aug 2018 20:28:52 -0400 Subject: [PATCH 60/92] Cmake now builds with CMAKE_BUILD_TYPE=Release by default, both while being invoked from the main Makefile (via cmakebuild) or directly from the build/cmake directory. Suggested by @pdknsk (#1081). --- Makefile | 2 +- README.md | 2 ++ build/.gitignore | 11 +++++++++++ build/cmake/CMakeLists.txt | 4 ++++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f67791275..6c4ab9c9b 100644 --- a/Makefile +++ b/Makefile @@ -114,7 +114,7 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT)) HOST_OS = POSIX -CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON +CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release .PHONY: list list: diff --git a/README.md b/README.md index 17edecb71..dc99dc0fd 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,8 @@ A `cmake` project generator is provided within `build/cmake`. It can generate Makefiles or other build scripts to create `zstd` binary, and `libzstd` dynamic and static libraries. +By default, `CMAKE_BUILD_TYPE` is set to `Release`. + #### Meson A Meson project is provided within `contrib/meson`. diff --git a/build/.gitignore b/build/.gitignore index b00e709ba..1ceb70ebc 100644 --- a/build/.gitignore +++ b/build/.gitignore @@ -18,3 +18,14 @@ Studio* # CMake cmake/build/ +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +Makefile +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake +build +lib diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt index fd9bc2b1e..1e2921de8 100644 --- a/build/cmake/CMakeLists.txt +++ b/build/cmake/CMakeLists.txt @@ -10,6 +10,10 @@ PROJECT(zstd) CMAKE_MINIMUM_REQUIRED(VERSION 2.8.9) SET(ZSTD_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..") + +# Ensure Release build even if not invoked via Makefile +SET(CMAKE_BUILD_TYPE "Release") + LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules") INCLUDE(GNUInstallDirs) From 614aaa3ae152aa13c3b17ae01a39f151be6c1c56 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 13 Aug 2018 16:38:51 -0700 Subject: [PATCH 61/92] rebase clevel --- tests/paramgrill.c | 63 ++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 955032e84..58f2e13bc 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -65,6 +65,7 @@ static const int g_maxNbVariations = 64; #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) #define CUSTOM_LEVEL 99 +#define BASE_CLEVEL 1 /* indices for each of the variables */ typedef enum { @@ -112,7 +113,7 @@ static U32 g_singleRun = 0; static U32 g_optimizer = 0; static U32 g_target = 0; static U32 g_noSeed = 0; -static ZSTD_compressionParameters g_params; +static ZSTD_compressionParameters g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ @@ -523,6 +524,7 @@ static void freeBuffers(const buffers_t b) { free(b.resPtrs[0]); } free(b.resPtrs); + free(b.resSizes); } /* allocates buffer's arguments. returns success / failuere */ @@ -532,7 +534,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab size_t pos = 0; size_t n; U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); - size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); + const size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles; U32 blockNb = 0; @@ -671,6 +673,7 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) { freeContexts(*ctx); return 1; } + fclose(f); return 0; } @@ -824,7 +827,7 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t static int BMK_benchParam(BMK_result_t* resultPtr, buffers_t buf, contexts_t ctx, const ZSTD_compressionParameters cParams) { - BMK_return_t res = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_timeMode, 3); + BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3); *resultPtr = res.result; return res.error; } @@ -840,16 +843,16 @@ static int BMK_benchParam(BMK_result_t* resultPtr, #define SIZE_RESULT 5 /* maybe have epsilon-eq to limit table size? */ static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) { - if(r1.cSpeed > r2.cSpeed) { - if(r1.cSize <= r2.cSize) { - return WORSE_RESULT; - } - return SIZE_RESULT; /* r2 is smaller but not faster. */ - } else { + if(r1.cSpeed < r2.cSpeed) { if(r1.cSize >= r2.cSize) { return BETTER_RESULT; } - return SPEED_RESULT; /* r2 is faster but not smaller */ + return SPEED_RESULT; /* r2 is smaller but not faster. */ + } else { + if(r1.cSize <= r2.cSize) { + return WORSE_RESULT; + } + return SIZE_RESULT; /* r2 is faster but not smaller */ } } @@ -875,12 +878,12 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { } while(cur_node->next != NULL) { - switch(speedSizeCompare(r, cur_node->res.result)) { - case BETTER_RESULT: + switch(speedSizeCompare(cur_node->res.result, r)) { + case WORSE_RESULT: { return 1; /* never insert if better */ } - case WORSE_RESULT: + case BETTER_RESULT: { winner_ll_node* tmp; cur_node->res = cur_node->next->res; @@ -889,12 +892,12 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { free(tmp); break; } - case SPEED_RESULT: + case SIZE_RESULT: { cur_node = cur_node->next; break; } - case SIZE_RESULT: /* insert after first size result, then return */ + case SPEED_RESULT: /* insert after first size result, then return */ { winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); if(newnode == NULL) { @@ -911,17 +914,17 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { } assert(cur_node->next == NULL); - switch(speedSizeCompare(r, cur_node->res.result)) { - case BETTER_RESULT: + switch(speedSizeCompare(cur_node->res.result, r)) { + case WORSE_RESULT: { return 1; /* never insert if better */ } - case WORSE_RESULT: + case BETTER_RESULT: { cur_node->res = w; return 0; } - case SPEED_RESULT: + case SIZE_RESULT: { winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); if(newnode == NULL) { @@ -932,7 +935,7 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { cur_node->next = newnode; return 0; } - case SIZE_RESULT: /* insert before first size result, then return */ + case SPEED_RESULT: /* insert before first size result, then return */ { winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); if(newnode == NULL) { @@ -1448,7 +1451,7 @@ static void freeMemoTableArray(U8** mtAll) { /* takes unsanitized varyParams */ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { varInds_t varNew[NUM_PARAMS]; - U8** mtAll = calloc(sizeof(U8*),(ZSTD_btultra + 1)); + U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1)); int i; if(mtAll == NULL) { return NULL; @@ -1467,7 +1470,7 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co return mtAll; } -static ZSTD_compressionParameters maskParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { +static ZSTD_compressionParameters overwriteParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog; base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog; base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog; @@ -1744,7 +1747,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam DISPLAY("using %d Files : \n", nbFiles); } - g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); + g_params = ZSTD_adjustCParams(overwriteParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); if(g_singleRun) { ret = benchOnce(buf, ctx); @@ -1771,7 +1774,7 @@ static int allBench(BMK_result_t* resultPtr, double winnerRS; /* initial benchmarking, gives exact ratio and memory, warms up future runs */ - benchres = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_both, BMK_iterMode, 1); + benchres = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_iterMode, 1); winnerRS = resultScore(*winnerResult, buf.srcSize, target); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); @@ -1806,14 +1809,14 @@ static int allBench(BMK_result_t* resultPtr, /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_iterMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } benchres = benchres2; } if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_iterMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_iterMode, 1); if(benchres2.error) { return ERROR_RESULT; } @@ -1836,7 +1839,7 @@ static int allBench(BMK_result_t* resultPtr, /* Final full run if estimates are unclear */ if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_compressOnly, BMK_timeMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } @@ -1844,7 +1847,7 @@ static int allBench(BMK_result_t* resultPtr, } if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, 0, &cParams, BMK_decodeOnly, BMK_timeMode, 1); + BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_timeMode, 1); if(benchres2.error) { return ERROR_RESULT; } @@ -2245,7 +2248,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int i; for (i=1; i<=maxSeeds; i++) { int ec; - CParams = maskParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget); + CParams = overwriteParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget); ec = BMK_benchParam(&candidate, buf, ctx, CParams); BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize); @@ -2439,11 +2442,11 @@ int main(int argc, const char** argv) PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed); PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); - PARSE_SUB_ARGS("level=", "lvl=", cLevel); PARSE_SUB_ARGS("strict=", "stc=", g_strictness); PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier); PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier); PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries); + if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; } DISPLAY("invalid optimization parameter \n"); return 1; From 76acba025da5ead78532bfd0dff4de6af67a6506 Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 14 Aug 2018 11:57:15 -0700 Subject: [PATCH 62/92] scan-build --- tests/paramgrill.c | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 58f2e13bc..7e85bf832 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -176,14 +176,14 @@ static size_t BMK_findMaxMem(U64 requiredMem) requiredMem = (((requiredMem >> 26) + 1) << 26); if (requiredMem > maxMemory) requiredMem = maxMemory; - requiredMem += 2*step; - while (!testmem) { - requiredMem -= step; + requiredMem += 2 * step; + while (!testmem && requiredMem > 0) { testmem = malloc ((size_t)requiredMem); + requiredMem -= step; } free (testmem); - return (size_t) (requiredMem - step); + return (size_t) requiredMem; } @@ -536,9 +536,14 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); const size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; - U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / blockSize) + (U32)nbFiles; + U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / MAX(blockSize, 1)) + (U32)nbFiles; U32 blockNb = 0; + if(!totalSizeToLoad || !benchedSize) { + DISPLAY("Nothing to Bench\n"); + return 1; + } + buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); @@ -555,6 +560,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab return 1; } + buff->srcBuffer = malloc(benchedSize); buff->srcPtrs[0] = (const void*)buff->srcBuffer; buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024)); @@ -613,6 +619,12 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab fclose(f); } + if(!blockNb) { + DISPLAY("Failed to load any files\n"); + freeBuffers(*buff); + return 1; + } + buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); buff->dstSizes[0] = buff->dstCapacities[0]; buff->resSizes[0] = buff->srcSizes[0]; @@ -957,8 +969,6 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) { char lvlstr[15] = "Custom Level"; - const U64 time = UTIL_clockSpanNano(g_time); - const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); fprintf(f, "\r%79s\r", ""); @@ -974,7 +984,11 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); - if(TIMED) { fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } + if(TIMED) { + const U64 time = UTIL_clockSpanNano(g_time); + const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); + fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); + } fprintf(f, "\n"); } @@ -2160,7 +2174,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(paramTarget.strategy) { varInds_t varNew[NUM_PARAMS]; int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.strategy); - allMT = calloc(sizeof(U8), (ZSTD_btultra + 1)); + allMT = (U8**)calloc(sizeof(U8*), (ZSTD_btultra + 1)); if(allMT == NULL) { ret = 57; goto _cleanUp; From f581ccd267244da3d0a4d8d6ba1cb8a1b191bfd8 Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 31 Jul 2018 18:47:27 -0700 Subject: [PATCH 63/92] Doc Updates Add option to pass in existing parameters in use --- tests/README.md | 4 ++-- tests/paramgrill.c | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/README.md b/tests/README.md index 736916936..4d2f314d2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -105,7 +105,8 @@ Full list of arguments t# - targetLength S# - strategy L# - level - --zstd= : Single run, parameter selection syntax same as zstdcli + --zstd= : Single run, parameter selection syntax same as zstdcli. + When invoked with --optimize, this represents the sample to exceed. --optimize= : find parameters to maximize compression ratio given parameters Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints cSpeed= : Minimum compression speed @@ -120,7 +121,6 @@ Full list of arguments when determining overall winner (default 1 for both, higher = more valued). tries= : Maximum number of random restarts on a single strategy before switching (Default 5) Higher values will make optimizer run longer, more chances to find better solution. - --optimize= : same as -O with more verbose syntax -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 7e85bf832..47e0a9421 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -2234,6 +2234,19 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } + if(g_singleRun) { + BMK_result_t res; + g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); + if(BMK_benchParam(&res, buf, ctx, g_params)) { + ret = 45; + goto _cleanUp; + } + if(compareResultLT(winner.result, res, relaxTarget(target), buf.srcSize)) { + winner.result = res; + winner.params = g_params; + } + } + /* bench */ DISPLAY("\r%79s\r", ""); if(nbFiles == 1) { From 88dda922854550959bda7640f554b5f02e5d0d4f Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 6 Aug 2018 15:08:35 -0700 Subject: [PATCH 64/92] Reduce Duplication Change Defaults Asserts actually disabled in paramgrill + fullbench --- tests/Makefile | 4 +- tests/README.md | 4 +- tests/paramgrill.c | 287 +++++++++++++++++++++------------------------ 3 files changed, 140 insertions(+), 155 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 81e685780..88a5d763c 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -129,7 +129,7 @@ zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c fullbench32: CPPFLAGS += -m32 fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD) -fullbench fullbench32 : DEBUGFLAGS = # turn off assert() for speed measurements +fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements fullbench fullbench32 : $(ZSTD_FILES) fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/bench.c fullbench.c $(CC) $(FLAGS) $^ -o $@$(EXT) @@ -200,7 +200,7 @@ zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) -paramgrill : DEBUGFLAGS = # turn off assert() for speed measurements +paramgrill : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements paramgrill : $(ZSTD_FILES) $(PRGDIR)/bench.c $(PRGDIR)/datagen.c paramgrill.c $(CC) $(FLAGS) $^ -lm -o $@$(EXT) diff --git a/tests/README.md b/tests/README.md index 4d2f314d2..3c35ecf01 100644 --- a/tests/README.md +++ b/tests/README.md @@ -118,8 +118,8 @@ Full list of arguments (Lower value will begin with stronger strategies) (Default 90%) preferSpeed= / preferRatio= : Only affects lvl = invocations. Defines value placed on compression speed or ratio - when determining overall winner (default 1 for both, higher = more valued). - tries= : Maximum number of random restarts on a single strategy before switching (Default 5) + when determining overall winner (default speed = 1, ratio = 5 for both, higher = more valued). + tries= : Maximum number of random restarts on a single strategy before switching (Default 3) Higher values will make optimizer run longer, more chances to find better solution. -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 47e0a9421..193b2ed20 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -139,7 +139,7 @@ static BMK_result_t g_lvltarget; static int g_optmode = 0; static U32 g_speedMultiplier = 1; -static U32 g_ratioMultiplier = 1; +static U32 g_ratioMultiplier = 5; /* g_mode? */ @@ -382,7 +382,7 @@ typedef struct { } contexts_t; /*-******************************************************* -* From Paramgrill +* From bench.c *********************************************************/ static void BMK_initCCtx(ZSTD_CCtx* ctx, @@ -411,7 +411,6 @@ static void BMK_initCCtx(ZSTD_CCtx* ctx, ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); } - static void BMK_initDCtx(ZSTD_DCtx* dctx, const void* dictBuffer, const size_t dictBufferSize) { ZSTD_DCtx_reset(dctx); @@ -503,7 +502,7 @@ static size_t local_defaultDecompress( } /*-******************************************************* -* From Paramgrill End +* From bench.c End *********************************************************/ static void freeBuffers(const buffers_t b) { @@ -527,23 +526,25 @@ static void freeBuffers(const buffers_t b) { free(b.resSizes); } -/* allocates buffer's arguments. returns success / failuere */ -static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, - const size_t nbFiles) +/* srcBuffer will be freed by freeBuffers now */ +static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbFiles, + const size_t* fileSizes) { - size_t pos = 0; - size_t n; - U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); - const size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); - const size_t blockSize = g_blockSize ? g_blockSize : totalSizeToLoad; - U32 const maxNbBlocks = (U32) ((totalSizeToLoad + (blockSize-1)) / MAX(blockSize, 1)) + (U32)nbFiles; - U32 blockNb = 0; + size_t pos = 0, n, blockSize; + U32 maxNbBlocks, blockNb = 0; + buff->srcSize = 0; + for(n = 0; n < nbFiles; n++) { + buff->srcSize += fileSizes[n]; + } - if(!totalSizeToLoad || !benchedSize) { - DISPLAY("Nothing to Bench\n"); + if(buff->srcSize == 0) { + DISPLAY("No data to bench\n"); return 1; } - + + blockSize = g_blockSize ? g_blockSize : buff->srcSize; + maxNbBlocks = (U32) ((buff->srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; + buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); @@ -560,18 +561,62 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab return 1; } - - buff->srcBuffer = malloc(benchedSize); + buff->srcBuffer = srcBuffer; buff->srcPtrs[0] = (const void*)buff->srcBuffer; - buff->dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + (maxNbBlocks * 1024)); - buff->resPtrs[0] = malloc(benchedSize); + buff->dstPtrs[0] = malloc(ZSTD_compressBound(buff->srcSize) + (maxNbBlocks * 1024)); + buff->resPtrs[0] = malloc(buff->srcSize); - if(!buff->srcPtrs[0] || !buff->dstPtrs[0] || !buff->resPtrs[0]) { + if(!buff->dstPtrs[0] || !buff->resPtrs[0]) { DISPLAY("alloc error\n"); freeBuffers(*buff); return 1; } + for(n = 0; n < nbFiles; n++) { + size_t pos_end = pos + fileSizes[n]; + for(; pos < pos_end; blockNb++) { + buff->srcPtrs[blockNb] = (const void*)((char*)srcBuffer + pos); + buff->srcSizes[blockNb] = blockSize; + pos += blockSize; + } + + if(fileSizes[n] > 0) { buff->srcSizes[blockNb - 1] = ((fileSizes[n] - 1) % blockSize) + 1; } + pos = pos_end; + } + + buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); + buff->dstSizes[0] = buff->dstCapacities[0]; + buff->resSizes[0] = buff->srcSizes[0]; + + for(n = 1; n < blockNb; n++) { + buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1]; + buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1]; + buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]); + buff->dstSizes[n] = buff->dstCapacities[n]; + buff->resSizes[n] = buff->srcSizes[n]; + } + + buff->nbBlocks = blockNb; + + return 0; +} + +/* allocates buffer's arguments. returns success / failuere */ +static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, + size_t nbFiles) { + size_t pos = 0; + size_t n; + size_t totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); + size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); + size_t* fileSizes = calloc(sizeof(size_t), nbFiles); + void* srcBuffer = malloc(benchedSize); + int ret = 0; + + + if(!fileSizes || !srcBuffer) { + return 1; + } + for(n = 0; n < nbFiles; n++) { FILE* f; U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); @@ -586,62 +631,35 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab f = fopen(fileNamesTable[n], "rb"); if (f==NULL) { DISPLAY("impossible to open file %s\n", fileNamesTable[n]); - freeBuffers(*buff); + free(fileSizes); + free(srcBuffer); fclose(f); return 10; } DISPLAY("Loading %s... \r", fileNamesTable[n]); - if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, n=nbFiles; /* buffer too small - stop after this file */ + if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ { - char* buffer = (char*)(buff->srcBuffer); + char* buffer = (char*)(srcBuffer); size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); - size_t blocked = 0; - while(blocked < readSize) { - buff->srcPtrs[blockNb] = (const void*)((buffer) + (pos + blocked)); - buff->srcSizes[blockNb] = blockSize; - blocked += blockSize; - blockNb++; - } - if(readSize > 0) { buff->srcSizes[blockNb - 1] = ((readSize - 1) % blockSize) + 1; } - if (readSize != (size_t)fileSize) { + if (readSize != (size_t)fileSize) { /* should we accept partial read? */ DISPLAY("could not read %s", fileNamesTable[n]); - freeBuffers(*buff); - fclose(f); + free(fileSizes); + free(srcBuffer); return 1; } + fileSizes[n] = readSize; pos += readSize; - } fclose(f); } - if(!blockNb) { - DISPLAY("Failed to load any files\n"); - freeBuffers(*buff); - return 1; - } - - buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); - buff->dstSizes[0] = buff->dstCapacities[0]; - buff->resSizes[0] = buff->srcSizes[0]; - - for(n = 1; n < blockNb; n++) { - buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1]; - buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1]; - buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]); - buff->dstSizes[n] = buff->dstCapacities[n]; - buff->resSizes[n] = buff->srcSizes[n]; - } - buff->srcSize = pos; - buff->nbBlocks = blockNb; - - if (pos == 0) { DISPLAY("\nno data to bench\n"); return 1; } - - return 0; + ret = createBuffersFromMemory(buff, srcBuffer, nbFiles, fileSizes); + free(fileSizes); + return ret; } static void freeContexts(const contexts_t ctx) { @@ -1611,7 +1629,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl BMK_init_level_constraints(g_target * (1 MB)); } else { /* baseline config for level 1 */ - ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); //is dictionary ever even useful here? + ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); BMK_result_t testResult; BMK_benchParam(&testResult, buf, ctx, l1params); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); @@ -1641,82 +1659,6 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl fclose(f); } -static int benchSample(void) -{ - const char* const name = "Sample 10MB"; - size_t const benchedSize = 10 MB; - U32 blockSize = g_blockSize ? g_blockSize : benchedSize; - U32 const maxNbBlocks = (U32) ((benchedSize + (blockSize-1)) / blockSize) + 1; - size_t splitSize = 0; - - buffers_t buf; - contexts_t ctx; - - buf.srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); - buf.dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); - buf.resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); - buf.srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - buf.dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - buf.dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - buf.resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); - buf.srcSize = benchedSize; - - if(!buf.srcPtrs || !buf.dstPtrs || !buf.resPtrs || !buf.srcSizes || !buf.dstSizes || !buf.dstCapacities || !buf.resSizes) { - DISPLAY("Allocation Error\n"); - freeBuffers(buf); - return 1; - } - - buf.srcBuffer = malloc(benchedSize); - buf.srcPtrs[0] = (const void*)buf.srcBuffer; - buf.dstPtrs[0] = malloc(ZSTD_compressBound(benchedSize) + 1024 * maxNbBlocks); - buf.resPtrs[0] = malloc(benchedSize); - - if(!buf.srcPtrs[0] || !buf.dstPtrs[0] || !buf.resPtrs[0]) { - DISPLAY("Allocation Error\n"); - freeBuffers(buf); - return 1; - } - - - splitSize = MIN(benchedSize, blockSize); - buf.srcSizes[0] = splitSize; - buf.dstCapacities[0] = ZSTD_compressBound(splitSize); - buf.resSizes[0] = splitSize; - - for(buf.nbBlocks = 1; splitSize < benchedSize; buf.nbBlocks++) { - const size_t i = buf.nbBlocks; - const size_t nextBlockSize = MIN(benchedSize - splitSize, blockSize); - buf.srcSizes[i] = nextBlockSize; - buf.dstCapacities[i] = ZSTD_compressBound(nextBlockSize); - buf.resSizes[i] = nextBlockSize; - buf.srcPtrs[i] = (const void*)(((const char*)buf.srcPtrs[i-1]) + buf.srcSizes[i-1]); - buf.dstPtrs[i] = (void*)(((char*)buf.dstPtrs[i-1]) + buf.dstSizes[i-1]); - buf.resPtrs[i] = (void*)(((char*)buf.resPtrs[i-1]) + buf.resSizes[i-1]); - splitSize += nextBlockSize; - } - - if(createContexts(&ctx, NULL)) { - DISPLAY("Context Creation Error\n"); - freeBuffers(buf); - return 1; - } - - RDG_genBuffer(buf.srcBuffer, benchedSize, g_compressibility, 0.0, 0); - - /* bench */ - DISPLAY("\r%79s\r", ""); - DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100)); - - BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize)); - - freeBuffers(buf); - freeContexts(ctx); - - return 0; -} - - static int benchOnce(buffers_t buf, contexts_t ctx) { BMK_result_t testResult; @@ -1726,9 +1668,56 @@ static int benchOnce(buffers_t buf, contexts_t ctx) { } BMK_printWinner(stdout, CUSTOM_LEVEL, testResult, g_params, buf.srcSize); + return 0; } +static int benchSample(void) +{ + const char* const name = "Sample 10MB"; + size_t const benchedSize = 10 MB; + U32 blockSize = g_blockSize ? g_blockSize : benchedSize; + void* srcBuffer = malloc(benchedSize); + int ret = 0; + + buffers_t buf; + contexts_t ctx; + + if(srcBuffer == NULL) { + DISPLAY("Out of Memory\n"); + return 2; + } + + RDG_genBuffer(srcBuffer, benchedSize, g_compressibility, 0.0, 0); + + if(createBuffersFromMemory(&buf, srcBuffer, 1, &benchedSize)) { + DISPLAY("Buffer Creation Error\n"); + free(srcBuffer); + return 3; + } + + if(createContexts(&ctx, NULL)) { + DISPLAY("Context Creation Error\n"); + freeBuffers(buf); + return 1; + } + + /* bench */ + DISPLAY("\r%79s\r", ""); + DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100)); + + if(g_singleRun) { + ret = benchOnce(buf, ctx); + } else { + BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize)); + } + + freeBuffers(buf); + freeContexts(ctx); + + return ret; +} + /* benchFiles() : * note: while this function takes a table of filenames, * in practice, only the first filename will be used */ @@ -1776,6 +1765,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ #define VARIANCE 1.2 +#define HIGH_VARIANCE 100.0 static int allBench(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, const ZSTD_compressionParameters cParams, @@ -1784,7 +1774,7 @@ static int allBench(BMK_result_t* resultPtr, BMK_return_t benchres; BMK_result_t resultMax; U64 loopDurationC = 0, loopDurationD = 0; - double uncertaintyConstantC, uncertaintyConstantD; + double uncertaintyConstantC = 3., uncertaintyConstantD = 3.; double winnerRS; /* initial benchmarking, gives exact ratio and memory, warms up future runs */ @@ -1802,18 +1792,12 @@ static int allBench(BMK_result_t* resultPtr, /* calculate uncertainty in compression / decompression runs */ if(benchres.result.cSpeed) { loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); - uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC) * VARIANCE; - } else { - loopDurationC = 0; - uncertaintyConstantC = 3; + uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC); } if(benchres.result.dSpeed) { loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); - uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD) * VARIANCE; - } else { - loopDurationD = 0; - uncertaintyConstantD = 3; + uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD); } /* anything with worse ratio in feas is definitely worse, discard */ @@ -1841,8 +1825,8 @@ static int allBench(BMK_result_t* resultPtr, /* optimistic assumption of benchres.result */ resultMax = benchres.result; - resultMax.cSpeed *= uncertaintyConstantC; - resultMax.dSpeed *= uncertaintyConstantD; + resultMax.cSpeed *= uncertaintyConstantC * VARIANCE; + resultMax.dSpeed *= uncertaintyConstantD * VARIANCE; /* disregard infeasible results in feas mode */ /* disregard if resultMax < winner in infeas mode */ @@ -2125,7 +2109,7 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) */ -static int g_maxTries = 5; +static int g_maxTries = 3; #define TRY_DECAY 1 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) @@ -2140,8 +2124,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ contexts_t ctx; buffers_t buf; - g_time = UTIL_getTime(); - /* Init */ if(!cParamValid(paramTarget)) { return 1; @@ -2234,6 +2216,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); } + /* Don't want it to return anything worse than the best known result */ if(g_singleRun) { BMK_result_t res; g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); @@ -2450,6 +2433,8 @@ int main(int argc, const char** argv) assert(argc>=1); /* for exename */ + g_time = UTIL_getTime(); + /* Welcome message */ DISPLAY(WELCOME_MESSAGE); From e3c679484aed822ec5574ae426a77490966b35aa Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 6 Aug 2018 16:52:17 -0700 Subject: [PATCH 65/92] Add Time Checks Fix double -> U64 display --- tests/paramgrill.c | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 193b2ed20..af4459187 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -17,7 +17,6 @@ #include /* fprintf, fopen, ftello64 */ #include /* strcmp */ #include /* log */ -#include #include #include "mem.h" @@ -96,6 +95,9 @@ typedef enum { #define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1) /* TLEN_RANGE picked manually */ +#define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } } +#define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } } + static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE }; static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; /*-************************************ @@ -104,7 +106,7 @@ static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48 typedef BYTE U8; -static double g_grillDuration_s = 99999; /* about 27 hours */ +static U32 g_timeLimit_s = 99999; /* about 27 hours */ static U32 g_nbIterations = NBLOOPS; static double g_compressibility = COMPRESSIBILITY_DEFAULT; static U32 g_blockSize = 0; @@ -166,7 +168,7 @@ void BMK_SetNbIterations(int nbLoops) *********************************************************/ /* accuracy in seconds only, span can be multiple years */ -static double BMK_timeSpan(time_t tStart) { return difftime(time(NULL), tStart); } +static U32 BMK_timeSpan(UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); } static size_t BMK_findMaxMem(U64 requiredMem) { @@ -1252,8 +1254,7 @@ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInd if( !((varArray[i] == clog_ind && strat == ZSTD_fast) || (varArray[i] == slog_ind && strat == ZSTD_fast) || (varArray[i] == slog_ind && strat == ZSTD_dfast) - || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast) - /* || varArray[i] == strt_ind */ )) { + || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { varNew[j] = varArray[i]; j++; } @@ -1645,10 +1646,10 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl BMK_printWinners(f, winners, buf.srcSize); /* start tests */ - { const time_t grillStart = time(NULL); + { const UTIL_time_t grillStart = UTIL_getTime(); do { BMK_selectRandomStart(f, winners, buf, ctx); - } while (BMK_timeSpan(grillStart) < g_grillDuration_s); + } while (BMK_timeSpan(grillStart) < g_timeLimit_s); } /* end summary */ @@ -1893,7 +1894,6 @@ static int benchMemo(BMK_result_t* resultPtr, return res; } - /* One iteration of hill climbing. Specifically, it first tries all * valid parameter configurations w/ manhattan distance 1 and picks the best one * failing that, it progressively tries candidates further and further away (up to #dim + 2) @@ -1944,6 +1944,7 @@ static winnerInfo_t climbOnce(const constraint_t target, /* all dist-1 candidates */ for(i = 0; i < varLen; i++) { for(offset = -1; offset <= 1; offset += 2) { + CHECKTIME(winnerInfo); candidateInfo.params = cparam; paramVaryOnce(varArray[i], offset, &candidateInfo.params); @@ -1953,8 +1954,7 @@ static winnerInfo_t climbOnce(const constraint_t target, strat = candidateInfo.params.strategy; varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); } - res = benchMemo(&candidateInfo.result, - buf, ctx, + res = benchMemo(&candidateInfo.result, buf, ctx, sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat], varNew, varLenNew, feas); if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ @@ -1976,6 +1976,7 @@ static winnerInfo_t climbOnce(const constraint_t target, for(dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */ for(i = 0; i < (1 << varLen) / varLen + 2; i++) { int res; + CHECKTIME(winnerInfo); candidateInfo.params = cparam; /* param error checking already done here */ paramVariation(&candidateInfo.params, varArray, varLen, dist); @@ -1985,8 +1986,7 @@ static winnerInfo_t climbOnce(const constraint_t target, varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); } - res = benchMemo(&candidateInfo.result, - buf, ctx, + res = benchMemo(&candidateInfo.result, buf, ctx, sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat], varNew, varLenNew, feas); if(res == BETTER_RESULT) { /* synonymous with better in this case*/ @@ -2058,7 +2058,7 @@ static winnerInfo_t optimizeFixedStrategy( BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); i = 0; } - + CHECKTIME(winnerInfo); i++; } return winnerInfo; @@ -2123,6 +2123,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ size_t maxBlockSize = 0; contexts_t ctx; buffers_t buf; + g_time = UTIL_getTime(); /* Init */ if(!cParamValid(paramTarget)) { @@ -2174,7 +2175,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize); } - if(!allMT) { DISPLAY("MemoTable Init Error\n"); ret = 2; @@ -2267,6 +2267,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ winner.params = CParams; } + CHECKTIMEGT(ret, 0, _cleanUp); /* if pass time limit, stop */ /* if the current params are too slow, just stop. */ if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; } } @@ -2290,6 +2291,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) { winner = w1; } + CHECKTIMEGT(ret, 0, _cleanUp); } while(st && tries > 0) { @@ -2307,6 +2309,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ st = nextStrategy(st, bestStrategy); tries -= TRY_DECAY; } + CHECKTIMEGT(ret, 0, _cleanUp); } } else { winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, @@ -2388,7 +2391,7 @@ static int usage_advanced(void) DISPLAY( " -S : Single run \n"); DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n"); DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); - DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", (U32)g_grillDuration_s, g_grillDuration_s / 3600); + DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", g_timeLimit_s, (double)g_timeLimit_s / 3600); DISPLAY( " -v : Prints Benchmarking output\n"); DISPLAY( " -D : Next argument dictionary file\n"); DISPLAY( " -s : Seperate Files\n"); @@ -2433,8 +2436,6 @@ int main(int argc, const char** argv) assert(argc>=1); /* for exename */ - g_time = UTIL_getTime(); - /* Welcome message */ DISPLAY(WELCOME_MESSAGE); @@ -2580,7 +2581,7 @@ int main(int argc, const char** argv) /* caps runtime (in seconds) */ case 't': argument++; - g_grillDuration_s = (double)readU32FromChar(&argument); + g_timeLimit_s = readU32FromChar(&argument); break; case 's': @@ -2609,6 +2610,7 @@ int main(int argc, const char** argv) /* first provided filename is input */ if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } } + if (filenamesStart==0) { if (g_optimizer) { DISPLAY("Optimizer Expects File\n"); From 3f2d024dca7723b4235618d99ce755287e530472 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 9 Aug 2018 10:42:35 -0700 Subject: [PATCH 66/92] forceAttachDict --- tests/paramgrill.c | 746 ++++++++++++++++++++++++--------------------- 1 file changed, 397 insertions(+), 349 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index af4459187..aab46ede5 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -66,22 +66,10 @@ static const int g_maxNbVariations = 64; #define CUSTOM_LEVEL 99 #define BASE_CLEVEL 1 -/* indices for each of the variables */ -typedef enum { - wlog_ind = 0, - clog_ind = 1, - hlog_ind = 2, - slog_ind = 3, - slen_ind = 4, - tlen_ind = 5, - strt_ind = 6 -} varInds_t; - -#define NUM_PARAMS 7 -/* just don't use strategy as a param. */ - #undef ZSTD_WINDOWLOG_MAX #define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. +#define FADT_MIN 0 +#define FADT_MAX ((U32)-1) #define ZSTD_TARGETLENGTH_MIN 0 #define ZSTD_TARGETLENGTH_MAX 999 @@ -93,13 +81,146 @@ typedef enum { #define SLEN_RANGE (ZSTD_SEARCHLENGTH_MAX - ZSTD_SEARCHLENGTH_MIN + 1) #define TLEN_RANGE 17 #define STRT_RANGE (ZSTD_btultra - ZSTD_fast + 1) -/* TLEN_RANGE picked manually */ +#define FADT_RANGE 3 #define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } } #define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } } -static const int rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE }; +#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt */ + +/*-************************************ +* Setup for Adding new params +**************************************/ + +/* indices for each of the variables */ +typedef enum { + wlog_ind = 0, + clog_ind = 1, + hlog_ind = 2, + slog_ind = 3, + slen_ind = 4, + tlen_ind = 5, + strt_ind = 6, + fadt_ind = 7, /* forceAttachDict */ + NUM_PARAMS = 8 +} varInds_t; + +/* maximum value of parameters */ +static const U32 mintable[NUM_PARAMS] = + { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN }; + +/* minimum value of parameters */ +static const U32 maxtable[NUM_PARAMS] = + { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_SEARCHLENGTH_MAX, ZSTD_TARGETLENGTH_MAX, ZSTD_btultra, FADT_MAX }; + +/* # of values parameters can take on */ +static const U32 rangetable[NUM_PARAMS] = + { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE }; + +static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] = + { ZSTD_p_windowLog, ZSTD_p_chainLog, ZSTD_p_hashLog, ZSTD_p_searchLog, ZSTD_p_minMatch, ZSTD_p_targetLength, ZSTD_p_compressionStrategy, ZSTD_p_forceAttachDict }; + +static const char* g_paramNames[NUM_PARAMS] = + { "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict"}; + + static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; +/* maps value from 0 to rangetable[param] - 1 to valid paramvalue */ +static U32 rangeMap(varInds_t param, U32 ind) { + ind = MIN(ind, rangetable[param] - 1); + switch(param) { + case tlen_ind: + return tlen_table[ind]; + case fadt_ind: /* 0, 1, 2 -> -1, 0, 1 */ + return ind - 1; + case wlog_ind: /* using default: triggers -Wswitch-enum */ + case clog_ind: + case hlog_ind: + case slog_ind: + case slen_ind: + case strt_ind: + return mintable[param] + ind; + case NUM_PARAMS: + return (U32)-1; + } + return 0; /* should never happen, stop compiler warnings */ +} + +/* inverse of rangeMap */ +static U32 invRangeMap(varInds_t param, U32 value) { + value = MIN(MAX(mintable[param], value), maxtable[param]); + switch(param) { + case tlen_ind: /* bin search */ + { + int lo = 0; + int hi = TLEN_RANGE; + while(lo < hi) { + int mid = (lo + hi) / 2; + if(tlen_table[mid] < value) { + lo = mid + 1; + } if(tlen_table[mid] == value) { + return mid; + } else { + hi = mid; + } + } + return lo; + } + case fadt_ind: + return value + 1; + case wlog_ind: + case clog_ind: + case hlog_ind: + case slog_ind: + case slen_ind: + case strt_ind: + return value - mintable[param]; + case NUM_PARAMS: + return (U32)-1; + } + return 0; /* should never happen, stop compiler warnings */ +} + +typedef struct { + U32 vals[NUM_PARAMS]; +} paramValues_t; + +//TODO: unset -> 0? +static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) { + ZSTD_compressionParameters c; + c.windowLog = p.vals[wlog_ind]; + c.chainLog = p.vals[clog_ind]; + c.hashLog = p.vals[hlog_ind]; + c.searchLog = p.vals[slog_ind]; + c.searchLength = p.vals[slen_ind]; + c.targetLength = p.vals[tlen_ind]; + c.strategy = p.vals[strt_ind]; + /* no forceAttachDict */ + return c; +} + +/* 0 = auto for fadt */ +static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) { + paramValues_t p; + p.vals[wlog_ind] = c.windowLog; + p.vals[clog_ind] = c.chainLog; + p.vals[hlog_ind] = c.hashLog; + p.vals[slog_ind] = c.searchLog; + p.vals[slen_ind] = c.searchLength; + p.vals[tlen_ind] = c.targetLength; + p.vals[strt_ind] = c.strategy; + p.vals[fadt_ind] = 0; + return p; +} + +/* equivalent of ZSTD_adjustCParams for paramValues_t */ +static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) { + U32 fval = p.vals[fadt_ind]; + p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize)); + p.vals[fadt_ind] = fval; + return p; +} + /*-************************************ * Benchmark Parameters/Global Variables **************************************/ @@ -115,15 +236,20 @@ static U32 g_singleRun = 0; static U32 g_optimizer = 0; static U32 g_target = 0; static U32 g_noSeed = 0; -static ZSTD_compressionParameters g_params; /* Initialized at the beginning of main w/ emptyParams() function */ +static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ typedef struct { BMK_result_t result; - ZSTD_compressionParameters params; + paramValues_t params; } winnerInfo_t; +typedef struct { + BMK_result_t result; + ZSTD_compressionParameters params; +} oldWinnerInfo_t; + typedef struct { U32 cSpeed; /* bytes / sec */ U32 dSpeed; @@ -242,8 +368,9 @@ static void findClockGranularity(void) { DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity); } +/* allows zeros */ #define CLAMPCHECK(val,min,max) { \ - if (val && (((val)<(min)) | ((val)>(max)))) { \ + if (((val)<(min)) | ((val)>(max))) { \ DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \ return 0; \ } } @@ -251,36 +378,53 @@ static void findClockGranularity(void) { /* Like ZSTD_checkCParams() but allows 0's */ /* no check on targetLen? */ -static int cParamValid(ZSTD_compressionParameters paramTarget) { - CLAMPCHECK(paramTarget.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); - CLAMPCHECK(paramTarget.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); - CLAMPCHECK(paramTarget.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); - CLAMPCHECK(paramTarget.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); - CLAMPCHECK(paramTarget.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); - if(paramTarget.targetLength > ZSTD_TARGETLENGTH_MAX) { - DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); - return 0; - } - if(paramTarget.strategy > ZSTD_btultra) { - DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); - return 0; +static int paramValid(paramValues_t paramTarget) { + U32 i; + for(i = 0; i < NUM_PARAMS; i++) { + CLAMPCHECK(paramTarget.vals[i], mintable[i], maxtable[i]); } + //TODO: Strategy could be valid at 0 before, is that right? return 1; } -static void cParamZeroMin(ZSTD_compressionParameters* paramTarget) { - paramTarget->windowLog = paramTarget->windowLog ? paramTarget->windowLog : ZSTD_WINDOWLOG_MIN; - paramTarget->searchLog = paramTarget->searchLog ? paramTarget->searchLog : ZSTD_SEARCHLOG_MIN; - paramTarget->chainLog = paramTarget->chainLog ? paramTarget->chainLog : ZSTD_CHAINLOG_MIN; - paramTarget->hashLog = paramTarget->hashLog ? paramTarget->hashLog : ZSTD_HASHLOG_MIN; - paramTarget->searchLength = paramTarget->searchLength ? paramTarget->searchLength : ZSTD_SEARCHLENGTH_MIN; - paramTarget->targetLength = paramTarget->targetLength ? paramTarget->targetLength : 0; +//TODO: doesn't affect strategy? +static paramValues_t cParamUnsetMin(paramValues_t paramTarget) { + varInds_t i; + for(i = 0; i < NUM_PARAMS; i++) { + if(paramTarget.vals[i] == PARAM_UNSET) { + paramTarget.vals[i] = mintable[i]; + } + } + return paramTarget; } -static void BMK_translateAdvancedParams(const ZSTD_compressionParameters params) -{ - DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); +static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { + U32 i; + fprintf(f,"--zstd="); + for(i = 0; i < NUM_PARAMS; i++) { + fprintf(f,"%s", g_paramNames[i]); + fprintf(f,"=%u", params.vals[i]); + if(i != NUM_PARAMS - 1) { + fprintf(f, ","); + } + } + fprintf(f, "\n"); +} + + +static const char* g_stratName[ZSTD_btultra+1] = { + "(none) ", "ZSTD_fast ", "ZSTD_dfast ", + "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", + "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; + +static void BMK_displayOneResult(FILE* f, winnerInfo_t res, size_t srcSize) { + res.params = cParamUnsetMin(res.params); + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u,%3d, %s}, ", + res.params.vals[wlog_ind], res.params.vals[clog_ind], res.params.vals[hlog_ind], res.params.vals[slog_ind], res.params.vals[slen_ind], + res.params.vals[tlen_ind], (int)res.params.vals[fadt_ind], g_stratName[res.params.vals[strt_ind]]); + fprintf(f, + " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB)); } /* checks results are feasible */ @@ -343,17 +487,16 @@ static constraint_t relaxTarget(constraint_t target) { * Bench functions *********************************************************/ -const char* g_stratName[ZSTD_btultra+1] = { - "(none) ", "ZSTD_fast ", "ZSTD_dfast ", - "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", - "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; - -static ZSTD_compressionParameters emptyParams(void) { - ZSTD_compressionParameters p = { 0, 0, 0, 0, 0, 0, (ZSTD_strategy)0 }; +static paramValues_t emptyParams(void) { + U32 i; + paramValues_t p; + for(i = 0; i < NUM_PARAMS; i++) { + p.vals[i] = PARAM_UNSET; + } return p; } -static winnerInfo_t initWinnerInfo(ZSTD_compressionParameters p) { +static winnerInfo_t initWinnerInfo(paramValues_t p) { winnerInfo_t w1; w1.result.cSpeed = 0.; w1.result.dSpeed = 0.; @@ -389,27 +532,16 @@ typedef struct { static void BMK_initCCtx(ZSTD_CCtx* ctx, const void* dictBuffer, const size_t dictBufferSize, const int cLevel, - const ZSTD_compressionParameters* comprParams, const BMK_advancedParams_t* adv) { + const paramValues_t* comprParams) { + varInds_t i; ZSTD_CCtx_reset(ctx); ZSTD_CCtx_resetParameters(ctx); - if (adv->nbWorkers==1) { - ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, 0); - } else { - ZSTD_CCtx_setParameter(ctx, ZSTD_p_nbWorkers, adv->nbWorkers); - } ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_enableLongDistanceMatching, adv->ldmFlag); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmMinMatch, adv->ldmMinMatch); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashLog, adv->ldmHashLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmBucketSizeLog, adv->ldmBucketSizeLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_ldmHashEveryLog, adv->ldmHashEveryLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_windowLog, comprParams->windowLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_hashLog, comprParams->hashLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_chainLog, comprParams->chainLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_searchLog, comprParams->searchLog); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_minMatch, comprParams->searchLength); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_targetLength, comprParams->targetLength); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionStrategy, comprParams->strategy); + + for(i = 0; i < NUM_PARAMS; i++) { + if(comprParams->vals[i] != PARAM_UNSET) + ZSTD_CCtx_setParameter(ctx, cctxSetParamTable[i], comprParams->vals[i]); + } ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); } @@ -424,13 +556,12 @@ typedef struct { const void* dictBuffer; size_t dictBufferSize; int cLevel; - const ZSTD_compressionParameters* comprParams; - const BMK_advancedParams_t* adv; + const paramValues_t* comprParams; } BMK_initCCtxArgs; static size_t local_initCCtx(void* payload) { const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload; - BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv); + BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams); return 0; } @@ -507,10 +638,7 @@ static size_t local_defaultDecompress( * From bench.c End *********************************************************/ -static void freeBuffers(const buffers_t b) { - if(b.srcPtrs != NULL) { - free(b.srcBuffer); - } +static void freeNonSrcBuffers(const buffers_t b) { free(b.srcPtrs); free(b.srcSizes); @@ -528,6 +656,13 @@ static void freeBuffers(const buffers_t b) { free(b.resSizes); } +static void freeBuffers(const buffers_t b) { + if(b.srcPtrs != NULL) { + free(b.srcBuffer); + } + freeNonSrcBuffers(b); +} + /* srcBuffer will be freed by freeBuffers now */ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbFiles, const size_t* fileSizes) @@ -559,7 +694,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) { DISPLAY("alloc error\n"); - freeBuffers(*buff); + freeNonSrcBuffers(*buff); return 1; } @@ -570,7 +705,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF if(!buff->dstPtrs[0] || !buff->resPtrs[0]) { DISPLAY("alloc error\n"); - freeBuffers(*buff); + freeNonSrcBuffers(*buff); return 1; } @@ -611,12 +746,20 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab size_t totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); size_t* fileSizes = calloc(sizeof(size_t), nbFiles); - void* srcBuffer = malloc(benchedSize); + void* srcBuffer = NULL; int ret = 0; + if(!totalSizeToLoad || !benchedSize) { + ret = 1; + DISPLAY("Nothing to Bench\n"); + goto _cleanUp; + } + + srcBuffer = malloc(benchedSize); if(!fileSizes || !srcBuffer) { - return 1; + ret = 1; + goto _cleanUp; } for(n = 0; n < nbFiles; n++) { @@ -633,10 +776,9 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab f = fopen(fileNamesTable[n], "rb"); if (f==NULL) { DISPLAY("impossible to open file %s\n", fileNamesTable[n]); - free(fileSizes); - free(srcBuffer); fclose(f); - return 10; + ret = 10; + goto _cleanUp; } DISPLAY("Loading %s... \r", fileNamesTable[n]); @@ -645,21 +787,22 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab { char* buffer = (char*)(srcBuffer); size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); - + fclose(f); if (readSize != (size_t)fileSize) { /* should we accept partial read? */ DISPLAY("could not read %s", fileNamesTable[n]); - free(fileSizes); - free(srcBuffer); - return 1; + ret = 1; + goto _cleanUp; } fileSizes[n] = readSize; pos += readSize; } - fclose(f); } ret = createBuffersFromMemory(buff, srcBuffer, nbFiles, fileSizes); + +_cleanUp: + if(ret) { free(srcBuffer); } free(fileSizes); return ret; } @@ -717,7 +860,7 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) { /* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ /* dictionary nullable, nothing else though. */ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx, - const int cLevel, const ZSTD_compressionParameters* comprParams, + const int cLevel, const paramValues_t* comprParams, const BMK_mode_t mode, const BMK_loopMode_t loopMode, const unsigned nbSeconds) { U32 i; @@ -736,11 +879,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ZSTD_CCtx* cctx = ctx.cctx; ZSTD_DCtx* dctx = ctx.dctx; - BMK_advancedParams_t adv = BMK_initAdvancedParams(); - adv.mode = mode; - adv.loopMode = loopMode; - adv.nbSeconds = nbSeconds; - /* warmimg up memory */ /* can't do this if decode only */ for(i = 0; i < buf.nbBlocks; i++) { @@ -762,7 +900,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t cctxprep.dictBufferSize = dictBufferSize; cctxprep.cLevel = cLevel; cctxprep.comprParams = comprParams; - cctxprep.adv = &adv; dctxprep.dctx = dctx; dctxprep.dictBuffer = dictBuffer; dctxprep.dictBufferSize = dictBufferSize; @@ -852,13 +989,13 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t } } /* Bench */ - results.result.cMem = (1 << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx); + results.result.cMem = (1 << (comprParams->vals[wlog_ind])) + ZSTD_sizeof_CCtx(cctx); return results; } static int BMK_benchParam(BMK_result_t* resultPtr, buffers_t buf, contexts_t ctx, - const ZSTD_compressionParameters cParams) { + const paramValues_t cParams) { BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3); *resultPtr = res.result; return res.error; @@ -986,23 +1123,21 @@ static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { /* Writes to f the results of a parameter benchmark */ /* when used with --optimize, will only print results better than previously discovered */ -static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const size_t srcSize) +static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize) { char lvlstr[15] = "Custom Level"; + winnerInfo_t w; + w.params = params; + w.result = result; fprintf(f, "\r%79s\r", ""); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, - params.targetLength, g_stratName[(U32)(params.strategy)]); - if(cLevel != CUSTOM_LEVEL) { snprintf(lvlstr, 15, " Level %2u ", cLevel); } - fprintf(f, - "/* %s */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */", - lvlstr, (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); + fprintf(f, "/* %s */ ", lvlstr); + BMK_displayOneResult(f, w, srcSize); if(TIMED) { const U64 time = UTIL_clockSpanNano(g_time); @@ -1012,11 +1147,10 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result fprintf(f, "\n"); } -static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const ZSTD_compressionParameters params, const constraint_t targetConstraints, const size_t srcSize) +static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize) { /* global winner used for constraints */ - static winnerInfo_t g_winner = { { 0, 0, (size_t)-1, (size_t)-1 } , { 0, 0, 0, 0, 0, 0, ZSTD_fast } }; - + static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } }; if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); @@ -1025,7 +1159,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res BMK_printWinner(f, cLevel, result, params, srcSize); if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { - BMK_translateAdvancedParams(params); + BMK_translateAdvancedParams(f, params); g_winner.result = result; g_winner.params = params; } @@ -1046,13 +1180,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res fprintf(f, "================================\n"); for(n = g_winners; n != NULL; n = n->next) { fprintf(f, "\r%79s\r", ""); - - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - n->res.params.windowLog, n->res.params.chainLog, n->res.params.hashLog, n->res.params.searchLog, n->res.params.searchLength, - n->res.params.targetLength, g_stratName[(U32)(n->res.params.strategy)]); - fprintf(f, - " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / n->res.result.cSize, (double)n->res.result.cSpeed / (1 MB), (double)n->res.result.dSpeed / (1 MB)); + BMK_displayOneResult(f, n->res, srcSize); } fprintf(f, "================================\n"); fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n", @@ -1060,27 +1188,15 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res fprintf(f, "Overall Winner: \n"); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - g_winner.params.windowLog, g_winner.params.chainLog, g_winner.params.hashLog, g_winner.params.searchLog, g_winner.params.searchLength, - g_winner.params.targetLength, g_stratName[(U32)(g_winner.params.strategy)]); - fprintf(f, - " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / g_winner.result.cSize, (double)g_winner.result.cSpeed / (1 MB), (double)g_winner.result.dSpeed / (1 MB)); - - BMK_translateAdvancedParams(g_winner.params); - - fprintf(f, "Latest BMK: \n"); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", - params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, - params.targetLength, g_stratName[(U32)(params.strategy)]); - fprintf(f, - " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / result.cSize, (double)result.cSpeed / (1 MB), (double)result.dSpeed / (1 MB)); + BMK_displayOneResult(f, g_winner, srcSize); + BMK_translateAdvancedParams(f, g_winner.params); + fprintf(f, "Latest BMK: \n");\ + BMK_displayOneResult(f, w, srcSize); } } -static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) +static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize) { int cLevel; @@ -1088,11 +1204,11 @@ static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSi fprintf(f, " /* W, C, H, S, L, T, strat */ \n"); for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++) - BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize); + BMK_printWinner(f, cLevel, winners[cLevel].result, cParamsToPVals(winners[cLevel].params), srcSize); } -static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSize) +static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize) { fseek(f, 0, SEEK_SET); BMK_printWinners2(f, winners, srcSize); @@ -1129,14 +1245,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1) } } } -static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, +static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters params, buffers_t buf, contexts_t ctx) { BMK_result_t testResult; int better = 0; int cLevel; - BMK_benchParam(&testResult, buf, ctx, params); + BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(params)); for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { @@ -1152,7 +1268,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* first solution for this cLevel */ winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); + BMK_printWinner(stdout, cLevel, testResult, cParamsToPVals(params), buf.srcSize); better = 1; continue; } @@ -1217,7 +1333,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); + BMK_printWinner(stdout, cLevel, testResult, cParamsToPVals(params), buf.srcSize); better = 1; } } @@ -1234,20 +1350,21 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters para /* no point in windowLog < chainLog (no point 2x chainLog for bt) */ /* now with built in bounds-checking */ /* no longer does anything with sanitizeVarArray + clampcheck */ -static ZSTD_compressionParameters sanitizeParams(ZSTD_compressionParameters params) +static paramValues_t sanitizeParams(paramValues_t params) { - if (params.strategy == ZSTD_fast) - params.chainLog = 0, params.searchLog = 0; - if (params.strategy == ZSTD_dfast) - params.searchLog = 0; - if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra && params.strategy != ZSTD_fast) - params.targetLength = 0; + if (params.vals[strt_ind] == ZSTD_fast) + params.vals[clog_ind] = 0, params.vals[slog_ind] = 0; + if (params.vals[strt_ind] == ZSTD_dfast) + params.vals[slog_ind] = 0; + if (params.vals[strt_ind] != ZSTD_btopt && params.vals[strt_ind] != ZSTD_btultra && params.vals[strt_ind] != ZSTD_fast) + params.vals[tlen_ind] = 0; return params; } -/* new length */ +/* return: new length */ /* keep old array, will need if iter over strategy. */ +/* prunes useless params */ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInds_t* varArray, const ZSTD_strategy strat) { int i, j = 0; for(i = 0; i < varLength; i++) { @@ -1264,80 +1381,31 @@ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInd } /* res should be NUM_PARAMS size */ -/* constructs varArray from ZSTD_compressionParameters style parameter */ -static int variableParams(const ZSTD_compressionParameters paramConstraints, varInds_t* res) { +/* constructs varArray from paramValues_t style parameter */ +/* pass in using dict. */ +static int variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) { + varInds_t i; int j = 0; - if(!paramConstraints.windowLog) { - res[j] = wlog_ind; - j++; - } - if(!paramConstraints.chainLog) { - res[j] = clog_ind; - j++; - } - if(!paramConstraints.hashLog) { - res[j] = hlog_ind; - j++; - } - if(!paramConstraints.searchLog) { - res[j] = slog_ind; - j++; - } - if(!paramConstraints.searchLength) { - res[j] = slen_ind; - j++; - } - if(!paramConstraints.targetLength) { - res[j] = tlen_ind; - j++; - } - if(!paramConstraints.strategy) { - res[j] = strt_ind; - j++; - } - return j; -} - -/* bin-search on tlen_table for correct index. */ -static int tlen_inv(U32 x) { - int lo = 0; - int hi = TLEN_RANGE; - while(lo < hi) { - int mid = (lo + hi) / 2; - if(tlen_table[mid] < x) { - lo = mid + 1; - } if(tlen_table[mid] == x) { - return mid; - } else { - hi = mid; + for(i = 0; i < NUM_PARAMS; i++) { + if(paramConstraints.vals[i] == PARAM_UNSET) { + if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */ + res[j] = i; j++; } } - return lo; + return j; } /* amt will probably always be \pm 1? */ /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ /* take max/min bounds into account as well? */ -static void paramVaryOnce(const varInds_t paramIndex, const int amt, ZSTD_compressionParameters* ptr) { - switch(paramIndex) - { - case wlog_ind: ptr->windowLog += amt; break; - case clog_ind: ptr->chainLog += amt; break; - case hlog_ind: ptr->hashLog += amt; break; - case slog_ind: ptr->searchLog += amt; break; - case slen_ind: ptr->searchLength += amt; break; - case tlen_ind: - ptr->targetLength = tlen_table[MAX(0, MIN(TLEN_RANGE - 1, tlen_inv(ptr->targetLength) + amt))]; - break; - case strt_ind: ptr->strategy += amt; break; - default: break; - } +static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) { + ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); //TODO: bounds check. } /* varies ptr by nbChanges respecting varyParams*/ -static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) +static void paramVariation(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) { - ZSTD_compressionParameters p; + paramValues_t p; U32 validated = 0; while (!validated) { U32 i; @@ -1346,7 +1414,7 @@ static void paramVariation(ZSTD_compressionParameters* ptr, const varInds_t* var const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } - validated = !ZSTD_isError(ZSTD_checkCParams(p)) && p.strategy > 0; + validated = paramValid(p); } *ptr = p; } @@ -1364,48 +1432,25 @@ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { } /* returns unique index in memotable of compression parameters */ -static unsigned memoTableInd(const ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen) { +static unsigned memoTableInd(const paramValues_t* ptr, const varInds_t* varyParams, const int varyLen) { int i; unsigned ind = 0; for(i = 0; i < varyLen; i++) { - switch(varyParams[i]) { - case wlog_ind: ind *= WLOG_RANGE; ind += ptr->windowLog - - ZSTD_WINDOWLOG_MIN ; break; - case clog_ind: ind *= CLOG_RANGE; ind += ptr->chainLog - - ZSTD_CHAINLOG_MIN ; break; - case hlog_ind: ind *= HLOG_RANGE; ind += ptr->hashLog - - ZSTD_HASHLOG_MIN ; break; - case slog_ind: ind *= SLOG_RANGE; ind += ptr->searchLog - - ZSTD_SEARCHLOG_MIN ; break; - case slen_ind: ind *= SLEN_RANGE; ind += ptr->searchLength - - ZSTD_SEARCHLENGTH_MIN; break; - case tlen_ind: ind *= TLEN_RANGE; ind += tlen_inv(ptr->targetLength) - - ZSTD_TARGETLENGTH_MIN; break; - case strt_ind: break; - } + varInds_t v = varyParams[i]; + if(v == strt_ind) continue; /* exclude strategy from memotable */ + ind *= rangetable[v]; ind += invRangeMap(v, ptr->vals[v]); } return ind; } /* inverse of above function (from index to parameters) */ -static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) { +static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) { int i; for(i = varyLen - 1; i >= 0; i--) { - switch(varyParams[i]) { - case wlog_ind: ptr->windowLog = ind % WLOG_RANGE + ZSTD_WINDOWLOG_MIN; - ind /= WLOG_RANGE; break; - case clog_ind: ptr->chainLog = ind % CLOG_RANGE + ZSTD_CHAINLOG_MIN; - ind /= CLOG_RANGE; break; - case hlog_ind: ptr->hashLog = ind % HLOG_RANGE + ZSTD_HASHLOG_MIN; - ind /= HLOG_RANGE; break; - case slog_ind: ptr->searchLog = ind % SLOG_RANGE + ZSTD_SEARCHLOG_MIN; - ind /= SLOG_RANGE; break; - case slen_ind: ptr->searchLength = ind % SLEN_RANGE + ZSTD_SEARCHLENGTH_MIN; - ind /= SLEN_RANGE; break; - case tlen_ind: ptr->targetLength = tlen_table[(ind % TLEN_RANGE)]; - ind /= TLEN_RANGE; break; - case strt_ind: break; - } + varInds_t v = varyParams[i]; + if(v == strt_ind) continue; + ptr->vals[v] = rangeMap(v, ind % rangetable[v]); + ind /= rangetable[v]; } } @@ -1414,36 +1459,36 @@ static void memoTableIndInv(ZSTD_compressionParameters* ptr, const varInds_t* va * redundant / obviously non-optimal parameter configurations (e.g. wlog - 1 larger) * than srcSize, clog > wlog, ... */ -static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +static void initMemoTable(U8* memoTable, paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); - int cwFixed = !paramConstraints.chainLog || !paramConstraints.windowLog; - int scFixed = !paramConstraints.searchLog || !paramConstraints.chainLog; - int whFixed = !paramConstraints.windowLog || !paramConstraints.hashLog; - int wFixed = !paramConstraints.windowLog; + int cwFixed = paramConstraints.vals[clog_ind] == PARAM_UNSET || paramConstraints.vals[wlog_ind] == PARAM_UNSET; + int scFixed = paramConstraints.vals[slog_ind] == PARAM_UNSET || paramConstraints.vals[clog_ind] == PARAM_UNSET; + int whFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET || paramConstraints.vals[hlog_ind] == PARAM_UNSET; + int wFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET; int j = 0; assert(memoTable != NULL); memset(memoTable, 0, arrayLen); - cParamZeroMin(¶mConstraints); + paramConstraints = cParamUnsetMin(paramConstraints); for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - if(ZSTD_estimateCStreamSize_usingCParams(paramConstraints) > (size_t)target.cMem) { + if(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramConstraints)) > (size_t)target.cMem) { memoTable[i] = 255; j++; } - if(wFixed && (1ULL << (paramConstraints.windowLog - 1)) > srcSize) { + if(wFixed && (1ULL << (paramConstraints.vals[wlog_ind] - 1)) >= srcSize && paramConstraints.vals[wlog_ind] != mintable[wlog_ind]) { memoTable[i] = 255; } /* nil out parameter sets equivalent to others. */ - if(cwFixed/* at most least 1 param fixed. */) { - if(paramConstraints.strategy == ZSTD_btlazy2 || paramConstraints.strategy == ZSTD_btopt || paramConstraints.strategy == ZSTD_btultra) { - if(paramConstraints.chainLog > paramConstraints.windowLog + 1) { + if(cwFixed) { + if(paramConstraints.vals[strt_ind] == ZSTD_btlazy2 || paramConstraints.vals[strt_ind] == ZSTD_btopt || paramConstraints.vals[strt_ind] == ZSTD_btultra) { + if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]+ 1) { if(memoTable[i] != 255) { j++; } memoTable[i] = 255; } } else { - if(paramConstraints.chainLog > paramConstraints.windowLog) { + if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]) { if(memoTable[i] != 255) { j++; } memoTable[i] = 255; } @@ -1451,14 +1496,14 @@ static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstra } if(scFixed) { - if(paramConstraints.searchLog > paramConstraints.chainLog) { + if(paramConstraints.vals[slog_ind] > paramConstraints.vals[clog_ind]) { if(memoTable[i] != 255) { j++; } memoTable[i] = 255; } } if(whFixed) { - if(paramConstraints.hashLog > paramConstraints.windowLog + 1) { + if(paramConstraints.vals[hlog_ind] > paramConstraints.vals[wlog_ind] + 1) { if(memoTable[i] != 255) { j++; } memoTable[i] = 255; } @@ -1466,7 +1511,7 @@ static void initMemoTable(U8* memoTable, ZSTD_compressionParameters paramConstra } DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i); if((int)i == j) { - DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.strategy) + DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.vals[strt_ind]); } } @@ -1482,7 +1527,7 @@ static void freeMemoTableArray(U8** mtAll) { /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ -static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +static U8** createMemoTableArray(paramValues_t paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { varInds_t varNew[NUM_PARAMS]; U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1)); int i; @@ -1503,14 +1548,13 @@ static U8** createMemoTableArray(ZSTD_compressionParameters paramConstraints, co return mtAll; } -static ZSTD_compressionParameters overwriteParams(ZSTD_compressionParameters base, ZSTD_compressionParameters mask) { - base.windowLog = mask.windowLog ? mask.windowLog : base.windowLog; - base.chainLog = mask.chainLog ? mask.chainLog : base.chainLog; - base.hashLog = mask.hashLog ? mask.hashLog : base.hashLog; - base.searchLog = mask.searchLog ? mask.searchLog : base.searchLog; - base.searchLength = mask.searchLength ? mask.searchLength : base.searchLength; - base.targetLength = mask.targetLength ? mask.targetLength : base.targetLength; - base.strategy = mask.strategy ? mask.strategy : base.strategy; +static paramValues_t overwriteParams(paramValues_t base, paramValues_t mask) { + U32 i; + for(i = 0; i < NUM_PARAMS; i++) { + if(mask.vals[i] != PARAM_UNSET) { + base.vals[i] = mask.vals[i]; + } + } return base; } @@ -1524,38 +1568,41 @@ static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */ g_alreadyTested[(XXH64(((void*)&sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK] */ static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { - ZSTD_compressionParameters p2 = sanitizeParams(p); + ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(cParamsToPVals(p))); return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK]; } -static void playAround(FILE* f, winnerInfo_t* winners, +static void playAround(FILE* f, oldWinnerInfo_t* winners, ZSTD_compressionParameters params, buffers_t buf, contexts_t ctx) { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); - const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; + const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; /* no fadt */ while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { - ZSTD_compressionParameters p = params; + paramValues_t p = cParamsToPVals(params); + ZSTD_compressionParameters p2; BYTE* b; if (nbVariations++ > g_maxNbVariations) break; paramVariation(&p, unconstrained, NUM_PARAMS, 4); + p2 = pvalsToCParams(p); + /* exclude faster if already played params */ - if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) + if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p2))-1)) continue; /* test */ - b = NB_TESTS_PLAYED(p); + b = NB_TESTS_PLAYED(p2); (*b)++; - if (!BMK_seed(winners, p, buf, ctx)) continue; + if (!BMK_seed(winners, p2, buf, ctx)) continue; /* improvement found => search more */ BMK_printWinners(f, winners, buf.srcSize); - playAround(f, winners, p, buf, ctx); + playAround(f, winners, p2, buf, ctx); } } @@ -1587,7 +1634,7 @@ static ZSTD_compressionParameters randomParams(void) } /* Sets pc to random unmeasured set of parameters */ -static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* varArray, int varLen, U8* memoTable) +static void randomConstrainedParams(paramValues_t* pc, varInds_t* varArray, int varLen, U8* memoTable) { size_t tries = memoTableLen(varArray, varLen); const size_t maxSize = memoTableLen(varArray, varLen); @@ -1601,7 +1648,7 @@ static void randomConstrainedParams(ZSTD_compressionParameters* pc, varInds_t* v } static void BMK_selectRandomStart( - FILE* f, winnerInfo_t* winners, + FILE* f, oldWinnerInfo_t* winners, buffers_t buf, contexts_t ctx) { U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); @@ -1617,7 +1664,7 @@ static void BMK_selectRandomStart( static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize) { ZSTD_compressionParameters params; - winnerInfo_t winners[NB_LEVELS_TRACKED+1]; + oldWinnerInfo_t winners[NB_LEVELS_TRACKED+1]; const char* const rfName = "grillResults.txt"; FILE* const f = fopen(rfName, "w"); @@ -1632,7 +1679,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl /* baseline config for level 1 */ ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); BMK_result_t testResult; - BMK_benchParam(&testResult, buf, ctx, l1params); + BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(l1params)); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); } @@ -1751,7 +1798,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam DISPLAY("using %d Files : \n", nbFiles); } - g_params = ZSTD_adjustCParams(overwriteParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); + g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize); if(g_singleRun) { ret = benchOnce(buf, ctx); @@ -1769,7 +1816,7 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam #define HIGH_VARIANCE 100.0 static int allBench(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, - const ZSTD_compressionParameters cParams, + const paramValues_t cParams, const constraint_t target, BMK_result_t* winnerResult, int feas) { BMK_return_t benchres; @@ -1870,7 +1917,7 @@ static int allBench(BMK_result_t* resultPtr, /* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ static int benchMemo(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, - const ZSTD_compressionParameters cParams, + const paramValues_t cParams, const constraint_t target, BMK_result_t* winnerResult, U8* const memoTable, const varInds_t* varyParams, const int varyLen, const int feas) { @@ -1914,13 +1961,13 @@ static winnerInfo_t climbOnce(const constraint_t target, const varInds_t* varArray, const int varLen, ZSTD_strategy strat, U8** memoTableArray, buffers_t buf, contexts_t ctx, - const ZSTD_compressionParameters init) { + const paramValues_t init) { /* * cparam - currently considered 'center' * candidate - params to benchmark/results * winner - best option found so far. */ - ZSTD_compressionParameters cparam = init; + paramValues_t cparam = init; winnerInfo_t candidateInfo, winnerInfo; int better = 1; int feas = 0; @@ -1948,10 +1995,10 @@ static winnerInfo_t climbOnce(const constraint_t target, candidateInfo.params = cparam; paramVaryOnce(varArray[i], offset, &candidateInfo.params); - if(!ZSTD_isError(ZSTD_checkCParams(candidateInfo.params)) && candidateInfo.params.strategy > 0) { + if(paramValid(candidateInfo.params)) { int res; - if(strat != candidateInfo.params.strategy) { /* maybe only try strategy switching after exhausting non-switching solutions? */ - strat = candidateInfo.params.strategy; + if(strat != candidateInfo.params.vals[strt_ind]) { /* maybe only try strategy switching after exhausting non-switching solutions? */ + strat = candidateInfo.params.vals[strt_ind]; varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); } res = benchMemo(&candidateInfo.result, buf, ctx, @@ -1981,8 +2028,8 @@ static winnerInfo_t climbOnce(const constraint_t target, /* param error checking already done here */ paramVariation(&candidateInfo.params, varArray, varLen, dist); - if(strat != candidateInfo.params.strategy) { - strat = candidateInfo.params.strategy; + if(strat != candidateInfo.params.vals[strt_ind]) { + strat = candidateInfo.params.vals[strt_ind]; varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); } @@ -2030,7 +2077,7 @@ static winnerInfo_t climbOnce(const constraint_t target, */ static winnerInfo_t optimizeFixedStrategy( const buffers_t buf, const contexts_t ctx, - const constraint_t target, ZSTD_compressionParameters paramTarget, + const constraint_t target, paramValues_t paramTarget, const ZSTD_strategy strat, const varInds_t* varArray, const int varLen, U8** memoTableArray, const int tries) { @@ -2038,14 +2085,14 @@ static winnerInfo_t optimizeFixedStrategy( varInds_t varNew[NUM_PARAMS]; int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); - ZSTD_compressionParameters init; + paramValues_t init; winnerInfo_t winnerInfo, candidateInfo; winnerInfo = initWinnerInfo(emptyParams()); /* so climb is given the right fixed strategy */ - paramTarget.strategy = strat; + paramTarget.vals[strt_ind] = strat; /* to pass ZSTD_checkCParams */ - cParamZeroMin(¶mTarget); + paramTarget = cParamUnsetMin(paramTarget); init = paramTarget; @@ -2112,25 +2159,23 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { static int g_maxTries = 3; #define TRY_DECAY 1 -static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, ZSTD_compressionParameters paramTarget, int cLevel) +static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget, int cLevelOpt, int cLevelRun) { varInds_t varArray [NUM_PARAMS]; int ret = 0; - const int varLen = variableParams(paramTarget, varArray); + const int varLen = variableParams(paramTarget, varArray, dictFileName != NULL); winnerInfo_t winner = initWinnerInfo(emptyParams()); U8** allMT = NULL; - size_t k; - size_t maxBlockSize = 0; + paramValues_t paramBase = cParamUnsetMin(paramTarget); + size_t k, maxBlockSize = 0; contexts_t ctx; buffers_t buf; g_time = UTIL_getTime(); - /* Init */ - if(!cParamValid(paramTarget)) { + if(!paramValid(paramBase)) { return 1; } - /* load dictionary*/ if(createBuffers(&buf, fileNamesTable, nbFiles)) { DISPLAY("unable to load files\n"); return 1; @@ -2154,23 +2199,23 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* if strategy is fixed, only init that part of memotable */ - if(paramTarget.strategy) { + if(paramTarget.vals[strt_ind] != PARAM_UNSET) { varInds_t varNew[NUM_PARAMS]; - int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.strategy); + int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.vals[strt_ind]); allMT = (U8**)calloc(sizeof(U8*), (ZSTD_btultra + 1)); if(allMT == NULL) { ret = 57; goto _cleanUp; } - allMT[paramTarget.strategy] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + allMT[paramTarget.vals[strt_ind]] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); - if(allMT[paramTarget.strategy] == NULL) { + if(allMT[paramTarget.vals[strt_ind]] == NULL) { ret = 58; goto _cleanUp; } - initMemoTable(allMT[paramTarget.strategy], paramTarget, target, varNew, varLenNew, maxBlockSize); + initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, maxBlockSize); } else { allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize); } @@ -2180,7 +2225,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ ret = 2; goto _cleanUp; } - + /* default strictness = Maximum for */ if(g_strictness == DEFAULT_STRICTNESS) { if(g_optmode) { @@ -2199,7 +2244,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* use level'ing mode instead of normal target mode */ /* Should lvl be parameter-masked here? */ if(g_optmode) { - winner.params = ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize); + winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, maxBlockSize, ctx.dictSize)); if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { ret = 3; goto _cleanUp; @@ -2213,13 +2258,13 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ target.cSpeed = (U32)g_lvltarget.cSpeed; target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is reasonable. - BMK_printWinnerOpt(stdout, cLevel, winner.result, winner.params, target, buf.srcSize); + BMK_printWinnerOpt(stdout, cLevelOpt, winner.result, winner.params, target, buf.srcSize); } /* Don't want it to return anything worse than the best known result */ if(g_singleRun) { BMK_result_t res; - g_params = ZSTD_adjustCParams(maskParams(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize), g_params), maxBlockSize, ctx.dictSize); + g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize); if(BMK_benchParam(&res, buf, ctx, g_params)) { ret = 45; goto _cleanUp; @@ -2246,19 +2291,19 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ findClockGranularity(); { - ZSTD_compressionParameters CParams; + paramValues_t CParams; /* find best solution from default params */ { /* strategy selection */ const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); DEBUGOUTPUT("Strategy Selection\n"); - if(paramTarget.strategy == 0) { + if(paramTarget.vals[strt_ind] == PARAM_UNSET) { BMK_result_t candidate; int i; for (i=1; i<=maxSeeds; i++) { int ec; - CParams = overwriteParams(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize), paramTarget); + CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize)), paramTarget); ec = BMK_benchParam(&candidate, buf, ctx, CParams); BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize); @@ -2271,17 +2316,18 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* if the current params are too slow, just stop. */ if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; } } + + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); + BMK_translateAdvancedParams(stdout, winner.params); } } - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); - BMK_translateAdvancedParams(winner.params); DEBUGOUTPUT("Real Opt\n"); /* start 'real' tests */ { - int bestStrategy = (int)winner.params.strategy; - if(paramTarget.strategy == 0) { - int st = (int)winner.params.strategy; + int bestStrategy = (int)winner.params.vals[strt_ind]; + if(paramTarget.vals[strt_ind] == PARAM_UNSET) { + int st = bestStrategy; int tries = g_maxTries; { @@ -2298,7 +2344,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ winnerInfo_t wc; DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); - wc = optimizeFixedStrategy(buf, ctx, target, paramTarget, + wc = optimizeFixedStrategy(buf, ctx, target, paramBase, st, varArray, varLen, allMT, tries); if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { @@ -2312,7 +2358,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ CHECKTIMEGT(ret, 0, _cleanUp); } } else { - winner = optimizeFixedStrategy(buf, ctx, target, paramTarget, paramTarget.strategy, + winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], varArray, varLen, allMT, g_maxTries); } @@ -2326,7 +2372,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* end summary */ BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); - BMK_translateAdvancedParams(winner.params); + BMK_translateAdvancedParams(stdout, winner.params); DISPLAY("grillParams size - optimizer completed \n"); } @@ -2350,7 +2396,9 @@ static void errorOut(const char* msg) static unsigned readU32FromChar(const char** stringPtr) { const char errorMsg[] = "error: numeric value too large"; + unsigned sign = 1; unsigned result = 0; + if(**stringPtr == '-') { sign = (unsigned)-1; (*stringPtr)++; } while ((**stringPtr >='0') && (**stringPtr <='9')) { unsigned const max = (((unsigned)(-1)) / 10) - 1; if (result > max) errorOut(errorMsg); @@ -2368,7 +2416,7 @@ static unsigned readU32FromChar(const char** stringPtr) if (**stringPtr=='i') (*stringPtr)++; if (**stringPtr=='B') (*stringPtr)++; } - return result; + return result * sign; } static int usage(const char* exename) @@ -2408,13 +2456,14 @@ static int badusage(const char* exename) #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } } #define PARSE_CPARAMS(variable) \ { \ - PARSE_SUB_ARGS("windowLog=", "wlog=", variable.windowLog); \ - PARSE_SUB_ARGS("chainLog=" , "clog=", variable.chainLog); \ - PARSE_SUB_ARGS("hashLog=", "hlog=", variable.hashLog); \ - PARSE_SUB_ARGS("searchLog=" , "slog=", variable.searchLog); \ - PARSE_SUB_ARGS("searchLength=", "slen=", variable.searchLength); \ - PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.targetLength); \ - PARSE_SUB_ARGS("strategy=", "strat=", variable.strategy); \ + PARSE_SUB_ARGS("windowLog=", "wlog=", variable.vals[wlog_ind]); \ + PARSE_SUB_ARGS("chainLog=" , "clog=", variable.vals[clog_ind]); \ + PARSE_SUB_ARGS("hashLog=", "hlog=", variable.vals[hlog_ind]); \ + PARSE_SUB_ARGS("searchLog=" , "slog=", variable.vals[slog_ind]); \ + PARSE_SUB_ARGS("searchLength=", "slen=", variable.vals[slen_ind]); \ + PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.vals[tlen_ind]); \ + PARSE_SUB_ARGS("strategy=", "strat=", variable.vals[strt_ind]); \ + PARSE_SUB_ARGS("forceAttachDict=", "fad=" , variable.vals[fadt_ind]); \ } int main(int argc, const char** argv) @@ -2426,12 +2475,11 @@ int main(int argc, const char** argv) const char* input_filename = NULL; const char* dictFileName = NULL; U32 main_pause = 0; - int cLevel = 0; + int cLevelOpt = 0, cLevelRun = 0; int seperateFiles = 0; - constraint_t target = { 0, 0, (U32)-1 }; - ZSTD_compressionParameters paramTarget = emptyParams(); + paramValues_t paramTarget = emptyParams(); g_params = emptyParams(); assert(argc>=1); /* for exename */ @@ -2441,9 +2489,7 @@ int main(int argc, const char** argv) for(i=1; i Date: Tue, 14 Aug 2018 14:44:47 -0700 Subject: [PATCH 67/92] Fix scan-build warnings in bench.c --- programs/bench.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 7662678ab..79ef42caa 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -597,15 +597,16 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( results.error = compressionResults.error; return results; } + if(compressionResults.result.nanoSecPerRun == 0) { results.result.cSpeed = 0; } else { results.result.cSpeed = srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun; } + results.result.cSize = compressionResults.result.sumOfReturn; { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - results.result.cSpeed = (srcSize * TIMELOOP_NANOSEC / compressionResults.result.nanoSecPerRun); cSize = compressionResults.result.sumOfReturn; results.result.cSize = cSize; ratio = (double)srcSize / results.result.cSize; @@ -626,6 +627,7 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( results.error = decompressionResults.error; return results; } + if(decompressionResults.result.nanoSecPerRun == 0) { results.result.dSpeed = 0; } else { @@ -634,7 +636,6 @@ static BMK_return_t BMK_benchMemAdvancedNoAlloc( { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; - results.result.dSpeed = (srcSize * TIMELOOP_NANOSEC/ decompressionResults.result.nanoSecPerRun); markNb = (markNb+1) % NB_MARKS; DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", marks[markNb], displayName, (U32)srcSize, (U32)results.result.cSize, @@ -737,14 +738,16 @@ BMK_return_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize); void* const compressedBuffer = dstBuffer ? dstBuffer : internalDstBuffer; - void* resultBuffer = malloc(srcSize); - BMK_return_t results = { { 0, 0, 0, 0 }, 0 }; + + int parametersConflict = !dstBuffer ^ !dstCapacity; + + void* resultBuffer = srcSize ? malloc(srcSize) : NULL; + int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs || !cSizes || !cCapacities || !resPtrs || !resSizes || !timeStateCompress || !timeStateDecompress || !compressedBuffer || !resultBuffer; - int parametersConflict = !dstBuffer ^ !dstCapacity; if (!allocationincomplete && !parametersConflict) { @@ -809,7 +812,7 @@ static size_t BMK_findMaxMem(U64 requiredMem) do { testmem = (BYTE*)malloc((size_t)requiredMem); requiredMem -= step; - } while (!testmem); + } while (!testmem && requiredMem > 0); free(testmem); return (size_t)(requiredMem); @@ -937,7 +940,8 @@ BMK_return_t BMK_benchFilesAdvanced(const char* const * const fileNamesTable, un if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; if (benchedSize < totalSizeToLoad) DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); - srcBuffer = malloc(benchedSize); + + srcBuffer = benchedSize ? malloc(benchedSize) : NULL; if (!srcBuffer) { free(dictBuffer); free(fileSizes); From 96725989ef376b4fec697958837216f66d962406 Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 9 Aug 2018 16:36:34 -0700 Subject: [PATCH 68/92] Temp fix perf regression --- tests/paramgrill.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index aab46ede5..4324678ad 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -185,9 +185,9 @@ typedef struct { U32 vals[NUM_PARAMS]; } paramValues_t; -//TODO: unset -> 0? static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) { ZSTD_compressionParameters c; + memset(&c, 0, sizeof(ZSTD_compressionParameters)); c.windowLog = p.vals[wlog_ind]; c.chainLog = p.vals[clog_ind]; c.hashLog = p.vals[hlog_ind]; @@ -383,11 +383,9 @@ static int paramValid(paramValues_t paramTarget) { for(i = 0; i < NUM_PARAMS; i++) { CLAMPCHECK(paramTarget.vals[i], mintable[i], maxtable[i]); } - //TODO: Strategy could be valid at 0 before, is that right? return 1; } -//TODO: doesn't affect strategy? static paramValues_t cParamUnsetMin(paramValues_t paramTarget) { varInds_t i; for(i = 0; i < NUM_PARAMS; i++) { @@ -1399,7 +1397,7 @@ static int variableParams(const paramValues_t paramConstraints, varInds_t* res, /* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ /* take max/min bounds into account as well? */ static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) { - ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); //TODO: bounds check. + ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); } /* varies ptr by nbChanges respecting varyParams*/ @@ -1447,10 +1445,23 @@ static unsigned memoTableInd(const paramValues_t* ptr, const varInds_t* varyPara static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) { int i; for(i = varyLen - 1; i >= 0; i--) { + /* This is cleaner/easier to generalize but slower varInds_t v = varyParams[i]; if(v == strt_ind) continue; ptr->vals[v] = rangeMap(v, ind % rangetable[v]); - ind /= rangetable[v]; + ind /= rangetable[v]; */ + + switch(varyParams[i]) { + case wlog_ind: ptr->vals[wlog_ind] = ind % rangetable[wlog_ind] + mintable[wlog_ind]; ind /= rangetable[wlog_ind]; break; + case clog_ind: ptr->vals[clog_ind] = ind % rangetable[clog_ind] + mintable[clog_ind]; ind /= rangetable[clog_ind]; break; + case hlog_ind: ptr->vals[hlog_ind] = ind % rangetable[hlog_ind] + mintable[hlog_ind]; ind /= rangetable[hlog_ind]; break; + case slog_ind: ptr->vals[slog_ind] = ind % rangetable[slog_ind] + mintable[slog_ind]; ind /= rangetable[slog_ind]; break; + case slen_ind: ptr->vals[slen_ind] = ind % rangetable[slen_ind] + mintable[slen_ind]; ind /= rangetable[slen_ind]; break; + case tlen_ind: ptr->vals[tlen_ind] = tlen_table[(ind % rangetable[tlen_ind])]; ind /= rangetable[tlen_ind]; break; + case fadt_ind: ptr->vals[fadt_ind] = ind % rangetable[fadt_ind] - 1; ind /= rangetable[fadt_ind]; break; + case strt_ind: + case NUM_PARAMS: break; + } } } @@ -1471,6 +1482,7 @@ static void initMemoTable(U8* memoTable, paramValues_t paramConstraints, const c memset(memoTable, 0, arrayLen); paramConstraints = cParamUnsetMin(paramConstraints); + for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); if(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramConstraints)) > (size_t)target.cMem) { From 8c918edd3a87a006200f704b9119434c5f3250f3 Mon Sep 17 00:00:00 2001 From: George Lu Date: Fri, 10 Aug 2018 16:14:12 -0700 Subject: [PATCH 69/92] MAke it easier to add params Make memoTable size limited --- tests/paramgrill.c | 499 ++++++++++++++++++++++++--------------------- 1 file changed, 272 insertions(+), 227 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 4324678ad..c2979cb83 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -86,7 +86,15 @@ static const int g_maxNbVariations = 64; #define CHECKTIME(r) { if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } } #define CHECKTIMEGT(ret, val, _gototag) {if(BMK_timeSpan(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } } -#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt */ +#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt uses -1 */ + +static const char* g_stratName[ZSTD_btultra+1] = { + "(none) ", "ZSTD_fast ", "ZSTD_dfast ", + "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", + "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; + + +static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; /*-************************************ * Setup for Adding new params @@ -105,6 +113,14 @@ typedef enum { NUM_PARAMS = 8 } varInds_t; +typedef struct { + U32 vals[NUM_PARAMS]; +} paramValues_t; + +/* list of parameters */ +static const varInds_t paramTable[NUM_PARAMS] = + { wlog_ind, clog_ind, hlog_ind, slog_ind, slen_ind, tlen_ind, strt_ind, fadt_ind }; + /* maximum value of parameters */ static const U32 mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN }; @@ -117,15 +133,19 @@ static const U32 maxtable[NUM_PARAMS] = static const U32 rangetable[NUM_PARAMS] = { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, SLEN_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE }; +/* ZSTD_cctxSetParameter() index to set */ static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] = { ZSTD_p_windowLog, ZSTD_p_chainLog, ZSTD_p_hashLog, ZSTD_p_searchLog, ZSTD_p_minMatch, ZSTD_p_targetLength, ZSTD_p_compressionStrategy, ZSTD_p_forceAttachDict }; +/* names of parameters */ static const char* g_paramNames[NUM_PARAMS] = - { "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict"}; + { "windowLog", "chainLog", "hashLog","searchLog", "searchLength", "targetLength", "strategy", "forceAttachDict" }; +/* shortened names of parameters */ +static const char* g_shortParamNames[NUM_PARAMS] = + { "wlog", "clog", "hlog","slog", "slen", "tlen", "strt", "fadt" }; -static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; -/* maps value from 0 to rangetable[param] - 1 to valid paramvalue */ +/* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */ static U32 rangeMap(varInds_t param, U32 ind) { ind = MIN(ind, rangetable[param] - 1); switch(param) { @@ -141,6 +161,7 @@ static U32 rangeMap(varInds_t param, U32 ind) { case strt_ind: return mintable[param] + ind; case NUM_PARAMS: + DISPLAY("Error, not a valid param\n "); return (U32)-1; } return 0; /* should never happen, stop compiler warnings */ @@ -176,49 +197,26 @@ static U32 invRangeMap(varInds_t param, U32 value) { case strt_ind: return value - mintable[param]; case NUM_PARAMS: + DISPLAY("Error, not a valid param\n "); return (U32)-1; } return 0; /* should never happen, stop compiler warnings */ } -typedef struct { - U32 vals[NUM_PARAMS]; -} paramValues_t; - -static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) { - ZSTD_compressionParameters c; - memset(&c, 0, sizeof(ZSTD_compressionParameters)); - c.windowLog = p.vals[wlog_ind]; - c.chainLog = p.vals[clog_ind]; - c.hashLog = p.vals[hlog_ind]; - c.searchLog = p.vals[slog_ind]; - c.searchLength = p.vals[slen_ind]; - c.targetLength = p.vals[tlen_ind]; - c.strategy = p.vals[strt_ind]; - /* no forceAttachDict */ - return c; -} - -/* 0 = auto for fadt */ -static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) { - paramValues_t p; - p.vals[wlog_ind] = c.windowLog; - p.vals[clog_ind] = c.chainLog; - p.vals[hlog_ind] = c.hashLog; - p.vals[slog_ind] = c.searchLog; - p.vals[slen_ind] = c.searchLength; - p.vals[tlen_ind] = c.targetLength; - p.vals[strt_ind] = c.strategy; - p.vals[fadt_ind] = 0; - return p; -} - -/* equivalent of ZSTD_adjustCParams for paramValues_t */ -static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) { - U32 fval = p.vals[fadt_ind]; - p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize)); - p.vals[fadt_ind] = fval; - return p; +/* display of params */ +static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) { + switch(param) { + case fadt_ind: if(width) { fprintf(f, "%*d", width, (int)value); } else { fprintf(f, "%d", (int)value); } break; + case strt_ind: if(width) { fprintf(f, "%*s", width, g_stratName[value]); } else { fprintf(f, "%s", g_stratName[value]); } break; + case wlog_ind: + case clog_ind: + case hlog_ind: + case slog_ind: + case slen_ind: + case tlen_ind: if(width) { fprintf(f, "%*u", width, value); } else { fprintf(f, "%u", value); } break; + case NUM_PARAMS: + DISPLAY("Error, not a valid param\n "); break; + } } /*-************************************ @@ -238,7 +236,7 @@ static U32 g_target = 0; static U32 g_noSeed = 0; static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ - +static U32 g_memoLimit = (U32)-1; //32 MB typedef struct { BMK_result_t result; @@ -293,6 +291,51 @@ void BMK_SetNbIterations(int nbLoops) * Private functions *********************************************************/ +static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) { + ZSTD_compressionParameters c; + memset(&c, 0, sizeof(ZSTD_compressionParameters)); + c.windowLog = p.vals[wlog_ind]; + c.chainLog = p.vals[clog_ind]; + c.hashLog = p.vals[hlog_ind]; + c.searchLog = p.vals[slog_ind]; + c.searchLength = p.vals[slen_ind]; + c.targetLength = p.vals[tlen_ind]; + c.strategy = p.vals[strt_ind]; + /* no forceAttachDict */ + return c; +} + +static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) { + paramValues_t p; + varInds_t i; + p.vals[wlog_ind] = c.windowLog; + p.vals[clog_ind] = c.chainLog; + p.vals[hlog_ind] = c.hashLog; + p.vals[slog_ind] = c.searchLog; + p.vals[slen_ind] = c.searchLength; + p.vals[tlen_ind] = c.targetLength; + p.vals[strt_ind] = c.strategy; + + /* set all other params to their minimum value */ + for(i = strt_ind + 1; i < NUM_PARAMS; i++) { + p.vals[i] = mintable[i]; + } + return p; +} + +/* equivalent of ZSTD_adjustCParams for paramValues_t */ +static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) { + paramValues_t ot = p; + varInds_t i; + p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize)); + + /* retain value of all other parameters */ + for(i = strt_ind + 1; i < NUM_PARAMS; i++) { + p.vals[i] = ot.vals[i]; + } + return p; +} + /* accuracy in seconds only, span can be multiple years */ static U32 BMK_timeSpan(UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); } @@ -375,9 +418,6 @@ static void findClockGranularity(void) { return 0; \ } } - -/* Like ZSTD_checkCParams() but allows 0's */ -/* no check on targetLen? */ static int paramValid(paramValues_t paramTarget) { U32 i; for(i = 0; i < NUM_PARAMS; i++) { @@ -400,8 +440,11 @@ static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { U32 i; fprintf(f,"--zstd="); for(i = 0; i < NUM_PARAMS; i++) { - fprintf(f,"%s", g_paramNames[i]); - fprintf(f,"=%u", params.vals[i]); + fprintf(f,"%s=", g_paramNames[i]); + + if(i == strt_ind) { fprintf(f,"%u", params.vals[i]); } + else { displayParamVal(f, i, params.vals[i], 0); } + if(i != NUM_PARAMS - 1) { fprintf(f, ","); } @@ -409,19 +452,16 @@ static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { fprintf(f, "\n"); } - -static const char* g_stratName[ZSTD_btultra+1] = { - "(none) ", "ZSTD_fast ", "ZSTD_dfast ", - "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", - "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra "}; - static void BMK_displayOneResult(FILE* f, winnerInfo_t res, size_t srcSize) { + varInds_t v; res.params = cParamUnsetMin(res.params); - fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u,%3d, %s}, ", - res.params.vals[wlog_ind], res.params.vals[clog_ind], res.params.vals[hlog_ind], res.params.vals[slog_ind], res.params.vals[slen_ind], - res.params.vals[tlen_ind], (int)res.params.vals[fadt_ind], g_stratName[res.params.vals[strt_ind]]); - fprintf(f, - " /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + fprintf(f," {"); + for(v = 0; v < NUM_PARAMS; v++) { + if(v != 0) { fprintf(f, ","); } + displayParamVal(f, v, res.params.vals[v], 3); + } + + fprintf(f, "}, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB)); } @@ -494,7 +534,7 @@ static paramValues_t emptyParams(void) { return p; } -static winnerInfo_t initWinnerInfo(paramValues_t p) { +static winnerInfo_t initWinnerInfo(const paramValues_t p) { winnerInfo_t w1; w1.result.cSpeed = 0.; w1.result.dSpeed = 0.; @@ -515,6 +555,7 @@ typedef struct { void** resPtrs; size_t* resSizes; size_t nbBlocks; + size_t maxBlockSize; } buffers_t; typedef struct { @@ -528,27 +569,6 @@ typedef struct { * From bench.c *********************************************************/ -static void BMK_initCCtx(ZSTD_CCtx* ctx, - const void* dictBuffer, const size_t dictBufferSize, const int cLevel, - const paramValues_t* comprParams) { - varInds_t i; - ZSTD_CCtx_reset(ctx); - ZSTD_CCtx_resetParameters(ctx); - ZSTD_CCtx_setParameter(ctx, ZSTD_p_compressionLevel, cLevel); - - for(i = 0; i < NUM_PARAMS; i++) { - if(comprParams->vals[i] != PARAM_UNSET) - ZSTD_CCtx_setParameter(ctx, cctxSetParamTable[i], comprParams->vals[i]); - } - ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize); -} - -static void BMK_initDCtx(ZSTD_DCtx* dctx, - const void* dictBuffer, const size_t dictBufferSize) { - ZSTD_DCtx_reset(dctx); - ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize); -} - typedef struct { ZSTD_CCtx* ctx; const void* dictBuffer; @@ -559,7 +579,17 @@ typedef struct { static size_t local_initCCtx(void* payload) { const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload; - BMK_initCCtx(ag->ctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams); + varInds_t i; + ZSTD_CCtx_reset(ag->ctx); + ZSTD_CCtx_resetParameters(ag->ctx); + ZSTD_CCtx_setParameter(ag->ctx, ZSTD_p_compressionLevel, ag->cLevel); + + for(i = 0; i < NUM_PARAMS; i++) { + if(ag->comprParams->vals[i] != PARAM_UNSET) + ZSTD_CCtx_setParameter(ag->ctx, cctxSetParamTable[i], ag->comprParams->vals[i]); + } + ZSTD_CCtx_loadDictionary(ag->ctx, ag->dictBuffer, ag->dictBufferSize); + return 0; } @@ -571,7 +601,8 @@ typedef struct { static size_t local_initDCtx(void* payload) { const BMK_initDCtxArgs* ag = (const BMK_initDCtxArgs*)payload; - BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize); + ZSTD_DCtx_reset(ag->dctx); + ZSTD_DCtx_loadDictionary(ag->dctx, ag->dictBuffer, ag->dictBufferSize); return 0; } @@ -636,6 +667,72 @@ static size_t local_defaultDecompress( * From bench.c End *********************************************************/ +static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) { + varInds_t v; + for(v = 0; v < NUM_PARAMS; v++) { + if(pc->vals[v] != PARAM_UNSET) { + U32 newval = MIN(MAX(pc->vals[v], mintable[v]), maxtable[v]); + if(newval != pc->vals[v]) { + pc->vals[v] = newval; + DISPLAY("Warning: parameter %s not in valid range, adjusting to ", g_paramNames[v]); displayParamVal(stderr, v, newval, 0); DISPLAY("\n"); + } + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET) { + + U32 sshb = maxBlockSize > 1 ? ZSTD_highbit32((U32)(maxBlockSize-1)) + 1 : 1; + /* edge case of highBit not working for 0 */ + + if(maxBlockSize < (1ULL << 31) && sshb + 1 < pc->vals[wlog_ind]) { + U32 adjust = MAX(mintable[wlog_ind], sshb); + if(adjust != pc->vals[wlog_ind]) { + pc->vals[wlog_ind] = adjust; + DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", pc->vals[wlog_ind]); + } + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) { + U32 maxclog; + if(pc->vals[strt_ind] == PARAM_UNSET || pc->vals[strt_ind] >= (U32)ZSTD_btlazy2) { + maxclog = pc->vals[wlog_ind] + 1; + } else { + maxclog = pc->vals[wlog_ind]; + } + + if(pc->vals[clog_ind] > maxclog) { + pc->vals[clog_ind] = maxclog; + DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", pc->vals[clog_ind]); + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[hlog_ind] != PARAM_UNSET) { + if(pc->vals[wlog_ind] + 1 < pc->vals[hlog_ind]) { + pc->vals[hlog_ind] = pc->vals[wlog_ind] + 1; + DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", pc->vals[hlog_ind]); + } + } + + if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) { + if(pc->vals[slog_ind] > pc->vals[clog_ind]) { + pc->vals[clog_ind] = pc->vals[slog_ind]; + DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", pc->vals[slog_ind]); + } + } +} + +/* what about low something like clog vs hlog in lvl 1? */ +static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t srcSize) { + return + (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */ + || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= srcSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */ + || (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/ + || (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */ + || (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */ + +} + static void freeNonSrcBuffers(const buffers_t b) { free(b.srcPtrs); free(b.srcSizes); @@ -722,6 +819,7 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); buff->dstSizes[0] = buff->dstCapacities[0]; buff->resSizes[0] = buff->srcSizes[0]; + buff->maxBlockSize = buff->srcSizes[0]; for(n = 1; n < blockNb; n++) { buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1]; @@ -729,6 +827,8 @@ static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbF buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]); buff->dstSizes[n] = buff->dstCapacities[n]; buff->resSizes[n] = buff->srcSizes[n]; + + buff->maxBlockSize = MAX(buff->maxBlockSize, buff->srcSizes[n]); } buff->nbBlocks = blockNb; @@ -786,7 +886,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab char* buffer = (char*)(srcBuffer); size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); fclose(f); - if (readSize != (size_t)fileSize) { /* should we accept partial read? */ + if (readSize != (size_t)fileSize) { DISPLAY("could not read %s", fileNamesTable[n]); ret = 1; goto _cleanUp; @@ -992,7 +1092,7 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t } static int BMK_benchParam(BMK_result_t* resultPtr, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const paramValues_t cParams) { BMK_return_t res = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_timeMode, 3); *resultPtr = res.result; @@ -1009,7 +1109,7 @@ static int BMK_benchParam(BMK_result_t* resultPtr, #define SPEED_RESULT 4 #define SIZE_RESULT 5 /* maybe have epsilon-eq to limit table size? */ -static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) { +static int speedSizeCompare(const BMK_result_t r1, const BMK_result_t r2) { if(r1.cSpeed < r2.cSpeed) { if(r1.cSize >= r2.cSize) { return BETTER_RESULT; @@ -1025,7 +1125,7 @@ static int speedSizeCompare(BMK_result_t r1, BMK_result_t r2) { /* 0 for insertion, 1 for no insert */ /* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */ -static int insertWinner(winnerInfo_t w, constraint_t targetConstraints) { +static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) { BMK_result_t r = w.result; winner_ll_node* cur_node = g_winners; /* first node to insert */ @@ -1194,7 +1294,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res } } -static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize) +static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize) { int cLevel; @@ -1206,7 +1306,7 @@ static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, size_t sr } -static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, size_t srcSize) +static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize) { fseek(f, 0, SEEK_SET); BMK_printWinners2(f, winners, srcSize); @@ -1244,7 +1344,7 @@ static void BMK_init_level_constraints(int bytePerSec_level1) } static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters params, - buffers_t buf, contexts_t ctx) + const buffers_t buf, const contexts_t ctx) { BMK_result_t testResult; int better = 0; @@ -1422,9 +1522,8 @@ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { size_t arrayLen = 1; int i; for(i = 0; i < varyLen; i++) { - if(varyParams[i] != strt_ind) { - arrayLen *= rangetable[varyParams[i]]; - } + if(varyParams[i] == strt_ind) continue; /* strategy separated by table */ + arrayLen *= rangetable[varyParams[i]]; } return arrayLen; } @@ -1452,13 +1551,13 @@ static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, con ind /= rangetable[v]; */ switch(varyParams[i]) { - case wlog_ind: ptr->vals[wlog_ind] = ind % rangetable[wlog_ind] + mintable[wlog_ind]; ind /= rangetable[wlog_ind]; break; - case clog_ind: ptr->vals[clog_ind] = ind % rangetable[clog_ind] + mintable[clog_ind]; ind /= rangetable[clog_ind]; break; - case hlog_ind: ptr->vals[hlog_ind] = ind % rangetable[hlog_ind] + mintable[hlog_ind]; ind /= rangetable[hlog_ind]; break; - case slog_ind: ptr->vals[slog_ind] = ind % rangetable[slog_ind] + mintable[slog_ind]; ind /= rangetable[slog_ind]; break; - case slen_ind: ptr->vals[slen_ind] = ind % rangetable[slen_ind] + mintable[slen_ind]; ind /= rangetable[slen_ind]; break; - case tlen_ind: ptr->vals[tlen_ind] = tlen_table[(ind % rangetable[tlen_ind])]; ind /= rangetable[tlen_ind]; break; - case fadt_ind: ptr->vals[fadt_ind] = ind % rangetable[fadt_ind] - 1; ind /= rangetable[fadt_ind]; break; + case wlog_ind: ptr->vals[wlog_ind] = ind % rangetable[wlog_ind] + mintable[wlog_ind]; ind /= rangetable[wlog_ind]; break; + case clog_ind: ptr->vals[clog_ind] = ind % rangetable[clog_ind] + mintable[clog_ind]; ind /= rangetable[clog_ind]; break; + case hlog_ind: ptr->vals[hlog_ind] = ind % rangetable[hlog_ind] + mintable[hlog_ind]; ind /= rangetable[hlog_ind]; break; + case slog_ind: ptr->vals[slog_ind] = ind % rangetable[slog_ind] + mintable[slog_ind]; ind /= rangetable[slog_ind]; break; + case slen_ind: ptr->vals[slen_ind] = ind % rangetable[slen_ind] + mintable[slen_ind]; ind /= rangetable[slen_ind]; break; + case tlen_ind: ptr->vals[tlen_ind] = tlen_table[(ind % rangetable[tlen_ind])]; ind /= rangetable[tlen_ind]; break; + case fadt_ind: ptr->vals[fadt_ind] = ind % rangetable[fadt_ind] - 1; ind /= rangetable[fadt_ind]; break; case strt_ind: case NUM_PARAMS: break; } @@ -1473,58 +1572,19 @@ static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, con static void initMemoTable(U8* memoTable, paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { size_t i; size_t arrayLen = memoTableLen(varyParams, varyLen); - int cwFixed = paramConstraints.vals[clog_ind] == PARAM_UNSET || paramConstraints.vals[wlog_ind] == PARAM_UNSET; - int scFixed = paramConstraints.vals[slog_ind] == PARAM_UNSET || paramConstraints.vals[clog_ind] == PARAM_UNSET; - int whFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET || paramConstraints.vals[hlog_ind] == PARAM_UNSET; - int wFixed = paramConstraints.vals[wlog_ind] == PARAM_UNSET; int j = 0; assert(memoTable != NULL); memset(memoTable, 0, arrayLen); paramConstraints = cParamUnsetMin(paramConstraints); - for(i = 0; i < arrayLen; i++) { memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - if(ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramConstraints)) > (size_t)target.cMem) { - memoTable[i] = 255; - j++; - } - if(wFixed && (1ULL << (paramConstraints.vals[wlog_ind] - 1)) >= srcSize && paramConstraints.vals[wlog_ind] != mintable[wlog_ind]) { - memoTable[i] = 255; - } - /* nil out parameter sets equivalent to others. */ - if(cwFixed) { - if(paramConstraints.vals[strt_ind] == ZSTD_btlazy2 || paramConstraints.vals[strt_ind] == ZSTD_btopt || paramConstraints.vals[strt_ind] == ZSTD_btultra) { - if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]+ 1) { - if(memoTable[i] != 255) { j++; } - memoTable[i] = 255; - } - } else { - if(paramConstraints.vals[clog_ind] > paramConstraints.vals[wlog_ind]) { - if(memoTable[i] != 255) { j++; } - memoTable[i] = 255; - } - } - } - - if(scFixed) { - if(paramConstraints.vals[slog_ind] > paramConstraints.vals[clog_ind]) { - if(memoTable[i] != 255) { j++; } - memoTable[i] = 255; - } - } - - if(whFixed) { - if(paramConstraints.vals[hlog_ind] > paramConstraints.vals[wlog_ind] + 1) { - if(memoTable[i] != 255) { j++; } - memoTable[i] = 255; - } + if(redundantParams(paramConstraints, target, srcSize)) { + memoTable[i] = 255; j++; } } - DEBUGOUTPUT("%d / %d Invalid\n", j, (int)i); - if((int)i == j) { - DEBUGOUTPUT("!!!Strategy %d totally infeasible\n", (int)paramConstraints.vals[strt_ind]); - } + + DEBUGOUTPUT("%d / %d Invalid\n", j, (int)arrayLen); } /* frees all allocated memotables */ @@ -1539,7 +1599,7 @@ static void freeMemoTableArray(U8** mtAll) { /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ -static U8** createMemoTableArray(paramValues_t paramConstraints, constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { +static U8** createMemoTableArray(paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { varInds_t varNew[NUM_PARAMS]; U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1)); int i; @@ -1549,18 +1609,21 @@ static U8** createMemoTableArray(paramValues_t paramConstraints, constraint_t ta for(i = 1; i <= (int)ZSTD_btultra; i++) { const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); - mtAll[i] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); + size_t mtl = memoTableLen(varNew, varLenNew); + if(mtl > g_memoLimit) { mtAll[i] = NULL; continue; } + mtAll[i] = malloc(sizeof(U8) * mtl); if(mtAll[i] == NULL) { freeMemoTableArray(mtAll); return NULL; } + paramConstraints.vals[strt_ind] = i; initMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); } return mtAll; } -static paramValues_t overwriteParams(paramValues_t base, paramValues_t mask) { +static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mask) { U32 i; for(i = 0; i < NUM_PARAMS; i++) { if(mask.vals[i] != PARAM_UNSET) { @@ -1586,7 +1649,7 @@ static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { static void playAround(FILE* f, oldWinnerInfo_t* winners, ZSTD_compressionParameters params, - buffers_t buf, contexts_t ctx) + const buffers_t buf, const contexts_t ctx) { int nbVariations = 0; UTIL_time_t const clockStart = UTIL_getTime(); @@ -1619,49 +1682,35 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners, } +/* Sets pc to random unmeasured set of parameters */ +/* Doesn't do strategy! */ +static void randomConstrainedParams(paramValues_t* pc, const varInds_t* varArray, const int varLen, const U8* memoTable) +{ + size_t j; + for(j = 0; j < MIN(g_memoLimit, memoTableLen(varArray, varLen)); j++) { + int i; + for(i = 0; i < NUM_PARAMS; i++) { + varInds_t v = varArray[i]; + if(v == strt_ind) continue; //don't do strategy (dependent on MT) + pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); + } + + if(memoTable == NULL || memoTable[memoTableInd(pc, varArray, varLen)]) { break; } + } +} + /* Completely random parameter selection */ static ZSTD_compressionParameters randomParams(void) { - ZSTD_compressionParameters p; - U32 validated = 0; - while (!validated) { - /* totally random entry */ - p.chainLog = (FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN)) - + ZSTD_CHAINLOG_MIN; - p.hashLog = (FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN)) - + ZSTD_HASHLOG_MIN; - p.searchLog = (FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN)) - + ZSTD_SEARCHLOG_MIN; - p.windowLog = (FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN)) - + ZSTD_WINDOWLOG_MIN; - p.searchLength=(FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN)) - + ZSTD_SEARCHLENGTH_MIN; - p.targetLength=(FUZ_rand(&g_rand) % (512)); - - p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); - - validated = !ZSTD_isError(ZSTD_checkCParams(p)); - } - return p; -} - -/* Sets pc to random unmeasured set of parameters */ -static void randomConstrainedParams(paramValues_t* pc, varInds_t* varArray, int varLen, U8* memoTable) -{ - size_t tries = memoTableLen(varArray, varLen); - const size_t maxSize = memoTableLen(varArray, varLen); - size_t ind; - do { - ind = (FUZ_rand(&g_rand)) % maxSize; - tries--; - } while(memoTable[ind] > 0 && tries > 0); - - memoTableIndInv(pc, varArray, varLen, (unsigned)ind); + paramValues_t p; + p.vals[strt_ind] = rangeMap(strt_ind, rangeMap(strt_ind, FUZ_rand(&g_rand) % rangetable[strt_ind])); + randomConstrainedParams(&p, paramTable, NUM_PARAMS, NULL); + return pvalsToCParams(p); } static void BMK_selectRandomStart( FILE* f, oldWinnerInfo_t* winners, - buffers_t buf, contexts_t ctx) + const buffers_t buf, const contexts_t ctx) { U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); if ((id==0) || (winners[id].params.windowLog==0)) { @@ -1673,7 +1722,7 @@ static void BMK_selectRandomStart( } } -static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBlockSize) +static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) { ZSTD_compressionParameters params; oldWinnerInfo_t winners[NB_LEVELS_TRACKED+1]; @@ -1689,7 +1738,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl BMK_init_level_constraints(g_target * (1 MB)); } else { /* baseline config for level 1 */ - ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, maxBlockSize, ctx.dictSize); + ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, buf.maxBlockSize, ctx.dictSize); BMK_result_t testResult; BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(l1params)); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); @@ -1699,7 +1748,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); int i; for (i=0; i<=maxSeeds; i++) { - params = ZSTD_getCParams(i, maxBlockSize, 0); + params = ZSTD_getCParams(i, buf.maxBlockSize, 0); BMK_seed(winners, params, buf, ctx); } } BMK_printWinners(f, winners, buf.srcSize); @@ -1719,7 +1768,7 @@ static void BMK_benchFullTable(buffers_t buf, contexts_t ctx, const size_t maxBl fclose(f); } -static int benchOnce(buffers_t buf, contexts_t ctx) { +static int benchOnce(const buffers_t buf, const contexts_t ctx) { BMK_result_t testResult; if(BMK_benchParam(&testResult, buf, ctx, g_params)) { @@ -1736,7 +1785,6 @@ static int benchSample(void) { const char* const name = "Sample 10MB"; size_t const benchedSize = 10 MB; - U32 blockSize = g_blockSize ? g_blockSize : benchedSize; void* srcBuffer = malloc(benchedSize); int ret = 0; @@ -1769,7 +1817,7 @@ static int benchSample(void) if(g_singleRun) { ret = benchOnce(buf, ctx); } else { - BMK_benchFullTable(buf, ctx, MIN(blockSize, benchedSize)); + BMK_benchFullTable(buf, ctx); } freeBuffers(buf); @@ -1785,7 +1833,6 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam { buffers_t buf; contexts_t ctx; - size_t maxBlockSize = 0, i; int ret = 0; if(createBuffers(&buf, fileNamesTable, nbFiles)) { @@ -1799,10 +1846,6 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam return 2; } - for(i = 0; i < buf.nbBlocks; i++) { - maxBlockSize = MAX(maxBlockSize, buf.srcSizes[i]); - } - DISPLAY("\r%79s\r", ""); if(nbFiles == 1) { DISPLAY("using %s : \n", fileNamesTable[0]); @@ -1810,12 +1853,12 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam DISPLAY("using %d Files : \n", nbFiles); } - g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize); + g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize); if(g_singleRun) { ret = benchOnce(buf, ctx); } else { - BMK_benchFullTable(buf, ctx, maxBlockSize); + BMK_benchFullTable(buf, ctx); } freeBuffers(buf); @@ -1937,7 +1980,7 @@ static int benchMemo(BMK_result_t* resultPtr, size_t memind = memoTableInd(&cParams, varyParams, varyLen); int res; - if(memoTable[memind] >= INFEASIBLE_THRESHOLD) { return WORSE_RESULT; } + if((memoTable == NULL && redundantParams(cParams, target, buf.maxBlockSize)) || ((memoTable != NULL) && memoTable[memind] >= INFEASIBLE_THRESHOLD)) { return WORSE_RESULT; } res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas); @@ -1947,7 +1990,7 @@ static int benchMemo(BMK_result_t* resultPtr, } BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize); - if(res == BETTER_RESULT || feas) { + if(memoTable != NULL && (res == BETTER_RESULT || feas)) { memoTable[memind] = 255; } return res; @@ -1972,7 +2015,7 @@ static int benchMemo(BMK_result_t* resultPtr, static winnerInfo_t climbOnce(const constraint_t target, const varInds_t* varArray, const int varLen, ZSTD_strategy strat, U8** memoTableArray, - buffers_t buf, contexts_t ctx, + const buffers_t buf, const contexts_t ctx, const paramValues_t init) { /* * cparam - currently considered 'center' @@ -2103,7 +2146,6 @@ static winnerInfo_t optimizeFixedStrategy( /* so climb is given the right fixed strategy */ paramTarget.vals[strt_ind] = strat; /* to pass ZSTD_checkCParams */ - paramTarget = cParamUnsetMin(paramTarget); init = paramTarget; @@ -2178,16 +2220,11 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ const int varLen = variableParams(paramTarget, varArray, dictFileName != NULL); winnerInfo_t winner = initWinnerInfo(emptyParams()); U8** allMT = NULL; - paramValues_t paramBase = cParamUnsetMin(paramTarget); - size_t k, maxBlockSize = 0; + paramValues_t paramBase; contexts_t ctx; buffers_t buf; g_time = UTIL_getTime(); - if(!paramValid(paramBase)) { - return 1; - } - if(createBuffers(&buf, fileNamesTable, nbFiles)) { DISPLAY("unable to load files\n"); return 1; @@ -2205,10 +2242,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ DISPLAY("Loading %lu Files... \r", (unsigned long)nbFiles); } - - for(k = 0; k < buf.nbBlocks; k++) { - maxBlockSize = MAX(buf.srcSizes[k], maxBlockSize); - } + /* sanitize paramTarget */ + optimizerAdjustInput(¶mTarget, buf.maxBlockSize); + paramBase = cParamUnsetMin(paramTarget); /* if strategy is fixed, only init that part of memotable */ if(paramTarget.vals[strt_ind] != PARAM_UNSET) { @@ -2227,9 +2263,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } - initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, maxBlockSize); + initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, buf.maxBlockSize); } else { - allMT = createMemoTableArray(paramTarget, target, varArray, varLen, maxBlockSize); + allMT = createMemoTableArray(paramTarget, target, varArray, varLen, buf.maxBlockSize); } if(!allMT) { @@ -2256,7 +2292,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* use level'ing mode instead of normal target mode */ /* Should lvl be parameter-masked here? */ if(g_optmode) { - winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, maxBlockSize, ctx.dictSize)); + winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, buf.maxBlockSize, ctx.dictSize)); if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { ret = 3; goto _cleanUp; @@ -2276,7 +2312,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ /* Don't want it to return anything worse than the best known result */ if(g_singleRun) { BMK_result_t res; - g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, maxBlockSize, ctx.dictSize)), g_params), maxBlockSize, ctx.dictSize); + g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize); if(BMK_benchParam(&res, buf, ctx, g_params)) { ret = 45; goto _cleanUp; @@ -2315,7 +2351,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ int i; for (i=1; i<=maxSeeds; i++) { int ec; - CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, maxBlockSize, ctx.dictSize)), paramTarget); + CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, buf.maxBlockSize, ctx.dictSize)), paramTarget); ec = BMK_benchParam(&candidate, buf, ctx, CParams); BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize); @@ -2466,16 +2502,24 @@ static int badusage(const char* exename) } #define PARSE_SUB_ARGS(stringLong, stringShort, variable) { if (longCommandWArg(&argument, stringLong) || longCommandWArg(&argument, stringShort)) { variable = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } } -#define PARSE_CPARAMS(variable) \ -{ \ - PARSE_SUB_ARGS("windowLog=", "wlog=", variable.vals[wlog_ind]); \ - PARSE_SUB_ARGS("chainLog=" , "clog=", variable.vals[clog_ind]); \ - PARSE_SUB_ARGS("hashLog=", "hlog=", variable.vals[hlog_ind]); \ - PARSE_SUB_ARGS("searchLog=" , "slog=", variable.vals[slog_ind]); \ - PARSE_SUB_ARGS("searchLength=", "slen=", variable.vals[slen_ind]); \ - PARSE_SUB_ARGS("targetLength=" , "tlen=", variable.vals[tlen_ind]); \ - PARSE_SUB_ARGS("strategy=", "strat=", variable.vals[strt_ind]); \ - PARSE_SUB_ARGS("forceAttachDict=", "fad=" , variable.vals[fadt_ind]); \ +/* 1 if successful parse, 0 otherwise */ +static int parse_params(const char** argptr, paramValues_t* pv) { + int matched = 0; + const char* argOrig = *argptr; + varInds_t v; + for(v = 0; v < NUM_PARAMS; v++) { + if(longCommandWArg(argptr,g_shortParamNames[v]) || longCommandWArg(argptr, g_paramNames[v])) { + if(**argptr == '=') { + (*argptr)++; + pv->vals[v] = readU32FromChar(argptr); + matched = 1; + break; + } + } + /* reset and try again */ + *argptr = argOrig; + } + return matched; } int main(int argc, const char** argv) @@ -2509,7 +2553,7 @@ int main(int argc, const char** argv) if (longCommandWArg(&argument, "--optimize=")) { g_optimizer = 1; for ( ; ;) { - PARSE_CPARAMS(paramTarget); + if(parse_params(&argument, ¶mTarget)) { if(argument[0] == ',') { argument++; continue; } else break; } PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed); PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); @@ -2517,6 +2561,7 @@ int main(int argc, const char** argv) PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier); PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier); PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries); + PARSE_SUB_ARGS("memoLimit=", "memo=", g_memoLimit); if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; } DISPLAY("invalid optimization parameter \n"); @@ -2532,7 +2577,7 @@ int main(int argc, const char** argv) /* Decode command (note : aggregated commands are allowed) */ g_singleRun = 1; for ( ; ;) { - PARSE_CPARAMS(g_params) + if(parse_params(&argument, &g_params)) { if(argument[0] == ',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelRun = readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; } DISPLAY("invalid compression parameter \n"); From 6e66bbf5dde09f555a2adb1dd763808c0de3c6c6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 14 Aug 2018 12:56:21 -0700 Subject: [PATCH 70/92] fixed several minor issues detected by scan-build only notable one : writeNCount() resists better vs invalid distributions (though it should never happen within zstd anyway) --- lib/common/zstd_internal.h | 3 +- lib/compress/fse_compress.c | 65 ++++++++++++++++++-------------- lib/compress/zstd_compress.c | 1 - lib/compress/zstd_opt.c | 2 +- lib/decompress/zstd_decompress.c | 26 ++++++------- lib/dictBuilder/zdict.c | 2 +- lib/legacy/zstd_v04.c | 1 + lib/legacy/zstd_v05.c | 1 + lib/legacy/zstd_v06.c | 2 +- lib/legacy/zstd_v07.c | 20 +++++----- 10 files changed, 67 insertions(+), 56 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index b4c1af53f..b36a2fbd0 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -79,8 +79,7 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; -#define ZSTD_FRAMEIDSIZE 4 -static const size_t ZSTD_frameIdSize = ZSTD_FRAMEIDSIZE; /* magic number size */ +#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ #define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index 70daae3bc..caa90edec 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -83,7 +83,9 @@ * wkspSize should be sized to handle worst case situation, which is `1< wkspSize) return ERROR(tableLog_tooLarge); tableU16[-2] = (U16) tableLog; tableU16[-1] = (U16) maxSymbolValue; - assert(tableLog < 16); /* required for the threshold strategy to work */ + assert(tableLog < 16); /* required for threshold strategy to work */ /* For explanations on how to distribute symbol values over the table : - * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ + * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ + + #ifdef __clang_analyzer__ + memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */ + #endif /* symbol start positions */ { U32 u; @@ -124,13 +130,15 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi U32 symbol; for (symbol=0; symbol<=maxSymbolValue; symbol++) { int nbOccurences; - for (nbOccurences=0; nbOccurences highThreshold) position = (position + step) & tableMask; /* Low proba area */ + while (position > highThreshold) + position = (position + step) & tableMask; /* Low proba area */ } } - if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */ + assert(position==0); /* Must have initialized all positions */ } /* Build table */ @@ -201,9 +209,10 @@ size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ } -static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, - const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, - unsigned writeIsSafe) +static size_t +FSE_writeNCount_generic (void* header, size_t headerBufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, + unsigned writeIsSafe) { BYTE* const ostart = (BYTE*) header; BYTE* out = ostart; @@ -212,13 +221,12 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, const int tableSize = 1 << tableLog; int remaining; int threshold; - U32 bitStream; - int bitCount; - unsigned charnum = 0; - int previous0 = 0; + U32 bitStream = 0; + int bitCount = 0; + unsigned symbol = 0; + unsigned const alphabetSize = maxSymbolValue + 1; + int previousIs0 = 0; - bitStream = 0; - bitCount = 0; /* Table Size */ bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; bitCount += 4; @@ -228,11 +236,11 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, threshold = tableSize; nbBits = tableLog+1; - while (remaining>1) { /* stops at 1 */ - if (previous0) { - unsigned start = charnum; - while (!normalizedCounter[charnum]) charnum++; - while (charnum >= start+24) { + while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */ + if (previousIs0) { + unsigned start = symbol; + while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++; + while (symbol >= start+24) { start+=24; bitStream += 0xFFFFU << bitCount; if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ @@ -241,12 +249,12 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, out+=2; bitStream>>=16; } - while (charnum >= start+3) { + while (symbol >= start+3) { start+=3; bitStream += 3 << bitCount; bitCount += 2; } - bitStream += (charnum-start) << bitCount; + bitStream += (symbol-start) << bitCount; bitCount += 2; if (bitCount>16) { if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ @@ -256,15 +264,15 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, bitStream >>= 16; bitCount -= 16; } } - { int count = normalizedCounter[charnum++]; - int const max = (2*threshold-1)-remaining; + { int count = normalizedCounter[symbol++]; + int const max = (2*threshold-1) - remaining; remaining -= count < 0 ? -count : count; count++; /* +1 for extra accuracy */ if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ bitStream += count << bitCount; bitCount += nbBits; bitCount -= (count>=1; } } @@ -277,14 +285,15 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, bitCount -= 16; } } + if (remaining != 1) return ERROR(GENERIC); /* incorrect normalized distribution */ + assert(symbol <= alphabetSize); + /* flush remaining bitStream */ if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ out[0] = (BYTE)bitStream; out[1] = (BYTE)(bitStream>>8); out+= (bitCount+7) /8; - if (charnum > maxSymbolValue + 1) return ERROR(GENERIC); - return (out-ostart); } @@ -297,7 +306,7 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalized if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); - return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */); } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index ed3aab871..29dce1250 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1147,7 +1147,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, if (zc->workSpace == NULL) return ERROR(memory_allocation); zc->workSpaceSize = neededSpace; zc->workSpaceOversizedDuration = 0; - ptr = zc->workSpace; /* Statically sized space. * entropyWorkspace never moves, diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 476cdc148..c4b9bb13b 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -970,7 +970,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ U32 seqPos = cur; DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)", - last_pos, cur); + last_pos, cur); (void)last_pos; assert(storeEnd < ZSTD_OPT_NUM); DEBUGLOG(6, "last sequence copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", storeEnd, lastSequence.litlen, lastSequence.mlen, lastSequence.off); diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 8f4589d13..16a08fb6b 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -185,7 +185,7 @@ size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } static size_t ZSTD_startingInputLength(ZSTD_format_e format) { size_t const startingInputLength = (format==ZSTD_f_zstd1_magicless) ? - ZSTD_frameHeaderSize_prefix - ZSTD_frameIdSize : + ZSTD_frameHeaderSize_prefix - ZSTD_FRAMEIDSIZE : ZSTD_frameHeaderSize_prefix; ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE); /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */ @@ -278,7 +278,7 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) * Note 3 : Skippable Frame Identifiers are considered valid. */ unsigned ZSTD_isFrame(const void* buffer, size_t size) { - if (size < ZSTD_frameIdSize) return 0; + if (size < ZSTD_FRAMEIDSIZE) return 0; { U32 const magic = MEM_readLE32(buffer); if (magic == ZSTD_MAGICNUMBER) return 1; if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1; @@ -331,6 +331,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s size_t const minInputSize = ZSTD_startingInputLength(format); if (srcSize < minInputSize) return minInputSize; + if (src==NULL) return ERROR(GENERIC); /* invalid parameter */ if ( (format != ZSTD_f_zstd1_magicless) && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { @@ -339,7 +340,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + frame length */ memset(zfhPtr, 0, sizeof(*zfhPtr)); - zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_frameIdSize); + zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); zfhPtr->frameType = ZSTD_skippableFrame; return 0; } @@ -451,7 +452,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) size_t skippableSize; if (srcSize < ZSTD_skippableHeaderSize) return ERROR(srcSize_wrong); - skippableSize = MEM_readLE32((const BYTE *)src + ZSTD_frameIdSize) + skippableSize = MEM_readLE32((const BYTE *)src + ZSTD_FRAMEIDSIZE) + ZSTD_skippableHeaderSize; if (srcSize < skippableSize) { return ZSTD_CONTENTSIZE_ERROR; @@ -1763,7 +1764,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) #endif if ( (srcSize >= ZSTD_skippableHeaderSize) && (MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START ) { - return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + ZSTD_frameIdSize); + return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + ZSTD_FRAMEIDSIZE); } else { const BYTE* ip = (const BYTE*)src; const BYTE* const ipstart = ip; @@ -1797,7 +1798,6 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) if (zfh.checksumFlag) { /* Final frame content checksum */ if (remainingSize < 4) return ERROR(srcSize_wrong); ip += 4; - remainingSize -= 4; } return ip - ipstart; @@ -1932,7 +1932,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, size_t skippableSize; if (srcSize < ZSTD_skippableHeaderSize) return ERROR(srcSize_wrong); - skippableSize = MEM_readLE32((const BYTE*)src + ZSTD_frameIdSize) + skippableSize = MEM_readLE32((const BYTE*)src + ZSTD_FRAMEIDSIZE) + ZSTD_skippableHeaderSize; if (srcSize < skippableSize) return ERROR(srcSize_wrong); @@ -2057,7 +2057,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c case ZSTDds_getFrameHeaderSize : assert(src != NULL); if (dctx->format == ZSTD_f_zstd1) { /* allows header */ - assert(srcSize >= ZSTD_frameIdSize); /* to read skippable magic number */ + assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */ if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ memcpy(dctx->headerBuffer, src, srcSize); dctx->expected = ZSTD_skippableHeaderSize - srcSize; /* remaining to load to get full skippable frame header */ @@ -2167,7 +2167,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c assert(src != NULL); assert(srcSize <= ZSTD_skippableHeaderSize); memcpy(dctx->headerBuffer + (ZSTD_skippableHeaderSize - srcSize), src, srcSize); /* complete skippable header */ - dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_frameIdSize); /* note : dctx->expected can grow seriously large, beyond local buffer size */ + dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ dctx->stage = ZSTDds_skipFrame; return 0; @@ -2268,7 +2268,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict if (magic != ZSTD_MAGIC_DICTIONARY) { return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ } } - dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_frameIdSize); + dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); /* load entropy tables */ { size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize); @@ -2381,7 +2381,7 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict, ZSTD_dictContentType_e return 0; /* pure content mode */ } } - ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_frameIdSize); + ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); /* load entropy tables */ CHECK_E( ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted ); @@ -2510,7 +2510,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) { if (dictSize < 8) return 0; if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0; - return MEM_readLE32((const char*)dict + ZSTD_frameIdSize); + return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); } /*! ZSTD_getDictID_fromDDict() : @@ -2855,7 +2855,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict)); if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ - zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_frameIdSize); + zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); zds->stage = ZSTDds_skipFrame; } else { CHECK_F(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize)); diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index b8a51789a..a4d0a4481 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -698,7 +698,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, short litLengthNCount[MaxLL+1]; U32 repOffset[MAXREPOFFSET]; offsetCount_t bestRepOffset[ZSTD_REP_NUM+1]; - EStats_ress_t esr; + EStats_ress_t esr = { NULL, NULL, NULL }; ZSTD_parameters params; U32 u, huffLog = 11, Offlog = OffFSELog, mlLog = MLFSELog, llLog = LLFSELog, total; size_t pos = 0, errorCode; diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index a2e2cfa80..15000db6d 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -1093,6 +1093,7 @@ static size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, un if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Init, lay down lowprob symbols */ + memset(tableDecode, 0, sizeof(FSE_DECODE_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */ DTableH.tableLog = (U16)tableLog; for (s=0; s<=maxSymbolValue; s++) { diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index a5e1b1ffc..89a5fe7e6 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -1224,6 +1224,7 @@ size_t FSEv05_buildDTable(FSEv05_DTable* dt, const short* normalizedCounter, uns if (tableLog > FSEv05_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Init, lay down lowprob symbols */ + memset(tableDecode, 0, sizeof(FSEv05_FUNCTION_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */ DTableH.tableLog = (U16)tableLog; for (s=0; s<=maxSymbolValue; s++) { if (normalizedCounter[s]==-1) { diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 8b068b3e5..d8de77ca9 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -4006,7 +4006,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd, if (ZSTDv06_isError(hSize)) return hSize; if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); - zbd->lhSize += iend-ip; ip = iend; notDone = 0; + zbd->lhSize += iend-ip; *dstCapacityPtr = 0; return (hSize - zbd->lhSize) + ZSTDv06_blockHeaderSize; /* remaining header bytes + next block header */ } diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index 70b170f0f..9f95f62ad 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -3150,10 +3150,10 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, const BYTE* ip = (const BYTE*)src; if (srcSize < ZSTDv07_frameHeaderSize_min) return ZSTDv07_frameHeaderSize_min; + memset(fparamsPtr, 0, sizeof(*fparamsPtr)); if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) { if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv07_MAGIC_SKIPPABLE_START) { if (srcSize < ZSTDv07_skippableHeaderSize) return ZSTDv07_skippableHeaderSize; /* magic number + skippable frame length */ - memset(fparamsPtr, 0, sizeof(*fparamsPtr)); fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4); fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */ return 0; @@ -3175,11 +3175,13 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, U32 windowSize = 0; U32 dictID = 0; U64 frameContentSize = 0; - if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */ + if ((fhdByte & 0x08) != 0) /* reserved bits, which must be zero */ + return ERROR(frameParameter_unsupported); if (!directMode) { BYTE const wlByte = ip[pos++]; U32 const windowLog = (wlByte >> 3) + ZSTDv07_WINDOWLOG_ABSOLUTEMIN; - if (windowLog > ZSTDv07_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported); + if (windowLog > ZSTDv07_WINDOWLOG_MAX) + return ERROR(frameParameter_unsupported); windowSize = (1U << windowLog); windowSize += (windowSize >> 3) * (wlByte&7); } @@ -3201,7 +3203,8 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, case 3 : frameContentSize = MEM_readLE64(ip+pos); break; } if (!windowSize) windowSize = (U32)frameContentSize; - if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported); + if (windowSize > windowSizeMax) + return ERROR(frameParameter_unsupported); fparamsPtr->frameContentSize = frameContentSize; fparamsPtr->windowSize = windowSize; fparamsPtr->dictID = dictID; @@ -3220,11 +3223,10 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, - frame header not completely provided (`srcSize` too small) */ unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize) { - { ZSTDv07_frameParams fparams; - size_t const frResult = ZSTDv07_getFrameParams(&fparams, src, srcSize); - if (frResult!=0) return 0; - return fparams.frameContentSize; - } + ZSTDv07_frameParams fparams; + size_t const frResult = ZSTDv07_getFrameParams(&fparams, src, srcSize); + if (frResult!=0) return 0; + return fparams.frameContentSize; } From b1d9ca737a318d153eb439b4f4ed786bd0cf4384 Mon Sep 17 00:00:00 2001 From: George Lu Date: Mon, 13 Aug 2018 12:51:22 -0700 Subject: [PATCH 71/92] Add memoTable options -hashing memotable -no memotable --- tests/README.md | 9 +- tests/paramgrill.c | 424 ++++++++++++++++++++------------------------- 2 files changed, 197 insertions(+), 236 deletions(-) diff --git a/tests/README.md b/tests/README.md index 3c35ecf01..bdc9fff97 100644 --- a/tests/README.md +++ b/tests/README.md @@ -113,14 +113,15 @@ Full list of arguments dSpeed= : Minimum decompression speed cMem= : Maximum compression memory lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, - stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 99%) + stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%) : In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters (Lower value will begin with stronger strategies) (Default 90%) - preferSpeed= / preferRatio= - : Only affects lvl = invocations. Defines value placed on compression speed or ratio - when determining overall winner (default speed = 1, ratio = 5 for both, higher = more valued). + speedRatio= (accepts decimals) + : determines value of gains in speed vs gains in ratio + when determining overall winner (default 5 (1% ratio = 5% speed)). tries= : Maximum number of random restarts on a single strategy before switching (Default 3) Higher values will make optimizer run longer, more chances to find better solution. + memLog : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output diff --git a/tests/paramgrill.c b/tests/paramgrill.c index c2979cb83..4a7fc387a 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -117,10 +117,6 @@ typedef struct { U32 vals[NUM_PARAMS]; } paramValues_t; -/* list of parameters */ -static const varInds_t paramTable[NUM_PARAMS] = - { wlog_ind, clog_ind, hlog_ind, slog_ind, slen_ind, tlen_ind, strt_ind, fadt_ind }; - /* maximum value of parameters */ static const U32 mintable[NUM_PARAMS] = { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLENGTH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_fast, FADT_MIN }; @@ -236,7 +232,21 @@ static U32 g_target = 0; static U32 g_noSeed = 0; static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ -static U32 g_memoLimit = (U32)-1; //32 MB +static U32 g_memoTableLog = PARAM_UNSET; + +typedef enum { + directMap, + xxhashMap, + noMemo +} memoTableType_t; + +typedef struct { + memoTableType_t tableType; + BYTE* table; + size_t tableLen; + varInds_t varArray[NUM_PARAMS]; + size_t varLen; +} memoTable_t; typedef struct { BMK_result_t result; @@ -264,14 +274,12 @@ static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSp static BMK_result_t g_lvltarget; static int g_optmode = 0; -static U32 g_speedMultiplier = 1; -static U32 g_ratioMultiplier = 5; +static double g_ratioMultiplier = 5.; /* g_mode? */ /* range 0 - 99, measure of how strict */ -#define DEFAULT_STRICTNESS 99999 -static U32 g_strictness = DEFAULT_STRICTNESS; +static U32 g_strictness = PARAM_UNSET; void BMK_SetNbIterations(int nbLoops) { @@ -281,7 +289,6 @@ void BMK_SetNbIterations(int nbLoops) /* * Additional Global Variables (Defined Above Use) - * g_stratName * g_level_constraint * g_alreadyTested * g_maxTries @@ -324,7 +331,7 @@ static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) { } /* equivalent of ZSTD_adjustCParams for paramValues_t */ -static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t dictSize) { +static paramValues_t adjustParams(paramValues_t p, const size_t maxBlockSize, const size_t dictSize) { paramValues_t ot = p; varInds_t i; p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize)); @@ -337,7 +344,7 @@ static paramValues_t adjustParams(paramValues_t p, size_t maxBlockSize, size_t d } /* accuracy in seconds only, span can be multiple years */ -static U32 BMK_timeSpan(UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); } +static U32 BMK_timeSpan(const UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); } static size_t BMK_findMaxMem(U64 requiredMem) { @@ -418,7 +425,7 @@ static void findClockGranularity(void) { return 0; \ } } -static int paramValid(paramValues_t paramTarget) { +static int paramValid(const paramValues_t paramTarget) { U32 i; for(i = 0; i < NUM_PARAMS; i++) { CLAMPCHECK(paramTarget.vals[i], mintable[i], maxtable[i]); @@ -452,7 +459,7 @@ static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { fprintf(f, "\n"); } -static void BMK_displayOneResult(FILE* f, winnerInfo_t res, size_t srcSize) { +static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) { varInds_t v; res.params = cParamUnsetMin(res.params); fprintf(f," {"); @@ -498,7 +505,7 @@ static double resultDistLvl(const BMK_result_t result1, const BMK_result_t lvlRe if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) { return 0.0; } - return normalizedRatioGain1 * g_ratioMultiplier + normalizedCSpeedGain1 * g_speedMultiplier; + return normalizedRatioGain1 * g_ratioMultiplier + normalizedCSpeedGain1; } /* return true if r2 strictly better than r1 */ @@ -723,10 +730,10 @@ static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) { } /* what about low something like clog vs hlog in lvl 1? */ -static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t srcSize) { +static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) { return (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */ - || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= srcSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */ + || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= maxBlockSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */ || (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/ || (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */ || (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */ @@ -759,7 +766,7 @@ static void freeBuffers(const buffers_t b) { } /* srcBuffer will be freed by freeBuffers now */ -static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, size_t nbFiles, +static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, const size_t nbFiles, const size_t* fileSizes) { size_t pos = 0, n, blockSize; @@ -978,7 +985,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ZSTD_DCtx* dctx = ctx.dctx; /* warmimg up memory */ - /* can't do this if decode only */ for(i = 0; i < buf.nbBlocks; i++) { if(mode != BMK_decodeOnly) { RDG_genBuffer(dstPtrs[i], dstCapacities[i], 0.10, 0.50, 1); @@ -988,7 +994,6 @@ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t } /* Bench */ - { /* init args */ BMK_initCCtxArgs cctxprep; @@ -1234,21 +1239,21 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result snprintf(lvlstr, 15, " Level %2u ", cLevel); } - fprintf(f, "/* %s */ ", lvlstr); - BMK_displayOneResult(f, w, srcSize); - - if(TIMED) { + if(TIMED) { const U64 time = UTIL_clockSpanNano(g_time); const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); - fprintf(f, " - %1lu:%2lu:%05.2f", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); + fprintf(f, "%1lu:%2lu:%05.2f - ", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); } - fprintf(f, "\n"); + + fprintf(f, "/* %s */ ", lvlstr); + BMK_displayOneResult(f, w, srcSize); } static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize) { /* global winner used for constraints */ - static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } }; + /* cSize, cSpeed, dSpeed, cMem */ + static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } }; if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); @@ -1445,9 +1450,7 @@ static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters p } /* nullified useless params, to ensure count stats */ -/* no point in windowLog < chainLog (no point 2x chainLog for bt) */ -/* now with built in bounds-checking */ -/* no longer does anything with sanitizeVarArray + clampcheck */ +/* cleans up params for memoizing / display */ static paramValues_t sanitizeParams(paramValues_t params) { if (params.vals[strt_ind] == ZSTD_fast) @@ -1463,8 +1466,8 @@ static paramValues_t sanitizeParams(paramValues_t params) /* return: new length */ /* keep old array, will need if iter over strategy. */ /* prunes useless params */ -static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInds_t* varArray, const ZSTD_strategy strat) { - int i, j = 0; +static size_t sanitizeVarArray(varInds_t* varNew, const size_t varLength, const varInds_t* varArray, const ZSTD_strategy strat) { + size_t i, j = 0; for(i = 0; i < varLength; i++) { if( !((varArray[i] == clog_ind && strat == ZSTD_fast) || (varArray[i] == slog_ind && strat == ZSTD_fast) @@ -1481,9 +1484,9 @@ static int sanitizeVarArray(varInds_t* varNew, const int varLength, const varInd /* res should be NUM_PARAMS size */ /* constructs varArray from paramValues_t style parameter */ /* pass in using dict. */ -static int variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) { +static size_t variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) { varInds_t i; - int j = 0; + size_t j = 0; for(i = 0; i < NUM_PARAMS; i++) { if(paramConstraints.vals[i] == PARAM_UNSET) { if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */ @@ -1501,7 +1504,7 @@ static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues } /* varies ptr by nbChanges respecting varyParams*/ -static void paramVariation(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, const U32 nbChanges) +static void paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges) { paramValues_t p; U32 validated = 0; @@ -1509,8 +1512,8 @@ static void paramVariation(paramValues_t* ptr, const varInds_t* varyParams, cons U32 i; p = *ptr; for (i = 0 ; i < nbChanges ; i++) { - const U32 changeID = FUZ_rand(&g_rand) % (varyLen << 1); - paramVaryOnce(varyParams[changeID >> 1], ((changeID & 1) << 1) - 1, &p); + const U32 changeID = (U32)FUZ_rand(&g_rand) % (mtAll[p.vals[strt_ind]].varLen << 1); + paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1], ((changeID & 1) << 1) - 1, &p); } validated = paramValid(p); } @@ -1518,9 +1521,9 @@ static void paramVariation(paramValues_t* ptr, const varInds_t* varyParams, cons } /* length of memo table given free variables */ -static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { +static size_t memoTableLen(const varInds_t* varyParams, const size_t varyLen) { size_t arrayLen = 1; - int i; + size_t i; for(i = 0; i < varyLen; i++) { if(varyParams[i] == strt_ind) continue; /* strategy separated by table */ arrayLen *= rangetable[varyParams[i]]; @@ -1529,8 +1532,8 @@ static size_t memoTableLen(const varInds_t* varyParams, const int varyLen) { } /* returns unique index in memotable of compression parameters */ -static unsigned memoTableInd(const paramValues_t* ptr, const varInds_t* varyParams, const int varyLen) { - int i; +static unsigned memoTableIndDirect(const paramValues_t* ptr, const varInds_t* varyParams, const size_t varyLen) { + size_t i; unsigned ind = 0; for(i = 0; i < varyLen; i++) { varInds_t v = varyParams[i]; @@ -1540,84 +1543,82 @@ static unsigned memoTableInd(const paramValues_t* ptr, const varInds_t* varyPara return ind; } -/* inverse of above function (from index to parameters) */ -static void memoTableIndInv(paramValues_t* ptr, const varInds_t* varyParams, const int varyLen, size_t ind) { - int i; - for(i = varyLen - 1; i >= 0; i--) { - /* This is cleaner/easier to generalize but slower - varInds_t v = varyParams[i]; - if(v == strt_ind) continue; - ptr->vals[v] = rangeMap(v, ind % rangetable[v]); - ind /= rangetable[v]; */ - - switch(varyParams[i]) { - case wlog_ind: ptr->vals[wlog_ind] = ind % rangetable[wlog_ind] + mintable[wlog_ind]; ind /= rangetable[wlog_ind]; break; - case clog_ind: ptr->vals[clog_ind] = ind % rangetable[clog_ind] + mintable[clog_ind]; ind /= rangetable[clog_ind]; break; - case hlog_ind: ptr->vals[hlog_ind] = ind % rangetable[hlog_ind] + mintable[hlog_ind]; ind /= rangetable[hlog_ind]; break; - case slog_ind: ptr->vals[slog_ind] = ind % rangetable[slog_ind] + mintable[slog_ind]; ind /= rangetable[slog_ind]; break; - case slen_ind: ptr->vals[slen_ind] = ind % rangetable[slen_ind] + mintable[slen_ind]; ind /= rangetable[slen_ind]; break; - case tlen_ind: ptr->vals[tlen_ind] = tlen_table[(ind % rangetable[tlen_ind])]; ind /= rangetable[tlen_ind]; break; - case fadt_ind: ptr->vals[fadt_ind] = ind % rangetable[fadt_ind] - 1; ind /= rangetable[fadt_ind]; break; - case strt_ind: - case NUM_PARAMS: break; - } +static size_t memoTableGet(const memoTable_t* memoTableArray, const paramValues_t p) { + const memoTable_t mt = memoTableArray[p.vals[strt_ind]]; + switch(mt.tableType) { + case directMap: + return mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)]; + case xxhashMap: + return mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen]; + case noMemo: + return 0; } + return 0; /* should never happen, stop compiler warnings */ } -/* Initialize memoization table, which tracks and prevents repeated benchmarking - * of the same set of parameters. In addition, it is also used to immediately mark - * redundant / obviously non-optimal parameter configurations (e.g. wlog - 1 larger) - * than srcSize, clog > wlog, ... - */ -static void initMemoTable(U8* memoTable, paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { - size_t i; - size_t arrayLen = memoTableLen(varyParams, varyLen); - int j = 0; - assert(memoTable != NULL); - memset(memoTable, 0, arrayLen); - paramConstraints = cParamUnsetMin(paramConstraints); - - for(i = 0; i < arrayLen; i++) { - memoTableIndInv(¶mConstraints, varyParams, varyLen, i); - if(redundantParams(paramConstraints, target, srcSize)) { - memoTable[i] = 255; j++; - } +static void memoTableSet(const memoTable_t* memoTableArray, const paramValues_t p, const BYTE value) { + const memoTable_t mt = memoTableArray[p.vals[strt_ind]]; + switch(mt.tableType) { + case directMap: + mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)] = value; break; + case xxhashMap: + mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen] = value; break; + case noMemo: + break; } - - DEBUGOUTPUT("%d / %d Invalid\n", j, (int)arrayLen); } /* frees all allocated memotables */ -static void freeMemoTableArray(U8** mtAll) { +static void freeMemoTableArray(memoTable_t* const mtAll) { int i; if(mtAll == NULL) { return; } for(i = 1; i <= (int)ZSTD_btultra; i++) { - free(mtAll[i]); + free(mtAll[i].table); } free(mtAll); } /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ -static U8** createMemoTableArray(paramValues_t paramConstraints, const constraint_t target, const varInds_t* varyParams, const int varyLen, const size_t srcSize) { - varInds_t varNew[NUM_PARAMS]; - U8** mtAll = (U8**)calloc(sizeof(U8*),(ZSTD_btultra + 1)); +static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, const size_t varyLen) { + memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1)); int i; + if(mtAll == NULL) { return NULL; } for(i = 1; i <= (int)ZSTD_btultra; i++) { - const int varLenNew = sanitizeVarArray(varNew, varyLen, varyParams, i); - size_t mtl = memoTableLen(varNew, varLenNew); - if(mtl > g_memoLimit) { mtAll[i] = NULL; continue; } - mtAll[i] = malloc(sizeof(U8) * mtl); - if(mtAll[i] == NULL) { + mtAll[i].varLen = sanitizeVarArray(mtAll[i].varArray, varyLen, varyParams, i); + } + + /* no memoization */ + if(g_memoTableLog == 0) { + for(i = 1; i <= (int)ZSTD_btultra; i++) { + mtAll[i].tableType = noMemo; + mtAll[i].table = NULL; + mtAll[i].tableLen = 0; + } + return mtAll; + } + + /* hash table if normal table is too big */ + for(i = 1; i <= (int)ZSTD_btultra; i++) { + size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen); + mtAll[i].tableType = directMap; + + if(g_memoTableLog != PARAM_UNSET && mtl > (1ULL << g_memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */ + mtAll[i].tableType = xxhashMap; + mtl = (1ULL << g_memoTableLog); + } + + mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl); + mtAll[i].tableLen = mtl; + + if(mtAll[i].table == NULL) { freeMemoTableArray(mtAll); return NULL; } - paramConstraints.vals[strt_ind] = i; - initMemoTable(mtAll[i], paramConstraints, target, varNew, varLenNew, srcSize); } return mtAll; @@ -1638,10 +1639,6 @@ static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mas #define PARAMTABLEMASK (PARAMTABLESIZE-1) static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */ -/* -#define NB_TESTS_PLAYED(p) \ - g_alreadyTested[(XXH64(((void*)&sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK] */ - static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(cParamsToPVals(p))); return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK]; @@ -1651,10 +1648,8 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners, ZSTD_compressionParameters params, const buffers_t buf, const contexts_t ctx) { - int nbVariations = 0; + int nbVariations = 0, i; UTIL_time_t const clockStart = UTIL_getTime(); - const U32 unconstrained[NUM_PARAMS] = { 0, 1, 2, 3, 4, 5, 6 }; /* no fadt */ - while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { paramValues_t p = cParamsToPVals(params); @@ -1662,7 +1657,9 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners, BYTE* b; if (nbVariations++ > g_maxNbVariations) break; - paramVariation(&p, unconstrained, NUM_PARAMS, 4); + + do { for(i = 0; i < 4; i++) { paramVaryOnce(FUZ_rand(&g_rand) % (strt_ind + 1), ((FUZ_rand(&g_rand) & 1) << 1) - 1, &p); } } + while(!paramValid(p)); p2 = pvalsToCParams(p); @@ -1683,28 +1680,31 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners, } /* Sets pc to random unmeasured set of parameters */ -/* Doesn't do strategy! */ -static void randomConstrainedParams(paramValues_t* pc, const varInds_t* varArray, const int varLen, const U8* memoTable) +/* specifiy strategy */ +static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTableArray, const ZSTD_strategy st) { size_t j; - for(j = 0; j < MIN(g_memoLimit, memoTableLen(varArray, varLen)); j++) { + const memoTable_t mt = memoTableArray[st]; + pc->vals[strt_ind] = st; + for(j = 0; j < MIN(1ULL << g_memoTableLog, memoTableLen(mt.varArray, mt.varLen)); j++) { int i; for(i = 0; i < NUM_PARAMS; i++) { - varInds_t v = varArray[i]; - if(v == strt_ind) continue; //don't do strategy (dependent on MT) + varInds_t v = mt.varArray[i]; + if(v == strt_ind) continue; //skip, already specified pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); } - if(memoTable == NULL || memoTable[memoTableInd(pc, varArray, varLen)]) { break; } + if(!(memoTableGet(memoTableArray, *pc))) break; //only pick unpicked params. } } /* Completely random parameter selection */ static ZSTD_compressionParameters randomParams(void) { - paramValues_t p; - p.vals[strt_ind] = rangeMap(strt_ind, rangeMap(strt_ind, FUZ_rand(&g_rand) % rangetable[strt_ind])); - randomConstrainedParams(&p, paramTable, NUM_PARAMS, NULL); + paramValues_t p; varInds_t v; + for(v = 0; v < NUM_PARAMS; v++) { + p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); + } return pvalsToCParams(p); } @@ -1866,68 +1866,62 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam return ret; } +#define CBENCHMARK(conditional, resultvar, tmpret, mode, loopmode, sec) { \ + if(conditional) { \ + BMK_return_t tmpret = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, mode, loopmode, sec); \ + if(tmpret.error) { DEBUGOUTPUT("Benchmarking failed\n"); return ERROR_RESULT; } \ + if(mode != BMK_decodeOnly) { \ + resultvar.cSpeed = tmpret.result.cSpeed; \ + resultvar.cSize = tmpret.result.cSize; \ + resultvar.cMem = tmpret.result.cMem; \ + } \ + if(mode != BMK_compressOnly) { resultvar.dSpeed = tmpret.result.dSpeed; } \ + } \ +} + /* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ #define VARIANCE 1.2 -#define HIGH_VARIANCE 100.0 static int allBench(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, const paramValues_t cParams, const constraint_t target, BMK_result_t* winnerResult, int feas) { - BMK_return_t benchres; - BMK_result_t resultMax; + BMK_result_t resultMax, benchres; U64 loopDurationC = 0, loopDurationD = 0; double uncertaintyConstantC = 3., uncertaintyConstantD = 3.; double winnerRS; - /* initial benchmarking, gives exact ratio and memory, warms up future runs */ - benchres = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, BMK_iterMode, 1); + CBENCHMARK(1, benchres, tmp, BMK_both, BMK_iterMode, 1); winnerRS = resultScore(*winnerResult, buf.srcSize, target); DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); - if(benchres.error) { - DEBUGOUTPUT("Benchmarking failed\n"); - return ERROR_RESULT; - } - *resultPtr = benchres.result; + *resultPtr = benchres; /* calculate uncertainty in compression / decompression runs */ - if(benchres.result.cSpeed) { - loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.cSpeed); + if(benchres.cSpeed) { + loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed); uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC); } - if(benchres.result.dSpeed) { - loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.result.dSpeed); + if(benchres.dSpeed) { + loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed); uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD); } /* anything with worse ratio in feas is definitely worse, discard */ - if(feas && benchres.result.cSize < winnerResult->cSize && !g_optmode) { + if(feas && benchres.cSize < winnerResult->cSize && !g_optmode) { return WORSE_RESULT; } /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ - if(loopDurationC < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } - benchres = benchres2; - } - if(loopDurationD < TIMELOOP_NANOSEC / 10) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_iterMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } - benchres.result.dSpeed = benchres2.result.dSpeed; - } + CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_compressOnly, BMK_iterMode, 1); + CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_decodeOnly, BMK_iterMode, 1); - *resultPtr = benchres.result; + *resultPtr = benchres; - /* optimistic assumption of benchres.result */ - resultMax = benchres.result; + /* optimistic assumption of benchres */ + resultMax = benchres; resultMax.cSpeed *= uncertaintyConstantC * VARIANCE; resultMax.dSpeed *= uncertaintyConstantD * VARIANCE; @@ -1938,29 +1932,15 @@ static int allBench(BMK_result_t* resultPtr, return WORSE_RESULT; } - /* Final full run if estimates are unclear */ - if(loopDurationC < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_compressOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } - benchres.result.cSpeed = benchres2.result.cSpeed; - } + CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC, benchres, tmp, BMK_compressOnly, BMK_timeMode, 1); + CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC, benchres, tmp, BMK_decodeOnly, BMK_timeMode, 1); - if(loopDurationD < TIMELOOP_NANOSEC) { - BMK_return_t benchres2 = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_decodeOnly, BMK_timeMode, 1); - if(benchres2.error) { - return ERROR_RESULT; - } - benchres.result.dSpeed = benchres2.result.dSpeed; - } - - *resultPtr = benchres.result; + *resultPtr = benchres; /* compare by resultScore when in infeas */ /* compare by compareResultLT when in feas */ - if((!feas && (resultScore(benchres.result, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || - (feas && (compareResultLT(*winnerResult, benchres.result, target, buf.srcSize))) ) { + if((!feas && (resultScore(benchres, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || + (feas && (compareResultLT(*winnerResult, benchres, target, buf.srcSize))) ) { return BETTER_RESULT; } else { return WORSE_RESULT; @@ -1968,19 +1948,17 @@ static int allBench(BMK_result_t* resultPtr, } #define INFEASIBLE_THRESHOLD 200 - /* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ static int benchMemo(BMK_result_t* resultPtr, const buffers_t buf, const contexts_t ctx, const paramValues_t cParams, const constraint_t target, - BMK_result_t* winnerResult, U8* const memoTable, - const varInds_t* varyParams, const int varyLen, const int feas) { + BMK_result_t* winnerResult, memoTable_t* const memoTableArray, + const int feas) { static int bmcount = 0; - size_t memind = memoTableInd(&cParams, varyParams, varyLen); int res; - if((memoTable == NULL && redundantParams(cParams, target, buf.maxBlockSize)) || ((memoTable != NULL) && memoTable[memind] >= INFEASIBLE_THRESHOLD)) { return WORSE_RESULT; } + if(memoTableGet(memoTableArray, cParams) >= INFEASIBLE_THRESHOLD || redundantParams(cParams, target, buf.maxBlockSize)) { return WORSE_RESULT; } res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas); @@ -1990,8 +1968,8 @@ static int benchMemo(BMK_result_t* resultPtr, } BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize); - if(memoTable != NULL && (res == BETTER_RESULT || feas)) { - memoTable[memind] = 255; + if(res == BETTER_RESULT || feas) { + memoTableSet(memoTableArray, cParams, 255); /* what happens if collisions are frequent */ } return res; } @@ -2013,8 +1991,7 @@ static int benchMemo(BMK_result_t* resultPtr, * all generation after random should be sanitized. (maybe sanitize random) */ static winnerInfo_t climbOnce(const constraint_t target, - const varInds_t* varArray, const int varLen, ZSTD_strategy strat, - U8** memoTableArray, + memoTable_t* mtAll, const buffers_t buf, const contexts_t ctx, const paramValues_t init) { /* @@ -2026,8 +2003,6 @@ static winnerInfo_t climbOnce(const constraint_t target, winnerInfo_t candidateInfo, winnerInfo; int better = 1; int feas = 0; - varInds_t varNew[NUM_PARAMS]; - int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); winnerInfo = initWinnerInfo(init); candidateInfo = winnerInfo; @@ -2037,7 +2012,9 @@ static winnerInfo_t climbOnce(const constraint_t target, DEBUGOUTPUT("Climb Part 1\n"); while(better) { - int i, dist, offset; + int offset; + size_t i, dist; + const size_t varLen = mtAll[cparam.vals[strt_ind]].varLen; better = 0; DEBUGOUTPUT("Start\n"); cparam = winnerInfo.params; @@ -2048,17 +2025,13 @@ static winnerInfo_t climbOnce(const constraint_t target, for(offset = -1; offset <= 1; offset += 2) { CHECKTIME(winnerInfo); candidateInfo.params = cparam; - paramVaryOnce(varArray[i], offset, &candidateInfo.params); + paramVaryOnce(mtAll[cparam.vals[strt_ind]].varArray[i], offset, &candidateInfo.params); if(paramValid(candidateInfo.params)) { int res; - if(strat != candidateInfo.params.vals[strt_ind]) { /* maybe only try strategy switching after exhausting non-switching solutions? */ - strat = candidateInfo.params.vals[strt_ind]; - varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); - } res = benchMemo(&candidateInfo.result, buf, ctx, - sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat], - varNew, varLenNew, feas); + sanitizeParams(candidateInfo.params), target, &winnerInfo.result, mtAll, feas); + DEBUGOUTPUT("Res: %d\n", res); if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ winnerInfo = candidateInfo; BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize); @@ -2081,16 +2054,11 @@ static winnerInfo_t climbOnce(const constraint_t target, CHECKTIME(winnerInfo); candidateInfo.params = cparam; /* param error checking already done here */ - paramVariation(&candidateInfo.params, varArray, varLen, dist); - - if(strat != candidateInfo.params.vals[strt_ind]) { - strat = candidateInfo.params.vals[strt_ind]; - varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); - } + paramVariation(&candidateInfo.params, mtAll, (U32)dist); res = benchMemo(&candidateInfo.result, buf, ctx, - sanitizeParams(candidateInfo.params), target, &winnerInfo.result, memoTableArray[strat], - varNew, varLenNew, feas); + sanitizeParams(candidateInfo.params), target, &winnerInfo.result, mtAll, feas); + DEBUGOUTPUT("Res: %d\n", res); if(res == BETTER_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize); @@ -2134,11 +2102,8 @@ static winnerInfo_t optimizeFixedStrategy( const buffers_t buf, const contexts_t ctx, const constraint_t target, paramValues_t paramTarget, const ZSTD_strategy strat, - const varInds_t* varArray, const int varLen, - U8** memoTableArray, const int tries) { + memoTable_t* memoTableArray, const int tries) { int i = 0; - varInds_t varNew[NUM_PARAMS]; - int varLenNew = sanitizeVarArray(varNew, varLen, varArray, strat); paramValues_t init; winnerInfo_t winnerInfo, candidateInfo; @@ -2150,14 +2115,15 @@ static winnerInfo_t optimizeFixedStrategy( init = paramTarget; - while(i < tries) { + for(i = 0; i < tries; i++) { DEBUGOUTPUT("Restart\n"); - randomConstrainedParams(&init, varNew, varLenNew, memoTableArray[strat]); - candidateInfo = climbOnce(target, varArray, varLen, strat, memoTableArray, buf, ctx, init); + do { randomConstrainedParams(&init, memoTableArray, strat); } while(redundantParams(init, target, buf.maxBlockSize)); //only non-redundant params + candidateInfo = climbOnce(target, memoTableArray, buf, ctx, init); if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { winnerInfo = candidateInfo; BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); i = 0; + continue; } CHECKTIME(winnerInfo); i++; @@ -2217,9 +2183,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ { varInds_t varArray [NUM_PARAMS]; int ret = 0; - const int varLen = variableParams(paramTarget, varArray, dictFileName != NULL); + const size_t varLen = variableParams(paramTarget, varArray, dictFileName != NULL); winnerInfo_t winner = initWinnerInfo(emptyParams()); - U8** allMT = NULL; + memoTable_t* allMT = NULL; paramValues_t paramBase; contexts_t ctx; buffers_t buf; @@ -2246,38 +2212,19 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ optimizerAdjustInput(¶mTarget, buf.maxBlockSize); paramBase = cParamUnsetMin(paramTarget); - /* if strategy is fixed, only init that part of memotable */ - if(paramTarget.vals[strt_ind] != PARAM_UNSET) { - varInds_t varNew[NUM_PARAMS]; - int varLenNew = sanitizeVarArray(varNew, varLen, varArray, paramTarget.vals[strt_ind]); - allMT = (U8**)calloc(sizeof(U8*), (ZSTD_btultra + 1)); - if(allMT == NULL) { - ret = 57; - goto _cleanUp; - } + // TODO: if strategy is fixed, only init that row + allMT = createMemoTableArray(varArray, varLen); - allMT[paramTarget.vals[strt_ind]] = malloc(sizeof(U8) * memoTableLen(varNew, varLenNew)); - - if(allMT[paramTarget.vals[strt_ind]] == NULL) { - ret = 58; - goto _cleanUp; - } - - initMemoTable(allMT[paramTarget.vals[strt_ind]], paramTarget, target, varNew, varLenNew, buf.maxBlockSize); - } else { - allMT = createMemoTableArray(paramTarget, target, varArray, varLen, buf.maxBlockSize); - } - if(!allMT) { DISPLAY("MemoTable Init Error\n"); ret = 2; goto _cleanUp; } - /* default strictness = Maximum for */ - if(g_strictness == DEFAULT_STRICTNESS) { + /* default strictnesses */ + if(g_strictness == PARAM_UNSET) { if(g_optmode) { - g_strictness = 99; + g_strictness = 100; } else { g_strictness = 90; } @@ -2371,7 +2318,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } DEBUGOUTPUT("Real Opt\n"); - /* start 'real' tests */ + /* start 'real' optimization */ { int bestStrategy = (int)winner.params.vals[strt_ind]; if(paramTarget.vals[strt_ind] == PARAM_UNSET) { @@ -2380,8 +2327,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ { /* one iterations of hill climbing with the level-defined parameters. */ - winnerInfo_t w1 = climbOnce(target, varArray, varLen, st, allMT, - buf, ctx, winner.params); + winnerInfo_t w1 = climbOnce(target, allMT, buf, ctx, winner.params); if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) { winner = w1; } @@ -2392,8 +2338,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ winnerInfo_t wc; DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); - wc = optimizeFixedStrategy(buf, ctx, target, paramBase, - st, varArray, varLen, allMT, tries); + wc = optimizeFixedStrategy(buf, ctx, target, paramBase, st, allMT, tries); if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { winner = wc; @@ -2406,8 +2351,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ CHECKTIMEGT(ret, 0, _cleanUp); } } else { - winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], - varArray, varLen, allMT, g_maxTries); + winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], allMT, g_maxTries); } } @@ -2467,6 +2411,22 @@ static unsigned readU32FromChar(const char** stringPtr) return result * sign; } +static double readDoubleFromChar(const char** stringPtr) +{ + double result = 0, divide = 10; + while ((**stringPtr >='0') && (**stringPtr <='9')) { + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + } + if(**stringPtr!='.') { + return result; + } + (*stringPtr)++; + while ((**stringPtr >='0') && (**stringPtr <='9')) { + result += (double)(**stringPtr - '0') / divide, divide *= 10, (*stringPtr)++ ; + } + return result; +} + static int usage(const char* exename) { DISPLAY( "Usage :\n"); @@ -2558,11 +2518,10 @@ int main(int argc, const char** argv) PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); PARSE_SUB_ARGS("strict=", "stc=", g_strictness); - PARSE_SUB_ARGS("preferSpeed=", "prfSpd=", g_speedMultiplier); - PARSE_SUB_ARGS("preferRatio=", "prfRto=", g_ratioMultiplier); PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries); - PARSE_SUB_ARGS("memoLimit=", "memo=", g_memoLimit); + PARSE_SUB_ARGS("memoLimitLog=", "memLog=", g_memoTableLog); if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "speedForRatio=") || longCommandWArg(&argument, "speedRatio=")) { g_ratioMultiplier = readDoubleFromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } DISPLAY("invalid optimization parameter \n"); return 1; @@ -2690,6 +2649,7 @@ int main(int argc, const char** argv) break; case 's': + argument++; seperateFiles = 1; break; From 3dcfe5cc2c19b5a4fd864c5d1c40aec4a9e5f1dc Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 14 Aug 2018 14:24:05 -0700 Subject: [PATCH 72/92] begin display changes --- tests/paramgrill.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 4a7fc387a..c56f0a71c 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -53,6 +53,8 @@ static const int g_maxNbVariations = 64; * Macros **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(n, ...) if(g_displayLevel >= n) { fprintf(stderr, __VA_ARGS__); } + #define TIMED 0 #ifndef DEBUG # define DEBUG 0 @@ -233,6 +235,7 @@ static U32 g_noSeed = 0; static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ static U32 g_memoTableLog = PARAM_UNSET; +static U32 g_displayLevel = 3; typedef enum { directMap, @@ -1282,7 +1285,6 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res /* the table */ fprintf(f, "================================\n"); for(n = g_winners; n != NULL; n = n->next) { - fprintf(f, "\r%79s\r", ""); BMK_displayOneResult(f, n->res, srcSize); } fprintf(f, "================================\n"); @@ -2313,7 +2315,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); - BMK_translateAdvancedParams(stdout, winner.params); } } @@ -2364,7 +2365,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* end summary */ BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); - BMK_translateAdvancedParams(stdout, winner.params); DISPLAY("grillParams size - optimizer completed \n"); } @@ -2653,6 +2653,14 @@ int main(int argc, const char** argv) seperateFiles = 1; break; + case 'q': + g_displayLevel--; + break; + + case 'v': + g_displayLevel++; + break; + /* load dictionary file (only applicable for optimizer rn) */ case 'D': if(i == argc - 1) { /* last argument, return error. */ From 4d9c6f51b8f34970fc292d82f23a2fa7f3bdede0 Mon Sep 17 00:00:00 2001 From: George Lu Date: Tue, 14 Aug 2018 15:54:07 -0700 Subject: [PATCH 73/92] -q -v options --- tests/paramgrill.c | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index c56f0a71c..ef6d6d46a 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -1257,22 +1257,22 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res /* global winner used for constraints */ /* cSize, cSpeed, dSpeed, cMem */ static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } }; - if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { + if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize) || g_displayLevel >= 4) { if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { DISPLAY("New Winner: \n"); } - BMK_printWinner(f, cLevel, result, params, srcSize); + if(g_displayLevel >= 2) { BMK_printWinner(f, cLevel, result, params, srcSize); } if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { - BMK_translateAdvancedParams(f, params); + if(g_displayLevel >= 1) { BMK_translateAdvancedParams(f, params); } g_winner.result = result; g_winner.params = params; } } //prints out tradeoff table if using lvloptimize - if(g_optmode && g_optimizer) { + if(g_optmode && g_optimizer && (DEBUG || g_displayLevel == 3)) { winnerInfo_t w; winner_ll_node* n; w.result = result; @@ -2205,9 +2205,9 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } if(nbFiles == 1) { - DISPLAY("Loading %s... \r", fileNamesTable[0]); + DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[0]); } else { - DISPLAY("Loading %lu Files... \r", (unsigned long)nbFiles); + DISPLAYLEVEL(2, "Loading %lu Files... \r", (unsigned long)nbFiles); } /* sanitize paramTarget */ @@ -2273,16 +2273,16 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* bench */ - DISPLAY("\r%79s\r", ""); + DISPLAYLEVEL(2, "\r%79s\r", ""); if(nbFiles == 1) { - DISPLAY("optimizing for %s", fileNamesTable[0]); + DISPLAYLEVEL(2, "optimizing for %s", fileNamesTable[0]); } else { - DISPLAY("optimizing for %lu Files", (unsigned long)nbFiles); + DISPLAYLEVEL(2, "optimizing for %lu Files", (unsigned long)nbFiles); } - if(target.cSpeed != 0) { DISPLAY(" - limit compression speed %u MB/s", target.cSpeed >> 20); } - if(target.dSpeed != 0) { DISPLAY(" - limit decompression speed %u MB/s", target.dSpeed >> 20); } - if(target.cMem != (U32)-1) { DISPLAY(" - limit memory %u MB", target.cMem >> 20); } + if(target.cSpeed != 0) { DISPLAYLEVEL(2," - limit compression speed %u MB/s", target.cSpeed >> 20); } + if(target.dSpeed != 0) { DISPLAYLEVEL(2, " - limit decompression speed %u MB/s", target.dSpeed >> 20); } + if(target.cMem != (U32)-1) { DISPLAYLEVEL(2, " - limit memory %u MB", target.cMem >> 20); } DISPLAY("\n"); findClockGranularity(); @@ -2364,7 +2364,8 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } /* end summary */ - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); + BMK_displayOneResult(stdout, winner, buf.srcSize); + BMK_translateAdvancedParams(stdout, winner.params); DISPLAY("grillParams size - optimizer completed \n"); } @@ -2501,7 +2502,7 @@ int main(int argc, const char** argv) assert(argc>=1); /* for exename */ /* Welcome message */ - DISPLAY(WELCOME_MESSAGE); + DISPLAYLEVEL(2, WELCOME_MESSAGE); for(i=1; i Date: Tue, 14 Aug 2018 16:51:39 -0700 Subject: [PATCH 74/92] Clean up repetitive display Add documentation --- tests/README.md | 2 ++ tests/paramgrill.c | 28 +++++++++++++--------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/README.md b/tests/README.md index bdc9fff97..60fcebfbc 100644 --- a/tests/README.md +++ b/tests/README.md @@ -127,6 +127,8 @@ Full list of arguments -v : Prints Benchmarking output -D : Next argument dictionary file -s : Benchmark all files separately + -q : Quiet, repeat for more quiet + -v : Verbose, cancels quiet, repeat for more volume ``` Any inputs afterwards are treated as files to benchmark. diff --git a/tests/paramgrill.c b/tests/paramgrill.c index ef6d6d46a..ef005dd9a 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -235,7 +235,7 @@ static U32 g_noSeed = 0; static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ static U32 g_memoTableLog = PARAM_UNSET; -static U32 g_displayLevel = 3; +static int g_displayLevel = 3; typedef enum { directMap, @@ -889,7 +889,7 @@ static int createBuffers(buffers_t* buff, const char* const * const fileNamesTab goto _cleanUp; } - DISPLAY("Loading %s... \r", fileNamesTable[n]); + DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]); if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ { @@ -2020,7 +2020,6 @@ static winnerInfo_t climbOnce(const constraint_t target, better = 0; DEBUGOUTPUT("Start\n"); cparam = winnerInfo.params; - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); candidateInfo.params = cparam; /* all dist-1 candidates */ for(i = 0; i < varLen; i++) { @@ -2036,7 +2035,6 @@ static winnerInfo_t climbOnce(const constraint_t target, DEBUGOUTPUT("Res: %d\n", res); if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ winnerInfo = candidateInfo; - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; @@ -2063,7 +2061,6 @@ static winnerInfo_t climbOnce(const constraint_t target, DEBUGOUTPUT("Res: %d\n", res); if(res == BETTER_RESULT) { /* synonymous with better in this case*/ winnerInfo = candidateInfo; - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, sanitizeParams(winnerInfo.params), target, buf.srcSize); better = 1; if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { bestFeasible1 = winnerInfo; @@ -2284,7 +2281,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(target.dSpeed != 0) { DISPLAYLEVEL(2, " - limit decompression speed %u MB/s", target.dSpeed >> 20); } if(target.cMem != (U32)-1) { DISPLAYLEVEL(2, " - limit memory %u MB", target.cMem >> 20); } - DISPLAY("\n"); + DISPLAYLEVEL(2, "\n"); findClockGranularity(); { @@ -2309,7 +2306,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ winner.params = CParams; } - CHECKTIMEGT(ret, 0, _cleanUp); /* if pass time limit, stop */ + CHECKTIMEGT(ret, 0, _displayCleanUp); /* if pass time limit, stop */ /* if the current params are too slow, just stop. */ if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; } } @@ -2332,7 +2329,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ if(compareResultLT(winner.result, w1.result, target, buf.srcSize)) { winner = w1; } - CHECKTIMEGT(ret, 0, _cleanUp); + CHECKTIMEGT(ret, 0, _displayCleanUp); } while(st && tries > 0) { @@ -2349,7 +2346,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ st = nextStrategy(st, bestStrategy); tries -= TRY_DECAY; } - CHECKTIMEGT(ret, 0, _cleanUp); + CHECKTIMEGT(ret, 0, _displayCleanUp); } } else { winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], allMT, g_maxTries); @@ -2364,12 +2361,13 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ goto _cleanUp; } /* end summary */ - BMK_displayOneResult(stdout, winner, buf.srcSize); +_displayCleanUp: + if(g_displayLevel >= 0) { BMK_displayOneResult(stdout, winner, buf.srcSize); } BMK_translateAdvancedParams(stdout, winner.params); - DISPLAY("grillParams size - optimizer completed \n"); + DISPLAYLEVEL(1, "grillParams size - optimizer completed \n"); } -_cleanUp: +_cleanUp: freeContexts(ctx); freeBuffers(buf); freeMemoTableArray(allMT); @@ -2501,9 +2499,6 @@ int main(int argc, const char** argv) assert(argc>=1); /* for exename */ - /* Welcome message */ - DISPLAYLEVEL(2, WELCOME_MESSAGE); - for(i=1; i Date: Tue, 14 Aug 2018 18:04:58 -0700 Subject: [PATCH 75/92] silencing params --- tests/README.md | 1 + tests/paramgrill.c | 70 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/tests/README.md b/tests/README.md index 60fcebfbc..3af08f7be 100644 --- a/tests/README.md +++ b/tests/README.md @@ -122,6 +122,7 @@ Full list of arguments tries= : Maximum number of random restarts on a single strategy before switching (Default 3) Higher values will make optimizer run longer, more chances to find better solution. memLog : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization + --display= : which params to display, uses all --zstd parameter names and 'cParams' to display only compression parameters. -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output diff --git a/tests/paramgrill.c b/tests/paramgrill.c index ef005dd9a..2f00c198c 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -237,6 +237,8 @@ static UTIL_time_t g_time; /* to be used to compare solution finding speeds to c static U32 g_memoTableLog = PARAM_UNSET; static int g_displayLevel = 3; +static BYTE g_silenceParams[NUM_PARAMS]; + typedef enum { directMap, xxhashMap, @@ -447,31 +449,34 @@ static paramValues_t cParamUnsetMin(paramValues_t paramTarget) { } static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { - U32 i; + varInds_t v; + int first = 1; fprintf(f,"--zstd="); - for(i = 0; i < NUM_PARAMS; i++) { - fprintf(f,"%s=", g_paramNames[i]); + for(v = 0; v < NUM_PARAMS; v++) { + if(g_silenceParams[v]) { continue; } + if(!first) { fprintf(f, ","); } + fprintf(f,"%s=", g_paramNames[v]); - if(i == strt_ind) { fprintf(f,"%u", params.vals[i]); } - else { displayParamVal(f, i, params.vals[i], 0); } - - if(i != NUM_PARAMS - 1) { - fprintf(f, ","); - } + if(v == strt_ind) { fprintf(f,"%u", params.vals[v]); } + else { displayParamVal(f, v, params.vals[v], 0); } + first = 0; } fprintf(f, "\n"); } static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) { varInds_t v; + int first = 1; res.params = cParamUnsetMin(res.params); fprintf(f," {"); for(v = 0; v < NUM_PARAMS; v++) { - if(v != 0) { fprintf(f, ","); } + if(g_silenceParams[v]) { continue; } + if(!first) { fprintf(f, ","); } displayParamVal(f, v, res.params.vals[v], 3); + first = 0; } - fprintf(f, "}, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + fprintf(f, " }, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB)); } @@ -2545,6 +2550,43 @@ int main(int argc, const char** argv) } continue; /* if not return, success */ + + } else if (longCommandWArg(&argument, "--display=")) { + /* Decode command (note : aggregated commands are allowed) */ + memset(g_silenceParams, 1, sizeof(g_silenceParams)); + for ( ; ;) { + int found = 0; + varInds_t v; + for(v = 0; v < NUM_PARAMS; v++) { + if(longCommandWArg(&argument, g_shortParamNames[v]) || longCommandWArg(&argument, g_paramNames[v])) { + g_silenceParams[v] = 0; + found = 1; + } + } + if(longCommandWArg(&argument, "compressionParameters") || longCommandWArg(&argument, "cParams")) { + for(v = 0; v <= strt_ind; v++) { + g_silenceParams[v] = 0; + } + found = 1; + } + + + if(found) { + if(argument[0]==',') { + continue; + } else { + break; + } + } + DISPLAY("invalid parameter name parameter \n"); + return 1; + } + + if (argument[0] != 0) { + DISPLAY("invalid --display format\n"); + return 1; /* check the end of string */ + } + continue; } else if (argument[0]=='-') { argument++; @@ -2650,13 +2692,11 @@ int main(int argc, const char** argv) break; case 'q': - argument++; - g_displayLevel--; + while (argument[0] == 'q') { argument++; g_displayLevel--; } break; case 'v': - argument++; - g_displayLevel++; + while (argument[0] == 'v') { argument++; g_displayLevel++; } break; /* load dictionary file (only applicable for optimizer rn) */ From ee77ddc28ddb0ff9bae63ee61e58a3ecfb315728 Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 15 Aug 2018 11:46:19 -0700 Subject: [PATCH 76/92] Fix wraparound --- tests/paramgrill.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 2f00c198c..0d814bf7c 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -144,8 +144,8 @@ static const char* g_shortParamNames[NUM_PARAMS] = { "wlog", "clog", "hlog","slog", "slen", "tlen", "strt", "fadt" }; /* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */ -static U32 rangeMap(varInds_t param, U32 ind) { - ind = MIN(ind, rangetable[param] - 1); +static U32 rangeMap(varInds_t param, int ind) { + ind = MAX(MIN(ind, (int)rangetable[param] - 1), 0); switch(param) { case tlen_ind: return tlen_table[ind]; @@ -166,7 +166,7 @@ static U32 rangeMap(varInds_t param, U32 ind) { } /* inverse of rangeMap */ -static U32 invRangeMap(varInds_t param, U32 value) { +static int invRangeMap(varInds_t param, U32 value) { value = MIN(MAX(mintable[param], value), maxtable[param]); switch(param) { case tlen_ind: /* bin search */ @@ -186,7 +186,7 @@ static U32 invRangeMap(varInds_t param, U32 value) { return lo; } case fadt_ind: - return value + 1; + return (int)value + 1; case wlog_ind: case clog_ind: case hlog_ind: @@ -196,7 +196,7 @@ static U32 invRangeMap(varInds_t param, U32 value) { return value - mintable[param]; case NUM_PARAMS: DISPLAY("Error, not a valid param\n "); - return (U32)-1; + return -2; } return 0; /* should never happen, stop compiler warnings */ } @@ -1545,7 +1545,7 @@ static unsigned memoTableIndDirect(const paramValues_t* ptr, const varInds_t* va for(i = 0; i < varyLen; i++) { varInds_t v = varyParams[i]; if(v == strt_ind) continue; /* exclude strategy from memotable */ - ind *= rangetable[v]; ind += invRangeMap(v, ptr->vals[v]); + ind *= rangetable[v]; ind += (unsigned)invRangeMap(v, ptr->vals[v]); } return ind; } From b234870c33745401138af634909acf46ec3363fc Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 15 Aug 2018 14:29:49 -0700 Subject: [PATCH 77/92] clarify display README --- tests/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 3af08f7be..442285dbc 100644 --- a/tests/README.md +++ b/tests/README.md @@ -122,7 +122,10 @@ Full list of arguments tries= : Maximum number of random restarts on a single strategy before switching (Default 3) Higher values will make optimizer run longer, more chances to find better solution. memLog : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization - --display= : which params to display, uses all --zstd parameter names and 'cParams' to display only compression parameters. + --display= : specifiy which parameters are included in the output + can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters + (Default: display all params available) + -P# : generated sample compressibility -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output From 42a02ab745976068255fe1a43915b70b8f9aa104 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 15 Aug 2018 14:35:38 -0700 Subject: [PATCH 78/92] fixed minor warnings issued by scan-build --- Makefile | 4 ++-- contrib/seekable_format/zstdseek_decompress.c | 5 ++++- lib/decompress/zstd_decompress.c | 1 + programs/dibio.c | 6 +++++- programs/dibio.h | 2 +- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 6c4ab9c9b..f8441e60d 100644 --- a/Makefile +++ b/Makefile @@ -114,7 +114,7 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT)) HOST_OS = POSIX -CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release +CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release .PHONY: list list: @@ -351,7 +351,7 @@ bmi32build: clean $(CC) -v CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test -staticAnalyze: clean +staticAnalyze: $(CC) -v CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all endif diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c index b006ff834..2b109b9d0 100644 --- a/contrib/seekable_format/zstdseek_decompress.c +++ b/contrib/seekable_format/zstdseek_decompress.c @@ -56,6 +56,7 @@ #include /* malloc, free */ #include /* FILE* */ +#include #define XXH_STATIC_LINKING_ONLY #define XXH_NAMESPACE ZSTD_ @@ -112,7 +113,7 @@ static int ZSTD_seekable_read_buff(void* opaque, void* buffer, size_t n) static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin) { - buffWrapper_t* buff = (buffWrapper_t*) opaque; + buffWrapper_t* const buff = (buffWrapper_t*) opaque; unsigned long long newOffset; switch (origin) { case SEEK_SET: @@ -124,6 +125,8 @@ static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin) case SEEK_END: newOffset = (unsigned long long)buff->size - offset; break; + default: + assert(0); /* not possible */ } if (newOffset > buff->size) { return -1; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 16a08fb6b..5d9f0ba0b 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -330,6 +330,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s const BYTE* ip = (const BYTE*)src; size_t const minInputSize = ZSTD_startingInputLength(format); + memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */ if (srcSize < minInputSize) return minInputSize; if (src==NULL) return ERROR(GENERIC); /* invalid parameter */ diff --git a/programs/dibio.c b/programs/dibio.c index 5d1f6d6c4..fbb8aa6fa 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -27,6 +27,7 @@ #include /* memset */ #include /* fprintf, fopen, ftello64 */ #include /* errno */ +#include #include "mem.h" /* read */ #include "error_private.h" @@ -165,6 +166,7 @@ static U32 DiB_rand(U32* src) static void DiB_shuffle(const char** fileNamesTable, unsigned nbFiles) { U32 seed = 0xFD2FB528; unsigned i; + assert(nbFiles >= 1); for (i = nbFiles - 1; i > 0; --i) { unsigned const j = DiB_rand(&seed) % (i + 1); const char* const tmp = fileNamesTable[j]; @@ -310,7 +312,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, /* Load input buffer */ DISPLAYLEVEL(3, "Shuffling input files\n"); DiB_shuffle(fileNamesTable, nbFiles); - nbFiles = DiB_loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel); + DiB_loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel); { size_t dictSize; if (params) { @@ -319,6 +321,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, srcBuffer, sampleSizes, fs.nbSamples, *params); } else if (optimizeCover) { + assert(coverParams != NULL); dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer, sampleSizes, fs.nbSamples, coverParams); @@ -327,6 +330,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d, coverParams->steps, splitPercentage); } } else { + assert(coverParams != NULL); dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer, sampleSizes, fs.nbSamples, *coverParams); } diff --git a/programs/dibio.h b/programs/dibio.h index 499e30365..31a6b4bdb 100644 --- a/programs/dibio.h +++ b/programs/dibio.h @@ -33,7 +33,7 @@ */ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, - ZDICT_legacy_params_t *params, ZDICT_cover_params_t *coverParams, + ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams, int optimizeCover); #endif From 46be2ef5d8f48b6509e6813f74f3622db0fbc09f Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 15 Aug 2018 14:00:57 -0700 Subject: [PATCH 79/92] Remove unused stuff --- tests/README.md | 3 +- tests/paramgrill.c | 164 ++++++++++++++++++++------------------------- 2 files changed, 73 insertions(+), 94 deletions(-) diff --git a/tests/README.md b/tests/README.md index 442285dbc..c1128571a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -95,7 +95,6 @@ Full list of arguments ``` -T# : set level 1 speed objective -B# : cut input into blocks of size # (default : single block) - -i# : iteration loops -S : benchmarks a single run (example command: -Sl3w10h12) w# - windowLog h# - hashLog @@ -119,7 +118,7 @@ Full list of arguments speedRatio= (accepts decimals) : determines value of gains in speed vs gains in ratio when determining overall winner (default 5 (1% ratio = 5% speed)). - tries= : Maximum number of random restarts on a single strategy before switching (Default 3) + tries= : Maximum number of random restarts on a single strategy before switching (Default 5) Higher values will make optimizer run longer, more chances to find better solution. memLog : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization --display= : specifiy which parameters are included in the output diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 0d814bf7c..e5e1e2a36 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -37,9 +37,6 @@ #define WELCOME_MESSAGE "*** %s %s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR #define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */ - -#define NBLOOPS 2 -#define TIMELOOP (2 * SEC_TO_MICRO) #define NB_LEVELS_TRACKED 22 /* ensured being >= ZSTD_maxCLevel() in BMK_init_level_constraints() */ static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); @@ -54,12 +51,12 @@ static const int g_maxNbVariations = 64; **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAYLEVEL(n, ...) if(g_displayLevel >= n) { fprintf(stderr, __VA_ARGS__); } +#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } #define TIMED 0 #ifndef DEBUG # define DEBUG 0 #endif -#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } #undef MIN #undef MAX @@ -223,21 +220,30 @@ static void displayParamVal(FILE* f, varInds_t param, U32 value, int width) { typedef BYTE U8; +/* General Utility */ static U32 g_timeLimit_s = 99999; /* about 27 hours */ -static U32 g_nbIterations = NBLOOPS; -static double g_compressibility = COMPRESSIBILITY_DEFAULT; +static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ static U32 g_blockSize = 0; static U32 g_rand = 1; + +/* Display */ +static int g_displayLevel = 3; +static BYTE g_silenceParams[NUM_PARAMS]; + +/* Mode Selection */ static U32 g_singleRun = 0; static U32 g_optimizer = 0; +static int g_optmode = 0; + +/* For cLevel Table generation */ static U32 g_target = 0; static U32 g_noSeed = 0; -static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ -static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ -static U32 g_memoTableLog = PARAM_UNSET; -static int g_displayLevel = 3; -static BYTE g_silenceParams[NUM_PARAMS]; +/* For optimizer */ +static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ +static double g_ratioMultiplier = 5.; +static U32 g_strictness = PARAM_UNSET; /* range 0 - 99, measure of how strict */ +static BMK_result_t g_lvltarget; typedef enum { directMap, @@ -258,11 +264,6 @@ typedef struct { paramValues_t params; } winnerInfo_t; -typedef struct { - BMK_result_t result; - ZSTD_compressionParameters params; -} oldWinnerInfo_t; - typedef struct { U32 cSpeed; /* bytes / sec */ U32 dSpeed; @@ -276,27 +277,13 @@ struct winner_ll_node { }; static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ -static BMK_result_t g_lvltarget; -static int g_optmode = 0; - -static double g_ratioMultiplier = 5.; - -/* g_mode? */ - -/* range 0 - 99, measure of how strict */ -static U32 g_strictness = PARAM_UNSET; - -void BMK_SetNbIterations(int nbLoops) -{ - g_nbIterations = nbLoops; - DISPLAY("- %u iterations -\n", g_nbIterations); -} /* * Additional Global Variables (Defined Above Use) * g_level_constraint * g_alreadyTested * g_maxTries + * g_clockGranularity */ /*-******************************************************* @@ -340,7 +327,7 @@ static paramValues_t adjustParams(paramValues_t p, const size_t maxBlockSize, co paramValues_t ot = p; varInds_t i; p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize)); - + if(!dictSize) { p.vals[fadt_ind] = 0; } /* retain value of all other parameters */ for(i = strt_ind + 1; i < NUM_PARAMS; i++) { p.vals[i] = ot.vals[i]; @@ -1306,7 +1293,7 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res } } -static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize) +static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, const size_t srcSize) { int cLevel; @@ -1314,11 +1301,11 @@ static void BMK_printWinners2(FILE* f, const oldWinnerInfo_t* winners, const siz fprintf(f, " /* W, C, H, S, L, T, strat */ \n"); for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++) - BMK_printWinner(f, cLevel, winners[cLevel].result, cParamsToPVals(winners[cLevel].params), srcSize); + BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize); } -static void BMK_printWinners(FILE* f, const oldWinnerInfo_t* winners, const size_t srcSize) +static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, const size_t srcSize) { fseek(f, 0, SEEK_SET); BMK_printWinners2(f, winners, srcSize); @@ -1355,14 +1342,14 @@ static void BMK_init_level_constraints(int bytePerSec_level1) } } } -static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters params, +static int BMK_seed(winnerInfo_t* winners, const paramValues_t params, const buffers_t buf, const contexts_t ctx) { BMK_result_t testResult; int better = 0; int cLevel; - BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(params)); + BMK_benchParam(&testResult, buf, ctx, params); for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { @@ -1370,15 +1357,15 @@ static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters p continue; /* not fast enough for this level */ if (testResult.dSpeed < g_level_constraint[cLevel].dSpeed_min) continue; /* not fast enough for this level */ - if (params.windowLog > g_level_constraint[cLevel].windowLog_max) + if (params.vals[wlog_ind] > g_level_constraint[cLevel].windowLog_max) continue; /* too much memory for this level */ - if (params.strategy > g_level_constraint[cLevel].strategy_max) + if (params.vals[strt_ind] > g_level_constraint[cLevel].strategy_max) continue; /* forbidden strategy for this level */ if (winners[cLevel].result.cSize==0) { /* first solution for this cLevel */ winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, cParamsToPVals(params), buf.srcSize); + BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); better = 1; continue; } @@ -1389,13 +1376,13 @@ static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters p double O_ratio = (double)buf.srcSize / winners[cLevel].result.cSize; double W_ratioNote = log (W_ratio); double O_ratioNote = log (O_ratio); - size_t W_DMemUsed = (1 << params.windowLog) + (16 KB); - size_t O_DMemUsed = (1 << winners[cLevel].params.windowLog) + (16 KB); + size_t W_DMemUsed = (1 << params.vals[wlog_ind]) + (16 KB); + size_t O_DMemUsed = (1 << winners[cLevel].params.vals[wlog_ind]) + (16 KB); double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); - size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(params); - size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(winners[cLevel].params); + size_t W_CMemUsed = (1 << params.vals[wlog_ind]) + ZSTD_estimateCCtxSize_usingCParams(pvalsToCParams(params)); + size_t O_CMemUsed = (1 << winners[cLevel].params.vals[wlog_ind]) + ZSTD_estimateCCtxSize_usingCParams(pvalsToCParams(winners[cLevel].params)); double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); @@ -1443,7 +1430,7 @@ static int BMK_seed(oldWinnerInfo_t* winners, const ZSTD_compressionParameters p winners[cLevel].result = testResult; winners[cLevel].params = params; - BMK_printWinner(stdout, cLevel, testResult, cParamsToPVals(params), buf.srcSize); + BMK_printWinner(stdout, cLevel, testResult, params, buf.srcSize); better = 1; } } @@ -1587,7 +1574,7 @@ static void freeMemoTableArray(memoTable_t* const mtAll) { /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ -static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, const size_t varyLen) { +static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, const size_t varyLen, U32 memoTableLog) { memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1)); int i; @@ -1600,7 +1587,7 @@ static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, cons } /* no memoization */ - if(g_memoTableLog == 0) { + if(memoTableLog == 0) { for(i = 1; i <= (int)ZSTD_btultra; i++) { mtAll[i].tableType = noMemo; mtAll[i].table = NULL; @@ -1614,9 +1601,9 @@ static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, cons size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen); mtAll[i].tableType = directMap; - if(g_memoTableLog != PARAM_UNSET && mtl > (1ULL << g_memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */ + if(memoTableLog != PARAM_UNSET && mtl > (1ULL << memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */ mtAll[i].tableType = xxhashMap; - mtl = (1ULL << g_memoTableLog); + mtl = (1ULL << memoTableLog); } mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl); @@ -1646,21 +1633,19 @@ static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mas #define PARAMTABLEMASK (PARAMTABLESIZE-1) static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */ -static BYTE* NB_TESTS_PLAYED(ZSTD_compressionParameters p) { - ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(cParamsToPVals(p))); +static BYTE* NB_TESTS_PLAYED(paramValues_t p) { + ZSTD_compressionParameters p2 = pvalsToCParams(sanitizeParams(p)); return &g_alreadyTested[(XXH64((void*)&p2, sizeof(p2), 0) >> 3) & PARAMTABLEMASK]; } -static void playAround(FILE* f, oldWinnerInfo_t* winners, - ZSTD_compressionParameters params, +static void playAround(FILE* f, winnerInfo_t* winners, + paramValues_t p, const buffers_t buf, const contexts_t ctx) { int nbVariations = 0, i; UTIL_time_t const clockStart = UTIL_getTime(); while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { - paramValues_t p = cParamsToPVals(params); - ZSTD_compressionParameters p2; BYTE* b; if (nbVariations++ > g_maxNbVariations) break; @@ -1668,20 +1653,18 @@ static void playAround(FILE* f, oldWinnerInfo_t* winners, do { for(i = 0; i < 4; i++) { paramVaryOnce(FUZ_rand(&g_rand) % (strt_ind + 1), ((FUZ_rand(&g_rand) & 1) << 1) - 1, &p); } } while(!paramValid(p)); - p2 = pvalsToCParams(p); - /* exclude faster if already played params */ - if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p2))-1)) + if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) continue; /* test */ - b = NB_TESTS_PLAYED(p2); + b = NB_TESTS_PLAYED(p); (*b)++; - if (!BMK_seed(winners, p2, buf, ctx)) continue; + if (!BMK_seed(winners, p, buf, ctx)) continue; /* improvement found => search more */ BMK_printWinners(f, winners, buf.srcSize); - playAround(f, winners, p2, buf, ctx); + playAround(f, winners, p, buf, ctx); } } @@ -1693,7 +1676,7 @@ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTa size_t j; const memoTable_t mt = memoTableArray[st]; pc->vals[strt_ind] = st; - for(j = 0; j < MIN(1ULL << g_memoTableLog, memoTableLen(mt.varArray, mt.varLen)); j++) { + for(j = 0; j < mt.tableLen; j++) { int i; for(i = 0; i < NUM_PARAMS; i++) { varInds_t v = mt.varArray[i]; @@ -1706,23 +1689,24 @@ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTa } /* Completely random parameter selection */ -static ZSTD_compressionParameters randomParams(void) +static paramValues_t randomParams(void) { - paramValues_t p; varInds_t v; - for(v = 0; v < NUM_PARAMS; v++) { + varInds_t v; paramValues_t p; + for(v = 0; v <= NUM_PARAMS; v++) { p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); } - return pvalsToCParams(p); + return p; } static void BMK_selectRandomStart( - FILE* f, oldWinnerInfo_t* winners, + FILE* f, winnerInfo_t* winners, const buffers_t buf, const contexts_t ctx) { U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); - if ((id==0) || (winners[id].params.windowLog==0)) { + if ((id==0) || (winners[id].params.vals[wlog_ind]==0)) { /* use some random entry */ - ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), buf.srcSize, 0); + paramValues_t const p = adjustParams(cParamsToPVals(pvalsToCParams(randomParams())), /* defaults nonCompression parameters */ + buf.srcSize, 0); playAround(f, winners, p, buf, ctx); } else { playAround(f, winners, winners[id].params, buf, ctx); @@ -1731,8 +1715,8 @@ static void BMK_selectRandomStart( static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) { - ZSTD_compressionParameters params; - oldWinnerInfo_t winners[NB_LEVELS_TRACKED+1]; + paramValues_t params; + winnerInfo_t winners[NB_LEVELS_TRACKED+1]; const char* const rfName = "grillResults.txt"; FILE* const f = fopen(rfName, "w"); @@ -1745,9 +1729,9 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) BMK_init_level_constraints(g_target * (1 MB)); } else { /* baseline config for level 1 */ - ZSTD_compressionParameters const l1params = ZSTD_getCParams(1, buf.maxBlockSize, ctx.dictSize); + paramValues_t const l1params = cParamsToPVals(ZSTD_getCParams(1, buf.maxBlockSize, ctx.dictSize)); BMK_result_t testResult; - BMK_benchParam(&testResult, buf, ctx, cParamsToPVals(l1params)); + BMK_benchParam(&testResult, buf, ctx, l1params); BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); } @@ -1755,7 +1739,7 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); int i; for (i=0; i<=maxSeeds; i++) { - params = ZSTD_getCParams(i, buf.maxBlockSize, 0); + params = cParamsToPVals(ZSTD_getCParams(i, buf.maxBlockSize, 0)); BMK_seed(winners, params, buf, ctx); } } BMK_printWinners(f, winners, buf.srcSize); @@ -1788,7 +1772,7 @@ static int benchOnce(const buffers_t buf, const contexts_t ctx) { return 0; } -static int benchSample(void) +static int benchSample(double compressibility) { const char* const name = "Sample 10MB"; size_t const benchedSize = 10 MB; @@ -1803,7 +1787,7 @@ static int benchSample(void) return 2; } - RDG_genBuffer(srcBuffer, benchedSize, g_compressibility, 0.0, 0); + RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); if(createBuffersFromMemory(&buf, srcBuffer, 1, &benchedSize)) { DISPLAY("Buffer Creation Error\n"); @@ -1819,7 +1803,7 @@ static int benchSample(void) /* bench */ DISPLAY("\r%79s\r", ""); - DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100)); + DISPLAY("using %s %i%%: \n", name, (int)(compressibility*100)); if(g_singleRun) { ret = benchOnce(buf, ctx); @@ -2180,10 +2164,11 @@ static int nextStrategy(const int currentStrategy, const int bestStrategy) { * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) */ -static int g_maxTries = 3; +static int g_maxTries = 5; #define TRY_DECAY 1 -static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget, int cLevelOpt, int cLevelRun) +static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget, + int cLevelOpt, int cLevelRun, U32 memoTableLog) { varInds_t varArray [NUM_PARAMS]; int ret = 0; @@ -2217,7 +2202,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ paramBase = cParamUnsetMin(paramTarget); // TODO: if strategy is fixed, only init that row - allMT = createMemoTableArray(varArray, varLen); + allMT = createMemoTableArray(varArray, varLen, memoTableLog); if(!allMT) { DISPLAY("MemoTable Init Error\n"); @@ -2446,7 +2431,6 @@ static int usage_advanced(void) DISPLAY( "\nAdvanced options :\n"); DISPLAY( " -T# : set level 1 speed objective \n"); DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n"); - DISPLAY( " -i# : iteration loops (default : %i) \n", NBLOOPS); DISPLAY( " --optimize= : same as -O with more verbose syntax (see README.md)\n"); DISPLAY( " -S : Single run \n"); DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n"); @@ -2497,6 +2481,8 @@ int main(int argc, const char** argv) U32 main_pause = 0; int cLevelOpt = 0, cLevelRun = 0; int seperateFiles = 0; + double compressibility = COMPRESSIBILITY_DEFAULT; + U32 memoTableLog = PARAM_UNSET; constraint_t target = { 0, 0, (U32)-1 }; paramValues_t paramTarget = emptyParams(); @@ -2520,7 +2506,7 @@ int main(int argc, const char** argv) PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); PARSE_SUB_ARGS("strict=", "stc=", g_strictness); PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries); - PARSE_SUB_ARGS("memoLimitLog=", "memLog=", g_memoTableLog); + PARSE_SUB_ARGS("memoLimitLog=", "memLog=", memoTableLog); if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; } if (longCommandWArg(&argument, "speedForRatio=") || longCommandWArg(&argument, "speedRatio=")) { g_ratioMultiplier = readDoubleFromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } @@ -2600,18 +2586,12 @@ int main(int argc, const char** argv) /* Pause at the end (hidden option) */ case 'p': main_pause = 1; argument++; break; - /* Modify Nb Iterations */ - - case 'i': - argument++; - g_nbIterations = readU32FromChar(&argument); - break; /* Sample compressibility (when no file provided) */ case 'P': argument++; { U32 const proba32 = readU32FromChar(&argument); - g_compressibility = (double)proba32 / 100.; + compressibility = (double)proba32 / 100.; } break; @@ -2730,13 +2710,13 @@ int main(int argc, const char** argv) DISPLAY("Optimizer Expects File\n"); return 1; } else { - result = benchSample(); + result = benchSample(compressibility); } } else { if(seperateFiles) { for(i = 0; i < argc - filenamesStart; i++) { if (g_optimizer) { - result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevelOpt, cLevelRun); + result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog); if(result) { DISPLAY("Error on File %d", i); return result; } } else { result = benchFiles(argv+filenamesStart + i, 1, dictFileName, cLevelRun); @@ -2745,7 +2725,7 @@ int main(int argc, const char** argv) } } else { if (g_optimizer) { - result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevelOpt, cLevelRun); + result = optimizeForSize(argv+filenamesStart, argc-filenamesStart, dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog); } else { result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevelRun); } From 3f8b10baa1b5e70530ec5d893ebaaf8cc9962d04 Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 15 Aug 2018 14:27:07 -0700 Subject: [PATCH 80/92] consts --- tests/paramgrill.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index e5e1e2a36..1025e4323 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -1263,7 +1263,6 @@ static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t res } } - //prints out tradeoff table if using lvloptimize if(g_optmode && g_optimizer && (DEBUG || g_displayLevel == 3)) { winnerInfo_t w; winner_ll_node* n; @@ -1574,9 +1573,9 @@ static void freeMemoTableArray(memoTable_t* const mtAll) { /* inits memotables for all (including mallocs), all strategies */ /* takes unsanitized varyParams */ -static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, const size_t varyLen, U32 memoTableLog) { +static memoTable_t* createMemoTableArray(const paramValues_t p, const varInds_t* const varyParams, const size_t varyLen, const U32 memoTableLog) { memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1)); - int i; + ZSTD_strategy i, stratMin = ZSTD_fast, stratMax = ZSTD_btultra; if(mtAll == NULL) { return NULL; @@ -1596,8 +1595,14 @@ static memoTable_t* createMemoTableArray(const varInds_t* const varyParams, cons return mtAll; } - /* hash table if normal table is too big */ - for(i = 1; i <= (int)ZSTD_btultra; i++) { + + if(p.vals[strt_ind] != PARAM_UNSET) { + stratMin = p.vals[strt_ind]; + stratMax = p.vals[strt_ind]; + } + + + for(i = stratMin; i <= stratMax; i++) { size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen); mtAll[i].tableType = directMap; @@ -1680,11 +1685,11 @@ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTa int i; for(i = 0; i < NUM_PARAMS; i++) { varInds_t v = mt.varArray[i]; - if(v == strt_ind) continue; //skip, already specified + if(v == strt_ind) continue; pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); } - if(!(memoTableGet(memoTableArray, *pc))) break; //only pick unpicked params. + if(!(memoTableGet(memoTableArray, *pc))) break; /* only pick unpicked params. */ } } @@ -2105,7 +2110,7 @@ static winnerInfo_t optimizeFixedStrategy( for(i = 0; i < tries; i++) { DEBUGOUTPUT("Restart\n"); - do { randomConstrainedParams(&init, memoTableArray, strat); } while(redundantParams(init, target, buf.maxBlockSize)); //only non-redundant params + do { randomConstrainedParams(&init, memoTableArray, strat); } while(redundantParams(init, target, buf.maxBlockSize)); candidateInfo = climbOnce(target, memoTableArray, buf, ctx, init); if(compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { winnerInfo = candidateInfo; @@ -2168,7 +2173,7 @@ static int g_maxTries = 5; #define TRY_DECAY 1 static int optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, const char* dictFileName, constraint_t target, paramValues_t paramTarget, - int cLevelOpt, int cLevelRun, U32 memoTableLog) + const int cLevelOpt, const int cLevelRun, const U32 memoTableLog) { varInds_t varArray [NUM_PARAMS]; int ret = 0; @@ -2201,8 +2206,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ optimizerAdjustInput(¶mTarget, buf.maxBlockSize); paramBase = cParamUnsetMin(paramTarget); - // TODO: if strategy is fixed, only init that row - allMT = createMemoTableArray(varArray, varLen, memoTableLog); + allMT = createMemoTableArray(paramTarget, varArray, varLen, memoTableLog); if(!allMT) { DISPLAY("MemoTable Init Error\n"); From 8a296d3e1f4f8a5177b084098170b122b7e33c26 Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 15 Aug 2018 14:57:10 -0700 Subject: [PATCH 81/92] Move Stuff around Group similar functions together, remove outdated comments --- tests/paramgrill.c | 1443 ++++++++++++++++++++++---------------------- 1 file changed, 734 insertions(+), 709 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 1025e4323..7ebcdeec0 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -65,8 +65,6 @@ static const int g_maxNbVariations = 64; #define CUSTOM_LEVEL 99 #define BASE_CLEVEL 1 -#undef ZSTD_WINDOWLOG_MAX -#define ZSTD_WINDOWLOG_MAX 27 //no long range stuff for now. #define FADT_MIN 0 #define FADT_MAX ((U32)-1) @@ -242,7 +240,7 @@ static U32 g_noSeed = 0; /* For optimizer */ static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ static double g_ratioMultiplier = 5.; -static U32 g_strictness = PARAM_UNSET; /* range 0 - 99, measure of how strict */ +static U32 g_strictness = PARAM_UNSET; /* range 1 - 100, measure of how strict */ static BMK_result_t g_lvltarget; typedef enum { @@ -287,9 +285,23 @@ static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSp */ /*-******************************************************* -* Private functions +* General Util Functions *********************************************************/ +/* nullified useless params, to ensure count stats */ +/* cleans up params for memoizing / display */ +static paramValues_t sanitizeParams(paramValues_t params) +{ + if (params.vals[strt_ind] == ZSTD_fast) + params.vals[clog_ind] = 0, params.vals[slog_ind] = 0; + if (params.vals[strt_ind] == ZSTD_dfast) + params.vals[slog_ind] = 0; + if (params.vals[strt_ind] != ZSTD_btopt && params.vals[strt_ind] != ZSTD_btultra && params.vals[strt_ind] != ZSTD_fast) + params.vals[tlen_ind] = 0; + + return params; +} + static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) { ZSTD_compressionParameters c; memset(&c, 0, sizeof(ZSTD_compressionParameters)); @@ -335,9 +347,6 @@ static paramValues_t adjustParams(paramValues_t p, const size_t maxBlockSize, co return p; } -/* accuracy in seconds only, span can be multiple years */ -static U32 BMK_timeSpan(const UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); } - static size_t BMK_findMaxMem(U64 requiredMem) { size_t const step = 64 MB; @@ -356,6 +365,8 @@ static size_t BMK_findMaxMem(U64 requiredMem) return (size_t) requiredMem; } +/* accuracy in seconds only, span can be multiple years */ +static U32 BMK_timeSpan(const UTIL_time_t tStart) { return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); } static U32 FUZ_rotl32(U32 x, U32 r) { @@ -374,42 +385,6 @@ U32 FUZ_rand(U32* src) return rand32 >> 5; } -/** longCommandWArg() : - * check if *stringPtr is the same as longCommand. - * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. - * @return 0 and doesn't modify *stringPtr otherwise. - * from zstdcli.c - */ -static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) -{ - size_t const comSize = strlen(longCommand); - int const result = !strncmp(*stringPtr, longCommand, comSize); - if (result) *stringPtr += comSize; - return result; -} - -static U64 g_clockGranularity = 100000000ULL; - -static void findClockGranularity(void) { - UTIL_time_t clockStart = UTIL_getTime(); - U64 el1 = 0, el2 = 0; - int i = 0; - do { - el1 = el2; - el2 = UTIL_clockSpanNano(clockStart); - if(el1 < el2) { - U64 iv = el2 - el1; - if(g_clockGranularity > iv) { - g_clockGranularity = iv; - i = 0; - } else { - i++; - } - } - } while(i < 10); - DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity); -} - /* allows zeros */ #define CLAMPCHECK(val,min,max) { \ if (((val)<(min)) | ((val)>(max))) { \ @@ -435,38 +410,95 @@ static paramValues_t cParamUnsetMin(paramValues_t paramTarget) { return paramTarget; } -static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { - varInds_t v; - int first = 1; - fprintf(f,"--zstd="); - for(v = 0; v < NUM_PARAMS; v++) { - if(g_silenceParams[v]) { continue; } - if(!first) { fprintf(f, ","); } - fprintf(f,"%s=", g_paramNames[v]); - - if(v == strt_ind) { fprintf(f,"%u", params.vals[v]); } - else { displayParamVal(f, v, params.vals[v], 0); } - first = 0; +static paramValues_t emptyParams(void) { + U32 i; + paramValues_t p; + for(i = 0; i < NUM_PARAMS; i++) { + p.vals[i] = PARAM_UNSET; } - fprintf(f, "\n"); + return p; } -static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) { - varInds_t v; - int first = 1; - res.params = cParamUnsetMin(res.params); - fprintf(f," {"); - for(v = 0; v < NUM_PARAMS; v++) { - if(g_silenceParams[v]) { continue; } - if(!first) { fprintf(f, ","); } - displayParamVal(f, v, res.params.vals[v], 3); - first = 0; +static winnerInfo_t initWinnerInfo(const paramValues_t p) { + winnerInfo_t w1; + w1.result.cSpeed = 0.; + w1.result.dSpeed = 0.; + w1.result.cMem = (size_t)-1; + w1.result.cSize = (size_t)-1; + w1.params = p; + return w1; +} + +static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mask) { + U32 i; + for(i = 0; i < NUM_PARAMS; i++) { + if(mask.vals[i] != PARAM_UNSET) { + base.vals[i] = mask.vals[i]; + } + } + return base; +} + +/* amt will probably always be \pm 1? */ +/* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ +/* take max/min bounds into account as well? */ +static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) { + ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); +} + +/* varies ptr by nbChanges respecting varyParams*/ +static void paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges) +{ + paramValues_t p; + U32 validated = 0; + while (!validated) { + U32 i; + p = *ptr; + for (i = 0 ; i < nbChanges ; i++) { + const U32 changeID = (U32)FUZ_rand(&g_rand) % (mtAll[p.vals[strt_ind]].varLen << 1); + paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1], ((changeID & 1) << 1) - 1, &p); + } + validated = paramValid(p); + } + *ptr = p; +} + +/* Completely random parameter selection */ +static paramValues_t randomParams(void) +{ + varInds_t v; paramValues_t p; + for(v = 0; v <= NUM_PARAMS; v++) { + p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); + } + return p; +} + +static U64 g_clockGranularity = 100000000ULL; + +static void findClockGranularity(void) { + UTIL_time_t clockStart = UTIL_getTime(); + U64 el1 = 0, el2 = 0; + int i = 0; + do { + el1 = el2; + el2 = UTIL_clockSpanNano(clockStart); + if(el1 < el2) { + U64 iv = el2 - el1; + if(g_clockGranularity > iv) { + g_clockGranularity = iv; + i = 0; + } else { + i++; } - - fprintf(f, " }, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", - (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB)); + } + } while(i < 10); + DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity); } +/*-************************************ +* Optimizer Util Functions +**************************************/ + /* checks results are feasible */ static int feasible(const BMK_result_t results, const constraint_t target) { return (results.cSpeed >= target.cSpeed) && (results.dSpeed >= target.dSpeed) && (results.cMem <= target.cMem) && (!g_optmode || results.cSize <= g_lvltarget.cSize); @@ -523,52 +555,324 @@ static constraint_t relaxTarget(constraint_t target) { return target; } -/*-******************************************************* -* Bench functions -*********************************************************/ - -static paramValues_t emptyParams(void) { - U32 i; - paramValues_t p; - for(i = 0; i < NUM_PARAMS; i++) { - p.vals[i] = PARAM_UNSET; +static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) { + varInds_t v; + for(v = 0; v < NUM_PARAMS; v++) { + if(pc->vals[v] != PARAM_UNSET) { + U32 newval = MIN(MAX(pc->vals[v], mintable[v]), maxtable[v]); + if(newval != pc->vals[v]) { + pc->vals[v] = newval; + DISPLAY("Warning: parameter %s not in valid range, adjusting to ", g_paramNames[v]); displayParamVal(stderr, v, newval, 0); DISPLAY("\n"); + } + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET) { + + U32 sshb = maxBlockSize > 1 ? ZSTD_highbit32((U32)(maxBlockSize-1)) + 1 : 1; + /* edge case of highBit not working for 0 */ + + if(maxBlockSize < (1ULL << 31) && sshb + 1 < pc->vals[wlog_ind]) { + U32 adjust = MAX(mintable[wlog_ind], sshb); + if(adjust != pc->vals[wlog_ind]) { + pc->vals[wlog_ind] = adjust; + DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", pc->vals[wlog_ind]); + } + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) { + U32 maxclog; + if(pc->vals[strt_ind] == PARAM_UNSET || pc->vals[strt_ind] >= (U32)ZSTD_btlazy2) { + maxclog = pc->vals[wlog_ind] + 1; + } else { + maxclog = pc->vals[wlog_ind]; + } + + if(pc->vals[clog_ind] > maxclog) { + pc->vals[clog_ind] = maxclog; + DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", pc->vals[clog_ind]); + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[hlog_ind] != PARAM_UNSET) { + if(pc->vals[wlog_ind] + 1 < pc->vals[hlog_ind]) { + pc->vals[hlog_ind] = pc->vals[wlog_ind] + 1; + DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", pc->vals[hlog_ind]); + } + } + + if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) { + if(pc->vals[slog_ind] > pc->vals[clog_ind]) { + pc->vals[clog_ind] = pc->vals[slog_ind]; + DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", pc->vals[slog_ind]); + } } - return p; } -static winnerInfo_t initWinnerInfo(const paramValues_t p) { - winnerInfo_t w1; - w1.result.cSpeed = 0.; - w1.result.dSpeed = 0.; - w1.result.cMem = (size_t)-1; - w1.result.cSize = (size_t)-1; - w1.params = p; - return w1; +/* what about low something like clog vs hlog in lvl 1? */ +static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) { + return + (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */ + || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= maxBlockSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */ + || (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/ + || (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */ + || (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */ + } -typedef struct { - void* srcBuffer; - size_t srcSize; - const void** srcPtrs; - size_t* srcSizes; - void** dstPtrs; - size_t* dstCapacities; - size_t* dstSizes; - void** resPtrs; - size_t* resSizes; - size_t nbBlocks; - size_t maxBlockSize; -} buffers_t; +/*-************************************ +* Display Functions +**************************************/ + +static void BMK_translateAdvancedParams(FILE* f, const paramValues_t params) { + varInds_t v; + int first = 1; + fprintf(f,"--zstd="); + for(v = 0; v < NUM_PARAMS; v++) { + if(g_silenceParams[v]) { continue; } + if(!first) { fprintf(f, ","); } + fprintf(f,"%s=", g_paramNames[v]); + + if(v == strt_ind) { fprintf(f,"%u", params.vals[v]); } + else { displayParamVal(f, v, params.vals[v], 0); } + first = 0; + } + fprintf(f, "\n"); +} + +static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) { + varInds_t v; + int first = 1; + res.params = cParamUnsetMin(res.params); + fprintf(f," {"); + for(v = 0; v < NUM_PARAMS; v++) { + if(g_silenceParams[v]) { continue; } + if(!first) { fprintf(f, ","); } + displayParamVal(f, v, res.params.vals[v], 3); + first = 0; + } + + fprintf(f, " }, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + (double)srcSize / res.result.cSize, (double)res.result.cSpeed / (1 MB), (double)res.result.dSpeed / (1 MB)); +} + +/* Writes to f the results of a parameter benchmark */ +/* when used with --optimize, will only print results better than previously discovered */ +static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize) +{ + char lvlstr[15] = "Custom Level"; + winnerInfo_t w; + w.params = params; + w.result = result; + + fprintf(f, "\r%79s\r", ""); + + if(cLevel != CUSTOM_LEVEL) { + snprintf(lvlstr, 15, " Level %2u ", cLevel); + } + + if(TIMED) { + const U64 time = UTIL_clockSpanNano(g_time); + const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); + fprintf(f, "%1lu:%2lu:%05.2f - ", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); + } + + fprintf(f, "/* %s */ ", lvlstr); + BMK_displayOneResult(f, w, srcSize); +} + +/* comparison function: */ +/* strictly better, strictly worse, equal, speed-side adv, size-side adv */ +//Maybe use compress_only for benchmark first run? +#define WORSE_RESULT 0 +#define BETTER_RESULT 1 +#define ERROR_RESULT 2 + +#define SPEED_RESULT 4 +#define SIZE_RESULT 5 +/* maybe have epsilon-eq to limit table size? */ +static int speedSizeCompare(const BMK_result_t r1, const BMK_result_t r2) { + if(r1.cSpeed < r2.cSpeed) { + if(r1.cSize >= r2.cSize) { + return BETTER_RESULT; + } + return SPEED_RESULT; /* r2 is smaller but not faster. */ + } else { + if(r1.cSize <= r2.cSize) { + return WORSE_RESULT; + } + return SIZE_RESULT; /* r2 is faster but not smaller */ + } +} + +/* 0 for insertion, 1 for no insert */ +/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */ +static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) { + BMK_result_t r = w.result; + winner_ll_node* cur_node = g_winners; + /* first node to insert */ + if(!feasible(r, targetConstraints)) { + return 1; + } + + if(g_winners == NULL) { + winner_ll_node* first_node = malloc(sizeof(winner_ll_node)); + if(first_node == NULL) { + return 1; + } + first_node->next = NULL; + first_node->res = w; + g_winners = first_node; + return 0; + } + + while(cur_node->next != NULL) { + switch(speedSizeCompare(cur_node->res.result, r)) { + case WORSE_RESULT: + { + return 1; /* never insert if better */ + } + case BETTER_RESULT: + { + winner_ll_node* tmp; + cur_node->res = cur_node->next->res; + tmp = cur_node->next; + cur_node->next = cur_node->next->next; + free(tmp); + break; + } + case SIZE_RESULT: + { + cur_node = cur_node->next; + break; + } + case SPEED_RESULT: /* insert after first size result, then return */ + { + winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); + if(newnode == NULL) { + return 1; + } + newnode->res = cur_node->res; + cur_node->res = w; + newnode->next = cur_node->next; + cur_node->next = newnode; + return 0; + } + } + + } + + assert(cur_node->next == NULL); + switch(speedSizeCompare(cur_node->res.result, r)) { + case WORSE_RESULT: + { + return 1; /* never insert if better */ + } + case BETTER_RESULT: + { + cur_node->res = w; + return 0; + } + case SIZE_RESULT: + { + winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); + if(newnode == NULL) { + return 1; + } + newnode->res = w; + newnode->next = NULL; + cur_node->next = newnode; + return 0; + } + case SPEED_RESULT: /* insert before first size result, then return */ + { + winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); + if(newnode == NULL) { + return 1; + } + newnode->res = cur_node->res; + cur_node->res = w; + newnode->next = cur_node->next; + cur_node->next = newnode; + return 0; + } + default: + return 1; + } +} + +static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize) +{ + /* global winner used for constraints */ + /* cSize, cSpeed, dSpeed, cMem */ + static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } }; + if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize) || g_displayLevel >= 4) { + if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { + DISPLAY("New Winner: \n"); + } + + if(g_displayLevel >= 2) { BMK_printWinner(f, cLevel, result, params, srcSize); } + + if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { + if(g_displayLevel >= 1) { BMK_translateAdvancedParams(f, params); } + g_winner.result = result; + g_winner.params = params; + } + } + + if(g_optmode && g_optimizer && (DEBUG || g_displayLevel == 3)) { + winnerInfo_t w; + winner_ll_node* n; + w.result = result; + w.params = params; + insertWinner(w, targetConstraints); + + if(!DEBUG) { fprintf(f, "\033c"); } + fprintf(f, "\n"); + + /* the table */ + fprintf(f, "================================\n"); + for(n = g_winners; n != NULL; n = n->next) { + BMK_displayOneResult(f, n->res, srcSize); + } + fprintf(f, "================================\n"); + fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n", + (double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / (1 MB)); + + + fprintf(f, "Overall Winner: \n"); + BMK_displayOneResult(f, g_winner, srcSize); + BMK_translateAdvancedParams(f, g_winner.params); + + fprintf(f, "Latest BMK: \n");\ + BMK_displayOneResult(f, w, srcSize); + } +} + +static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, const size_t srcSize) +{ + int cLevel; + + fprintf(f, "\n /* Proposed configurations : */ \n"); + fprintf(f, " /* W, C, H, S, L, T, strat */ \n"); + + for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++) + BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize); +} + + +static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, const size_t srcSize) +{ + fseek(f, 0, SEEK_SET); + BMK_printWinners2(f, winners, srcSize); + fflush(f); + BMK_printWinners2(stdout, winners, srcSize); +} -typedef struct { - size_t dictSize; - void* dictBuffer; - ZSTD_CCtx* cctx; - ZSTD_DCtx* dctx; -} contexts_t; /*-******************************************************* -* From bench.c +* Functions to Benchmark *********************************************************/ typedef struct { @@ -665,75 +969,30 @@ static size_t local_defaultDecompress( } -/*-******************************************************* -* From bench.c End -*********************************************************/ +/*-************************************ +* Data Initialization Functions +**************************************/ -static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) { - varInds_t v; - for(v = 0; v < NUM_PARAMS; v++) { - if(pc->vals[v] != PARAM_UNSET) { - U32 newval = MIN(MAX(pc->vals[v], mintable[v]), maxtable[v]); - if(newval != pc->vals[v]) { - pc->vals[v] = newval; - DISPLAY("Warning: parameter %s not in valid range, adjusting to ", g_paramNames[v]); displayParamVal(stderr, v, newval, 0); DISPLAY("\n"); - } - } - } +typedef struct { + void* srcBuffer; + size_t srcSize; + const void** srcPtrs; + size_t* srcSizes; + void** dstPtrs; + size_t* dstCapacities; + size_t* dstSizes; + void** resPtrs; + size_t* resSizes; + size_t nbBlocks; + size_t maxBlockSize; +} buffers_t; - if(pc->vals[wlog_ind] != PARAM_UNSET) { - - U32 sshb = maxBlockSize > 1 ? ZSTD_highbit32((U32)(maxBlockSize-1)) + 1 : 1; - /* edge case of highBit not working for 0 */ - - if(maxBlockSize < (1ULL << 31) && sshb + 1 < pc->vals[wlog_ind]) { - U32 adjust = MAX(mintable[wlog_ind], sshb); - if(adjust != pc->vals[wlog_ind]) { - pc->vals[wlog_ind] = adjust; - DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", pc->vals[wlog_ind]); - } - } - } - - if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) { - U32 maxclog; - if(pc->vals[strt_ind] == PARAM_UNSET || pc->vals[strt_ind] >= (U32)ZSTD_btlazy2) { - maxclog = pc->vals[wlog_ind] + 1; - } else { - maxclog = pc->vals[wlog_ind]; - } - - if(pc->vals[clog_ind] > maxclog) { - pc->vals[clog_ind] = maxclog; - DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", pc->vals[clog_ind]); - } - } - - if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[hlog_ind] != PARAM_UNSET) { - if(pc->vals[wlog_ind] + 1 < pc->vals[hlog_ind]) { - pc->vals[hlog_ind] = pc->vals[wlog_ind] + 1; - DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", pc->vals[hlog_ind]); - } - } - - if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) { - if(pc->vals[slog_ind] > pc->vals[clog_ind]) { - pc->vals[clog_ind] = pc->vals[slog_ind]; - DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", pc->vals[slog_ind]); - } - } -} - -/* what about low something like clog vs hlog in lvl 1? */ -static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) { - return - (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */ - || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= maxBlockSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */ - || (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/ - || (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */ - || (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */ - -} +typedef struct { + size_t dictSize; + void* dictBuffer; + ZSTD_CCtx* cctx; + ZSTD_DCtx* dctx; +} contexts_t; static void freeNonSrcBuffers(const buffers_t b) { free(b.srcPtrs); @@ -952,6 +1211,175 @@ static int createContexts(contexts_t* ctx, const char* dictFileName) { return 0; } +/*-************************************ +* Optimizer Memoization Functions +**************************************/ + +/* return: new length */ +/* keep old array, will need if iter over strategy. */ +/* prunes useless params */ +static size_t sanitizeVarArray(varInds_t* varNew, const size_t varLength, const varInds_t* varArray, const ZSTD_strategy strat) { + size_t i, j = 0; + for(i = 0; i < varLength; i++) { + if( !((varArray[i] == clog_ind && strat == ZSTD_fast) + || (varArray[i] == slog_ind && strat == ZSTD_fast) + || (varArray[i] == slog_ind && strat == ZSTD_dfast) + || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { + varNew[j] = varArray[i]; + j++; + } + } + return j; +} + +/* res should be NUM_PARAMS size */ +/* constructs varArray from paramValues_t style parameter */ +/* pass in using dict. */ +static size_t variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) { + varInds_t i; + size_t j = 0; + for(i = 0; i < NUM_PARAMS; i++) { + if(paramConstraints.vals[i] == PARAM_UNSET) { + if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */ + res[j] = i; j++; + } + } + return j; +} + +/* length of memo table given free variables */ +static size_t memoTableLen(const varInds_t* varyParams, const size_t varyLen) { + size_t arrayLen = 1; + size_t i; + for(i = 0; i < varyLen; i++) { + if(varyParams[i] == strt_ind) continue; /* strategy separated by table */ + arrayLen *= rangetable[varyParams[i]]; + } + return arrayLen; +} + +/* returns unique index in memotable of compression parameters */ +static unsigned memoTableIndDirect(const paramValues_t* ptr, const varInds_t* varyParams, const size_t varyLen) { + size_t i; + unsigned ind = 0; + for(i = 0; i < varyLen; i++) { + varInds_t v = varyParams[i]; + if(v == strt_ind) continue; /* exclude strategy from memotable */ + ind *= rangetable[v]; ind += (unsigned)invRangeMap(v, ptr->vals[v]); + } + return ind; +} + +static size_t memoTableGet(const memoTable_t* memoTableArray, const paramValues_t p) { + const memoTable_t mt = memoTableArray[p.vals[strt_ind]]; + switch(mt.tableType) { + case directMap: + return mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)]; + case xxhashMap: + return mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen]; + case noMemo: + return 0; + } + return 0; /* should never happen, stop compiler warnings */ +} + +static void memoTableSet(const memoTable_t* memoTableArray, const paramValues_t p, const BYTE value) { + const memoTable_t mt = memoTableArray[p.vals[strt_ind]]; + switch(mt.tableType) { + case directMap: + mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)] = value; break; + case xxhashMap: + mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen] = value; break; + case noMemo: + break; + } +} + +/* frees all allocated memotables */ +static void freeMemoTableArray(memoTable_t* const mtAll) { + int i; + if(mtAll == NULL) { return; } + for(i = 1; i <= (int)ZSTD_btultra; i++) { + free(mtAll[i].table); + } + free(mtAll); +} + +/* inits memotables for all (including mallocs), all strategies */ +/* takes unsanitized varyParams */ +static memoTable_t* createMemoTableArray(const paramValues_t p, const varInds_t* const varyParams, const size_t varyLen, const U32 memoTableLog) { + memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1)); + ZSTD_strategy i, stratMin = ZSTD_fast, stratMax = ZSTD_btultra; + + if(mtAll == NULL) { + return NULL; + } + + for(i = 1; i <= (int)ZSTD_btultra; i++) { + mtAll[i].varLen = sanitizeVarArray(mtAll[i].varArray, varyLen, varyParams, i); + } + + /* no memoization */ + if(memoTableLog == 0) { + for(i = 1; i <= (int)ZSTD_btultra; i++) { + mtAll[i].tableType = noMemo; + mtAll[i].table = NULL; + mtAll[i].tableLen = 0; + } + return mtAll; + } + + + if(p.vals[strt_ind] != PARAM_UNSET) { + stratMin = p.vals[strt_ind]; + stratMax = p.vals[strt_ind]; + } + + + for(i = stratMin; i <= stratMax; i++) { + size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen); + mtAll[i].tableType = directMap; + + if(memoTableLog != PARAM_UNSET && mtl > (1ULL << memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */ + mtAll[i].tableType = xxhashMap; + mtl = (1ULL << memoTableLog); + } + + mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl); + mtAll[i].tableLen = mtl; + + if(mtAll[i].table == NULL) { + freeMemoTableArray(mtAll); + return NULL; + } + } + + return mtAll; +} + +/* Sets pc to random unmeasured set of parameters */ +/* specifiy strategy */ +static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTableArray, const ZSTD_strategy st) +{ + size_t j; + const memoTable_t mt = memoTableArray[st]; + pc->vals[strt_ind] = st; + for(j = 0; j < mt.tableLen; j++) { + int i; + for(i = 0; i < NUM_PARAMS; i++) { + varInds_t v = mt.varArray[i]; + if(v == strt_ind) continue; + pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); + } + + if(!(memoTableGet(memoTableArray, *pc))) break; /* only pick unpicked params. */ + } +} + +/*-************************************ +* Benchmarking Functions +**************************************/ + /* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */ /* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */ /* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ @@ -1099,220 +1527,115 @@ static int BMK_benchParam(BMK_result_t* resultPtr, return res.error; } -/* comparison function: */ -/* strictly better, strictly worse, equal, speed-side adv, size-side adv */ -//Maybe use compress_only for benchmark first run? -#define WORSE_RESULT 0 -#define BETTER_RESULT 1 -#define ERROR_RESULT 2 -#define SPEED_RESULT 4 -#define SIZE_RESULT 5 -/* maybe have epsilon-eq to limit table size? */ -static int speedSizeCompare(const BMK_result_t r1, const BMK_result_t r2) { - if(r1.cSpeed < r2.cSpeed) { - if(r1.cSize >= r2.cSize) { - return BETTER_RESULT; - } - return SPEED_RESULT; /* r2 is smaller but not faster. */ - } else { - if(r1.cSize <= r2.cSize) { - return WORSE_RESULT; - } - return SIZE_RESULT; /* r2 is faster but not smaller */ +#define CBENCHMARK(conditional, resultvar, tmpret, mode, loopmode, sec) { \ + if(conditional) { \ + BMK_return_t tmpret = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, mode, loopmode, sec); \ + if(tmpret.error) { DEBUGOUTPUT("Benchmarking failed\n"); return ERROR_RESULT; } \ + if(mode != BMK_decodeOnly) { \ + resultvar.cSpeed = tmpret.result.cSpeed; \ + resultvar.cSize = tmpret.result.cSize; \ + resultvar.cMem = tmpret.result.cMem; \ + } \ + if(mode != BMK_compressOnly) { resultvar.dSpeed = tmpret.result.dSpeed; } \ + } \ +} + +/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ +#define VARIANCE 1.2 +static int allBench(BMK_result_t* resultPtr, + const buffers_t buf, const contexts_t ctx, + const paramValues_t cParams, + const constraint_t target, + BMK_result_t* winnerResult, int feas) { + BMK_result_t resultMax, benchres; + U64 loopDurationC = 0, loopDurationD = 0; + double uncertaintyConstantC = 3., uncertaintyConstantD = 3.; + double winnerRS; + /* initial benchmarking, gives exact ratio and memory, warms up future runs */ + CBENCHMARK(1, benchres, tmp, BMK_both, BMK_iterMode, 1); + + winnerRS = resultScore(*winnerResult, buf.srcSize, target); + DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); + + *resultPtr = benchres; + + /* calculate uncertainty in compression / decompression runs */ + if(benchres.cSpeed) { + loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed); + uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC); + } + + if(benchres.dSpeed) { + loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed); + uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD); + } + + /* anything with worse ratio in feas is definitely worse, discard */ + if(feas && benchres.cSize < winnerResult->cSize && !g_optmode) { + return WORSE_RESULT; + } + + /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ + CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_compressOnly, BMK_iterMode, 1); + CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_decodeOnly, BMK_iterMode, 1); + + *resultPtr = benchres; + + /* optimistic assumption of benchres */ + resultMax = benchres; + resultMax.cSpeed *= uncertaintyConstantC * VARIANCE; + resultMax.dSpeed *= uncertaintyConstantD * VARIANCE; + + /* disregard infeasible results in feas mode */ + /* disregard if resultMax < winner in infeas mode */ + if((feas && !feasible(resultMax, target)) || + (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) { + return WORSE_RESULT; + } + + CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC, benchres, tmp, BMK_compressOnly, BMK_timeMode, 1); + CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC, benchres, tmp, BMK_decodeOnly, BMK_timeMode, 1); + + *resultPtr = benchres; + + /* compare by resultScore when in infeas */ + /* compare by compareResultLT when in feas */ + if((!feas && (resultScore(benchres, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || + (feas && (compareResultLT(*winnerResult, benchres, target, buf.srcSize))) ) { + return BETTER_RESULT; + } else { + return WORSE_RESULT; } } -/* 0 for insertion, 1 for no insert */ -/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */ -static int insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) { - BMK_result_t r = w.result; - winner_ll_node* cur_node = g_winners; - /* first node to insert */ - if(!feasible(r, targetConstraints)) { - return 1; +#define INFEASIBLE_THRESHOLD 200 +/* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ +static int benchMemo(BMK_result_t* resultPtr, + const buffers_t buf, const contexts_t ctx, + const paramValues_t cParams, + const constraint_t target, + BMK_result_t* winnerResult, memoTable_t* const memoTableArray, + const int feas) { + static int bmcount = 0; + int res; + + if(memoTableGet(memoTableArray, cParams) >= INFEASIBLE_THRESHOLD || redundantParams(cParams, target, buf.maxBlockSize)) { return WORSE_RESULT; } + + res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas); + + if(DEBUG && !(bmcount % 250)) { + DISPLAY("Count: %d\n", bmcount); + bmcount++; } + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize); - if(g_winners == NULL) { - winner_ll_node* first_node = malloc(sizeof(winner_ll_node)); - if(first_node == NULL) { - return 1; - } - first_node->next = NULL; - first_node->res = w; - g_winners = first_node; - return 0; + if(res == BETTER_RESULT || feas) { + memoTableSet(memoTableArray, cParams, 255); /* what happens if collisions are frequent */ } - - while(cur_node->next != NULL) { - switch(speedSizeCompare(cur_node->res.result, r)) { - case WORSE_RESULT: - { - return 1; /* never insert if better */ - } - case BETTER_RESULT: - { - winner_ll_node* tmp; - cur_node->res = cur_node->next->res; - tmp = cur_node->next; - cur_node->next = cur_node->next->next; - free(tmp); - break; - } - case SIZE_RESULT: - { - cur_node = cur_node->next; - break; - } - case SPEED_RESULT: /* insert after first size result, then return */ - { - winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); - if(newnode == NULL) { - return 1; - } - newnode->res = cur_node->res; - cur_node->res = w; - newnode->next = cur_node->next; - cur_node->next = newnode; - return 0; - } - } - - } - - assert(cur_node->next == NULL); - switch(speedSizeCompare(cur_node->res.result, r)) { - case WORSE_RESULT: - { - return 1; /* never insert if better */ - } - case BETTER_RESULT: - { - cur_node->res = w; - return 0; - } - case SIZE_RESULT: - { - winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); - if(newnode == NULL) { - return 1; - } - newnode->res = w; - newnode->next = NULL; - cur_node->next = newnode; - return 0; - } - case SPEED_RESULT: /* insert before first size result, then return */ - { - winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); - if(newnode == NULL) { - return 1; - } - newnode->res = cur_node->res; - cur_node->res = w; - newnode->next = cur_node->next; - cur_node->next = newnode; - return 0; - } - default: - return 1; - } + return res; } -/* Writes to f the results of a parameter benchmark */ -/* when used with --optimize, will only print results better than previously discovered */ -static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize) -{ - char lvlstr[15] = "Custom Level"; - winnerInfo_t w; - w.params = params; - w.result = result; - - fprintf(f, "\r%79s\r", ""); - - if(cLevel != CUSTOM_LEVEL) { - snprintf(lvlstr, 15, " Level %2u ", cLevel); - } - - if(TIMED) { - const U64 time = UTIL_clockSpanNano(g_time); - const U64 minutes = time / (60ULL * TIMELOOP_NANOSEC); - fprintf(f, "%1lu:%2lu:%05.2f - ", (unsigned long) minutes / 60,(unsigned long) minutes % 60, (double)(time - minutes * TIMELOOP_NANOSEC * 60ULL)/TIMELOOP_NANOSEC); - } - - fprintf(f, "/* %s */ ", lvlstr); - BMK_displayOneResult(f, w, srcSize); -} - -static void BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize) -{ - /* global winner used for constraints */ - /* cSize, cSpeed, dSpeed, cMem */ - static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } }; - if(DEBUG || compareResultLT(g_winner.result, result, targetConstraints, srcSize) || g_displayLevel >= 4) { - if(DEBUG && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { - DISPLAY("New Winner: \n"); - } - - if(g_displayLevel >= 2) { BMK_printWinner(f, cLevel, result, params, srcSize); } - - if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { - if(g_displayLevel >= 1) { BMK_translateAdvancedParams(f, params); } - g_winner.result = result; - g_winner.params = params; - } - } - - if(g_optmode && g_optimizer && (DEBUG || g_displayLevel == 3)) { - winnerInfo_t w; - winner_ll_node* n; - w.result = result; - w.params = params; - insertWinner(w, targetConstraints); - - if(!DEBUG) { fprintf(f, "\033c"); } - fprintf(f, "\n"); - - /* the table */ - fprintf(f, "================================\n"); - for(n = g_winners; n != NULL; n = n->next) { - BMK_displayOneResult(f, n->res, srcSize); - } - fprintf(f, "================================\n"); - fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n", - (double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / (1 MB)); - - - fprintf(f, "Overall Winner: \n"); - BMK_displayOneResult(f, g_winner, srcSize); - BMK_translateAdvancedParams(f, g_winner.params); - - fprintf(f, "Latest BMK: \n");\ - BMK_displayOneResult(f, w, srcSize); - } -} - -static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, const size_t srcSize) -{ - int cLevel; - - fprintf(f, "\n /* Proposed configurations : */ \n"); - fprintf(f, " /* W, C, H, S, L, T, strat */ \n"); - - for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++) - BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize); -} - - -static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, const size_t srcSize) -{ - fseek(f, 0, SEEK_SET); - BMK_printWinners2(f, winners, srcSize); - fflush(f); - BMK_printWinners2(stdout, winners, srcSize); -} - - typedef struct { U64 cSpeed_min; U64 dSpeed_min; @@ -1437,201 +1760,9 @@ static int BMK_seed(winnerInfo_t* winners, const paramValues_t params, return better; } -/* bounds check in sanitize too? */ -#define CLAMP(var, lo, hi) { \ - var = MAX(MIN(var, hi), lo); \ -} - -/* nullified useless params, to ensure count stats */ -/* cleans up params for memoizing / display */ -static paramValues_t sanitizeParams(paramValues_t params) -{ - if (params.vals[strt_ind] == ZSTD_fast) - params.vals[clog_ind] = 0, params.vals[slog_ind] = 0; - if (params.vals[strt_ind] == ZSTD_dfast) - params.vals[slog_ind] = 0; - if (params.vals[strt_ind] != ZSTD_btopt && params.vals[strt_ind] != ZSTD_btultra && params.vals[strt_ind] != ZSTD_fast) - params.vals[tlen_ind] = 0; - - return params; -} - -/* return: new length */ -/* keep old array, will need if iter over strategy. */ -/* prunes useless params */ -static size_t sanitizeVarArray(varInds_t* varNew, const size_t varLength, const varInds_t* varArray, const ZSTD_strategy strat) { - size_t i, j = 0; - for(i = 0; i < varLength; i++) { - if( !((varArray[i] == clog_ind && strat == ZSTD_fast) - || (varArray[i] == slog_ind && strat == ZSTD_fast) - || (varArray[i] == slog_ind && strat == ZSTD_dfast) - || (varArray[i] == tlen_ind && strat != ZSTD_btopt && strat != ZSTD_btultra && strat != ZSTD_fast))) { - varNew[j] = varArray[i]; - j++; - } - } - return j; - -} - -/* res should be NUM_PARAMS size */ -/* constructs varArray from paramValues_t style parameter */ -/* pass in using dict. */ -static size_t variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) { - varInds_t i; - size_t j = 0; - for(i = 0; i < NUM_PARAMS; i++) { - if(paramConstraints.vals[i] == PARAM_UNSET) { - if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */ - res[j] = i; j++; - } - } - return j; -} - -/* amt will probably always be \pm 1? */ -/* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ -/* take max/min bounds into account as well? */ -static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) { - ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); -} - -/* varies ptr by nbChanges respecting varyParams*/ -static void paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges) -{ - paramValues_t p; - U32 validated = 0; - while (!validated) { - U32 i; - p = *ptr; - for (i = 0 ; i < nbChanges ; i++) { - const U32 changeID = (U32)FUZ_rand(&g_rand) % (mtAll[p.vals[strt_ind]].varLen << 1); - paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1], ((changeID & 1) << 1) - 1, &p); - } - validated = paramValid(p); - } - *ptr = p; -} - -/* length of memo table given free variables */ -static size_t memoTableLen(const varInds_t* varyParams, const size_t varyLen) { - size_t arrayLen = 1; - size_t i; - for(i = 0; i < varyLen; i++) { - if(varyParams[i] == strt_ind) continue; /* strategy separated by table */ - arrayLen *= rangetable[varyParams[i]]; - } - return arrayLen; -} - -/* returns unique index in memotable of compression parameters */ -static unsigned memoTableIndDirect(const paramValues_t* ptr, const varInds_t* varyParams, const size_t varyLen) { - size_t i; - unsigned ind = 0; - for(i = 0; i < varyLen; i++) { - varInds_t v = varyParams[i]; - if(v == strt_ind) continue; /* exclude strategy from memotable */ - ind *= rangetable[v]; ind += (unsigned)invRangeMap(v, ptr->vals[v]); - } - return ind; -} - -static size_t memoTableGet(const memoTable_t* memoTableArray, const paramValues_t p) { - const memoTable_t mt = memoTableArray[p.vals[strt_ind]]; - switch(mt.tableType) { - case directMap: - return mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)]; - case xxhashMap: - return mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen]; - case noMemo: - return 0; - } - return 0; /* should never happen, stop compiler warnings */ -} - -static void memoTableSet(const memoTable_t* memoTableArray, const paramValues_t p, const BYTE value) { - const memoTable_t mt = memoTableArray[p.vals[strt_ind]]; - switch(mt.tableType) { - case directMap: - mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)] = value; break; - case xxhashMap: - mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen] = value; break; - case noMemo: - break; - } -} - -/* frees all allocated memotables */ -static void freeMemoTableArray(memoTable_t* const mtAll) { - int i; - if(mtAll == NULL) { return; } - for(i = 1; i <= (int)ZSTD_btultra; i++) { - free(mtAll[i].table); - } - free(mtAll); -} - -/* inits memotables for all (including mallocs), all strategies */ -/* takes unsanitized varyParams */ -static memoTable_t* createMemoTableArray(const paramValues_t p, const varInds_t* const varyParams, const size_t varyLen, const U32 memoTableLog) { - memoTable_t* mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_btultra + 1)); - ZSTD_strategy i, stratMin = ZSTD_fast, stratMax = ZSTD_btultra; - - if(mtAll == NULL) { - return NULL; - } - - for(i = 1; i <= (int)ZSTD_btultra; i++) { - mtAll[i].varLen = sanitizeVarArray(mtAll[i].varArray, varyLen, varyParams, i); - } - - /* no memoization */ - if(memoTableLog == 0) { - for(i = 1; i <= (int)ZSTD_btultra; i++) { - mtAll[i].tableType = noMemo; - mtAll[i].table = NULL; - mtAll[i].tableLen = 0; - } - return mtAll; - } - - - if(p.vals[strt_ind] != PARAM_UNSET) { - stratMin = p.vals[strt_ind]; - stratMax = p.vals[strt_ind]; - } - - - for(i = stratMin; i <= stratMax; i++) { - size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen); - mtAll[i].tableType = directMap; - - if(memoTableLog != PARAM_UNSET && mtl > (1ULL << memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */ - mtAll[i].tableType = xxhashMap; - mtl = (1ULL << memoTableLog); - } - - mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl); - mtAll[i].tableLen = mtl; - - if(mtAll[i].table == NULL) { - freeMemoTableArray(mtAll); - return NULL; - } - } - - return mtAll; -} - -static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mask) { - U32 i; - for(i = 0; i < NUM_PARAMS; i++) { - if(mask.vals[i] != PARAM_UNSET) { - base.vals[i] = mask.vals[i]; - } - } - return base; -} +/*-************************************ +* Compression Level Table Generation Functions +**************************************/ #define PARAMTABLELOG 25 #define PARAMTABLESIZE (1<vals[strt_ind] = st; - for(j = 0; j < mt.tableLen; j++) { - int i; - for(i = 0; i < NUM_PARAMS; i++) { - varInds_t v = mt.varArray[i]; - if(v == strt_ind) continue; - pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); - } - - if(!(memoTableGet(memoTableArray, *pc))) break; /* only pick unpicked params. */ - } -} - -/* Completely random parameter selection */ -static paramValues_t randomParams(void) -{ - varInds_t v; paramValues_t p; - for(v = 0; v <= NUM_PARAMS; v++) { - p.vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); - } - return p; -} - static void BMK_selectRandomStart( FILE* f, winnerInfo_t* winners, const buffers_t buf, const contexts_t ctx) @@ -1764,6 +1866,10 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) fclose(f); } +/*-************************************ +* Single Benchmark Functions +**************************************/ + static int benchOnce(const buffers_t buf, const contexts_t ctx) { BMK_result_t testResult; @@ -1862,113 +1968,10 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam return ret; } -#define CBENCHMARK(conditional, resultvar, tmpret, mode, loopmode, sec) { \ - if(conditional) { \ - BMK_return_t tmpret = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, mode, loopmode, sec); \ - if(tmpret.error) { DEBUGOUTPUT("Benchmarking failed\n"); return ERROR_RESULT; } \ - if(mode != BMK_decodeOnly) { \ - resultvar.cSpeed = tmpret.result.cSpeed; \ - resultvar.cSize = tmpret.result.cSize; \ - resultvar.cMem = tmpret.result.cMem; \ - } \ - if(mode != BMK_compressOnly) { resultvar.dSpeed = tmpret.result.dSpeed; } \ - } \ -} -/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ -#define VARIANCE 1.2 -static int allBench(BMK_result_t* resultPtr, - const buffers_t buf, const contexts_t ctx, - const paramValues_t cParams, - const constraint_t target, - BMK_result_t* winnerResult, int feas) { - BMK_result_t resultMax, benchres; - U64 loopDurationC = 0, loopDurationD = 0; - double uncertaintyConstantC = 3., uncertaintyConstantD = 3.; - double winnerRS; - /* initial benchmarking, gives exact ratio and memory, warms up future runs */ - CBENCHMARK(1, benchres, tmp, BMK_both, BMK_iterMode, 1); - - winnerRS = resultScore(*winnerResult, buf.srcSize, target); - DEBUGOUTPUT("WinnerScore: %f\n ", winnerRS); - - *resultPtr = benchres; - - /* calculate uncertainty in compression / decompression runs */ - if(benchres.cSpeed) { - loopDurationC = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed); - uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC); - } - - if(benchres.dSpeed) { - loopDurationD = ((buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed); - uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD); - } - - /* anything with worse ratio in feas is definitely worse, discard */ - if(feas && benchres.cSize < winnerResult->cSize && !g_optmode) { - return WORSE_RESULT; - } - - /* second run, if first run is too short, gives approximate cSpeed + dSpeed */ - CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_compressOnly, BMK_iterMode, 1); - CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC / 10, benchres, tmp, BMK_decodeOnly, BMK_iterMode, 1); - - *resultPtr = benchres; - - /* optimistic assumption of benchres */ - resultMax = benchres; - resultMax.cSpeed *= uncertaintyConstantC * VARIANCE; - resultMax.dSpeed *= uncertaintyConstantD * VARIANCE; - - /* disregard infeasible results in feas mode */ - /* disregard if resultMax < winner in infeas mode */ - if((feas && !feasible(resultMax, target)) || - (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) { - return WORSE_RESULT; - } - - CBENCHMARK(loopDurationC < TIMELOOP_NANOSEC, benchres, tmp, BMK_compressOnly, BMK_timeMode, 1); - CBENCHMARK(loopDurationD < TIMELOOP_NANOSEC, benchres, tmp, BMK_decodeOnly, BMK_timeMode, 1); - - *resultPtr = benchres; - - /* compare by resultScore when in infeas */ - /* compare by compareResultLT when in feas */ - if((!feas && (resultScore(benchres, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || - (feas && (compareResultLT(*winnerResult, benchres, target, buf.srcSize))) ) { - return BETTER_RESULT; - } else { - return WORSE_RESULT; - } -} - -#define INFEASIBLE_THRESHOLD 200 -/* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ -static int benchMemo(BMK_result_t* resultPtr, - const buffers_t buf, const contexts_t ctx, - const paramValues_t cParams, - const constraint_t target, - BMK_result_t* winnerResult, memoTable_t* const memoTableArray, - const int feas) { - static int bmcount = 0; - int res; - - if(memoTableGet(memoTableArray, cParams) >= INFEASIBLE_THRESHOLD || redundantParams(cParams, target, buf.maxBlockSize)) { return WORSE_RESULT; } - - res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas); - - if(DEBUG && !(bmcount % 250)) { - DISPLAY("Count: %d\n", bmcount); - bmcount++; - } - BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize); - - if(res == BETTER_RESULT || feas) { - memoTableSet(memoTableArray, cParams, 255); /* what happens if collisions are frequent */ - } - return res; -} +/*-************************************ +* Local Optimization Functions +**************************************/ /* One iteration of hill climbing. Specifically, it first tries all * valid parameter configurations w/ manhattan distance 1 and picks the best one @@ -2368,6 +2371,24 @@ _cleanUp: return ret; } +/*-************************************ +* CLI parsing functions +**************************************/ + +/** longCommandWArg() : + * check if *stringPtr is the same as longCommand. + * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. + * @return 0 and doesn't modify *stringPtr otherwise. + * from zstdcli.c + */ +static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) +{ + size_t const comSize = strlen(longCommand); + int const result = !strncmp(*stringPtr, longCommand, comSize); + if (result) *stringPtr += comSize; + return result; +} + static void errorOut(const char* msg) { DISPLAY("%s \n", msg); exit(1); @@ -2474,6 +2495,10 @@ static int parse_params(const char** argptr, paramValues_t* pv) { return matched; } +/*-************************************ +* Main +**************************************/ + int main(int argc, const char** argv) { int i, From 239e114d62c2a1d3c73ac05534a18d8583b9654d Mon Sep 17 00:00:00 2001 From: George Lu Date: Wed, 15 Aug 2018 15:01:03 -0700 Subject: [PATCH 82/92] prune comments --- tests/paramgrill.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index 7ebcdeec0..e94eead0d 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -439,9 +439,6 @@ static paramValues_t overwriteParams(paramValues_t base, const paramValues_t mas return base; } -/* amt will probably always be \pm 1? */ -/* slight change from old paramVariation, targetLength can only take on powers of 2 now (999 ~= 1024?) */ -/* take max/min bounds into account as well? */ static void paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) { ptr->vals[paramIndex] = rangeMap(paramIndex, invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); } @@ -610,7 +607,6 @@ static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) { } } -/* what about low something like clog vs hlog in lvl 1? */ static int redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) { return (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */ @@ -684,7 +680,6 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result /* comparison function: */ /* strictly better, strictly worse, equal, speed-side adv, size-side adv */ -//Maybe use compress_only for benchmark first run? #define WORSE_RESULT 0 #define BETTER_RESULT 1 #define ERROR_RESULT 2 @@ -1385,7 +1380,7 @@ static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTa /* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ /* nbSeconds used in same way as in BMK_advancedParams_t, as nbIters when in iterMode */ -/* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs? */ +/* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs */ /* dictionary nullable, nothing else though. */ static BMK_return_t BMK_benchMemInvertible(const buffers_t buf, const contexts_t ctx, const int cLevel, const paramValues_t* comprParams, @@ -2088,11 +2083,9 @@ static winnerInfo_t climbOnce(const constraint_t target, /* Optimizes for a fixed strategy */ -/* flexible parameters: iterations of (failed?) climbing (or if we do non-random, maybe this is when everything is close to visitied) +/* flexible parameters: iterations of failed climbing (or if we do non-random, maybe this is when everything is close to visitied) weight more on visit for bad results, less on good results/more on later results / ones with more failures. allocate memoTable here. - only real use for paramTarget is to get the fixed values, right? - maybe allow giving it a first init? */ static winnerInfo_t optimizeFixedStrategy( const buffers_t buf, const contexts_t ctx, @@ -2233,7 +2226,6 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ } /* use level'ing mode instead of normal target mode */ - /* Should lvl be parameter-masked here? */ if(g_optmode) { winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, buf.maxBlockSize, ctx.dictSize)); if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { @@ -2247,7 +2239,7 @@ static int optimizeForSize(const char* const * const fileNamesTable, const size_ g_lvltarget.cSize /= ((double)g_strictness) / 100; target.cSpeed = (U32)g_lvltarget.cSpeed; - target.dSpeed = (U32)g_lvltarget.dSpeed; //See if this is reasonable. + target.dSpeed = (U32)g_lvltarget.dSpeed; BMK_printWinnerOpt(stdout, cLevelOpt, winner.result, winner.params, target, buf.srcSize); } From da55865e47bb8035cbf1675d3d80c61979bfc8d3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 15 Aug 2018 16:43:13 -0700 Subject: [PATCH 83/92] ensure dependency for zlib wrapper --- Makefile | 14 +++++++------- zlibWrapper/examples/zwrapbench.c | 6 +++--- zlibWrapper/gzlib.c | 4 ++-- zlibWrapper/gzwrite.c | 9 ++++++--- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index f8441e60d..d33757603 100644 --- a/Makefile +++ b/Makefile @@ -30,8 +30,7 @@ default: lib-release zstd-release all: allmost examples manual contrib .PHONY: allmost -allmost: allzstd - $(MAKE) -C $(ZWRAPDIR) all +allmost: allzstd zlibwrapper #skip zwrapper, can't build that on alternate architectures without the proper zlib installed .PHONY: allzstd @@ -44,8 +43,9 @@ all32: $(MAKE) -C $(PRGDIR) zstd32 $(MAKE) -C $(TESTDIR) all32 -.PHONY: lib lib-release -lib lib-release: +.PHONY: lib lib-release libzstd.a +lib : libzstd.a +lib lib-release libzstd.a: @$(MAKE) -C $(ZSTDDIR) $@ .PHONY: zstd zstd-release @@ -59,8 +59,8 @@ zstdmt: cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT) .PHONY: zlibwrapper -zlibwrapper: - $(MAKE) -C $(ZWRAPDIR) test +zlibwrapper: libzstd.a + $(MAKE) -C $(ZWRAPDIR) all .PHONY: test test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror @@ -353,5 +353,5 @@ bmi32build: clean staticAnalyze: $(CC) -v - CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all + CC=$(CC) CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all endif diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index a4dfbb6e8..d2d6073f9 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -573,10 +573,10 @@ static size_t BMK_findMaxMem(U64 requiredMem) do { testmem = (BYTE*)malloc((size_t)requiredMem); requiredMem -= step; - } while (!testmem); + } while (!testmem && requiredMem); /* do not allocate zero bytes */ free(testmem); - return (size_t)(requiredMem); + return (size_t)(requiredMem+1); /* avoid zero */ } static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, @@ -734,7 +734,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; if (benchedSize < totalSizeToLoad) DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); - srcBuffer = malloc(benchedSize); + srcBuffer = malloc(benchedSize + !benchedSize); if (!srcBuffer) EXM_THROW(12, "not enough memory"); /* Load input buffer */ diff --git a/zlibWrapper/gzlib.c b/zlibWrapper/gzlib.c index 8235cff4f..3070dd8b4 100644 --- a/zlibWrapper/gzlib.c +++ b/zlibWrapper/gzlib.c @@ -111,7 +111,7 @@ local gzFile gz_open(path, fd, mode) return NULL; /* allocate gzFile structure to return */ - state = (gz_statep)(gz_state*)malloc(sizeof(gz_state)); + state.state = (gz_state*)malloc(sizeof(gz_state)); if (state.state == NULL) return NULL; state.state->size = 0; /* no buffers allocated yet */ @@ -266,7 +266,7 @@ local gzFile gz_open(path, fd, mode) gz_reset(state); /* return stream */ - return (gzFile)state.file; + return state.file; } /* -- see zlib.h -- */ diff --git a/zlibWrapper/gzwrite.c b/zlibWrapper/gzwrite.c index d1250b900..21d5f8472 100644 --- a/zlibWrapper/gzwrite.c +++ b/zlibWrapper/gzwrite.c @@ -6,6 +6,8 @@ * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html */ +#include + #include "gzguts.h" /* Local functions */ @@ -24,7 +26,7 @@ local int gz_init(state) z_streamp strm = &(state.state->strm); /* allocate input buffer (double size for gzprintf) */ - state.state->in = (unsigned char *)malloc(state.state->want << 1); + state.state->in = (unsigned char*)malloc(state.state->want << 1); if (state.state->in == NULL) { gz_error(state, Z_MEM_ERROR, "out of memory"); return -1; @@ -33,7 +35,7 @@ local int gz_init(state) /* only need output buffer and deflate state if compressing */ if (!state.state->direct) { /* allocate output buffer */ - state.state->out = (unsigned char *)malloc(state.state->want); + state.state->out = (unsigned char*)malloc(state.state->want); if (state.state->out == NULL) { free(state.state->in); gz_error(state, Z_MEM_ERROR, "out of memory"); @@ -284,6 +286,7 @@ z_size_t ZEXPORT gzfwrite(buf, size, nitems, file) gz_statep state; /* get internal structure */ + assert(size != 0); if (file == NULL) return 0; state = (gz_statep)file; @@ -294,7 +297,7 @@ z_size_t ZEXPORT gzfwrite(buf, size, nitems, file) /* compute bytes to read -- error on overflow */ len = nitems * size; - if (size && len / size != nitems) { + if (size && (len / size != nitems)) { gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t"); return 0; } From 5291d9ac31a87b68e717e43bf5b14a71ebaf644d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 15 Aug 2018 17:41:44 -0700 Subject: [PATCH 84/92] fix scope of scan-build tests exclude zlib code --- Makefile | 11 ++++++----- lib/decompress/zstd_decompress.c | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index d33757603..e338c5ca1 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ all: allmost examples manual contrib .PHONY: allmost allmost: allzstd zlibwrapper -#skip zwrapper, can't build that on alternate architectures without the proper zlib installed +# skip zwrapper, can't build that on alternate architectures without the proper zlib installed .PHONY: allzstd allzstd: lib $(MAKE) -C $(PRGDIR) all @@ -44,8 +44,7 @@ all32: $(MAKE) -C $(TESTDIR) all32 .PHONY: lib lib-release libzstd.a -lib : libzstd.a -lib lib-release libzstd.a: +lib lib-release : @$(MAKE) -C $(ZSTDDIR) $@ .PHONY: zstd zstd-release @@ -59,7 +58,7 @@ zstdmt: cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT) .PHONY: zlibwrapper -zlibwrapper: libzstd.a +zlibwrapper: lib $(MAKE) -C $(ZWRAPDIR) all .PHONY: test @@ -351,7 +350,9 @@ bmi32build: clean $(CC) -v CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test +# static analyzer test uses clang's scan-build +# does not analyze zlibWrapper, due to detected issues in zlib source code staticAnalyze: $(CC) -v - CC=$(CC) CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all + CC=$(CC) CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) allzstd examples contrib endif diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 5d9f0ba0b..8fef7e51b 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -2399,6 +2399,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { ddict->dictBuffer = NULL; ddict->dictContent = dict; + if (!dict) dictSize = 0; } else { void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem); ddict->dictBuffer = internalBuffer; From 31224cc126f23d2b7dbac84940aaedbf55ea477b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 15 Aug 2018 17:44:27 -0700 Subject: [PATCH 85/92] added static analyzer tests to travis CI --- .travis.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 36df67052..a61b82876 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,23 +20,25 @@ matrix: - env: Cmd='make gcc8install && CC=gcc-8 CFLAGS="-Werror -O3" make -j all' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd' + - env: Cmd='make staticAnalyze' + - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest' - env: Cmd='make gcc6install libc6install && make clean && CC=gcc-6 CFLAGS=-m32 make uasan-fuzztest' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest' - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream' - - env: Cmd='make arminstall && make armfuzz' - - env: Cmd='make arminstall && make aarch64fuzz' - - env: Cmd='make ppcinstall && make ppcfuzz' - - env: Cmd='make ppcinstall && make ppc64fuzz' - - env: Cmd='make -j uasanregressiontest && make clean && make -j msanregressiontest' - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest && make clean && make -C tests test-fuzzer-stackmode' + - env: Cmd='make arminstall && make armfuzz' + - env: Cmd='make arminstall && make aarch64fuzz' + - env: Cmd='make ppcinstall && make ppcfuzz' + - env: Cmd='make ppcinstall && make ppc64fuzz' + - env: Cmd='make lz4install && make -C tests test-lz4 && make clean && make -C tests test-pool && make clean && bash tests/libzstd_partial_builds.sh' From 1515f0bb0d73138dd191a9fd5caad09e9868a8a0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 16 Aug 2018 14:40:47 -0700 Subject: [PATCH 86/92] fixed more issues detected by recent version of scan-build test run on Linux --- examples/multiple_streaming_compression.c | 3 ++- examples/streaming_compression.c | 22 ++++++++++++++++------ lib/compress/fse_compress.c | 22 +++++++++++++++------- lib/dictBuilder/divsufsort.c | 6 +++--- 4 files changed, 36 insertions(+), 17 deletions(-) diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c index e395aefbc..4308a2e4d 100644 --- a/examples/multiple_streaming_compression.c +++ b/examples/multiple_streaming_compression.c @@ -158,7 +158,8 @@ int main(int argc, const char** argv) } freeResources(ress); - /* success */ + free(ofnBuffer); + printf("compressed %i files \n", argc-1); return 0; diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c index f76364d84..9287ff398 100644 --- a/examples/streaming_compression.c +++ b/examples/streaming_compression.c @@ -73,7 +73,11 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve ZSTD_CStream* const cstream = ZSTD_createCStream(); if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); } size_t const initResult = ZSTD_initCStream(cstream, cLevel); - if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); } + if (ZSTD_isError(initResult)) { + fprintf(stderr, "ZSTD_initCStream() error : %s \n", + ZSTD_getErrorName(initResult)); + exit(11); + } size_t read, toRead = buffInSize; while( (read = fread_orDie(buffIn, toRead, fin)) ) { @@ -81,7 +85,11 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve while (input.pos < input.size) { ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; toRead = ZSTD_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */ - if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); } + if (ZSTD_isError(toRead)) { + fprintf(stderr, "ZSTD_compressStream() error : %s \n", + ZSTD_getErrorName(toRead)); + exit(12); + } if (toRead > buffInSize) toRead = buffInSize; /* Safely handle case when `buffInSize` is manually changed to a value < ZSTD_CStreamInSize()*/ fwrite_orDie(buffOut, output.pos, fout); } @@ -100,15 +108,15 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve } -static const char* createOutFilename_orDie(const char* filename) +static char* createOutFilename_orDie(const char* filename) { size_t const inL = strlen(filename); size_t const outL = inL + 5; - void* outSpace = malloc_orDie(outL); + void* const outSpace = malloc_orDie(outL); memset(outSpace, 0, outL); strcat(outSpace, filename); strcat(outSpace, ".zst"); - return (const char*)outSpace; + return (char*)outSpace; } int main(int argc, const char** argv) @@ -124,8 +132,10 @@ int main(int argc, const char** argv) const char* const inFilename = argv[1]; - const char* const outFilename = createOutFilename_orDie(inFilename); + char* const outFilename = createOutFilename_orDie(inFilename); compressFile_orDie(inFilename, outFilename, 1); + free(outFilename); /* not strictly required, since program execution stops there, + * but some static analyzer main complain otherwise */ return 0; } diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index caa90edec..4075db217 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -240,10 +240,12 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize, if (previousIs0) { unsigned start = symbol; while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++; + if (symbol == alphabetSize) break; /* incorrect distribution */ while (symbol >= start+24) { start+=24; bitStream += 0xFFFFU << bitCount; - if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + if ((!writeIsSafe) && (out > oend-2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ out[0] = (BYTE) bitStream; out[1] = (BYTE)(bitStream>>8); out+=2; @@ -257,7 +259,8 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize, bitStream += (symbol-start) << bitCount; bitCount += 2; if (bitCount>16) { - if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ out[0] = (BYTE)bitStream; out[1] = (BYTE)(bitStream>>8); out += 2; @@ -268,7 +271,8 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize, int const max = (2*threshold-1) - remaining; remaining -= count < 0 ? -count : count; count++; /* +1 for extra accuracy */ - if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ + if (count>=threshold) + count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ bitStream += count << bitCount; bitCount += nbBits; bitCount -= (count>=1; } } if (bitCount>16) { - if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ out[0] = (BYTE)bitStream; out[1] = (BYTE)(bitStream>>8); out += 2; @@ -285,11 +290,13 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize, bitCount -= 16; } } - if (remaining != 1) return ERROR(GENERIC); /* incorrect normalized distribution */ + if (remaining != 1) + return ERROR(GENERIC); /* incorrect normalized distribution */ assert(symbol <= alphabetSize); /* flush remaining bitStream */ - if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ out[0] = (BYTE)bitStream; out[1] = (BYTE)(bitStream>>8); out+= (bitCount+7) /8; @@ -298,7 +305,8 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize, } -size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +size_t FSE_writeNCount (void* buffer, size_t bufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */ if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ diff --git a/lib/dictBuilder/divsufsort.c b/lib/dictBuilder/divsufsort.c index 60cceb088..ead922044 100644 --- a/lib/dictBuilder/divsufsort.c +++ b/lib/dictBuilder/divsufsort.c @@ -1637,7 +1637,7 @@ construct_SA(const unsigned char *T, int *SA, if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } k = SA + BUCKET_B(c2 = c0, c1); } - assert(k < j); + assert(k < j); assert(k != NULL); *k-- = s; } else { assert(((s == 0) && (T[s] == c1)) || (s < 0)); @@ -1701,7 +1701,7 @@ construct_BWT(const unsigned char *T, int *SA, if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } k = SA + BUCKET_B(c2 = c0, c1); } - assert(k < j); + assert(k < j); assert(k != NULL); *k-- = s; } else if(s != 0) { *j = ~s; @@ -1785,7 +1785,7 @@ construct_BWT_indexes(const unsigned char *T, int *SA, if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } k = SA + BUCKET_B(c2 = c0, c1); } - assert(k < j); + assert(k < j); assert(k != NULL); *k-- = s; } else if(s != 0) { *j = ~s; From 36d6165a2d75ab45ddec7da1f681e4153d3f8153 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 16 Aug 2018 15:41:56 -0700 Subject: [PATCH 87/92] Makefile: added variable SCANBUILD so that a different version of scan-build can be selected --- Makefile | 3 ++- contrib/seekable_format/examples/seekable_compression.c | 7 ++++--- lib/decompress/zstd_decompress.c | 1 + lib/legacy/zstd_v05.c | 1 + lib/legacy/zstd_v06.c | 1 + 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index e338c5ca1..853ceee93 100644 --- a/Makefile +++ b/Makefile @@ -352,7 +352,8 @@ bmi32build: clean # static analyzer test uses clang's scan-build # does not analyze zlibWrapper, due to detected issues in zlib source code +staticAnalyze: SCANBUILD ?= scan-build staticAnalyze: $(CC) -v - CC=$(CC) CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) allzstd examples contrib + CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) allzstd examples contrib endif diff --git a/contrib/seekable_format/examples/seekable_compression.c b/contrib/seekable_format/examples/seekable_compression.c index 9485bf26f..9a331a895 100644 --- a/contrib/seekable_format/examples/seekable_compression.c +++ b/contrib/seekable_format/examples/seekable_compression.c @@ -101,7 +101,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve free(buffOut); } -static const char* createOutFilename_orDie(const char* filename) +static char* createOutFilename_orDie(const char* filename) { size_t const inL = strlen(filename); size_t const outL = inL + 5; @@ -109,7 +109,7 @@ static const char* createOutFilename_orDie(const char* filename) memset(outSpace, 0, outL); strcat(outSpace, filename); strcat(outSpace, ".zst"); - return (const char*)outSpace; + return (char*)outSpace; } int main(int argc, const char** argv) { @@ -124,8 +124,9 @@ int main(int argc, const char** argv) { { const char* const inFileName = argv[1]; unsigned const frameSize = (unsigned)atoi(argv[2]); - const char* const outFileName = createOutFilename_orDie(inFileName); + char* const outFileName = createOutFilename_orDie(inFileName); compressFile_orDie(inFileName, outFileName, 5, frameSize); + free(outFileName); } return 0; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 8fef7e51b..7c4769cd8 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -542,6 +542,7 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize) { + if (dst==NULL) return ERROR(dstSize_tooSmall); if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall); memcpy(dst, src, srcSize); return srcSize; diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 89a5fe7e6..65f07a3b5 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -2846,6 +2846,7 @@ size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize) { + if (dst==NULL) return ERROR(dstSize_tooSmall); if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall); memcpy(dst, src, srcSize); return srcSize; diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index d8de77ca9..ca982c35e 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -3041,6 +3041,7 @@ size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize) { + if (dst==NULL) return ERROR(dstSize_tooSmall); if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall); memcpy(dst, src, srcSize); return srcSize; From 8175b28f03bc49a269c5481f8402b52b712d644c Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 16 Aug 2018 16:13:02 -0700 Subject: [PATCH 88/92] Fix negative lvl display value Also fix synthetic benchmark parameter setting --- tests/paramgrill.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/paramgrill.c b/tests/paramgrill.c index e94eead0d..f42afe403 100644 --- a/tests/paramgrill.c +++ b/tests/paramgrill.c @@ -655,7 +655,7 @@ static void BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize /* Writes to f the results of a parameter benchmark */ /* when used with --optimize, will only print results better than previously discovered */ -static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize) +static void BMK_printWinner(FILE* f, const int cLevel, const BMK_result_t result, const paramValues_t params, const size_t srcSize) { char lvlstr[15] = "Custom Level"; winnerInfo_t w; @@ -665,7 +665,7 @@ static void BMK_printWinner(FILE* f, const U32 cLevel, const BMK_result_t result fprintf(f, "\r%79s\r", ""); if(cLevel != CUSTOM_LEVEL) { - snprintf(lvlstr, 15, " Level %2u ", cLevel); + snprintf(lvlstr, 15, " Level %2d ", cLevel); } if(TIMED) { @@ -1865,8 +1865,9 @@ static void BMK_benchFullTable(const buffers_t buf, const contexts_t ctx) * Single Benchmark Functions **************************************/ -static int benchOnce(const buffers_t buf, const contexts_t ctx) { +static int benchOnce(const buffers_t buf, const contexts_t ctx, const int cLevel) { BMK_result_t testResult; + g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize); if(BMK_benchParam(&testResult, buf, ctx, g_params)) { DISPLAY("Error during benchmarking\n"); @@ -1878,7 +1879,7 @@ static int benchOnce(const buffers_t buf, const contexts_t ctx) { return 0; } -static int benchSample(double compressibility) +static int benchSample(double compressibility, int cLevel) { const char* const name = "Sample 10MB"; size_t const benchedSize = 10 MB; @@ -1912,7 +1913,7 @@ static int benchSample(double compressibility) DISPLAY("using %s %i%%: \n", name, (int)(compressibility*100)); if(g_singleRun) { - ret = benchOnce(buf, ctx); + ret = benchOnce(buf, ctx, cLevel); } else { BMK_benchFullTable(buf, ctx); } @@ -1926,7 +1927,7 @@ static int benchSample(double compressibility) /* benchFiles() : * note: while this function takes a table of filenames, * in practice, only the first filename will be used */ -int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName, int cLevel) +int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileName, const int cLevel) { buffers_t buf; contexts_t ctx; @@ -1950,10 +1951,8 @@ int benchFiles(const char** fileNamesTable, int nbFiles, const char* dictFileNam DISPLAY("using %d Files : \n", nbFiles); } - g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize); - if(g_singleRun) { - ret = benchOnce(buf, ctx); + ret = benchOnce(buf, ctx, cLevel); } else { BMK_benchFullTable(buf, ctx); } @@ -2731,7 +2730,7 @@ int main(int argc, const char** argv) DISPLAY("Optimizer Expects File\n"); return 1; } else { - result = benchSample(compressibility); + result = benchSample(compressibility, cLevelRun); } } else { if(seperateFiles) { From 3959ba15e633776456ebd3ed6bb96db5d775d1cd Mon Sep 17 00:00:00 2001 From: George Lu Date: Thu, 16 Aug 2018 17:22:29 -0700 Subject: [PATCH 89/92] Clarify README --- tests/README.md | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/README.md b/tests/README.md index c1128571a..f28766bd1 100644 --- a/tests/README.md +++ b/tests/README.md @@ -104,34 +104,40 @@ Full list of arguments t# - targetLength S# - strategy L# - level - --zstd= : Single run, parameter selection syntax same as zstdcli. - When invoked with --optimize, this represents the sample to exceed. + --zstd= : Single run, parameter selection syntax same as zstdcli with more parameters + (Added forceAttachDictionary / fadt) + When invoked with --optimize, this represents the sample to exceed. --optimize= : find parameters to maximize compression ratio given parameters - Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints + Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints cSpeed= : Minimum compression speed dSpeed= : Minimum decompression speed cMem= : Maximum compression memory lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%) : In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters - (Lower value will begin with stronger strategies) (Default 90%) + (Lower value will begin with stronger strategies) (Default 90%) speedRatio= (accepts decimals) : determines value of gains in speed vs gains in ratio - when determining overall winner (default 5 (1% ratio = 5% speed)). + when determining overall winner (default 5 (1% ratio = 5% speed)). tries= : Maximum number of random restarts on a single strategy before switching (Default 5) - Higher values will make optimizer run longer, more chances to find better solution. - memLog : Limits the log of the size of each memotable (1 per strategy). Setting memLog = 0 turns off memoization + Higher values will make optimizer run longer, more chances to find better solution. + memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size. + Setting memLog = 0 turns off memoization --display= : specifiy which parameters are included in the output - can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters - (Default: display all params available) - - -P# : generated sample compressibility + can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters + (Default: display all params available) + -P# : generated sample compressibility (when no file is provided) -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) -v : Prints Benchmarking output -D : Next argument dictionary file -s : Benchmark all files separately -q : Quiet, repeat for more quiet + -q Prints parameters + results whenever a new best is found + -qq Only prints parameters whenever a new best is found, prints final parameters + results + -qqq Only print final parameters + results + -qqqq Only prints final parameter set in the form --zstd= -v : Verbose, cancels quiet, repeat for more volume + -v Prints all candidate parameters and results ``` Any inputs afterwards are treated as files to benchmark. From e400a86f1727918dbc2f073ce74025a0726eda8e Mon Sep 17 00:00:00 2001 From: Timo Gurr Date: Fri, 17 Aug 2018 13:31:55 +0200 Subject: [PATCH 90/92] Use GNUInstallDirs DOCDIR for installing the manual --- build/cmake/contrib/gen_html/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/cmake/contrib/gen_html/CMakeLists.txt b/build/cmake/contrib/gen_html/CMakeLists.txt index 958e9d173..c4062d475 100644 --- a/build/cmake/contrib/gen_html/CMakeLists.txt +++ b/build/cmake/contrib/gen_html/CMakeLists.txt @@ -27,4 +27,4 @@ ADD_CUSTOM_TARGET(zstd_manual.html ALL ${GENHTML_BINARY} "${LIBVERSION}" "${LIBRARY_DIR}/zstd.h" "${PROJECT_BINARY_DIR}/zstd_manual.html" DEPENDS gen_html COMMENT "Update zstd manual") -INSTALL(FILES "${PROJECT_BINARY_DIR}/zstd_manual.html" DESTINATION "${CMAKE_INSTALL_PREFIX}/${DOC_INSTALL_DIR}") +INSTALL(FILES "${PROJECT_BINARY_DIR}/zstd_manual.html" DESTINATION "${CMAKE_INSTALL_DOCDIR}") From 78af534f82e32277d3272881d4351363ab1e3488 Mon Sep 17 00:00:00 2001 From: Eden Zik Date: Mon, 20 Aug 2018 22:15:24 -0400 Subject: [PATCH 91/92] Fixed unsafe string copy and concat in `fileio.c`. Per warnings from flawfinder: "Does not check for buffer overflows when copying to destination [MS-banned] (CWE-120). Consider using snprintf, strcpy_s, or strlcpy (warning: strncpy easily misused).". Replaced called to strcpy and strcat in `fileio.c` to calls with a specified size (`strncpy` and `strncat`). Tested the changes on OSX, Linux, Windows. On OSX + Linux, changes were tested with ASAN. The following flags were used: 'check_initialization_order=1:strict_init_order=1:detect_odr_violation=1:detect_stack_use_after_return=1' To reproduce warning: ./flawfinder.py ./programs/fileio.c --- programs/fileio.c | 4 ++-- tests/.gitignore | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 39b2c741c..5f10958d7 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -1011,8 +1011,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile if (!dstFileName) { EXM_THROW(30, "zstd: %s", strerror(errno)); } } - strcpy(dstFileName, inFileNamesTable[u]); - strcat(dstFileName, suffix); + strncpy(dstFileName, inFileNamesTable[u], ifnSize+1 /* Include null */); + strncat(dstFileName, suffix, suffixSize); missed_files += FIO_compressFilename_dstFile(ress, dstFileName, inFileNamesTable[u], compressionLevel); } } diff --git a/tests/.gitignore b/tests/.gitignore index 4911b2d62..da536251d 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -26,6 +26,7 @@ invalidDictionaries checkTag zcat zstdcat +tm # Tmp test directory zstdtest From 9d6ed9def34fa042291d48637dabc695bfa5d23f Mon Sep 17 00:00:00 2001 From: Jennifer Liu Date: Thu, 23 Aug 2018 12:06:20 -0700 Subject: [PATCH 92/92] Merge fastCover into DictBuilder (#1274) * Minor fix * Run non-optimize FASTCOVER 5 times in benchmark * Merge fastCover into dictBuilder * Fix mixed declaration issue * Add fastcover to symbol.c * Add fastCover.c and cover.h to build * Change fastCover.c to fastcover.c * Update benchmark to run FASTCOVER in dictBuilder * Undo spliting fastcover_param into cover_param and f * Remove convert param functions * Assign f to parameter * Add zdict.h to Makefile in lib * Add cover.h to BUCK * Cast 1 to U64 before shifting * Remove trimming of zero freq head and tail in selectSegment and rebenchmark * Remove f as a separate parameter of tryParam * Read 8 bytes when d is 6 * Add trimming off zero frequency head and tail * Use best functions from COVER and remove trimming part(which leads to worse compression ratio after previous bugs were fixed) * Add finalize= argument to FASTCOVER to specify percentage of training samples passed to ZDICT_finalizeDictionary * Change nbDmer to always read 8 bytes even when d=6 * Add skip=# argument to allow skipping dmers in computeFrequency in FASTCOVER * Update comments and benchmarking result * Change default method of ZDICT_trainFromBuffer to ZDICT_optimizeTrainFromBuffer_fastCover * Add dictType enum and fix bug about passing zParam when converting to coverParam * Combine finalize and skip into a single parameter * Update acceleration parameters and benchmark on 3 sample sets * Change default splitPoint of FASTCOVER to 0.75 and benchmark first 3 sample sets * Initialize variables outside of for loop in benchmark.c * Update benchmark result for hg-manifest * Remove cover.h from install-includes * Add explanation of f * Set default compression level for trainFromBuffer to 3 * Add assertion of fastCoverParams in DiB_trainFromFiles * Add checkTotalCompressedSize function + some minor fixes * Add test for multithreading fastCovr * Initialize segmentFreqs in every FASTCOVER_selectSegment and move mutex_unnlock to end of COVER_best_finish * Free segmentFreqs * Initialize segmentFreqs before calling FASTCOVER_buildDictionary instead of in FASTCOVER_selectSegment * Add FASTCOVER_MEMMULT * Minor fix * Update benchmarking result --- build/VS2008/fuzzer/fuzzer.vcproj | 8 + build/VS2008/zstd/zstd.vcproj | 8 + build/VS2008/zstdlib/zstdlib.vcproj | 8 + build/VS2010/fuzzer/fuzzer.vcxproj | 2 + build/VS2010/libzstd-dll/libzstd-dll.vcxproj | 1 + build/VS2010/libzstd/libzstd.vcxproj | 1 + build/VS2010/zstd/zstd.vcxproj | 2 + build/cmake/lib/CMakeLists.txt | 3 + .../benchmarkDictBuilder/Makefile | 10 +- .../benchmarkDictBuilder/README.md | 925 ++++++++++++++++-- .../benchmarkDictBuilder/benchmark.c | 57 +- .../fastCover/fastCover.c | 4 +- lib/BUCK | 1 + lib/dictBuilder/cover.c | 147 ++- lib/dictBuilder/cover.h | 83 ++ lib/dictBuilder/fastcover.c | 701 +++++++++++++ lib/dictBuilder/zdict.c | 10 +- lib/dictBuilder/zdict.h | 59 +- programs/README.md | 1 + programs/dibio.c | 50 +- programs/dibio.h | 2 +- programs/zstd.1 | 21 +- programs/zstd.1.md | 20 + programs/zstdcli.c | 76 +- tests/playTests.sh | 43 +- tests/symbols.c | 2 + 26 files changed, 1999 insertions(+), 246 deletions(-) create mode 100644 lib/dictBuilder/cover.h create mode 100644 lib/dictBuilder/fastcover.c diff --git a/build/VS2008/fuzzer/fuzzer.vcproj b/build/VS2008/fuzzer/fuzzer.vcproj index f6ea1f855..4d444caef 100644 --- a/build/VS2008/fuzzer/fuzzer.vcproj +++ b/build/VS2008/fuzzer/fuzzer.vcproj @@ -336,6 +336,10 @@ RelativePath="..\..\..\lib\dictBuilder\cover.c" > + + @@ -482,6 +486,10 @@ RelativePath="..\..\..\lib\dictBuilder\zdict.h" > + + diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj index 5d9f68327..46d7c85dd 100644 --- a/build/VS2008/zstd/zstd.vcproj +++ b/build/VS2008/zstd/zstd.vcproj @@ -348,6 +348,10 @@ RelativePath="..\..\..\lib\dictBuilder\cover.c" > + + @@ -522,6 +526,10 @@ RelativePath="..\..\..\lib\dictBuilder\zdict.h" > + + diff --git a/build/VS2008/zstdlib/zstdlib.vcproj b/build/VS2008/zstdlib/zstdlib.vcproj index 7234b02ac..950a6f39c 100644 --- a/build/VS2008/zstdlib/zstdlib.vcproj +++ b/build/VS2008/zstdlib/zstdlib.vcproj @@ -332,6 +332,10 @@ RelativePath="..\..\..\lib\dictBuilder\cover.c" > + + @@ -502,6 +506,10 @@ RelativePath="..\..\..\lib\dictBuilder\zdict.h" > + + diff --git a/build/VS2010/fuzzer/fuzzer.vcxproj b/build/VS2010/fuzzer/fuzzer.vcxproj index f0d1ab066..6077cd2c1 100644 --- a/build/VS2010/fuzzer/fuzzer.vcxproj +++ b/build/VS2010/fuzzer/fuzzer.vcxproj @@ -176,6 +176,7 @@ + @@ -199,6 +200,7 @@ + diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj index 92d518d3d..6e14e020e 100644 --- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj +++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj @@ -43,6 +43,7 @@ + diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj index c306fcec9..18f5cb533 100644 --- a/build/VS2010/libzstd/libzstd.vcxproj +++ b/build/VS2010/libzstd/libzstd.vcxproj @@ -43,6 +43,7 @@ + diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj index 4af281326..936960b58 100644 --- a/build/VS2010/zstd/zstd.vcxproj +++ b/build/VS2010/zstd/zstd.vcxproj @@ -40,6 +40,7 @@ + @@ -61,6 +62,7 @@ + diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index e84e06301..ffc196ddc 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -47,6 +47,7 @@ SET(Sources ${LIBRARY_DIR}/decompress/huf_decompress.c ${LIBRARY_DIR}/decompress/zstd_decompress.c ${LIBRARY_DIR}/dictBuilder/cover.c + ${LIBRARY_DIR}/dictBuilder/fastcover.c ${LIBRARY_DIR}/dictBuilder/divsufsort.c ${LIBRARY_DIR}/dictBuilder/zdict.c ${LIBRARY_DIR}/deprecated/zbuff_common.c @@ -74,6 +75,7 @@ SET(Headers ${LIBRARY_DIR}/compress/zstd_ldm.h ${LIBRARY_DIR}/compress/zstdmt_compress.h ${LIBRARY_DIR}/dictBuilder/zdict.h + ${LIBRARY_DIR}/dictBuilder/cover.h ${LIBRARY_DIR}/deprecated/zbuff.h) IF (ZSTD_LEGACY_SUPPORT) @@ -178,6 +180,7 @@ INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/deprecated/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h + ${LIBRARY_DIR}/dictBuilder/cover.h ${LIBRARY_DIR}/common/zstd_errors.h DESTINATION "include") diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile b/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile index 681494888..72ce04f2a 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile @@ -2,10 +2,9 @@ ARG := CC ?= gcc CFLAGS ?= -O3 -INCLUDES := -I ../randomDictBuilder -I ../fastCover -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder +INCLUDES := -I ../randomDictBuilder -I ../../../programs -I ../../../lib/common -I ../../../lib -I ../../../lib/dictBuilder RANDOM_FILE := ../randomDictBuilder/random.c -FAST_FILE := ../fastCover/fastCover.c IO_FILE := ../randomDictBuilder/io.c all: run clean @@ -22,8 +21,8 @@ test: benchmarkTest clean benchmarkTest: benchmark test.sh sh test.sh -benchmark: benchmark.o io.o random.o fastCover.o libzstd.a - $(CC) $(CFLAGS) benchmark.o io.o random.o fastCover.o libzstd.a -o benchmark +benchmark: benchmark.o io.o random.o libzstd.a + $(CC) $(CFLAGS) benchmark.o io.o random.o libzstd.a -o benchmark benchmark.o: benchmark.c $(CC) $(CFLAGS) $(INCLUDES) -c benchmark.c @@ -31,9 +30,6 @@ benchmark.o: benchmark.c random.o: $(RANDOM_FILE) $(CC) $(CFLAGS) $(INCLUDES) -c $(RANDOM_FILE) -fastCover.o: $(FAST_FILE) - $(CC) $(CFLAGS) $(INCLUDES) -c $(FAST_FILE) - io.o: $(IO_FILE) $(CC) $(CFLAGS) $(INCLUDES) -c $(IO_FILE) diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md index 559776e2b..6a6c7f1d2 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md @@ -14,113 +14,836 @@ make ARG="in=../../../lib/dictBuilder in=../../../lib/compress" ###Benchmarking Result: - First Cover is optimize cover, second Cover uses optimized d and k from first one. -- For every f value of fastCover, the first one is optimize fastCover and the second one uses optimized d and k from first one. +- For every f value of fastCover, the first one is optimize fastCover and the second one uses optimized d and k from first one. This is run for accel values from 1 to 10. - Fourth column is chosen d and fifth column is chosen k github: -NODICT 0.000025 2.999642 -RANDOM 0.030101 8.791189 -LEGACY 0.913108 8.173529 -COVER 59.234160 10.652243 8 1298 -COVER 6.258459 10.652243 8 1298 -FAST15 9.959246 10.555630 8 1874 -FAST15 0.077719 10.555630 8 1874 -FAST16 10.028343 10.701698 8 1106 -FAST16 0.078117 10.701698 8 1106 -FAST17 10.567355 10.650652 8 1106 -FAST17 0.124833 10.650652 8 1106 -FAST18 11.795287 10.499142 8 1826 -FAST18 0.086992 10.499142 8 1826 -FAST19 13.132451 10.527140 8 1826 -FAST19 0.134716 10.527140 8 1826 -FAST20 14.366314 10.494710 8 1826 -FAST20 0.128844 10.494710 8 1826 -FAST21 14.941238 10.503488 8 1778 -FAST21 0.134975 10.503488 8 1778 -FAST22 15.146226 10.509284 8 1826 -FAST22 0.146918 10.509284 8 1826 -FAST23 16.260552 10.509284 8 1826 -FAST23 0.158494 10.509284 8 1826 -FAST24 16.806037 10.512369 8 1826 -FAST24 0.190464 10.512369 8 1826 +NODICT 0.000004 2.999642 +RANDOM 0.024560 8.791189 +LEGACY 0.727109 8.173529 +COVER 40.565676 10.652243 8 1298 +COVER 3.608284 10.652243 8 1298 +FAST f=15 a=1 4.181024 10.570882 8 1154 +FAST f=15 a=1 0.040788 10.570882 8 1154 +FAST f=15 a=2 3.548352 10.574287 6 1970 +FAST f=15 a=2 0.035535 10.574287 6 1970 +FAST f=15 a=3 3.287364 10.613950 6 1010 +FAST f=15 a=3 0.032182 10.613950 6 1010 +FAST f=15 a=4 3.184976 10.573883 6 1058 +FAST f=15 a=4 0.029878 10.573883 6 1058 +FAST f=15 a=5 3.045513 10.580640 8 1154 +FAST f=15 a=5 0.022162 10.580640 8 1154 +FAST f=15 a=6 3.003296 10.583677 6 1010 +FAST f=15 a=6 0.028091 10.583677 6 1010 +FAST f=15 a=7 2.952655 10.622551 6 1106 +FAST f=15 a=7 0.02724 10.622551 6 1106 +FAST f=15 a=8 2.945674 10.614657 6 1010 +FAST f=15 a=8 0.027264 10.614657 6 1010 +FAST f=15 a=9 3.153439 10.564018 8 1154 +FAST f=15 a=9 0.020635 10.564018 8 1154 +FAST f=15 a=10 2.950416 10.511454 6 1010 +FAST f=15 a=10 0.026606 10.511454 6 1010 +FAST f=16 a=1 3.970029 10.681035 8 1154 +FAST f=16 a=1 0.038188 10.681035 8 1154 +FAST f=16 a=2 3.422892 10.484978 6 1874 +FAST f=16 a=2 0.034702 10.484978 6 1874 +FAST f=16 a=3 3.215836 10.632631 8 1154 +FAST f=16 a=3 0.026084 10.632631 8 1154 +FAST f=16 a=4 3.081353 10.626533 6 1106 +FAST f=16 a=4 0.030032 10.626533 6 1106 +FAST f=16 a=5 3.041241 10.545027 8 1922 +FAST f=16 a=5 0.022882 10.545027 8 1922 +FAST f=16 a=6 2.989390 10.638284 6 1874 +FAST f=16 a=6 0.028308 10.638284 6 1874 +FAST f=16 a=7 3.001581 10.797136 6 1106 +FAST f=16 a=7 0.027479 10.797136 6 1106 +FAST f=16 a=8 2.984107 10.658356 8 1058 +FAST f=16 a=8 0.021099 10.658356 8 1058 +FAST f=16 a=9 2.925788 10.523869 6 1010 +FAST f=16 a=9 0.026905 10.523869 6 1010 +FAST f=16 a=10 2.889605 10.745841 6 1874 +FAST f=16 a=10 0.026846 10.745841 6 1874 +FAST f=17 a=1 4.031953 10.672080 8 1202 +FAST f=17 a=1 0.040658 10.672080 8 1202 +FAST f=17 a=2 3.458107 10.589352 8 1106 +FAST f=17 a=2 0.02926 10.589352 8 1106 +FAST f=17 a=3 3.291189 10.662714 8 1154 +FAST f=17 a=3 0.026531 10.662714 8 1154 +FAST f=17 a=4 3.154950 10.549456 8 1346 +FAST f=17 a=4 0.024991 10.549456 8 1346 +FAST f=17 a=5 3.092271 10.541670 6 1202 +FAST f=17 a=5 0.038285 10.541670 6 1202 +FAST f=17 a=6 3.166146 10.729112 6 1874 +FAST f=17 a=6 0.038217 10.729112 6 1874 +FAST f=17 a=7 3.035467 10.810485 6 1106 +FAST f=17 a=7 0.036655 10.810485 6 1106 +FAST f=17 a=8 3.035668 10.530532 6 1058 +FAST f=17 a=8 0.037715 10.530532 6 1058 +FAST f=17 a=9 2.987917 10.589802 8 1922 +FAST f=17 a=9 0.02217 10.589802 8 1922 +FAST f=17 a=10 2.981647 10.722579 8 1106 +FAST f=17 a=10 0.021948 10.722579 8 1106 +FAST f=18 a=1 4.067144 10.634943 8 1154 +FAST f=18 a=1 0.041386 10.634943 8 1154 +FAST f=18 a=2 3.507377 10.546230 6 1970 +FAST f=18 a=2 0.037572 10.546230 6 1970 +FAST f=18 a=3 3.323015 10.648061 8 1154 +FAST f=18 a=3 0.028306 10.648061 8 1154 +FAST f=18 a=4 3.216735 10.705402 6 1010 +FAST f=18 a=4 0.030755 10.705402 6 1010 +FAST f=18 a=5 3.175794 10.588154 8 1874 +FAST f=18 a=5 0.025315 10.588154 8 1874 +FAST f=18 a=6 3.127459 10.751104 8 1106 +FAST f=18 a=6 0.023897 10.751104 8 1106 +FAST f=18 a=7 3.083017 10.780402 6 1106 +FAST f=18 a=7 0.029158 10.780402 6 1106 +FAST f=18 a=8 3.069700 10.547226 8 1346 +FAST f=18 a=8 0.024046 10.547226 8 1346 +FAST f=18 a=9 3.056591 10.674759 6 1010 +FAST f=18 a=9 0.028496 10.674759 6 1010 +FAST f=18 a=10 3.063588 10.737578 8 1106 +FAST f=18 a=10 0.023033 10.737578 8 1106 +FAST f=19 a=1 4.164041 10.650333 8 1154 +FAST f=19 a=1 0.042906 10.650333 8 1154 +FAST f=19 a=2 3.585409 10.577066 6 1058 +FAST f=19 a=2 0.038994 10.577066 6 1058 +FAST f=19 a=3 3.439643 10.639403 8 1154 +FAST f=19 a=3 0.028427 10.639403 8 1154 +FAST f=19 a=4 3.268869 10.554410 8 1298 +FAST f=19 a=4 0.026866 10.554410 8 1298 +FAST f=19 a=5 3.238225 10.615109 6 1010 +FAST f=19 a=5 0.03078 10.615109 6 1010 +FAST f=19 a=6 3.199558 10.609782 6 1874 +FAST f=19 a=6 0.030099 10.609782 6 1874 +FAST f=19 a=7 3.132395 10.794753 6 1106 +FAST f=19 a=7 0.028964 10.794753 6 1106 +FAST f=19 a=8 3.148446 10.554842 8 1298 +FAST f=19 a=8 0.024277 10.554842 8 1298 +FAST f=19 a=9 3.108324 10.668763 6 1010 +FAST f=19 a=9 0.02896 10.668763 6 1010 +FAST f=19 a=10 3.159863 10.757347 8 1106 +FAST f=19 a=10 0.023351 10.757347 8 1106 +FAST f=20 a=1 4.462698 10.661788 8 1154 +FAST f=20 a=1 0.047174 10.661788 8 1154 +FAST f=20 a=2 3.820269 10.678612 6 1106 +FAST f=20 a=2 0.040807 10.678612 6 1106 +FAST f=20 a=3 3.644955 10.648424 8 1154 +FAST f=20 a=3 0.031398 10.648424 8 1154 +FAST f=20 a=4 3.546257 10.559756 8 1298 +FAST f=20 a=4 0.029856 10.559756 8 1298 +FAST f=20 a=5 3.485248 10.646637 6 1010 +FAST f=20 a=5 0.033756 10.646637 6 1010 +FAST f=20 a=6 3.490438 10.775824 8 1106 +FAST f=20 a=6 0.028338 10.775824 8 1106 +FAST f=20 a=7 3.631289 10.801795 6 1106 +FAST f=20 a=7 0.035228 10.801795 6 1106 +FAST f=20 a=8 3.758936 10.545116 8 1346 +FAST f=20 a=8 0.027495 10.545116 8 1346 +FAST f=20 a=9 3.707024 10.677454 6 1010 +FAST f=20 a=9 0.031326 10.677454 6 1010 +FAST f=20 a=10 3.586593 10.756017 8 1106 +FAST f=20 a=10 0.027122 10.756017 8 1106 +FAST f=21 a=1 5.701396 10.655398 8 1154 +FAST f=21 a=1 0.067744 10.655398 8 1154 +FAST f=21 a=2 5.270542 10.650743 6 1106 +FAST f=21 a=2 0.052999 10.650743 6 1106 +FAST f=21 a=3 4.945294 10.652380 8 1154 +FAST f=21 a=3 0.052678 10.652380 8 1154 +FAST f=21 a=4 4.894079 10.543185 8 1298 +FAST f=21 a=4 0.04997 10.543185 8 1298 +FAST f=21 a=5 4.785417 10.630321 6 1010 +FAST f=21 a=5 0.045294 10.630321 6 1010 +FAST f=21 a=6 4.789381 10.664477 6 1874 +FAST f=21 a=6 0.046578 10.664477 6 1874 +FAST f=21 a=7 4.302955 10.805179 6 1106 +FAST f=21 a=7 0.041205 10.805179 6 1106 +FAST f=21 a=8 4.034630 10.551211 8 1298 +FAST f=21 a=8 0.040121 10.551211 8 1298 +FAST f=21 a=9 4.523868 10.799114 6 1010 +FAST f=21 a=9 0.043592 10.799114 6 1010 +FAST f=21 a=10 4.760736 10.750255 8 1106 +FAST f=21 a=10 0.043483 10.750255 8 1106 +FAST f=22 a=1 6.743064 10.640537 8 1154 +FAST f=22 a=1 0.086967 10.640537 8 1154 +FAST f=22 a=2 6.121739 10.626638 6 1970 +FAST f=22 a=2 0.066337 10.626638 6 1970 +FAST f=22 a=3 5.248851 10.640688 8 1154 +FAST f=22 a=3 0.054935 10.640688 8 1154 +FAST f=22 a=4 5.436579 10.588333 8 1298 +FAST f=22 a=4 0.064113 10.588333 8 1298 +FAST f=22 a=5 5.812815 10.652653 6 1010 +FAST f=22 a=5 0.058189 10.652653 6 1010 +FAST f=22 a=6 5.745472 10.666437 6 1874 +FAST f=22 a=6 0.057188 10.666437 6 1874 +FAST f=22 a=7 5.716393 10.806911 6 1106 +FAST f=22 a=7 0.056 10.806911 6 1106 +FAST f=22 a=8 5.698799 10.530784 8 1298 +FAST f=22 a=8 0.0583 10.530784 8 1298 +FAST f=22 a=9 5.710533 10.777391 6 1010 +FAST f=22 a=9 0.054945 10.777391 6 1010 +FAST f=22 a=10 5.685395 10.745023 8 1106 +FAST f=22 a=10 0.056526 10.745023 8 1106 +FAST f=23 a=1 7.836923 10.638828 8 1154 +FAST f=23 a=1 0.099522 10.638828 8 1154 +FAST f=23 a=2 6.627834 10.631061 6 1970 +FAST f=23 a=2 0.066769 10.631061 6 1970 +FAST f=23 a=3 5.602533 10.647288 8 1154 +FAST f=23 a=3 0.064513 10.647288 8 1154 +FAST f=23 a=4 6.005580 10.568747 8 1298 +FAST f=23 a=4 0.062022 10.568747 8 1298 +FAST f=23 a=5 5.481816 10.676921 6 1010 +FAST f=23 a=5 0.058959 10.676921 6 1010 +FAST f=23 a=6 5.460444 10.666194 6 1874 +FAST f=23 a=6 0.057687 10.666194 6 1874 +FAST f=23 a=7 5.659822 10.800377 6 1106 +FAST f=23 a=7 0.06783 10.800377 6 1106 +FAST f=23 a=8 6.826940 10.522167 8 1298 +FAST f=23 a=8 0.070533 10.522167 8 1298 +FAST f=23 a=9 6.804757 10.577799 8 1682 +FAST f=23 a=9 0.069949 10.577799 8 1682 +FAST f=23 a=10 6.774933 10.742093 8 1106 +FAST f=23 a=10 0.068395 10.742093 8 1106 +FAST f=24 a=1 8.444110 10.632783 8 1154 +FAST f=24 a=1 0.094357 10.632783 8 1154 +FAST f=24 a=2 7.289578 10.631061 6 1970 +FAST f=24 a=2 0.098515 10.631061 6 1970 +FAST f=24 a=3 8.619780 10.646289 8 1154 +FAST f=24 a=3 0.098041 10.646289 8 1154 +FAST f=24 a=4 8.508455 10.555199 8 1298 +FAST f=24 a=4 0.093885 10.555199 8 1298 +FAST f=24 a=5 8.471145 10.674363 6 1010 +FAST f=24 a=5 0.088676 10.674363 6 1010 +FAST f=24 a=6 8.426727 10.667228 6 1874 +FAST f=24 a=6 0.087247 10.667228 6 1874 +FAST f=24 a=7 8.356826 10.803027 6 1106 +FAST f=24 a=7 0.085835 10.803027 6 1106 +FAST f=24 a=8 6.756811 10.522049 8 1298 +FAST f=24 a=8 0.07107 10.522049 8 1298 +FAST f=24 a=9 6.548169 10.571882 8 1682 +FAST f=24 a=9 0.0713 10.571882 8 1682 +FAST f=24 a=10 8.238079 10.736453 8 1106 +FAST f=24 a=10 0.07004 10.736453 8 1106 + hg-commands: -NODICT 0.000026 2.425291 -RANDOM 0.046270 3.490331 -LEGACY 0.847904 3.911682 -COVER 71.691804 4.132653 8 386 -COVER 3.187085 4.132653 8 386 -FAST15 11.593687 3.920720 6 1106 -FAST15 0.082431 3.920720 6 1106 -FAST16 11.775958 4.033306 8 674 -FAST16 0.092587 4.033306 8 674 -FAST17 11.965064 4.064132 8 1490 -FAST17 0.106382 4.064132 8 1490 -FAST18 11.438197 4.086714 8 290 -FAST18 0.097293 4.086714 8 290 -FAST19 12.292512 4.097947 8 578 -FAST19 0.104406 4.097947 8 578 -FAST20 13.857857 4.102851 8 434 -FAST20 0.139467 4.102851 8 434 -FAST21 14.599613 4.105350 8 530 -FAST21 0.189416 4.105350 8 530 -FAST22 15.966109 4.104100 8 530 -FAST22 0.183817 4.104100 8 530 -FAST23 18.033645 4.098110 8 914 -FAST23 0.246641 4.098110 8 914 -FAST24 22.992891 4.117367 8 722 -FAST24 0.285994 4.117367 8 722 +NODICT 0.000005 2.425276 +RANDOM 0.046332 3.490331 +LEGACY 0.720351 3.911682 +COVER 45.507731 4.132653 8 386 +COVER 1.868810 4.132653 8 386 +FAST f=15 a=1 4.561427 3.866894 8 1202 +FAST f=15 a=1 0.048946 3.866894 8 1202 +FAST f=15 a=2 3.574462 3.892119 8 1538 +FAST f=15 a=2 0.033677 3.892119 8 1538 +FAST f=15 a=3 3.230227 3.888791 6 1346 +FAST f=15 a=3 0.034312 3.888791 6 1346 +FAST f=15 a=4 3.042388 3.899739 8 1010 +FAST f=15 a=4 0.024307 3.899739 8 1010 +FAST f=15 a=5 2.800148 3.896220 8 818 +FAST f=15 a=5 0.022331 3.896220 8 818 +FAST f=15 a=6 2.706518 3.882039 8 578 +FAST f=15 a=6 0.020955 3.882039 8 578 +FAST f=15 a=7 2.701820 3.885430 6 866 +FAST f=15 a=7 0.026074 3.885430 6 866 +FAST f=15 a=8 2.604445 3.906932 8 1826 +FAST f=15 a=8 0.021789 3.906932 8 1826 +FAST f=15 a=9 2.598568 3.870324 6 1682 +FAST f=15 a=9 0.026004 3.870324 6 1682 +FAST f=15 a=10 2.575920 3.920783 8 1442 +FAST f=15 a=10 0.020228 3.920783 8 1442 +FAST f=16 a=1 4.630623 4.001430 8 770 +FAST f=16 a=1 0.047497 4.001430 8 770 +FAST f=16 a=2 3.674721 3.974431 8 1874 +FAST f=16 a=2 0.035761 3.974431 8 1874 +FAST f=16 a=3 3.338384 3.978703 8 1010 +FAST f=16 a=3 0.029436 3.978703 8 1010 +FAST f=16 a=4 3.004412 3.983035 8 1010 +FAST f=16 a=4 0.025744 3.983035 8 1010 +FAST f=16 a=5 2.881892 3.987710 8 770 +FAST f=16 a=5 0.023211 3.987710 8 770 +FAST f=16 a=6 2.807410 3.952717 8 1298 +FAST f=16 a=6 0.023199 3.952717 8 1298 +FAST f=16 a=7 2.819623 3.994627 8 770 +FAST f=16 a=7 0.021806 3.994627 8 770 +FAST f=16 a=8 2.740092 3.954032 8 1826 +FAST f=16 a=8 0.0226 3.954032 8 1826 +FAST f=16 a=9 2.682564 3.969879 6 1442 +FAST f=16 a=9 0.026324 3.969879 6 1442 +FAST f=16 a=10 2.657959 3.969755 8 674 +FAST f=16 a=10 0.020413 3.969755 8 674 +FAST f=17 a=1 4.729228 4.046000 8 530 +FAST f=17 a=1 0.049703 4.046000 8 530 +FAST f=17 a=2 3.764510 3.991519 8 1970 +FAST f=17 a=2 0.038195 3.991519 8 1970 +FAST f=17 a=3 3.416992 4.006296 6 914 +FAST f=17 a=3 0.036244 4.006296 6 914 +FAST f=17 a=4 3.145626 3.979182 8 1970 +FAST f=17 a=4 0.028676 3.979182 8 1970 +FAST f=17 a=5 2.995070 4.050070 8 770 +FAST f=17 a=5 0.025707 4.050070 8 770 +FAST f=17 a=6 2.911833 4.040024 8 770 +FAST f=17 a=6 0.02453 4.040024 8 770 +FAST f=17 a=7 2.894796 4.015884 8 818 +FAST f=17 a=7 0.023956 4.015884 8 818 +FAST f=17 a=8 2.789962 4.039303 8 530 +FAST f=17 a=8 0.023219 4.039303 8 530 +FAST f=17 a=9 2.787625 3.996762 8 1634 +FAST f=17 a=9 0.023651 3.996762 8 1634 +FAST f=17 a=10 2.754796 4.005059 8 1058 +FAST f=17 a=10 0.022537 4.005059 8 1058 +FAST f=18 a=1 4.779117 4.038214 8 242 +FAST f=18 a=1 0.048814 4.038214 8 242 +FAST f=18 a=2 3.829753 4.045768 8 722 +FAST f=18 a=2 0.036541 4.045768 8 722 +FAST f=18 a=3 3.495053 4.021497 8 770 +FAST f=18 a=3 0.032648 4.021497 8 770 +FAST f=18 a=4 3.221395 4.039623 8 770 +FAST f=18 a=4 0.027818 4.039623 8 770 +FAST f=18 a=5 3.059369 4.050414 8 530 +FAST f=18 a=5 0.026296 4.050414 8 530 +FAST f=18 a=6 3.019292 4.010714 6 962 +FAST f=18 a=6 0.031104 4.010714 6 962 +FAST f=18 a=7 2.949322 4.031439 6 770 +FAST f=18 a=7 0.030745 4.031439 6 770 +FAST f=18 a=8 2.876425 4.032088 6 386 +FAST f=18 a=8 0.027407 4.032088 6 386 +FAST f=18 a=9 2.850958 4.053372 8 674 +FAST f=18 a=9 0.023799 4.053372 8 674 +FAST f=18 a=10 2.884352 4.020148 8 1730 +FAST f=18 a=10 0.024401 4.020148 8 1730 +FAST f=19 a=1 4.815669 4.061203 8 674 +FAST f=19 a=1 0.051425 4.061203 8 674 +FAST f=19 a=2 3.951356 4.013822 8 1442 +FAST f=19 a=2 0.039968 4.013822 8 1442 +FAST f=19 a=3 3.554682 4.050425 8 722 +FAST f=19 a=3 0.032725 4.050425 8 722 +FAST f=19 a=4 3.242585 4.054677 8 722 +FAST f=19 a=4 0.028194 4.054677 8 722 +FAST f=19 a=5 3.105909 4.064524 8 818 +FAST f=19 a=5 0.02675 4.064524 8 818 +FAST f=19 a=6 3.059901 4.036857 8 1250 +FAST f=19 a=6 0.026396 4.036857 8 1250 +FAST f=19 a=7 3.016151 4.068234 6 770 +FAST f=19 a=7 0.031501 4.068234 6 770 +FAST f=19 a=8 2.962902 4.077509 8 530 +FAST f=19 a=8 0.023333 4.077509 8 530 +FAST f=19 a=9 2.899607 4.067328 8 530 +FAST f=19 a=9 0.024553 4.067328 8 530 +FAST f=19 a=10 2.950978 4.059901 8 434 +FAST f=19 a=10 0.023852 4.059901 8 434 +FAST f=20 a=1 5.259834 4.027579 8 1634 +FAST f=20 a=1 0.061123 4.027579 8 1634 +FAST f=20 a=2 4.382150 4.025093 8 1634 +FAST f=20 a=2 0.048009 4.025093 8 1634 +FAST f=20 a=3 4.104323 4.060842 8 530 +FAST f=20 a=3 0.040965 4.060842 8 530 +FAST f=20 a=4 3.853340 4.023504 6 914 +FAST f=20 a=4 0.041072 4.023504 6 914 +FAST f=20 a=5 3.728841 4.018089 6 1634 +FAST f=20 a=5 0.037469 4.018089 6 1634 +FAST f=20 a=6 3.683045 4.069138 8 578 +FAST f=20 a=6 0.028011 4.069138 8 578 +FAST f=20 a=7 3.726973 4.063160 8 722 +FAST f=20 a=7 0.028437 4.063160 8 722 +FAST f=20 a=8 3.555073 4.057690 8 386 +FAST f=20 a=8 0.027588 4.057690 8 386 +FAST f=20 a=9 3.551095 4.067253 8 482 +FAST f=20 a=9 0.025976 4.067253 8 482 +FAST f=20 a=10 3.490127 4.068518 8 530 +FAST f=20 a=10 0.025971 4.068518 8 530 +FAST f=21 a=1 7.343816 4.064945 8 770 +FAST f=21 a=1 0.085035 4.064945 8 770 +FAST f=21 a=2 5.930894 4.048206 8 386 +FAST f=21 a=2 0.067349 4.048206 8 386 +FAST f=21 a=3 6.770775 4.063417 8 578 +FAST f=21 a=3 0.077104 4.063417 8 578 +FAST f=21 a=4 6.889409 4.066761 8 626 +FAST f=21 a=4 0.0717 4.066761 8 626 +FAST f=21 a=5 6.714896 4.051813 8 914 +FAST f=21 a=5 0.071026 4.051813 8 914 +FAST f=21 a=6 6.539890 4.047263 8 1922 +FAST f=21 a=6 0.07127 4.047263 8 1922 +FAST f=21 a=7 6.511052 4.068373 8 482 +FAST f=21 a=7 0.065467 4.068373 8 482 +FAST f=21 a=8 6.458788 4.071597 8 482 +FAST f=21 a=8 0.063817 4.071597 8 482 +FAST f=21 a=9 6.377591 4.052905 8 434 +FAST f=21 a=9 0.063112 4.052905 8 434 +FAST f=21 a=10 6.360752 4.047773 8 530 +FAST f=21 a=10 0.063606 4.047773 8 530 +FAST f=22 a=1 10.523471 4.040812 8 962 +FAST f=22 a=1 0.14214 4.040812 8 962 +FAST f=22 a=2 9.454758 4.059396 8 914 +FAST f=22 a=2 0.118343 4.059396 8 914 +FAST f=22 a=3 9.043197 4.043019 8 1922 +FAST f=22 a=3 0.109798 4.043019 8 1922 +FAST f=22 a=4 8.716261 4.044819 8 770 +FAST f=22 a=4 0.099687 4.044819 8 770 +FAST f=22 a=5 8.529472 4.070576 8 530 +FAST f=22 a=5 0.093127 4.070576 8 530 +FAST f=22 a=6 8.424241 4.070565 8 722 +FAST f=22 a=6 0.093703 4.070565 8 722 +FAST f=22 a=7 8.403391 4.070591 8 578 +FAST f=22 a=7 0.089763 4.070591 8 578 +FAST f=22 a=8 8.285221 4.089171 8 530 +FAST f=22 a=8 0.087716 4.089171 8 530 +FAST f=22 a=9 8.282506 4.047470 8 722 +FAST f=22 a=9 0.089773 4.047470 8 722 +FAST f=22 a=10 8.241809 4.064151 8 818 +FAST f=22 a=10 0.090413 4.064151 8 818 +FAST f=23 a=1 12.389208 4.051635 6 530 +FAST f=23 a=1 0.147796 4.051635 6 530 +FAST f=23 a=2 11.300910 4.042835 6 914 +FAST f=23 a=2 0.133178 4.042835 6 914 +FAST f=23 a=3 10.879455 4.047415 8 626 +FAST f=23 a=3 0.129571 4.047415 8 626 +FAST f=23 a=4 10.522718 4.038269 6 914 +FAST f=23 a=4 0.118121 4.038269 6 914 +FAST f=23 a=5 10.348043 4.066884 8 434 +FAST f=23 a=5 0.112098 4.066884 8 434 +FAST f=23 a=6 10.238630 4.048635 8 1010 +FAST f=23 a=6 0.120281 4.048635 8 1010 +FAST f=23 a=7 10.213255 4.061809 8 530 +FAST f=23 a=7 0.1121 4.061809 8 530 +FAST f=23 a=8 10.107879 4.074104 8 818 +FAST f=23 a=8 0.116544 4.074104 8 818 +FAST f=23 a=9 10.063424 4.064811 8 674 +FAST f=23 a=9 0.109045 4.064811 8 674 +FAST f=23 a=10 10.035801 4.054918 8 530 +FAST f=23 a=10 0.108735 4.054918 8 530 +FAST f=24 a=1 14.963878 4.073490 8 722 +FAST f=24 a=1 0.206344 4.073490 8 722 +FAST f=24 a=2 13.833472 4.036100 8 962 +FAST f=24 a=2 0.17486 4.036100 8 962 +FAST f=24 a=3 13.404631 4.026281 6 1106 +FAST f=24 a=3 0.153961 4.026281 6 1106 +FAST f=24 a=4 13.041164 4.065448 8 674 +FAST f=24 a=4 0.155509 4.065448 8 674 +FAST f=24 a=5 12.879412 4.054636 8 674 +FAST f=24 a=5 0.148282 4.054636 8 674 +FAST f=24 a=6 12.773736 4.081376 8 530 +FAST f=24 a=6 0.142563 4.081376 8 530 +FAST f=24 a=7 12.711310 4.059834 8 770 +FAST f=24 a=7 0.149321 4.059834 8 770 +FAST f=24 a=8 12.635459 4.052050 8 1298 +FAST f=24 a=8 0.15095 4.052050 8 1298 +FAST f=24 a=9 12.558104 4.076516 8 722 +FAST f=24 a=9 0.144361 4.076516 8 722 +FAST f=24 a=10 10.661348 4.062137 8 818 +FAST f=24 a=10 0.108232 4.062137 8 818 + hg-changelog: -NODICT 0.000007 1.377613 -RANDOM 0.297345 2.097487 -LEGACY 2.633992 2.058907 -COVER 219.179786 2.189685 8 98 -COVER 6.620852 2.189685 8 98 -FAST15 47.635082 2.130794 6 386 -FAST15 0.321297 2.130794 6 386 -FAST16 43.837676 2.144845 8 194 -FAST16 0.312640 2.144845 8 194 -FAST17 49.349017 2.156099 8 242 -FAST17 0.348459 2.156099 8 242 -FAST18 51.153784 2.172439 6 98 -FAST18 0.353106 2.172439 6 98 -FAST19 52.627045 2.180321 6 98 -FAST19 0.390612 2.180321 6 98 -FAST20 63.748782 2.187431 6 98 -FAST20 0.489544 2.187431 6 98 -FAST21 68.709198 2.184185 6 146 -FAST21 0.530852 2.184185 6 146 -FAST22 68.491639 2.182830 6 98 -FAST22 0.645699 2.182830 6 98 -FAST23 72.558688 2.186399 8 98 -FAST23 0.593539 2.186399 8 98 -FAST24 76.137195 2.185608 6 98 -FAST24 0.680132 2.185608 6 98 +NODICT 0.000017 1.377590 +RANDOM 0.186171 2.097487 +LEGACY 1.670867 2.058907 +COVER 173.561948 2.189685 8 98 +COVER 4.811180 2.189685 8 98 +FAST f=15 a=1 18.685906 2.129682 8 434 +FAST f=15 a=1 0.173376 2.129682 8 434 +FAST f=15 a=2 12.928259 2.131890 8 482 +FAST f=15 a=2 0.102582 2.131890 8 482 +FAST f=15 a=3 11.132343 2.128027 8 386 +FAST f=15 a=3 0.077122 2.128027 8 386 +FAST f=15 a=4 10.120683 2.125797 8 434 +FAST f=15 a=4 0.065175 2.125797 8 434 +FAST f=15 a=5 9.479092 2.127697 8 386 +FAST f=15 a=5 0.057905 2.127697 8 386 +FAST f=15 a=6 9.159523 2.127132 8 1682 +FAST f=15 a=6 0.058604 2.127132 8 1682 +FAST f=15 a=7 8.724003 2.129914 8 434 +FAST f=15 a=7 0.0493 2.129914 8 434 +FAST f=15 a=8 8.595001 2.127137 8 338 +FAST f=15 a=8 0.0474 2.127137 8 338 +FAST f=15 a=9 8.356405 2.125512 8 482 +FAST f=15 a=9 0.046126 2.125512 8 482 +FAST f=15 a=10 8.207111 2.126066 8 338 +FAST f=15 a=10 0.043292 2.126066 8 338 +FAST f=16 a=1 18.464436 2.144040 8 242 +FAST f=16 a=1 0.172156 2.144040 8 242 +FAST f=16 a=2 12.844825 2.148171 8 194 +FAST f=16 a=2 0.099619 2.148171 8 194 +FAST f=16 a=3 11.082568 2.140837 8 290 +FAST f=16 a=3 0.079165 2.140837 8 290 +FAST f=16 a=4 10.066749 2.144405 8 386 +FAST f=16 a=4 0.068411 2.144405 8 386 +FAST f=16 a=5 9.501121 2.140720 8 386 +FAST f=16 a=5 0.061316 2.140720 8 386 +FAST f=16 a=6 9.179332 2.139478 8 386 +FAST f=16 a=6 0.056322 2.139478 8 386 +FAST f=16 a=7 8.849438 2.142412 8 194 +FAST f=16 a=7 0.050493 2.142412 8 194 +FAST f=16 a=8 8.810919 2.143454 8 434 +FAST f=16 a=8 0.051304 2.143454 8 434 +FAST f=16 a=9 8.553900 2.140339 8 194 +FAST f=16 a=9 0.047285 2.140339 8 194 +FAST f=16 a=10 8.398027 2.143130 8 386 +FAST f=16 a=10 0.046386 2.143130 8 386 +FAST f=17 a=1 18.644657 2.157192 8 98 +FAST f=17 a=1 0.173884 2.157192 8 98 +FAST f=17 a=2 13.071242 2.159830 8 146 +FAST f=17 a=2 0.10388 2.159830 8 146 +FAST f=17 a=3 11.332366 2.153654 6 194 +FAST f=17 a=3 0.08983 2.153654 6 194 +FAST f=17 a=4 10.362413 2.156813 8 242 +FAST f=17 a=4 0.070389 2.156813 8 242 +FAST f=17 a=5 9.808159 2.155098 6 338 +FAST f=17 a=5 0.072661 2.155098 6 338 +FAST f=17 a=6 9.451165 2.153845 6 146 +FAST f=17 a=6 0.064959 2.153845 6 146 +FAST f=17 a=7 9.163097 2.155424 6 242 +FAST f=17 a=7 0.064323 2.155424 6 242 +FAST f=17 a=8 9.047276 2.156640 8 242 +FAST f=17 a=8 0.053382 2.156640 8 242 +FAST f=17 a=9 8.807671 2.152396 8 146 +FAST f=17 a=9 0.049617 2.152396 8 146 +FAST f=17 a=10 8.649827 2.152370 8 146 +FAST f=17 a=10 0.047849 2.152370 8 146 +FAST f=18 a=1 18.809502 2.168116 8 98 +FAST f=18 a=1 0.175226 2.168116 8 98 +FAST f=18 a=2 13.756502 2.170870 6 242 +FAST f=18 a=2 0.119507 2.170870 6 242 +FAST f=18 a=3 12.059748 2.163094 6 98 +FAST f=18 a=3 0.093912 2.163094 6 98 +FAST f=18 a=4 11.410294 2.172372 8 98 +FAST f=18 a=4 0.073048 2.172372 8 98 +FAST f=18 a=5 10.560297 2.166388 8 98 +FAST f=18 a=5 0.065136 2.166388 8 98 +FAST f=18 a=6 10.071390 2.162672 8 98 +FAST f=18 a=6 0.059402 2.162672 8 98 +FAST f=18 a=7 10.084214 2.166624 6 194 +FAST f=18 a=7 0.073276 2.166624 6 194 +FAST f=18 a=8 9.953226 2.167454 8 98 +FAST f=18 a=8 0.053659 2.167454 8 98 +FAST f=18 a=9 8.982461 2.161593 6 146 +FAST f=18 a=9 0.05955 2.161593 6 146 +FAST f=18 a=10 8.986092 2.164373 6 242 +FAST f=18 a=10 0.059135 2.164373 6 242 +FAST f=19 a=1 18.908277 2.176021 8 98 +FAST f=19 a=1 0.177316 2.176021 8 98 +FAST f=19 a=2 13.471313 2.176103 8 98 +FAST f=19 a=2 0.106344 2.176103 8 98 +FAST f=19 a=3 11.571406 2.172812 8 98 +FAST f=19 a=3 0.083293 2.172812 8 98 +FAST f=19 a=4 10.632775 2.177770 6 146 +FAST f=19 a=4 0.079864 2.177770 6 146 +FAST f=19 a=5 10.030190 2.175574 6 146 +FAST f=19 a=5 0.07223 2.175574 6 146 +FAST f=19 a=6 9.717818 2.169997 8 98 +FAST f=19 a=6 0.060049 2.169997 8 98 +FAST f=19 a=7 9.397531 2.172770 8 146 +FAST f=19 a=7 0.057188 2.172770 8 146 +FAST f=19 a=8 9.281061 2.175822 8 98 +FAST f=19 a=8 0.053711 2.175822 8 98 +FAST f=19 a=9 9.165242 2.169849 6 146 +FAST f=19 a=9 0.059898 2.169849 6 146 +FAST f=19 a=10 9.048763 2.173394 8 98 +FAST f=19 a=10 0.049757 2.173394 8 98 +FAST f=20 a=1 21.166917 2.183923 6 98 +FAST f=20 a=1 0.205425 2.183923 6 98 +FAST f=20 a=2 15.642753 2.182349 6 98 +FAST f=20 a=2 0.135957 2.182349 6 98 +FAST f=20 a=3 14.053730 2.173544 6 98 +FAST f=20 a=3 0.11266 2.173544 6 98 +FAST f=20 a=4 15.270019 2.183656 8 98 +FAST f=20 a=4 0.107892 2.183656 8 98 +FAST f=20 a=5 15.497927 2.174661 6 98 +FAST f=20 a=5 0.100305 2.174661 6 98 +FAST f=20 a=6 13.973505 2.172391 8 98 +FAST f=20 a=6 0.087565 2.172391 8 98 +FAST f=20 a=7 14.083296 2.172443 8 98 +FAST f=20 a=7 0.078062 2.172443 8 98 +FAST f=20 a=8 12.560048 2.175581 8 98 +FAST f=20 a=8 0.070282 2.175581 8 98 +FAST f=20 a=9 13.078645 2.173975 6 146 +FAST f=20 a=9 0.081041 2.173975 6 146 +FAST f=20 a=10 12.823328 2.177778 8 98 +FAST f=20 a=10 0.074522 2.177778 8 98 +FAST f=21 a=1 29.825370 2.183057 6 98 +FAST f=21 a=1 0.334453 2.183057 6 98 +FAST f=21 a=2 29.476474 2.182752 8 98 +FAST f=21 a=2 0.286602 2.182752 8 98 +FAST f=21 a=3 25.937186 2.175867 8 98 +FAST f=21 a=3 0.17626 2.175867 8 98 +FAST f=21 a=4 20.413865 2.179780 8 98 +FAST f=21 a=4 0.206085 2.179780 8 98 +FAST f=21 a=5 20.541889 2.178328 6 146 +FAST f=21 a=5 0.199157 2.178328 6 146 +FAST f=21 a=6 21.090670 2.174443 6 146 +FAST f=21 a=6 0.190645 2.174443 6 146 +FAST f=21 a=7 20.221569 2.177384 6 146 +FAST f=21 a=7 0.184278 2.177384 6 146 +FAST f=21 a=8 20.322357 2.179456 6 98 +FAST f=21 a=8 0.178458 2.179456 6 98 +FAST f=21 a=9 20.683912 2.174396 6 146 +FAST f=21 a=9 0.190829 2.174396 6 146 +FAST f=21 a=10 20.840865 2.174905 8 98 +FAST f=21 a=10 0.172515 2.174905 8 98 +FAST f=22 a=1 36.822827 2.181612 6 98 +FAST f=22 a=1 0.437389 2.181612 6 98 +FAST f=22 a=2 30.616902 2.183142 8 98 +FAST f=22 a=2 0.324284 2.183142 8 98 +FAST f=22 a=3 28.472482 2.178130 8 98 +FAST f=22 a=3 0.236538 2.178130 8 98 +FAST f=22 a=4 25.847028 2.181878 8 98 +FAST f=22 a=4 0.263744 2.181878 8 98 +FAST f=22 a=5 27.095881 2.180775 8 98 +FAST f=22 a=5 0.24988 2.180775 8 98 +FAST f=22 a=6 25.939172 2.170916 8 98 +FAST f=22 a=6 0.240033 2.170916 8 98 +FAST f=22 a=7 27.064194 2.177849 8 98 +FAST f=22 a=7 0.242383 2.177849 8 98 +FAST f=22 a=8 25.140221 2.178216 8 98 +FAST f=22 a=8 0.237601 2.178216 8 98 +FAST f=22 a=9 25.505283 2.177455 6 146 +FAST f=22 a=9 0.223217 2.177455 6 146 +FAST f=22 a=10 24.529362 2.176705 6 98 +FAST f=22 a=10 0.222876 2.176705 6 98 +FAST f=23 a=1 39.127310 2.183006 6 98 +FAST f=23 a=1 0.417338 2.183006 6 98 +FAST f=23 a=2 32.468161 2.183524 6 98 +FAST f=23 a=2 0.351645 2.183524 6 98 +FAST f=23 a=3 31.577620 2.172604 6 98 +FAST f=23 a=3 0.319659 2.172604 6 98 +FAST f=23 a=4 30.129247 2.183932 6 98 +FAST f=23 a=4 0.307239 2.183932 6 98 +FAST f=23 a=5 29.103376 2.183529 6 146 +FAST f=23 a=5 0.285533 2.183529 6 146 +FAST f=23 a=6 29.776045 2.174367 8 98 +FAST f=23 a=6 0.276846 2.174367 8 98 +FAST f=23 a=7 28.940407 2.178022 6 146 +FAST f=23 a=7 0.274082 2.178022 6 146 +FAST f=23 a=8 29.256009 2.179462 6 98 +FAST f=23 a=8 0.26949 2.179462 6 98 +FAST f=23 a=9 29.347312 2.170407 8 98 +FAST f=23 a=9 0.265034 2.170407 8 98 +FAST f=23 a=10 29.140081 2.171762 8 98 +FAST f=23 a=10 0.259183 2.171762 8 98 +FAST f=24 a=1 44.871179 2.182115 6 98 +FAST f=24 a=1 0.509433 2.182115 6 98 +FAST f=24 a=2 38.694867 2.180549 8 98 +FAST f=24 a=2 0.406695 2.180549 8 98 +FAST f=24 a=3 38.363769 2.172821 8 98 +FAST f=24 a=3 0.359581 2.172821 8 98 +FAST f=24 a=4 36.580797 2.184142 8 98 +FAST f=24 a=4 0.340614 2.184142 8 98 +FAST f=24 a=5 33.125701 2.183301 8 98 +FAST f=24 a=5 0.324874 2.183301 8 98 +FAST f=24 a=6 34.776068 2.173019 6 146 +FAST f=24 a=6 0.340397 2.173019 6 146 +FAST f=24 a=7 34.417625 2.176561 6 146 +FAST f=24 a=7 0.308223 2.176561 6 146 +FAST f=24 a=8 35.470291 2.182161 6 98 +FAST f=24 a=8 0.307724 2.182161 6 98 +FAST f=24 a=9 34.927252 2.172682 6 146 +FAST f=24 a=9 0.300598 2.172682 6 146 +FAST f=24 a=10 33.238355 2.173395 6 98 +FAST f=24 a=10 0.249916 2.173395 6 98 + hg-manifest: -NODICT 0.000026 1.866385 -RANDOM 0.784554 2.309436 -LEGACY 10.193714 2.506977 -COVER 988.206583 2.582528 8 434 -COVER 39.726199 2.582528 8 434 -FAST15 168.388819 2.392920 6 1826 -FAST15 1.272178 2.392920 6 1826 -FAST16 161.822607 2.480762 6 1922 -FAST16 1.164908 2.480762 6 1922 -FAST17 157.688544 2.548285 6 1682 -FAST17 1.222439 2.548285 6 1682 -FAST18 154.529585 2.567634 6 386 -FAST18 1.217596 2.567634 6 386 -FAST19 160.244979 2.581653 8 338 -FAST19 1.282450 2.581653 8 338 -FAST20 191.503297 2.586881 8 194 -FAST20 2.009748 2.586881 8 194 -FAST21 226.389709 2.590051 6 242 -FAST21 2.494543 2.590051 6 242 -FAST22 217.859055 2.591376 6 194 -FAST22 2.295693 2.591376 6 194 -FAST23 236.819791 2.591131 8 434 -FAST23 2.744711 2.591131 8 434 -FAST24 269.187800 2.591548 6 290 -FAST24 2.923671 2.591548 6 290 +NODICT 0.000004 1.866377 +RANDOM 0.696346 2.309436 +LEGACY 7.064527 2.506977 +COVER 876.312865 2.582528 8 434 +COVER 35.684533 2.582528 8 434 +FAST f=15 a=1 76.618201 2.404013 8 1202 +FAST f=15 a=1 0.700722 2.404013 8 1202 +FAST f=15 a=2 49.213058 2.409248 6 1826 +FAST f=15 a=2 0.473393 2.409248 6 1826 +FAST f=15 a=3 41.753197 2.409677 8 1490 +FAST f=15 a=3 0.336848 2.409677 8 1490 +FAST f=15 a=4 38.648295 2.407996 8 1538 +FAST f=15 a=4 0.283952 2.407996 8 1538 +FAST f=15 a=5 36.144936 2.402895 8 1874 +FAST f=15 a=5 0.270128 2.402895 8 1874 +FAST f=15 a=6 35.484675 2.394873 8 1586 +FAST f=15 a=6 0.251637 2.394873 8 1586 +FAST f=15 a=7 34.280599 2.397311 8 1778 +FAST f=15 a=7 0.23984 2.397311 8 1778 +FAST f=15 a=8 32.122572 2.396089 6 1490 +FAST f=15 a=8 0.251508 2.396089 6 1490 +FAST f=15 a=9 29.909842 2.390092 6 1970 +FAST f=15 a=9 0.251233 2.390092 6 1970 +FAST f=15 a=10 30.102938 2.400086 6 1682 +FAST f=15 a=10 0.23688 2.400086 6 1682 +FAST f=16 a=1 67.750401 2.475460 6 1346 +FAST f=16 a=1 0.796035 2.475460 6 1346 +FAST f=16 a=2 52.812027 2.480860 6 1730 +FAST f=16 a=2 0.480384 2.480860 6 1730 +FAST f=16 a=3 44.179259 2.469304 8 1970 +FAST f=16 a=3 0.332657 2.469304 8 1970 +FAST f=16 a=4 37.612728 2.478208 6 1970 +FAST f=16 a=4 0.32498 2.478208 6 1970 +FAST f=16 a=5 35.056222 2.475568 6 1298 +FAST f=16 a=5 0.302824 2.475568 6 1298 +FAST f=16 a=6 34.713012 2.486079 8 1730 +FAST f=16 a=6 0.24755 2.486079 8 1730 +FAST f=16 a=7 33.713687 2.477180 6 1682 +FAST f=16 a=7 0.280358 2.477180 6 1682 +FAST f=16 a=8 31.571412 2.475418 8 1538 +FAST f=16 a=8 0.241241 2.475418 8 1538 +FAST f=16 a=9 31.608069 2.478263 8 1922 +FAST f=16 a=9 0.241764 2.478263 8 1922 +FAST f=16 a=10 31.358002 2.472263 8 1442 +FAST f=16 a=10 0.221661 2.472263 8 1442 +FAST f=17 a=1 66.185775 2.536085 6 1346 +FAST f=17 a=1 0.713549 2.536085 6 1346 +FAST f=17 a=2 50.365000 2.546105 8 1298 +FAST f=17 a=2 0.467846 2.546105 8 1298 +FAST f=17 a=3 42.712843 2.536250 8 1298 +FAST f=17 a=3 0.34047 2.536250 8 1298 +FAST f=17 a=4 39.514227 2.535555 8 1442 +FAST f=17 a=4 0.302989 2.535555 8 1442 +FAST f=17 a=5 35.189292 2.524925 8 1202 +FAST f=17 a=5 0.273451 2.524925 8 1202 +FAST f=17 a=6 35.791683 2.523466 8 1202 +FAST f=17 a=6 0.268261 2.523466 8 1202 +FAST f=17 a=7 37.416136 2.526625 6 1010 +FAST f=17 a=7 0.277558 2.526625 6 1010 +FAST f=17 a=8 37.084707 2.533274 6 1250 +FAST f=17 a=8 0.285104 2.533274 6 1250 +FAST f=17 a=9 34.183814 2.532765 8 1298 +FAST f=17 a=9 0.235133 2.532765 8 1298 +FAST f=17 a=10 31.149235 2.528722 8 1346 +FAST f=17 a=10 0.232679 2.528722 8 1346 +FAST f=18 a=1 72.942176 2.559857 6 386 +FAST f=18 a=1 0.718618 2.559857 6 386 +FAST f=18 a=2 51.690440 2.559572 8 290 +FAST f=18 a=2 0.403978 2.559572 8 290 +FAST f=18 a=3 45.344908 2.561040 8 962 +FAST f=18 a=3 0.357205 2.561040 8 962 +FAST f=18 a=4 39.804522 2.558446 8 1010 +FAST f=18 a=4 0.310526 2.558446 8 1010 +FAST f=18 a=5 38.134888 2.561811 8 626 +FAST f=18 a=5 0.273743 2.561811 8 626 +FAST f=18 a=6 35.091890 2.555518 8 722 +FAST f=18 a=6 0.260135 2.555518 8 722 +FAST f=18 a=7 34.639523 2.562938 8 290 +FAST f=18 a=7 0.234294 2.562938 8 290 +FAST f=18 a=8 36.076431 2.563567 8 1586 +FAST f=18 a=8 0.274075 2.563567 8 1586 +FAST f=18 a=9 36.376433 2.560950 8 722 +FAST f=18 a=9 0.240106 2.560950 8 722 +FAST f=18 a=10 32.624790 2.559340 8 578 +FAST f=18 a=10 0.234704 2.559340 8 578 +FAST f=19 a=1 70.513761 2.572441 8 194 +FAST f=19 a=1 0.726112 2.572441 8 194 +FAST f=19 a=2 59.263032 2.574560 8 482 +FAST f=19 a=2 0.451554 2.574560 8 482 +FAST f=19 a=3 51.509594 2.571546 6 194 +FAST f=19 a=3 0.393014 2.571546 6 194 +FAST f=19 a=4 55.393906 2.573386 8 482 +FAST f=19 a=4 0.38819 2.573386 8 482 +FAST f=19 a=5 43.201736 2.567589 8 674 +FAST f=19 a=5 0.292155 2.567589 8 674 +FAST f=19 a=6 42.911687 2.572666 6 434 +FAST f=19 a=6 0.303988 2.572666 6 434 +FAST f=19 a=7 44.687591 2.573613 6 290 +FAST f=19 a=7 0.308721 2.573613 6 290 +FAST f=19 a=8 37.372868 2.571039 6 194 +FAST f=19 a=8 0.287137 2.571039 6 194 +FAST f=19 a=9 36.074230 2.566473 6 482 +FAST f=19 a=9 0.280721 2.566473 6 482 +FAST f=19 a=10 33.731720 2.570306 8 194 +FAST f=19 a=10 0.224073 2.570306 8 194 +FAST f=20 a=1 79.670634 2.581146 6 290 +FAST f=20 a=1 0.899986 2.581146 6 290 +FAST f=20 a=2 58.827141 2.579782 8 386 +FAST f=20 a=2 0.602288 2.579782 8 386 +FAST f=20 a=3 51.289004 2.579627 8 722 +FAST f=20 a=3 0.446091 2.579627 8 722 +FAST f=20 a=4 47.711068 2.581508 8 722 +FAST f=20 a=4 0.473007 2.581508 8 722 +FAST f=20 a=5 47.402929 2.578062 6 434 +FAST f=20 a=5 0.497131 2.578062 6 434 +FAST f=20 a=6 54.797102 2.577365 8 482 +FAST f=20 a=6 0.515061 2.577365 8 482 +FAST f=20 a=7 51.370877 2.583050 8 386 +FAST f=20 a=7 0.402878 2.583050 8 386 +FAST f=20 a=8 51.437931 2.574875 6 242 +FAST f=20 a=8 0.453094 2.574875 6 242 +FAST f=20 a=9 44.105456 2.576700 6 242 +FAST f=20 a=9 0.456633 2.576700 6 242 +FAST f=20 a=10 44.447580 2.578305 8 338 +FAST f=20 a=10 0.409121 2.578305 8 338 +FAST f=21 a=1 113.031686 2.582449 6 242 +FAST f=21 a=1 1.456971 2.582449 6 242 +FAST f=21 a=2 97.700932 2.582124 8 194 +FAST f=21 a=2 1.072078 2.582124 8 194 +FAST f=21 a=3 96.563648 2.585479 8 434 +FAST f=21 a=3 0.949528 2.585479 8 434 +FAST f=21 a=4 90.597813 2.582366 6 386 +FAST f=21 a=4 0.76944 2.582366 6 386 +FAST f=21 a=5 86.815980 2.579043 8 434 +FAST f=21 a=5 0.858167 2.579043 8 434 +FAST f=21 a=6 91.235820 2.578378 8 530 +FAST f=21 a=6 0.684274 2.578378 8 530 +FAST f=21 a=7 84.392788 2.581243 8 386 +FAST f=21 a=7 0.814386 2.581243 8 386 +FAST f=21 a=8 82.052310 2.582547 8 338 +FAST f=21 a=8 0.822633 2.582547 8 338 +FAST f=21 a=9 74.696074 2.579319 8 194 +FAST f=21 a=9 0.811028 2.579319 8 194 +FAST f=21 a=10 76.211170 2.578766 8 290 +FAST f=21 a=10 0.809715 2.578766 8 290 +FAST f=22 a=1 138.976871 2.580478 8 194 +FAST f=22 a=1 1.748932 2.580478 8 194 +FAST f=22 a=2 120.164097 2.583633 8 386 +FAST f=22 a=2 1.333239 2.583633 8 386 +FAST f=22 a=3 111.986474 2.582566 6 194 +FAST f=22 a=3 1.305734 2.582566 6 194 +FAST f=22 a=4 108.548148 2.583068 6 194 +FAST f=22 a=4 1.314026 2.583068 6 194 +FAST f=22 a=5 103.173017 2.583495 6 290 +FAST f=22 a=5 1.228664 2.583495 6 290 +FAST f=22 a=6 108.421262 2.582349 8 530 +FAST f=22 a=6 1.076773 2.582349 8 530 +FAST f=22 a=7 103.284127 2.581022 8 386 +FAST f=22 a=7 1.112117 2.581022 8 386 +FAST f=22 a=8 96.330279 2.581073 8 290 +FAST f=22 a=8 1.109303 2.581073 8 290 +FAST f=22 a=9 97.651348 2.580075 6 194 +FAST f=22 a=9 0.933032 2.580075 6 194 +FAST f=22 a=10 101.660621 2.584886 8 194 +FAST f=22 a=10 0.796823 2.584886 8 194 +FAST f=23 a=1 159.322978 2.581474 6 242 +FAST f=23 a=1 2.015878 2.581474 6 242 +FAST f=23 a=2 134.331775 2.581619 8 194 +FAST f=23 a=2 1.545845 2.581619 8 194 +FAST f=23 a=3 127.724552 2.579888 6 338 +FAST f=23 a=3 1.444496 2.579888 6 338 +FAST f=23 a=4 126.077675 2.578137 6 242 +FAST f=23 a=4 1.364394 2.578137 6 242 +FAST f=23 a=5 124.914027 2.580843 8 338 +FAST f=23 a=5 1.116059 2.580843 8 338 +FAST f=23 a=6 122.874153 2.577637 6 338 +FAST f=23 a=6 1.164584 2.577637 6 338 +FAST f=23 a=7 123.099257 2.582715 6 386 +FAST f=23 a=7 1.354042 2.582715 6 386 +FAST f=23 a=8 122.026753 2.577681 8 194 +FAST f=23 a=8 1.210966 2.577681 8 194 +FAST f=23 a=9 121.164312 2.584599 6 290 +FAST f=23 a=9 1.174859 2.584599 6 290 +FAST f=23 a=10 117.462222 2.580358 8 194 +FAST f=23 a=10 1.075258 2.580358 8 194 +FAST f=24 a=1 169.539659 2.581642 6 194 +FAST f=24 a=1 1.916804 2.581642 6 194 +FAST f=24 a=2 160.539270 2.580421 6 290 +FAST f=24 a=2 1.71087 2.580421 6 290 +FAST f=24 a=3 155.455874 2.580449 6 242 +FAST f=24 a=3 1.60307 2.580449 6 242 +FAST f=24 a=4 147.630320 2.582953 6 338 +FAST f=24 a=4 1.396364 2.582953 6 338 +FAST f=24 a=5 133.767428 2.580589 6 290 +FAST f=24 a=5 1.19933 2.580589 6 290 +FAST f=24 a=6 146.437535 2.579453 8 194 +FAST f=24 a=6 1.385405 2.579453 8 194 +FAST f=24 a=7 147.227507 2.584155 8 386 +FAST f=24 a=7 1.48942 2.584155 8 386 +FAST f=24 a=8 138.005773 2.584115 8 194 +FAST f=24 a=8 1.352 2.584115 8 194 +FAST f=24 a=9 141.442625 2.582902 8 290 +FAST f=24 a=9 1.39647 2.582902 8 290 +FAST f=24 a=10 142.157446 2.582701 8 434 +FAST f=24 a=10 1.498889 2.582701 8 434 diff --git a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c index d92e8d5cb..b19345692 100644 --- a/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c +++ b/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c @@ -5,7 +5,6 @@ #include #include #include "random.h" -#include "fastCover.h" #include "dictBuilder.h" #include "zstd_internal.h" /* includes zstd.h */ #include "io.h" @@ -149,7 +148,7 @@ double compressWithDict(sampleInfo *srcInfo, dictInfo* dInfo, int compressionLev /* Allocate dst with enough space to compress the maximum sized sample */ { size_t maxSampleSize = 0; - for (int i = 0; i < srcInfo->nbSamples; i++) { + for (i = 0; i < srcInfo->nbSamples; i++) { maxSampleSize = MAX(srcInfo->samplesSizes[i], maxSampleSize); } dstCapacity = ZSTD_compressBound(maxSampleSize); @@ -291,6 +290,9 @@ int main(int argCount, const char* argv[]) /* Initialize arguments to default values */ unsigned k = 200; unsigned d = 8; + unsigned f; + unsigned accel; + unsigned i; const unsigned cLevel = DEFAULT_CLEVEL; const unsigned dictID = 0; const unsigned maxDictSize = g_defaultMaxDictSize; @@ -305,7 +307,7 @@ int main(int argCount, const char* argv[]) const char** extendedFileList = NULL; /* Parse arguments */ - for (int i = 1; i < argCount; i++) { + for (i = 1; i < argCount; i++) { const char* argument = argv[i]; if (longCommandWArg(&argument, "in=")) { filenameTable[filenameIdx] = argument; @@ -375,6 +377,7 @@ int main(int argCount, const char* argv[]) /* for cover */ { + /* for cover (optimizing k and d) */ ZDICT_cover_params_t coverParam; memset(&coverParam, 0, sizeof(coverParam)); coverParam.zParams = zParams; @@ -388,6 +391,7 @@ int main(int argCount, const char* argv[]) goto _cleanup; } + /* for cover (with k and d provided) */ const int coverResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, &coverParam, NULL, NULL); DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParam.k, coverParam.d, coverParam.steps, (unsigned)(coverParam.splitPoint * 100)); if(coverResult) { @@ -398,29 +402,34 @@ int main(int argCount, const char* argv[]) } /* for fastCover */ - for (unsigned f = 15; f < 25; f++){ + for (f = 15; f < 25; f++){ DISPLAYLEVEL(2, "current f is %u\n", f); - /* for fastCover (optimizing k and d) */ - ZDICT_fastCover_params_t fastParam; - memset(&fastParam, 0, sizeof(fastParam)); - fastParam.zParams = zParams; - fastParam.splitPoint = 1.0; - fastParam.f = f; - fastParam.steps = 40; - fastParam.nbThreads = 1; - const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); - DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); - if(fastOptResult) { - result = 1; - goto _cleanup; - } + for (accel = 1; accel < 11; accel++) { + DISPLAYLEVEL(2, "current accel is %u\n", accel); + /* for fastCover (optimizing k and d) */ + ZDICT_fastCover_params_t fastParam; + memset(&fastParam, 0, sizeof(fastParam)); + fastParam.zParams = zParams; + fastParam.f = f; + fastParam.steps = 40; + fastParam.nbThreads = 1; + fastParam.accel = accel; + const int fastOptResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100), fastParam.accel); + if(fastOptResult) { + result = 1; + goto _cleanup; + } - /* for fastCover (with k and d provided) */ - const int fastResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); - DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100)); - if(fastResult) { - result = 1; - goto _cleanup; + /* for fastCover (with k and d provided) */ + for (i = 0; i < 5; i++) { + const int fastResult = benchmarkDictBuilder(srcInfo, maxDictSize, NULL, NULL, NULL, &fastParam); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastParam.k, fastParam.d, fastParam.f, fastParam.steps, (unsigned)(fastParam.splitPoint * 100), fastParam.accel); + if(fastResult) { + result = 1; + goto _cleanup; + } + } } } diff --git a/contrib/experimental_dict_builders/fastCover/fastCover.c b/contrib/experimental_dict_builders/fastCover/fastCover.c index 84d841b10..02c155a81 100644 --- a/contrib/experimental_dict_builders/fastCover/fastCover.c +++ b/contrib/experimental_dict_builders/fastCover/fastCover.c @@ -197,7 +197,7 @@ static FASTCOVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, bestSegment.end = newEnd; } { - /* Half the frequency of hash value of each dmer covered by the chosen segment. */ + /* Zero the frequency of hash value of each dmer covered by the chosen segment. */ U32 pos; for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { const size_t i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, parameters.f, ctx->d); @@ -300,7 +300,7 @@ static int FASTCOVER_ctx_init(FASTCOVER_ctx_t *ctx, const void *samplesBuffer, if (totalSamplesSize < MAX(d, sizeof(U64)) || totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) { DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n", - (U32)(totalSamplesSize>>20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20)); + (U32)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20)); return 0; } /* Check if there are at least 5 training samples */ diff --git a/lib/BUCK b/lib/BUCK index dbe8885fc..bd93b082a 100644 --- a/lib/BUCK +++ b/lib/BUCK @@ -69,6 +69,7 @@ cxx_library( ]), headers=subdir_glob([ ('dictBuilder', 'divsufsort.h'), + ('dictBuilder', 'cover.h'), ]), srcs=glob(['dictBuilder/*.c']), deps=[':common'], diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index e32991652..74b70ef82 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -29,6 +29,7 @@ #include "mem.h" /* read */ #include "pool.h" #include "threading.h" +#include "cover.h" #include "zstd_internal.h" /* includes zstd.h */ #ifndef ZDICT_STATIC_LINKING_ONLY #define ZDICT_STATIC_LINKING_ONLY @@ -185,7 +186,7 @@ static void COVER_map_remove(COVER_map_t *map, U32 key) { } /** - * Destroyes a map that is inited with COVER_map_init(). + * Destroys a map that is inited with COVER_map_init(). */ static void COVER_map_destroy(COVER_map_t *map) { if (map->data) { @@ -223,7 +224,7 @@ static COVER_ctx_t *g_ctx = NULL; /** * Returns the sum of the sample sizes. */ -static size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) { +size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) { size_t sum = 0; unsigned i; for (i = 0; i < nbSamples; ++i) { @@ -380,14 +381,6 @@ static void COVER_group(COVER_ctx_t *ctx, const void *group, ctx->suffix[dmerId] = freq; } -/** - * A segment is a range in the source as well as the score of the segment. - */ -typedef struct { - U32 begin; - U32 end; - U32 score; -} COVER_segment_t; /** * Selects the best segment in an epoch. @@ -691,7 +684,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( BYTE* const dict = (BYTE*)dictBuffer; COVER_ctx_t ctx; COVER_map_t activeDmers; - parameters.splitPoint = 1.0; + parameters.splitPoint = parameters.splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters.splitPoint; /* Initialize global data */ g_displayLevel = parameters.zParams.notificationLevel; /* Checks */ @@ -737,28 +730,65 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( } } -/** - * COVER_best_t is used for two purposes: - * 1. Synchronizing threads. - * 2. Saving the best parameters and dictionary. - * - * All of the methods except COVER_best_init() are thread safe if zstd is - * compiled with multithreaded support. - */ -typedef struct COVER_best_s { - ZSTD_pthread_mutex_t mutex; - ZSTD_pthread_cond_t cond; - size_t liveJobs; - void *dict; - size_t dictSize; - ZDICT_cover_params_t parameters; - size_t compressedSize; -} COVER_best_t; + + +size_t COVER_checkTotalCompressedSize(const ZDICT_cover_params_t parameters, + const size_t *samplesSizes, const BYTE *samples, + size_t *offsets, + size_t nbTrainSamples, size_t nbSamples, + BYTE *const dict, size_t dictBufferCapacity) { + size_t totalCompressedSize = ERROR(GENERIC); + /* Pointers */ + ZSTD_CCtx *cctx; + ZSTD_CDict *cdict; + void *dst; + /* Local variables */ + size_t dstCapacity; + size_t i; + /* Allocate dst with enough space to compress the maximum sized sample */ + { + size_t maxSampleSize = 0; + i = parameters.splitPoint < 1.0 ? nbTrainSamples : 0; + for (; i < nbSamples; ++i) { + maxSampleSize = MAX(samplesSizes[i], maxSampleSize); + } + dstCapacity = ZSTD_compressBound(maxSampleSize); + dst = malloc(dstCapacity); + } + /* Create the cctx and cdict */ + cctx = ZSTD_createCCtx(); + cdict = ZSTD_createCDict(dict, dictBufferCapacity, + parameters.zParams.compressionLevel); + if (!dst || !cctx || !cdict) { + goto _compressCleanup; + } + /* Compress each sample and sum their sizes (or error) */ + totalCompressedSize = dictBufferCapacity; + i = parameters.splitPoint < 1.0 ? nbTrainSamples : 0; + for (; i < nbSamples; ++i) { + const size_t size = ZSTD_compress_usingCDict( + cctx, dst, dstCapacity, samples + offsets[i], + samplesSizes[i], cdict); + if (ZSTD_isError(size)) { + totalCompressedSize = ERROR(GENERIC); + goto _compressCleanup; + } + totalCompressedSize += size; + } +_compressCleanup: + ZSTD_freeCCtx(cctx); + ZSTD_freeCDict(cdict); + if (dst) { + free(dst); + } + return totalCompressedSize; +} + /** * Initialize the `COVER_best_t`. */ -static void COVER_best_init(COVER_best_t *best) { +void COVER_best_init(COVER_best_t *best) { if (best==NULL) return; /* compatible with init on NULL */ (void)ZSTD_pthread_mutex_init(&best->mutex, NULL); (void)ZSTD_pthread_cond_init(&best->cond, NULL); @@ -772,7 +802,7 @@ static void COVER_best_init(COVER_best_t *best) { /** * Wait until liveJobs == 0. */ -static void COVER_best_wait(COVER_best_t *best) { +void COVER_best_wait(COVER_best_t *best) { if (!best) { return; } @@ -786,7 +816,7 @@ static void COVER_best_wait(COVER_best_t *best) { /** * Call COVER_best_wait() and then destroy the COVER_best_t. */ -static void COVER_best_destroy(COVER_best_t *best) { +void COVER_best_destroy(COVER_best_t *best) { if (!best) { return; } @@ -802,7 +832,7 @@ static void COVER_best_destroy(COVER_best_t *best) { * Called when a thread is about to be launched. * Increments liveJobs. */ -static void COVER_best_start(COVER_best_t *best) { +void COVER_best_start(COVER_best_t *best) { if (!best) { return; } @@ -816,7 +846,7 @@ static void COVER_best_start(COVER_best_t *best) { * Decrements liveJobs and signals any waiting threads if liveJobs == 0. * If this dictionary is the best so far save it and its parameters. */ -static void COVER_best_finish(COVER_best_t *best, size_t compressedSize, +void COVER_best_finish(COVER_best_t *best, size_t compressedSize, ZDICT_cover_params_t parameters, void *dict, size_t dictSize) { if (!best) { @@ -847,10 +877,10 @@ static void COVER_best_finish(COVER_best_t *best, size_t compressedSize, best->parameters = parameters; best->compressedSize = compressedSize; } - ZSTD_pthread_mutex_unlock(&best->mutex); if (liveJobs == 0) { ZSTD_pthread_cond_broadcast(&best->cond); } + ZSTD_pthread_mutex_unlock(&best->mutex); } } @@ -904,51 +934,10 @@ static void COVER_tryParameters(void *opaque) { } } /* Check total compressed size */ - { - /* Pointers */ - ZSTD_CCtx *cctx; - ZSTD_CDict *cdict; - void *dst; - /* Local variables */ - size_t dstCapacity; - size_t i; - /* Allocate dst with enough space to compress the maximum sized sample */ - { - size_t maxSampleSize = 0; - i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0; - for (; i < ctx->nbSamples; ++i) { - maxSampleSize = MAX(ctx->samplesSizes[i], maxSampleSize); - } - dstCapacity = ZSTD_compressBound(maxSampleSize); - dst = malloc(dstCapacity); - } - /* Create the cctx and cdict */ - cctx = ZSTD_createCCtx(); - cdict = ZSTD_createCDict(dict, dictBufferCapacity, - parameters.zParams.compressionLevel); - if (!dst || !cctx || !cdict) { - goto _compressCleanup; - } - /* Compress each sample and sum their sizes (or error) */ - totalCompressedSize = dictBufferCapacity; - i = parameters.splitPoint < 1.0 ? ctx->nbTrainSamples : 0; - for (; i < ctx->nbSamples; ++i) { - const size_t size = ZSTD_compress_usingCDict( - cctx, dst, dstCapacity, ctx->samples + ctx->offsets[i], - ctx->samplesSizes[i], cdict); - if (ZSTD_isError(size)) { - totalCompressedSize = ERROR(GENERIC); - goto _compressCleanup; - } - totalCompressedSize += size; - } - _compressCleanup: - ZSTD_freeCCtx(cctx); - ZSTD_freeCDict(cdict); - if (dst) { - free(dst); - } - } + totalCompressedSize = COVER_checkTotalCompressedSize(parameters, ctx->samplesSizes, + ctx->samples, ctx->offsets, + ctx->nbTrainSamples, ctx->nbSamples, + dict, dictBufferCapacity); _cleanup: COVER_best_finish(data->best, totalCompressedSize, parameters, dict, diff --git a/lib/dictBuilder/cover.h b/lib/dictBuilder/cover.h new file mode 100644 index 000000000..82e2e1cea --- /dev/null +++ b/lib/dictBuilder/cover.h @@ -0,0 +1,83 @@ +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* memset */ +#include /* clock */ +#include "mem.h" /* read */ +#include "pool.h" +#include "threading.h" +#include "zstd_internal.h" /* includes zstd.h */ +#ifndef ZDICT_STATIC_LINKING_ONLY +#define ZDICT_STATIC_LINKING_ONLY +#endif +#include "zdict.h" + +/** + * COVER_best_t is used for two purposes: + * 1. Synchronizing threads. + * 2. Saving the best parameters and dictionary. + * + * All of the methods except COVER_best_init() are thread safe if zstd is + * compiled with multithreaded support. + */ +typedef struct COVER_best_s { + ZSTD_pthread_mutex_t mutex; + ZSTD_pthread_cond_t cond; + size_t liveJobs; + void *dict; + size_t dictSize; + ZDICT_cover_params_t parameters; + size_t compressedSize; +} COVER_best_t; + +/** + * A segment is a range in the source as well as the score of the segment. + */ +typedef struct { + U32 begin; + U32 end; + U32 score; +} COVER_segment_t; + +/** + * Checks total compressed size of a dictionary + */ +size_t COVER_checkTotalCompressedSize(const ZDICT_cover_params_t parameters, + const size_t *samplesSizes, const BYTE *samples, + size_t *offsets, + size_t nbTrainSamples, size_t nbSamples, + BYTE *const dict, size_t dictBufferCapacity); + +/** + * Returns the sum of the sample sizes. + */ +size_t COVER_sum(const size_t *samplesSizes, unsigned nbSamples) ; + +/** + * Initialize the `COVER_best_t`. + */ +void COVER_best_init(COVER_best_t *best); + +/** + * Wait until liveJobs == 0. + */ +void COVER_best_wait(COVER_best_t *best); + +/** + * Call COVER_best_wait() and then destroy the COVER_best_t. + */ +void COVER_best_destroy(COVER_best_t *best); + +/** + * Called when a thread is about to be launched. + * Increments liveJobs. + */ +void COVER_best_start(COVER_best_t *best); + +/** + * Called when a thread finishes executing, both on error or success. + * Decrements liveJobs and signals any waiting threads if liveJobs == 0. + * If this dictionary is the best so far save it and its parameters. + */ +void COVER_best_finish(COVER_best_t *best, size_t compressedSize, + ZDICT_cover_params_t parameters, void *dict, + size_t dictSize); diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c new file mode 100644 index 000000000..9a41ac2f9 --- /dev/null +++ b/lib/dictBuilder/fastcover.c @@ -0,0 +1,701 @@ +/*-************************************* +* Dependencies +***************************************/ +#include /* fprintf */ +#include /* malloc, free, qsort */ +#include /* memset */ +#include /* clock */ + +#include "mem.h" /* read */ +#include "pool.h" +#include "threading.h" +#include "cover.h" +#include "zstd_internal.h" /* includes zstd.h */ +#ifndef ZDICT_STATIC_LINKING_ONLY +#define ZDICT_STATIC_LINKING_ONLY +#endif +#include "zdict.h" + + +/*-************************************* +* Constants +***************************************/ +#define FASTCOVER_MAX_SAMPLES_SIZE (sizeof(size_t) == 8 ? ((U32)-1) : ((U32)1 GB)) +#define FASTCOVER_MAX_F 31 +#define FASTCOVER_MAX_ACCEL 10 +#define DEFAULT_SPLITPOINT 0.75 +#define DEFAULT_F 18 +#define DEFAULT_ACCEL 1 + + +/*-************************************* +* Console display +***************************************/ +static int g_displayLevel = 2; +#define DISPLAY(...) \ + { \ + fprintf(stderr, __VA_ARGS__); \ + fflush(stderr); \ + } +#define LOCALDISPLAYLEVEL(displayLevel, l, ...) \ + if (displayLevel >= l) { \ + DISPLAY(__VA_ARGS__); \ + } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ +#define DISPLAYLEVEL(l, ...) LOCALDISPLAYLEVEL(g_displayLevel, l, __VA_ARGS__) + +#define LOCALDISPLAYUPDATE(displayLevel, l, ...) \ + if (displayLevel >= l) { \ + if ((clock() - g_time > refreshRate) || (displayLevel >= 4)) { \ + g_time = clock(); \ + DISPLAY(__VA_ARGS__); \ + } \ + } +#define DISPLAYUPDATE(l, ...) LOCALDISPLAYUPDATE(g_displayLevel, l, __VA_ARGS__) +static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; + + +/*-************************************* +* Hash Functions +***************************************/ +static const U64 prime6bytes = 227718039650203ULL; +static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; } +static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); } + +static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; +static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } +static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } + + +/** + * Hash the d-byte value pointed to by p and mod 2^f + */ +static size_t FASTCOVER_hashPtrToIndex(const void* p, U32 h, unsigned d) { + if (d == 6) { + return ZSTD_hash6Ptr(p, h) & ((1 << h) - 1); + } + return ZSTD_hash8Ptr(p, h) & ((1 << h) - 1); +} + + +/*-************************************* +* Acceleration +***************************************/ +typedef struct { + unsigned finalize; /* Percentage of training samples used for ZDICT_finalizeDictionary */ + unsigned skip; /* Number of dmer skipped between each dmer counted in computeFrequency */ +} FASTCOVER_accel_t; + + +static const FASTCOVER_accel_t FASTCOVER_defaultAccelParameters[FASTCOVER_MAX_ACCEL+1] = { + { 100, 0 }, /* accel = 0, should not happen because accel = 0 defaults to accel = 1 */ + { 100, 0 }, /* accel = 1 */ + { 50, 1 }, /* accel = 2 */ + { 34, 2 }, /* accel = 3 */ + { 25, 3 }, /* accel = 4 */ + { 20, 4 }, /* accel = 5 */ + { 17, 5 }, /* accel = 6 */ + { 14, 6 }, /* accel = 7 */ + { 13, 7 }, /* accel = 8 */ + { 11, 8 }, /* accel = 9 */ + { 10, 9 }, /* accel = 10 */ +}; + + +/*-************************************* +* Context +***************************************/ +typedef struct { + const BYTE *samples; + size_t *offsets; + const size_t *samplesSizes; + size_t nbSamples; + size_t nbTrainSamples; + size_t nbTestSamples; + size_t nbDmers; + U32 *freqs; + unsigned d; + unsigned f; + FASTCOVER_accel_t accelParams; +} FASTCOVER_ctx_t; + + +/*-************************************* +* Helper functions +***************************************/ +/** + * Selects the best segment in an epoch. + * Segments of are scored according to the function: + * + * Let F(d) be the frequency of all dmers with hash value d. + * Let S_i be hash value of the dmer at position i of segment S which has length k. + * + * Score(S) = F(S_1) + F(S_2) + ... + F(S_{k-d+1}) + * + * Once the dmer with hash value d is in the dictionay we set F(d) = 0. + */ +static COVER_segment_t FASTCOVER_selectSegment(const FASTCOVER_ctx_t *ctx, + U32 *freqs, U32 begin, U32 end, + ZDICT_cover_params_t parameters, + U16* segmentFreqs) { + /* Constants */ + const U32 k = parameters.k; + const U32 d = parameters.d; + const U32 f = ctx->f; + const U32 dmersInK = k - d + 1; + + /* Try each segment (activeSegment) and save the best (bestSegment) */ + COVER_segment_t bestSegment = {0, 0, 0}; + COVER_segment_t activeSegment; + + /* Reset the activeDmers in the segment */ + /* The activeSegment starts at the beginning of the epoch. */ + activeSegment.begin = begin; + activeSegment.end = begin; + activeSegment.score = 0; + + /* Slide the activeSegment through the whole epoch. + * Save the best segment in bestSegment. + */ + while (activeSegment.end < end) { + /* Get hash value of current dmer */ + const size_t index = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.end, f, d); + + /* Add frequency of this index to score if this is the first occurence of index in active segment */ + if (segmentFreqs[index] == 0) { + activeSegment.score += freqs[index]; + } + /* Increment end of segment and segmentFreqs*/ + activeSegment.end += 1; + segmentFreqs[index] += 1; + /* If the window is now too large, drop the first position */ + if (activeSegment.end - activeSegment.begin == dmersInK + 1) { + /* Get hash value of the dmer to be eliminated from active segment */ + const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, f, d); + segmentFreqs[delIndex] -= 1; + /* Subtract frequency of this index from score if this is the last occurrence of this index in active segment */ + if (segmentFreqs[delIndex] == 0) { + activeSegment.score -= freqs[delIndex]; + } + /* Increment start of segment */ + activeSegment.begin += 1; + } + + /* If this segment is the best so far save it */ + if (activeSegment.score > bestSegment.score) { + bestSegment = activeSegment; + } + } + + /* Zero out rest of segmentFreqs array */ + while (activeSegment.begin < end) { + const size_t delIndex = FASTCOVER_hashPtrToIndex(ctx->samples + activeSegment.begin, f, d); + segmentFreqs[delIndex] -= 1; + activeSegment.begin += 1; + } + + { + /* Zero the frequency of hash value of each dmer covered by the chosen segment. */ + U32 pos; + for (pos = bestSegment.begin; pos != bestSegment.end; ++pos) { + const size_t i = FASTCOVER_hashPtrToIndex(ctx->samples + pos, f, d); + freqs[i] = 0; + } + } + + return bestSegment; +} + + +static int FASTCOVER_checkParameters(ZDICT_cover_params_t parameters, + size_t maxDictSize, unsigned f, + unsigned accel) { + /* k, d, and f are required parameters */ + if (parameters.d == 0 || parameters.k == 0) { + return 0; + } + /* d has to be 6 or 8 */ + if (parameters.d != 6 && parameters.d != 8) { + return 0; + } + /* k <= maxDictSize */ + if (parameters.k > maxDictSize) { + return 0; + } + /* d <= k */ + if (parameters.d > parameters.k) { + return 0; + } + /* 0 < f <= FASTCOVER_MAX_F*/ + if (f > FASTCOVER_MAX_F || f == 0) { + return 0; + } + /* 0 < splitPoint <= 1 */ + if (parameters.splitPoint <= 0 || parameters.splitPoint > 1) { + return 0; + } + /* 0 < accel <= 10 */ + if (accel > 10 || accel == 0) { + return 0; + } + return 1; +} + + +/** + * Clean up a context initialized with `FASTCOVER_ctx_init()`. + */ +static void FASTCOVER_ctx_destroy(FASTCOVER_ctx_t *ctx) { + if (!ctx) { + return; + } + + free(ctx->freqs); + ctx->freqs = NULL; + + free(ctx->offsets); + ctx->offsets = NULL; +} + + +/** + * Calculate for frequency of hash value of each dmer in ctx->samples + */ +static void FASTCOVER_computeFrequency(U32 *freqs, FASTCOVER_ctx_t *ctx){ + const unsigned f = ctx->f; + const unsigned d = ctx->d; + const unsigned skip = ctx->accelParams.skip; + const unsigned readLength = MAX(d, 8); + size_t start; /* start of current dmer */ + size_t i; + for (i = 0; i < ctx->nbTrainSamples; i++) { + size_t currSampleStart = ctx->offsets[i]; + size_t currSampleEnd = ctx->offsets[i+1]; + start = currSampleStart; + while (start + readLength <= currSampleEnd) { + const size_t dmerIndex = FASTCOVER_hashPtrToIndex(ctx->samples + start, f, d); + freqs[dmerIndex]++; + start = start + skip + 1; + } + } +} + + +/** + * Prepare a context for dictionary building. + * The context is only dependent on the parameter `d` and can used multiple + * times. + * Returns 1 on success or zero on error. + * The context must be destroyed with `FASTCOVER_ctx_destroy()`. + */ +static int FASTCOVER_ctx_init(FASTCOVER_ctx_t *ctx, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + unsigned d, double splitPoint, unsigned f, + FASTCOVER_accel_t accelParams) { + const BYTE *const samples = (const BYTE *)samplesBuffer; + const size_t totalSamplesSize = COVER_sum(samplesSizes, nbSamples); + /* Split samples into testing and training sets */ + const unsigned nbTrainSamples = splitPoint < 1.0 ? (unsigned)((double)nbSamples * splitPoint) : nbSamples; + const unsigned nbTestSamples = splitPoint < 1.0 ? nbSamples - nbTrainSamples : nbSamples; + const size_t trainingSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes, nbTrainSamples) : totalSamplesSize; + const size_t testSamplesSize = splitPoint < 1.0 ? COVER_sum(samplesSizes + nbTrainSamples, nbTestSamples) : totalSamplesSize; + /* Checks */ + if (totalSamplesSize < MAX(d, sizeof(U64)) || + totalSamplesSize >= (size_t)FASTCOVER_MAX_SAMPLES_SIZE) { + DISPLAYLEVEL(1, "Total samples size is too large (%u MB), maximum size is %u MB\n", + (U32)(totalSamplesSize >> 20), (FASTCOVER_MAX_SAMPLES_SIZE >> 20)); + return 0; + } + /* Check if there are at least 5 training samples */ + if (nbTrainSamples < 5) { + DISPLAYLEVEL(1, "Total number of training samples is %u and is invalid\n", nbTrainSamples); + return 0; + } + /* Check if there's testing sample */ + if (nbTestSamples < 1) { + DISPLAYLEVEL(1, "Total number of testing samples is %u and is invalid.\n", nbTestSamples); + return 0; + } + /* Zero the context */ + memset(ctx, 0, sizeof(*ctx)); + DISPLAYLEVEL(2, "Training on %u samples of total size %u\n", nbTrainSamples, + (U32)trainingSamplesSize); + DISPLAYLEVEL(2, "Testing on %u samples of total size %u\n", nbTestSamples, + (U32)testSamplesSize); + + ctx->samples = samples; + ctx->samplesSizes = samplesSizes; + ctx->nbSamples = nbSamples; + ctx->nbTrainSamples = nbTrainSamples; + ctx->nbTestSamples = nbTestSamples; + ctx->nbDmers = trainingSamplesSize - MAX(d, sizeof(U64)) + 1; + ctx->d = d; + ctx->f = f; + ctx->accelParams = accelParams; + + /* The offsets of each file */ + ctx->offsets = (size_t *)malloc((nbSamples + 1) * sizeof(size_t)); + if (!ctx->offsets) { + DISPLAYLEVEL(1, "Failed to allocate scratch buffers\n"); + FASTCOVER_ctx_destroy(ctx); + return 0; + } + + /* Fill offsets from the samplesSizes */ + { + U32 i; + ctx->offsets[0] = 0; + for (i = 1; i <= nbSamples; ++i) { + ctx->offsets[i] = ctx->offsets[i - 1] + samplesSizes[i - 1]; + } + } + + /* Initialize frequency array of size 2^f */ + ctx->freqs = (U32 *)calloc(((U64)1 << f), sizeof(U32)); + + DISPLAYLEVEL(2, "Computing frequencies\n"); + FASTCOVER_computeFrequency(ctx->freqs, ctx); + + return 1; +} + + +/** + * Given the prepared context build the dictionary. + */ +static size_t FASTCOVER_buildDictionary(const FASTCOVER_ctx_t *ctx, U32 *freqs, + void *dictBuffer, size_t dictBufferCapacity, + ZDICT_cover_params_t parameters, U16* segmentFreqs){ + BYTE *const dict = (BYTE *)dictBuffer; + size_t tail = dictBufferCapacity; + /* Divide the data up into epochs of equal size. + * We will select at least one segment from each epoch. + */ + const U32 epochs = MAX(1, (U32)(dictBufferCapacity / parameters.k)); + const U32 epochSize = (U32)(ctx->nbDmers / epochs); + size_t epoch; + DISPLAYLEVEL(2, "Breaking content into %u epochs of size %u\n", epochs, + epochSize); + /* Loop through the epochs until there are no more segments or the dictionary + * is full. + */ + for (epoch = 0; tail > 0; epoch = (epoch + 1) % epochs) { + const U32 epochBegin = (U32)(epoch * epochSize); + const U32 epochEnd = epochBegin + epochSize; + size_t segmentSize; + /* Select a segment */ + COVER_segment_t segment = FASTCOVER_selectSegment( + ctx, freqs, epochBegin, epochEnd, parameters, segmentFreqs); + + /* If the segment covers no dmers, then we are out of content */ + if (segment.score == 0) { + break; + } + + /* Trim the segment if necessary and if it is too small then we are done */ + segmentSize = MIN(segment.end - segment.begin + parameters.d - 1, tail); + if (segmentSize < parameters.d) { + break; + } + + /* We fill the dictionary from the back to allow the best segments to be + * referenced with the smallest offsets. + */ + tail -= segmentSize; + memcpy(dict + tail, ctx->samples + segment.begin, segmentSize); + DISPLAYUPDATE( + 2, "\r%u%% ", + (U32)(((dictBufferCapacity - tail) * 100) / dictBufferCapacity)); + } + DISPLAYLEVEL(2, "\r%79s\r", ""); + return tail; +} + + +/** + * Parameters for FASTCOVER_tryParameters(). + */ +typedef struct FASTCOVER_tryParameters_data_s { + const FASTCOVER_ctx_t *ctx; + COVER_best_t *best; + size_t dictBufferCapacity; + ZDICT_cover_params_t parameters; +} FASTCOVER_tryParameters_data_t; + + +/** + * Tries a set of parameters and updates the COVER_best_t with the results. + * This function is thread safe if zstd is compiled with multithreaded support. + * It takes its parameters as an *OWNING* opaque pointer to support threading. + */ +static void FASTCOVER_tryParameters(void *opaque) { + /* Save parameters as local variables */ + FASTCOVER_tryParameters_data_t *const data = (FASTCOVER_tryParameters_data_t *)opaque; + const FASTCOVER_ctx_t *const ctx = data->ctx; + const ZDICT_cover_params_t parameters = data->parameters; + size_t dictBufferCapacity = data->dictBufferCapacity; + size_t totalCompressedSize = ERROR(GENERIC); + /* Initialize array to keep track of frequency of dmer within activeSegment */ + U16* segmentFreqs = (U16 *)calloc(((U64)1 << ctx->f), sizeof(U16)); + /* Allocate space for hash table, dict, and freqs */ + BYTE *const dict = (BYTE * const)malloc(dictBufferCapacity); + U32 *freqs = (U32*) malloc(((U64)1 << ctx->f) * sizeof(U32)); + if (!segmentFreqs || !dict || !freqs) { + DISPLAYLEVEL(1, "Failed to allocate buffers: out of memory\n"); + goto _cleanup; + } + /* Copy the frequencies because we need to modify them */ + memcpy(freqs, ctx->freqs, ((U64)1 << ctx->f) * sizeof(U32)); + /* Build the dictionary */ + { + const size_t tail = FASTCOVER_buildDictionary(ctx, freqs, dict, dictBufferCapacity, + parameters, segmentFreqs); + const unsigned nbFinalizeSamples = (unsigned)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100); + dictBufferCapacity = ZDICT_finalizeDictionary( + dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail, + ctx->samples, ctx->samplesSizes, nbFinalizeSamples, parameters.zParams); + if (ZDICT_isError(dictBufferCapacity)) { + DISPLAYLEVEL(1, "Failed to finalize dictionary\n"); + goto _cleanup; + } + } + /* Check total compressed size */ + totalCompressedSize = COVER_checkTotalCompressedSize(parameters, ctx->samplesSizes, + ctx->samples, ctx->offsets, + ctx->nbTrainSamples, ctx->nbSamples, + dict, dictBufferCapacity); +_cleanup: + COVER_best_finish(data->best, totalCompressedSize, parameters, dict, + dictBufferCapacity); + free(data); + free(segmentFreqs); + free(dict); + free(freqs); +} + + + +static void FASTCOVER_convertToCoverParams(ZDICT_fastCover_params_t fastCoverParams, + ZDICT_cover_params_t *coverParams) { + coverParams->k = fastCoverParams.k; + coverParams->d = fastCoverParams.d; + coverParams->steps = fastCoverParams.steps; + coverParams->nbThreads = fastCoverParams.nbThreads; + coverParams->splitPoint = fastCoverParams.splitPoint; + coverParams->zParams = fastCoverParams.zParams; +} + + +static void FASTCOVER_convertToFastCoverParams(ZDICT_cover_params_t coverParams, + ZDICT_fastCover_params_t *fastCoverParams, + unsigned f, unsigned accel) { + fastCoverParams->k = coverParams.k; + fastCoverParams->d = coverParams.d; + fastCoverParams->steps = coverParams.steps; + fastCoverParams->nbThreads = coverParams.nbThreads; + fastCoverParams->splitPoint = coverParams.splitPoint; + fastCoverParams->f = f; + fastCoverParams->accel = accel; + fastCoverParams->zParams = coverParams.zParams; +} + + +ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover( + void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t parameters) { + BYTE* const dict = (BYTE*)dictBuffer; + FASTCOVER_ctx_t ctx; + ZDICT_cover_params_t coverParams; + FASTCOVER_accel_t accelParams; + /* Initialize global data */ + g_displayLevel = parameters.zParams.notificationLevel; + /* Assign splitPoint and f if not provided */ + parameters.splitPoint = parameters.splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters.splitPoint; + parameters.f = parameters.f == 0 ? DEFAULT_F : parameters.f; + parameters.accel = parameters.accel == 0 ? DEFAULT_ACCEL : parameters.accel; + /* Convert to cover parameter */ + memset(&coverParams, 0 , sizeof(coverParams)); + FASTCOVER_convertToCoverParams(parameters, &coverParams); + /* Checks */ + if (!FASTCOVER_checkParameters(coverParams, dictBufferCapacity, parameters.f, + parameters.accel)) { + DISPLAYLEVEL(1, "FASTCOVER parameters incorrect\n"); + return ERROR(GENERIC); + } + if (nbSamples == 0) { + DISPLAYLEVEL(1, "FASTCOVER must have at least one input file\n"); + return ERROR(GENERIC); + } + if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { + DISPLAYLEVEL(1, "dictBufferCapacity must be at least %u\n", + ZDICT_DICTSIZE_MIN); + return ERROR(dstSize_tooSmall); + } + /* Assign corresponding FASTCOVER_accel_t to accelParams*/ + accelParams = FASTCOVER_defaultAccelParameters[parameters.accel]; + /* Initialize context */ + if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, + coverParams.d, parameters.splitPoint, parameters.f, + accelParams)) { + DISPLAYLEVEL(1, "Failed to initialize context\n"); + return ERROR(GENERIC); + } + /* Build the dictionary */ + DISPLAYLEVEL(2, "Building dictionary\n"); + { + /* Initialize array to keep track of frequency of dmer within activeSegment */ + U16* segmentFreqs = (U16 *)calloc(((U64)1 << parameters.f), sizeof(U16)); + const size_t tail = FASTCOVER_buildDictionary(&ctx, ctx.freqs, dictBuffer, + dictBufferCapacity, coverParams, segmentFreqs); + const unsigned nbFinalizeSamples = (unsigned)(ctx.nbTrainSamples * ctx.accelParams.finalize / 100); + const size_t dictionarySize = ZDICT_finalizeDictionary( + dict, dictBufferCapacity, dict + tail, dictBufferCapacity - tail, + samplesBuffer, samplesSizes, nbFinalizeSamples, coverParams.zParams); + if (!ZSTD_isError(dictionarySize)) { + DISPLAYLEVEL(2, "Constructed dictionary of size %u\n", + (U32)dictionarySize); + } + FASTCOVER_ctx_destroy(&ctx); + free(segmentFreqs); + return dictionarySize; + } +} + + +ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover( + void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t *parameters) { + ZDICT_cover_params_t coverParams; + FASTCOVER_accel_t accelParams; + /* constants */ + const unsigned nbThreads = parameters->nbThreads; + const double splitPoint = + parameters->splitPoint <= 0.0 ? DEFAULT_SPLITPOINT : parameters->splitPoint; + const unsigned kMinD = parameters->d == 0 ? 6 : parameters->d; + const unsigned kMaxD = parameters->d == 0 ? 8 : parameters->d; + const unsigned kMinK = parameters->k == 0 ? 50 : parameters->k; + const unsigned kMaxK = parameters->k == 0 ? 2000 : parameters->k; + const unsigned kSteps = parameters->steps == 0 ? 40 : parameters->steps; + const unsigned kStepSize = MAX((kMaxK - kMinK) / kSteps, 1); + const unsigned kIterations = + (1 + (kMaxD - kMinD) / 2) * (1 + (kMaxK - kMinK) / kStepSize); + const unsigned f = parameters->f == 0 ? DEFAULT_F : parameters->f; + const unsigned accel = parameters->accel == 0 ? DEFAULT_ACCEL : parameters->accel; + /* Local variables */ + const int displayLevel = parameters->zParams.notificationLevel; + unsigned iteration = 1; + unsigned d; + unsigned k; + COVER_best_t best; + POOL_ctx *pool = NULL; + /* Checks */ + if (splitPoint <= 0 || splitPoint > 1) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect splitPoint\n"); + return ERROR(GENERIC); + } + if (accel == 0 || accel > FASTCOVER_MAX_ACCEL) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect accel\n"); + return ERROR(GENERIC); + } + if (kMinK < kMaxD || kMaxK < kMinK) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Incorrect k\n"); + return ERROR(GENERIC); + } + if (nbSamples == 0) { + LOCALDISPLAYLEVEL(displayLevel, 1, "FASTCOVER must have at least one input file\n"); + return ERROR(GENERIC); + } + if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) { + LOCALDISPLAYLEVEL(displayLevel, 1, "dictBufferCapacity must be at least %u\n", + ZDICT_DICTSIZE_MIN); + return ERROR(dstSize_tooSmall); + } + if (nbThreads > 1) { + pool = POOL_create(nbThreads, 1); + if (!pool) { + return ERROR(memory_allocation); + } + } + /* Initialization */ + COVER_best_init(&best); + memset(&coverParams, 0 , sizeof(coverParams)); + FASTCOVER_convertToCoverParams(*parameters, &coverParams); + accelParams = FASTCOVER_defaultAccelParameters[accel]; + /* Turn down global display level to clean up display at level 2 and below */ + g_displayLevel = displayLevel == 0 ? 0 : displayLevel - 1; + /* Loop through d first because each new value needs a new context */ + LOCALDISPLAYLEVEL(displayLevel, 2, "Trying %u different sets of parameters\n", + kIterations); + for (d = kMinD; d <= kMaxD; d += 2) { + /* Initialize the context for this value of d */ + FASTCOVER_ctx_t ctx; + LOCALDISPLAYLEVEL(displayLevel, 3, "d=%u\n", d); + if (!FASTCOVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d, splitPoint, f, accelParams)) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n"); + COVER_best_destroy(&best); + POOL_free(pool); + return ERROR(GENERIC); + } + /* Loop through k reusing the same context */ + for (k = kMinK; k <= kMaxK; k += kStepSize) { + /* Prepare the arguments */ + FASTCOVER_tryParameters_data_t *data = (FASTCOVER_tryParameters_data_t *)malloc( + sizeof(FASTCOVER_tryParameters_data_t)); + LOCALDISPLAYLEVEL(displayLevel, 3, "k=%u\n", k); + if (!data) { + LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n"); + COVER_best_destroy(&best); + FASTCOVER_ctx_destroy(&ctx); + POOL_free(pool); + return ERROR(GENERIC); + } + data->ctx = &ctx; + data->best = &best; + data->dictBufferCapacity = dictBufferCapacity; + data->parameters = coverParams; + data->parameters.k = k; + data->parameters.d = d; + data->parameters.splitPoint = splitPoint; + data->parameters.steps = kSteps; + data->parameters.zParams.notificationLevel = g_displayLevel; + /* Check the parameters */ + if (!FASTCOVER_checkParameters(data->parameters, dictBufferCapacity, + data->ctx->f, accel)) { + DISPLAYLEVEL(1, "FASTCOVER parameters incorrect\n"); + free(data); + continue; + } + /* Call the function and pass ownership of data to it */ + COVER_best_start(&best); + if (pool) { + POOL_add(pool, &FASTCOVER_tryParameters, data); + } else { + FASTCOVER_tryParameters(data); + } + /* Print status */ + LOCALDISPLAYUPDATE(displayLevel, 2, "\r%u%% ", + (U32)((iteration * 100) / kIterations)); + ++iteration; + } + COVER_best_wait(&best); + FASTCOVER_ctx_destroy(&ctx); + } + LOCALDISPLAYLEVEL(displayLevel, 2, "\r%79s\r", ""); + /* Fill the output buffer and parameters with output of the best parameters */ + { + const size_t dictSize = best.dictSize; + if (ZSTD_isError(best.compressedSize)) { + const size_t compressedSize = best.compressedSize; + COVER_best_destroy(&best); + POOL_free(pool); + return compressedSize; + } + FASTCOVER_convertToFastCoverParams(best.parameters, parameters, f, accel); + memcpy(dictBuffer, best.dict, dictSize); + COVER_best_destroy(&best); + POOL_free(pool); + return dictSize; + } + +} diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index a4d0a4481..9acceb40b 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -863,8 +863,8 @@ _cleanup: size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity, const void* customDictContent, size_t dictContentSize, - const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, - ZDICT_params_t params) + const void* samplesBuffer, const size_t* samplesSizes, + unsigned nbSamples, ZDICT_params_t params) { size_t hSize; #define HBUFFSIZE 256 /* should prove large enough for all entropy headers */ @@ -1078,17 +1078,17 @@ size_t ZDICT_trainFromBuffer_legacy(void* dictBuffer, size_t dictBufferCapacity, size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples) { - ZDICT_cover_params_t params; + ZDICT_fastCover_params_t params; DEBUGLOG(3, "ZDICT_trainFromBuffer"); memset(¶ms, 0, sizeof(params)); params.d = 8; params.steps = 4; /* Default to level 6 since no compression level information is available */ - params.zParams.compressionLevel = 6; + params.zParams.compressionLevel = 3; #if defined(DEBUGLEVEL) && (DEBUGLEVEL>=1) params.zParams.notificationLevel = DEBUGLEVEL; #endif - return ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, dictBufferCapacity, + return ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, dictBufferCapacity, samplesBuffer, samplesSizes, nbSamples, ¶ms); } diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 4094669d1..c838fd4a7 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -39,7 +39,8 @@ extern "C" { /*! ZDICT_trainFromBuffer(): * Train a dictionary from an array of samples. - * Redirect towards ZDICT_optimizeTrainFromBuffer_cover() single-threaded, with d=8 and steps=4. + * Redirect towards ZDICT_optimizeTrainFromBuffer_fastCover() single-threaded, with d=8, steps=4, + * f=18, and accel=1. * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. * The resulting dictionary will be saved into `dictBuffer`. @@ -90,6 +91,16 @@ typedef struct { ZDICT_params_t zParams; } ZDICT_cover_params_t; +typedef struct { + unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ + unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ + unsigned f; /* log of size of frequency array : constraint: 0 < f <= 31 : 1 means default(18)*/ + unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */ + unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ + double splitPoint; /* Percentage of samples used for training: the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */ + unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */ + ZDICT_params_t zParams; +} ZDICT_fastCover_params_t; /*! ZDICT_trainFromBuffer_cover(): * Train a dictionary from an array of samples using the COVER algorithm. @@ -116,9 +127,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( * dictionary constructed with those parameters is stored in `dictBuffer`. * * All of the parameters d, k, steps are optional. - * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8, 10, 12, 14, 16}. + * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}. * if steps is zero it defaults to its default value. - * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [16, 2048]. + * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000]. * * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) * or an error code, which can be tested with ZDICT_isError(). @@ -130,6 +141,48 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_cover_params_t* parameters); +/*! ZDICT_trainFromBuffer_fastCover(): + * Train a dictionary from an array of samples using a modified version of COVER algorithm. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * d and k are required. + * All other parameters are optional, will use default values if not provided + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * Note: ZDICT_trainFromBuffer_fastCover() requires about 1 bytes of memory for each input byte and additionally another 6 * 2^f bytes of memory . + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer, + size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t parameters); + +/*! ZDICT_optimizeTrainFromBuffer_fastCover(): + * The same requirements as above hold for all the parameters except `parameters`. + * This function tries many parameter combinations (specifically, k and d combinations) + * and picks the best parameters. `*parameters` is filled with the best parameters found, + * dictionary constructed with those parameters is stored in `dictBuffer`. + * All of the parameters d, k, steps, f, and accel are optional. + * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}. + * if steps is zero it defaults to its default value. + * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000]. + * If f is zero, default value of 18 is used. + * If accel is zero, default value of 1 is used. + * + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * On success `*parameters` contains the parameters selected. + * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 1 byte of memory for each input byte and additionally another 6 * 2^f bytes of memory for each thread. + */ +ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer, + size_t dictBufferCapacity, const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t* parameters); + /*! ZDICT_finalizeDictionary(): * Given a custom content as a basis for dictionary, and a set of samples, * finalize dictionary by adding headers and statistics. diff --git a/programs/README.md b/programs/README.md index 22a00409c..0fa033ccf 100644 --- a/programs/README.md +++ b/programs/README.md @@ -151,6 +151,7 @@ Advanced arguments : Dictionary builder : --train ## : create a dictionary from a training set of files --train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args +--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#] : use the fastcover algorithm with optional args --train-legacy[=s=#] : use the legacy algorithm with selectivity (default: 9) -o file : `file` is dictionary name (default: dictionary) --maxdict=# : limit dictionary to specified size (default: 112640) diff --git a/programs/dibio.c b/programs/dibio.c index fbb8aa6fa..4b68be6c9 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -44,6 +44,7 @@ #define SAMPLESIZE_MAX (128 KB) #define MEMMULT 11 /* rough estimation : memory cost to analyze 1 byte of sample */ #define COVER_MEMMULT 9 /* rough estimation : memory cost to analyze 1 byte of sample */ +#define FASTCOVER_MEMMULT 1 /* rough estimation : memory cost to analyze 1 byte of sample */ static const size_t g_maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t)); #define NOISELENGTH 32 @@ -271,16 +272,19 @@ size_t ZDICT_trainFromBuffer_unsafe_legacy(void* dictBuffer, size_t dictBufferCa int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, - ZDICT_legacy_params_t *params, ZDICT_cover_params_t *coverParams, - int optimizeCover) + ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams, + ZDICT_fastCover_params_t* fastCoverParams, int optimize) { unsigned const displayLevel = params ? params->zParams.notificationLevel : coverParams ? coverParams->zParams.notificationLevel : + fastCoverParams ? fastCoverParams->zParams.notificationLevel : 0; /* should never happen */ void* const dictBuffer = malloc(maxDictSize); fileStats const fs = DiB_fileStats(fileNamesTable, nbFiles, chunkSize, displayLevel); size_t* const sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t)); - size_t const memMult = params ? MEMMULT : COVER_MEMMULT; + size_t const memMult = params ? MEMMULT : + coverParams ? COVER_MEMMULT: + FASTCOVER_MEMMULT; size_t const maxMem = DiB_findMaxMem(fs.totalSizeToLoad * memMult) / memMult; size_t loadedSize = (size_t) MIN ((unsigned long long)maxMem, fs.totalSizeToLoad); void* const srcBuffer = malloc(loadedSize+NOISELENGTH); @@ -312,6 +316,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, /* Load input buffer */ DISPLAYLEVEL(3, "Shuffling input files\n"); DiB_shuffle(fileNamesTable, nbFiles); + DiB_loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel); { size_t dictSize; @@ -320,19 +325,36 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, dictSize = ZDICT_trainFromBuffer_unsafe_legacy(dictBuffer, maxDictSize, srcBuffer, sampleSizes, fs.nbSamples, *params); - } else if (optimizeCover) { - assert(coverParams != NULL); - dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize, - srcBuffer, sampleSizes, fs.nbSamples, - coverParams); - if (!ZDICT_isError(dictSize)) { - unsigned splitPercentage = (unsigned)(coverParams->splitPoint * 100); - DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d, coverParams->steps, splitPercentage); + } else if (coverParams) { + if (optimize) { + dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize, + srcBuffer, sampleSizes, fs.nbSamples, + coverParams); + if (!ZDICT_isError(dictSize)) { + unsigned splitPercentage = (unsigned)(coverParams->splitPoint * 100); + DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d, + coverParams->steps, splitPercentage); + } + } else { + dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer, + sampleSizes, fs.nbSamples, *coverParams); } } else { - assert(coverParams != NULL); - dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer, - sampleSizes, fs.nbSamples, *coverParams); + assert(fastCoverParams != NULL); + if (optimize) { + dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(dictBuffer, maxDictSize, + srcBuffer, sampleSizes, fs.nbSamples, + fastCoverParams); + if (!ZDICT_isError(dictSize)) { + unsigned splitPercentage = (unsigned)(fastCoverParams->splitPoint * 100); + DISPLAYLEVEL(2, "k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", fastCoverParams->k, + fastCoverParams->d, fastCoverParams->f, fastCoverParams->steps, splitPercentage, + fastCoverParams->accel); + } + } else { + dictSize = ZDICT_trainFromBuffer_fastCover(dictBuffer, maxDictSize, srcBuffer, + sampleSizes, fs.nbSamples, *fastCoverParams); + } } if (ZDICT_isError(dictSize)) { DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */ diff --git a/programs/dibio.h b/programs/dibio.h index 31a6b4bdb..ea163fe6a 100644 --- a/programs/dibio.h +++ b/programs/dibio.h @@ -34,6 +34,6 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams, - int optimizeCover); + ZDICT_fastCover_params_t* fastCoverParams, int optimize); #endif diff --git a/programs/zstd.1 b/programs/zstd.1 index 58473102e..51825af74 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -194,7 +194,7 @@ All arguments after \fB\-\-\fR are treated as files Use FILEs as training set to create a dictionary\. The training set should contain a lot of small files (> 100), and weight typically 100x the target dictionary size (for example, 10 MB for a 100 KB dictionary)\. . .IP -Supports multithreading if \fBzstd\fR is compiled with threading support\. Additional parameters can be specified with \fB\-\-train\-cover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. Equivalent to \fB\-\-train\-cover=d=8,steps=4\fR\. +Supports multithreading if \fBzstd\fR is compiled with threading support\. Additional parameters can be specified with \fB\-\-train\-cover\fR\ or \fB\-\-train\-fastcover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. Equivalent to \fB\-\-train\-fastCover=d=8,steps=4\fR\. . .TP \fB\-o file\fR @@ -240,6 +240,25 @@ Examples: \fBzstd \-\-train\-cover=k=50 FILEs\fR . .TP +\fB\-\-train\-fastcover[=k#,d=#,f=#,steps=#,split=#,accel=#]\fR +Same as cover but with extra parameters \fIf\fR and \fIaccel\fR and different default value of split +. +.IP +If \fIsplit\fR is not specified, then it tries \fIsplit\fR = 75. If \fIf\fR is not specified, then it tries \fIf\fR = 18. Requires that 0 < \fIf\fR < 32. If \fIaccel\fR is not specified, then it tries \fIaccel\fR = 1. Requires that 0 < \fIaccel\fR <= 10. Requires that \fId\fR = 6 or \fId\fR = 8. +. +.IP +\fIf\fR is log of size of array that keeps track of frequency of subsegments of size \fId\fR. The subsegment is hashed to an index in the range [0,2^\fIf\fR - 1]. It is possible that 2 different subsegments are hashed to the same index, and they are considered as the same subsegment when computing frequency. Using a higher \fIf\fR reduces collision but takes longer. +. +.IP +Examples: +. +.IP +\fBzstd \-\-train\-fastcover FILEs\fR +. +.IP +\fBzstd \-\-train\-fastcover=d=8,f=15,accel=2 FILEs\fR +. +.TP \fB\-\-train\-legacy[=selectivity=#]\fR Use legacy dictionary builder algorithm with the given dictionary \fIselectivity\fR (default: 9)\. The smaller the \fIselectivity\fR value, the denser the dictionary, improving its efficiency but reducing its possible maximum size\. \fB\-\-train\-legacy=s=#\fR is also accepted\. . diff --git a/programs/zstd.1.md b/programs/zstd.1.md index 055c5c244..324b765b7 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -254,6 +254,26 @@ Compression of small files similar to the sample set will be greatly improved. `zstd --train-cover=k=50,split=60 FILEs` +* `--train-fastcover[=k#,d=#,f=#,steps=#,split=#,accel=#]`: + Same as cover but with extra parameters _f_ and _accel_ and different default value of split + If _split_ is not specified, then it tries _split_ = 75. + If _f_ is not specified, then it tries _f_ = 18. + Requires that 0 < _f_ < 32. + If _accel_ is not specified, then it tries _accel_ = 1. + Requires that 0 < _accel_ <= 10. + Requires that _d_ = 6 or _d_ = 8. + + _f_ is log of size of array that keeps track of frequency of subsegments of size _d_. + The subsegment is hashed to an index in the range [0,2^_f_ - 1]. + It is possible that 2 different subsegments are hashed to the same index, and they are considered as the same subsegment when computing frequency. + Using a higher _f_ reduces collision but takes longer. + + Examples: + + `zstd --train-fastcover FILEs` + + `zstd --train-fastcover=d=8,f=15,accel=2 FILEs` + * `--train-legacy[=selectivity=#]`: Use legacy dictionary builder algorithm with the given dictionary _selectivity_ (default: 9). diff --git a/programs/zstdcli.c b/programs/zstdcli.c index d5a2216d6..f7c5934d9 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -84,7 +84,10 @@ static U32 g_ldmMinMatch = 0; static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT; static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT; -#define DEFAULT_SPLITPOINT 1.0 + +#define DEFAULT_ACCEL 1 + +typedef enum { cover, fastCover, legacy } dictType; /*-************************************ * Display Macros @@ -172,6 +175,7 @@ static int usage_advanced(const char* programName) DISPLAY( "Dictionary builder : \n"); DISPLAY( "--train ## : create a dictionary from a training set of files \n"); DISPLAY( "--train-cover[=k=#,d=#,steps=#,split=#] : use the cover algorithm with optional args\n"); + DISPLAY( "--train-fastcover[=k=#,d=#,f=#,steps=#,split=#,accel=#] : use the fast cover algorithm with optional args\n"); DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel); DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName); DISPLAY( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize); @@ -295,6 +299,33 @@ static unsigned parseCoverParameters(const char* stringPtr, ZDICT_cover_params_t return 1; } +/** + * parseFastCoverParameters() : + * reads fastcover parameters from *stringPtr (e.g. "--train-fastcover=k=48,d=8,f=20,steps=32,accel=2") into *params + * @return 1 means that fastcover parameters were correct + * @return 0 in case of malformed parameters + */ +static unsigned parseFastCoverParameters(const char* stringPtr, ZDICT_fastCover_params_t* params) +{ + memset(params, 0, sizeof(*params)); + for (; ;) { + if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "f=")) { params->f = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "accel=")) { params->accel = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; } + if (longCommandWArg(&stringPtr, "split=")) { + unsigned splitPercentage = readU32FromChar(&stringPtr); + params->splitPoint = (double)splitPercentage / 100.0; + if (stringPtr[0]==',') { stringPtr++; continue; } else break; + } + return 0; + } + if (stringPtr[0] != 0) return 0; + DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nf=%u\nsteps=%u\nsplit=%u\naccel=%u\n", params->k, params->d, params->f, params->steps, (unsigned)(params->splitPoint * 100), params->accel); + return 1; +} + /** * parseLegacyParameters() : * reads legacy dictioanry builter parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity @@ -316,7 +347,19 @@ static ZDICT_cover_params_t defaultCoverParams(void) memset(¶ms, 0, sizeof(params)); params.d = 8; params.steps = 4; - params.splitPoint = DEFAULT_SPLITPOINT; + params.splitPoint = 1.0; + return params; +} + +static ZDICT_fastCover_params_t defaultFastCoverParams(void) +{ + ZDICT_fastCover_params_t params; + memset(¶ms, 0, sizeof(params)); + params.d = 8; + params.f = 18; + params.steps = 4; + params.splitPoint = 0.75; /* different from default splitPoint of cover */ + params.accel = DEFAULT_ACCEL; return params; } #endif @@ -431,7 +474,8 @@ int main(int argCount, const char* argv[]) #endif #ifndef ZSTD_NODICT ZDICT_cover_params_t coverParams = defaultCoverParams(); - int cover = 1; + ZDICT_fastCover_params_t fastCoverParams = defaultFastCoverParams(); + dictType dict = fastCover; #endif #ifndef ZSTD_NOBENCH BMK_advancedParams_t benchParams = BMK_initAdvancedParams(); @@ -530,18 +574,29 @@ int main(int argCount, const char* argv[]) operation = zom_train; if (outFileName == NULL) outFileName = g_defaultDictName; - cover = 1; + dict = cover; /* Allow optional arguments following an = */ if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); } else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); } else if (!parseCoverParameters(argument, &coverParams)) { CLEAN_RETURN(badusage(programName)); } continue; } + if (longCommandWArg(&argument, "--train-fastcover")) { + operation = zom_train; + if (outFileName == NULL) + outFileName = g_defaultDictName; + dict = fastCover; + /* Allow optional arguments following an = */ + if (*argument == 0) { memset(&fastCoverParams, 0, sizeof(fastCoverParams)); } + else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); } + else if (!parseFastCoverParameters(argument, &fastCoverParams)) { CLEAN_RETURN(badusage(programName)); } + continue; + } if (longCommandWArg(&argument, "--train-legacy")) { operation = zom_train; if (outFileName == NULL) outFileName = g_defaultDictName; - cover = 0; + dict = legacy; /* Allow optional arguments following an = */ if (*argument == 0) { continue; } else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); } @@ -881,17 +936,22 @@ int main(int argCount, const char* argv[]) zParams.compressionLevel = dictCLevel; zParams.notificationLevel = g_displayLevel; zParams.dictID = dictID; - if (cover) { + if (dict == cover) { int const optimize = !coverParams.k || !coverParams.d; coverParams.nbThreads = nbWorkers; coverParams.zParams = zParams; - operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, optimize); + operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, NULL, optimize); + } else if (dict == fastCover) { + int const optimize = !fastCoverParams.k || !fastCoverParams.d; + fastCoverParams.nbThreads = nbWorkers; + fastCoverParams.zParams = zParams; + operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, NULL, &fastCoverParams, optimize); } else { ZDICT_legacy_params_t dictParams; memset(&dictParams, 0, sizeof(dictParams)); dictParams.selectivityLevel = dictSelect; dictParams.zParams = zParams; - operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, 0); + operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, NULL, 0); } #endif goto _end; diff --git a/tests/playTests.sh b/tests/playTests.sh index 0a1f96c02..fa0685c6d 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -427,7 +427,7 @@ $ECHO "- Create second (different) dictionary" $ZSTD --train-cover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC $ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" $ECHO "- Create dictionary with short dictID" -$ZSTD --train-cover=k=46,d=8 *.c ../programs/*.c --dictID=1 -o tmpDict1 +$ZSTD --train-cover=k=46,d=8,split=80 *.c ../programs/*.c --dictID=1 -o tmpDict1 cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" $ECHO "- Create dictionary with size limit" $ZSTD --train-cover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K @@ -444,6 +444,47 @@ $ZSTD --train-cover *.c ../programs/*.c test -f dictionary rm tmp* dictionary + +$ECHO "\n===> fastCover dictionary builder : advanced options " + +TESTFILE=../programs/zstdcli.c +./datagen > tmpDict +$ECHO "- Create first dictionary" +$ZSTD --train-fastcover=k=46,d=8,f=15,split=80 *.c ../programs/*.c -o tmpDict +cp $TESTFILE tmp +$ZSTD -f tmp -D tmpDict +$ZSTD -d tmp.zst -D tmpDict -fo result +$DIFF $TESTFILE result +$ECHO "- Create second (different) dictionary" +$ZSTD --train-fastcover=k=56,d=8 *.c ../programs/*.c ../programs/*.h -o tmpDictC +$ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" +$ECHO "- Create dictionary with short dictID" +$ZSTD --train-fastcover=k=46,d=8,f=15,split=80 *.c ../programs/*.c --dictID=1 -o tmpDict1 +cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" +$ECHO "- Create dictionary with size limit" +$ZSTD --train-fastcover=steps=8 *.c ../programs/*.c -o tmpDict2 --maxdict=4K +$ECHO "- Compare size of dictionary from 90% training samples with 80% training samples" +$ZSTD --train-fastcover=split=90 -r *.c ../programs/*.c +$ZSTD --train-fastcover=split=80 -r *.c ../programs/*.c +$ECHO "- Create dictionary using all samples for both training and testing" +$ZSTD --train-fastcover=split=100 -r *.c ../programs/*.c +$ECHO "- Create dictionary using f=16" +$ZSTD --train-fastcover=f=16 -r *.c ../programs/*.c +$ECHO "- Create dictionary using accel=2" +$ZSTD --train-fastcover=accel=2 -r *.c ../programs/*.c +$ECHO "- Create dictionary using accel=10" +$ZSTD --train-fastcover=accel=10 -r *.c ../programs/*.c +$ECHO "- Create dictionary with multithreading" +$ZSTD --train-fastcover -T4 -r *.c ../programs/*.c +$ECHO "- Test -o before --train-fastcover" +rm -f tmpDict dictionary +$ZSTD -o tmpDict --train-fastcover *.c ../programs/*.c +test -f tmpDict +$ZSTD --train-fastcover *.c ../programs/*.c +test -f dictionary +rm tmp* dictionary + + $ECHO "\n===> legacy dictionary builder " TESTFILE=../programs/zstdcli.c diff --git a/tests/symbols.c b/tests/symbols.c index c0bed2e5d..b37082131 100644 --- a/tests/symbols.c +++ b/tests/symbols.c @@ -144,6 +144,8 @@ static const void *symbols[] = { /* zdict.h: advanced functions */ &ZDICT_trainFromBuffer_cover, &ZDICT_optimizeTrainFromBuffer_cover, + &ZDICT_trainFromBuffer_fastCover, + &ZDICT_optimizeTrainFromBuffer_fastCover, &ZDICT_finalizeDictionary, &ZDICT_trainFromBuffer_legacy, &ZDICT_addEntropyTablesFromBuffer,