From ff54fced641bac595a4120108b563ee523650adc Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 28 Jul 2017 15:30:46 -0700 Subject: [PATCH 01/10] patched style errors, add ability to bound compression level variation --- contrib/adaptive-compression/adapt.c | 46 ++++++++++++++++++---------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/contrib/adaptive-compression/adapt.c b/contrib/adaptive-compression/adapt.c index 5cf3b9708..5e26a0db6 100644 --- a/contrib/adaptive-compression/adapt.c +++ b/contrib/adaptive-compression/adapt.c @@ -42,6 +42,8 @@ static size_t g_streamedSize = 0; static unsigned g_useProgressBar = 1; static UTIL_freq_t g_ticksPerSecond; static unsigned g_forceCompressionLevel = 0; +static unsigned g_minCLevel = 1; +static unsigned g_maxCLevel = 22; typedef struct { void* start; @@ -420,7 +422,7 @@ static void adaptCompressionLevel(adaptCCtx* ctx) /* use whichever one waited less because it was slower */ double const completion = MAX(createWaitCompressionCompletion, writeWaitCompressionCompletion); unsigned const change = convertCompletionToChange(completion); - unsigned const boundChange = MIN(change, ctx->compressionLevel - 1); + unsigned const boundChange = ctx->compressionLevel >= g_minCLevel ? MIN(change, ctx->compressionLevel - g_minCLevel) : 0; if (ctx->convergenceCounter >= CONVERGENCE_LOWER_BOUND && boundChange != 0) { /* reset convergence counter, might have been a spike */ ctx->convergenceCounter = 0; @@ -438,7 +440,7 @@ static void adaptCompressionLevel(adaptCCtx* ctx) /* compress waiting on write */ double const completion = MIN(compressWaitWriteCompletion, compressWaitCreateCompletion); unsigned const change = convertCompletionToChange(completion); - unsigned const boundChange = MIN(change, ZSTD_maxCLevel() - ctx->compressionLevel); + unsigned const boundChange = g_maxCLevel >= ctx->compressionLevel ? MIN(change, g_maxCLevel - ctx->compressionLevel) : 0; if (ctx->convergenceCounter >= CONVERGENCE_LOWER_BOUND && boundChange != 0) { /* reset convergence counter, might have been a spike */ ctx->convergenceCounter = 0; @@ -620,17 +622,19 @@ static void* compressionThread(void* arg) static void displayProgress(unsigned cLevel, unsigned last) { if (!g_useProgressBar) return; - UTIL_time_t currTime; - UTIL_getTime(&currTime); - double const timeElapsed = (double)(UTIL_getSpanTimeMicro(g_ticksPerSecond, g_startTime, currTime) / 1000.0); - double const sizeMB = (double)g_streamedSize / (1 << 20); - double const avgCompRate = sizeMB * 1000 / timeElapsed; - fprintf(stderr, "\r| Comp. Level: %2u | Time Elapsed: %7.2f s | Data Size: %7.1f MB | Avg Comp. Rate: %6.2f MB/s |", cLevel, timeElapsed/1000.0, sizeMB, avgCompRate); - if (last) { - fprintf(stderr, "\n"); - } - else { - fflush(stderr); + { + UTIL_time_t currTime; + UTIL_getTime(&currTime); + double const timeElapsed = (double)(UTIL_getSpanTimeMicro(g_ticksPerSecond, g_startTime, currTime) / 1000.0); + double const sizeMB = (double)g_streamedSize / (1 << 20); + double const avgCompRate = sizeMB * 1000 / timeElapsed; + fprintf(stderr, "\r| Comp. Level: %2u | Time Elapsed: %7.2f s | Data Size: %7.1f MB | Avg Comp. Rate: %6.2f MB/s |", cLevel, timeElapsed/1000.0, sizeMB, avgCompRate); + if (last) { + fprintf(stderr, "\n"); + } + else { + fflush(stderr); + } } } @@ -928,9 +932,9 @@ static int freeFileCompressionResources(fcResources* fcr) static int compressFilename(const char* const srcFilename, const char* const dstFilenameOrNull) { int ret = 0; + fcResources fcr = createFileCompressionResources(srcFilename, dstFilenameOrNull); UTIL_getTime(&g_startTime); g_streamedSize = 0; - fcResources fcr = createFileCompressionResources(srcFilename, dstFilenameOrNull); ret |= performCompression(fcr.ctx, fcr.srcFile, fcr.otArg); ret |= freeFileCompressionResources(&fcr); return ret; @@ -973,7 +977,7 @@ static unsigned readU32FromChar(const char** stringPtr) return result; } -static void help() +static void help(void) { PRINT("Usage:\n"); PRINT(" ./multi [options] [file(s)]\n"); @@ -986,6 +990,8 @@ static void help() PRINT(" -c : force write to stdout\n"); PRINT(" -p : hide progress bar\n"); PRINT(" -q : quiet mode -- do not show progress bar or other information\n"); + PRINT(" -l# : provide lower bound for compression level\n"); + PRINT(" -u# : provide upper bound for compression level\n"); } /* return 0 if successful, else return error */ int main(int argCount, const char* argv[]) @@ -993,10 +999,10 @@ int main(int argCount, const char* argv[]) const char* outFilename = NULL; const char** filenameTable = (const char**)malloc(argCount*sizeof(const char*)); unsigned filenameIdx = 0; - filenameTable[0] = stdinmark; unsigned forceStdout = 0; int ret = 0; int argNum; + filenameTable[0] = stdinmark; UTIL_initTimer(&g_ticksPerSecond); @@ -1036,6 +1042,14 @@ int main(int argCount, const char* argv[]) g_useProgressBar = 0; g_displayLevel = 0; break; + case 'l': + argument += 2; + g_minCLevel = readU32FromChar(&argument); + break; + case 'u': + argument += 2; + g_maxCLevel = readU32FromChar(&argument); + break; default: DISPLAY("Error: invalid argument provided\n"); ret = 1; From 0f4cb67b0050503aeb89a8c91e738c3d685cf89a Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 28 Jul 2017 15:55:02 -0700 Subject: [PATCH 02/10] add tests for compression bounds, fix another warning --- contrib/adaptive-compression/adapt.c | 5 +++-- contrib/adaptive-compression/test-correctness.sh | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/contrib/adaptive-compression/adapt.c b/contrib/adaptive-compression/adapt.c index 5e26a0db6..55f4148fb 100644 --- a/contrib/adaptive-compression/adapt.c +++ b/contrib/adaptive-compression/adapt.c @@ -621,10 +621,11 @@ static void* compressionThread(void* arg) static void displayProgress(unsigned cLevel, unsigned last) { + + UTIL_time_t currTime; + UTIL_getTime(&currTime); if (!g_useProgressBar) return; { - UTIL_time_t currTime; - UTIL_getTime(&currTime); double const timeElapsed = (double)(UTIL_getSpanTimeMicro(g_ticksPerSecond, g_startTime, currTime) / 1000.0); double const sizeMB = (double)g_streamedSize / (1 << 20); double const avgCompRate = sizeMB * 1000 / timeElapsed; diff --git a/contrib/adaptive-compression/test-correctness.sh b/contrib/adaptive-compression/test-correctness.sh index 8ae6604af..3bea867b9 100755 --- a/contrib/adaptive-compression/test-correctness.sh +++ b/contrib/adaptive-compression/test-correctness.sh @@ -242,4 +242,11 @@ echo -e "\ncorrectness tests -- window size test" ./datagen -s39 -g1GB | pv -L 25m -q | ./adapt -i1 | pv -q > tmp.zst zstd -d tmp.zst rm tmp* + +echo -e "\ncorrectness tests -- testing bounds" +./datagen -s40 -g1GB | pv -L 25m -q | ./adapt -i1 -u4 | pv -q > tmp.zst +rm tmp* + +./datagen -s41 -g1GB | ./adapt -i14 -l4 > tmp.zst +rm tmp* make clean From 4d904ac800920286fc6d22b28e132577572cd4de Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 28 Jul 2017 16:12:58 -0700 Subject: [PATCH 03/10] add flags for multithreading --- contrib/adaptive-compression/Makefile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/contrib/adaptive-compression/Makefile b/contrib/adaptive-compression/Makefile index 9bc19ee15..0e081a670 100644 --- a/contrib/adaptive-compression/Makefile +++ b/contrib/adaptive-compression/Makefile @@ -6,6 +6,15 @@ ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) +# Define *.exe as extension for Windows systems +ifneq (,$(filter Windows%,$(OS))) +EXT =.exe +MULTITHREAD_LD = +else +EXT = +MULTITHREAD_LD = -pthread +endif + DEBUGFLAGS= -g -DZSTD_DEBUG=1 CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) @@ -17,7 +26,7 @@ CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wredundant-decls CFLAGS += $(DEBUGFLAGS) CFLAGS += $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LD) all: adapt datagen From 51788225db8d3a2fac58614d72e0dd918fa380fb Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 28 Jul 2017 17:27:36 -0700 Subject: [PATCH 04/10] remove exe extension from makefile, reinclude pthread flag --- contrib/adaptive-compression/Makefile | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/contrib/adaptive-compression/Makefile b/contrib/adaptive-compression/Makefile index 0e081a670..b1c498c93 100644 --- a/contrib/adaptive-compression/Makefile +++ b/contrib/adaptive-compression/Makefile @@ -6,15 +6,7 @@ ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) -# Define *.exe as extension for Windows systems -ifneq (,$(filter Windows%,$(OS))) -EXT =.exe -MULTITHREAD_LD = -else -EXT = -MULTITHREAD_LD = -pthread -endif - +MULTITHREAD_LDFLAGS = -pthread DEBUGFLAGS= -g -DZSTD_DEBUG=1 CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) @@ -26,7 +18,7 @@ CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wredundant-decls CFLAGS += $(DEBUGFLAGS) CFLAGS += $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LD) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS) all: adapt datagen From cb9af53e773c1fd1df9765028c2ffec636838158 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 28 Jul 2017 17:28:25 -0700 Subject: [PATCH 05/10] delete empty line --- contrib/adaptive-compression/adapt.c | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/adaptive-compression/adapt.c b/contrib/adaptive-compression/adapt.c index 55f4148fb..755b7e976 100644 --- a/contrib/adaptive-compression/adapt.c +++ b/contrib/adaptive-compression/adapt.c @@ -621,7 +621,6 @@ static void* compressionThread(void* arg) static void displayProgress(unsigned cLevel, unsigned last) { - UTIL_time_t currTime; UTIL_getTime(&currTime); if (!g_useProgressBar) return; From e22b60cb76a09af2dc0365bca2ee20437003f2db Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 28 Jul 2017 17:46:51 -0700 Subject: [PATCH 06/10] removed ternary operation, added assert statement, check to make sure initial compression level is within bounds --- contrib/adaptive-compression/adapt.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/contrib/adaptive-compression/adapt.c b/contrib/adaptive-compression/adapt.c index 755b7e976..3a57c3723 100644 --- a/contrib/adaptive-compression/adapt.c +++ b/contrib/adaptive-compression/adapt.c @@ -414,6 +414,8 @@ static void adaptCompressionLevel(adaptCCtx* ctx) pthread_mutex_unlock(&ctx->createCompletion_mutex.pMutex); DEBUG(2, "convergence counter: %u\n", ctx->convergenceCounter); + assert(g_minCLevel <= ctx->compressionLevel && g_maxCLevel >= ctx->compressionLevel); + /* adaptation logic */ if (ctx->cooldown) ctx->cooldown--; @@ -422,7 +424,7 @@ static void adaptCompressionLevel(adaptCCtx* ctx) /* use whichever one waited less because it was slower */ double const completion = MAX(createWaitCompressionCompletion, writeWaitCompressionCompletion); unsigned const change = convertCompletionToChange(completion); - unsigned const boundChange = ctx->compressionLevel >= g_minCLevel ? MIN(change, ctx->compressionLevel - g_minCLevel) : 0; + unsigned const boundChange = MIN(change, ctx->compressionLevel - g_minCLevel); if (ctx->convergenceCounter >= CONVERGENCE_LOWER_BOUND && boundChange != 0) { /* reset convergence counter, might have been a spike */ ctx->convergenceCounter = 0; @@ -440,7 +442,7 @@ static void adaptCompressionLevel(adaptCCtx* ctx) /* compress waiting on write */ double const completion = MIN(compressWaitWriteCompletion, compressWaitCreateCompletion); unsigned const change = convertCompletionToChange(completion); - unsigned const boundChange = g_maxCLevel >= ctx->compressionLevel ? MIN(change, g_maxCLevel - ctx->compressionLevel) : 0; + unsigned const boundChange = MIN(change, g_maxCLevel - ctx->compressionLevel); if (ctx->convergenceCounter >= CONVERGENCE_LOWER_BOUND && boundChange != 0) { /* reset convergence counter, might have been a spike */ ctx->convergenceCounter = 0; @@ -1000,6 +1002,7 @@ int main(int argCount, const char* argv[]) const char** filenameTable = (const char**)malloc(argCount*sizeof(const char*)); unsigned filenameIdx = 0; unsigned forceStdout = 0; + unsigned providedInitialCLevel = 0; int ret = 0; int argNum; filenameTable[0] = stdinmark; @@ -1024,6 +1027,7 @@ int main(int argCount, const char* argv[]) case 'i': argument += 2; g_compressionLevel = readU32FromChar(&argument); + providedInitialCLevel = 1; break; case 'h': help(); @@ -1062,6 +1066,20 @@ int main(int argCount, const char* argv[]) filenameTable[filenameIdx++] = argument; } + /* check initial, max, and min compression levels */ + { + unsigned const minMaxInconsistent = g_minCLevel > g_maxCLevel; + unsigned const initialNotInRange = g_minCLevel > g_compressionLevel || g_maxCLevel < g_compressionLevel; + if (minMaxInconsistent || (initialNotInRange && providedInitialCLevel)) { + DISPLAY("Error: provided compression level parameters are invalid\n"); + ret = 1; + goto _main_exit; + } + else if (initialNotInRange) { + g_compressionLevel = g_minCLevel; + } + } + /* error checking with number of files */ if (filenameIdx > 1 && (outFilename != NULL && strcmp(outFilename, stdoutmark))) { DISPLAY("Error: multiple input files provided, cannot use specified output file\n"); From f60cd3f99bccc482d89c1afac73b329a2d9adb29 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 31 Jul 2017 09:47:09 -0700 Subject: [PATCH 07/10] print defaults and range, remove EXT --- contrib/adaptive-compression/Makefile | 2 +- contrib/adaptive-compression/adapt.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/adaptive-compression/Makefile b/contrib/adaptive-compression/Makefile index b1c498c93..87b61b417 100644 --- a/contrib/adaptive-compression/Makefile +++ b/contrib/adaptive-compression/Makefile @@ -29,7 +29,7 @@ adapt-debug: $(ZSTD_FILES) adapt.c $(CC) $(FLAGS) -DDEBUG_MODE=2 $^ -o adapt datagen : $(PRGDIR)/datagen.c datagencli.c - $(CC) $(FLAGS) $^ -o $@$(EXT) + $(CC) $(FLAGS) $^ -o $@ test-adapt-correctness: datagen adapt @./test-correctness.sh diff --git a/contrib/adaptive-compression/adapt.c b/contrib/adaptive-compression/adapt.c index 3a57c3723..5cec227ec 100644 --- a/contrib/adaptive-compression/adapt.c +++ b/contrib/adaptive-compression/adapt.c @@ -986,14 +986,14 @@ static void help(void) PRINT("\n"); PRINT("Options:\n"); PRINT(" -oFILE : specify the output file name\n"); - PRINT(" -i# : provide initial compression level\n"); + PRINT(" -i# : provide initial compression level -- default %d, must be in the range [L, U] where L and U are bound values (see below for defaults)\n", DEFAULT_COMPRESSION_LEVEL); PRINT(" -h : display help/information\n"); PRINT(" -f : force the compression level to stay constant\n"); PRINT(" -c : force write to stdout\n"); PRINT(" -p : hide progress bar\n"); PRINT(" -q : quiet mode -- do not show progress bar or other information\n"); - PRINT(" -l# : provide lower bound for compression level\n"); - PRINT(" -u# : provide upper bound for compression level\n"); + PRINT(" -l# : provide lower bound for compression level -- default 1\n"); + PRINT(" -u# : provide upper bound for compression level -- default 22\n"); } /* return 0 if successful, else return error */ int main(int argCount, const char* argv[]) From 9ea7df03de87a581b3f3be1eb6e555aece56888c Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 31 Jul 2017 11:04:17 -0700 Subject: [PATCH 08/10] add install target in makefile --- contrib/adaptive-compression/Makefile | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/contrib/adaptive-compression/Makefile b/contrib/adaptive-compression/Makefile index 87b61b417..c64fce954 100644 --- a/contrib/adaptive-compression/Makefile +++ b/contrib/adaptive-compression/Makefile @@ -46,3 +46,31 @@ clean: @$(RM) -f tests/*.zst @$(RM) -f tests/tmp* @echo "finished cleaning" + +#----------------------------------------------------------------------------- +# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets +#----------------------------------------------------------------------------- +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) + +ifneq (,$(filter $(shell uname),SunOS)) +INSTALL ?= ginstall +else +INSTALL ?= install +endif + +PREFIX ?= /usr/local +DESTDIR ?= +BINDIR ?= $(PREFIX)/bin + +INSTALL_PROGRAM ?= $(INSTALL) -m 755 + +install: adapt + @echo Installing binaries + @$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ + @$(INSTALL_PROGRAM) adapt $(DESTDIR)$(BINDIR)/zstd-adaptive + @echo zstd-adaptive installation completed + +uninstall: + @$(RM) $(DESTDIR)$(BINDIR)/zstd-adaptive + @echo zstd-adaptive programs successfully uninstalled +endif From 0295737ad770ebeafd02f93cf61e1446a2cd4913 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 31 Jul 2017 13:43:03 -0700 Subject: [PATCH 09/10] change signal to broadcast for jobCompressed condition varaible since multiple threads waiting --- contrib/adaptive-compression/adapt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/adaptive-compression/adapt.c b/contrib/adaptive-compression/adapt.c index 5cec227ec..eeb4c2ea9 100644 --- a/contrib/adaptive-compression/adapt.c +++ b/contrib/adaptive-compression/adapt.c @@ -333,7 +333,7 @@ static void signalErrorToThreads(adaptCCtx* ctx) pthread_mutex_unlock(&ctx->jobReady_mutex.pMutex); pthread_mutex_lock(&ctx->jobCompressed_mutex.pMutex); - pthread_cond_signal(&ctx->jobCompressed_cond.pCond); + pthread_cond_broadcast(&ctx->jobCompressed_cond.pCond); pthread_mutex_unlock(&ctx->jobReady_mutex.pMutex); pthread_mutex_lock(&ctx->jobWrite_mutex.pMutex); From e100a311ebb88e59f1bf155f0a31f22ba04e172b Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 7 Aug 2017 13:11:07 -0700 Subject: [PATCH 10/10] removed direct assignment of 22, used ZSTD_maxCLevel() instead --- contrib/adaptive-compression/adapt.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/contrib/adaptive-compression/adapt.c b/contrib/adaptive-compression/adapt.c index eeb4c2ea9..40ebb0722 100644 --- a/contrib/adaptive-compression/adapt.c +++ b/contrib/adaptive-compression/adapt.c @@ -43,7 +43,7 @@ static unsigned g_useProgressBar = 1; static UTIL_freq_t g_ticksPerSecond; static unsigned g_forceCompressionLevel = 0; static unsigned g_minCLevel = 1; -static unsigned g_maxCLevel = 22; +static unsigned g_maxCLevel; typedef struct { void* start; @@ -993,7 +993,7 @@ static void help(void) PRINT(" -p : hide progress bar\n"); PRINT(" -q : quiet mode -- do not show progress bar or other information\n"); PRINT(" -l# : provide lower bound for compression level -- default 1\n"); - PRINT(" -u# : provide upper bound for compression level -- default 22\n"); + PRINT(" -u# : provide upper bound for compression level -- default %u\n", ZSTD_maxCLevel()); } /* return 0 if successful, else return error */ int main(int argCount, const char* argv[]) @@ -1006,6 +1006,7 @@ int main(int argCount, const char* argv[]) int ret = 0; int argNum; filenameTable[0] = stdinmark; + g_maxCLevel = ZSTD_maxCLevel(); UTIL_initTimer(&g_ticksPerSecond);