From 003c7a85684220726d098ac39354c5bfb6538a98 Mon Sep 17 00:00:00 2001 From: inikep Date: Wed, 27 Jul 2016 11:07:13 +0200 Subject: [PATCH 1/4] optimal parser: removed ZSTD_REP_INIT --- Makefile | 2 +- lib/common/zstd_internal.h | 9 +++++---- lib/compress/zstd_opt.h | 28 +++++++++++++++------------- lib/decompress/zstd_decompress.c | 8 ++++---- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 9572497d8..45782af9f 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,7 @@ armtest: clean ppctest: clean $(MAKE) -C $(PRGDIR) datagen # use native, faster - $(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-static" + $(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" ppc64test: clean $(MAKE) -C $(PRGDIR) datagen # use native, faster diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 8e50da347..b12877c3c 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -52,8 +52,9 @@ * Common constants ***************************************/ #define ZSTD_OPT_DEBUG 0 /* 3 = compression stats; 5 = check encoded sequences; 9 = full logs */ -#include #if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9 + #include + #include #define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__) #define ZSTD_LOG_ENCODE(...) printf(__VA_ARGS__) #define ZSTD_LOG_BLOCK(...) printf(__VA_ARGS__) @@ -66,8 +67,8 @@ #define ZSTD_OPT_NUM (1<<12) #define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7 */ -#define ZSTD_REP_NUM 3 -#define ZSTD_REP_INIT ZSTD_REP_NUM +#define ZSTD_REP_NUM 3 /* number of repcodes */ +#define ZSTD_REP_CHECK (ZSTD_REP_NUM-0) /* number of repcodes to check by the optimal parser */ #define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; @@ -169,7 +170,7 @@ typedef struct { U32 off; U32 mlen; U32 litlen; - U32 rep[ZSTD_REP_INIT]; + U32 rep[ZSTD_REP_NUM]; } ZSTD_optimal_t; #if ZSTD_OPT_DEBUG == 3 diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index ef394f198..0764d945f 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -464,13 +464,14 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, ZSTD_optimal_t* opt = seqStorePtr->priceTable; ZSTD_match_t* matches = seqStorePtr->matchTable; const BYTE* inr; - U32 offset, rep[ZSTD_REP_INIT]; + U32 offset, rep[ZSTD_REP_NUM]; /* init */ ctx->nextToUpdate3 = ctx->nextToUpdate; ZSTD_rescaleFreqs(seqStorePtr); ip += (ip==prefixStart); - { U32 i; for (i=0; irep[i]; } + { U32 i; for (i=0; irep[i]; } + inr = ip; ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_GENERIC srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len); @@ -484,7 +485,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, /* check repCode */ { U32 i; - for (i=0; i sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { - ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); best_mlen = mlen; best_off = i; last_pos = cur + 1; + ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); goto _storeSequence; } @@ -755,12 +756,13 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, const BYTE* inr; /* init */ - U32 offset, rep[ZSTD_REP_INIT]; - { U32 i; for (i=0; irep[i]; } + U32 offset, rep[ZSTD_REP_NUM]; + { U32 i; for (i=0; irep[i]; } ctx->nextToUpdate3 = ctx->nextToUpdate; ZSTD_rescaleFreqs(seqStorePtr); ip += (ip==prefixStart); + inr = ip; ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_EXTDICT srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len); @@ -776,7 +778,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, /* check repCode */ { U32 i; - for (i=0; i sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { @@ -878,7 +880,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, best_mlen = 0; { U32 i; - for (i=0; i sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { - ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); best_mlen = mlen; best_off = i; last_pos = cur + 1; + ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); goto _storeSequence; } @@ -1033,7 +1035,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ } } /* for (cur=0; cur < last_pos; ) */ /* Save reps for next block */ - ctx->savedRep[0] = rep[0]; ctx->savedRep[1] = rep[1]; ctx->savedRep[2] = rep[2]; + { int i; for (i=0; isavedRep[i] = rep[i]; } /* Last Literals */ { size_t lastLLSize = iend - anchor; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index cc61627f2..ecffb4c6d 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -118,7 +118,7 @@ struct ZSTD_DCtx_s const void* vBase; const void* dictEnd; size_t expected; - U32 rep[3]; + U32 rep[ZSTD_REP_NUM]; ZSTD_frameParams fParams; blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ ZSTD_dStage stage; @@ -571,7 +571,7 @@ typedef struct { FSE_DState_t stateLL; FSE_DState_t stateOffb; FSE_DState_t stateML; - size_t prevOffset[ZSTD_REP_INIT]; + size_t prevOffset[ZSTD_REP_NUM]; } seqState_t; @@ -751,7 +751,7 @@ static size_t ZSTD_decompressSequences( if (nbSeq) { seqState_t seqState; dctx->fseEntropy = 1; - { U32 i; for (i=0; irep[i]; } + { U32 i; for (i=0; irep[i]; } { size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip); if (ERR_isError(errorCode)) return ERROR(corruption_detected); } FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL); @@ -769,7 +769,7 @@ static size_t ZSTD_decompressSequences( /* check if reached exact end */ if (nbSeq) return ERROR(corruption_detected); /* save reps for next block */ - { U32 i; for (i=0; irep[i] = (U32)(seqState.prevOffset[i]); } + { U32 i; for (i=0; irep[i] = (U32)(seqState.prevOffset[i]); } } /* last literal segment */ From 59b86fc141eb84ffcba297e44df66bdf988102bf Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 29 Jul 2016 11:00:33 +0200 Subject: [PATCH 2/4] zstd_opt.h: fixed checking of rep codes --- lib/compress/zstd_opt.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index 0764d945f..387d62400 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -782,7 +782,8 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, const U32 repIndex = (U32)(current - rep[i]); const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; - if ( (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ + if ((rep[i]<(U32)(ip-prefixStart)) + && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { /* repcode detected we should take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; @@ -884,7 +885,8 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, const U32 repIndex = (U32)(current+cur - opt[cur].rep[i]); const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; - if ( (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ + if ((rep[i]<(U32)(ip-prefixStart)) + && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { /* repcode detected */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; @@ -1018,7 +1020,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ ml2 = ZSTD_count_2segments(ip, match, iend, dictEnd, prefixStart); ZSTD_LOG_PARSER("%d: ZSTD_count_2segments=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)ml2, (int)best_off, dictBase, dictEnd, prefixStart, ip, match); } - else ml2 = (U32)ZSTD_count(ip, ip-offset, iend); + else ml2 = (U32)ZSTD_count(ip, ip-best_off, iend); } else ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend); if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { From c4b5106f7f67b8b2dfeeee06b8fa6daabeda7555 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 29 Jul 2016 16:11:37 +0200 Subject: [PATCH 3/4] test-zstd-speed.py: force to compile zstd32 --- tests/test-zstd-speed.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py index a837cf3a8..562d0790c 100755 --- a/tests/test-zstd-speed.py +++ b/tests/test-zstd-speed.py @@ -169,7 +169,7 @@ def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, hav local_branch = string.split(branch, '/')[1] version = local_branch.rpartition('-')[2] + '_' + commit if not args.dry_run: - execute('make -C programs clean zstd zstd32 MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version) + execute('make -C programs clean zstd MOREFLAGS="-DZSTD_GIT_COMMIT=%s" && make -B -C programs zstd32 MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % (version, version)) logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt" text_to_send = [] results_files = "" @@ -284,6 +284,7 @@ if __name__ == '__main__': email_topic = '%s:%s ERROR in %s:%s' % (email_header, pid, branch, commit) send_email(args.emails, email_topic, stack, have_mutt, have_mail) print(stack) + time.sleep(args.sleepTime) except KeyboardInterrupt: os.unlink(pidfile) send_email(args.emails, email_header + ':%s test-zstd-speed.py has been stopped' % pid, args.message, have_mutt, have_mail) From 6b68ba20794b51a3e976cb32384844f5af7100c7 Mon Sep 17 00:00:00 2001 From: inikep Date: Fri, 29 Jul 2016 16:45:39 +0200 Subject: [PATCH 4/4] zstd_opt.h: fixed checking of rep codes (2) --- lib/compress/zstd_opt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index 387d62400..9496ed358 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -885,7 +885,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, const U32 repIndex = (U32)(current+cur - opt[cur].rep[i]); const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; - if ((rep[i]<(U32)(ip-prefixStart)) + if ((opt[cur].rep[i]<(U32)(ip-prefixStart)) && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { /* repcode detected */