diff --git a/Makefile b/Makefile index 9572497d8..45782af9f 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,7 @@ armtest: clean ppctest: clean $(MAKE) -C $(PRGDIR) datagen # use native, faster - $(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-static" + $(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" ppc64test: clean $(MAKE) -C $(PRGDIR) datagen # use native, faster diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index f801959e7..61c5354b6 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -52,8 +52,9 @@ * Common constants ***************************************/ #define ZSTD_OPT_DEBUG 0 /* 3 = compression stats; 5 = check encoded sequences; 9 = full logs */ -#include #if defined(ZSTD_OPT_DEBUG) && ZSTD_OPT_DEBUG>=9 + #include + #include #define ZSTD_LOG_PARSER(...) printf(__VA_ARGS__) #define ZSTD_LOG_ENCODE(...) printf(__VA_ARGS__) #define ZSTD_LOG_BLOCK(...) printf(__VA_ARGS__) @@ -66,8 +67,8 @@ #define ZSTD_OPT_NUM (1<<12) #define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7 */ -#define ZSTD_REP_NUM 3 -#define ZSTD_REP_INIT ZSTD_REP_NUM +#define ZSTD_REP_NUM 3 /* number of repcodes */ +#define ZSTD_REP_CHECK (ZSTD_REP_NUM-0) /* number of repcodes to check by the optimal parser */ #define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; @@ -169,7 +170,7 @@ typedef struct { U32 off; U32 mlen; U32 litlen; - U32 rep[ZSTD_REP_INIT]; + U32 rep[ZSTD_REP_NUM]; } ZSTD_optimal_t; #if ZSTD_OPT_DEBUG == 3 diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index ef394f198..9496ed358 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -464,13 +464,14 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, ZSTD_optimal_t* opt = seqStorePtr->priceTable; ZSTD_match_t* matches = seqStorePtr->matchTable; const BYTE* inr; - U32 offset, rep[ZSTD_REP_INIT]; + U32 offset, rep[ZSTD_REP_NUM]; /* init */ ctx->nextToUpdate3 = ctx->nextToUpdate; ZSTD_rescaleFreqs(seqStorePtr); ip += (ip==prefixStart); - { U32 i; for (i=0; irep[i]; } + { U32 i; for (i=0; irep[i]; } + inr = ip; ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_GENERIC srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len); @@ -484,7 +485,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, /* check repCode */ { U32 i; - for (i=0; i sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { - ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); best_mlen = mlen; best_off = i; last_pos = cur + 1; + ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); goto _storeSequence; } @@ -755,12 +756,13 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, const BYTE* inr; /* init */ - U32 offset, rep[ZSTD_REP_INIT]; - { U32 i; for (i=0; irep[i]; } + U32 offset, rep[ZSTD_REP_NUM]; + { U32 i; for (i=0; irep[i]; } ctx->nextToUpdate3 = ctx->nextToUpdate; ZSTD_rescaleFreqs(seqStorePtr); ip += (ip==prefixStart); + inr = ip; ZSTD_LOG_BLOCK("%d: COMPBLOCK_OPT_EXTDICT srcSz=%d maxSrch=%d mls=%d sufLen=%d\n", (int)(ip-base), (int)srcSize, maxSearches, mls, sufficient_len); @@ -776,11 +778,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, /* check repCode */ { U32 i; - for (i=0; i= 3) & (repIndex>lowestIndex)) /* intentional overflow */ + if ((rep[i]<(U32)(ip-prefixStart)) + && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { /* repcode detected we should take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; @@ -807,7 +810,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, ZSTD_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-base), match_num, last_pos); if (!last_pos && !match_num) { ip++; continue; } - { U32 i; for (i=0; i sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { @@ -878,11 +881,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, best_mlen = 0; { U32 i; - for (i=0; i= 3) & (repIndex>lowestIndex)) /* intentional overflow */ + if ((opt[cur].rep[i]<(U32)(ip-prefixStart)) + && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { /* repcode detected */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; @@ -890,8 +894,8 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, ZSTD_LOG_PARSER("%d: Found REP %d/%d mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), i, ZSTD_REP_NUM, mlen, i, opt[cur].rep[i], cur, opt[cur].off); if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { - ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); best_mlen = mlen; best_off = i; last_pos = cur + 1; + ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); goto _storeSequence; } @@ -1016,7 +1020,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ ml2 = ZSTD_count_2segments(ip, match, iend, dictEnd, prefixStart); ZSTD_LOG_PARSER("%d: ZSTD_count_2segments=%d offset=%d dictBase=%p dictEnd=%p prefixStart=%p ip=%p match=%p\n", (int)current, (int)ml2, (int)best_off, dictBase, dictEnd, prefixStart, ip, match); } - else ml2 = (U32)ZSTD_count(ip, ip-offset, iend); + else ml2 = (U32)ZSTD_count(ip, ip-best_off, iend); } else ml2 = (U32)ZSTD_count(ip, ip-rep[0], iend); if ((offset >= 8) && (ml2 < mlen || ml2 < minMatch)) { @@ -1033,7 +1037,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ } } /* for (cur=0; cur < last_pos; ) */ /* Save reps for next block */ - ctx->savedRep[0] = rep[0]; ctx->savedRep[1] = rep[1]; ctx->savedRep[2] = rep[2]; + { int i; for (i=0; isavedRep[i] = rep[i]; } /* Last Literals */ { size_t lastLLSize = iend - anchor; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 972a8143d..e1ac20049 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -119,7 +119,7 @@ struct ZSTD_DCtx_s const void* vBase; const void* dictEnd; size_t expected; - U32 rep[3]; + U32 rep[ZSTD_REP_NUM]; ZSTD_frameParams fParams; blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ ZSTD_dStage stage; @@ -583,7 +583,7 @@ typedef struct { FSE_DState_t stateLL; FSE_DState_t stateOffb; FSE_DState_t stateML; - size_t prevOffset[ZSTD_REP_INIT]; + size_t prevOffset[ZSTD_REP_NUM]; } seqState_t; @@ -763,7 +763,7 @@ static size_t ZSTD_decompressSequences( if (nbSeq) { seqState_t seqState; dctx->fseEntropy = 1; - { U32 i; for (i=0; irep[i]; } + { U32 i; for (i=0; irep[i]; } { size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip); if (ERR_isError(errorCode)) return ERROR(corruption_detected); } FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL); @@ -781,7 +781,7 @@ static size_t ZSTD_decompressSequences( /* check if reached exact end */ if (nbSeq) return ERROR(corruption_detected); /* save reps for next block */ - { U32 i; for (i=0; irep[i] = (U32)(seqState.prevOffset[i]); } + { U32 i; for (i=0; irep[i] = (U32)(seqState.prevOffset[i]); } } /* last literal segment */ diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py index a837cf3a8..562d0790c 100755 --- a/tests/test-zstd-speed.py +++ b/tests/test-zstd-speed.py @@ -169,7 +169,7 @@ def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, hav local_branch = string.split(branch, '/')[1] version = local_branch.rpartition('-')[2] + '_' + commit if not args.dry_run: - execute('make -C programs clean zstd zstd32 MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version) + execute('make -C programs clean zstd MOREFLAGS="-DZSTD_GIT_COMMIT=%s" && make -B -C programs zstd32 MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % (version, version)) logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt" text_to_send = [] results_files = "" @@ -284,6 +284,7 @@ if __name__ == '__main__': email_topic = '%s:%s ERROR in %s:%s' % (email_header, pid, branch, commit) send_email(args.emails, email_topic, stack, have_mutt, have_mail) print(stack) + time.sleep(args.sleepTime) except KeyboardInterrupt: os.unlink(pidfile) send_email(args.emails, email_header + ':%s test-zstd-speed.py has been stopped' % pid, args.message, have_mutt, have_mail)