From b59bf963a8afd634769976c3e89affeddd170da8 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 4 Apr 2016 14:53:16 +0200 Subject: [PATCH 1/4] fixed Visual conversion warnings --- lib/zstd_compress.c | 2 +- lib/zstd_static.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 219b13f82..fcb6e36bc 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -182,7 +182,7 @@ void ZSTD_adjustCParams(ZSTD_compressionParameters* params, U64 srcSize, size_t /* resize params, to use less memory when necessary */ { U32 const minSrcSize = (srcSize==0) ? 500 : 0; U64 const rSize = srcSize + dictSize + minSrcSize; - if (rSize < (1<windowLog > srcLog) params->windowLog = srcLog; } } diff --git a/lib/zstd_static.h b/lib/zstd_static.h index 2f11ceeb6..333861255 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -57,7 +57,7 @@ extern "C" { /*-************************************* * Types ***************************************/ -#define ZSTD_WINDOWLOG_MAX (MEM_32bits() ? 25 : 27) +#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? 25 : 27)) #define ZSTD_WINDOWLOG_MIN 18 #define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1) #define ZSTD_CHAINLOG_MIN 4 From 485371600a99473d68006f923ded85d9ff8be27b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 7 Apr 2016 15:24:29 +0200 Subject: [PATCH 2/4] Implemented default coding table for lengths and offset --- lib/zstd_compress.c | 24 ++++++++++++------------ lib/zstd_decompress.c | 36 ++++-------------------------------- lib/zstd_internal.h | 21 ++++++++++++--------- lib/zstd_opt.h | 6 +++--- 4 files changed, 31 insertions(+), 56 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index f06f55639..0d84176b3 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -224,7 +224,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, const size_t tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); /* Check if workSpace is large enough, alloc a new one if needed */ - { size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (1<seqStore.litLengthFreq = zc->seqStore.litFreq + (1<seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1); zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1); - zc->seqStore.matchTable = (ZSTD_match_t*)((void*)(zc->seqStore.offCodeFreq + (1<seqStore.matchTable = (ZSTD_match_t*)((void*)(zc->seqStore.offCodeFreq + (MaxOff+1))); zc->seqStore.priceTable = (ZSTD_optimal_t*)((void*)(zc->seqStore.matchTable + ZSTD_OPT_NUM+1)); zc->seqStore.buffer = zc->seqStore.priceTable + ZSTD_OPT_NUM+1; zc->seqStore.litLengthSum = 0; @@ -738,8 +738,8 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc, Offtype = FSE_ENCODING_RLE; } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { Offtype = FSE_ENCODING_STATIC; - } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (Offbits-1)))) { - FSE_buildCTable_raw(CTable_OffsetBits, Offbits); + } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog-1)))) { + FSE_buildCTable(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog); Offtype = FSE_ENCODING_RAW; } else { size_t nbSeq_1 = nbSeq; @@ -1287,7 +1287,7 @@ U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip) U32 idx = zc->nextToUpdate3; const U32 target = zc->nextToUpdate3 = (U32)(ip - base); const size_t hash3 = ZSTD_hash3Ptr(ip, hashLog3); - + while(idx < target) { hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; idx++; @@ -1335,9 +1335,9 @@ size_t ZSTD_HcFindBestMatch_generic ( } /* save best solution */ - if (currentMl > ml) { - ml = currentMl; *offsetPtr = ZSTD_REP_MOVE + current - matchIndex3; - if (ip+currentMl == iLimit) return (ml>=MINMATCH) ? ml : 0; /* best possible, and avoid read overflow*/ + if (currentMl > ml) { + ml = currentMl; *offsetPtr = ZSTD_REP_MOVE + current - matchIndex3; + if (ip+currentMl == iLimit) return (ml>=MINMATCH) ? ml : 0; /* best possible, and avoid read overflow*/ } } } #endif @@ -1752,7 +1752,7 @@ _storeSequence: ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, matchLength-MINMATCH); anchor = ip = start + matchLength; } - + /* check immediate repcode */ while ( (ip <= ilimit) && (MEM_read32(ip) == MEM_read32(ip - rep[1])) ) { @@ -1841,7 +1841,7 @@ void ZSTD_compressBlock_greedy_extDict_generic(ZSTD_CCtx* ctx, const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } rep[1] = rep[0]; rep[0] = (U32)(offset - ZSTD_REP_MOVE); - } + } _storeSequence: /* store sequence */ @@ -2206,7 +2206,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, } /* store sequence */ - { + { if (offset >= ZSTD_REP_NUM) { rep[2] = rep[1]; rep[1] = rep[0]; @@ -2527,7 +2527,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) { - U32 hashLog3 = (pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN); + U32 hashLog3 = (pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN); zc->hashLog3 = (params.cParams.searchLength==3) ? hashLog3 : 0; // printf("windowLog=%d hashLog=%d hashLog3=%d \n", params.windowLog, params.hashLog, zc->hashLog3); diff --git a/lib/zstd_decompress.c b/lib/zstd_decompress.c index 2dbe5ce89..60595b45c 100644 --- a/lib/zstd_decompress.c +++ b/lib/zstd_decompress.c @@ -500,34 +500,6 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, @return : nb bytes read from src, or an error code if it fails, testable with ZSTD_isError() */ -FORCE_INLINE size_t ZSTD_buildSeqTableOff(FSE_DTable* DTable, U32 type, U32 rawBits, U32 maxLog, - const void* src, size_t srcSize) -{ - U32 max = (1< max, data is corrupted */ - return 1; - case FSE_ENCODING_RAW : - FSE_buildDTable_raw(DTable, rawBits); - return 0; - case FSE_ENCODING_STATIC: - return 0; - default : /* impossible */ - case FSE_ENCODING_DYNAMIC : - { U32 tableLog; - S16 norm[MaxSeq+1]; - size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); - if (FSE_isError(headerSize)) return ERROR(corruption_detected); - if (tableLog > maxLog) return ERROR(corruption_detected); - FSE_buildDTable(DTable, norm, max, tableLog); - return headerSize; - } } -} - - FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, U32 type, U32 max, U32 maxLog, const void* src, size_t srcSize, const S16* defaultNorm, U32 defaultLog) @@ -594,7 +566,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr, if (ZSTD_isError(bhSize)) return ERROR(corruption_detected); ip += bhSize; } - { size_t const bhSize = ZSTD_buildSeqTableOff(DTableOffb, Offtype, Offbits, OffFSELog, ip, iend-ip); + { size_t const bhSize = ZSTD_buildSeqTable(DTableOffb, Offtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog); if (ZSTD_isError(bhSize)) return ERROR(corruption_detected); ip += bhSize; } @@ -650,7 +622,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, - 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, /*fake*/ 1, 1, 1, 1, 1 }; + 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, /*fake*/ 1, 1 }; /* sequence */ { size_t offset; @@ -660,7 +632,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) offset = OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits); /* <= 26 bits */ if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); } - + if (offset < ZSTD_REP_NUM) { if (llCode == 0 && offset <= 1) offset = 1-offset; @@ -678,7 +650,7 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState) } else { offset -= ZSTD_REP_MOVE; seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[0] = offset; } seq->offset = offset; diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 34d8b5441..9e6ffcafb 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -104,11 +104,10 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t; #define REPCODE_STARTVALUE 1 #define Litbits 8 -#define Offbits 5 #define MaxLit ((1<litSum = (2<litLengthSum = MaxLL+1; ssPtr->matchLengthSum = MaxML+1; - ssPtr->offCodeSum = (1<offCodeSum = (MaxOff+1); ssPtr->matchSum = (2<= ZSTD_REP_NUM) { opt[cur].rep[2] = opt[cur-mlen].rep[1]; opt[cur].rep[1] = opt[cur-mlen].rep[0]; - opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } else { opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; @@ -838,7 +838,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, if (opt[cur].off >= ZSTD_REP_NUM) { opt[cur].rep[2] = opt[cur-mlen].rep[1]; opt[cur].rep[1] = opt[cur-mlen].rep[0]; - opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE; ZSTD_LOG_ENCODE("%d: COPYREP_OFF cur=%d mlen=%d rep[0]=%d rep[1]=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep[0], opt[cur].rep[1]); } else { opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; From 5d39357bc4a030819778e2ec054825cb72a70af3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 7 Apr 2016 17:19:00 +0200 Subject: [PATCH 3/4] minor compression speed optimization --- lib/zstd_compress.c | 29 +++++++++++++++++------------ lib/zstd_internal.h | 3 ++- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 0d84176b3..e6b13ca3e 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -80,6 +80,7 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr) ssPtr->lit = ssPtr->litStart; ssPtr->litLength = ssPtr->litLengthStart; ssPtr->matchLength = ssPtr->matchLengthStart; + ssPtr->longLengthID = 0; } @@ -620,14 +621,16 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq) 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24 }; const BYTE LL_deltaCode = 19; - U16* const llTable = seqStorePtr->litLengthStart; + const U16* const llTable = seqStorePtr->litLengthStart; BYTE* const llCodeTable = seqStorePtr->llCodeStart; size_t u; for (u=0; ulongLength; llTable[u] = (U16)ll; } + U32 const ll = llTable[u]; llCodeTable[u] = (ll>63) ? (BYTE)ZSTD_highbit(ll) + LL_deltaCode : LL_Code[ll]; - } } + } + if (seqStorePtr->longLengthID==1) + llCodeTable[seqStorePtr->longLengthPos] = MaxLL; + } /* Offset codes */ { const U32* const offsetTable = seqStorePtr->offsetStart; @@ -646,14 +649,16 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq) 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; const BYTE ML_deltaCode = 36; - U16* const mlTable = seqStorePtr->matchLengthStart; + const U16* const mlTable = seqStorePtr->matchLengthStart; BYTE* const mlCodeTable = seqStorePtr->mlCodeStart; size_t u; for (u=0; ulongLength; mlTable[u] = (U16)ml; } + U32 const ml = mlTable[u]; mlCodeTable[u] = (ml>127) ? (BYTE)ZSTD_highbit(ml) + ML_deltaCode : ML_Code[ml]; - } } + } + if (seqStorePtr->longLengthID==2) + mlCodeTable[seqStorePtr->longLengthPos] = MaxML; + } } @@ -864,15 +869,15 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const B seqStorePtr->lit += litLength; /* literal Length */ - if (litLength>=65535) { *(seqStorePtr->litLength++) = 65535; seqStorePtr->longLength = (U32)litLength; } - else *seqStorePtr->litLength++ = (U16)litLength; + if (litLength>0xFFFF) { seqStorePtr->longLengthID = 1; seqStorePtr->longLengthPos = (U32)(seqStorePtr->litLength - seqStorePtr->litLengthStart); } + *seqStorePtr->litLength++ = (U16)litLength; /* match offset */ *(seqStorePtr->offset++) = (U32)offsetCode + 1; /* match Length */ - if (matchCode>=65535) { *(seqStorePtr->matchLength++) = 65535; seqStorePtr->longLength = (U32)matchCode; } - else *seqStorePtr->matchLength++ = (U16)matchCode; + if (matchCode>0xFFFF) { seqStorePtr->longLengthID = 2; seqStorePtr->longLengthPos = (U32)(seqStorePtr->matchLength - seqStorePtr->matchLengthStart); } + *seqStorePtr->matchLength++ = (U16)matchCode; } diff --git a/lib/zstd_internal.h b/lib/zstd_internal.h index 9e6ffcafb..31a99cbde 100644 --- a/lib/zstd_internal.h +++ b/lib/zstd_internal.h @@ -222,7 +222,8 @@ typedef struct { U16* matchLengthStart; U16* matchLength; BYTE* mlCodeStart; - U32 longLength; + U32 longLengthID; /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */ + U32 longLengthPos; /* opt */ ZSTD_optimal_t* priceTable; ZSTD_match_t* matchTable; From d7bee33caf22703dad5233fbcff5d52c03d984a5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 7 Apr 2016 18:16:42 +0200 Subject: [PATCH 4/4] Fixed make install (reported by Oliver Lange) --- lib/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 6bdf2f876..ce66ebbe0 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -118,8 +118,8 @@ install: libzstd libzstd.pc @cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h - @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h - @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zdict.h + @install -m 644 zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h + @install -m 644 zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd static and shared library installed uninstall: @@ -129,8 +129,8 @@ uninstall: @[ -x $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) ] && rm -f $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) @[ -f $(DESTDIR)$(LIBDIR)/libzstd.a ] && rm -f $(DESTDIR)$(LIBDIR)/libzstd.a @[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zstd.h - @[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zbuff.h - @[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zdict.h + @[ -f $(DESTDIR)$(INCLUDEDIR)/zbuff.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zbuff.h + @[ -f $(DESTDIR)$(INCLUDEDIR)/zdict.h ] && rm -f $(DESTDIR)$(INCLUDEDIR)/zdict.h @echo zstd libraries successfully uninstalled endif