ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/dev' into fixes

Browse Source 
* upstream/dev:
  added doc\zstd_manual.html
  added contrib\gen_html
  zstd_compression_format.md moved to doc/
  Fix small bug in ZSTD_execSequence()
  improved ZSTD_compressBlock_opt_extDict_generic
  protect ZSTD_decodeFrameHeader() from invalid usage, as suggested by @spaskob
  zstd_opt.h: small improvement in compression ratio
  improved dicitonary segment merge
  use implicit rules to compile zstd_decompress.c
  detect early impossible decompression scenario in legacy decoder v0.5
  no repeat mode in legacy v0.5
  fixed invalid invocation of dictionary in legacy decoder v0.5
  fix edge case
  fix command line interpretation
  fixed minor corner case
  zstd.h: added the Introduction section
  fixed clang 3.5 warnings
  zstd.h: updated comments
This commit is contained in:
Nick Terrell
2016-10-24 13:10:13 -07:00
parent f9c9af3c2e 4239a207dd
commit f698ad6deb
20 changed files with 1009 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/huf_compress.c
+4 -3
View File
@@ -155,13 +155,14 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si
} }
/* fill val */
{ U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0};
{ U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */
U16 valPerRank[HUF_TABLELOG_MAX+2] = {0};
{ U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; }
/* determine stating value per rank */
valPerRank[tableLog+1] = 0; /* for w==0 */
{ U16 min = 0;
U32 n; for (n=HUF_TABLELOG_MAX; n>0; n--) {
valPerRank[n] = min; /* get starting value within each rank */
U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */
valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n];
min >>= 1;
} }
lib/compress/zstd_compress.c
+1 -1
View File
@@ -377,7 +377,7 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
* Block entropic compression
*********************************************************/
/* See zstd_compression_format.md for detailed format description */
/* See doc/zstd_compression_format.md for detailed format description */
size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
lib/compress/zstd_opt.h
+29 -35
View File
@@ -1,5 +1,5 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
@@ -16,6 +16,7 @@
#define ZSTD_FREQ_DIV 5
#define ZSTD_MAX_PRICE (1<<30)
/*-*************************************
* Price functions for optimal parser
@@ -171,7 +172,7 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \
{ \
while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \
while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } \
opt[pos].mlen = mlen_; \
opt[pos].off = offset_; \
opt[pos].litlen = litlen_; \
@@ -401,7 +402,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
ZSTD_rescaleFreqs(seqStorePtr);
ip += (ip==prefixStart);
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; }
//inr = ip;
/* Match Loop */
while (ip < ilimit) {
@@ -496,7 +496,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
}
best_mlen = minMatch;
best_mlen = minMatch;
{ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
for (i=(opt[cur].mlen != 1); i<last_i; i++) { /* check rep */
const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (opt[cur].mlen != 1)) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
@@ -510,21 +510,20 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
}
best_off = i - (opt[cur].mlen != 1);
if (mlen > best_mlen) best_mlen = mlen;
if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen;
if (cur > litlen) {
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH);
} else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
} else {
litlen = 0;
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH);
}
do {
if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen;
if (cur > litlen) {
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH);
} else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
} else {
litlen = 0;
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH);
}
if (mlen > best_mlen) best_mlen = mlen;
do {
if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
SET_PRICE(cur + mlen, mlen, i, litlen, price);
mlen--;
@@ -657,7 +656,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
ctx->nextToUpdate3 = ctx->nextToUpdate;
ZSTD_rescaleFreqs(seqStorePtr);
ip += (ip==prefixStart);
//inr = ip;
/* Match Loop */
while (ip < ilimit) {
@@ -666,7 +664,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
U32 current = (U32)(ip-base);
memset(opt, 0, sizeof(ZSTD_optimal_t));
last_pos = 0;
//inr = ip;
opt[0].litlen = (U32)(ip - anchor);
/* check repCode */
@@ -765,8 +762,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
}
best_mlen = 0;
best_mlen = minMatch;
{ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
for (i = (mlen != 1); i<last_i; i++) {
const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (opt[cur].mlen != 1)) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
@@ -786,20 +782,20 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
}
best_off = i - (opt[cur].mlen != 1);
if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen;
if (cur > litlen) {
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH);
} else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
} else {
litlen = 0;
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH);
}
best_mlen = mlen;
if (mlen > best_mlen) best_mlen = mlen;
do {
if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen;
if (cur > litlen) {
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH);
} else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
} else {
litlen = 0;
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH);
}
if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
SET_PRICE(cur + mlen, mlen, i, litlen, price);
mlen--;
@@ -815,8 +811,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
goto _storeSequence;
}
best_mlen = (best_mlen > minMatch) ? best_mlen : minMatch;
/* set prices using matches at position = cur */
for (u = 0; u < match_num; u++) {
mlen = (u>0) ? matches[u-1].len+1 : best_mlen;