ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

removed special case all-1 huffman distribution

Browse Source
This commit is contained in:
Yann Collet
2016-07-24 15:35:59 +02:00
parent 7ed5e33b89
commit 38b75ddeb2
3 changed files with 50 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/common/entropy_common.c
+18 -27
View File
@@ -38,10 +38,9 @@
#include "mem.h"
#include "error_private.h" /* ERR_*, ERROR */
#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */
#include "fse.h" /* FSE_isError, FSE_getErrorName */
#include "fse.h"
#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */
#include "huf.h" /* HUF_isError, HUF_getErrorName */
#include "huf.h"
/*-****************************************
@@ -90,7 +89,7 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
threshold = 1<<nbBits;
nbBits++;
while ((remaining>1) && (charnum<=*maxSVPtr)) {
while ((remaining>1) & (charnum<=*maxSVPtr)) {
if (previous0) {
unsigned n0 = charnum;
while ((bitStream & 0xFFFF) == 0xFFFF) {
@@ -115,10 +114,9 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
ip += bitCount>>3;
bitCount &= 7;
bitStream = MEM_readLE32(ip) >> bitCount;
}
else
} else {
bitStream >>= 2;
}
} }
{ short const max = (short)((2*threshold-1)-remaining);
short count;
@@ -148,12 +146,11 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
ip = iend - 4;
}
bitStream = MEM_readLE32(ip) >> (bitCount & 31);
} } /* while ((remaining>1) && (charnum<=*maxSVPtr)) */
if (remaining != 1) return ERROR(GENERIC);
} } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */
if (remaining != 1) return ERROR(corruption_detected);
*maxSVPtr = charnum-1;
ip += (bitCount+7)>>3;
if ((size_t)(ip-istart) > hbSize) return ERROR(srcSize_wrong);
return ip-istart;
}
@@ -162,7 +159,7 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t
Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer.
@return : size read from `src` , or an error Code .
Note : Needed by HUF_readCTable() and HUF_readDTableXn() .
Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
*/
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32* nbSymbolsPtr, U32* tableLogPtr,
@@ -176,22 +173,16 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
/* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */
if (iSize >= (242)) { /* RLE */
static U32 l[14] = { 1, 2, 3, 4, 7, 8, 15, 16, 31, 32, 63, 64, 127, 128 };
oSize = l[iSize-242];
memset(huffWeight, 1, hwSize);
iSize = 0;
} else { /* Incompressible */
oSize = iSize - 127;
iSize = ((oSize+1)/2);
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
if (oSize >= hwSize) return ERROR(corruption_detected);
ip += 1;
{ U32 n;
for (n=0; n<oSize; n+=2) {
huffWeight[n] = ip[n/2] >> 4;
huffWeight[n+1] = ip[n/2] & 15;
} } } }
oSize = iSize - 127;
iSize = ((oSize+1)/2);
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
if (oSize >= hwSize) return ERROR(corruption_detected);
ip += 1;
{ U32 n;
for (n=0; n<oSize; n+=2) {
huffWeight[n] = ip[n/2] >> 4;
huffWeight[n+1] = ip[n/2] & 15;
} } }
else { /* header compressed with FSE (normal case) */
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
lib/compress/huf_compress.c
+19 -48
View File
@@ -105,68 +105,39 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
{
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1];
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
U32 n;
BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
BYTE* op = (BYTE*)dst;
size_t size;
U32 n;
/* check conditions */
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX + 1)
return ERROR(GENERIC);
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
/* convert to weight */
bitsToWeight[0] = 0;
for (n=1; n<=huffLog; n++)
for (n=1; n<huffLog+1; n++)
bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
for (n=0; n<maxSymbolValue; n++)
huffWeight[n] = bitsToWeight[CTable[n].nbBits];
size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue); /* don't need last symbol stat : implied */
if (HUF_isError(size)) return size;
if (size >= 128) return ERROR(GENERIC); /* should never happen, since maxSymbolValue <= 255 */
if ((size <= 1) || (size >= maxSymbolValue/2)) {
if (size==1) { /* RLE */
/* only possible case : series of 1 (because there are at least 2) */
/* can only be 2^n or (2^n-1), otherwise not an huffman tree */
BYTE code;
switch(maxSymbolValue)
{
case 1: code = 0; break;
case 2: code = 1; break;
case 3: code = 2; break;
case 4: code = 3; break;
case 7: code = 4; break;
case 8: code = 5; break;
case 15: code = 6; break;
case 16: code = 7; break;
case 31: code = 8; break;
case 32: code = 9; break;
case 63: code = 10; break;
case 64: code = 11; break;
case 127: code = 12; break;
case 128: code = 13; break;
default : return ERROR(corruption_detected);
}
op[0] = (BYTE)(255-13 + code);
return 1;
}
/* Not compressible */
if (maxSymbolValue > (241-128)) return ERROR(GENERIC); /* not implemented (not possible with current format) */
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
op[0] = (BYTE)(128 /*special case*/ + 0 /* Not Compressible */ + (maxSymbolValue-1));
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
for (n=0; n<maxSymbolValue; n+=2)
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
return ((maxSymbolValue+1)/2) + 1;
}
{ size_t const size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue);
if (FSE_isError(size)) return size;
if ((size>1) & (size < maxSymbolValue/2)) { /* FSE compressed */
op[0] = (BYTE)size;
return size+1;
} }
/* raw values */
if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen */
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1));
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
for (n=0; n<maxSymbolValue; n+=2)
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
return ((maxSymbolValue+1)/2) + 1;
/* normal header case */
op[0] = (BYTE)size;
return size+1;
}
size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
{
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];