ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/Cyan4973/dev' into dev

Browse Source
This commit is contained in:
inikep
2016-05-24 17:15:04 +02:00
parent c8f5509b32 ddbb8e27bf
commit 02c244bf78
33 changed files with 683 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/fse_compress.c
+10 -7
View File
@@ -111,8 +111,6 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned
U32 highThreshold = tableSize-1;
/* CTable header */
tableU16[-2] = (U16) tableLog;
tableU16[-1] = (U16) maxSymbolValue;
@@ -147,10 +145,10 @@ size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned
}
/* Build table */
{ U32 u; for (u=0; u<tableSize; u++) {
{ U32 u; for (u=0; u<tableSize; u++) {
FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */
tableU16[cumul[s]++] = (U16) (tableSize+u); /* TableU16 : sorted by symbol order; gives next state value */
}}
} }
/* Build Symbol Transformation Table */
{ unsigned total = 0;
@@ -444,7 +442,7 @@ FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
return (FSE_CTable*)malloc(size);
}
void FSE_freeCTable (FSE_CTable* ct) { free(ct); }
void FSE_freeCTable (FSE_CTable* ct) { free(ct); }
/* provides the minimum logSize to safely represent a distribution */
static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
@@ -455,9 +453,9 @@ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
return minBits;
}
unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus)
{
U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - 2;
U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus;
U32 tableLog = maxTableLog;
U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
@@ -468,6 +466,11 @@ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
return tableLog;
}
unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
{
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
}
/* Secondary normalization method.
To be used when primary method fails. */
lib/compress/huf_compress.c
+87 -72
View File
@@ -64,7 +64,7 @@
#include <stdio.h> /* printf (debug) */
#include "huf_static.h"
#include "bitstream.h"
#include "fse.h" /* header compression */
#include "fse_static.h" /* header compression */
/* **************************************************************
@@ -73,6 +73,15 @@
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/* **************************************************************
* Utils
****************************************************************/
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
{
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
}
/* *******************************************************
* HUF : Huffman block compression
*********************************************************/
@@ -94,14 +103,14 @@ typedef struct nodeElt_s {
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
{
BYTE bitsToWeight[HUF_MAX_TABLELOG + 1];
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1];
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
U32 n;
BYTE* op = (BYTE*)dst;
size_t size;
/* check conditions */
if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE + 1)
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX + 1)
return ERROR(GENERIC);
/* convert to weight */
@@ -116,7 +125,7 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
if (size >= 128) return ERROR(GENERIC); /* should never happen, since maxSymbolValue <= 255 */
if ((size <= 1) || (size >= maxSymbolValue/2)) {
if (size==1) { /* RLE */
/* only possible case : serie of 1 (because there are at least 2) */
/* only possible case : series of 1 (because there are at least 2) */
/* can only be 2^n or (2^n-1), otherwise not an huffman tree */
BYTE code;
switch(maxSymbolValue)
@@ -159,19 +168,19 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
{
BYTE huffWeight[HUF_MAX_SYMBOL_VALUE + 1];
U32 rankVal[HUF_ABSOLUTEMAX_TABLELOG + 1]; /* large enough for values from 0 to 16 */
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
U32 tableLog = 0;
size_t readSize;
U32 nbSymbols = 0;
//memset(huffWeight, 0, sizeof(huffWeight)); /* is not necessary, even though some analyzer complain ... */
/* get symbol weights */
readSize = HUF_readStats(huffWeight, HUF_MAX_SYMBOL_VALUE+1, rankVal, &nbSymbols, &tableLog, src, srcSize);
readSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize);
if (HUF_isError(readSize)) return readSize;
/* check result */
if (tableLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall);
/* Prepare base value per rank */
@@ -189,12 +198,12 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si
}}
/* fill val */
{ U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0};
U16 valPerRank[HUF_MAX_TABLELOG+1] = {0};
{ U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
{ U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; }
/* determine stating value per rank */
{ U16 min = 0;
U32 n; for (n=HUF_MAX_TABLELOG; n>0; n--) {
U32 n; for (n=HUF_TABLELOG_MAX; n>0; n--) {
valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n];
min >>= 1;
@@ -229,7 +238,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
/* repay normalized cost */
{ U32 const noSymbol = 0xF0F0F0F0;
U32 rankLast[HUF_MAX_TABLELOG+1];
U32 rankLast[HUF_TABLELOG_MAX+1];
int pos;
/* Get pos of last (smallest) symbol per rank */
@@ -253,7 +262,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
if (highTotal <= lowTotal) break;
} }
/* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
while ((nBitsToDecrease<=HUF_MAX_TABLELOG) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
nBitsToDecrease ++;
totalCost -= 1 << (nBitsToDecrease-1);
if (rankLast[nBitsToDecrease-1] == noSymbol)
@@ -312,10 +321,10 @@ static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
}
#define STARTNODE (HUF_MAX_SYMBOL_VALUE+1)
#define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
{
nodeElt huffNode0[2*HUF_MAX_SYMBOL_VALUE+1 +1];
nodeElt huffNode0[2*HUF_SYMBOLVALUE_MAX+1 +1];
nodeElt* huffNode = huffNode0 + 1;
U32 n, nonNullRank;
int lowS, lowN;
@@ -323,8 +332,8 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
U32 nodeRoot;
/* safety checks */
if (maxNbBits == 0) maxNbBits = HUF_DEFAULT_TABLELOG;
if (maxSymbolValue > HUF_MAX_SYMBOL_VALUE) return ERROR(GENERIC);
if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
memset(huffNode0, 0, sizeof(huffNode0));
/* sort, decreasing order */
@@ -360,9 +369,9 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
/* fill result into tree (val, nbBits) */
{ U16 nbPerRank[HUF_MAX_TABLELOG+1] = {0};
U16 valPerRank[HUF_MAX_TABLELOG+1] = {0};
if (maxNbBits > HUF_MAX_TABLELOG) return ERROR(GENERIC); /* check fit into table */
{ U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */
for (n=0; n<=nonNullRank; n++)
nbPerRank[huffNode[n].nbBits]++;
/* determine stating value per rank */
@@ -391,10 +400,10 @@ size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
#define HUF_FLUSHBITS_1(stream) \
if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*2+7) HUF_FLUSHBITS(stream)
if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream)
#define HUF_FLUSHBITS_2(stream) \
if (sizeof((stream)->bitContainer)*8 < HUF_MAX_TABLELOG*4+7) HUF_FLUSHBITS(stream)
if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream)
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
{
@@ -441,44 +450,47 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
{
size_t segmentSize = (srcSize+3)/4; /* first 3 segments */
size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */
const BYTE* ip = (const BYTE*) src;
const BYTE* const iend = ip + srcSize;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
BYTE* op = ostart;
size_t errorCode;
if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */
if (srcSize < 12) return 0; /* no saving possible : too small input */
op += 6; /* jumpTable */
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode==0) return 0;
MEM_writeLE16(ostart, (U16)errorCode);
{ size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
if (HUF_isError(cSize)) return cSize;
if (cSize==0) return 0;
MEM_writeLE16(ostart, (U16)cSize);
op += cSize;
}
ip += segmentSize;
op += errorCode;
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode==0) return 0;
MEM_writeLE16(ostart+2, (U16)errorCode);
{ size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
if (HUF_isError(cSize)) return cSize;
if (cSize==0) return 0;
MEM_writeLE16(ostart+2, (U16)cSize);
op += cSize;
}
ip += segmentSize;
op += errorCode;
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode==0) return 0;
MEM_writeLE16(ostart+4, (U16)errorCode);
{ size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
if (HUF_isError(cSize)) return cSize;
if (cSize==0) return 0;
MEM_writeLE16(ostart+4, (U16)cSize);
op += cSize;
}
ip += segmentSize;
op += errorCode;
errorCode = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode==0) return 0;
{ size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable);
if (HUF_isError(cSize)) return cSize;
if (cSize==0) return 0;
op += cSize;
}
op += errorCode;
return op-ostart;
}
@@ -493,43 +505,46 @@ static size_t HUF_compress_internal (
BYTE* const oend = ostart + dstSize;
BYTE* op = ostart;
U32 count[HUF_MAX_SYMBOL_VALUE+1];
HUF_CElt CTable[HUF_MAX_SYMBOL_VALUE+1];
size_t errorCode;
U32 count[HUF_SYMBOLVALUE_MAX+1];
HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1];
/* checks & inits */
if (srcSize < 1) return 0; /* Uncompressed - note : 1 means rle, so first byte must be correct */
if (dstSize < 1) return 0; /* not compressible within dst budget */
if (srcSize > 128 * 1024) return ERROR(srcSize_wrong); /* current block size limit */
if (huffLog > HUF_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
if (!maxSymbolValue) maxSymbolValue = HUF_MAX_SYMBOL_VALUE;
if (!huffLog) huffLog = HUF_DEFAULT_TABLELOG;
if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */
if (!dstSize) return 0; /* cannot fit within dst budget */
if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */
if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
/* Scan input and build symbol stats */
errorCode = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; }
if (errorCode <= (srcSize >> 7)+1) return 0; /* Heuristic : not compressible enough */
{ size_t const largest = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
if (HUF_isError(largest)) return largest;
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* rle */
if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */
}
/* Build Huffman Tree */
errorCode = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
if (HUF_isError(errorCode)) return errorCode;
huffLog = (U32)errorCode;
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
{ size_t const maxBits = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
if (HUF_isError(maxBits)) return maxBits;
huffLog = (U32)maxBits;
}
/* Write table description header */
errorCode = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode + 12 >= srcSize) return 0; /* not useful to try compression */
op += errorCode;
{ size_t const hSize = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog);
if (HUF_isError(hSize)) return hSize;
if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */
op += hSize;
}
/* Compress */
if (singleStream)
errorCode = HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable); /* single segment */
else
errorCode = HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
if (HUF_isError(errorCode)) return errorCode;
if (errorCode==0) return 0;
op += errorCode;
{ size_t const cSize = (singleStream) ?
HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : /* single segment */
HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
if (HUF_isError(cSize)) return cSize;
if (cSize==0) return 0; /* uncompressible */
op += cSize;
}
/* check compressibility */
if ((size_t)(op-ostart) >= srcSize-1)
@@ -556,5 +571,5 @@ size_t HUF_compress2 (void* dst, size_t dstSize,
size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_DEFAULT_TABLELOG);
return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT);
}
lib/compress/zbuff_compress.c
+1 -1
View File
@@ -81,7 +81,7 @@ static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE;
typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage;
/* *** Ressources *** */
/* *** Resources *** */
struct ZBUFF_CCtx_s {
ZSTD_CCtx* zc;
char* inBuff;
lib/compress/zstd_compress.c
+8 -11
View File
@@ -973,7 +973,7 @@ static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLim
const BYTE* const pStart = pIn;
while ((pIn<pInLimit-(sizeof(size_t)-1))) {
size_t diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
pIn += ZSTD_NbCommonBytes(diff);
return (size_t)(pIn - pStart);
@@ -990,10 +990,8 @@ static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLim
*/
static size_t ZSTD_count_2segments(const BYTE* ip, const BYTE* match, const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart)
{
size_t matchLength;
const BYTE* vEnd = ip + (mEnd - match);
if (vEnd > iEnd) vEnd = iEnd;
matchLength = ZSTD_count(ip, match, vEnd);
const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd);
size_t matchLength = ZSTD_count(ip, match, vEnd);
if (match + matchLength == mEnd)
matchLength += ZSTD_count(ip+matchLength, iStart, iEnd);
return matchLength;
@@ -2464,7 +2462,7 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
/*-===== Pre-defined compression levels =====-*/
#define ZSTD_DEFAULT_CLEVEL 5
#define ZSTD_DEFAULT_CLEVEL 1
#define ZSTD_MAX_CLEVEL 22
unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
@@ -2575,17 +2573,16 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
},
};
/*! ZSTD_getParams() :
* @return ZSTD_parameters structure for a selected compression level and srcSize.
* `srcSize` value is optional, select 0 if not known */
/*! ZSTD_getCParams() :
* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`.
* Size values are optional, provide 0 if not known or unused */
ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize)
{
ZSTD_compressionParameters cp;
size_t const addedSize = srcSize ? 0 : 500;
U64 const rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1;
U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
if (compressionLevel < 0) compressionLevel = ZSTD_DEFAULT_CLEVEL;
if (compressionLevel==0) compressionLevel = 1;
if (compressionLevel <= 0) compressionLevel = ZSTD_DEFAULT_CLEVEL; /* 0 == default; no negative compressionLevel yet */
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
cp = ZSTD_defaultCParameters[tableID][compressionLevel];
if (MEM_32bits()) { /* auto-correction, for 32-bits mode */