add basic tests

This commit is contained in:
Yann Collet
2016-01-28 15:39:52 +01:00
parent b31de73435
commit 9cadd0853c
8 changed files with 256 additions and 298 deletions
+4
View File
@@ -36,6 +36,7 @@
PRGDIR = programs PRGDIR = programs
ZSTDDIR = lib ZSTDDIR = lib
DICTDIR = dictBuilder
# Define nul output # Define nul output
ifneq (,$(filter Windows%,$(OS))) ifneq (,$(filter Windows%,$(OS)))
@@ -51,6 +52,7 @@ default: zstdprogram
all: all:
$(MAKE) -C $(ZSTDDIR) $@ $(MAKE) -C $(ZSTDDIR) $@
$(MAKE) -C $(PRGDIR) $@ $(MAKE) -C $(PRGDIR) $@
$(MAKE) -C $(DICTDIR) $@
zstdprogram: zstdprogram:
$(MAKE) -C $(PRGDIR) $(MAKE) -C $(PRGDIR)
@@ -58,6 +60,7 @@ zstdprogram:
clean: clean:
@$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) @$(MAKE) -C $(ZSTDDIR) $@ > $(VOID)
@$(MAKE) -C $(PRGDIR) $@ > $(VOID) @$(MAKE) -C $(PRGDIR) $@ > $(VOID)
@$(MAKE) -C $(DICTDIR) $@ > $(VOID)
@echo Cleaning completed @echo Cleaning completed
@@ -78,6 +81,7 @@ travis-install:
test: test:
$(MAKE) -C $(PRGDIR) $@ $(MAKE) -C $(PRGDIR) $@
$(MAKE) -C $(DICTDIR) $@
cmaketest: cmaketest:
cd contrib/cmake ; cmake . ; $(MAKE) cd contrib/cmake ; cmake . ; $(MAKE)
+3 -1
View File
@@ -55,7 +55,9 @@ clean:
dictBuilder$(EXT) dictBuilder$(EXT)
@echo Cleaning completed @echo Cleaning completed
test: clean dictBuilder test: dictBuilder
./dictBuilder *
@rm dictionary
clangtest: CC = clang clangtest: CC = clang
clangtest: CFLAGS += -Werror clangtest: CFLAGS += -Werror
+1 -1
View File
@@ -153,7 +153,7 @@ int main(int argCount, const char** argv)
operationResult=0, operationResult=0,
nextArgumentIsMaxDict=0, nextArgumentIsMaxDict=0,
nextArgumentIsDictFileName=0; nextArgumentIsDictFileName=0;
size_t maxDictSize = maxDictSizeDefault; unsigned maxDictSize = maxDictSizeDefault;
unsigned selectionLevel = selectionLevelDefault; unsigned selectionLevel = selectionLevelDefault;
const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */ const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
unsigned filenameIdx = 0; unsigned filenameIdx = 0;
+12 -12
View File
@@ -419,7 +419,7 @@ static dictItem DiB_analyzePos(
/* largest useful length */ /* largest useful length */
memset(cumulLength, 0, sizeof(cumulLength)); memset(cumulLength, 0, sizeof(cumulLength));
cumulLength[maxLength-1] = lengthList[maxLength-1]; cumulLength[maxLength-1] = lengthList[maxLength-1];
for (i=maxLength-2; i>=0; i--) for (i=(int)(maxLength-2); i>=0; i--)
cumulLength[i] = cumulLength[i+1] + lengthList[i]; cumulLength[i] = cumulLength[i+1] + lengthList[i];
for (i=LLIMIT-1; i>=MINMATCHLENGTH; i--) if (cumulLength[i]>=minRatio) break; for (i=LLIMIT-1; i>=MINMATCHLENGTH; i--) if (cumulLength[i]>=minRatio) break;
@@ -427,7 +427,7 @@ static dictItem DiB_analyzePos(
/* reduce maxLength in case of final into repetitive data */ /* reduce maxLength in case of final into repetitive data */
{ {
U32 l = maxLength; U32 l = (U32)maxLength;
BYTE c = b[pos + maxLength-1]; BYTE c = b[pos + maxLength-1];
while (b[pos+l-2]==c) l--; while (b[pos+l-2]==c) l--;
maxLength = l; maxLength = l;
@@ -442,8 +442,8 @@ static dictItem DiB_analyzePos(
DISPLAYLEVEL(4, "Selected ref at position %u, of length %u : saves %u (ratio: %.2f) \n", DISPLAYLEVEL(4, "Selected ref at position %u, of length %u : saves %u (ratio: %.2f) \n",
(U32)pos, (U32)maxLength, savings[maxLength], (double)savings[maxLength] / maxLength); (U32)pos, (U32)maxLength, savings[maxLength], (double)savings[maxLength] / maxLength);
solution.pos = pos; solution.pos = (U32)pos;
solution.length = maxLength; solution.length = (U32)maxLength;
solution.savings = savings[maxLength]; solution.savings = savings[maxLength];
/* mark positions done */ /* mark positions done */
@@ -459,7 +459,7 @@ static dictItem DiB_analyzePos(
length = DiB_count(b+pos, b+testedPos); length = DiB_count(b+pos, b+testedPos);
if (length > solution.length) length = solution.length; if (length > solution.length) length = solution.length;
} }
pEnd = testedPos + length; pEnd = (U32)(testedPos + length);
for (p=testedPos; p<pEnd; p++) for (p=testedPos; p<pEnd; p++)
doneMarks[p] = 1; doneMarks[p] = 1;
} } } } } }
@@ -522,7 +522,7 @@ static void DiB_removeDictItem(dictItem* table, U32 id)
U32 max = table->pos; U32 max = table->pos;
U32 u; U32 u;
if (!id) return; /* protection, should never happen */ if (!id) return; /* protection, should never happen */
for (u=id; u<max; u++) for (u=id; u<max-1; u++)
table[u] = table[u+1]; table[u] = table[u+1];
table->pos--; table->pos--;
} }
@@ -570,7 +570,7 @@ static U32 DiB_dictSize(const dictItem* dictList)
static void DiB_trainBuffer(dictItem* dictList, U32 dictListSize, static void DiB_trainBuffer(dictItem* dictList, U32 dictListSize,
const void* const buffer, const size_t bufferSize, /* buffer must end with noisy guard band */ const void* const buffer, const size_t bufferSize, /* buffer must end with noisy guard band */
const char* displayName, const char* displayName,
const size_t* fileSizes, size_t nbFiles, unsigned maxDictSize, const size_t* fileSizes, unsigned nbFiles, unsigned maxDictSize,
U32 shiftRatio) U32 shiftRatio)
{ {
saidx_t* const suffix0 = (saidx_t*)malloc((bufferSize+2)*sizeof(*suffix0)); saidx_t* const suffix0 = (saidx_t*)malloc((bufferSize+2)*sizeof(*suffix0));
@@ -590,19 +590,19 @@ static void DiB_trainBuffer(dictItem* dictList, U32 dictListSize,
/* sort */ /* sort */
DISPLAYLEVEL(2, "sorting %s ...\n", displayName); DISPLAYLEVEL(2, "sorting %s ...\n", displayName);
errorCode = divsufsort((const sauchar_t*)buffer, suffix, bufferSize); errorCode = divsufsort((const sauchar_t*)buffer, suffix, (saidx_t)bufferSize);
if (errorCode != 0) EXM_THROW(2, "sort failed"); if (errorCode != 0) EXM_THROW(2, "sort failed");
suffix[bufferSize] = bufferSize; /* leads into noise */ suffix[bufferSize] = (saidx_t)bufferSize; /* leads into noise */
suffix0[0] = bufferSize; /* leads into noise */ suffix0[0] = (saidx_t)bufferSize; /* leads into noise */
{ {
/* build reverse suffix sort */ /* build reverse suffix sort */
size_t pos; size_t pos;
for (pos=0; pos < bufferSize; pos++) for (pos=0; pos < bufferSize; pos++)
reverseSuffix[suffix[pos]] = pos; reverseSuffix[suffix[pos]] = (U32)pos;
/* build file pos */ /* build file pos */
filePos[0] = 0; filePos[0] = 0;
for (pos=1; pos<nbFiles; pos++) for (pos=1; pos<nbFiles; pos++)
filePos[pos] = filePos[pos-1] + fileSizes[pos-1]; filePos[pos] = (U32)(filePos[pos-1] + fileSizes[pos-1]);
} }
DISPLAYLEVEL(2, "finding patterns ... \n"); DISPLAYLEVEL(2, "finding patterns ... \n");
+6
View File
@@ -24,6 +24,12 @@
* OTHER DEALINGS IN THE SOFTWARE. * OTHER DEALINGS IN THE SOFTWARE.
*/ */
/*- Compiler specifics -*/
#ifdef __clang__
#pragma clang diagnostic ignored "-Wshorten-64-to-32"
#endif
/*- Dependencies -*/
#include "divsufsort_private.h" #include "divsufsort_private.h"
#ifdef _OPENMP #ifdef _OPENMP
# include <omp.h> # include <omp.h>
+6
View File
@@ -24,6 +24,12 @@
* OTHER DEALINGS IN THE SOFTWARE. * OTHER DEALINGS IN THE SOFTWARE.
*/ */
/*- Compiler specifics -*/
#ifdef __clang__
#pragma clang diagnostic ignored "-Wshorten-64-to-32"
#endif
/*- Dependencies -*/
#include "divsufsort_private.h" #include "divsufsort_private.h"
+6
View File
@@ -24,6 +24,12 @@
* OTHER DEALINGS IN THE SOFTWARE. * OTHER DEALINGS IN THE SOFTWARE.
*/ */
/*- Compiler specifics -*/
#ifdef __clang__
#pragma clang diagnostic ignored "-Wshorten-64-to-32"
#endif
/*- Dependencies -*/
#include "divsufsort_private.h" #include "divsufsort_private.h"
+218 -284
View File
@@ -129,10 +129,8 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue); /* don't need last symbol stat : implied */ size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue); /* don't need last symbol stat : implied */
if (HUF_isError(size)) return size; if (HUF_isError(size)) return size;
if (size >= 128) return ERROR(GENERIC); /* should never happen, since maxSymbolValue <= 255 */ if (size >= 128) return ERROR(GENERIC); /* should never happen, since maxSymbolValue <= 255 */
if ((size <= 1) || (size >= maxSymbolValue/2)) if ((size <= 1) || (size >= maxSymbolValue/2)) {
{ if (size==1) { /* RLE */
if (size==1) /* RLE */
{
/* only possible case : serie of 1 (because there are at least 2) */ /* only possible case : serie of 1 (because there are at least 2) */
/* can only be 2^n or (2^n-1), otherwise not an huffman tree */ /* can only be 2^n or (2^n-1), otherwise not an huffman tree */
BYTE code; BYTE code;
@@ -220,13 +218,11 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si
{ {
/* determine stating value per rank */ /* determine stating value per rank */
U16 min = 0; U16 min = 0;
for (n=HUF_MAX_TABLELOG; n>0; n--) for (n=HUF_MAX_TABLELOG; n>0; n--) {
{
valPerRank[n] = min; /* get starting value within each rank */ valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n]; min += nbPerRank[n];
min >>= 1; min >>= 1;
} } }
}
for (n=0; n<=maxSymbolValue; n++) for (n=0; n<=maxSymbolValue; n++)
CTable[n].val = valPerRank[CTable[n].nbBits]++; /* assign value within rank, symbol order */ CTable[n].val = valPerRank[CTable[n].nbBits]++; /* assign value within rank, symbol order */
} }
@@ -248,8 +244,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
const U32 baseCost = 1 << (largestBits - maxNbBits); const U32 baseCost = 1 << (largestBits - maxNbBits);
U32 n = lastNonNull; U32 n = lastNonNull;
while (huffNode[n].nbBits > maxNbBits) while (huffNode[n].nbBits > maxNbBits) {
{
totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
huffNode[n].nbBits = (BYTE)maxNbBits; huffNode[n].nbBits = (BYTE)maxNbBits;
n --; n --;
@@ -268,18 +263,15 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
/* Get pos of last (smallest) symbol per rank */ /* Get pos of last (smallest) symbol per rank */
memset(rankLast, 0xF0, sizeof(rankLast)); memset(rankLast, 0xF0, sizeof(rankLast));
for (pos=n ; pos >= 0; pos--) for (pos=n ; pos >= 0; pos--) {
{
if (huffNode[pos].nbBits >= currentNbBits) continue; if (huffNode[pos].nbBits >= currentNbBits) continue;
currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */
rankLast[maxNbBits-currentNbBits] = pos; rankLast[maxNbBits-currentNbBits] = pos;
} }
while (totalCost > 0) while (totalCost > 0) {
{
U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1;
for ( ; nBitsToDecrease > 1; nBitsToDecrease--) for ( ; nBitsToDecrease > 1; nBitsToDecrease--) {
{
U32 highPos = rankLast[nBitsToDecrease]; U32 highPos = rankLast[nBitsToDecrease];
U32 lowPos = rankLast[nBitsToDecrease-1]; U32 lowPos = rankLast[nBitsToDecrease-1];
if (highPos == noSymbol) continue; if (highPos == noSymbol) continue;
@@ -288,8 +280,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
U32 highTotal = huffNode[highPos].count; U32 highTotal = huffNode[highPos].count;
U32 lowTotal = 2 * huffNode[lowPos].count; U32 lowTotal = 2 * huffNode[lowPos].count;
if (highTotal <= lowTotal) break; if (highTotal <= lowTotal) break;
} } }
}
/* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
while ((nBitsToDecrease<=HUF_MAX_TABLELOG) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ while ((nBitsToDecrease<=HUF_MAX_TABLELOG) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
nBitsToDecrease ++; nBitsToDecrease ++;
@@ -299,18 +290,14 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
huffNode[rankLast[nBitsToDecrease]].nbBits ++; huffNode[rankLast[nBitsToDecrease]].nbBits ++;
if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */
rankLast[nBitsToDecrease] = noSymbol; rankLast[nBitsToDecrease] = noSymbol;
else else {
{
rankLast[nBitsToDecrease]--; rankLast[nBitsToDecrease]--;
if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
} } }
}
while (totalCost < 0) /* Sometimes, cost correction overshoot */ while (totalCost < 0) { /* Sometimes, cost correction overshoot */
{ if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */
if (rankLast[1] == noSymbol) /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */
{
while (huffNode[n].nbBits == maxNbBits) n--; while (huffNode[n].nbBits == maxNbBits) n--;
huffNode[n+1].nbBits--; huffNode[n+1].nbBits--;
rankLast[1] = n+1; rankLast[1] = n+1;
@@ -320,9 +307,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
huffNode[ rankLast[1] + 1 ].nbBits--; huffNode[ rankLast[1] + 1 ].nbBits--;
rankLast[1]++; rankLast[1]++;
totalCost ++; totalCost ++;
} } } }
}
}
return maxNbBits; return maxNbBits;
} }
@@ -339,15 +324,13 @@ static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
U32 n; U32 n;
memset(rank, 0, sizeof(rank)); memset(rank, 0, sizeof(rank));
for (n=0; n<=maxSymbolValue; n++) for (n=0; n<=maxSymbolValue; n++) {
{
U32 r = BIT_highbit32(count[n] + 1); U32 r = BIT_highbit32(count[n] + 1);
rank[r].base ++; rank[r].base ++;
} }
for (n=30; n>0; n--) rank[n-1].base += rank[n].base; for (n=30; n>0; n--) rank[n-1].base += rank[n].base;
for (n=0; n<32; n++) rank[n].current = rank[n].base; for (n=0; n<32; n++) rank[n].current = rank[n].base;
for (n=0; n<=maxSymbolValue; n++) for (n=0; n<=maxSymbolValue; n++) {
{
U32 c = count[n]; U32 c = count[n];
U32 r = BIT_highbit32(c+1) + 1; U32 r = BIT_highbit32(c+1) + 1;
U32 pos = rank[r].current++; U32 pos = rank[r].current++;
@@ -387,8 +370,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
huffNode0[0].count = (U32)(1U<<31); huffNode0[0].count = (U32)(1U<<31);
/* create parents */ /* create parents */
while (nodeNb <= nodeRoot) while (nodeNb <= nodeRoot) {
{
U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
@@ -416,8 +398,7 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
{ {
/* determine stating value per rank */ /* determine stating value per rank */
U16 min = 0; U16 min = 0;
for (n=maxNbBits; n>0; n--) for (n=maxNbBits; n>0; n--) {
{
valPerRank[n] = min; /* get starting value within each rank */ valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n]; min += nbPerRank[n];
min >>= 1; min >>= 1;
@@ -476,8 +457,7 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si
default: ; default: ;
} }
for (; n>0; n-=4) /* note : n&3==0 at this stage */ for (; n>0; n-=4) { /* note : n&3==0 at this stage */
{
HUF_encodeSymbol(&bitC, ip[n- 1], CTable); HUF_encodeSymbol(&bitC, ip[n- 1], CTable);
HUF_FLUSHBITS_1(&bitC); HUF_FLUSHBITS_1(&bitC);
HUF_encodeSymbol(&bitC, ip[n- 2], CTable); HUF_encodeSymbol(&bitC, ip[n- 2], CTable);
@@ -666,8 +646,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
/* collect weight stats */ /* collect weight stats */
memset(rankStats, 0, (HUF_ABSOLUTEMAX_TABLELOG + 1) * sizeof(U32)); memset(rankStats, 0, (HUF_ABSOLUTEMAX_TABLELOG + 1) * sizeof(U32));
weightTotal = 0; weightTotal = 0;
for (n=0; n<oSize; n++) for (n=0; n<oSize; n++) {
{
if (huffWeight[n] >= HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected); if (huffWeight[n] >= HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected);
rankStats[huffWeight[n]]++; rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1; weightTotal += (1 << huffWeight[n]) >> 1;
@@ -676,7 +655,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
/* get last non-null symbol weight (implied, total must be 2^n) */ /* get last non-null symbol weight (implied, total must be 2^n) */
tableLog = BIT_highbit32(weightTotal) + 1; tableLog = BIT_highbit32(weightTotal) + 1;
if (tableLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected); if (tableLog > HUF_ABSOLUTEMAX_TABLELOG) return ERROR(corruption_detected);
{ { /* determine last weight */
U32 total = 1 << tableLog; U32 total = 1 << tableLog;
U32 rest = total - weightTotal; U32 rest = total - weightTotal;
U32 verif = 1 << BIT_highbit32(rest); U32 verif = 1 << BIT_highbit32(rest);
@@ -724,16 +703,14 @@ size_t HUF_readDTableX2 (U16* DTable, const void* src, size_t srcSize)
/* Prepare ranks */ /* Prepare ranks */
nextRankStart = 0; nextRankStart = 0;
for (n=1; n<=tableLog; n++) for (n=1; n<=tableLog; n++) {
{
U32 current = nextRankStart; U32 current = nextRankStart;
nextRankStart += (rankVal[n] << (n-1)); nextRankStart += (rankVal[n] << (n-1));
rankVal[n] = current; rankVal[n] = current;
} }
/* fill DTable */ /* fill DTable */
for (n=0; n<nbSymbols; n++) for (n=0; n<nbSymbols; n++) {
{
const U32 w = huffWeight[n]; const U32 w = huffWeight[n];
const U32 length = (1 << w) >> 1; const U32 length = (1 << w) >> 1;
U32 i; U32 i;
@@ -771,8 +748,7 @@ static inline size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, B
BYTE* const pStart = p; BYTE* const pStart = p;
/* up to 4 symbols at a time */ /* up to 4 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4)) while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4)) {
{
HUF_DECODE_SYMBOLX2_2(p, bitDPtr); HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
HUF_DECODE_SYMBOLX2_1(p, bitDPtr); HUF_DECODE_SYMBOLX2_1(p, bitDPtr);
HUF_DECODE_SYMBOLX2_2(p, bitDPtr); HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
@@ -834,94 +810,91 @@ size_t HUF_decompress4X2_usingDTable(
const void* cSrc, size_t cSrcSize, const void* cSrc, size_t cSrcSize,
const U16* DTable) const U16* DTable)
{ {
const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const void* const dtPtr = DTable;
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr) +1;
const U32 dtLog = DTable[0];
size_t errorCode;
/* Check */
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
{ /* Init */
const BYTE* const istart = (const BYTE*) cSrc; BIT_DStream_t bitD1;
BYTE* const ostart = (BYTE*) dst; BIT_DStream_t bitD2;
BYTE* const oend = ostart + dstSize; BIT_DStream_t bitD3;
const void* const dtPtr = DTable; BIT_DStream_t bitD4;
const HUF_DEltX2* const dt = ((const HUF_DEltX2*)dtPtr) +1; const size_t length1 = MEM_readLE16(istart);
const U32 dtLog = DTable[0]; const size_t length2 = MEM_readLE16(istart+2);
size_t errorCode; const size_t length3 = MEM_readLE16(istart+4);
size_t length4;
const BYTE* const istart1 = istart + 6; /* jumpTable */
const BYTE* const istart2 = istart1 + length1;
const BYTE* const istart3 = istart2 + length2;
const BYTE* const istart4 = istart3 + length3;
const size_t segmentSize = (dstSize+3) / 4;
BYTE* const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize;
BYTE* op1 = ostart;
BYTE* op2 = opStart2;
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
/* Init */ length4 = cSrcSize - (length1 + length2 + length3 + 6);
BIT_DStream_t bitD1; if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
BIT_DStream_t bitD2; errorCode = BIT_initDStream(&bitD1, istart1, length1);
BIT_DStream_t bitD3; if (HUF_isError(errorCode)) return errorCode;
BIT_DStream_t bitD4; errorCode = BIT_initDStream(&bitD2, istart2, length2);
const size_t length1 = MEM_readLE16(istart); if (HUF_isError(errorCode)) return errorCode;
const size_t length2 = MEM_readLE16(istart+2); errorCode = BIT_initDStream(&bitD3, istart3, length3);
const size_t length3 = MEM_readLE16(istart+4); if (HUF_isError(errorCode)) return errorCode;
size_t length4; errorCode = BIT_initDStream(&bitD4, istart4, length4);
const BYTE* const istart1 = istart + 6; /* jumpTable */ if (HUF_isError(errorCode)) return errorCode;
const BYTE* const istart2 = istart1 + length1;
const BYTE* const istart3 = istart2 + length2;
const BYTE* const istart4 = istart3 + length3;
const size_t segmentSize = (dstSize+3) / 4;
BYTE* const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize;
BYTE* op1 = ostart;
BYTE* op2 = opStart2;
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
length4 = cSrcSize - (length1 + length2 + length3 + 6); /* 16-32 symbols per loop (4-8 symbols per stream) */
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
errorCode = BIT_initDStream(&bitD1, istart1, length1); for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) {
if (HUF_isError(errorCode)) return errorCode; HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
errorCode = BIT_initDStream(&bitD2, istart2, length2); HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
if (HUF_isError(errorCode)) return errorCode; HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
errorCode = BIT_initDStream(&bitD3, istart3, length3); HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
if (HUF_isError(errorCode)) return errorCode; HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
errorCode = BIT_initDStream(&bitD4, istart4, length4); HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
if (HUF_isError(errorCode)) return errorCode; HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
/* 16-32 symbols per loop (4-8 symbols per stream) */ HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; )
{
HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
}
/* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected);
if (op2 > opStart3) return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected);
/* note : op4 supposed already verified within main loop */
/* finish bitStreams one by one */
HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
/* check */
endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
if (!endSignal) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
} }
/* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected);
if (op2 > opStart3) return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected);
/* note : op4 supposed already verified within main loop */
/* finish bitStreams one by one */
HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
/* check */
endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
if (!endSignal) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
} }
@@ -958,8 +931,7 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co
memcpy(rankVal, rankValOrigin, sizeof(rankVal)); memcpy(rankVal, rankValOrigin, sizeof(rankVal));
/* fill skipped values */ /* fill skipped values */
if (minWeight>1) if (minWeight>1) {
{
U32 i, skipSize = rankVal[minWeight]; U32 i, skipSize = rankVal[minWeight];
MEM_writeLE16(&(DElt.sequence), baseSeq); MEM_writeLE16(&(DElt.sequence), baseSeq);
DElt.nbBits = (BYTE)(consumed); DElt.nbBits = (BYTE)(consumed);
@@ -969,8 +941,7 @@ static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 co
} }
/* fill DTable */ /* fill DTable */
for (s=0; s<sortedListSize; s++) /* note : sortedSymbols already skipped */ for (s=0; s<sortedListSize; s++) { /* note : sortedSymbols already skipped */
{
const U32 symbol = sortedSymbols[s].symbol; const U32 symbol = sortedSymbols[s].symbol;
const U32 weight = sortedSymbols[s].weight; const U32 weight = sortedSymbols[s].weight;
const U32 nbBits = nbBitsBaseline - weight; const U32 nbBits = nbBitsBaseline - weight;
@@ -1003,16 +974,14 @@ static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
memcpy(rankVal, rankValOrigin, sizeof(rankVal)); memcpy(rankVal, rankValOrigin, sizeof(rankVal));
/* fill DTable */ /* fill DTable */
for (s=0; s<sortedListSize; s++) for (s=0; s<sortedListSize; s++) {
{
const U16 symbol = sortedList[s].symbol; const U16 symbol = sortedList[s].symbol;
const U32 weight = sortedList[s].weight; const U32 weight = sortedList[s].weight;
const U32 nbBits = nbBitsBaseline - weight; const U32 nbBits = nbBitsBaseline - weight;
const U32 start = rankVal[weight]; const U32 start = rankVal[weight];
const U32 length = 1 << (targetLog-nbBits); const U32 length = 1 << (targetLog-nbBits);
if (targetLog-nbBits >= minBits) /* enough room for a second symbol */ if (targetLog-nbBits >= minBits) { /* enough room for a second symbol */
{
U32 sortedRank; U32 sortedRank;
int minWeight = nbBits + scaleLog; int minWeight = nbBits + scaleLog;
if (minWeight < 1) minWeight = 1; if (minWeight < 1) minWeight = 1;
@@ -1021,9 +990,7 @@ static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
rankValOrigin[nbBits], minWeight, rankValOrigin[nbBits], minWeight,
sortedList+sortedRank, sortedListSize-sortedRank, sortedList+sortedRank, sortedListSize-sortedRank,
nbBitsBaseline, symbol); nbBitsBaseline, symbol);
} } else {
else
{
U32 i; U32 i;
const U32 end = start + length; const U32 end = start + length;
HUF_DEltX4 DElt; HUF_DEltX4 DElt;
@@ -1068,8 +1035,7 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
/* Get start index of each weight */ /* Get start index of each weight */
{ {
U32 w, nextRankStart = 0; U32 w, nextRankStart = 0;
for (w=1; w<=maxW; w++) for (w=1; w<=maxW; w++) {
{
U32 current = nextRankStart; U32 current = nextRankStart;
nextRankStart += rankStats[w]; nextRankStart += rankStats[w];
rankStart[w] = current; rankStart[w] = current;
@@ -1081,8 +1047,7 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
/* sort symbols by weight */ /* sort symbols by weight */
{ {
U32 s; U32 s;
for (s=0; s<nbSymbols; s++) for (s=0; s<nbSymbols; s++) {
{
U32 w = weightList[s]; U32 w = weightList[s];
U32 r = rankStart[w]++; U32 r = rankStart[w]++;
sortedSymbol[r].symbol = (BYTE)s; sortedSymbol[r].symbol = (BYTE)s;
@@ -1098,21 +1063,16 @@ size_t HUF_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
U32 w, consumed; U32 w, consumed;
const int rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */ const int rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */
U32* rankVal0 = rankVal[0]; U32* rankVal0 = rankVal[0];
for (w=1; w<=maxW; w++) for (w=1; w<=maxW; w++) {
{
U32 current = nextRankVal; U32 current = nextRankVal;
nextRankVal += rankStats[w] << (w+rescale); nextRankVal += rankStats[w] << (w+rescale);
rankVal0[w] = current; rankVal0[w] = current;
} }
for (consumed = minBits; consumed <= memLog - minBits; consumed++) for (consumed = minBits; consumed <= memLog - minBits; consumed++) {
{
U32* rankValPtr = rankVal[consumed]; U32* rankValPtr = rankVal[consumed];
for (w = 1; w <= maxW; w++) for (w = 1; w <= maxW; w++) {
{
rankValPtr[w] = rankVal0[w] >> consumed; rankValPtr[w] = rankVal0[w] >> consumed;
} } } }
}
}
HUF_fillDTableX4(dt, memLog, HUF_fillDTableX4(dt, memLog,
sortedSymbol, sizeOfSort, sortedSymbol, sizeOfSort,
@@ -1136,15 +1096,12 @@ static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DE
const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
memcpy(op, dt+val, 1); memcpy(op, dt+val, 1);
if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits);
else else {
{ if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8))
{
BIT_skipBits(DStream, dt[val].nbBits); BIT_skipBits(DStream, dt[val].nbBits);
if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
} } }
}
return 1; return 1;
} }
@@ -1165,8 +1122,7 @@ static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c
BYTE* const pStart = p; BYTE* const pStart = p;
/* up to 8 symbols at a time */ /* up to 8 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd-7)) while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd-7)) {
{
HUF_DECODE_SYMBOLX4_2(p, bitDPtr); HUF_DECODE_SYMBOLX4_2(p, bitDPtr);
HUF_DECODE_SYMBOLX4_1(p, bitDPtr); HUF_DECODE_SYMBOLX4_1(p, bitDPtr);
HUF_DECODE_SYMBOLX4_2(p, bitDPtr); HUF_DECODE_SYMBOLX4_2(p, bitDPtr);
@@ -1282,8 +1238,7 @@ size_t HUF_decompress4X4_usingDTable(
/* 16-32 symbols per loop (4-8 symbols per stream) */ /* 16-32 symbols per loop (4-8 symbols per stream) */
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) {
{
HUF_DECODE_SYMBOLX4_2(op1, &bitD1); HUF_DECODE_SYMBOLX4_2(op1, &bitD1);
HUF_DECODE_SYMBOLX4_2(op2, &bitD2); HUF_DECODE_SYMBOLX4_2(op2, &bitD2);
HUF_DECODE_SYMBOLX4_2(op3, &bitD3); HUF_DECODE_SYMBOLX4_2(op3, &bitD3);
@@ -1363,22 +1318,18 @@ static void HUF_fillDTableX6LevelN(HUF_DDescX6* DDescription, HUF_DSeqX6* DSeque
memcpy(rankVal, rankValOrigin[consumed], sizeof(rankVal)); memcpy(rankVal, rankValOrigin[consumed], sizeof(rankVal));
/* fill skipped values */ /* fill skipped values */
if (minWeight>1) if (minWeight>1) {
{
U32 i; U32 i;
const U32 skipSize = rankVal[minWeight]; const U32 skipSize = rankVal[minWeight];
for (i = 0; i < skipSize; i++) for (i = 0; i < skipSize; i++) {
{
DSequence[i] = baseSeq; DSequence[i] = baseSeq;
DDescription[i] = DDesc; DDescription[i] = DDesc;
} } }
}
/* fill DTable */ /* fill DTable */
DDesc.nbBytes++; DDesc.nbBytes++;
symbolStartPos = rankStart[minWeight]; symbolStartPos = rankStart[minWeight];
for (s=symbolStartPos; s<sortedListSize; s++) for (s=symbolStartPos; s<sortedListSize; s++) {
{
const BYTE symbol = sortedSymbols[s].symbol; const BYTE symbol = sortedSymbols[s].symbol;
const U32 weight = sortedSymbols[s].weight; /* >= 1 (sorted) */ const U32 weight = sortedSymbols[s].weight; /* >= 1 (sorted) */
const int nbBits = nbBitsBaseline - weight; /* >= 1 (by construction) */ const int nbBits = nbBitsBaseline - weight; /* >= 1 (by construction) */
@@ -1388,25 +1339,20 @@ static void HUF_fillDTableX6LevelN(HUF_DDescX6* DDescription, HUF_DSeqX6* DSeque
baseSeq.byte[level] = symbol; baseSeq.byte[level] = symbol;
DDesc.nbBits = (BYTE)totalBits; DDesc.nbBits = (BYTE)totalBits;
if ((level<3) && (sizeLog-totalBits >= minBits)) /* enough room for another symbol */ if ((level<3) && (sizeLog-totalBits >= minBits)) { /* enough room for another symbol */
{
int nextMinWeight = totalBits + scaleLog; int nextMinWeight = totalBits + scaleLog;
if (nextMinWeight < 1) nextMinWeight = 1; if (nextMinWeight < 1) nextMinWeight = 1;
HUF_fillDTableX6LevelN(DDescription+start, DSequence+start, sizeLog-nbBits, HUF_fillDTableX6LevelN(DDescription+start, DSequence+start, sizeLog-nbBits,
rankValOrigin, totalBits, nextMinWeight, maxWeight, rankValOrigin, totalBits, nextMinWeight, maxWeight,
sortedSymbols, sortedListSize, rankStart, sortedSymbols, sortedListSize, rankStart,
nbBitsBaseline, baseSeq, DDesc); /* recursive (max : level 3) */ nbBitsBaseline, baseSeq, DDesc); /* recursive (max : level 3) */
} } else {
else
{
U32 i; U32 i;
const U32 end = start + length; const U32 end = start + length;
for (i = start; i < end; i++) for (i = start; i < end; i++) {
{
DDescription[i] = DDesc; DDescription[i] = DDesc;
DSequence[i] = baseSeq; DSequence[i] = baseSeq;
} } }
}
rankVal[weight] += length; rankVal[weight] += length;
} }
} }
@@ -1440,8 +1386,7 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
/* Get start index of each weight */ /* Get start index of each weight */
{ {
U32 w, nextRankStart = 0; U32 w, nextRankStart = 0;
for (w=1; w<=maxW; w++) for (w=1; w<=maxW; w++) {
{
U32 current = nextRankStart; U32 current = nextRankStart;
nextRankStart += rankStats[w]; nextRankStart += rankStats[w];
rankStart[w] = current; rankStart[w] = current;
@@ -1453,8 +1398,7 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
/* sort symbols by weight */ /* sort symbols by weight */
{ {
U32 s; U32 s;
for (s=0; s<nbSymbols; s++) for (s=0; s<nbSymbols; s++) {
{
U32 w = weightList[s]; U32 w = weightList[s];
U32 r = rankStart[w]++; U32 r = rankStart[w]++;
sortedSymbol[r].symbol = (BYTE)s; sortedSymbol[r].symbol = (BYTE)s;
@@ -1470,21 +1414,16 @@ size_t HUF_readDTableX6 (U32* DTable, const void* src, size_t srcSize)
U32 w, consumed; U32 w, consumed;
const int rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */ const int rescale = (memLog-tableLog) - 1; /* tableLog <= memLog */
U32* rankVal0 = rankVal[0]; U32* rankVal0 = rankVal[0];
for (w=1; w<=maxW; w++) for (w=1; w<=maxW; w++) {
{
U32 current = nextRankVal; U32 current = nextRankVal;
nextRankVal += rankStats[w] << (w+rescale); nextRankVal += rankStats[w] << (w+rescale);
rankVal0[w] = current; rankVal0[w] = current;
} }
for (consumed = minBits; consumed <= memLog - minBits; consumed++) for (consumed = minBits; consumed <= memLog - minBits; consumed++) {
{
U32* rankValPtr = rankVal[consumed]; U32* rankValPtr = rankVal[consumed];
for (w = 1; w <= maxW; w++) for (w = 1; w <= maxW; w++) {
{
rankValPtr[w] = rankVal0[w] >> consumed; rankValPtr[w] = rankVal0[w] >> consumed;
} } } }
}
}
/* fill tables */ /* fill tables */
{ {
@@ -1520,15 +1459,13 @@ static U32 HUF_decodeLastSymbolsX6(void* op, const U32 maxL, BIT_DStream_t* DStr
{ {
const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ const size_t val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
U32 length = dd[val].nbBytes; U32 length = dd[val].nbBytes;
if (length <= maxL) if (length <= maxL) {
{
memcpy(op, ds+val, length); memcpy(op, ds+val, length);
BIT_skipBits(DStream, dd[val].nbBits); BIT_skipBits(DStream, dd[val].nbBits);
return length; return length;
} }
memcpy(op, ds+val, maxL); memcpy(op, ds+val, maxL);
if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
{
BIT_skipBits(DStream, dd[val].nbBits); BIT_skipBits(DStream, dd[val].nbBits);
if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
@@ -1557,8 +1494,7 @@ static inline size_t HUF_decodeStreamX6(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* c
BYTE* const pStart = p; BYTE* const pStart = p;
/* up to 16 symbols at a time */ /* up to 16 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-16)) while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-16)) {
{
HUF_DECODE_SYMBOLX6_2(p, bitDPtr); HUF_DECODE_SYMBOLX6_2(p, bitDPtr);
HUF_DECODE_SYMBOLX6_1(p, bitDPtr); HUF_DECODE_SYMBOLX6_1(p, bitDPtr);
HUF_DECODE_SYMBOLX6_2(p, bitDPtr); HUF_DECODE_SYMBOLX6_2(p, bitDPtr);
@@ -1623,97 +1559,95 @@ size_t HUF_decompress4X6_usingDTable(
const void* cSrc, size_t cSrcSize, const void* cSrc, size_t cSrcSize,
const U32* DTable) const U32* DTable)
{ {
const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const U32 dtLog = DTable[0];
const void* const ddPtr = DTable+1;
const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr;
const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1));
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr;
size_t errorCode;
/* Check */
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
{ /* Init */
const BYTE* const istart = (const BYTE*) cSrc; BIT_DStream_t bitD1;
BYTE* const ostart = (BYTE*) dst; BIT_DStream_t bitD2;
BYTE* const oend = ostart + dstSize; BIT_DStream_t bitD3;
BIT_DStream_t bitD4;
const size_t length1 = MEM_readLE16(istart);
const size_t length2 = MEM_readLE16(istart+2);
const size_t length3 = MEM_readLE16(istart+4);
size_t length4;
const BYTE* const istart1 = istart + 6; /* jumpTable */
const BYTE* const istart2 = istart1 + length1;
const BYTE* const istart3 = istart2 + length2;
const BYTE* const istart4 = istart3 + length3;
const size_t segmentSize = (dstSize+3) / 4;
BYTE* const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize;
BYTE* op1 = ostart;
BYTE* op2 = opStart2;
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
const U32 dtLog = DTable[0]; length4 = cSrcSize - (length1 + length2 + length3 + 6);
const void* const ddPtr = DTable+1; if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
const HUF_DDescX6* dd = (const HUF_DDescX6*)ddPtr; errorCode = BIT_initDStream(&bitD1, istart1, length1);
const void* const dsPtr = DTable + 1 + ((size_t)1<<(dtLog-1)); if (HUF_isError(errorCode)) return errorCode;
const HUF_DSeqX6* ds = (const HUF_DSeqX6*)dsPtr; errorCode = BIT_initDStream(&bitD2, istart2, length2);
size_t errorCode; if (HUF_isError(errorCode)) return errorCode;
errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode)) return errorCode;
errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode)) return errorCode;
/* Init */ /* 16-64 symbols per loop (4-16 symbols per stream) */
BIT_DStream_t bitD1; endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
BIT_DStream_t bitD2; for ( ; (op3 <= opStart4) && (endSignal==BIT_DStream_unfinished) && (op4<=(oend-16)) ; ) {
BIT_DStream_t bitD3; HUF_DECODE_SYMBOLX6_2(op1, &bitD1);
BIT_DStream_t bitD4; HUF_DECODE_SYMBOLX6_2(op2, &bitD2);
const size_t length1 = MEM_readLE16(istart); HUF_DECODE_SYMBOLX6_2(op3, &bitD3);
const size_t length2 = MEM_readLE16(istart+2); HUF_DECODE_SYMBOLX6_2(op4, &bitD4);
const size_t length3 = MEM_readLE16(istart+4); HUF_DECODE_SYMBOLX6_1(op1, &bitD1);
size_t length4; HUF_DECODE_SYMBOLX6_1(op2, &bitD2);
const BYTE* const istart1 = istart + 6; /* jumpTable */ HUF_DECODE_SYMBOLX6_1(op3, &bitD3);
const BYTE* const istart2 = istart1 + length1; HUF_DECODE_SYMBOLX6_1(op4, &bitD4);
const BYTE* const istart3 = istart2 + length2; HUF_DECODE_SYMBOLX6_2(op1, &bitD1);
const BYTE* const istart4 = istart3 + length3; HUF_DECODE_SYMBOLX6_2(op2, &bitD2);
const size_t segmentSize = (dstSize+3) / 4; HUF_DECODE_SYMBOLX6_2(op3, &bitD3);
BYTE* const opStart2 = ostart + segmentSize; HUF_DECODE_SYMBOLX6_2(op4, &bitD4);
BYTE* const opStart3 = opStart2 + segmentSize; HUF_DECODE_SYMBOLX6_0(op1, &bitD1);
BYTE* const opStart4 = opStart3 + segmentSize; HUF_DECODE_SYMBOLX6_0(op2, &bitD2);
BYTE* op1 = ostart; HUF_DECODE_SYMBOLX6_0(op3, &bitD3);
BYTE* op2 = opStart2; HUF_DECODE_SYMBOLX6_0(op4, &bitD4);
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
length4 = cSrcSize - (length1 + length2 + length3 + 6);
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
errorCode = BIT_initDStream(&bitD1, istart1, length1);
if (HUF_isError(errorCode)) return errorCode;
errorCode = BIT_initDStream(&bitD2, istart2, length2);
if (HUF_isError(errorCode)) return errorCode;
errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode)) return errorCode;
errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode)) return errorCode;
/* 16-64 symbols per loop (4-16 symbols per stream) */
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
for ( ; (op3 <= opStart4) && (endSignal==BIT_DStream_unfinished) && (op4<=(oend-16)) ; )
{
HUF_DECODE_SYMBOLX6_2(op1, &bitD1);
HUF_DECODE_SYMBOLX6_2(op2, &bitD2);
HUF_DECODE_SYMBOLX6_2(op3, &bitD3);
HUF_DECODE_SYMBOLX6_2(op4, &bitD4);
HUF_DECODE_SYMBOLX6_1(op1, &bitD1);
HUF_DECODE_SYMBOLX6_1(op2, &bitD2);
HUF_DECODE_SYMBOLX6_1(op3, &bitD3);
HUF_DECODE_SYMBOLX6_1(op4, &bitD4);
HUF_DECODE_SYMBOLX6_2(op1, &bitD1);
HUF_DECODE_SYMBOLX6_2(op2, &bitD2);
HUF_DECODE_SYMBOLX6_2(op3, &bitD3);
HUF_DECODE_SYMBOLX6_2(op4, &bitD4);
HUF_DECODE_SYMBOLX6_0(op1, &bitD1);
HUF_DECODE_SYMBOLX6_0(op2, &bitD2);
HUF_DECODE_SYMBOLX6_0(op3, &bitD3);
HUF_DECODE_SYMBOLX6_0(op4, &bitD4);
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
}
/* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected);
if (op2 > opStart3) return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected);
/* note : op4 supposed already verified within main loop */
/* finish bitStreams one by one */
HUF_decodeStreamX6(op1, &bitD1, opStart2, DTable, dtLog);
HUF_decodeStreamX6(op2, &bitD2, opStart3, DTable, dtLog);
HUF_decodeStreamX6(op3, &bitD3, opStart4, DTable, dtLog);
HUF_decodeStreamX6(op4, &bitD4, oend, DTable, dtLog);
/* check */
endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
if (!endSignal) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
} }
/* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected);
if (op2 > opStart3) return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected);
/* note : op4 supposed already verified within main loop */
/* finish bitStreams one by one */
HUF_decodeStreamX6(op1, &bitD1, opStart2, DTable, dtLog);
HUF_decodeStreamX6(op2, &bitD2, opStart3, DTable, dtLog);
HUF_decodeStreamX6(op3, &bitD3, opStart4, DTable, dtLog);
HUF_decodeStreamX6(op4, &bitD4, oend, DTable, dtLog);
/* check */
endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
if (!endSignal) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
} }