ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

created entropy structure type

Browse Source
This commit is contained in:
Yann Collet
2017-02-25 18:33:31 -08:00
parent 8dff956dbf
commit 8629f0e41f
1 changed files with 44 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/decompress/zstd_decompress.c
+44 -38
View File
@@ -87,22 +87,26 @@ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
typedef struct {
FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
U32 rep[ZSTD_REP_NUM];
} ZSTD_entropyTables_t;
struct ZSTD_DCtx_s struct ZSTD_DCtx_s
{ {
const FSE_DTable* LLTptr; const FSE_DTable* LLTptr;
const FSE_DTable* MLTptr; const FSE_DTable* MLTptr;
const FSE_DTable* OFTptr; const FSE_DTable* OFTptr;
const HUF_DTable* HUFptr; const HUF_DTable* HUFptr;
FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; ZSTD_entropyTables_t entropy;
FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
const void* previousDstEnd; /* detect continuity */ const void* previousDstEnd; /* detect continuity */
const void* base; /* start of current segment */ const void* base; /* start of current segment */
const void* vBase; /* virtual start of previous segment if it was just before current one */ const void* vBase; /* virtual start of previous segment if it was just before current one */
const void* dictEnd; /* end of previous segment */ const void* dictEnd; /* end of previous segment */
size_t expected; size_t expected;
U32 rep[ZSTD_REP_NUM];
ZSTD_frameParams fParams; ZSTD_frameParams fParams;
blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
ZSTD_dStage stage; ZSTD_dStage stage;
@@ -131,15 +135,15 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
dctx->base = NULL; dctx->base = NULL;
dctx->vBase = NULL; dctx->vBase = NULL;
dctx->dictEnd = NULL; dctx->dictEnd = NULL;
dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
dctx->litEntropy = dctx->fseEntropy = 0; dctx->litEntropy = dctx->fseEntropy = 0;
dctx->dictID = 0; dctx->dictID = 0;
MEM_STATIC_ASSERT(sizeof(dctx->rep) == sizeof(repStartValue)); MEM_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
memcpy(dctx->rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
dctx->LLTptr = dctx->LLTable; dctx->LLTptr = dctx->entropy.LLTable;
dctx->MLTptr = dctx->MLTable; dctx->MLTptr = dctx->entropy.MLTable;
dctx->OFTptr = dctx->OFTable; dctx->OFTptr = dctx->entropy.OFTable;
dctx->HUFptr = dctx->hufTable; dctx->HUFptr = dctx->entropy.hufTable;
return 0; return 0;
} }
@@ -186,13 +190,13 @@ static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
dstDCtx->dictID = srcDCtx->dictID; dstDCtx->dictID = srcDCtx->dictID;
dstDCtx->litEntropy = srcDCtx->litEntropy; dstDCtx->litEntropy = srcDCtx->litEntropy;
dstDCtx->fseEntropy = srcDCtx->fseEntropy; dstDCtx->fseEntropy = srcDCtx->fseEntropy;
dstDCtx->LLTptr = srcDCtx->LLTable; dstDCtx->LLTptr = srcDCtx->entropy.LLTable;
dstDCtx->MLTptr = srcDCtx->MLTable; dstDCtx->MLTptr = srcDCtx->entropy.MLTable;
dstDCtx->OFTptr = srcDCtx->OFTable; dstDCtx->OFTptr = srcDCtx->entropy.OFTable;
dstDCtx->HUFptr = srcDCtx->hufTable; dstDCtx->HUFptr = srcDCtx->entropy.hufTable;
dstDCtx->rep[0] = srcDCtx->rep[0]; dstDCtx->entropy.rep[0] = srcDCtx->entropy.rep[0];
dstDCtx->rep[1] = srcDCtx->rep[1]; dstDCtx->entropy.rep[1] = srcDCtx->entropy.rep[1];
dstDCtx->rep[2] = srcDCtx->rep[2]; dstDCtx->entropy.rep[2] = srcDCtx->entropy.rep[2];
} }
} }
@@ -506,14 +510,14 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) : HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) :
HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) : HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) :
( singleStream ? ( singleStream ?
HUF_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) : HUF_decompress1X2_DCtx(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) :
HUF_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) )) HUF_decompress4X_hufOnly (dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) ))
return ERROR(corruption_detected); return ERROR(corruption_detected);
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize; dctx->litSize = litSize;
dctx->litEntropy = 1; dctx->litEntropy = 1;
if (litEncType==set_compressed) dctx->HUFptr = dctx->hufTable; if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable;
memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize; return litCSize + lhSize;
} }
@@ -830,19 +834,19 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
ip++; ip++;
/* Build DTables */ /* Build DTables */
{ size_t const llhSize = ZSTD_buildSeqTable(dctx->LLTable, &dctx->LLTptr, { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr,
LLtype, MaxLL, LLFSELog, LLtype, MaxLL, LLFSELog,
ip, iend-ip, LL_defaultDTable, dctx->fseEntropy); ip, iend-ip, LL_defaultDTable, dctx->fseEntropy);
if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); if (ZSTD_isError(llhSize)) return ERROR(corruption_detected);
ip += llhSize; ip += llhSize;
} }
{ size_t const ofhSize = ZSTD_buildSeqTable(dctx->OFTable, &dctx->OFTptr, { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr,
OFtype, MaxOff, OffFSELog, OFtype, MaxOff, OffFSELog,
ip, iend-ip, OF_defaultDTable, dctx->fseEntropy); ip, iend-ip, OF_defaultDTable, dctx->fseEntropy);
if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected);
ip += ofhSize; ip += ofhSize;
} }
{ size_t const mlhSize = ZSTD_buildSeqTable(dctx->MLTable, &dctx->MLTptr, { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr,
MLtype, MaxML, MLFSELog, MLtype, MaxML, MLFSELog,
ip, iend-ip, ML_defaultDTable, dctx->fseEntropy); ip, iend-ip, ML_defaultDTable, dctx->fseEntropy);
if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected);
@@ -1104,7 +1108,7 @@ static size_t ZSTD_decompressSequences(
if (nbSeq) { if (nbSeq) {
seqState_t seqState; seqState_t seqState;
dctx->fseEntropy = 1; dctx->fseEntropy = 1;
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->rep[i]; } { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
@@ -1121,7 +1125,7 @@ static size_t ZSTD_decompressSequences(
/* check if reached exact end */ /* check if reached exact end */
if (nbSeq) return ERROR(corruption_detected); if (nbSeq) return ERROR(corruption_detected);
/* save reps for next block */ /* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); } { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
} }
/* last literal segment */ /* last literal segment */
@@ -1330,7 +1334,7 @@ static size_t ZSTD_decompressSequencesLong(
seqState_t seqState; seqState_t seqState;
int seqNb; int seqNb;
dctx->fseEntropy = 1; dctx->fseEntropy = 1;
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->rep[i]; } { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
seqState.base = base; seqState.base = base;
seqState.pos = (size_t)(op-base); seqState.pos = (size_t)(op-base);
seqState.gotoDict = (iPtrDiff)(dictEnd - base); seqState.gotoDict = (iPtrDiff)(dictEnd - base);
@@ -1365,7 +1369,7 @@ static size_t ZSTD_decompressSequencesLong(
} }
/* save reps for next block */ /* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->rep[i] = (U32)(seqState.prevOffset[i]); } { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
} }
/* last literal segment */ /* last literal segment */
@@ -1843,7 +1847,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
if (dictSize <= 8) return ERROR(dictionary_corrupted); if (dictSize <= 8) return ERROR(dictionary_corrupted);
dictPtr += 8; /* skip header = magic + dictID */ dictPtr += 8; /* skip header = magic + dictID */
{ size_t const hSize = HUF_readDTableX4(dctx->hufTable, dictPtr, dictEnd-dictPtr); { size_t const hSize = HUF_readDTableX4(dctx->entropy.hufTable, dictPtr, dictEnd-dictPtr);
if (HUF_isError(hSize)) return ERROR(dictionary_corrupted); if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
dictPtr += hSize; dictPtr += hSize;
} }
@@ -1853,7 +1857,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); CHECK_E(FSE_buildDTable(dctx->entropy.OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
dictPtr += offcodeHeaderSize; dictPtr += offcodeHeaderSize;
} }
@@ -1862,7 +1866,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); CHECK_E(FSE_buildDTable(dctx->entropy.MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
dictPtr += matchlengthHeaderSize; dictPtr += matchlengthHeaderSize;
} }
@@ -1871,17 +1875,18 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); CHECK_E(FSE_buildDTable(dctx->entropy.LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
dictPtr += litlengthHeaderSize; dictPtr += litlengthHeaderSize;
} }
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] == 0 || dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); { int i;
dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] == 0 || dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); for (i=0; i<3; i++) {
dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] == 0 || dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
dictPtr += 12; if (rep==0 || rep >= dictSize) return ERROR(dictionary_corrupted);
dctx->entropy.rep[i] = rep;
} }
dctx->litEntropy = dctx->fseEntropy = 1;
return dictPtr - (const BYTE*)dict; return dictPtr - (const BYTE*)dict;
} }
@@ -1900,6 +1905,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
dict = (const char*)dict + eSize; dict = (const char*)dict + eSize;
dictSize -= eSize; dictSize -= eSize;
} }
dctx->litEntropy = dctx->fseEntropy = 1;
/* reference dictionary content */ /* reference dictionary content */
return ZSTD_refDictContent(dctx, dict, dictSize); return ZSTD_refDictContent(dctx, dict, dictSize);