From 95424409eab01b82c6dea5186e4235a219d090f2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 9 Feb 2018 04:25:15 -0800 Subject: [PATCH 1/8] addBits and baseline into FSE decoding table note : unfinished - need new default tables - need modify long mode --- lib/common/zstd_internal.h | 7 +- lib/compress/zstd_compress.c | 6 +- lib/decompress/zstd_decompress.c | 284 ++++++++++++++++++++++++------- 3 files changed, 228 insertions(+), 69 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 49f19a010..65c08a825 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -132,14 +132,15 @@ typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingTy #define Litbits 8 #define MaxLit ((1<tableLog = 0; + DTableH->fastMode = 0; + + cell->nbBits = 0; + cell->nextState = 0; + cell->nbAdditionalBits = 0; + cell->baseValue = symbolValue; +} + + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) */ +static void +ZSTD_buildFSETable(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U32* nbAdditionalBits, + unsigned tableLog) +{ + ZSTD_seqSymbol* const tableDecode = dt+1; + U16 symbolNext[MaxSeq+1]; + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize-1; + + /* Sanity Checks */ + assert(maxSymbolValue <= MaxSeq); + assert(tableLog <= MaxFSELog); + + /* Init, lay down lowprob symbols */ + { ZSTD_seqSymbol_header DTableH; + DTableH.tableLog = tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + symbolNext[s] = normalizedCounter[s]; + } } } + memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + { U32 const tableMask = tableSize-1; + U32 const step = FSE_TABLESTEP(tableSize); + U32 s, position = 0; + for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; u max) return ERROR(corruption_detected); - FSE_buildDTable_rle(DTableSpace, *(const BYTE*)src); + ZSTD_buildSeqTable_rle(DTableSpace, *(const BYTE*)src); *DTablePtr = DTableSpace; return 1; case set_basic : + assert(0); /* need to rebuild all default tables */ *DTablePtr = &defaultTable->dtable; return 0; case set_repeat: @@ -755,12 +850,40 @@ static size_t ZSTD_buildSeqTable(FSE_DTable* DTableSpace, const FSE_DTable** DTa size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); if (FSE_isError(headerSize)) return ERROR(corruption_detected); if (tableLog > maxLog) return ERROR(corruption_detected); - FSE_buildDTable(DTableSpace, norm, max, tableLog); + ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog); *DTablePtr = DTableSpace; return headerSize; } } } +static const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; + +static const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; + +static const U32 OF_bits[MaxOff+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31 }; + +static const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, + 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, const void* src, size_t srcSize) { @@ -797,19 +920,27 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, /* Build DTables */ { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, LLtype, MaxLL, LLFSELog, - ip, iend-ip, LL_defaultDTable, dctx->fseEntropy); + ip, iend-ip, + LL_base, LL_bits, + LL_defaultDTable, dctx->fseEntropy); if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); ip += llhSize; } + { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, OFtype, MaxOff, OffFSELog, - ip, iend-ip, OF_defaultDTable, dctx->fseEntropy); + ip, iend-ip, + OF_base, OF_bits, + OF_defaultDTable, dctx->fseEntropy); if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); ip += ofhSize; } + { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, MLtype, MaxML, MLFSELog, - ip, iend-ip, ML_defaultDTable, dctx->fseEntropy); + ip, iend-ip, + ML_base, ML_bits, + ML_defaultDTable, dctx->fseEntropy); if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); ip += mlhSize; } @@ -826,11 +957,16 @@ typedef struct { const BYTE* match; } seq_t; +typedef struct { + size_t state; + const ZSTD_seqSymbol* table; +} ZSTD_fseState; + typedef struct { BIT_DStream_t DStream; - FSE_DState_t stateLL; - FSE_DState_t stateOffb; - FSE_DState_t stateML; + ZSTD_fseState stateLL; + ZSTD_fseState stateOffb; + ZSTD_fseState stateML; size_t prevOffset[ZSTD_REP_NUM]; const BYTE* prefixStart; const BYTE* dictEnd; @@ -885,6 +1021,16 @@ size_t ZSTD_execSequenceLast7(BYTE* op, } +static void +ZSTD_updateFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD) +{ + ZSTD_seqSymbol const DInfo = DStatePtr->table[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.nextState + lowBits; +} + + typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; /* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum @@ -900,41 +1046,17 @@ typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset static seq_t ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) { seq_t seq; - - U32 const llCode = FSE_peekSymbol(&seqState->stateLL); - U32 const mlCode = FSE_peekSymbol(&seqState->stateML); - U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= MaxOff, by table construction */ - - U32 const llBits = LL_bits[llCode]; - U32 const mlBits = ML_bits[mlCode]; - U32 const ofBits = ofCode; + U32 const llBits = seqState->stateLL.table[seqState->stateLL.state].nbAdditionalBits; + U32 const mlBits = seqState->stateML.table[seqState->stateML.state].nbAdditionalBits; + U32 const ofBits = seqState->stateOffb.table[seqState->stateOffb.state].nbAdditionalBits; U32 const totalBits = llBits+mlBits+ofBits; - - static const U32 LL_base[MaxLL+1] = { - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 18, 20, 22, 24, 28, 32, 40, - 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, - 0x2000, 0x4000, 0x8000, 0x10000 }; - - static const U32 ML_base[MaxML+1] = { - 3, 4, 5, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 29, 30, 31, 32, 33, 34, - 35, 37, 39, 41, 43, 47, 51, 59, - 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, - 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; - - static const U32 OF_base[MaxOff+1] = { - 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, - 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, - 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, - 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; + U32 const llBase = seqState->stateLL.table[seqState->stateLL.state].baseValue; + U32 const mlBase = seqState->stateML.table[seqState->stateML.state].baseValue; + U32 const ofBase = seqState->stateOffb.table[seqState->stateOffb.state].baseValue; /* sequence */ { size_t offset; - if (!ofCode) + if (!ofBits) offset = 0; else { ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); @@ -942,18 +1064,18 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e l assert(ofBits <= MaxOff); if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed); - offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); BIT_reloadDStream(&seqState->DStream); if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32); /* to avoid another reload */ } else { - offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); } } - if (ofCode <= 1) { - offset += (llCode==0); + if (ofBits <= 1) { + offset += (llBase==0); if (offset) { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ @@ -971,8 +1093,8 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e l seq.offset = offset; } - seq.matchLength = ML_base[mlCode] - + ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/) : 0); /* <= 16 bits */ + seq.matchLength = mlBase + + ((mlBits>0) ? BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/) : 0); /* <= 16 bits */ if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) BIT_reloadDStream(&seqState->DStream); if (MEM_64bits() && (totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) @@ -980,8 +1102,8 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e l /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); - seq.litLength = LL_base[llCode] - + ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits/*>0*/) : 0); /* <= 16 bits */ + seq.litLength = llBase + + ((llBits>0) ? BIT_readBitsFast(&seqState->DStream, llBits/*>0*/) : 0); /* <= 16 bits */ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); @@ -989,10 +1111,10 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e l (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); /* ANS state update */ - FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ - FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ + ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ + ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ + ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ return seq; } @@ -1079,6 +1201,18 @@ size_t ZSTD_execSequence(BYTE* op, } +static void +ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) +{ + const void* ptr = dt; + const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits", + (U32)DStatePtr->state, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + static size_t ZSTD_decompressSequences( ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, @@ -1112,9 +1246,12 @@ static size_t ZSTD_decompressSequences( dctx->fseEntropy = 1; { U32 i; for (i=0; ientropy.rep[i]; } CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); - FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); - FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); - FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + DEBUGLOG(6, "ZSTD_decompressSequences: init stateLL"); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + DEBUGLOG(6, "ZSTD_decompressSequences: init stateOffb"); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + DEBUGLOG(6, "ZSTD_decompressSequences: init stateML"); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { nbSeq--; @@ -1143,6 +1280,8 @@ static size_t ZSTD_decompressSequences( } +#if 0 + HINT_INLINE seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const longOffsets) { @@ -1407,6 +1546,7 @@ static size_t ZSTD_decompressSequencesLong( return op-ostart; } +#endif static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, @@ -1430,10 +1570,12 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, ip += litCSize; srcSize -= litCSize; } +#if 0 if ( frame /* windowSize exists */ && (dctx->fParams.windowSize > (1<<24)) && MEM_64bits() /* x86 benefits less from long mode than x64 */ ) return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, isLongOffset); +#endif return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, isLongOffset); } @@ -1929,8 +2071,12 @@ static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy, const void* const U32 offcodeMaxValue = MaxOff, offcodeLog; size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + if (offcodeMaxValue > MaxOff) return ERROR(dictionary_corrupted); if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); - CHECK_E(FSE_buildDTable(entropy->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); + ZSTD_buildFSETable(entropy->OFTable, + offcodeNCount, offcodeMaxValue, + OF_base, OF_bits, + offcodeLog); dictPtr += offcodeHeaderSize; } @@ -1938,8 +2084,12 @@ static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy, const void* const unsigned matchlengthMaxValue = MaxML, matchlengthLog; size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (matchlengthMaxValue > MaxML) return ERROR(dictionary_corrupted); if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); - CHECK_E(FSE_buildDTable(entropy->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); + ZSTD_buildFSETable(entropy->MLTable, + matchlengthNCount, matchlengthMaxValue, + ML_base, ML_bits, + matchlengthLog); dictPtr += matchlengthHeaderSize; } @@ -1947,8 +2097,12 @@ static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy, const void* const unsigned litlengthMaxValue = MaxLL, litlengthLog; size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (litlengthMaxValue > MaxLL) return ERROR(dictionary_corrupted); if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); - CHECK_E(FSE_buildDTable(entropy->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); + ZSTD_buildFSETable(entropy->LLTable, + litlengthNCount, litlengthMaxValue, + LL_base, LL_bits, + litlengthLog); dictPtr += litlengthHeaderSize; } From 421a2716d8e9c6ed796357810c8502aa4002222e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 9 Feb 2018 04:50:58 -0800 Subject: [PATCH 2/8] fixed default fse distributions but would be better to pre-calculate tables, for speed --- lib/decompress/zstd_decompress.c | 33 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 215dfd482..0181351b1 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -82,8 +82,8 @@ typedef enum { zdss_init=0, zdss_loadHeader, typedef struct { - U32 tableLog; U32 fastMode; + U32 tableLog; } ZSTD_seqSymbol_header; typedef struct { @@ -665,28 +665,22 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, } } - -typedef union { - FSE_decode_t realData; - FSE_DTable dtable; - U32 alignedBy4; -} FSE_decode_t4; - +#if 0 /* Default FSE distribution table for Literal Lengths */ -static const FSE_decode_t4 LL_defaultDTable[(1<dtable; + //*DTablePtr = &defaultTable->dtable; + ZSTD_buildFSETable(DTableSpace, defaultNorm, max, baseValue, nbAdditionalBits, defaultLog); + *DTablePtr = DTableSpace; return 0; case set_repeat: if (!flagRepeatTable) return ERROR(corruption_detected); @@ -922,7 +919,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_base, LL_bits, - LL_defaultDTable, dctx->fseEntropy); + LL_defaultNorm, LL_defaultNormLog, dctx->fseEntropy); if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); ip += llhSize; } @@ -931,7 +928,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_base, OF_bits, - OF_defaultDTable, dctx->fseEntropy); + OF_defaultNorm, OF_defaultNormLog, dctx->fseEntropy); if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); ip += ofhSize; } @@ -940,7 +937,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_base, ML_bits, - ML_defaultDTable, dctx->fseEntropy); + ML_defaultNorm, ML_defaultNormLog, dctx->fseEntropy); if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); ip += mlhSize; } From ab75df21ed9ab71a521f5d7f43f9e31f3f064d24 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 9 Feb 2018 05:12:13 -0800 Subject: [PATCH 3/8] fixed mono-symbol distribution --- lib/decompress/zstd_decompress.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 0181351b1..bc0e3fb3d 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -734,7 +734,7 @@ static const FSE_decode_t4 OF_defaultDTable[(1<nbBits = 0; cell->nextState = 0; - cell->nbAdditionalBits = 0; - cell->baseValue = symbolValue; + assert(nbAddBits < 255); + cell->nbAdditionalBits = (BYTE)nbAddBits; + cell->baseValue = baseValue; } @@ -828,12 +829,15 @@ static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymb case set_rle : if (!srcSize) return ERROR(srcSize_wrong); if ( (*(const BYTE*)src) > max) return ERROR(corruption_detected); - ZSTD_buildSeqTable_rle(DTableSpace, *(const BYTE*)src); + { U32 const symbol = *(const BYTE*)src; + U32 const baseline = baseValue[symbol]; + U32 const nbBits = nbAdditionalBits[symbol]; + ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); + } *DTablePtr = DTableSpace; return 1; case set_basic : - assert(0); /* need to rebuild all default tables */ - //*DTablePtr = &defaultTable->dtable; + //*DTablePtr = &defaultTable->dtable; // when default tables will be pre-built ZSTD_buildFSETable(DTableSpace, defaultNorm, max, baseValue, nbAdditionalBits, defaultLog); *DTablePtr = DTableSpace; return 0; From 1850597eaa42ecda915e8578844197be16e74876 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 9 Feb 2018 06:01:02 -0800 Subject: [PATCH 4/8] pre-calculated default decoding tables --- lib/decompress/zstd_decompress.c | 166 ++++++++++++++++++------------- 1 file changed, 98 insertions(+), 68 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index bc0e3fb3d..c09d73b29 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -665,74 +665,105 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, } } -#if 0 /* Default FSE distribution table for Literal Lengths */ static const ZSTD_seqSymbol LL_defaultDTable[(1<dtable; // when default tables will be pre-built - ZSTD_buildFSETable(DTableSpace, defaultNorm, max, baseValue, nbAdditionalBits, defaultLog); - *DTablePtr = DTableSpace; + *DTablePtr = defaultTable; return 0; case set_repeat: if (!flagRepeatTable) return ERROR(corruption_detected); @@ -885,6 +914,7 @@ static const U32 ML_base[MaxML+1] = { 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, const void* src, size_t srcSize) { @@ -923,7 +953,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_base, LL_bits, - LL_defaultNorm, LL_defaultNormLog, dctx->fseEntropy); + LL_defaultDTable, dctx->fseEntropy); if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); ip += llhSize; } @@ -932,7 +962,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_base, OF_bits, - OF_defaultNorm, OF_defaultNormLog, dctx->fseEntropy); + OF_defaultDTable, dctx->fseEntropy); if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); ip += ofhSize; } @@ -941,7 +971,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_base, ML_bits, - ML_defaultNorm, ML_defaultNormLog, dctx->fseEntropy); + ML_defaultDTable, dctx->fseEntropy); if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); ip += mlhSize; } From 6bfe50ad480472814de2b43616e30876042fd6d6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 9 Feb 2018 09:14:25 -0800 Subject: [PATCH 5/8] re-enabled ZSTD_decompressSequencesLong() --- lib/decompress/zstd_decompress.c | 73 ++++++++++---------------------- 1 file changed, 22 insertions(+), 51 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index c09d73b29..d63526920 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1277,11 +1277,8 @@ static size_t ZSTD_decompressSequences( dctx->fseEntropy = 1; { U32 i; for (i=0; ientropy.rep[i]; } CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); - DEBUGLOG(6, "ZSTD_decompressSequences: init stateLL"); ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); - DEBUGLOG(6, "ZSTD_decompressSequences: init stateOffb"); ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); - DEBUGLOG(6, "ZSTD_decompressSequences: init stateML"); ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { @@ -1294,7 +1291,7 @@ static size_t ZSTD_decompressSequences( } } /* check if reached exact end */ - DEBUGLOG(5, "after decode loop, remaining nbSeq : %i", nbSeq); + DEBUGLOG(5, "ZSTD_decompressSequences: after decode loop, remaining nbSeq : %i", nbSeq); if (nbSeq) return ERROR(corruption_detected); /* save reps for next block */ { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } @@ -1311,47 +1308,22 @@ static size_t ZSTD_decompressSequences( } -#if 0 HINT_INLINE seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const longOffsets) { seq_t seq; - - U32 const llCode = FSE_peekSymbol(&seqState->stateLL); - U32 const mlCode = FSE_peekSymbol(&seqState->stateML); - U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= MaxOff, by table construction */ - - U32 const llBits = LL_bits[llCode]; - U32 const mlBits = ML_bits[mlCode]; - U32 const ofBits = ofCode; + U32 const llBits = seqState->stateLL.table[seqState->stateLL.state].nbAdditionalBits; + U32 const mlBits = seqState->stateML.table[seqState->stateML.state].nbAdditionalBits; + U32 const ofBits = seqState->stateOffb.table[seqState->stateOffb.state].nbAdditionalBits; U32 const totalBits = llBits+mlBits+ofBits; - - static const U32 LL_base[MaxLL+1] = { - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 18, 20, 22, 24, 28, 32, 40, - 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, - 0x2000, 0x4000, 0x8000, 0x10000 }; - - static const U32 ML_base[MaxML+1] = { - 3, 4, 5, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 29, 30, 31, 32, 33, 34, - 35, 37, 39, 41, 43, 47, 51, 59, - 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, - 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; - - static const U32 OF_base[MaxOff+1] = { - 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, - 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, - 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, - 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; + U32 const llBase = seqState->stateLL.table[seqState->stateLL.state].baseValue; + U32 const mlBase = seqState->stateML.table[seqState->stateML.state].baseValue; + U32 const ofBase = seqState->stateOffb.table[seqState->stateOffb.state].baseValue; /* sequence */ { size_t offset; - if (!ofCode) + if (!ofBits) offset = 0; else { ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); @@ -1359,17 +1331,17 @@ seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const long assert(ofBits <= MaxOff); if (MEM_32bits() && longOffsets) { U32 const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN_32-1); - offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); if (MEM_32bits() || extraBits) BIT_reloadDStream(&seqState->DStream); if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); } else { - offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); } } - if (ofCode <= 1) { - offset += (llCode==0); + if (ofBits <= 1) { + offset += (llBase==0); if (offset) { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ @@ -1387,7 +1359,7 @@ seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const long seq.offset = offset; } - seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ + seq.matchLength = mlBase + ((mlBits>0) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) BIT_reloadDStream(&seqState->DStream); if (MEM_64bits() && (totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) @@ -1395,7 +1367,7 @@ seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const long /* Verify that there is enough bits to read the rest of the data in 64-bit mode. */ ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); - seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ + seq.litLength = llBase + ((llBits>0) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); @@ -1407,10 +1379,10 @@ seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const long } /* ANS state update */ - FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ - FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ + ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ + ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ + ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ return seq; } @@ -1534,9 +1506,9 @@ static size_t ZSTD_decompressSequencesLong( seqState.pos = (size_t)(op-prefixStart); seqState.dictEnd = dictEnd; CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); - FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); - FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); - FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); /* prepare in advance */ for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNbfParams.windowSize > (1<<24)) && MEM_64bits() /* x86 benefits less from long mode than x64 */ ) return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, isLongOffset); -#endif + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, isLongOffset); } From 71c07966bbedf478b21dfe62f1507b2a4b96f85f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 12 Feb 2018 16:52:15 -0800 Subject: [PATCH 6/8] added SEQSYMBOL_TABLE_SIZE() as suggested by @terrelln's comment --- lib/decompress/zstd_decompress.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index d63526920..d65dd5fd9 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -93,10 +93,12 @@ typedef struct { U32 baseValue; } ZSTD_seqSymbol; +#define SEQSYMBOL_TABLE_SIZE(log) (1 + (1< STREAM_ACCUMULATOR_MIN_32 \ - ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \ +#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \ + (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \ + ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \ : 0) static seq_t ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) From 2524cbd8475add0c962f4e7ab0f5fd9eb5b30085 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 13 Feb 2018 10:02:25 -0800 Subject: [PATCH 7/8] added code comment on how to generate default tables as suggested by @terrelln --- lib/decompress/zstd_decompress.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index d65dd5fd9..36dfc6376 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -667,6 +667,15 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, } } +/* Default FSE distribution tables. + * These are pre-calculated FSE decoding tables using default distributions as defined in specification : + * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#default-distributions + * They were generated programmatically with following method : + * - start from default distributions, present in /lib/common/zstd_internal.h + * - generate tables normally, using ZSTD_buildFSETable() + * - printout the content of tables + * - pretify output, report below, test with fuzzer to ensure it's correct */ + /* Default FSE distribution table for Literal Lengths */ static const ZSTD_seqSymbol LL_defaultDTable[(1< Date: Tue, 13 Feb 2018 10:09:01 -0800 Subject: [PATCH 8/8] pretify last unit test output --- tests/fuzzer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index e7c92edc0..aecc06449 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1209,7 +1209,7 @@ static int basicUnitTests(U32 seed, double compressibility) if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error; DISPLAYLEVEL(3, "OK \n"); - DISPLAYLEVEL(4, "test%3i : testing ZSTD dictionary sizes : ", testNb++); + DISPLAYLEVEL(3, "test%3i : testing ZSTD dictionary sizes : ", testNb++); RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); { size_t const size = MIN(128 KB, CNBuffSize); @@ -1230,6 +1230,7 @@ static int basicUnitTests(U32 seed, double compressibility) ZSTD_freeCDict(lgCDict); ZSTD_freeCCtx(cctx); } + DISPLAYLEVEL(3, "OK \n"); _end: free(CNBuffer);