ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

adding traces to better track processing of literals

Browse Source
This commit is contained in:
Yann Collet
2022-01-26 14:47:21 -08:00
parent a0acf9aa49
commit 7616e39f3b
5 changed files with 135 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/huf_compress.c
+97 -34
View File
@@ -272,7 +272,7 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void
U32 HUF_getNbBitsFromCTable(HUF_CElt const* CTable, U32 symbolValue) U32 HUF_getNbBitsFromCTable(HUF_CElt const* CTable, U32 symbolValue)
{ {
const HUF_CElt* ct = CTable + 1; const HUF_CElt* const ct = CTable + 1;
assert(symbolValue <= HUF_SYMBOLVALUE_MAX); assert(symbolValue <= HUF_SYMBOLVALUE_MAX);
return (U32)HUF_getNbBits(ct[symbolValue]); return (U32)HUF_getNbBits(ct[symbolValue]);
} }
@@ -287,53 +287,56 @@ typedef struct nodeElt_s {
/** /**
* HUF_setMaxHeight(): * HUF_setMaxHeight():
* Enforces maxNbBits on the Huffman tree described in huffNode. * Try to enforce @targetNbBits on the Huffman tree described in @huffNode.
* *
* It sets all nodes with nbBits > maxNbBits to be maxNbBits. Then it adjusts * It attempts to convert all nodes with nbBits > @targetNbBits
* the tree to so that it is a valid canonical Huffman tree. * to employ @targetNbBits instead. Then it adjusts the tree
* so that it remains a valid canonical Huffman tree.
* *
* @pre The sum of the ranks of each symbol == 2^largestBits, * @pre The sum of the ranks of each symbol == 2^largestBits,
* where largestBits == huffNode[lastNonNull].nbBits. * where largestBits == huffNode[lastNonNull].nbBits.
* @post The sum of the ranks of each symbol == 2^largestBits, * @post The sum of the ranks of each symbol == 2^largestBits,
* where largestBits is the return value <= maxNbBits. * where largestBits is the return value (expected <= targetNbBits).
* *
* @param huffNode The Huffman tree modified in place to enforce maxNbBits. * @param huffNode The Huffman tree modified in place to enforce targetNbBits.
* It's presumed sorted, from most frequent to rarest symbol.
* @param lastNonNull The symbol with the lowest count in the Huffman tree. * @param lastNonNull The symbol with the lowest count in the Huffman tree.
* @param maxNbBits The maximum allowed number of bits, which the Huffman tree * @param targetNbBits The allowed number of bits, which the Huffman tree
* may not respect. After this function the Huffman tree will * may not respect. After this function the Huffman tree will
* respect maxNbBits. * respect targetNbBits.
* @return The maximum number of bits of the Huffman tree after adjustment, * @return The maximum number of bits of the Huffman tree after adjustment.
* necessarily no more than maxNbBits.
*/ */
static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 targetNbBits)
{ {
const U32 largestBits = huffNode[lastNonNull].nbBits; const U32 largestBits = huffNode[lastNonNull].nbBits;
/* early exit : no elt > maxNbBits, so the tree is already valid. */ /* early exit : no elt > targetNbBits, so the tree is already valid. */
if (largestBits <= maxNbBits) return largestBits; if (largestBits <= targetNbBits) return largestBits;
DEBUGLOG(5, "HUF_setMaxHeight (targetNbBits = %u)", targetNbBits);
/* there are several too large elements (at least >= 2) */ /* there are several too large elements (at least >= 2) */
{ int totalCost = 0; { int totalCost = 0;
const U32 baseCost = 1 << (largestBits - maxNbBits); const U32 baseCost = 1 << (largestBits - targetNbBits);
int n = (int)lastNonNull; int n = (int)lastNonNull;
/* Adjust any ranks > maxNbBits to maxNbBits. /* Adjust any ranks > targetNbBits to targetNbBits.
* Compute totalCost, which is how far the sum of the ranks is * Compute totalCost, which is how far the sum of the ranks is
* we are over 2^largestBits after adjust the offending ranks. * we are over 2^largestBits after adjust the offending ranks.
*/ */
while (huffNode[n].nbBits > maxNbBits) { while (huffNode[n].nbBits > targetNbBits) {
totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
huffNode[n].nbBits = (BYTE)maxNbBits; huffNode[n].nbBits = (BYTE)targetNbBits;
n--; n--;
} }
/* n stops at huffNode[n].nbBits <= maxNbBits */ /* n stops at huffNode[n].nbBits <= targetNbBits */
assert(huffNode[n].nbBits <= maxNbBits); assert(huffNode[n].nbBits <= targetNbBits);
/* n end at index of smallest symbol using < maxNbBits */ /* n end at index of smallest symbol using < targetNbBits */
while (huffNode[n].nbBits == maxNbBits) --n; while (huffNode[n].nbBits == targetNbBits) --n;
/* renorm totalCost from 2^largestBits to 2^maxNbBits /* renorm totalCost from 2^largestBits to 2^targetNbBits
* note : totalCost is necessarily a multiple of baseCost */ * note : totalCost is necessarily a multiple of baseCost */
assert((totalCost & (baseCost - 1)) == 0); assert((totalCost & (baseCost - 1)) == 0);
totalCost >>= (largestBits - maxNbBits); totalCost >>= (largestBits - targetNbBits);
assert(totalCost > 0); assert(totalCost > 0);
/* repay normalized cost */ /* repay normalized cost */
@@ -342,12 +345,12 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
/* Get pos of last (smallest = lowest cum. count) symbol per rank */ /* Get pos of last (smallest = lowest cum. count) symbol per rank */
ZSTD_memset(rankLast, 0xF0, sizeof(rankLast)); ZSTD_memset(rankLast, 0xF0, sizeof(rankLast));
{ U32 currentNbBits = maxNbBits; { U32 currentNbBits = targetNbBits;
int pos; int pos;
for (pos=n ; pos >= 0; pos--) { for (pos=n ; pos >= 0; pos--) {
if (huffNode[pos].nbBits >= currentNbBits) continue; if (huffNode[pos].nbBits >= currentNbBits) continue;
currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ currentNbBits = huffNode[pos].nbBits; /* < targetNbBits */
rankLast[maxNbBits-currentNbBits] = (U32)pos; rankLast[targetNbBits-currentNbBits] = (U32)pos;
} } } }
while (totalCost > 0) { while (totalCost > 0) {
@@ -394,7 +397,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
rankLast[nBitsToDecrease] = noSymbol; rankLast[nBitsToDecrease] = noSymbol;
else { else {
rankLast[nBitsToDecrease]--; rankLast[nBitsToDecrease]--;
if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) if (huffNode[rankLast[nBitsToDecrease]].nbBits != targetNbBits-nBitsToDecrease)
rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
} }
} /* while (totalCost > 0) */ } /* while (totalCost > 0) */
@@ -406,11 +409,11 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
* TODO. * TODO.
*/ */
while (totalCost < 0) { /* Sometimes, cost correction overshoot */ while (totalCost < 0) { /* Sometimes, cost correction overshoot */
/* special case : no rank 1 symbol (using maxNbBits-1); /* special case : no rank 1 symbol (using targetNbBits-1);
* let's create one from largest rank 0 (using maxNbBits). * let's create one from largest rank 0 (using targetNbBits).
*/ */
if (rankLast[1] == noSymbol) { if (rankLast[1] == noSymbol) {
while (huffNode[n].nbBits == maxNbBits) n--; while (huffNode[n].nbBits == targetNbBits) n--;
huffNode[n+1].nbBits--; huffNode[n+1].nbBits--;
assert(n >= 0); assert(n >= 0);
rankLast[1] = (U32)(n+1); rankLast[1] = (U32)(n+1);
@@ -424,7 +427,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
} /* repay normalized cost */ } /* repay normalized cost */
} /* there are several too large elements (at least >= 2) */ } /* there are several too large elements (at least >= 2) */
return maxNbBits; return targetNbBits;
} }
typedef struct { typedef struct {
@@ -594,6 +597,28 @@ static void HUF_sort(nodeElt huffNode[], const unsigned count[], U32 const maxSy
assert(HUF_isSorted(huffNode, maxSymbolValue1)); assert(HUF_isSorted(huffNode, maxSymbolValue1));
} }
size_t showHNodeSymbols(const nodeElt* hnode, size_t size)
{
size_t u;
for (u=0; u<size; u++) {
RAWLOG(6, " %u", hnode[u].byte);
}
RAWLOG(6, " \n");
return size;
}
size_t showHNodeBits(const nodeElt* hnode, size_t size)
{
size_t u;
for (u=0; u<size; u++) {
RAWLOG(6, " %u", hnode[u].nbBits);
}
RAWLOG(6, " \n");
return size;
}
/** HUF_buildCTable_wksp() : /** HUF_buildCTable_wksp() :
* Same as HUF_buildCTable(), but using externally allocated scratch buffer. * Same as HUF_buildCTable(), but using externally allocated scratch buffer.
* `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as sizeof(HUF_buildCTable_wksp_tables). * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as sizeof(HUF_buildCTable_wksp_tables).
@@ -614,6 +639,7 @@ static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue)
int lowS, lowN; int lowS, lowN;
int nodeNb = STARTNODE; int nodeNb = STARTNODE;
int n, nodeRoot; int n, nodeRoot;
DEBUGLOG(5, "HUF_buildTree (alphabet size = %u)", maxSymbolValue + 1);
/* init for parents */ /* init for parents */
nonNullRank = (int)maxSymbolValue; nonNullRank = (int)maxSymbolValue;
while(huffNode[nonNullRank].count == 0) nonNullRank--; while(huffNode[nonNullRank].count == 0) nonNullRank--;
@@ -640,6 +666,8 @@ static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue)
for (n=0; n<=nonNullRank; n++) for (n=0; n<=nonNullRank; n++)
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
DEBUGLOG(6, "Initial distribution of bits completed (%zu sorted symbols)", showHNodeBits(huffNode, maxSymbolValue+1)); (void)showHNodeBits;
return nonNullRank; return nonNullRank;
} }
@@ -677,13 +705,18 @@ static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, i
CTable[0] = maxNbBits; CTable[0] = maxNbBits;
} }
size_t HUF_buildCTable_wksp (HUF_CElt* CTable, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize) size_t
HUF_buildCTable_wksp(HUF_CElt* CTable, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
void* workSpace, size_t wkspSize)
{ {
HUF_buildCTable_wksp_tables* const wksp_tables = (HUF_buildCTable_wksp_tables*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(U32)); HUF_buildCTable_wksp_tables* const wksp_tables =
(HUF_buildCTable_wksp_tables*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(U32));
nodeElt* const huffNode0 = wksp_tables->huffNodeTbl; nodeElt* const huffNode0 = wksp_tables->huffNodeTbl;
nodeElt* const huffNode = huffNode0+1; nodeElt* const huffNode = huffNode0+1;
int nonNullRank; int nonNullRank;
DEBUGLOG(5, "HUF_buildCTable_wksp (alphabet size = %u)", maxSymbolValue+1);
/* safety checks */ /* safety checks */
if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) if (wkspSize < sizeof(HUF_buildCTable_wksp_tables))
return ERROR(workSpace_tooSmall); return ERROR(workSpace_tooSmall);
@@ -694,11 +727,12 @@ size_t HUF_buildCTable_wksp (HUF_CElt* CTable, const unsigned* count, U32 maxSym
/* sort, decreasing order */ /* sort, decreasing order */
HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition); HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition);
DEBUGLOG(6, "sorted symbols completed (%zu symbols)", showHNodeSymbols(huffNode, maxSymbolValue+1)); (void)showHNodeSymbols;
/* build tree */ /* build tree */
nonNullRank = HUF_buildTree(huffNode, maxSymbolValue); nonNullRank = HUF_buildTree(huffNode, maxSymbolValue);
/* enforce maxTableLog */ /* determine and enforce maxTableLog */
maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits); maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits);
if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */
@@ -1158,6 +1192,28 @@ static size_t HUF_compressCTable_internal(
return (size_t)(op-ostart); return (size_t)(op-ostart);
} }
static size_t showU32(const U32* arr, size_t size)
{
size_t u;
for (u=0; u<size; u++) {
RAWLOG(6, " %u", arr[u]);
}
RAWLOG(6, " \n");
return size;
}
static size_t showCTableBits(const HUF_CElt* ctable, size_t size)
{
size_t u;
for (u=0; u<size; u++) {
RAWLOG(6, " %zu", HUF_getNbBits(ctable[u]));
}
RAWLOG(6, " \n");
return size;
}
typedef struct { typedef struct {
unsigned count[HUF_SYMBOLVALUE_MAX + 1]; unsigned count[HUF_SYMBOLVALUE_MAX + 1];
HUF_CElt CTable[HUF_CTABLE_SIZE_ST(HUF_SYMBOLVALUE_MAX)]; HUF_CElt CTable[HUF_CTABLE_SIZE_ST(HUF_SYMBOLVALUE_MAX)];
@@ -1188,6 +1244,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
BYTE* const oend = ostart + dstSize; BYTE* const oend = ostart + dstSize;
BYTE* op = ostart; BYTE* op = ostart;
DEBUGLOG(5, "HUF_compress_internal (srcSize=%zu)", srcSize);
HUF_STATIC_ASSERT(sizeof(*table) + HUF_WORKSPACE_MAX_ALIGNMENT <= HUF_WORKSPACE_SIZE); HUF_STATIC_ASSERT(sizeof(*table) + HUF_WORKSPACE_MAX_ALIGNMENT <= HUF_WORKSPACE_SIZE);
/* checks & inits */ /* checks & inits */
@@ -1211,6 +1268,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
DEBUG_STATIC_ASSERT(SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO >= 2); DEBUG_STATIC_ASSERT(SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO >= 2);
if (suspectUncompressible && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) { if (suspectUncompressible && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) {
size_t largestTotal = 0; size_t largestTotal = 0;
DEBUGLOG(5, "input suspected incompressible : sampling to check");
{ unsigned maxSymbolValueBegin = maxSymbolValue; { unsigned maxSymbolValueBegin = maxSymbolValue;
CHECK_V_F(largestBegin, HIST_count_simple (table->count, &maxSymbolValueBegin, (const BYTE*)src, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) ); CHECK_V_F(largestBegin, HIST_count_simple (table->count, &maxSymbolValueBegin, (const BYTE*)src, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) );
largestTotal += largestBegin; largestTotal += largestBegin;
@@ -1227,6 +1285,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */ if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */
} }
DEBUGLOG(6, "histogram detail completed (%zu symbols)", showU32(table->count, maxSymbolValue+1)); (void)showU32;
/* Check validity of previous table */ /* Check validity of previous table */
if ( repeat if ( repeat
@@ -1248,6 +1307,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
&table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp)); &table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp));
CHECK_F(maxBits); CHECK_F(maxBits);
huffLog = (U32)maxBits; huffLog = (U32)maxBits;
DEBUGLOG(6, "bit distribution completed (%zu symbols)", showCTableBits(table->CTable + 1, maxSymbolValue+1)); (void)showCTableBits;
} }
/* Zero unused symbols in CTable, so we can check it for validity */ /* Zero unused symbols in CTable, so we can check it for validity */
{ {
@@ -1300,6 +1360,7 @@ size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat,
int bmi2, unsigned suspectUncompressible) int bmi2, unsigned suspectUncompressible)
{ {
DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize);
return HUF_compress_internal(dst, dstSize, src, srcSize, return HUF_compress_internal(dst, dstSize, src, srcSize,
maxSymbolValue, huffLog, HUF_singleStream, maxSymbolValue, huffLog, HUF_singleStream,
workSpace, wkspSize, hufTable, workSpace, wkspSize, hufTable,
@@ -1314,6 +1375,7 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
unsigned maxSymbolValue, unsigned huffLog, unsigned maxSymbolValue, unsigned huffLog,
void* workSpace, size_t wkspSize) void* workSpace, size_t wkspSize)
{ {
DEBUGLOG(5, "HUF_compress4X_wksp (srcSize = %zu)", srcSize);
return HUF_compress_internal(dst, dstSize, src, srcSize, return HUF_compress_internal(dst, dstSize, src, srcSize,
maxSymbolValue, huffLog, HUF_fourStreams, maxSymbolValue, huffLog, HUF_fourStreams,
workSpace, wkspSize, workSpace, wkspSize,
@@ -1330,6 +1392,7 @@ size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
void* workSpace, size_t wkspSize, void* workSpace, size_t wkspSize,
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible) HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible)
{ {
DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize);
return HUF_compress_internal(dst, dstSize, src, srcSize, return HUF_compress_internal(dst, dstSize, src, srcSize,
maxSymbolValue, huffLog, HUF_fourStreams, maxSymbolValue, huffLog, HUF_fourStreams,
workSpace, wkspSize, workSpace, wkspSize,
lib/compress/zstd_compress.c
+7 -6
View File
@@ -5850,12 +5850,13 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
return 0; return 0;
} }
/* Returns the number of bytes to move the current read position back by. Only non-zero /* Returns the number of bytes to move the current read position back by.
* if we ended up splitting a sequence. Otherwise, it may return a ZSTD error if something * Only non-zero if we ended up splitting a sequence.
* went wrong. * Otherwise, it may return a ZSTD error if something went wrong.
* *
* This function will attempt to scan through blockSize bytes represented by the sequences * This function will attempt to scan through blockSize bytes
* in inSeqs, storing any (partial) sequences. * represented by the sequences in @inSeqs,
* storing any (partial) sequences.
* *
* Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to
* avoid splitting a match, or to avoid splitting a match such that it would produce a match * avoid splitting a match, or to avoid splitting a match such that it would produce a match
@@ -5883,7 +5884,7 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
} else { } else {
dictSize = 0; dictSize = 0;
} }
DEBUGLOG(5, "ZSTD_copySequencesToSeqStore: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize); DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreNoBlockDelim: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize);
DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength);
ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t));
while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) { while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) {
lib/compress/zstd_compress_literals.c
+25 -11
View File
@@ -36,7 +36,7 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
} }
ZSTD_memcpy(ostart + flSize, src, srcSize); ZSTD_memcpy(ostart + flSize, src, srcSize);
DEBUGLOG(5, "Raw literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize)); DEBUGLOG(5, "Raw (uncompressed) literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize));
return srcSize + flSize; return srcSize + flSize;
} }
@@ -67,6 +67,17 @@ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void*
return flSize+1; return flSize+1;
} }
static size_t showHexa(const void* src, size_t srcSize)
{
const BYTE* const ip = (const BYTE*)src;
size_t u;
for (u=0; u<srcSize; u++) {
RAWLOG(6, " %02X", ip[u]);
}
RAWLOG(6, " \n");
return srcSize;
}
size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
ZSTD_hufCTables_t* nextHuf, ZSTD_hufCTables_t* nextHuf,
ZSTD_strategy strategy, int disableLiteralCompression, ZSTD_strategy strategy, int disableLiteralCompression,
@@ -86,6 +97,8 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i srcSize=%u)", DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i srcSize=%u)",
disableLiteralCompression, (U32)srcSize); disableLiteralCompression, (U32)srcSize);
DEBUGLOG(6, "Completed literals listing (%zu bytes)", showHexa(src, srcSize)); (void)showHexa;
/* Prepare nextEntropy assuming reusing the existing table */ /* Prepare nextEntropy assuming reusing the existing table */
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
@@ -100,17 +113,18 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression"); RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression");
{ HUF_repeat repeat = prevHuf->repeatMode; { HUF_repeat repeat = prevHuf->repeatMode;
int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; int const preferRepeat = (strategy < ZSTD_lazy) ? srcSize <= 1024 : 0;
typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int, int, unsigned);
huf_compress_f huf_compress;
if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
cLitSize = singleStream ? huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat;
HUF_compress1X_repeat( cLitSize = huf_compress(ostart+lhSize, dstCapacity-lhSize,
ostart+lhSize, dstCapacity-lhSize, src, srcSize, src, srcSize,
HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize, HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT,
(HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible) : entropyWorkspace, entropyWorkspaceSize,
HUF_compress4X_repeat( (HUF_CElt*)nextHuf->CTable,
ostart+lhSize, dstCapacity-lhSize, src, srcSize, &repeat, preferRepeat,
HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize, bmi2, suspectUncompressible);
(HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible);
if (repeat != HUF_repeat_none) { if (repeat != HUF_repeat_none) {
/* reused the existing table */ /* reused the existing table */
DEBUGLOG(5, "Reusing previous huffman table"); DEBUGLOG(5, "Reusing previous huffman table");
lib/decompress/zstd_decompress_block.c
+1 -1
View File
@@ -1993,7 +1993,7 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
/* Decode literals section */ /* Decode literals section */
{ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming); { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming);
DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize); DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize);
if (ZSTD_isError(litCSize)) return litCSize; if (ZSTD_isError(litCSize)) return litCSize;
ip += litCSize; ip += litCSize;
srcSize -= litCSize; srcSize -= litCSize;
lib/zstd.h
+3 -1
View File
@@ -1409,7 +1409,9 @@ ZSTD_generateSequences( ZSTD_CCtx* zc,
ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
/*! ZSTD_compressSequences() : /*! ZSTD_compressSequences() :
* Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst. * Compress an array of ZSTD_Sequence, associted with @src buffer, into dst.
* @src contains the entire input (not just the literals).
* If @srcSize > sum(sequence.length), the remaining bytes are considered all literals
* If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
* The entire source is compressed into a single frame. * The entire source is compressed into a single frame.
* *