ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix long sequences (> 64 KB)

Browse Source
This commit is contained in:
Yann Collet
2024-02-23 15:35:12 -08:00
parent 6b11fc436c
commit 3b40100058
2 changed files with 9 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress_superblock.c
+8 -5
View File
@@ -136,6 +136,7 @@ ZSTD_seqDecompressedSize(seqStore_t const* seqStore,
size_t matchLengthSum = 0; size_t matchLengthSum = 0;
size_t litLengthSum = 0; size_t litLengthSum = 0;
(void)(litLengthSum); /* suppress unused variable warning on some environments */ (void)(litLengthSum); /* suppress unused variable warning on some environments */
DEBUGLOG(6, "ZSTD_seqDecompressedSize (%u sequences from %p) (last==%i)", (unsigned)nbSeq, sp, lastSequence);
while (sp < send) { while (sp < send) {
ZSTD_sequenceLength const seqLen = ZSTD_getSequenceLength(seqStore, sp); ZSTD_sequenceLength const seqLen = ZSTD_getSequenceLength(seqStore, sp);
litLengthSum += seqLen.litLength; litLengthSum += seqLen.litLength;
@@ -279,7 +280,8 @@ static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy,
litSize, nbSeq, writeLitEntropy, writeSeqEntropy, lastBlock); litSize, nbSeq, writeLitEntropy, writeSeqEntropy, lastBlock);
{ size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable, { size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable,
&entropyMetadata->hufMetadata, literals, litSize, &entropyMetadata->hufMetadata, literals, litSize,
op, oend-op, bmi2, writeLitEntropy, litEntropyWritten); op, (size_t)(oend-op),
bmi2, writeLitEntropy, litEntropyWritten);
FORWARD_IF_ERROR(cLitSize, "ZSTD_compressSubBlock_literal failed"); FORWARD_IF_ERROR(cLitSize, "ZSTD_compressSubBlock_literal failed");
if (cLitSize == 0) return 0; if (cLitSize == 0) return 0;
op += cLitSize; op += cLitSize;
@@ -289,7 +291,7 @@ static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy,
sequences, nbSeq, sequences, nbSeq,
llCode, mlCode, ofCode, llCode, mlCode, ofCode,
cctxParams, cctxParams,
op, oend-op, op, (size_t)(oend-op),
bmi2, writeSeqEntropy, seqEntropyWritten); bmi2, writeSeqEntropy, seqEntropyWritten);
FORWARD_IF_ERROR(cSeqSize, "ZSTD_compressSubBlock_sequences failed"); FORWARD_IF_ERROR(cSeqSize, "ZSTD_compressSubBlock_sequences failed");
if (cSeqSize == 0) return 0; if (cSeqSize == 0) return 0;
@@ -419,13 +421,14 @@ static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMe
return 0; return 0;
} }
static size_t countLiterals(const seqDef* sp, size_t seqCount) static size_t countLiterals(seqStore_t const* seqStore, const seqDef* sp, size_t seqCount)
{ {
size_t n, total = 0; size_t n, total = 0;
assert(sp != NULL); assert(sp != NULL);
for (n=0; n<seqCount; n++) { for (n=0; n<seqCount; n++) {
total += sp[n].litLength; total += ZSTD_getSequenceLength(seqStore, sp+n).litLength;
} }
DEBUGLOG(6, "countLiterals for %zu sequences from %p => %zu bytes", seqCount, sp, total);
return total; return total;
} }
@@ -504,7 +507,7 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
int lastSubBlock = (n==nbSubBlocks-1); int lastSubBlock = (n==nbSubBlocks-1);
size_t const nbSeqsLastSubBlock = nbSeqs - (nbSubBlocks-1) * nbSeqsPerBlock; size_t const nbSeqsLastSubBlock = nbSeqs - (nbSubBlocks-1) * nbSeqsPerBlock;
size_t seqCount = lastSubBlock ? nbSeqsLastSubBlock : nbSeqsPerBlock; size_t seqCount = lastSubBlock ? nbSeqsLastSubBlock : nbSeqsPerBlock;
size_t litSize = lastSubBlock ? (size_t)(lend-lp) : countLiterals(sp, seqCount); size_t litSize = lastSubBlock ? (size_t)(lend-lp) : countLiterals(seqStorePtr, sp, seqCount);
int litEntropyWritten = 0; int litEntropyWritten = 0;
int seqEntropyWritten = 0; int seqEntropyWritten = 0;
const size_t decompressedSize = const size_t decompressedSize =
tests/Makefile
+1 -1
View File
@@ -314,7 +314,7 @@ check: shortest
fuzztest: test-fuzzer test-zstream test-decodecorpus fuzztest: test-fuzzer test-zstream test-decodecorpus
.PHONY: test .PHONY: test
test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus test-cli-tests test: test-zstd test-cli-tests test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus
ifeq ($(QEMU_SYS),) ifeq ($(QEMU_SYS),)
test: test-pool test: test-pool
endif endif