ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2393 from senhuang42/fix_sequence_extractions_api

Browse Source 
Improve repcode handling in sequence extraction API
This commit is contained in:
sen
2020-11-15 18:29:52 -05:00
committed by GitHub
parent 69b8361b0c 9d936d61d2
commit 9736c497ce
2 changed files with 18 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+16 -16
View File
@@ -2497,12 +2497,14 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc)
ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex]; ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex];
size_t i; size_t i;
int repIdx; repcodes_t updatedRepcodes;
assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences); assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences);
/* Ensure we have enough space for last literals "sequence" */ /* Ensure we have enough space for last literals "sequence" */
assert(zc->seqCollector.maxSequences >= seqStoreSeqSize + 1); assert(zc->seqCollector.maxSequences >= seqStoreSeqSize + 1);
ZSTD_memcpy(updatedRepcodes.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t));
for (i = 0; i < seqStoreSeqSize; ++i) { for (i = 0; i < seqStoreSeqSize; ++i) {
U32 rawOffset = seqStoreSeqs[i].offset - ZSTD_REP_NUM;
outSeqs[i].litLength = seqStoreSeqs[i].litLength; outSeqs[i].litLength = seqStoreSeqs[i].litLength;
outSeqs[i].matchLength = seqStoreSeqs[i].matchLength + MINMATCH; outSeqs[i].matchLength = seqStoreSeqs[i].matchLength + MINMATCH;
outSeqs[i].rep = 0; outSeqs[i].rep = 0;
@@ -2516,27 +2518,26 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc)
} }
if (seqStoreSeqs[i].offset <= ZSTD_REP_NUM) { if (seqStoreSeqs[i].offset <= ZSTD_REP_NUM) {
/* Derive the correct offset corresponding to a repcode */
outSeqs[i].rep = seqStoreSeqs[i].offset; outSeqs[i].rep = seqStoreSeqs[i].offset;
repIdx = (unsigned int)i - seqStoreSeqs[i].offset; if (outSeqs[i].litLength != 0) {
rawOffset = updatedRepcodes.rep[outSeqs[i].rep - 1];
if (seqStoreSeqs[i].litLength == 0) { } else {
if (seqStoreSeqs[i].offset < 3) { if (outSeqs[i].rep == 3) {
--repIdx; rawOffset = updatedRepcodes.rep[0] - 1;
} else { } else {
repIdx = (unsigned int)i - 1; rawOffset = updatedRepcodes.rep[outSeqs[i].rep];
} }
} }
assert(repIdx >= -3);
outSeqs[i].offset = repIdx >= 0 ? outSeqs[repIdx].offset : repStartValue[-repIdx - 1];
if (outSeqs[i].rep == 3 && outSeqs[i].litLength == 0) {
--outSeqs[i].offset;
}
} else {
outSeqs[i].offset = seqStoreSeqs[i].offset - ZSTD_REP_NUM;
} }
outSeqs[i].offset = rawOffset;
/* seqStoreSeqs[i].offset == offCode+1, and ZSTD_updateRep() expects offCode
so we provide seqStoreSeqs[i].offset - 1 */
updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep,
seqStoreSeqs[i].offset - 1,
seqStoreSeqs[i].litLength == 0);
literalsRead += outSeqs[i].litLength; literalsRead += outSeqs[i].litLength;
} }
/* Insert last literals (if any exist) in the block as a sequence with ml == off == 0. /* Insert last literals (if any exist) in the block as a sequence with ml == off == 0.
* If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker * If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker
* for the block boundary, according to the API. * for the block boundary, according to the API.
@@ -2546,7 +2547,6 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc)
outSeqs[i].litLength = (U32)lastLLSize; outSeqs[i].litLength = (U32)lastLLSize;
outSeqs[i].matchLength = outSeqs[i].offset = outSeqs[i].rep = 0; outSeqs[i].matchLength = outSeqs[i].offset = outSeqs[i].rep = 0;
seqStoreSeqSize++; seqStoreSeqSize++;
zc->seqCollector.seqIndex += seqStoreSeqSize; zc->seqCollector.seqIndex += seqStoreSeqSize;
} }
tests/fuzzer.c
+2 -2
View File
@@ -2721,9 +2721,9 @@ static int basicUnitTests(U32 const seed, double compressibility)
if (seqs == NULL) goto _output_error; if (seqs == NULL) goto _output_error;
assert(cctx != NULL); assert(cctx != NULL);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19);
/* Populate src with random data */ /* Populate src with random data */
RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed); RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed);
/* Test with block delimiters roundtrip */ /* Test with block delimiters roundtrip */
seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize); seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);