ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix unit tests to agree with new changes

Browse Source
This commit is contained in:
senhuang42
2020-11-16 11:36:37 -05:00
parent 7f563b0519
commit 55b90ef010
2 changed files with 34 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+28 -19
View File
@@ -4489,7 +4489,7 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
typedef struct { typedef struct {
U32 idx; /* Index in array of ZSTD_Sequence */ U32 idx; /* Index in array of ZSTD_Sequence */
U32 posInSequence; /* Position within sequence at idx */ U32 posInSequence; /* Position within sequence at idx */
U64 posInSrc; /* Position in src stream */ U64 posInSrc; /* Number of bytes given by sequences provided so far */
} ZSTD_sequencePosition; } ZSTD_sequencePosition;
#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6) #if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6)
@@ -4510,7 +4510,7 @@ static size_t ZSTD_validateSequence(U32 offCode, U32 repCode, U32 matchLength,
} }
#endif #endif
/* Returns offset code, given a raw offset and repcode array */ /* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */
static U32 ZSTD_finalizeOffCode(U32 rawOffset, const U32* const rep, U32 ll0) { static U32 ZSTD_finalizeOffCode(U32 rawOffset, const U32* const rep, U32 ll0) {
U32 offCode = rawOffset + ZSTD_REP_MOVE; U32 offCode = rawOffset + ZSTD_REP_MOVE;
U32 repCode = 0; U32 repCode = 0;
@@ -4525,17 +4525,21 @@ static U32 ZSTD_finalizeOffCode(U32 rawOffset, const U32* const rep, U32 ll0) {
repCode = 3; repCode = 3;
} }
if (repCode) { if (repCode) {
/* ZSTD_storeSeq expects a number in the range [0, 2] to represent a repcode */
offCode = repCode - 1; offCode = repCode - 1;
} }
return offCode; return offCode;
} }
/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of
* ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter.
*/
static size_t ZSTD_copySequencesToSeqStoreBlockDelim(seqStore_t* seqStore, ZSTD_sequencePosition* seqPos, static size_t ZSTD_copySequencesToSeqStoreBlockDelim(seqStore_t* seqStore, ZSTD_sequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
const void* src, size_t blockSize, ZSTD_CCtx* cctx) { const void* src, size_t blockSize, ZSTD_CCtx* cctx) {
size_t idx = seqPos->idx; size_t idx = seqPos->idx;
BYTE const* ip = (BYTE const*)(src); BYTE const* ip = (BYTE const*)(src);
BYTE const* iend = ip + blockSize; const BYTE const* iend = ip + blockSize;
repcodes_t updatedRepcodes; repcodes_t updatedRepcodes;
U32 dictSize; U32 dictSize;
U32 litLength; U32 litLength;
@@ -4545,12 +4549,11 @@ static size_t ZSTD_copySequencesToSeqStoreBlockDelim(seqStore_t* seqStore, ZSTD_
if (cctx->cdict) { if (cctx->cdict) {
dictSize = cctx->cdict->dictContentSize; dictSize = cctx->cdict->dictContentSize;
} else if (cctx->prefixDict.dictSize) { } else if (cctx->prefixDict.dict) {
dictSize = cctx->prefixDict.dictSize; dictSize = cctx->prefixDict.dictSize;
} else { } else {
dictSize = 0; dictSize = 0;
} }
ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t));
for (; (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0) && idx < inSeqsSize; ++idx) { for (; (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0) && idx < inSeqsSize; ++idx) {
litLength = inSeqs[idx].litLength; litLength = inSeqs[idx].litLength;
@@ -4587,7 +4590,15 @@ static size_t ZSTD_copySequencesToSeqStoreBlockDelim(seqStore_t* seqStore, ZSTD_
} }
/* Returns the number of bytes to move the current read position back by. Only non-zero /* Returns the number of bytes to move the current read position back by. Only non-zero
* if we ended up splitting a sequence. * if we ended up splitting a sequence. Otherwise, it may return a ZSTD error if something
* went wrong.
*
* This function will attempt to scan through blockSize bytes represented by the sequences
* in inSeqs, storing any (partial) sequences.
*
* Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to
* avoid splitting a match, or to avoid splitting a match such that it would produce a match
* smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block.
*/ */
static size_t ZSTD_copySequencesToSeqStore(seqStore_t* seqStore, ZSTD_sequencePosition* seqPos, static size_t ZSTD_copySequencesToSeqStore(seqStore_t* seqStore, ZSTD_sequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
@@ -4596,7 +4607,7 @@ static size_t ZSTD_copySequencesToSeqStore(seqStore_t* seqStore, ZSTD_sequencePo
size_t startPosInSequence = seqPos->posInSequence; size_t startPosInSequence = seqPos->posInSequence;
size_t endPosInSequence = seqPos->posInSequence + blockSize; size_t endPosInSequence = seqPos->posInSequence + blockSize;
BYTE const* ip = (BYTE const*)(src); BYTE const* ip = (BYTE const*)(src);
BYTE const* iend = ip + blockSize; BYTE const* iend = ip + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */
repcodes_t updatedRepcodes; repcodes_t updatedRepcodes;
U32 bytesAdjustment = 0; U32 bytesAdjustment = 0;
U32 finalMatchSplit = 0; U32 finalMatchSplit = 0;
@@ -4608,11 +4619,11 @@ static size_t ZSTD_copySequencesToSeqStore(seqStore_t* seqStore, ZSTD_sequencePo
U32 repCode; U32 repCode;
if (cctx->cdict) { if (cctx->cdict) {
dictSize = ZSTD_sizeof_CDict(cctx->cdict); dictSize = cctx->cdict->dictContentSize;
} else if (cctx->prefixDict.dictSize) { } else if (cctx->prefixDict.dict) {
dictSize = cctx->prefixDict.dictSize; dictSize = cctx->prefixDict.dictSize;
} else if (ZSTD_sizeof_localDict(cctx->localDict)) { } else {
dictSize = ZSTD_sizeof_localDict(cctx->localDict); dictSize = 0;
} }
DEBUGLOG(5, "ZSTD_copySequencesToSeqStore: idx: %zu PIS: %u blockSize: %zu windowSize: %u", idx, startPosInSequence, blockSize, windowSize); DEBUGLOG(5, "ZSTD_copySequencesToSeqStore: idx: %zu PIS: %u blockSize: %zu windowSize: %u", idx, startPosInSequence, blockSize, windowSize);
DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength);
@@ -4638,9 +4649,10 @@ static size_t ZSTD_copySequencesToSeqStore(seqStore_t* seqStore, ZSTD_sequencePo
startPosInSequence = 0; startPosInSequence = 0;
idx++; idx++;
} else { } else {
/* This is the final sequence we're adding from inSeqs, and endPosInSequence /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence
does not reach the end of the match. So, we have to split the sequence */ does not reach the end of the match. So, we have to split the sequence */
DEBUGLOG(6, "Require a split: diff: %u, idx: %u PIS: %u", currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence); DEBUGLOG(6, "Require a split: diff: %u, idx: %u PIS: %u",
currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence);
if (endPosInSequence > litLength) { if (endPosInSequence > litLength) {
litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence; litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence;
U32 firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength; U32 firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength;
@@ -4660,7 +4672,8 @@ static size_t ZSTD_copySequencesToSeqStore(seqStore_t* seqStore, ZSTD_sequencePo
} else { } else {
/* Move the position in sequence backwards so that we don't split match, and break to store /* Move the position in sequence backwards so that we don't split match, and break to store
* the last literals. We use the original currSeq.litLength as a marker for where endPosInSequence * the last literals. We use the original currSeq.litLength as a marker for where endPosInSequence
* should go. * should go. We prefer to do this whenever it is not necessary to split the match, or if doing so
* would cause the first half of the match to be too small
*/ */
bytesAdjustment = endPosInSequence - currSeq.litLength; bytesAdjustment = endPosInSequence - currSeq.litLength;
endPosInSequence = currSeq.litLength; endPosInSequence = currSeq.litLength;
@@ -4692,7 +4705,6 @@ static size_t ZSTD_copySequencesToSeqStore(seqStore_t* seqStore, ZSTD_sequencePo
assert(endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength); assert(endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength);
seqPos->idx = idx; seqPos->idx = idx;
seqPos->posInSequence = endPosInSequence; seqPos->posInSequence = endPosInSequence;
/* Update repcodes */
ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t));
iend -= bytesAdjustment; iend -= bytesAdjustment;
@@ -4712,7 +4724,7 @@ static size_t ZSTD_copySequencesToSeqStore(seqStore_t* seqStore, ZSTD_sequencePo
/* Compress, block-by-block, all of the sequences given. /* Compress, block-by-block, all of the sequences given.
* *
* Returns the cumulative size of all compressed blocks (including their headers), otherwise a ZSTD error * Returns the cumulative size of all compressed blocks (including their headers), otherwise a ZSTD error.
*/ */
static size_t ZSTD_compressSequences_internal(void* dst, size_t dstCapacity, static size_t ZSTD_compressSequences_internal(void* dst, size_t dstCapacity,
ZSTD_CCtx* cctx, ZSTD_CCtx* cctx,
@@ -4849,12 +4861,9 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapaci
op += frameHeaderSize; op += frameHeaderSize;
dstCapacity -= frameHeaderSize; dstCapacity -= frameHeaderSize;
cSize += frameHeaderSize; cSize += frameHeaderSize;
/* Update checksum if requested */
if (cctx->appliedParams.fParams.checksumFlag && srcSize) { if (cctx->appliedParams.fParams.checksumFlag && srcSize) {
XXH64_update(&cctx->xxhState, src, srcSize); XXH64_update(&cctx->xxhState, src, srcSize);
} }
/* cSize includes block header size and compressed sequences size */ /* cSize includes block header size and compressed sequences size */
compressedBlocksSize = ZSTD_compressSequences_internal(op, dstCapacity, compressedBlocksSize = ZSTD_compressSequences_internal(op, dstCapacity,
cctx, inSeqs, inSeqsSize, cctx, inSeqs, inSeqsSize,
tests/fuzzer.c
+4 -2
View File
@@ -2764,7 +2764,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* Test with block delimiters roundtrip */ /* Test with block delimiters roundtrip */
seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize); seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
compressedSize = ZSTD_compressSequences(cctx, dst, dstSize, seqs, seqsSize, src, srcSize, ZSTD_sf_explicitBlockDelimiters); ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
compressedSize = ZSTD_compressSequences(cctx, dst, dstSize, seqs, seqsSize, src, srcSize);
if (ZSTD_isError(compressedSize)) { if (ZSTD_isError(compressedSize)) {
DISPLAY("Error in sequence compression with block delims\n"); DISPLAY("Error in sequence compression with block delims\n");
goto _output_error; goto _output_error;
@@ -2779,7 +2780,8 @@ static int basicUnitTests(U32 const seed, double compressibility)
/* Test with no block delimiters roundtrip */ /* Test with no block delimiters roundtrip */
seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize); seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize);
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
compressedSize = ZSTD_compressSequences(cctx, dst, dstSize, seqs, seqsSize, src, srcSize, ZSTD_sf_noBlockDelimiters); ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters);
compressedSize = ZSTD_compressSequences(cctx, dst, dstSize, seqs, seqsSize, src, srcSize);
if (ZSTD_isError(compressedSize)) { if (ZSTD_isError(compressedSize)) {
DISPLAY("Error in sequence compression with no block delims\n"); DISPLAY("Error in sequence compression with no block delims\n");
goto _output_error; goto _output_error;