ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

introduce macros STORE_OFFSET() and STORE_REPCODE()

Browse Source 
this meant to abstract the sumtype representation required
to transfert `offcode` to `ZSTD_storeSeq()`.

Unfortunately, the sumtype numeric representation is currently a leaky abstraction
that has permeated many other parts of the code,
especially within `zstd_lazy.c` and also within `zstd_opt.c` and `zstd_compress.c`.

While this PR makes a good job a transfering a large nb of call sites
to using the new macros, there are still a few sites where this transformation is more complex,
or where the numeric representation itself it used "as is".

One of the problematics area is the decision to use the numeric format of the sumtype
within the match finders of `zstd_lazy`.

This commit doesn't change the behavior, it only introduces and employes the macros,
but eventually the resulting code remains identical.

At target, if the numeric representation of the sumtype can be completely abstracted
and no other part of the code depends on it,
it will be possible to move it towards something slightly more efficient.
This commit is contained in:
Yann Collet
2021-12-23 22:03:30 -08:00
parent bec7bbb5a4
commit 1aed962216
8 changed files with 158 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+61 -55
View File
@@ -3449,11 +3449,16 @@ static U32 ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32
/** /**
* ZSTD_seqStore_resolveOffCodes() reconciles any possible divergences in offset history that may arise * ZSTD_seqStore_resolveOffCodes() reconciles any possible divergences in offset history that may arise
* due to emission of RLE/raw blocks that disturb the offset history, and replaces any repcodes within * due to emission of RLE/raw blocks that disturb the offset history,
* the seqStore that may be invalid. * and replaces any repcodes within the seqStore that may be invalid.
* *
* dRepcodes are updated as would be on the decompression side. cRepcodes are updated exactly in * dRepcodes are updated as would be on the decompression side.
* accordance with the seqStore. * cRepcodes are updated exactly in accordance with the seqStore.
*
* Note : this function assumes seq->offBase respects the following numbering scheme :
* 0 : invalid
* 1-3 : repcode 1-3
* 4+ : real_offset+3
*/ */
static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes, static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes,
seqStore_t* const seqStore, U32 const nbSeq) { seqStore_t* const seqStore, U32 const nbSeq) {
@@ -3461,9 +3466,9 @@ static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_
for (; idx < nbSeq; ++idx) { for (; idx < nbSeq; ++idx) {
seqDef* const seq = seqStore->sequencesStart + idx; seqDef* const seq = seqStore->sequencesStart + idx;
U32 const ll0 = (seq->litLength == 0); U32 const ll0 = (seq->litLength == 0);
U32 offCode = seq->offBase - 1; U32 const offCode = seq->offBase - 1;
assert(seq->offBase > 0); assert(seq->offBase > 0);
if (offCode <= ZSTD_REP_MOVE) { if (offCode < ZSTD_REP_NUM) {
U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offCode, ll0); U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offCode, ll0);
U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offCode, ll0); U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offCode, ll0);
/* Adjust simulated decompression repcode history if we come across a mismatch. Replace /* Adjust simulated decompression repcode history if we come across a mismatch. Replace
@@ -5738,39 +5743,39 @@ typedef struct {
size_t posInSrc; /* Number of bytes given by sequences provided so far */ size_t posInSrc; /* Number of bytes given by sequences provided so far */
} ZSTD_sequencePosition; } ZSTD_sequencePosition;
/* Returns a ZSTD error code if sequence is not valid */ /* ZSTD_validateSequence() :
static size_t ZSTD_validateSequence(U32 offCode, U32 matchLength, * @offCode : is presumed to follow format required by ZSTD_storeSeq()
size_t posInSrc, U32 windowLog, size_t dictSize, U32 minMatch) { * @returns a ZSTD error code if sequence is not valid
size_t offsetBound; */
U32 windowSize = 1 << windowLog; static size_t
ZSTD_validateSequence(U32 offCode, U32 matchLength,
size_t posInSrc, U32 windowLog, size_t dictSize)
{
U32 const windowSize = 1 << windowLog;
/* posInSrc represents the amount of data the the decoder would decode up to this point. /* posInSrc represents the amount of data the the decoder would decode up to this point.
* As long as the amount of data decoded is less than or equal to window size, offsets may be * As long as the amount of data decoded is less than or equal to window size, offsets may be
* larger than the total length of output decoded in order to reference the dict, even larger than * larger than the total length of output decoded in order to reference the dict, even larger than
* window size. After output surpasses windowSize, we're limited to windowSize offsets again. * window size. After output surpasses windowSize, we're limited to windowSize offsets again.
*/ */
offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize; size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize;
RETURN_ERROR_IF(offCode > offsetBound + ZSTD_REP_MOVE, corruption_detected, "Offset too large!"); RETURN_ERROR_IF(offCode > STORE_OFFSET(offsetBound), corruption_detected, "Offset too large!");
RETURN_ERROR_IF(matchLength < minMatch, corruption_detected, "Matchlength too small"); RETURN_ERROR_IF(matchLength < MINMATCH, corruption_detected, "Matchlength too small");
return 0; return 0;
} }
/* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */ /* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */
static U32 ZSTD_finalizeOffCode(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) { static U32 ZSTD_finalizeOffCode(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0)
U32 offCode = rawOffset + ZSTD_REP_MOVE; {
U32 repCode = 0; U32 offCode = STORE_OFFSET(rawOffset);
if (!ll0 && rawOffset == rep[0]) { if (!ll0 && rawOffset == rep[0]) {
repCode = 1; offCode = STORE_REPCODE_1;
} else if (rawOffset == rep[1]) { } else if (rawOffset == rep[1]) {
repCode = 2 - ll0; offCode = STORE_REPCODE(2 - ll0);
} else if (rawOffset == rep[2]) { } else if (rawOffset == rep[2]) {
repCode = 3 - ll0; offCode = STORE_REPCODE(3 - ll0);
} else if (ll0 && rawOffset == rep[0] - 1) { } else if (ll0 && rawOffset == rep[0] - 1) {
repCode = 3; offCode = STORE_REPCODE_3;
}
if (repCode) {
/* ZSTD_storeSeq expects a number in the range [0, 2] to represent a repcode */
offCode = repCode - 1;
} }
return offCode; return offCode;
} }
@@ -5778,18 +5783,17 @@ static U32 ZSTD_finalizeOffCode(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32
/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of /* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of
* ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter. * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter.
*/ */
static size_t ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, static size_t
ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
ZSTD_sequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
const void* src, size_t blockSize) { const void* src, size_t blockSize)
{
U32 idx = seqPos->idx; U32 idx = seqPos->idx;
BYTE const* ip = (BYTE const*)(src); BYTE const* ip = (BYTE const*)(src);
const BYTE* const iend = ip + blockSize; const BYTE* const iend = ip + blockSize;
repcodes_t updatedRepcodes; repcodes_t updatedRepcodes;
U32 dictSize; U32 dictSize;
U32 litLength;
U32 matchLength;
U32 ll0;
U32 offCode;
if (cctx->cdict) { if (cctx->cdict) {
dictSize = (U32)cctx->cdict->dictContentSize; dictSize = (U32)cctx->cdict->dictContentSize;
@@ -5800,18 +5804,17 @@ static size_t ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, ZS
} }
ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t));
for (; (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0) && idx < inSeqsSize; ++idx) { for (; (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0) && idx < inSeqsSize; ++idx) {
litLength = inSeqs[idx].litLength; U32 const litLength = inSeqs[idx].litLength;
matchLength = inSeqs[idx].matchLength; U32 const ll0 = (litLength == 0);
ll0 = litLength == 0; U32 const matchLength = inSeqs[idx].matchLength;
offCode = ZSTD_finalizeOffCode(inSeqs[idx].offset, updatedRepcodes.rep, ll0); U32 const offCode = ZSTD_finalizeOffCode(inSeqs[idx].offset, updatedRepcodes.rep, ll0);
updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0);
DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength);
if (cctx->appliedParams.validateSequences) { if (cctx->appliedParams.validateSequences) {
seqPos->posInSrc += litLength + matchLength; seqPos->posInSrc += litLength + matchLength;
FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc, FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc,
cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.cParams.windowLog, dictSize),
cctx->appliedParams.cParams.minMatch),
"Sequence validation failed"); "Sequence validation failed");
} }
RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation, RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation,
@@ -5843,9 +5846,11 @@ static size_t ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, ZS
* avoid splitting a match, or to avoid splitting a match such that it would produce a match * avoid splitting a match, or to avoid splitting a match such that it would produce a match
* smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block. * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block.
*/ */
static size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, static size_t
ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
const void* src, size_t blockSize) { const void* src, size_t blockSize)
{
U32 idx = seqPos->idx; U32 idx = seqPos->idx;
U32 startPosInSequence = seqPos->posInSequence; U32 startPosInSequence = seqPos->posInSequence;
U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize; U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize;
@@ -5855,10 +5860,6 @@ static size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_seq
repcodes_t updatedRepcodes; repcodes_t updatedRepcodes;
U32 bytesAdjustment = 0; U32 bytesAdjustment = 0;
U32 finalMatchSplit = 0; U32 finalMatchSplit = 0;
U32 litLength;
U32 matchLength;
U32 rawOffset;
U32 offCode;
if (cctx->cdict) { if (cctx->cdict) {
dictSize = cctx->cdict->dictContentSize; dictSize = cctx->cdict->dictContentSize;
@@ -5872,9 +5873,10 @@ static size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_seq
ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t));
while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) { while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) {
const ZSTD_Sequence currSeq = inSeqs[idx]; const ZSTD_Sequence currSeq = inSeqs[idx];
litLength = currSeq.litLength; U32 litLength = currSeq.litLength;
matchLength = currSeq.matchLength; U32 matchLength = currSeq.matchLength;
rawOffset = currSeq.offset; U32 const rawOffset = currSeq.offset;
U32 offCode;
/* Modify the sequence depending on where endPosInSequence lies */ /* Modify the sequence depending on where endPosInSequence lies */
if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) { if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) {
@@ -5927,7 +5929,7 @@ static size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_seq
} }
} }
/* Check if this offset can be represented with a repcode */ /* Check if this offset can be represented with a repcode */
{ U32 ll0 = (litLength == 0); { U32 const ll0 = (litLength == 0);
offCode = ZSTD_finalizeOffCode(rawOffset, updatedRepcodes.rep, ll0); offCode = ZSTD_finalizeOffCode(rawOffset, updatedRepcodes.rep, ll0);
updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); updatedRepcodes = ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0);
} }
@@ -5935,8 +5937,7 @@ static size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_seq
if (cctx->appliedParams.validateSequences) { if (cctx->appliedParams.validateSequences) {
seqPos->posInSrc += litLength + matchLength; seqPos->posInSrc += litLength + matchLength;
FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc, FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc,
cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.cParams.windowLog, dictSize),
cctx->appliedParams.cParams.minMatch),
"Sequence validation failed"); "Sequence validation failed");
} }
DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength);
@@ -5967,7 +5968,8 @@ static size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_seq
typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
const void* src, size_t blockSize); const void* src, size_t blockSize);
static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode) { static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode)
{
ZSTD_sequenceCopier sequenceCopier = NULL; ZSTD_sequenceCopier sequenceCopier = NULL;
assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, mode)); assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, mode));
if (mode == ZSTD_sf_explicitBlockDelimiters) { if (mode == ZSTD_sf_explicitBlockDelimiters) {
@@ -5981,12 +5983,15 @@ static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode)
/* Compress, block-by-block, all of the sequences given. /* Compress, block-by-block, all of the sequences given.
* *
* Returns the cumulative size of all compressed blocks (including their headers), otherwise a ZSTD error. * Returns the cumulative size of all compressed blocks (including their headers),
* otherwise a ZSTD error.
*/ */
static size_t ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, static size_t
ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize, const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
const void* src, size_t srcSize) { const void* src, size_t srcSize)
{
size_t cSize = 0; size_t cSize = 0;
U32 lastBlock; U32 lastBlock;
size_t blockSize; size_t blockSize;
@@ -5996,7 +6001,7 @@ static size_t ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
BYTE const* ip = (BYTE const*)src; BYTE const* ip = (BYTE const*)src;
BYTE* op = (BYTE*)dst; BYTE* op = (BYTE*)dst;
ZSTD_sequenceCopier sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters); ZSTD_sequenceCopier const sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters);
DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize); DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize);
/* Special case: empty frame */ /* Special case: empty frame */
@@ -6096,7 +6101,8 @@ static size_t ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapacity, size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapacity,
const ZSTD_Sequence* inSeqs, size_t inSeqsSize, const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
const void* src, size_t srcSize) { const void* src, size_t srcSize)
{
BYTE* op = (BYTE*)dst; BYTE* op = (BYTE*)dst;
size_t cSize = 0; size_t cSize = 0;
size_t compressedBlocksSize = 0; size_t compressedBlocksSize = 0;
lib/compress/zstd_compress_internal.h
+20 -7
View File
@@ -501,15 +501,20 @@ typedef struct repcodes_s {
U32 rep[3]; U32 rep[3];
} repcodes_t; } repcodes_t;
MEM_STATIC repcodes_t ZSTD_updateRep(U32 const rep[3], U32 const offset, U32 const ll0)
/* ZSTD_updateRep() :
* @offcode : expects a scale where 0,1,2 represent repcodes 1-3, and 2+ represents real_offset+2
*/
MEM_STATIC repcodes_t
ZSTD_updateRep(U32 const rep[3], U32 const offcode, U32 const ll0)
{ {
repcodes_t newReps; repcodes_t newReps;
if (offset >= ZSTD_REP_NUM) { /* full offset */ if (offcode >= ZSTD_REP_NUM) { /* full offset */
newReps.rep[2] = rep[1]; newReps.rep[2] = rep[1];
newReps.rep[1] = rep[0]; newReps.rep[1] = rep[0];
newReps.rep[0] = offset - ZSTD_REP_MOVE; newReps.rep[0] = offcode - ZSTD_REP_MOVE;
} else { /* repcode */ } else { /* repcode */
U32 const repCode = offset + ll0; U32 const repCode = offcode + ll0;
if (repCode > 0) { /* note : if repCode==0, no change */ if (repCode > 0) { /* note : if repCode==0, no change */
U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
newReps.rep[2] = (repCode >= 2) ? rep[1] : rep[2]; newReps.rep[2] = (repCode >= 2) ? rep[1] : rep[2];
@@ -600,14 +605,22 @@ static void ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const ie
while (ip < iend) *op++ = *ip++; while (ip < iend) *op++ = *ip++;
} }
#define STORE_REPCODE_1 STORE_REPCODE(1)
#define STORE_REPCODE_2 STORE_REPCODE(2)
#define STORE_REPCODE_3 STORE_REPCODE(3)
#define STORE_REPCODE(r) (assert((r)>=1), assert((r)<=3), (r)-1)
#define STORE_OFFSET(o) (assert((o)>0), o + ZSTD_REP_MOVE)
/*! ZSTD_storeSeq() : /*! ZSTD_storeSeq() :
* Store a sequence (litlen, litPtr, offCode and matchLength) into seqStore_t. * Store a sequence (litlen, litPtr, offCode and matchLength) into seqStore_t.
* `offCode` : distance to match + ZSTD_REP_MOVE (values <= ZSTD_REP_MOVE are repCodes). * @offBase_minus1 : distance to match + ZSTD_REP_MOVE (values <= ZSTD_REP_MOVE are repCodes).
* Users should not specify the encoded value directly,
* instead use macros STORE_REPCODE_X and STORE_OFFSET().
* @matchLength : must be >= MINMATCH * @matchLength : must be >= MINMATCH
* Allowed to overread literals up to litLimit. * Allowed to overread literals up to litLimit.
*/ */
HINT_INLINE UNUSED_ATTR HINT_INLINE UNUSED_ATTR
void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, U32 offCode, size_t matchLength) void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, U32 offBase_minus1, size_t matchLength)
{ {
BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH; BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH;
BYTE const* const litEnd = literals + litLength; BYTE const* const litEnd = literals + litLength;
@@ -647,7 +660,7 @@ void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* litera
seqStorePtr->sequences[0].litLength = (U16)litLength; seqStorePtr->sequences[0].litLength = (U16)litLength;
/* match offset */ /* match offset */
seqStorePtr->sequences[0].offBase = offCode + 1; seqStorePtr->sequences[0].offBase = offBase_minus1 + 1;
/* match Length */ /* match Length */
assert(matchLength >= MINMATCH); assert(matchLength >= MINMATCH);
lib/compress/zstd_double_fast.c
+10 -10
View File
@@ -131,7 +131,7 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic(
if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {
mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
ip++; ip++;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength); ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength);
goto _match_stored; goto _match_stored;
} }
@@ -217,7 +217,7 @@ _match_found: /* requires ip, offset, mLength */
hashLong[hl1] = (U32)(ip1 - base); hashLong[hl1] = (U32)(ip1 - base);
} }
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength); ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
_match_stored: _match_stored:
/* match found */ /* match found */
@@ -243,7 +243,7 @@ _match_stored:
U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base);
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base);
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, rLength); ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, rLength);
ip += rLength; ip += rLength;
anchor = ip; anchor = ip;
continue; /* faster when present ... (?) */ continue; /* faster when present ... (?) */
@@ -328,7 +328,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
ip++; ip++;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength); ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength);
goto _match_stored; goto _match_stored;
} }
@@ -419,7 +419,7 @@ _match_found:
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength); ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
_match_stored: _match_stored:
/* match found */ /* match found */
@@ -448,7 +448,7 @@ _match_stored:
const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4;
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2); ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, repLength2);
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
ip += repLength2; ip += repLength2;
@@ -585,7 +585,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
ip++; ip++;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength); ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength);
} else { } else {
if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) {
const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend;
@@ -596,7 +596,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength); ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
} else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) {
size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
@@ -621,7 +621,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
} }
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength); ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
} else { } else {
ip += ((ip-anchor) >> kSearchStrength) + 1; ip += ((ip-anchor) >> kSearchStrength) + 1;
@@ -653,7 +653,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2); ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, repLength2);
hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
ip += repLength2; ip += repLength2;
lib/compress/zstd_fast.c
+10 -10
View File
@@ -180,7 +180,7 @@ _start: /* Requires: ip0 */
mLength = ip0[-1] == match0[-1]; mLength = ip0[-1] == match0[-1];
ip0 -= mLength; ip0 -= mLength;
match0 -= mLength; match0 -= mLength;
offcode = 0; offcode = STORE_REPCODE_1;
mLength += 4; mLength += 4;
goto _match; goto _match;
} }
@@ -267,7 +267,7 @@ _offset: /* Requires: ip0, idx */
match0 = base + idx; match0 = base + idx;
rep_offset2 = rep_offset1; rep_offset2 = rep_offset1;
rep_offset1 = (U32)(ip0-match0); rep_offset1 = (U32)(ip0-match0);
offcode = rep_offset1 + ZSTD_REP_MOVE; offcode = STORE_OFFSET(rep_offset1);
mLength = 4; mLength = 4;
/* Count the backwards match length. */ /* Count the backwards match length. */
@@ -306,7 +306,7 @@ _match: /* Requires: ip0, match0, offcode */
{ U32 const tmpOff = rep_offset2; rep_offset2 = rep_offset1; rep_offset1 = tmpOff; } /* swap rep_offset2 <=> rep_offset1 */ { U32 const tmpOff = rep_offset2; rep_offset2 = rep_offset1; rep_offset1 = tmpOff; } /* swap rep_offset2 <=> rep_offset1 */
hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base);
ip0 += rLength; ip0 += rLength;
ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, 0 /*offCode*/, rLength); ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, STORE_REPCODE_1, rLength);
anchor = ip0; anchor = ip0;
continue; /* faster when present (confirmed on gcc-8) ... (?) */ continue; /* faster when present (confirmed on gcc-8) ... (?) */
} } } } } }
@@ -439,7 +439,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
ip++; ip++;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength); ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength);
} else if ( (matchIndex <= prefixStartIndex) ) { } else if ( (matchIndex <= prefixStartIndex) ) {
size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls); size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls);
U32 const dictMatchIndex = dictHashTable[dictHash]; U32 const dictMatchIndex = dictHashTable[dictHash];
@@ -459,7 +459,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
} /* catch up */ } /* catch up */
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength); ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
} }
} else if (MEM_read32(match) != MEM_read32(ip)) { } else if (MEM_read32(match) != MEM_read32(ip)) {
/* it's not a match, and we're not going to check the dictionary */ /* it's not a match, and we're not going to check the dictionary */
@@ -474,7 +474,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
&& (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
offset_2 = offset_1; offset_2 = offset_1;
offset_1 = offset; offset_1 = offset;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength); ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
} }
/* match found */ /* match found */
@@ -499,7 +499,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2); ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, repLength2);
hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2;
ip += repLength2; ip += repLength2;
anchor = ip; anchor = ip;
@@ -598,7 +598,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
size_t const rLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4; size_t const rLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4;
ip++; ip++;
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, rLength); ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, rLength);
ip += rLength; ip += rLength;
anchor = ip; anchor = ip;
} else { } else {
@@ -614,7 +614,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
size_t mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; size_t mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4;
while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
offset_2 = offset_1; offset_1 = offset; /* update offset history */ offset_2 = offset_1; offset_1 = offset; /* update offset history */
ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength); ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
ip += mLength; ip += mLength;
anchor = ip; anchor = ip;
} } } }
@@ -633,7 +633,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
{ U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */ { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, 0 /*offcode*/, repLength2); ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, STORE_REPCODE_1, repLength2);
hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2;
ip += repLength2; ip += repLength2;
anchor = ip; anchor = ip;
lib/compress/zstd_lazy.c
+14 -12
View File
@@ -197,8 +197,8 @@ ZSTD_DUBT_findBetterDictMatch (
U32 matchIndex = dictMatchIndex + dictIndexDelta; U32 matchIndex = dictMatchIndex + dictIndexDelta;
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) { if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) {
DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)", DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)",
curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, ZSTD_REP_MOVE + curr - matchIndex, dictMatchIndex, matchIndex); curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, STORE_OFFSET(curr - matchIndex), dictMatchIndex, matchIndex);
bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex; bestLength = matchLength, *offsetPtr = STORE_OFFSET(curr - matchIndex);
} }
if (ip+matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */ if (ip+matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */
break; /* drop, to guarantee consistency (miss a little bit of compression) */ break; /* drop, to guarantee consistency (miss a little bit of compression) */
@@ -328,7 +328,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
if (matchLength > matchEndIdx - matchIndex) if (matchLength > matchEndIdx - matchIndex)
matchEndIdx = matchIndex + (U32)matchLength; matchEndIdx = matchIndex + (U32)matchLength;
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) )
bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + curr - matchIndex; bestLength = matchLength, *offsetPtr = STORE_OFFSET(curr - matchIndex);
if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */
if (dictMode == ZSTD_dictMatchState) { if (dictMode == ZSTD_dictMatchState) {
nbCompares = 0; /* in addition to avoiding checking any nbCompares = 0; /* in addition to avoiding checking any
@@ -561,7 +561,7 @@ size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nb
/* save best solution */ /* save best solution */
if (currentMl > ml) { if (currentMl > ml) {
ml = currentMl; ml = currentMl;
*offsetPtr = curr - (matchIndex + ddsIndexDelta) + ZSTD_REP_MOVE; *offsetPtr = STORE_OFFSET(curr - (matchIndex + ddsIndexDelta));
if (ip+currentMl == iLimit) { if (ip+currentMl == iLimit) {
/* best possible, avoids read overflow on next attempt */ /* best possible, avoids read overflow on next attempt */
return ml; return ml;
@@ -598,7 +598,7 @@ size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nb
/* save best solution */ /* save best solution */
if (currentMl > ml) { if (currentMl > ml) {
ml = currentMl; ml = currentMl;
*offsetPtr = curr - (matchIndex + ddsIndexDelta) + ZSTD_REP_MOVE; *offsetPtr = STORE_OFFSET(curr - (matchIndex + ddsIndexDelta));
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
} }
} }
@@ -703,7 +703,7 @@ size_t ZSTD_HcFindBestMatch(
/* save best solution */ /* save best solution */
if (currentMl > ml) { if (currentMl > ml) {
ml = currentMl; ml = currentMl;
*offsetPtr = curr - matchIndex + ZSTD_REP_MOVE; *offsetPtr = STORE_OFFSET(curr - matchIndex);
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
} }
@@ -738,7 +738,8 @@ size_t ZSTD_HcFindBestMatch(
/* save best solution */ /* save best solution */
if (currentMl > ml) { if (currentMl > ml) {
ml = currentMl; ml = currentMl;
*offsetPtr = curr - (matchIndex + dmsIndexDelta) + ZSTD_REP_MOVE; assert(curr > matchIndex + dmsIndexDelta);
*offsetPtr = STORE_OFFSET(curr - (matchIndex + dmsIndexDelta));
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
} }
@@ -1244,7 +1245,7 @@ size_t ZSTD_RowFindBestMatch(
/* Save best solution */ /* Save best solution */
if (currentMl > ml) { if (currentMl > ml) {
ml = currentMl; ml = currentMl;
*offsetPtr = curr - matchIndex + ZSTD_REP_MOVE; *offsetPtr = STORE_OFFSET(curr - matchIndex);
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
} }
} }
@@ -1292,7 +1293,8 @@ size_t ZSTD_RowFindBestMatch(
if (currentMl > ml) { if (currentMl > ml) {
ml = currentMl; ml = currentMl;
*offsetPtr = curr - (matchIndex + dmsIndexDelta) + ZSTD_REP_MOVE; assert(curr > matchIndex + dmsIndexDelta);
*offsetPtr = STORE_OFFSET(curr - (matchIndex + dmsIndexDelta));
if (ip+currentMl == iLimit) break; if (ip+currentMl == iLimit) break;
} }
} }
@@ -1685,7 +1687,7 @@ _storeSequence:
const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend; const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4; matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4;
offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset_2 <=> offset_1 */ offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset_2 <=> offset_1 */
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength); ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength);
ip += matchLength; ip += matchLength;
anchor = ip; anchor = ip;
continue; continue;
@@ -1700,7 +1702,7 @@ _storeSequence:
/* store sequence */ /* store sequence */
matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */ offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength); ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength);
ip += matchLength; ip += matchLength;
anchor = ip; anchor = ip;
continue; /* faster when present ... (?) */ continue; /* faster when present ... (?) */
@@ -2028,7 +2030,7 @@ _storeSequence:
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset history */ offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset history */
ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength); ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength);
ip += matchLength; ip += matchLength;
anchor = ip; anchor = ip;
continue; /* faster when present ... (?) */ continue; /* faster when present ... (?) */
lib/compress/zstd_ldm.c
+1 -1
View File
@@ -709,7 +709,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
rep[0] = sequence.offset; rep[0] = sequence.offset;
/* Store the sequence */ /* Store the sequence */
ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend, ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend,
sequence.offset + ZSTD_REP_MOVE, STORE_OFFSET(sequence.offset),
sequence.matchLength); sequence.matchLength);
ip += sequence.matchLength; ip += sequence.matchLength;
} }
lib/compress/zstd_opt.c
+22 -19
View File
@@ -280,15 +280,17 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP
/* ZSTD_getMatchPrice() : /* ZSTD_getMatchPrice() :
* Provides the cost of the match part (offset + matchLength) of a sequence * Provides the cost of the match part (offset + matchLength) of a sequence
* Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence. * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence.
* optLevel: when <2, favors small offset for decompression speed (improved cache efficiency) */ * @offcode : expects a scale where 0,1,2 are repcodes 1-3, and 3+ are real_offsets+2
* @optLevel: when <2, favors small offset for decompression speed (improved cache efficiency)
*/
FORCE_INLINE_TEMPLATE U32 FORCE_INLINE_TEMPLATE U32
ZSTD_getMatchPrice(U32 const offset, ZSTD_getMatchPrice(U32 const offcode,
U32 const matchLength, U32 const matchLength,
const optState_t* const optPtr, const optState_t* const optPtr,
int const optLevel) int const optLevel)
{ {
U32 price; U32 price;
U32 const offCode = ZSTD_highbit32(offset+1); U32 const offCode = ZSTD_highbit32(offcode+1);
U32 const mlBase = matchLength - MINMATCH; U32 const mlBase = matchLength - MINMATCH;
assert(matchLength >= MINMATCH); assert(matchLength >= MINMATCH);
@@ -631,7 +633,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u", DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u",
repCode, ll0, repOffset, repLen); repCode, ll0, repOffset, repLen);
bestLength = repLen; bestLength = repLen;
matches[mnum].off = repCode - ll0; matches[mnum].off = STORE_REPCODE(repCode - ll0 + 1); /* expect value between 1 and 3 */
matches[mnum].len = (U32)repLen; matches[mnum].len = (U32)repLen;
mnum++; mnum++;
if ( (repLen > sufficient_len) if ( (repLen > sufficient_len)
@@ -660,7 +662,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
bestLength = mlen; bestLength = mlen;
assert(curr > matchIndex3); assert(curr > matchIndex3);
assert(mnum==0); /* no prior solution */ assert(mnum==0); /* no prior solution */
matches[0].off = (curr - matchIndex3) + ZSTD_REP_MOVE; matches[0].off = STORE_OFFSET(curr - matchIndex3);
matches[0].len = (U32)mlen; matches[0].len = (U32)mlen;
mnum = 1; mnum = 1;
if ( (mlen > sufficient_len) | if ( (mlen > sufficient_len) |
@@ -694,12 +696,12 @@ U32 ZSTD_insertBtAndGetAllMatches (
if (matchLength > bestLength) { if (matchLength > bestLength) {
DEBUGLOG(8, "found match of length %u at distance %u (offCode=%u)", DEBUGLOG(8, "found match of length %u at distance %u (offCode=%u)",
(U32)matchLength, curr - matchIndex, curr - matchIndex + ZSTD_REP_MOVE); (U32)matchLength, curr - matchIndex, STORE_OFFSET(curr - matchIndex));
assert(matchEndIdx > matchIndex); assert(matchEndIdx > matchIndex);
if (matchLength > matchEndIdx - matchIndex) if (matchLength > matchEndIdx - matchIndex)
matchEndIdx = matchIndex + (U32)matchLength; matchEndIdx = matchIndex + (U32)matchLength;
bestLength = matchLength; bestLength = matchLength;
matches[mnum].off = (curr - matchIndex) + ZSTD_REP_MOVE; matches[mnum].off = STORE_OFFSET(curr - matchIndex);
matches[mnum].len = (U32)matchLength; matches[mnum].len = (U32)matchLength;
mnum++; mnum++;
if ( (matchLength > ZSTD_OPT_NUM) if ( (matchLength > ZSTD_OPT_NUM)
@@ -742,11 +744,11 @@ U32 ZSTD_insertBtAndGetAllMatches (
if (matchLength > bestLength) { if (matchLength > bestLength) {
matchIndex = dictMatchIndex + dmsIndexDelta; matchIndex = dictMatchIndex + dmsIndexDelta;
DEBUGLOG(8, "found dms match of length %u at distance %u (offCode=%u)", DEBUGLOG(8, "found dms match of length %u at distance %u (offCode=%u)",
(U32)matchLength, curr - matchIndex, curr - matchIndex + ZSTD_REP_MOVE); (U32)matchLength, curr - matchIndex, STORE_OFFSET(curr - matchIndex));
if (matchLength > matchEndIdx - matchIndex) if (matchLength > matchEndIdx - matchIndex)
matchEndIdx = matchIndex + (U32)matchLength; matchEndIdx = matchIndex + (U32)matchLength;
bestLength = matchLength; bestLength = matchLength;
matches[mnum].off = (curr - matchIndex) + ZSTD_REP_MOVE; matches[mnum].off = STORE_OFFSET(curr - matchIndex);
matches[mnum].len = (U32)matchLength; matches[mnum].len = (U32)matchLength;
mnum++; mnum++;
if ( (matchLength > ZSTD_OPT_NUM) if ( (matchLength > ZSTD_OPT_NUM)
@@ -938,11 +940,12 @@ static void ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 cu
* and 'matchEndPosInBlock', into 'matches'. Maintains the correct ordering of 'matches' * and 'matchEndPosInBlock', into 'matches'. Maintains the correct ordering of 'matches'
*/ */
static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
ZSTD_optLdm_t* optLdm, U32 currPosInBlock) { ZSTD_optLdm_t* optLdm, U32 currPosInBlock)
U32 posDiff = currPosInBlock - optLdm->startPosInBlock; {
U32 const posDiff = currPosInBlock - optLdm->startPosInBlock;
/* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */ /* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */
U32 candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff; U32 const candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff;
U32 candidateOffCode = optLdm->offset + ZSTD_REP_MOVE; U32 const candidateOffCode = STORE_OFFSET(optLdm->offset);
/* Ensure that current block position is not outside of the match */ /* Ensure that current block position is not outside of the match */
if (currPosInBlock < optLdm->startPosInBlock if (currPosInBlock < optLdm->startPosInBlock
@@ -1075,14 +1078,14 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
/* large match -> immediate encoding */ /* large match -> immediate encoding */
{ U32 const maxML = matches[nbMatches-1].len; { U32 const maxML = matches[nbMatches-1].len;
U32 const maxOffset = matches[nbMatches-1].off; U32 const maxOffcode = matches[nbMatches-1].off;
DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffCode=%u at cPos=%u => start new series", DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffCode=%u at cPos=%u => start new series",
nbMatches, maxML, maxOffset, (U32)(ip-prefixStart)); nbMatches, maxML, maxOffcode, (U32)(ip-prefixStart));
if (maxML > sufficient_len) { if (maxML > sufficient_len) {
lastSequence.litlen = litlen; lastSequence.litlen = litlen;
lastSequence.mlen = maxML; lastSequence.mlen = maxML;
lastSequence.off = maxOffset; lastSequence.off = maxOffcode;
DEBUGLOG(6, "large match (%u>%u), immediate encoding", DEBUGLOG(6, "large match (%u>%u), immediate encoding",
maxML, sufficient_len); maxML, sufficient_len);
cur = 0; cur = 0;
@@ -1099,15 +1102,15 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
opt[pos].price = ZSTD_MAX_PRICE; /* mlen, litlen and price will be fixed during forward scanning */ opt[pos].price = ZSTD_MAX_PRICE; /* mlen, litlen and price will be fixed during forward scanning */
} }
for (matchNb = 0; matchNb < nbMatches; matchNb++) { for (matchNb = 0; matchNb < nbMatches; matchNb++) {
U32 const offset = matches[matchNb].off; U32 const offcode = matches[matchNb].off;
U32 const end = matches[matchNb].len; U32 const end = matches[matchNb].len;
for ( ; pos <= end ; pos++ ) { for ( ; pos <= end ; pos++ ) {
U32 const matchPrice = ZSTD_getMatchPrice(offset, pos, optStatePtr, optLevel); U32 const matchPrice = ZSTD_getMatchPrice(offcode, pos, optStatePtr, optLevel);
U32 const sequencePrice = literalsPrice + matchPrice; U32 const sequencePrice = literalsPrice + matchPrice;
DEBUGLOG(7, "rPos:%u => set initial price : %.2f", DEBUGLOG(7, "rPos:%u => set initial price : %.2f",
pos, ZSTD_fCost(sequencePrice)); pos, ZSTD_fCost(sequencePrice));
opt[pos].mlen = pos; opt[pos].mlen = pos;
opt[pos].off = offset; opt[pos].off = offcode;
opt[pos].litlen = litlen; opt[pos].litlen = litlen;
opt[pos].price = (int)sequencePrice; opt[pos].price = (int)sequencePrice;
} } } }
tests/decodecorpus.c
+15 -18
View File
@@ -633,17 +633,16 @@ static inline void initSeqStore(seqStore_t *seqStore) {
} }
/* Randomly generate sequence commands */ /* Randomly generate sequence commands */
static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, static U32
generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
size_t contentSize, size_t literalsSize, dictInfo info) size_t contentSize, size_t literalsSize, dictInfo info)
{ {
/* The total length of all the matches */ /* The total length of all the matches */
size_t const remainingMatch = contentSize - literalsSize; size_t const remainingMatch = contentSize - literalsSize;
size_t excessMatch = 0; size_t excessMatch = 0;
U32 numSequences = 0; U32 numSequences = 0;
U32 i; U32 i;
const BYTE* literals = LITERAL_BUFFER; const BYTE* literals = LITERAL_BUFFER;
BYTE* srcPtr = frame->src; BYTE* srcPtr = frame->src;
@@ -703,32 +702,31 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
lenPastStart = MIN(lenPastStart+MIN_SEQ_LEN, (U32)info.dictContentSize); lenPastStart = MIN(lenPastStart+MIN_SEQ_LEN, (U32)info.dictContentSize);
offset = (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart) + lenPastStart; offset = (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart) + lenPastStart;
} }
{ { U32 const matchLenBound = MIN(frame->header.windowSize, lenPastStart);
U32 const matchLenBound = MIN(frame->header.windowSize, lenPastStart);
matchLen = MIN(matchLen, matchLenBound); matchLen = MIN(matchLen, matchLenBound);
} }
} }
} }
offsetCode = offset + ZSTD_REP_MOVE; offsetCode = STORE_OFFSET(offset);
repIndex = 2; repIndex = 2;
} else { } else {
/* do a repeat offset */ /* do a repeat offset */
offsetCode = RAND(seed) % 3; U32 const randomRepIndex = RAND(seed) % 3;
offsetCode = STORE_REPCODE(randomRepIndex + 1); /* expects values between 1 & 3 */
if (literalLen > 0) { if (literalLen > 0) {
offset = frame->stats.rep[offsetCode]; offset = frame->stats.rep[randomRepIndex];
repIndex = offsetCode; repIndex = randomRepIndex;
} else { } else {
/* special case */ /* special case : literalLen == 0 */
offset = offsetCode == 2 ? frame->stats.rep[0] - 1 offset = randomRepIndex == 2 ? frame->stats.rep[0] - 1
: frame->stats.rep[offsetCode + 1]; : frame->stats.rep[randomRepIndex + 1];
repIndex = MIN(2, offsetCode + 1); repIndex = MIN(2, randomRepIndex + 1);
} }
} }
} while (((!info.useDict) && (offset > (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) || offset == 0); } while (((!info.useDict) && (offset > (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) || offset == 0);
{ { BYTE* const dictEnd = info.dictContent + info.dictContentSize;
size_t j; size_t j;
BYTE* const dictEnd = info.dictContent + info.dictContentSize;
for (j = 0; j < matchLen; j++) { for (j = 0; j < matchLen; j++) {
if ((U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart) < offset) { if ((U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart) < offset) {
/* copy from dictionary instead of literals */ /* copy from dictionary instead of literals */
@@ -739,8 +737,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
*srcPtr = *(srcPtr-offset); *srcPtr = *(srcPtr-offset);
} }
srcPtr++; srcPtr++;
} } }
}
{ int r; { int r;
for (r = repIndex; r > 0; r--) { for (r = repIndex; r > 0; r--) {
@@ -754,7 +751,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
DISPLAYLEVEL(7, " srcPos: %8u seqNb: %3u", DISPLAYLEVEL(7, " srcPos: %8u seqNb: %3u",
(unsigned)((BYTE*)srcPtr - (BYTE*)frame->srcStart), (unsigned)i); (unsigned)((BYTE*)srcPtr - (BYTE*)frame->srcStart), (unsigned)i);
DISPLAYLEVEL(6, "\n"); DISPLAYLEVEL(6, "\n");
if (offsetCode < 3) { if (offsetCode < ZSTD_REP_NUM) { /* expects @offsetCode to use 0-2 to represents repCodes */
DISPLAYLEVEL(7, " repeat offset: %d\n", (int)repIndex); DISPLAYLEVEL(7, " repeat offset: %d\n", (int)repIndex);
} }
/* use libzstd sequence handling */ /* use libzstd sequence handling */