ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'dev' into flexibleLevel

Browse Source
This commit is contained in:
Yann Collet
2018-02-10 11:54:49 -08:00
parent 75689838e4 4e3db17cab
commit 9945e60ac4
6 changed files with 72 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+26 -13
View File
@@ -1247,32 +1247,44 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
#define ZSTD_ROWSIZE 16
/*! ZSTD_reduceTable_internal() :
* reduce table indexes by `reducerValue`
* presume table size is a multiple of ZSTD_ROWSIZE.
* Helps auto-vectorization */
static void ZSTD_reduceTable_internal (U32* const table, int const nbRows, U32 const reducerValue)
/*! ZSTD_reduceTable() :
* reduce table indexes by `reducerValue`, or squash to zero.
* PreserveMark preserves "unsorted mark" for btlazy2 strategy.
* It must be set to a clear 0/1 value, to remove branch during inlining.
* Presume table size is a multiple of ZSTD_ROWSIZE
* to help auto-vectorization */
FORCE_INLINE_TEMPLATE void
ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerValue, int const preserveMark)
{
int const nbRows = (int)size / ZSTD_ROWSIZE;
int cellNb = 0;
int rowNb;
assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */
assert(size < (1U<<31)); /* can be casted to int */
for (rowNb=0 ; rowNb < nbRows ; rowNb++) {
int column;
for (column=0; column<ZSTD_ROWSIZE; column++) {
if (preserveMark) {
U32 const adder = (table[cellNb] == ZSTD_DUBT_UNSORTED_MARK) ? reducerValue : 0;
table[cellNb] += adder;
}
if (table[cellNb] < reducerValue) table[cellNb] = 0;
else table[cellNb] -= reducerValue;
cellNb++;
} }
}
/*! ZSTD_reduceTable() :
* reduce table indexes by `reducerValue` */
static void ZSTD_reduceTable (U32* const table, U32 const size, U32 const reducerValue)
static void ZSTD_reduceTable(U32* const table, U32 const size, U32 const reducerValue)
{
assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */
assert(size < (1U<<31)); /* can be casted to int */
ZSTD_reduceTable_internal(table, size/ZSTD_ROWSIZE, reducerValue);
ZSTD_reduceTable_internal(table, size, reducerValue, 0);
}
static void ZSTD_reduceTable_btlazy2(U32* const table, U32 const size, U32 const reducerValue)
{
ZSTD_reduceTable_internal(table, size, reducerValue, 1);
}
/*! ZSTD_ldm_reduceTable() :
* reduce table indexes by `reducerValue` */
static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size,
@@ -1297,8 +1309,9 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
if (zc->appliedParams.cParams.strategy != ZSTD_fast) {
U32 const chainSize = (U32)1 << zc->appliedParams.cParams.chainLog;
if (zc->appliedParams.cParams.strategy == ZSTD_btlazy2)
ZSTD_preserveUnsortedMark(ms->chainTable, chainSize, reducerValue);
ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue);
ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue);
else
ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue);
}
if (ms->hashLog3) {
lib/compress/zstd_compress_internal.h
+6
View File
@@ -33,6 +33,12 @@ extern "C" {
#define kSearchStrength 8
#define HASH_READ_SIZE 8
#define ZSTD_CLEVEL_CUSTOM 999
#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index 1 now means "unsorted".
It could be confused for a real successor at index "1", if sorted as larger than its predecessor.
It's not a big deal though : candidate will just be sorted again.
Additionnally, candidate position 1 will be lost.
But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss.
The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be misdhandled after table re-use with a different strategy */
/*-*************************************
lib/compress/zstd_lazy.c
-29
View File
@@ -15,35 +15,6 @@
/*-*************************************
* Binary Tree search
***************************************/
#define ZSTD_DUBT_UNSORTED_MARK 1 /* note : index 1 will now be confused with "unsorted" if sorted as larger than its predecessor.
It's not a big deal though : the candidate will just be considered unsorted, and be sorted again.
Additionnally, candidate position 1 will be lost.
But candidate 1 cannot hide a large tree of candidates, so it's a moderate loss.
The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be misdhandled by a table re-use using a different strategy */
/*! ZSTD_preserveUnsortedMark() :
* pre-emptively increase value of ZSTD_DUBT_UNSORTED_MARK before ZSTD_reduceTable()
* so that combined operation preserves its value.
* Without it, ZSTD_DUBT_UNSORTED_MARK==1 would be squashed to 0.
* As a consequence, the list of unsorted elements would stop on the first element,
* removing candidates, resulting in a negligible loss to compression ratio
* (since overflow protection with ZSTD_reduceTable() is relatively rare).
* Another potential risk is that a position will be promoted from *unsorted*
* to *sorted=>smaller:0*, meaning the next candidate will be considered smaller.
* This could be wrong, and result in data corruption.
* On second thought, this corruption might be impossible,
* because unsorted elements are always at the beginning of the list,
* and squashing to zero reduce the list to a single element,
* which needs to be sorted anyway.
* I haven't spent much thoughts into this possible scenario,
* and just felt it was safer to implement ZSTD_preserveUnsortedMark() */
void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue)
{
U32 u;
for (u=0; u<size; u++)
if (table[u] == ZSTD_DUBT_UNSORTED_MARK)
table[u] = ZSTD_DUBT_UNSORTED_MARK + reducerValue;
}
void ZSTD_updateDUBT(
ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams,