feat(compress): port the optimal parser to Rust
Move the binary-tree optimal and ultra block parsers into Rust with a narrow C projection of match-state, sequence-store, and entropy-table fields. Keep the established C entry points as wrappers, and avoid reading uninitialized entropy costs during dictionary tree updates. Test Plan: - cargo test --manifest-path rust/Cargo.toml - cargo clippy --manifest-path rust/Cargo.toml --all-targets -- -D warnings - cargo +nightly fmt --manifest-path rust/Cargo.toml --all -- --check - make -B -C tests test-zstream V=1 - git diff --check Refs: rust/src/zstd_opt.rs, lib/compress/zstd_opt.c
This commit is contained in:
+176
-1425
@@ -5,19 +5,20 @@
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
* You may select, at your option, one or both of these licenses.
|
||||
*/
|
||||
|
||||
/* The optimal parser is implemented in rust/src/zstd_opt.rs. Keep the
|
||||
* private ZSTD_MatchState_t and entropy-table layouts on the C side: this
|
||||
* translation unit only projects the leaves Rust needs and preserves the
|
||||
* established C entry points. */
|
||||
#include "zstd_compress_internal.h"
|
||||
#include "hist.h"
|
||||
#include "zstd_opt.h"
|
||||
|
||||
#if !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \
|
||||
|| !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \
|
||||
|| !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR)
|
||||
|
||||
/* The Rust tree updater receives this leaf view instead of the private,
|
||||
* configuration-dependent ZSTD_MatchState_t layout. */
|
||||
typedef struct {
|
||||
U32* hashTable;
|
||||
U32* chainTable;
|
||||
@@ -32,1461 +33,211 @@ typedef struct {
|
||||
U32 searchLog;
|
||||
U32 windowLog;
|
||||
int useCPredict;
|
||||
} ZSTD_RustOptTreeState;
|
||||
U32* hashTable3;
|
||||
U32 hashLog3;
|
||||
const BYTE* nextSrc;
|
||||
U32 minMatch;
|
||||
U32 targetLength;
|
||||
optState_t* opt;
|
||||
const void* dictMatchState;
|
||||
const RawSeqStore_t* ldmSeqStore;
|
||||
const BYTE** windowBase;
|
||||
U32* windowDictLimit;
|
||||
U32* windowLowLimit;
|
||||
const size_t* hufCTable;
|
||||
int hufRepeatValid;
|
||||
const U32* fseLitLengthCTable;
|
||||
const U32* fseMatchLengthCTable;
|
||||
const U32* fseOffCodeCTable;
|
||||
} ZSTD_RustOptState;
|
||||
|
||||
static ZSTD_RustOptTreeState ZSTD_rustOptTreeState(
|
||||
typedef char ZSTD_rust_opt_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1];
|
||||
typedef char ZSTD_rust_opt_match_layout[(sizeof(ZSTD_match_t) == 8) ? 1 : -1];
|
||||
typedef char ZSTD_rust_opt_rawseq_layout[(sizeof(rawSeq) == 12) ? 1 : -1];
|
||||
typedef char ZSTD_rust_opt_seqstore_layout[
|
||||
(offsetof(SeqStore_t, longLengthPos) == 9 * sizeof(size_t) + 4) ? 1 : -1];
|
||||
typedef char ZSTD_rust_optimal_layout[
|
||||
(sizeof(ZSTD_optimal_t) == 4 * sizeof(U32) + sizeof(U32) * ZSTD_REP_NUM) ? 1 : -1];
|
||||
typedef char ZSTD_rust_opt_rep_count[(ZSTD_REP_NUM == 3) ? 1 : -1];
|
||||
|
||||
static void ZSTD_rustOptState_init(
|
||||
ZSTD_RustOptState* const out,
|
||||
ZSTD_MatchState_t* const ms,
|
||||
const ZSTD_RustOptState* const dms)
|
||||
{
|
||||
out->hashTable = ms->hashTable;
|
||||
out->chainTable = ms->chainTable;
|
||||
out->base = ms->window.base;
|
||||
out->dictBase = ms->window.dictBase;
|
||||
out->dictLimit = ms->window.dictLimit;
|
||||
out->lowLimit = ms->window.lowLimit;
|
||||
out->loadedDictEnd = ms->loadedDictEnd;
|
||||
out->nextToUpdate = &ms->nextToUpdate;
|
||||
out->hashLog = ms->cParams.hashLog;
|
||||
out->chainLog = ms->cParams.chainLog;
|
||||
out->searchLog = ms->cParams.searchLog;
|
||||
#ifdef ZSTD_C_PREDICT
|
||||
out->useCPredict = 1;
|
||||
#else
|
||||
out->useCPredict = 0;
|
||||
#endif
|
||||
out->windowLog = ms->cParams.windowLog;
|
||||
out->hashTable3 = ms->hashTable3;
|
||||
out->hashLog3 = ms->hashLog3;
|
||||
out->nextSrc = ms->window.nextSrc;
|
||||
out->minMatch = ms->cParams.minMatch;
|
||||
out->targetLength = ms->cParams.targetLength;
|
||||
out->opt = &ms->opt;
|
||||
out->dictMatchState = dms;
|
||||
out->ldmSeqStore = ms->ldmSeqStore;
|
||||
out->windowBase = &ms->window.base;
|
||||
out->windowDictLimit = &ms->window.dictLimit;
|
||||
out->windowLowLimit = &ms->window.lowLimit;
|
||||
out->hufCTable = NULL;
|
||||
out->hufRepeatValid = 0;
|
||||
out->fseLitLengthCTable = NULL;
|
||||
out->fseMatchLengthCTable = NULL;
|
||||
out->fseOffCodeCTable = NULL;
|
||||
}
|
||||
|
||||
static void ZSTD_rustOptState_projectCosts(
|
||||
ZSTD_RustOptState* const out,
|
||||
ZSTD_MatchState_t* const ms)
|
||||
{
|
||||
ZSTD_RustOptTreeState state;
|
||||
state.hashTable = ms->hashTable;
|
||||
state.chainTable = ms->chainTable;
|
||||
state.base = ms->window.base;
|
||||
state.dictBase = ms->window.dictBase;
|
||||
state.dictLimit = ms->window.dictLimit;
|
||||
state.lowLimit = ms->window.lowLimit;
|
||||
state.loadedDictEnd = ms->loadedDictEnd;
|
||||
state.nextToUpdate = &ms->nextToUpdate;
|
||||
state.hashLog = ms->cParams.hashLog;
|
||||
state.chainLog = ms->cParams.chainLog;
|
||||
state.searchLog = ms->cParams.searchLog;
|
||||
state.windowLog = ms->cParams.windowLog;
|
||||
#ifdef ZSTD_C_PREDICT
|
||||
state.useCPredict = 1;
|
||||
#else
|
||||
state.useCPredict = 0;
|
||||
#endif
|
||||
return state;
|
||||
const ZSTD_entropyCTables_t* const costs = ms->opt.symbolCosts;
|
||||
out->hufCTable = costs ? costs->huf.CTable : NULL;
|
||||
out->hufRepeatValid = costs && costs->huf.repeatMode == HUF_repeat_valid;
|
||||
out->fseLitLengthCTable = costs ? costs->fse.litlengthCTable : NULL;
|
||||
out->fseMatchLengthCTable = costs ? costs->fse.matchlengthCTable : NULL;
|
||||
out->fseOffCodeCTable = costs ? costs->fse.offcodeCTable : NULL;
|
||||
}
|
||||
|
||||
void ZSTD_rust_opt_updateTreeInternal(
|
||||
ZSTD_RustOptTreeState* state,
|
||||
static void ZSTD_rustOptState_init_with_dict(
|
||||
ZSTD_RustOptState* const out,
|
||||
ZSTD_MatchState_t* const ms)
|
||||
{
|
||||
ZSTD_rustOptState_init(out, ms, NULL);
|
||||
}
|
||||
|
||||
static void ZSTD_rustOptState_init_with_costs(
|
||||
ZSTD_RustOptState* const out,
|
||||
ZSTD_MatchState_t* const ms,
|
||||
const ZSTD_RustOptState* const dms)
|
||||
{
|
||||
ZSTD_rustOptState_init(out, ms, dms);
|
||||
ZSTD_rustOptState_projectCosts(out, ms);
|
||||
}
|
||||
|
||||
void ZSTD_rust_opt_updateTree(
|
||||
ZSTD_RustOptState* state,
|
||||
const void* ip, const void* iend,
|
||||
U32 mls, int extDict);
|
||||
U32 mls, int dictMode);
|
||||
|
||||
#define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */
|
||||
#define ZSTD_MAX_PRICE (1<<30)
|
||||
size_t ZSTD_rust_compressBlock_opt(
|
||||
ZSTD_RustOptState* state, void* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize, int optLevel, int dictMode);
|
||||
|
||||
#define ZSTD_PREDEF_THRESHOLD 8 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */
|
||||
void ZSTD_rust_initStats_ultra(
|
||||
ZSTD_RustOptState* state, void* seqStore, const U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
size_t ZSTD_rust_compressBlock_btultra2(
|
||||
ZSTD_RustOptState* state, void* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize);
|
||||
|
||||
/*-*************************************
|
||||
* Price functions for optimal parser
|
||||
***************************************/
|
||||
enum {
|
||||
ZSTD_rust_dict_noDict = 0,
|
||||
ZSTD_rust_dict_extDict = 1,
|
||||
ZSTD_rust_dict_dictMatchState = 2,
|
||||
};
|
||||
|
||||
#if 0 /* approximation at bit level (for tests) */
|
||||
# define BITCOST_ACCURACY 0
|
||||
# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
|
||||
# define WEIGHT(stat, opt) ((void)(opt), ZSTD_bitWeight(stat))
|
||||
#elif 0 /* fractional bit accuracy (for tests) */
|
||||
# define BITCOST_ACCURACY 8
|
||||
# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
|
||||
# define WEIGHT(stat,opt) ((void)(opt), ZSTD_fracWeight(stat))
|
||||
#else /* opt==approx, ultra==accurate */
|
||||
# define BITCOST_ACCURACY 8
|
||||
# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
|
||||
# define WEIGHT(stat,opt) ((opt) ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat))
|
||||
#endif
|
||||
|
||||
/* ZSTD_bitWeight() :
|
||||
* provide estimated "cost" of a stat in full bits only */
|
||||
MEM_STATIC U32 ZSTD_bitWeight(U32 stat)
|
||||
void ZSTD_updateTree(ZSTD_MatchState_t* const ms, const BYTE* const ip,
|
||||
const BYTE* const iend)
|
||||
{
|
||||
return (ZSTD_highbit32(stat+1) * BITCOST_MULTIPLIER);
|
||||
ZSTD_RustOptState state;
|
||||
ZSTD_rustOptState_init_with_dict(&state, ms);
|
||||
ZSTD_rust_opt_updateTree(&state, ip, iend, ms->cParams.minMatch,
|
||||
ZSTD_rust_dict_noDict);
|
||||
}
|
||||
|
||||
/* ZSTD_fracWeight() :
|
||||
* provide fractional-bit "cost" of a stat,
|
||||
* using linear interpolation approximation */
|
||||
MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat)
|
||||
{
|
||||
U32 const stat = rawStat + 1;
|
||||
U32 const hb = ZSTD_highbit32(stat);
|
||||
U32 const BWeight = hb * BITCOST_MULTIPLIER;
|
||||
/* Fweight was meant for "Fractional weight"
|
||||
* but it's effectively a value between 1 and 2
|
||||
* using fixed point arithmetic */
|
||||
U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb;
|
||||
U32 const weight = BWeight + FWeight;
|
||||
assert(hb + BITCOST_ACCURACY < 31);
|
||||
return weight;
|
||||
}
|
||||
|
||||
#if (DEBUGLEVEL>=2)
|
||||
/* debugging function,
|
||||
* @return price in bytes as fractional value
|
||||
* for debug messages only */
|
||||
MEM_STATIC double ZSTD_fCost(int price)
|
||||
{
|
||||
return (double)price / (BITCOST_MULTIPLIER*8);
|
||||
}
|
||||
#endif
|
||||
|
||||
static int ZSTD_compressedLiterals(optState_t const* const optPtr)
|
||||
{
|
||||
return optPtr->literalCompressionMode != ZSTD_ps_disable;
|
||||
}
|
||||
|
||||
static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel)
|
||||
{
|
||||
if (ZSTD_compressedLiterals(optPtr))
|
||||
optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel);
|
||||
optPtr->litLengthSumBasePrice = WEIGHT(optPtr->litLengthSum, optLevel);
|
||||
optPtr->matchLengthSumBasePrice = WEIGHT(optPtr->matchLengthSum, optLevel);
|
||||
optPtr->offCodeSumBasePrice = WEIGHT(optPtr->offCodeSum, optLevel);
|
||||
}
|
||||
|
||||
|
||||
static U32 sum_u32(const unsigned table[], size_t nbElts)
|
||||
{
|
||||
size_t n;
|
||||
U32 total = 0;
|
||||
for (n=0; n<nbElts; n++) {
|
||||
total += table[n];
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
typedef enum { base_0possible=0, base_1guaranteed=1 } base_directive_e;
|
||||
|
||||
static U32
|
||||
ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift, base_directive_e base1)
|
||||
{
|
||||
U32 s, sum=0;
|
||||
DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)",
|
||||
(unsigned)lastEltIndex+1, (unsigned)shift );
|
||||
assert(shift < 30);
|
||||
for (s=0; s<lastEltIndex+1; s++) {
|
||||
unsigned const base = base1 ? 1 : (table[s]>0);
|
||||
unsigned const newStat = base + (table[s] >> shift);
|
||||
sum += newStat;
|
||||
table[s] = newStat;
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
|
||||
/* ZSTD_scaleStats() :
|
||||
* reduce all elt frequencies in table if sum too large
|
||||
* return the resulting sum of elements */
|
||||
static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget)
|
||||
{
|
||||
U32 const prevsum = sum_u32(table, lastEltIndex+1);
|
||||
U32 const factor = prevsum >> logTarget;
|
||||
DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget);
|
||||
assert(logTarget < 30);
|
||||
if (factor <= 1) return prevsum;
|
||||
return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor), base_1guaranteed);
|
||||
}
|
||||
|
||||
/* ZSTD_rescaleFreqs() :
|
||||
* if first block (detected by optPtr->litLengthSum == 0) : init statistics
|
||||
* take hints from dictionary if there is one
|
||||
* and init from zero if there is none,
|
||||
* using src for literals stats, and baseline stats for sequence symbols
|
||||
* otherwise downscale existing stats, to be used as seed for next block.
|
||||
*/
|
||||
static void
|
||||
ZSTD_rescaleFreqs(optState_t* const optPtr,
|
||||
const BYTE* const src, size_t const srcSize,
|
||||
int const optLevel)
|
||||
{
|
||||
int const compressedLiterals = ZSTD_compressedLiterals(optPtr);
|
||||
DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize);
|
||||
optPtr->priceType = zop_dynamic;
|
||||
|
||||
if (optPtr->litLengthSum == 0) { /* no literals stats collected -> first block assumed -> init */
|
||||
|
||||
/* heuristic: use pre-defined stats for too small inputs */
|
||||
if (srcSize <= ZSTD_PREDEF_THRESHOLD) {
|
||||
DEBUGLOG(5, "srcSize <= %i : use predefined stats", ZSTD_PREDEF_THRESHOLD);
|
||||
optPtr->priceType = zop_predef;
|
||||
}
|
||||
|
||||
assert(optPtr->symbolCosts != NULL);
|
||||
if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) {
|
||||
|
||||
/* huffman stats covering the full value set : table presumed generated by dictionary */
|
||||
optPtr->priceType = zop_dynamic;
|
||||
|
||||
if (compressedLiterals) {
|
||||
/* generate literals statistics from huffman table */
|
||||
unsigned lit;
|
||||
assert(optPtr->litFreq != NULL);
|
||||
optPtr->litSum = 0;
|
||||
for (lit=0; lit<=MaxLit; lit++) {
|
||||
U32 const scaleLog = 11; /* scale to 2K */
|
||||
U32 const bitCost = HUF_getNbBitsFromCTable(optPtr->symbolCosts->huf.CTable, lit);
|
||||
assert(bitCost <= scaleLog);
|
||||
optPtr->litFreq[lit] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/;
|
||||
optPtr->litSum += optPtr->litFreq[lit];
|
||||
} }
|
||||
|
||||
{ unsigned ll;
|
||||
FSE_CState_t llstate;
|
||||
FSE_initCState(&llstate, optPtr->symbolCosts->fse.litlengthCTable);
|
||||
optPtr->litLengthSum = 0;
|
||||
for (ll=0; ll<=MaxLL; ll++) {
|
||||
U32 const scaleLog = 10; /* scale to 1K */
|
||||
U32 const bitCost = FSE_getMaxNbBits(llstate.symbolTT, ll);
|
||||
assert(bitCost < scaleLog);
|
||||
optPtr->litLengthFreq[ll] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/;
|
||||
optPtr->litLengthSum += optPtr->litLengthFreq[ll];
|
||||
} }
|
||||
|
||||
{ unsigned ml;
|
||||
FSE_CState_t mlstate;
|
||||
FSE_initCState(&mlstate, optPtr->symbolCosts->fse.matchlengthCTable);
|
||||
optPtr->matchLengthSum = 0;
|
||||
for (ml=0; ml<=MaxML; ml++) {
|
||||
U32 const scaleLog = 10;
|
||||
U32 const bitCost = FSE_getMaxNbBits(mlstate.symbolTT, ml);
|
||||
assert(bitCost < scaleLog);
|
||||
optPtr->matchLengthFreq[ml] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/;
|
||||
optPtr->matchLengthSum += optPtr->matchLengthFreq[ml];
|
||||
} }
|
||||
|
||||
{ unsigned of;
|
||||
FSE_CState_t ofstate;
|
||||
FSE_initCState(&ofstate, optPtr->symbolCosts->fse.offcodeCTable);
|
||||
optPtr->offCodeSum = 0;
|
||||
for (of=0; of<=MaxOff; of++) {
|
||||
U32 const scaleLog = 10;
|
||||
U32 const bitCost = FSE_getMaxNbBits(ofstate.symbolTT, of);
|
||||
assert(bitCost < scaleLog);
|
||||
optPtr->offCodeFreq[of] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/;
|
||||
optPtr->offCodeSum += optPtr->offCodeFreq[of];
|
||||
} }
|
||||
|
||||
} else { /* first block, no dictionary */
|
||||
|
||||
assert(optPtr->litFreq != NULL);
|
||||
if (compressedLiterals) {
|
||||
/* base initial cost of literals on direct frequency within src */
|
||||
unsigned lit = MaxLit;
|
||||
HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */
|
||||
optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8, base_0possible);
|
||||
}
|
||||
|
||||
{ unsigned const baseLLfreqs[MaxLL+1] = {
|
||||
4, 2, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1
|
||||
};
|
||||
ZSTD_memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs));
|
||||
optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1);
|
||||
}
|
||||
|
||||
{ unsigned ml;
|
||||
for (ml=0; ml<=MaxML; ml++)
|
||||
optPtr->matchLengthFreq[ml] = 1;
|
||||
}
|
||||
optPtr->matchLengthSum = MaxML+1;
|
||||
|
||||
{ unsigned const baseOFCfreqs[MaxOff+1] = {
|
||||
6, 2, 1, 1, 2, 3, 4, 4,
|
||||
4, 3, 2, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1
|
||||
};
|
||||
ZSTD_memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs));
|
||||
optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
} else { /* new block : scale down accumulated statistics */
|
||||
|
||||
if (compressedLiterals)
|
||||
optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12);
|
||||
optPtr->litLengthSum = ZSTD_scaleStats(optPtr->litLengthFreq, MaxLL, 11);
|
||||
optPtr->matchLengthSum = ZSTD_scaleStats(optPtr->matchLengthFreq, MaxML, 11);
|
||||
optPtr->offCodeSum = ZSTD_scaleStats(optPtr->offCodeFreq, MaxOff, 11);
|
||||
}
|
||||
|
||||
ZSTD_setBasePrices(optPtr, optLevel);
|
||||
}
|
||||
|
||||
/* ZSTD_rawLiteralsCost() :
|
||||
* price of literals (only) in specified segment (which length can be 0).
|
||||
* does not include price of literalLength symbol */
|
||||
static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength,
|
||||
const optState_t* const optPtr,
|
||||
int optLevel)
|
||||
{
|
||||
DEBUGLOG(8, "ZSTD_rawLiteralsCost (%u literals)", litLength);
|
||||
if (litLength == 0) return 0;
|
||||
|
||||
if (!ZSTD_compressedLiterals(optPtr))
|
||||
return (litLength << 3) * BITCOST_MULTIPLIER; /* Uncompressed - 8 bytes per literal. */
|
||||
|
||||
if (optPtr->priceType == zop_predef)
|
||||
return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */
|
||||
|
||||
/* dynamic statistics */
|
||||
{ U32 price = optPtr->litSumBasePrice * litLength;
|
||||
U32 const litPriceMax = optPtr->litSumBasePrice - BITCOST_MULTIPLIER;
|
||||
U32 u;
|
||||
assert(optPtr->litSumBasePrice >= BITCOST_MULTIPLIER);
|
||||
for (u=0; u < litLength; u++) {
|
||||
U32 litPrice = WEIGHT(optPtr->litFreq[literals[u]], optLevel);
|
||||
if (UNLIKELY(litPrice > litPriceMax)) litPrice = litPriceMax;
|
||||
price -= litPrice;
|
||||
}
|
||||
return price;
|
||||
}
|
||||
}
|
||||
|
||||
/* ZSTD_litLengthPrice() :
|
||||
* cost of literalLength symbol */
|
||||
static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr, int optLevel)
|
||||
{
|
||||
assert(litLength <= ZSTD_BLOCKSIZE_MAX);
|
||||
if (optPtr->priceType == zop_predef)
|
||||
return WEIGHT(litLength, optLevel);
|
||||
|
||||
/* ZSTD_LLcode() can't compute litLength price for sizes >= ZSTD_BLOCKSIZE_MAX
|
||||
* because it isn't representable in the zstd format.
|
||||
* So instead just pretend it would cost 1 bit more than ZSTD_BLOCKSIZE_MAX - 1.
|
||||
* In such a case, the block would be all literals.
|
||||
*/
|
||||
if (litLength == ZSTD_BLOCKSIZE_MAX)
|
||||
return BITCOST_MULTIPLIER + ZSTD_litLengthPrice(ZSTD_BLOCKSIZE_MAX - 1, optPtr, optLevel);
|
||||
|
||||
/* dynamic statistics */
|
||||
{ U32 const llCode = ZSTD_LLcode(litLength);
|
||||
return (LL_bits[llCode] * BITCOST_MULTIPLIER)
|
||||
+ optPtr->litLengthSumBasePrice
|
||||
- WEIGHT(optPtr->litLengthFreq[llCode], optLevel);
|
||||
}
|
||||
}
|
||||
|
||||
/* ZSTD_getMatchPrice() :
|
||||
* Provides the cost of the match part (offset + matchLength) of a sequence.
|
||||
* Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence.
|
||||
* @offBase : sumtype, representing an offset or a repcode, and using numeric representation of ZSTD_storeSeq()
|
||||
* @optLevel: when <2, favors small offset for decompression speed (improved cache efficiency)
|
||||
*/
|
||||
FORCE_INLINE_TEMPLATE U32
|
||||
ZSTD_getMatchPrice(U32 const offBase,
|
||||
U32 const matchLength,
|
||||
const optState_t* const optPtr,
|
||||
int const optLevel)
|
||||
{
|
||||
U32 price;
|
||||
U32 const offCode = ZSTD_highbit32(offBase);
|
||||
U32 const mlBase = matchLength - MINMATCH;
|
||||
assert(matchLength >= MINMATCH);
|
||||
|
||||
if (optPtr->priceType == zop_predef) /* fixed scheme, does not use statistics */
|
||||
return WEIGHT(mlBase, optLevel)
|
||||
+ ((16 + offCode) * BITCOST_MULTIPLIER); /* emulated offset cost */
|
||||
|
||||
/* dynamic statistics */
|
||||
price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel));
|
||||
if ((optLevel<2) /*static*/ && offCode >= 20)
|
||||
price += (offCode-19)*2 * BITCOST_MULTIPLIER; /* handicap for long distance offsets, favor decompression speed */
|
||||
|
||||
/* match Length */
|
||||
{ U32 const mlCode = ZSTD_MLcode(mlBase);
|
||||
price += (ML_bits[mlCode] * BITCOST_MULTIPLIER) + (optPtr->matchLengthSumBasePrice - WEIGHT(optPtr->matchLengthFreq[mlCode], optLevel));
|
||||
}
|
||||
|
||||
price += BITCOST_MULTIPLIER / 5; /* heuristic : make matches a bit more costly to favor less sequences -> faster decompression speed */
|
||||
|
||||
DEBUGLOG(8, "ZSTD_getMatchPrice(ml:%u) = %u", matchLength, price);
|
||||
return price;
|
||||
}
|
||||
|
||||
/* ZSTD_updateStats() :
|
||||
* assumption : literals + litLength <= iend */
|
||||
static void ZSTD_updateStats(optState_t* const optPtr,
|
||||
U32 litLength, const BYTE* literals,
|
||||
U32 offBase, U32 matchLength)
|
||||
{
|
||||
/* literals */
|
||||
if (ZSTD_compressedLiterals(optPtr)) {
|
||||
U32 u;
|
||||
for (u=0; u < litLength; u++)
|
||||
optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD;
|
||||
optPtr->litSum += litLength*ZSTD_LITFREQ_ADD;
|
||||
}
|
||||
|
||||
/* literal Length */
|
||||
{ U32 const llCode = ZSTD_LLcode(litLength);
|
||||
optPtr->litLengthFreq[llCode]++;
|
||||
optPtr->litLengthSum++;
|
||||
}
|
||||
|
||||
/* offset code : follows storeSeq() numeric representation */
|
||||
{ U32 const offCode = ZSTD_highbit32(offBase);
|
||||
assert(offCode <= MaxOff);
|
||||
optPtr->offCodeFreq[offCode]++;
|
||||
optPtr->offCodeSum++;
|
||||
}
|
||||
|
||||
/* match Length */
|
||||
{ U32 const mlBase = matchLength - MINMATCH;
|
||||
U32 const mlCode = ZSTD_MLcode(mlBase);
|
||||
optPtr->matchLengthFreq[mlCode]++;
|
||||
optPtr->matchLengthSum++;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* ZSTD_readMINMATCH() :
|
||||
* function safe only for comparisons
|
||||
* assumption : memPtr must be at least 4 bytes before end of buffer */
|
||||
MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length)
|
||||
{
|
||||
switch (length)
|
||||
{
|
||||
default :
|
||||
case 4 : return MEM_read32(memPtr);
|
||||
case 3 : if (MEM_isLittleEndian())
|
||||
return MEM_read32(memPtr)<<8;
|
||||
else
|
||||
return MEM_read32(memPtr)>>8;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* Update hashTable3 up to ip (excluded)
|
||||
Assumption : always within prefix (i.e. not within extDict) */
|
||||
static
|
||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
U32 ZSTD_insertAndFindFirstIndexHash3 (const ZSTD_MatchState_t* ms,
|
||||
U32* nextToUpdate3,
|
||||
const BYTE* const ip)
|
||||
{
|
||||
U32* const hashTable3 = ms->hashTable3;
|
||||
U32 const hashLog3 = ms->hashLog3;
|
||||
const BYTE* const base = ms->window.base;
|
||||
U32 idx = *nextToUpdate3;
|
||||
U32 const target = (U32)(ip - base);
|
||||
size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3);
|
||||
assert(hashLog3 > 0);
|
||||
|
||||
while(idx < target) {
|
||||
hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx;
|
||||
idx++;
|
||||
}
|
||||
|
||||
*nextToUpdate3 = target;
|
||||
return hashTable3[hash3];
|
||||
}
|
||||
|
||||
|
||||
/*-*************************************
|
||||
* Binary Tree search
|
||||
***************************************/
|
||||
FORCE_INLINE_TEMPLATE
|
||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
void ZSTD_updateTree_internal(
|
||||
ZSTD_MatchState_t* ms,
|
||||
const BYTE* const ip, const BYTE* const iend,
|
||||
const U32 mls, const ZSTD_dictMode_e dictMode)
|
||||
{
|
||||
ZSTD_RustOptTreeState state = ZSTD_rustOptTreeState(ms);
|
||||
ZSTD_rust_opt_updateTreeInternal(&state, ip, iend, mls,
|
||||
dictMode == ZSTD_extDict);
|
||||
}
|
||||
|
||||
void ZSTD_updateTree(ZSTD_MatchState_t* ms, const BYTE* ip, const BYTE* iend) {
|
||||
ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict);
|
||||
}
|
||||
|
||||
FORCE_INLINE_TEMPLATE
|
||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
U32
|
||||
ZSTD_insertBtAndGetAllMatches (
|
||||
ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */
|
||||
ZSTD_MatchState_t* ms,
|
||||
U32* nextToUpdate3,
|
||||
const BYTE* const ip, const BYTE* const iLimit,
|
||||
const ZSTD_dictMode_e dictMode,
|
||||
const U32 rep[ZSTD_REP_NUM],
|
||||
const U32 ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */
|
||||
const U32 lengthToBeat,
|
||||
const U32 mls /* template */)
|
||||
{
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
|
||||
const BYTE* const base = ms->window.base;
|
||||
U32 const curr = (U32)(ip-base);
|
||||
U32 const hashLog = cParams->hashLog;
|
||||
U32 const minMatch = (mls==3) ? 3 : 4;
|
||||
U32* const hashTable = ms->hashTable;
|
||||
size_t const h = ZSTD_hashPtr(ip, hashLog, mls);
|
||||
U32 matchIndex = hashTable[h];
|
||||
U32* const bt = ms->chainTable;
|
||||
U32 const btLog = cParams->chainLog - 1;
|
||||
U32 const btMask= (1U << btLog) - 1;
|
||||
size_t commonLengthSmaller=0, commonLengthLarger=0;
|
||||
const BYTE* const dictBase = ms->window.dictBase;
|
||||
U32 const dictLimit = ms->window.dictLimit;
|
||||
const BYTE* const dictEnd = dictBase + dictLimit;
|
||||
const BYTE* const prefixStart = base + dictLimit;
|
||||
U32 const btLow = (btMask >= curr) ? 0 : curr - btMask;
|
||||
U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog);
|
||||
U32 const matchLow = windowLow ? windowLow : 1;
|
||||
U32* smallerPtr = bt + 2*(curr&btMask);
|
||||
U32* largerPtr = bt + 2*(curr&btMask) + 1;
|
||||
U32 matchEndIdx = curr+8+1; /* farthest referenced position of any match => detects repetitive patterns */
|
||||
U32 dummy32; /* to be nullified at the end */
|
||||
U32 mnum = 0;
|
||||
U32 nbCompares = 1U << cParams->searchLog;
|
||||
|
||||
const ZSTD_MatchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL;
|
||||
const ZSTD_compressionParameters* const dmsCParams =
|
||||
dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL;
|
||||
const BYTE* const dmsBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL;
|
||||
const BYTE* const dmsEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL;
|
||||
U32 const dmsHighLimit = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0;
|
||||
U32 const dmsLowLimit = dictMode == ZSTD_dictMatchState ? dms->window.lowLimit : 0;
|
||||
U32 const dmsIndexDelta = dictMode == ZSTD_dictMatchState ? windowLow - dmsHighLimit : 0;
|
||||
U32 const dmsHashLog = dictMode == ZSTD_dictMatchState ? dmsCParams->hashLog : hashLog;
|
||||
U32 const dmsBtLog = dictMode == ZSTD_dictMatchState ? dmsCParams->chainLog - 1 : btLog;
|
||||
U32 const dmsBtMask = dictMode == ZSTD_dictMatchState ? (1U << dmsBtLog) - 1 : 0;
|
||||
U32 const dmsBtLow = dictMode == ZSTD_dictMatchState && dmsBtMask < dmsHighLimit - dmsLowLimit ? dmsHighLimit - dmsBtMask : dmsLowLimit;
|
||||
|
||||
size_t bestLength = lengthToBeat-1;
|
||||
DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", curr);
|
||||
|
||||
/* check repCode */
|
||||
assert(ll0 <= 1); /* necessarily 1 or 0 */
|
||||
{ U32 const lastR = ZSTD_REP_NUM + ll0;
|
||||
U32 repCode;
|
||||
for (repCode = ll0; repCode < lastR; repCode++) {
|
||||
U32 const repOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
|
||||
U32 const repIndex = curr - repOffset;
|
||||
U32 repLen = 0;
|
||||
assert(curr >= dictLimit);
|
||||
if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < curr-dictLimit) { /* equivalent to `curr > repIndex >= dictLimit` */
|
||||
/* We must validate the repcode offset because when we're using a dictionary the
|
||||
* valid offset range shrinks when the dictionary goes out of bounds.
|
||||
*/
|
||||
if ((repIndex >= windowLow) & (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch))) {
|
||||
repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch;
|
||||
}
|
||||
} else { /* repIndex < dictLimit || repIndex >= curr */
|
||||
const BYTE* const repMatch = dictMode == ZSTD_dictMatchState ?
|
||||
dmsBase + repIndex - dmsIndexDelta :
|
||||
dictBase + repIndex;
|
||||
assert(curr >= windowLow);
|
||||
if ( dictMode == ZSTD_extDict
|
||||
&& ( ((repOffset-1) /*intentional overflow*/ < curr - windowLow) /* equivalent to `curr > repIndex >= windowLow` */
|
||||
& (ZSTD_index_overlap_check(dictLimit, repIndex)) )
|
||||
&& (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
|
||||
repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch;
|
||||
}
|
||||
if (dictMode == ZSTD_dictMatchState
|
||||
&& ( ((repOffset-1) /*intentional overflow*/ < curr - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `curr > repIndex >= dmsLowLimit` */
|
||||
& (ZSTD_index_overlap_check(dictLimit, repIndex)) )
|
||||
&& (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
|
||||
repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch;
|
||||
} }
|
||||
/* save longer solution */
|
||||
if (repLen > bestLength) {
|
||||
DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u",
|
||||
repCode, ll0, repOffset, repLen);
|
||||
bestLength = repLen;
|
||||
matches[mnum].off = REPCODE_TO_OFFBASE(repCode - ll0 + 1); /* expect value between 1 and 3 */
|
||||
matches[mnum].len = (U32)repLen;
|
||||
mnum++;
|
||||
if ( (repLen > sufficient_len)
|
||||
| (ip+repLen == iLimit) ) { /* best possible */
|
||||
return mnum;
|
||||
} } } }
|
||||
|
||||
/* HC3 match finder */
|
||||
if ((mls == 3) /*static*/ && (bestLength < mls)) {
|
||||
U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, nextToUpdate3, ip);
|
||||
if ((matchIndex3 >= matchLow)
|
||||
& (curr - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) {
|
||||
size_t mlen;
|
||||
if ((dictMode == ZSTD_noDict) /*static*/ || (dictMode == ZSTD_dictMatchState) /*static*/ || (matchIndex3 >= dictLimit)) {
|
||||
const BYTE* const match = base + matchIndex3;
|
||||
mlen = ZSTD_count(ip, match, iLimit);
|
||||
} else {
|
||||
const BYTE* const match = dictBase + matchIndex3;
|
||||
mlen = ZSTD_count_2segments(ip, match, iLimit, dictEnd, prefixStart);
|
||||
}
|
||||
|
||||
/* save best solution */
|
||||
if (mlen >= mls /* == 3 > bestLength */) {
|
||||
DEBUGLOG(8, "found small match with hlog3, of length %u",
|
||||
(U32)mlen);
|
||||
bestLength = mlen;
|
||||
assert(curr > matchIndex3);
|
||||
assert(mnum==0); /* no prior solution */
|
||||
matches[0].off = OFFSET_TO_OFFBASE(curr - matchIndex3);
|
||||
matches[0].len = (U32)mlen;
|
||||
mnum = 1;
|
||||
if ( (mlen > sufficient_len) |
|
||||
(ip+mlen == iLimit) ) { /* best possible length */
|
||||
ms->nextToUpdate = curr+1; /* skip insertion */
|
||||
return 1;
|
||||
} } }
|
||||
/* no dictMatchState lookup: dicts don't have a populated HC3 table */
|
||||
} /* if (mls == 3) */
|
||||
|
||||
hashTable[h] = curr; /* Update Hash Table */
|
||||
|
||||
for (; nbCompares && (matchIndex >= matchLow); --nbCompares) {
|
||||
U32* const nextPtr = bt + 2*(matchIndex & btMask);
|
||||
const BYTE* match;
|
||||
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
|
||||
assert(curr > matchIndex);
|
||||
|
||||
if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) {
|
||||
assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */
|
||||
match = base + matchIndex;
|
||||
if (matchIndex >= dictLimit) assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */
|
||||
matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit);
|
||||
} else {
|
||||
match = dictBase + matchIndex;
|
||||
assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */
|
||||
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart);
|
||||
if (matchIndex+matchLength >= dictLimit)
|
||||
match = base + matchIndex; /* prepare for match[matchLength] read */
|
||||
}
|
||||
|
||||
if (matchLength > bestLength) {
|
||||
DEBUGLOG(8, "found match of length %u at distance %u (offBase=%u)",
|
||||
(U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex));
|
||||
assert(matchEndIdx > matchIndex);
|
||||
if (matchLength > matchEndIdx - matchIndex)
|
||||
matchEndIdx = matchIndex + (U32)matchLength;
|
||||
bestLength = matchLength;
|
||||
matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex);
|
||||
matches[mnum].len = (U32)matchLength;
|
||||
mnum++;
|
||||
if ( (matchLength > ZSTD_OPT_NUM)
|
||||
| (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) {
|
||||
if (dictMode == ZSTD_dictMatchState) nbCompares = 0; /* break should also skip searching dms */
|
||||
break; /* drop, to preserve bt consistency (miss a little bit of compression) */
|
||||
} }
|
||||
|
||||
if (match[matchLength] < ip[matchLength]) {
|
||||
/* match smaller than current */
|
||||
*smallerPtr = matchIndex; /* update smaller idx */
|
||||
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
|
||||
if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */
|
||||
smallerPtr = nextPtr+1; /* new candidate => larger than match, which was smaller than current */
|
||||
matchIndex = nextPtr[1]; /* new matchIndex, larger than previous, closer to current */
|
||||
} else {
|
||||
*largerPtr = matchIndex;
|
||||
commonLengthLarger = matchLength;
|
||||
if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */
|
||||
largerPtr = nextPtr;
|
||||
matchIndex = nextPtr[0];
|
||||
} }
|
||||
|
||||
*smallerPtr = *largerPtr = 0;
|
||||
|
||||
assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */
|
||||
if (dictMode == ZSTD_dictMatchState && nbCompares) {
|
||||
size_t const dmsH = ZSTD_hashPtr(ip, dmsHashLog, mls);
|
||||
U32 dictMatchIndex = dms->hashTable[dmsH];
|
||||
const U32* const dmsBt = dms->chainTable;
|
||||
commonLengthSmaller = commonLengthLarger = 0;
|
||||
for (; nbCompares && (dictMatchIndex > dmsLowLimit); --nbCompares) {
|
||||
const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & dmsBtMask);
|
||||
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
|
||||
const BYTE* match = dmsBase + dictMatchIndex;
|
||||
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dmsEnd, prefixStart);
|
||||
if (dictMatchIndex+matchLength >= dmsHighLimit)
|
||||
match = base + dictMatchIndex + dmsIndexDelta; /* to prepare for next usage of match[matchLength] */
|
||||
|
||||
if (matchLength > bestLength) {
|
||||
matchIndex = dictMatchIndex + dmsIndexDelta;
|
||||
DEBUGLOG(8, "found dms match of length %u at distance %u (offBase=%u)",
|
||||
(U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex));
|
||||
if (matchLength > matchEndIdx - matchIndex)
|
||||
matchEndIdx = matchIndex + (U32)matchLength;
|
||||
bestLength = matchLength;
|
||||
matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex);
|
||||
matches[mnum].len = (U32)matchLength;
|
||||
mnum++;
|
||||
if ( (matchLength > ZSTD_OPT_NUM)
|
||||
| (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) {
|
||||
break; /* drop, to guarantee consistency (miss a little bit of compression) */
|
||||
} }
|
||||
|
||||
if (dictMatchIndex <= dmsBtLow) { break; } /* beyond tree size, stop the search */
|
||||
if (match[matchLength] < ip[matchLength]) {
|
||||
commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */
|
||||
dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */
|
||||
} else {
|
||||
/* match is larger than current */
|
||||
commonLengthLarger = matchLength;
|
||||
dictMatchIndex = nextPtr[0];
|
||||
} } } /* if (dictMode == ZSTD_dictMatchState) */
|
||||
|
||||
assert(matchEndIdx > curr+8);
|
||||
ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */
|
||||
return mnum;
|
||||
}
|
||||
|
||||
typedef U32 (*ZSTD_getAllMatchesFn)(
|
||||
ZSTD_match_t*,
|
||||
ZSTD_MatchState_t*,
|
||||
U32*,
|
||||
const BYTE*,
|
||||
const BYTE*,
|
||||
const U32 rep[ZSTD_REP_NUM],
|
||||
U32 const ll0,
|
||||
U32 const lengthToBeat);
|
||||
|
||||
FORCE_INLINE_TEMPLATE
|
||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
U32 ZSTD_btGetAllMatches_internal(
|
||||
ZSTD_match_t* matches,
|
||||
ZSTD_MatchState_t* ms,
|
||||
U32* nextToUpdate3,
|
||||
const BYTE* ip,
|
||||
const BYTE* const iHighLimit,
|
||||
const U32 rep[ZSTD_REP_NUM],
|
||||
U32 const ll0,
|
||||
U32 const lengthToBeat,
|
||||
const ZSTD_dictMode_e dictMode,
|
||||
const U32 mls)
|
||||
{
|
||||
assert(BOUNDED(3, ms->cParams.minMatch, 6) == mls);
|
||||
DEBUGLOG(8, "ZSTD_BtGetAllMatches(dictMode=%d, mls=%u)", (int)dictMode, mls);
|
||||
if (ip < ms->window.base + ms->nextToUpdate)
|
||||
return 0; /* skipped area */
|
||||
ZSTD_updateTree_internal(ms, ip, iHighLimit, mls, dictMode);
|
||||
return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, mls);
|
||||
}
|
||||
|
||||
#define ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls) ZSTD_btGetAllMatches_##dictMode##_##mls
|
||||
|
||||
#define GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, mls) \
|
||||
static U32 ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls)( \
|
||||
ZSTD_match_t* matches, \
|
||||
ZSTD_MatchState_t* ms, \
|
||||
U32* nextToUpdate3, \
|
||||
const BYTE* ip, \
|
||||
const BYTE* const iHighLimit, \
|
||||
const U32 rep[ZSTD_REP_NUM], \
|
||||
U32 const ll0, \
|
||||
U32 const lengthToBeat) \
|
||||
{ \
|
||||
return ZSTD_btGetAllMatches_internal( \
|
||||
matches, ms, nextToUpdate3, ip, iHighLimit, \
|
||||
rep, ll0, lengthToBeat, ZSTD_##dictMode, mls); \
|
||||
}
|
||||
|
||||
#define GEN_ZSTD_BT_GET_ALL_MATCHES(dictMode) \
|
||||
GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 3) \
|
||||
GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 4) \
|
||||
GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 5) \
|
||||
GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 6)
|
||||
|
||||
GEN_ZSTD_BT_GET_ALL_MATCHES(noDict)
|
||||
GEN_ZSTD_BT_GET_ALL_MATCHES(extDict)
|
||||
GEN_ZSTD_BT_GET_ALL_MATCHES(dictMatchState)
|
||||
|
||||
#define ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMode) \
|
||||
{ \
|
||||
ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 3), \
|
||||
ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 4), \
|
||||
ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 5), \
|
||||
ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 6) \
|
||||
}
|
||||
|
||||
static ZSTD_getAllMatchesFn
|
||||
ZSTD_selectBtGetAllMatches(ZSTD_MatchState_t const* ms, ZSTD_dictMode_e const dictMode)
|
||||
{
|
||||
ZSTD_getAllMatchesFn const getAllMatchesFns[3][4] = {
|
||||
ZSTD_BT_GET_ALL_MATCHES_ARRAY(noDict),
|
||||
ZSTD_BT_GET_ALL_MATCHES_ARRAY(extDict),
|
||||
ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMatchState)
|
||||
};
|
||||
U32 const mls = BOUNDED(3, ms->cParams.minMatch, 6);
|
||||
assert((U32)dictMode < 3);
|
||||
assert(mls - 3 < 4);
|
||||
return getAllMatchesFns[(int)dictMode][mls - 3];
|
||||
}
|
||||
|
||||
/*************************
|
||||
* LDM helper functions *
|
||||
*************************/
|
||||
|
||||
/* Struct containing info needed to make decision about ldm inclusion */
|
||||
typedef struct {
|
||||
RawSeqStore_t seqStore; /* External match candidates store for this block */
|
||||
U32 startPosInBlock; /* Start position of the current match candidate */
|
||||
U32 endPosInBlock; /* End position of the current match candidate */
|
||||
U32 offset; /* Offset of the match candidate */
|
||||
} ZSTD_optLdm_t;
|
||||
|
||||
/* ZSTD_optLdm_skipRawSeqStoreBytes():
|
||||
* Moves forward in @rawSeqStore by @nbBytes,
|
||||
* which will update the fields 'pos' and 'posInSequence'.
|
||||
*/
|
||||
static void ZSTD_optLdm_skipRawSeqStoreBytes(RawSeqStore_t* rawSeqStore, size_t nbBytes)
|
||||
{
|
||||
U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
|
||||
while (currPos && rawSeqStore->pos < rawSeqStore->size) {
|
||||
rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos];
|
||||
if (currPos >= currSeq.litLength + currSeq.matchLength) {
|
||||
currPos -= currSeq.litLength + currSeq.matchLength;
|
||||
rawSeqStore->pos++;
|
||||
} else {
|
||||
rawSeqStore->posInSequence = currPos;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) {
|
||||
rawSeqStore->posInSequence = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* ZSTD_opt_getNextMatchAndUpdateSeqStore():
|
||||
* Calculates the beginning and end of the next match in the current block.
|
||||
* Updates 'pos' and 'posInSequence' of the ldmSeqStore.
|
||||
*/
|
||||
static void
|
||||
ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock,
|
||||
U32 blockBytesRemaining)
|
||||
{
|
||||
rawSeq currSeq;
|
||||
U32 currBlockEndPos;
|
||||
U32 literalsBytesRemaining;
|
||||
U32 matchBytesRemaining;
|
||||
|
||||
/* Setting match end position to MAX to ensure we never use an LDM during this block */
|
||||
if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) {
|
||||
optLdm->startPosInBlock = UINT_MAX;
|
||||
optLdm->endPosInBlock = UINT_MAX;
|
||||
return;
|
||||
}
|
||||
/* Calculate appropriate bytes left in matchLength and litLength
|
||||
* after adjusting based on ldmSeqStore->posInSequence */
|
||||
currSeq = optLdm->seqStore.seq[optLdm->seqStore.pos];
|
||||
assert(optLdm->seqStore.posInSequence <= currSeq.litLength + currSeq.matchLength);
|
||||
currBlockEndPos = currPosInBlock + blockBytesRemaining;
|
||||
literalsBytesRemaining = (optLdm->seqStore.posInSequence < currSeq.litLength) ?
|
||||
currSeq.litLength - (U32)optLdm->seqStore.posInSequence :
|
||||
0;
|
||||
matchBytesRemaining = (literalsBytesRemaining == 0) ?
|
||||
currSeq.matchLength - ((U32)optLdm->seqStore.posInSequence - currSeq.litLength) :
|
||||
currSeq.matchLength;
|
||||
|
||||
/* If there are more literal bytes than bytes remaining in block, no ldm is possible */
|
||||
if (literalsBytesRemaining >= blockBytesRemaining) {
|
||||
optLdm->startPosInBlock = UINT_MAX;
|
||||
optLdm->endPosInBlock = UINT_MAX;
|
||||
ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, blockBytesRemaining);
|
||||
return;
|
||||
}
|
||||
|
||||
/* Matches may be < minMatch by this process. In that case, we will reject them
|
||||
when we are deciding whether or not to add the ldm */
|
||||
optLdm->startPosInBlock = currPosInBlock + literalsBytesRemaining;
|
||||
optLdm->endPosInBlock = optLdm->startPosInBlock + matchBytesRemaining;
|
||||
optLdm->offset = currSeq.offset;
|
||||
|
||||
if (optLdm->endPosInBlock > currBlockEndPos) {
|
||||
/* Match ends after the block ends, we can't use the whole match */
|
||||
optLdm->endPosInBlock = currBlockEndPos;
|
||||
ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, currBlockEndPos - currPosInBlock);
|
||||
} else {
|
||||
/* Consume nb of bytes equal to size of sequence left */
|
||||
ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, literalsBytesRemaining + matchBytesRemaining);
|
||||
}
|
||||
}
|
||||
|
||||
/* ZSTD_optLdm_maybeAddMatch():
|
||||
* Adds a match if it's long enough,
|
||||
* based on it's 'matchStartPosInBlock' and 'matchEndPosInBlock',
|
||||
* into 'matches'. Maintains the correct ordering of 'matches'.
|
||||
*/
|
||||
static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
|
||||
const ZSTD_optLdm_t* optLdm, U32 currPosInBlock,
|
||||
U32 minMatch)
|
||||
{
|
||||
U32 const posDiff = currPosInBlock - optLdm->startPosInBlock;
|
||||
/* Note: ZSTD_match_t actually contains offBase and matchLength (before subtracting MINMATCH) */
|
||||
U32 const candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff;
|
||||
|
||||
/* Ensure that current block position is not outside of the match */
|
||||
if (currPosInBlock < optLdm->startPosInBlock
|
||||
|| currPosInBlock >= optLdm->endPosInBlock
|
||||
|| candidateMatchLength < minMatch) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (*nbMatches == 0 || ((candidateMatchLength > matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM)) {
|
||||
U32 const candidateOffBase = OFFSET_TO_OFFBASE(optLdm->offset);
|
||||
DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offBase: %u matchLength %u) at block position=%u",
|
||||
candidateOffBase, candidateMatchLength, currPosInBlock);
|
||||
matches[*nbMatches].len = candidateMatchLength;
|
||||
matches[*nbMatches].off = candidateOffBase;
|
||||
(*nbMatches)++;
|
||||
}
|
||||
}
|
||||
|
||||
/* ZSTD_optLdm_processMatchCandidate():
|
||||
* Wrapper function to update ldm seq store and call ldm functions as necessary.
|
||||
*/
|
||||
static void
|
||||
ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm,
|
||||
ZSTD_match_t* matches, U32* nbMatches,
|
||||
U32 currPosInBlock, U32 remainingBytes,
|
||||
U32 minMatch)
|
||||
{
|
||||
if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (currPosInBlock >= optLdm->endPosInBlock) {
|
||||
if (currPosInBlock > optLdm->endPosInBlock) {
|
||||
/* The position at which ZSTD_optLdm_processMatchCandidate() is called is not necessarily
|
||||
* at the end of a match from the ldm seq store, and will often be some bytes
|
||||
* over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots"
|
||||
*/
|
||||
U32 const posOvershoot = currPosInBlock - optLdm->endPosInBlock;
|
||||
ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, posOvershoot);
|
||||
}
|
||||
ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes);
|
||||
}
|
||||
ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock, minMatch);
|
||||
}
|
||||
|
||||
|
||||
/*-*******************************
|
||||
* Optimal parser
|
||||
*********************************/
|
||||
|
||||
#if 0 /* debug */
|
||||
|
||||
static void
|
||||
listStats(const U32* table, int lastEltID)
|
||||
{
|
||||
int const nbElts = lastEltID + 1;
|
||||
int enb;
|
||||
for (enb=0; enb < nbElts; enb++) {
|
||||
(void)table;
|
||||
/* RAWLOG(2, "%3i:%3i, ", enb, table[enb]); */
|
||||
RAWLOG(2, "%4i,", table[enb]);
|
||||
}
|
||||
RAWLOG(2, " \n");
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#define LIT_PRICE(_p) (int)ZSTD_rawLiteralsCost(_p, 1, optStatePtr, optLevel)
|
||||
#define LL_PRICE(_l) (int)ZSTD_litLengthPrice(_l, optStatePtr, optLevel)
|
||||
#define LL_INCPRICE(_l) (LL_PRICE(_l) - LL_PRICE(_l-1))
|
||||
|
||||
FORCE_INLINE_TEMPLATE
|
||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
size_t
|
||||
ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms,
|
||||
SeqStore_t* seqStore,
|
||||
U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize,
|
||||
const int optLevel,
|
||||
const ZSTD_dictMode_e dictMode)
|
||||
{
|
||||
optState_t* const optStatePtr = &ms->opt;
|
||||
const BYTE* const istart = (const BYTE*)src;
|
||||
const BYTE* ip = istart;
|
||||
const BYTE* anchor = istart;
|
||||
const BYTE* const iend = istart + srcSize;
|
||||
const BYTE* const ilimit = iend - 8;
|
||||
const BYTE* const base = ms->window.base;
|
||||
const BYTE* const prefixStart = base + ms->window.dictLimit;
|
||||
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
||||
|
||||
ZSTD_getAllMatchesFn getAllMatches = ZSTD_selectBtGetAllMatches(ms, dictMode);
|
||||
|
||||
U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
|
||||
U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4;
|
||||
U32 nextToUpdate3 = ms->nextToUpdate;
|
||||
|
||||
ZSTD_optimal_t* const opt = optStatePtr->priceTable;
|
||||
ZSTD_match_t* const matches = optStatePtr->matchTable;
|
||||
ZSTD_optimal_t lastStretch;
|
||||
ZSTD_optLdm_t optLdm;
|
||||
|
||||
ZSTD_memset(&lastStretch, 0, sizeof(ZSTD_optimal_t));
|
||||
|
||||
optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore;
|
||||
optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0;
|
||||
ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip));
|
||||
|
||||
/* init */
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u",
|
||||
(U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate);
|
||||
assert(optLevel <= 2);
|
||||
ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel);
|
||||
ip += (ip==prefixStart);
|
||||
|
||||
/* Match Loop */
|
||||
while (ip < ilimit) {
|
||||
U32 cur, last_pos = 0;
|
||||
|
||||
/* find first match */
|
||||
{ U32 const litlen = (U32)(ip - anchor);
|
||||
U32 const ll0 = !litlen;
|
||||
U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, ip, iend, rep, ll0, minMatch);
|
||||
ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches,
|
||||
(U32)(ip-istart), (U32)(iend-ip),
|
||||
minMatch);
|
||||
if (!nbMatches) {
|
||||
DEBUGLOG(8, "no match found at cPos %u", (unsigned)(ip-istart));
|
||||
ip++;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Match found: let's store this solution, and eventually find more candidates.
|
||||
* During this forward pass, @opt is used to store stretches,
|
||||
* defined as "a match followed by N literals".
|
||||
* Note how this is different from a Sequence, which is "N literals followed by a match".
|
||||
* Storing stretches allows us to store different match predecessors
|
||||
* for each literal position part of a literals run. */
|
||||
|
||||
/* initialize opt[0] */
|
||||
opt[0].mlen = 0; /* there are only literals so far */
|
||||
opt[0].litlen = litlen;
|
||||
/* No need to include the actual price of the literals before the first match
|
||||
* because it is static for the duration of the forward pass, and is included
|
||||
* in every subsequent price. But, we include the literal length because
|
||||
* the cost variation of litlen depends on the value of litlen.
|
||||
*/
|
||||
opt[0].price = LL_PRICE(litlen);
|
||||
ZSTD_STATIC_ASSERT(sizeof(opt[0].rep[0]) == sizeof(rep[0]));
|
||||
ZSTD_memcpy(&opt[0].rep, rep, sizeof(opt[0].rep));
|
||||
|
||||
/* large match -> immediate encoding */
|
||||
{ U32 const maxML = matches[nbMatches-1].len;
|
||||
U32 const maxOffBase = matches[nbMatches-1].off;
|
||||
DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffBase=%u at cPos=%u => start new series",
|
||||
nbMatches, maxML, maxOffBase, (U32)(ip-prefixStart));
|
||||
|
||||
if (maxML > sufficient_len) {
|
||||
lastStretch.litlen = 0;
|
||||
lastStretch.mlen = maxML;
|
||||
lastStretch.off = maxOffBase;
|
||||
DEBUGLOG(6, "large match (%u>%u) => immediate encoding",
|
||||
maxML, sufficient_len);
|
||||
cur = 0;
|
||||
last_pos = maxML;
|
||||
goto _shortestPath;
|
||||
} }
|
||||
|
||||
/* set prices for first matches starting position == 0 */
|
||||
assert(opt[0].price >= 0);
|
||||
{ U32 pos;
|
||||
U32 matchNb;
|
||||
for (pos = 1; pos < minMatch; pos++) {
|
||||
opt[pos].price = ZSTD_MAX_PRICE;
|
||||
opt[pos].mlen = 0;
|
||||
opt[pos].litlen = litlen + pos;
|
||||
}
|
||||
for (matchNb = 0; matchNb < nbMatches; matchNb++) {
|
||||
U32 const offBase = matches[matchNb].off;
|
||||
U32 const end = matches[matchNb].len;
|
||||
for ( ; pos <= end ; pos++ ) {
|
||||
int const matchPrice = (int)ZSTD_getMatchPrice(offBase, pos, optStatePtr, optLevel);
|
||||
int const sequencePrice = opt[0].price + matchPrice;
|
||||
DEBUGLOG(7, "rPos:%u => set initial price : %.2f",
|
||||
pos, ZSTD_fCost(sequencePrice));
|
||||
opt[pos].mlen = pos;
|
||||
opt[pos].off = offBase;
|
||||
opt[pos].litlen = 0; /* end of match */
|
||||
opt[pos].price = sequencePrice + LL_PRICE(0);
|
||||
}
|
||||
}
|
||||
last_pos = pos-1;
|
||||
opt[pos].price = ZSTD_MAX_PRICE;
|
||||
}
|
||||
}
|
||||
|
||||
/* check further positions */
|
||||
for (cur = 1; cur <= last_pos; cur++) {
|
||||
const BYTE* const inr = ip + cur;
|
||||
assert(cur <= ZSTD_OPT_NUM);
|
||||
DEBUGLOG(7, "cPos:%i==rPos:%u", (int)(inr-istart), cur);
|
||||
|
||||
/* Fix current position with one literal if cheaper */
|
||||
{ U32 const litlen = opt[cur-1].litlen + 1;
|
||||
int const price = opt[cur-1].price
|
||||
+ LIT_PRICE(ip+cur-1)
|
||||
+ LL_INCPRICE(litlen);
|
||||
assert(price < 1000000000); /* overflow check */
|
||||
if (price <= opt[cur].price) {
|
||||
ZSTD_optimal_t const prevMatch = opt[cur];
|
||||
DEBUGLOG(7, "cPos:%i==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)",
|
||||
(int)(inr-istart), cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen,
|
||||
opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]);
|
||||
opt[cur] = opt[cur-1];
|
||||
opt[cur].litlen = litlen;
|
||||
opt[cur].price = price;
|
||||
if ( (optLevel >= 1) /* additional check only for higher modes */
|
||||
&& (prevMatch.litlen == 0) /* replace a match */
|
||||
&& (LL_INCPRICE(1) < 0) /* ll1 is cheaper than ll0 */
|
||||
&& LIKELY(ip + cur < iend)
|
||||
) {
|
||||
/* check next position, in case it would be cheaper */
|
||||
int with1literal = prevMatch.price + LIT_PRICE(ip+cur) + LL_INCPRICE(1);
|
||||
int withMoreLiterals = price + LIT_PRICE(ip+cur) + LL_INCPRICE(litlen+1);
|
||||
DEBUGLOG(7, "then at next rPos %u : match+1lit %.2f vs %ulits %.2f",
|
||||
cur+1, ZSTD_fCost(with1literal), litlen+1, ZSTD_fCost(withMoreLiterals));
|
||||
if ( (with1literal < withMoreLiterals)
|
||||
&& (with1literal < opt[cur+1].price) ) {
|
||||
/* update offset history - before it disappears */
|
||||
U32 const prev = cur - prevMatch.mlen;
|
||||
Repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, prevMatch.off, opt[prev].litlen==0);
|
||||
assert(cur >= prevMatch.mlen);
|
||||
DEBUGLOG(7, "==> match+1lit is cheaper (%.2f < %.2f) (hist:%u,%u,%u) !",
|
||||
ZSTD_fCost(with1literal), ZSTD_fCost(withMoreLiterals),
|
||||
newReps.rep[0], newReps.rep[1], newReps.rep[2] );
|
||||
opt[cur+1] = prevMatch; /* mlen & offbase */
|
||||
ZSTD_memcpy(opt[cur+1].rep, &newReps, sizeof(Repcodes_t));
|
||||
opt[cur+1].litlen = 1;
|
||||
opt[cur+1].price = with1literal;
|
||||
if (last_pos < cur+1) last_pos = cur+1;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
DEBUGLOG(7, "cPos:%i==rPos:%u : literal would cost more (%.2f>%.2f)",
|
||||
(int)(inr-istart), cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price));
|
||||
}
|
||||
}
|
||||
|
||||
/* Offset history is not updated during match comparison.
|
||||
* Do it here, now that the match is selected and confirmed.
|
||||
*/
|
||||
ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(Repcodes_t));
|
||||
assert(cur >= opt[cur].mlen);
|
||||
if (opt[cur].litlen == 0) {
|
||||
/* just finished a match => alter offset history */
|
||||
U32 const prev = cur - opt[cur].mlen;
|
||||
Repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[prev].litlen==0);
|
||||
ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(Repcodes_t));
|
||||
}
|
||||
|
||||
/* last match must start at a minimum distance of 8 from oend */
|
||||
if (inr > ilimit) continue;
|
||||
|
||||
if (cur == last_pos) break;
|
||||
|
||||
if ( (optLevel==0) /*static_test*/
|
||||
&& (opt[cur+1].price <= opt[cur].price + (BITCOST_MULTIPLIER/2)) ) {
|
||||
DEBUGLOG(7, "skip current position : next rPos(%u) price is cheaper", cur+1);
|
||||
continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */
|
||||
}
|
||||
|
||||
assert(opt[cur].price >= 0);
|
||||
{ U32 const ll0 = (opt[cur].litlen == 0);
|
||||
int const previousPrice = opt[cur].price;
|
||||
int const basePrice = previousPrice + LL_PRICE(0);
|
||||
U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, inr, iend, opt[cur].rep, ll0, minMatch);
|
||||
U32 matchNb;
|
||||
|
||||
ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches,
|
||||
(U32)(inr-istart), (U32)(iend-inr),
|
||||
minMatch);
|
||||
|
||||
if (!nbMatches) {
|
||||
DEBUGLOG(7, "rPos:%u : no match found", cur);
|
||||
continue;
|
||||
}
|
||||
|
||||
{ U32 const longestML = matches[nbMatches-1].len;
|
||||
DEBUGLOG(7, "cPos:%i==rPos:%u, found %u matches, of longest ML=%u",
|
||||
(int)(inr-istart), cur, nbMatches, longestML);
|
||||
|
||||
if ( (longestML > sufficient_len)
|
||||
|| (cur + longestML >= ZSTD_OPT_NUM)
|
||||
|| (ip + cur + longestML >= iend) ) {
|
||||
lastStretch.mlen = longestML;
|
||||
lastStretch.off = matches[nbMatches-1].off;
|
||||
lastStretch.litlen = 0;
|
||||
last_pos = cur + longestML;
|
||||
goto _shortestPath;
|
||||
} }
|
||||
|
||||
/* set prices using matches found at position == cur */
|
||||
for (matchNb = 0; matchNb < nbMatches; matchNb++) {
|
||||
U32 const offset = matches[matchNb].off;
|
||||
U32 const lastML = matches[matchNb].len;
|
||||
U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch;
|
||||
U32 mlen;
|
||||
|
||||
DEBUGLOG(7, "testing match %u => offBase=%4u, mlen=%2u, llen=%2u",
|
||||
matchNb, matches[matchNb].off, lastML, opt[cur].litlen);
|
||||
|
||||
for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */
|
||||
U32 const pos = cur + mlen;
|
||||
int const price = basePrice + (int)ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel);
|
||||
|
||||
if ((pos > last_pos) || (price < opt[pos].price)) {
|
||||
DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)",
|
||||
pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price));
|
||||
while (last_pos < pos) {
|
||||
/* fill empty positions, for future comparisons */
|
||||
last_pos++;
|
||||
opt[last_pos].price = ZSTD_MAX_PRICE;
|
||||
opt[last_pos].litlen = !0; /* just needs to be != 0, to mean "not an end of match" */
|
||||
}
|
||||
opt[pos].mlen = mlen;
|
||||
opt[pos].off = offset;
|
||||
opt[pos].litlen = 0;
|
||||
opt[pos].price = price;
|
||||
} else {
|
||||
DEBUGLOG(7, "rPos:%u (ml=%2u) => new price is worse (%.2f>=%.2f)",
|
||||
pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price));
|
||||
if (optLevel==0) break; /* early update abort; gets ~+10% speed for about -0.01 ratio loss */
|
||||
}
|
||||
} } }
|
||||
opt[last_pos+1].price = ZSTD_MAX_PRICE;
|
||||
} /* for (cur = 1; cur <= last_pos; cur++) */
|
||||
|
||||
lastStretch = opt[last_pos];
|
||||
assert(cur >= lastStretch.mlen);
|
||||
cur = last_pos - lastStretch.mlen;
|
||||
|
||||
_shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
|
||||
assert(opt[0].mlen == 0);
|
||||
assert(last_pos >= lastStretch.mlen);
|
||||
assert(cur == last_pos - lastStretch.mlen);
|
||||
|
||||
if (lastStretch.mlen==0) {
|
||||
/* no solution : all matches have been converted into literals */
|
||||
assert(lastStretch.litlen == (ip - anchor) + last_pos);
|
||||
ip += last_pos;
|
||||
continue;
|
||||
}
|
||||
assert(lastStretch.off > 0);
|
||||
|
||||
/* Update offset history */
|
||||
if (lastStretch.litlen == 0) {
|
||||
/* finishing on a match : update offset history */
|
||||
Repcodes_t const reps = ZSTD_newRep(opt[cur].rep, lastStretch.off, opt[cur].litlen==0);
|
||||
ZSTD_memcpy(rep, &reps, sizeof(Repcodes_t));
|
||||
} else {
|
||||
ZSTD_memcpy(rep, lastStretch.rep, sizeof(Repcodes_t));
|
||||
assert(cur >= lastStretch.litlen);
|
||||
cur -= lastStretch.litlen;
|
||||
}
|
||||
|
||||
/* Let's write the shortest path solution.
|
||||
* It is stored in @opt in reverse order,
|
||||
* starting from @storeEnd (==cur+2),
|
||||
* effectively partially @opt overwriting.
|
||||
* Content is changed too:
|
||||
* - So far, @opt stored stretches, aka a match followed by literals
|
||||
* - Now, it will store sequences, aka literals followed by a match
|
||||
*/
|
||||
{ U32 const storeEnd = cur + 2;
|
||||
U32 storeStart = storeEnd;
|
||||
U32 stretchPos = cur;
|
||||
|
||||
DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)",
|
||||
last_pos, cur); (void)last_pos;
|
||||
assert(storeEnd < ZSTD_OPT_SIZE);
|
||||
DEBUGLOG(6, "last stretch copied into pos=%u (llen=%u,mlen=%u,ofc=%u)",
|
||||
storeEnd, lastStretch.litlen, lastStretch.mlen, lastStretch.off);
|
||||
if (lastStretch.litlen > 0) {
|
||||
/* last "sequence" is unfinished: just a bunch of literals */
|
||||
opt[storeEnd].litlen = lastStretch.litlen;
|
||||
opt[storeEnd].mlen = 0;
|
||||
storeStart = storeEnd-1;
|
||||
opt[storeStart] = lastStretch;
|
||||
} {
|
||||
opt[storeEnd] = lastStretch; /* note: litlen will be fixed */
|
||||
storeStart = storeEnd;
|
||||
}
|
||||
while (1) {
|
||||
ZSTD_optimal_t nextStretch = opt[stretchPos];
|
||||
opt[storeStart].litlen = nextStretch.litlen;
|
||||
DEBUGLOG(6, "selected sequence (llen=%u,mlen=%u,ofc=%u)",
|
||||
opt[storeStart].litlen, opt[storeStart].mlen, opt[storeStart].off);
|
||||
if (nextStretch.mlen == 0) {
|
||||
/* reaching beginning of segment */
|
||||
break;
|
||||
}
|
||||
storeStart--;
|
||||
opt[storeStart] = nextStretch; /* note: litlen will be fixed */
|
||||
assert(nextStretch.litlen + nextStretch.mlen <= stretchPos);
|
||||
stretchPos -= nextStretch.litlen + nextStretch.mlen;
|
||||
}
|
||||
|
||||
/* save sequences */
|
||||
DEBUGLOG(6, "sending selected sequences into seqStore");
|
||||
{ U32 storePos;
|
||||
for (storePos=storeStart; storePos <= storeEnd; storePos++) {
|
||||
U32 const llen = opt[storePos].litlen;
|
||||
U32 const mlen = opt[storePos].mlen;
|
||||
U32 const offBase = opt[storePos].off;
|
||||
U32 const advance = llen + mlen;
|
||||
DEBUGLOG(6, "considering seq starting at %i, llen=%u, mlen=%u",
|
||||
(int)(anchor - istart), (unsigned)llen, (unsigned)mlen);
|
||||
|
||||
if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */
|
||||
assert(storePos == storeEnd); /* must be last sequence */
|
||||
ip = anchor + llen; /* last "sequence" is a bunch of literals => don't progress anchor */
|
||||
continue; /* will finish */
|
||||
}
|
||||
|
||||
assert(anchor + llen <= iend);
|
||||
ZSTD_updateStats(optStatePtr, llen, anchor, offBase, mlen);
|
||||
ZSTD_storeSeq(seqStore, llen, anchor, iend, offBase, mlen);
|
||||
anchor += advance;
|
||||
ip = anchor;
|
||||
} }
|
||||
DEBUGLOG(7, "new offset history : %u, %u, %u", rep[0], rep[1], rep[2]);
|
||||
|
||||
/* update all costs */
|
||||
ZSTD_setBasePrices(optStatePtr, optLevel);
|
||||
}
|
||||
} /* while (ip < ilimit) */
|
||||
|
||||
/* Return the last literals size */
|
||||
return (size_t)(iend - anchor);
|
||||
}
|
||||
#endif /* build exclusions */
|
||||
|
||||
#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR
|
||||
static size_t ZSTD_compressBlock_opt0(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode)
|
||||
{
|
||||
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /* optLevel */, dictMode);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR
|
||||
static size_t ZSTD_compressBlock_opt2(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode)
|
||||
{
|
||||
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /* optLevel */, dictMode);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR
|
||||
size_t ZSTD_compressBlock_btopt(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize)
|
||||
ZSTD_MatchState_t* const ms, SeqStore_t* const seqStore,
|
||||
U32 rep[ZSTD_REP_NUM], const void* const src, size_t const srcSize)
|
||||
{
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_btopt");
|
||||
return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_noDict);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
|
||||
#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR
|
||||
/* ZSTD_initStats_ultra():
|
||||
* make a first compression pass, just to seed stats with more accurate starting values.
|
||||
* only works on first block, with no dictionary and no ldm.
|
||||
* this function cannot error out, its narrow contract must be respected.
|
||||
*/
|
||||
static
|
||||
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
|
||||
void ZSTD_initStats_ultra(ZSTD_MatchState_t* ms,
|
||||
SeqStore_t* seqStore,
|
||||
U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */
|
||||
ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep));
|
||||
|
||||
DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize);
|
||||
assert(ms->opt.litLengthSum == 0); /* first block */
|
||||
assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */
|
||||
assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */
|
||||
assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */
|
||||
|
||||
ZSTD_compressBlock_opt2(ms, seqStore, tmpRep, src, srcSize, ZSTD_noDict); /* generate stats into ms->opt*/
|
||||
|
||||
/* invalidate first scan from history, only keep entropy stats */
|
||||
ZSTD_resetSeqStore(seqStore);
|
||||
ms->window.base -= srcSize;
|
||||
ms->window.dictLimit += (U32)srcSize;
|
||||
ms->window.lowLimit = ms->window.dictLimit;
|
||||
ms->nextToUpdate = ms->window.dictLimit;
|
||||
|
||||
ZSTD_RustOptState state;
|
||||
ZSTD_rustOptState_init_with_costs(&state, ms, NULL);
|
||||
return ZSTD_rust_compressBlock_opt(&state, seqStore, rep, src, srcSize,
|
||||
0, ZSTD_rust_dict_noDict);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_btultra(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize);
|
||||
return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_btultra2(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize)
|
||||
{
|
||||
U32 const curr = (U32)((const BYTE*)src - ms->window.base);
|
||||
DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize);
|
||||
|
||||
/* 2-passes strategy:
|
||||
* this strategy makes a first pass over first block to collect statistics
|
||||
* in order to seed next round's statistics with it.
|
||||
* After 1st pass, function forgets history, and starts a new block.
|
||||
* Consequently, this can only work if no data has been previously loaded in tables,
|
||||
* aka, no dictionary, no prefix, no ldm preprocessing.
|
||||
* The compression ratio gain is generally small (~0.5% on first block),
|
||||
* the cost is 2x cpu time on first block. */
|
||||
assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
|
||||
if ( (ms->opt.litLengthSum==0) /* first block */
|
||||
&& (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
|
||||
&& (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */
|
||||
&& (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */
|
||||
&& (srcSize > ZSTD_PREDEF_THRESHOLD) /* input large enough to not employ default stats */
|
||||
) {
|
||||
ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize);
|
||||
}
|
||||
|
||||
return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR
|
||||
size_t ZSTD_compressBlock_btopt_dictMatchState(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize)
|
||||
ZSTD_MatchState_t* const ms, SeqStore_t* const seqStore,
|
||||
U32 rep[ZSTD_REP_NUM], const void* const src, size_t const srcSize)
|
||||
{
|
||||
return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState);
|
||||
ZSTD_RustOptState state;
|
||||
ZSTD_RustOptState dictState;
|
||||
ZSTD_MatchState_t dictCopy = *ms->dictMatchState;
|
||||
ZSTD_rustOptState_init(&dictState, &dictCopy, NULL);
|
||||
ZSTD_rustOptState_init_with_costs(&state, ms, &dictState);
|
||||
return ZSTD_rust_compressBlock_opt(&state, seqStore, rep, src, srcSize,
|
||||
0, ZSTD_rust_dict_dictMatchState);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_btopt_extDict(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize)
|
||||
ZSTD_MatchState_t* const ms, SeqStore_t* const seqStore,
|
||||
U32 rep[ZSTD_REP_NUM], const void* const src, size_t const srcSize)
|
||||
{
|
||||
return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_extDict);
|
||||
ZSTD_RustOptState state;
|
||||
ZSTD_rustOptState_init_with_costs(&state, ms, NULL);
|
||||
return ZSTD_rust_compressBlock_opt(&state, seqStore, rep, src, srcSize,
|
||||
0, ZSTD_rust_dict_extDict);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR
|
||||
size_t ZSTD_compressBlock_btultra_dictMatchState(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize)
|
||||
size_t ZSTD_compressBlock_btultra(
|
||||
ZSTD_MatchState_t* const ms, SeqStore_t* const seqStore,
|
||||
U32 rep[ZSTD_REP_NUM], const void* const src, size_t const srcSize)
|
||||
{
|
||||
return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState);
|
||||
ZSTD_RustOptState state;
|
||||
ZSTD_rustOptState_init_with_costs(&state, ms, NULL);
|
||||
return ZSTD_rust_compressBlock_opt(&state, seqStore, rep, src, srcSize,
|
||||
2, ZSTD_rust_dict_noDict);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_btultra_dictMatchState(
|
||||
ZSTD_MatchState_t* const ms, SeqStore_t* const seqStore,
|
||||
U32 rep[ZSTD_REP_NUM], const void* const src, size_t const srcSize)
|
||||
{
|
||||
ZSTD_RustOptState state;
|
||||
ZSTD_RustOptState dictState;
|
||||
ZSTD_MatchState_t dictCopy = *ms->dictMatchState;
|
||||
ZSTD_rustOptState_init(&dictState, &dictCopy, NULL);
|
||||
ZSTD_rustOptState_init_with_costs(&state, ms, &dictState);
|
||||
return ZSTD_rust_compressBlock_opt(&state, seqStore, rep, src, srcSize,
|
||||
2, ZSTD_rust_dict_dictMatchState);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_btultra_extDict(
|
||||
ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
||||
const void* src, size_t srcSize)
|
||||
ZSTD_MatchState_t* const ms, SeqStore_t* const seqStore,
|
||||
U32 rep[ZSTD_REP_NUM], const void* const src, size_t const srcSize)
|
||||
{
|
||||
return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_extDict);
|
||||
ZSTD_RustOptState state;
|
||||
ZSTD_rustOptState_init_with_costs(&state, ms, NULL);
|
||||
return ZSTD_rust_compressBlock_opt(&state, seqStore, rep, src, srcSize,
|
||||
2, ZSTD_rust_dict_extDict);
|
||||
}
|
||||
|
||||
size_t ZSTD_compressBlock_btultra2(
|
||||
ZSTD_MatchState_t* const ms, SeqStore_t* const seqStore,
|
||||
U32 rep[ZSTD_REP_NUM], const void* const src, size_t const srcSize)
|
||||
{
|
||||
ZSTD_RustOptState state;
|
||||
ZSTD_rustOptState_init_with_costs(&state, ms, NULL);
|
||||
return ZSTD_rust_compressBlock_btultra2(&state, seqStore, rep, src,
|
||||
srcSize);
|
||||
}
|
||||
#endif
|
||||
|
||||
/* note : no btultra2 variant for extDict nor dictMatchState,
|
||||
* because btultra2 is not meant to work with dictionaries
|
||||
* and is only specific for the first block (no prefix) */
|
||||
#endif /* any optimal parser entry point is enabled */
|
||||
|
||||
@@ -55,6 +55,8 @@ pub mod zstd_lazy;
|
||||
#[cfg(feature = "compression")]
|
||||
pub mod zstd_ldm;
|
||||
#[cfg(feature = "compression")]
|
||||
pub mod zstd_opt;
|
||||
#[cfg(feature = "compression")]
|
||||
pub mod zstd_opt_tree;
|
||||
#[cfg(feature = "compression")]
|
||||
pub mod zstd_presplit;
|
||||
|
||||
@@ -0,0 +1,1836 @@
|
||||
#![allow(non_camel_case_types)]
|
||||
#![allow(non_snake_case)]
|
||||
#![allow(clippy::missing_safety_doc)]
|
||||
#![allow(clippy::too_many_arguments)]
|
||||
#![allow(clippy::not_unsafe_ptr_arg_deref)]
|
||||
|
||||
//! Optimal block parsing for the binary-tree compression strategies.
|
||||
//!
|
||||
//! The compression context remains private to C. `zstd_opt.c` projects the
|
||||
//! fields used by this module into `ZSTD_RustOptState`; all buffers and tables
|
||||
//! remain owned by the caller. This keeps the ABI independent of the private
|
||||
//! `ZSTD_MatchState_t` layout while moving the price model, match enumeration,
|
||||
//! optimal parse, and sequence emission into Rust.
|
||||
|
||||
use crate::bits::ZSTD_highbit32;
|
||||
use crate::common::{LL_BITS, MAX_LIT, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS};
|
||||
use crate::mem::{
|
||||
MEM_64bits, MEM_isLittleEndian, MEM_read16, MEM_read32, MEM_readLE32, MEM_readLE64, MEM_readST,
|
||||
};
|
||||
use std::cmp::min;
|
||||
use std::ffi::c_void;
|
||||
use std::mem::size_of;
|
||||
use std::os::raw::{c_int, c_uint};
|
||||
use std::ptr;
|
||||
|
||||
const ZSTD_REP_NUM: usize = 3;
|
||||
const HASH_READ_SIZE: usize = 8;
|
||||
const ZSTD_OPT_NUM: u32 = 1 << 12;
|
||||
const ZSTD_BLOCKSIZE_MAX: usize = 1 << 17;
|
||||
const ZSTD_LITFREQ_ADD: u32 = 2;
|
||||
const ZSTD_MAX_PRICE: i32 = 1 << 30;
|
||||
const ZSTD_PREDEF_THRESHOLD: usize = 8;
|
||||
const BITCOST_ACCURACY: u32 = 8;
|
||||
const BITCOST_MULTIPLIER: u32 = 1 << BITCOST_ACCURACY;
|
||||
const OFFSET_OFFBASE: u32 = ZSTD_REP_NUM as u32;
|
||||
const PRIME3BYTES: u32 = 506_832_829;
|
||||
|
||||
const ZOP_DYNAMIC: c_int = 0;
|
||||
const ZOP_PREDEF: c_int = 1;
|
||||
const ZSTD_PS_DISABLE: c_int = 2;
|
||||
|
||||
const DICT_NO_DICT: c_int = 0;
|
||||
const DICT_EXT: c_int = 1;
|
||||
const DICT_MATCH_STATE: c_int = 2;
|
||||
|
||||
const LL_CODE: [u8; 64] = [
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20,
|
||||
20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24,
|
||||
24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
|
||||
];
|
||||
|
||||
#[allow(dead_code)]
|
||||
const ML_CODE: [u8; 149] = [
|
||||
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
|
||||
26, 27, 28, 29, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 38, 38,
|
||||
38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
|
||||
40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
|
||||
41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41,
|
||||
41, 41, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
|
||||
42, 42, 42,
|
||||
];
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy)]
|
||||
struct SeqDef {
|
||||
offBase: u32,
|
||||
litLength: u16,
|
||||
mlBase: u16,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
struct SeqStore_t {
|
||||
sequencesStart: *mut SeqDef,
|
||||
sequences: *mut SeqDef,
|
||||
litStart: *mut u8,
|
||||
lit: *mut u8,
|
||||
llCode: *mut u8,
|
||||
mlCode: *mut u8,
|
||||
ofCode: *mut u8,
|
||||
maxNbSeq: usize,
|
||||
maxNbLit: usize,
|
||||
longLengthType: c_int,
|
||||
longLengthPos: u32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct ZSTD_match_t {
|
||||
off: u32,
|
||||
len: u32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct ZSTD_optimal_t {
|
||||
price: c_int,
|
||||
off: u32,
|
||||
mlen: u32,
|
||||
litlen: u32,
|
||||
rep: [u32; ZSTD_REP_NUM],
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct RawSeq {
|
||||
offset: u32,
|
||||
litLength: u32,
|
||||
matchLength: u32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy)]
|
||||
struct RawSeqStore {
|
||||
seq: *const RawSeq,
|
||||
pos: usize,
|
||||
posInSequence: usize,
|
||||
size: usize,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
impl Default for RawSeqStore {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
seq: ptr::null(),
|
||||
pos: 0,
|
||||
posInSequence: 0,
|
||||
size: 0,
|
||||
capacity: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
struct OptState {
|
||||
litFreq: *mut c_uint,
|
||||
litLengthFreq: *mut c_uint,
|
||||
matchLengthFreq: *mut c_uint,
|
||||
offCodeFreq: *mut c_uint,
|
||||
matchTable: *mut ZSTD_match_t,
|
||||
priceTable: *mut ZSTD_optimal_t,
|
||||
litSum: u32,
|
||||
litLengthSum: u32,
|
||||
matchLengthSum: u32,
|
||||
offCodeSum: u32,
|
||||
litSumBasePrice: u32,
|
||||
litLengthSumBasePrice: u32,
|
||||
matchLengthSumBasePrice: u32,
|
||||
offCodeSumBasePrice: u32,
|
||||
priceType: c_int,
|
||||
symbolCosts: *const c_void,
|
||||
literalCompressionMode: c_int,
|
||||
}
|
||||
|
||||
/* The first fields intentionally match zstd_opt_tree's public leaf view. */
|
||||
#[repr(C)]
|
||||
struct OptTreePrefix {
|
||||
hash_table: *mut u32,
|
||||
chain_table: *mut u32,
|
||||
base: *const u8,
|
||||
dict_base: *const u8,
|
||||
dict_limit: u32,
|
||||
low_limit: u32,
|
||||
loaded_dict_end: u32,
|
||||
next_to_update: *mut u32,
|
||||
hash_log: u32,
|
||||
chain_log: u32,
|
||||
search_log: u32,
|
||||
window_log: u32,
|
||||
use_c_predict: c_int,
|
||||
}
|
||||
|
||||
/// Private-state projection populated by `lib/compress/zstd_opt.c`.
|
||||
#[repr(C)]
|
||||
pub struct ZSTD_RustOptState {
|
||||
hash_table: *mut u32,
|
||||
chain_table: *mut u32,
|
||||
base: *const u8,
|
||||
dict_base: *const u8,
|
||||
dict_limit: u32,
|
||||
low_limit: u32,
|
||||
loaded_dict_end: u32,
|
||||
next_to_update: *mut u32,
|
||||
hash_log: u32,
|
||||
chain_log: u32,
|
||||
search_log: u32,
|
||||
window_log: u32,
|
||||
use_c_predict: c_int,
|
||||
hash_table3: *mut u32,
|
||||
hash_log3: u32,
|
||||
next_src: *const u8,
|
||||
min_match: u32,
|
||||
target_length: u32,
|
||||
opt: *mut OptState,
|
||||
dict_match_state: *const ZSTD_RustOptState,
|
||||
ldm_seq_store: *const RawSeqStore,
|
||||
window_base: *mut *const u8,
|
||||
window_dict_limit: *mut u32,
|
||||
window_low_limit: *mut u32,
|
||||
huf_ctable: *const usize,
|
||||
huf_repeat_valid: c_int,
|
||||
fse_litlength_ctable: *const u32,
|
||||
fse_matchlength_ctable: *const u32,
|
||||
fse_offcode_ctable: *const u32,
|
||||
}
|
||||
|
||||
#[repr(C)]
|
||||
#[derive(Clone, Copy)]
|
||||
struct FseSymbolCompressionTransform {
|
||||
deltaFindState: c_int,
|
||||
deltaNbBits: u32,
|
||||
}
|
||||
|
||||
extern "C" {
|
||||
fn ZSTD_rust_opt_updateTreeInternal(
|
||||
state: *mut OptTreePrefix,
|
||||
ip: *const c_void,
|
||||
iend: *const c_void,
|
||||
mls: u32,
|
||||
extDict: c_int,
|
||||
);
|
||||
fn HUF_getNbBitsFromCTable(ctable: *const usize, symbolValue: u32) -> u32;
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn ptr_diff(left: *const u8, right: *const u8) -> usize {
|
||||
(left as usize).wrapping_sub(right as usize)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn index_from(base: *const u8, value: *const u8) -> u32 {
|
||||
ptr_diff(value, base) as u32
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn count(mut input: *const u8, mut matched: *const u8, input_limit: *const u8) -> usize {
|
||||
let input_start = input;
|
||||
let word_size = size_of::<usize>();
|
||||
while ptr_diff(input_limit, input) >= word_size {
|
||||
let difference =
|
||||
unsafe { MEM_readST(matched.cast::<c_void>()) ^ MEM_readST(input.cast::<c_void>()) };
|
||||
if difference != 0 {
|
||||
let common = if MEM_isLittleEndian() {
|
||||
difference.trailing_zeros()
|
||||
} else {
|
||||
difference.leading_zeros()
|
||||
} as usize
|
||||
/ 8;
|
||||
return ptr_diff(input, input_start) + common;
|
||||
}
|
||||
input = input.wrapping_add(word_size);
|
||||
matched = matched.wrapping_add(word_size);
|
||||
}
|
||||
if MEM_64bits()
|
||||
&& ptr_diff(input_limit, input) >= 4
|
||||
&& unsafe { MEM_read32(matched.cast::<c_void>()) == MEM_read32(input.cast::<c_void>()) }
|
||||
{
|
||||
input = input.wrapping_add(4);
|
||||
matched = matched.wrapping_add(4);
|
||||
}
|
||||
if ptr_diff(input_limit, input) >= 2
|
||||
&& unsafe { MEM_read16(matched.cast::<c_void>()) == MEM_read16(input.cast::<c_void>()) }
|
||||
{
|
||||
input = input.wrapping_add(2);
|
||||
matched = matched.wrapping_add(2);
|
||||
}
|
||||
if ptr_diff(input, input_limit) != 0 && unsafe { *matched == *input } {
|
||||
input = input.wrapping_add(1);
|
||||
}
|
||||
ptr_diff(input, input_start)
|
||||
}
|
||||
|
||||
unsafe fn count_2segments(
|
||||
input: *const u8,
|
||||
matched: *const u8,
|
||||
input_end: *const u8,
|
||||
match_end: *const u8,
|
||||
input_start: *const u8,
|
||||
) -> usize {
|
||||
let match_remaining = ptr_diff(match_end, matched);
|
||||
let input_remaining = ptr_diff(input_end, input);
|
||||
let first_end = input.wrapping_add(min(match_remaining, input_remaining));
|
||||
let first_count = unsafe { count(input, matched, first_end) };
|
||||
if matched.wrapping_add(first_count) != match_end {
|
||||
return first_count;
|
||||
}
|
||||
first_count + unsafe { count(input.wrapping_add(first_count), input_start, input_end) }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn hash_ptr(input: *const u8, hbits: u32, mls: u32) -> usize {
|
||||
let hash_shift32 = |value: u32| {
|
||||
if hbits == 0 {
|
||||
0
|
||||
} else {
|
||||
(value >> (32 - hbits)) as usize
|
||||
}
|
||||
};
|
||||
let hash_shift64 = |value: u64| {
|
||||
if hbits == 0 {
|
||||
0
|
||||
} else {
|
||||
(value >> (64 - hbits)) as usize
|
||||
}
|
||||
};
|
||||
match mls {
|
||||
5 => hash_shift64(
|
||||
unsafe { MEM_readLE64(input.cast::<c_void>()) }
|
||||
.wrapping_shl(24)
|
||||
.wrapping_mul(889_523_592_379),
|
||||
),
|
||||
6 => hash_shift64(
|
||||
unsafe { MEM_readLE64(input.cast::<c_void>()) }
|
||||
.wrapping_shl(16)
|
||||
.wrapping_mul(227_718_039_650_203),
|
||||
),
|
||||
7 => hash_shift64(
|
||||
unsafe { MEM_readLE64(input.cast::<c_void>()) }
|
||||
.wrapping_shl(8)
|
||||
.wrapping_mul(58_295_818_150_454_627),
|
||||
),
|
||||
8 => hash_shift64(
|
||||
unsafe { MEM_readLE64(input.cast::<c_void>()) }.wrapping_mul(0xCF1B_BCDC_B7A5_6463),
|
||||
),
|
||||
_ => hash_shift32(
|
||||
unsafe { MEM_readLE32(input.cast::<c_void>()) }.wrapping_mul(2_654_435_761),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn hash3_ptr(input: *const u8, hbits: u32) -> usize {
|
||||
if hbits == 0 {
|
||||
0
|
||||
} else {
|
||||
let value = unsafe { MEM_readLE32(input.cast::<c_void>()) };
|
||||
(value.wrapping_shl(8).wrapping_mul(PRIME3BYTES) >> (32 - hbits)) as usize
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn table_get(table: *const u32, index: usize) -> u32 {
|
||||
unsafe { *table.add(index) }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn table_set(table: *mut u32, index: usize, value: u32) {
|
||||
unsafe { *table.add(index) = value };
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn lowest_match_index(state: &ZSTD_RustOptState, current: u32) -> u32 {
|
||||
let max_distance = 1u32.wrapping_shl(state.window_log);
|
||||
let within_window = if current.wrapping_sub(state.low_limit) > max_distance {
|
||||
current.wrapping_sub(max_distance)
|
||||
} else {
|
||||
state.low_limit
|
||||
};
|
||||
if state.loaded_dict_end != 0 {
|
||||
state.low_limit
|
||||
} else {
|
||||
within_window
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool {
|
||||
prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3
|
||||
}
|
||||
|
||||
#[inline]
|
||||
unsafe fn read_min_match(input: *const u8, length: u32) -> u32 {
|
||||
let value = unsafe { MEM_read32(input.cast::<c_void>()) };
|
||||
if length == 3 && MEM_isLittleEndian() {
|
||||
value << 8
|
||||
} else if length == 3 {
|
||||
value >> 8
|
||||
} else {
|
||||
value
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn ll_code(value: u32) -> usize {
|
||||
if value > 63 {
|
||||
(ZSTD_highbit32(value) + 19) as usize
|
||||
} else {
|
||||
LL_CODE[value as usize] as usize
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn ml_code(value: u32) -> usize {
|
||||
match value {
|
||||
0..=31 => value as usize,
|
||||
32..=33 => 32,
|
||||
34..=35 => 33,
|
||||
36..=37 => 34,
|
||||
38..=39 => 35,
|
||||
40..=43 => 36,
|
||||
44..=47 => 37,
|
||||
48..=51 => 38,
|
||||
52..=55 => 39,
|
||||
56..=71 => 40,
|
||||
72..=87 => 41,
|
||||
88..=127 => 42,
|
||||
_ => (ZSTD_highbit32(value) + 36) as usize,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn bit_weight(stat: u32) -> u32 {
|
||||
ZSTD_highbit32(stat.wrapping_add(1)) * BITCOST_MULTIPLIER
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn frac_weight(raw_stat: u32) -> u32 {
|
||||
let stat = raw_stat.wrapping_add(1);
|
||||
let high_bit = ZSTD_highbit32(stat);
|
||||
let base = high_bit * BITCOST_MULTIPLIER;
|
||||
base + (stat << BITCOST_ACCURACY >> high_bit)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn weight(stat: u32, opt_level: c_int) -> u32 {
|
||||
if opt_level >= 2 {
|
||||
frac_weight(stat)
|
||||
} else {
|
||||
bit_weight(stat)
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn sum_u32(table: *const c_uint, count: usize) -> u32 {
|
||||
let mut total: u32 = 0;
|
||||
for index in 0..count {
|
||||
total = total.wrapping_add(unsafe { *table.add(index) });
|
||||
}
|
||||
total
|
||||
}
|
||||
|
||||
unsafe fn downscale_stats(
|
||||
table: *mut c_uint,
|
||||
last_index: usize,
|
||||
shift: u32,
|
||||
base_one: bool,
|
||||
) -> u32 {
|
||||
let mut sum: u32 = 0;
|
||||
for index in 0..=last_index {
|
||||
let old = unsafe { *table.add(index) };
|
||||
let value = u32::from(base_one) + (old >> shift);
|
||||
unsafe { *table.add(index) = value };
|
||||
sum = sum.wrapping_add(value);
|
||||
}
|
||||
sum
|
||||
}
|
||||
|
||||
unsafe fn scale_stats(table: *mut c_uint, last_index: usize, log_target: u32) -> u32 {
|
||||
let previous = unsafe { sum_u32(table, last_index + 1) };
|
||||
let factor = previous >> log_target;
|
||||
if factor <= 1 {
|
||||
previous
|
||||
} else {
|
||||
unsafe { downscale_stats(table, last_index, ZSTD_highbit32(factor), true) }
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn fse_max_nb_bits(table: *const u32, symbol: usize) -> u32 {
|
||||
let table_log = unsafe { MEM_read16(table.cast::<c_void>()) } as usize;
|
||||
let transform_offset = 1 + if table_log == 0 {
|
||||
1
|
||||
} else {
|
||||
1 << (table_log - 1)
|
||||
};
|
||||
let transform = unsafe {
|
||||
table
|
||||
.add(transform_offset)
|
||||
.cast::<FseSymbolCompressionTransform>()
|
||||
.add(symbol)
|
||||
.read()
|
||||
};
|
||||
(transform.deltaNbBits.wrapping_add(0xFFFF)) >> 16
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn compressed_literals(opt: &OptState) -> bool {
|
||||
opt.literalCompressionMode != ZSTD_PS_DISABLE
|
||||
}
|
||||
|
||||
unsafe fn set_base_prices(opt: &mut OptState, opt_level: c_int) {
|
||||
if compressed_literals(opt) {
|
||||
opt.litSumBasePrice = weight(opt.litSum, opt_level);
|
||||
}
|
||||
opt.litLengthSumBasePrice = weight(opt.litLengthSum, opt_level);
|
||||
opt.matchLengthSumBasePrice = weight(opt.matchLengthSum, opt_level);
|
||||
opt.offCodeSumBasePrice = weight(opt.offCodeSum, opt_level);
|
||||
}
|
||||
|
||||
unsafe fn rescale_freqs(
|
||||
state: &ZSTD_RustOptState,
|
||||
src: *const u8,
|
||||
src_size: usize,
|
||||
opt_level: c_int,
|
||||
) {
|
||||
let opt = unsafe { &mut *state.opt };
|
||||
let compressed = compressed_literals(opt);
|
||||
opt.priceType = ZOP_DYNAMIC;
|
||||
if opt.litLengthSum == 0 {
|
||||
if src_size <= ZSTD_PREDEF_THRESHOLD {
|
||||
opt.priceType = ZOP_PREDEF;
|
||||
}
|
||||
|
||||
if state.huf_repeat_valid == 0 {
|
||||
/* No valid dictionary table: seed from the first source block. */
|
||||
if compressed {
|
||||
for index in 0..=MAX_LIT {
|
||||
unsafe { *opt.litFreq.add(index) = 0 };
|
||||
}
|
||||
for index in 0..src_size {
|
||||
let byte = unsafe { *src.add(index) } as usize;
|
||||
unsafe { *opt.litFreq.add(byte) = (*opt.litFreq.add(byte)).wrapping_add(1) };
|
||||
}
|
||||
opt.litSum = unsafe { downscale_stats(opt.litFreq, MAX_LIT, 8, false) };
|
||||
}
|
||||
let base_ll: [u32; MAX_LL + 1] = [
|
||||
4, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1, 1, 1, 1, 1,
|
||||
];
|
||||
unsafe {
|
||||
ptr::copy_nonoverlapping(base_ll.as_ptr(), opt.litLengthFreq, base_ll.len());
|
||||
}
|
||||
opt.litLengthSum = unsafe { sum_u32(base_ll.as_ptr(), base_ll.len()) };
|
||||
for index in 0..=MAX_ML {
|
||||
unsafe { *opt.matchLengthFreq.add(index) = 1 };
|
||||
}
|
||||
opt.matchLengthSum = (MAX_ML + 1) as u32;
|
||||
let base_of: [u32; MAX_OFF + 1] = [
|
||||
6, 2, 1, 1, 2, 3, 4, 4, 4, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||
1, 1, 1, 1,
|
||||
];
|
||||
unsafe {
|
||||
ptr::copy_nonoverlapping(base_of.as_ptr(), opt.offCodeFreq, base_of.len());
|
||||
}
|
||||
opt.offCodeSum = unsafe { sum_u32(base_of.as_ptr(), base_of.len()) };
|
||||
} else {
|
||||
/* Dictionary-seeded tables are projected as leaf pointers by C. */
|
||||
if compressed && !state.huf_ctable.is_null() {
|
||||
opt.litSum = 0;
|
||||
for lit in 0..=MAX_LIT {
|
||||
let bit_cost = unsafe { HUF_getNbBitsFromCTable(state.huf_ctable, lit as u32) };
|
||||
let value = if bit_cost != 0 {
|
||||
1u32 << (11 - bit_cost)
|
||||
} else {
|
||||
1
|
||||
};
|
||||
unsafe { *opt.litFreq.add(lit) = value };
|
||||
opt.litSum = opt.litSum.wrapping_add(value);
|
||||
}
|
||||
}
|
||||
if !state.fse_litlength_ctable.is_null() {
|
||||
opt.litLengthSum = 0;
|
||||
for symbol in 0..=MAX_LL {
|
||||
let bits = unsafe { fse_max_nb_bits(state.fse_litlength_ctable, symbol) };
|
||||
let value = if bits != 0 { 1u32 << (10 - bits) } else { 1 };
|
||||
unsafe { *opt.litLengthFreq.add(symbol) = value };
|
||||
opt.litLengthSum = opt.litLengthSum.wrapping_add(value);
|
||||
}
|
||||
}
|
||||
if !state.fse_matchlength_ctable.is_null() {
|
||||
opt.matchLengthSum = 0;
|
||||
for symbol in 0..=MAX_ML {
|
||||
let bits = unsafe { fse_max_nb_bits(state.fse_matchlength_ctable, symbol) };
|
||||
let value = if bits != 0 { 1u32 << (10 - bits) } else { 1 };
|
||||
unsafe { *opt.matchLengthFreq.add(symbol) = value };
|
||||
opt.matchLengthSum = opt.matchLengthSum.wrapping_add(value);
|
||||
}
|
||||
}
|
||||
if !state.fse_offcode_ctable.is_null() {
|
||||
opt.offCodeSum = 0;
|
||||
for symbol in 0..=MAX_OFF {
|
||||
let bits = unsafe { fse_max_nb_bits(state.fse_offcode_ctable, symbol) };
|
||||
let value = if bits != 0 { 1u32 << (10 - bits) } else { 1 };
|
||||
unsafe { *opt.offCodeFreq.add(symbol) = value };
|
||||
opt.offCodeSum = opt.offCodeSum.wrapping_add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if compressed {
|
||||
opt.litSum = unsafe { scale_stats(opt.litFreq, MAX_LIT, 12) };
|
||||
}
|
||||
opt.litLengthSum = unsafe { scale_stats(opt.litLengthFreq, MAX_LL, 11) };
|
||||
opt.matchLengthSum = unsafe { scale_stats(opt.matchLengthFreq, MAX_ML, 11) };
|
||||
opt.offCodeSum = unsafe { scale_stats(opt.offCodeFreq, MAX_OFF, 11) };
|
||||
}
|
||||
unsafe { set_base_prices(opt, opt_level) };
|
||||
}
|
||||
|
||||
unsafe fn raw_literals_cost(
|
||||
literals: *const u8,
|
||||
lit_length: u32,
|
||||
opt: &OptState,
|
||||
opt_level: c_int,
|
||||
) -> u32 {
|
||||
if lit_length == 0 {
|
||||
return 0;
|
||||
}
|
||||
if !compressed_literals(opt) {
|
||||
return lit_length.wrapping_mul(8).wrapping_mul(BITCOST_MULTIPLIER);
|
||||
}
|
||||
if opt.priceType == ZOP_PREDEF {
|
||||
return lit_length.wrapping_mul(6).wrapping_mul(BITCOST_MULTIPLIER);
|
||||
}
|
||||
let mut price = opt.litSumBasePrice.wrapping_mul(lit_length);
|
||||
let max_price = opt.litSumBasePrice.wrapping_sub(BITCOST_MULTIPLIER);
|
||||
for index in 0..lit_length as usize {
|
||||
let literal = unsafe { *literals.add(index) } as usize;
|
||||
let frequency = unsafe { *opt.litFreq.add(literal) };
|
||||
price = price.wrapping_sub(min(weight(frequency, opt_level), max_price));
|
||||
}
|
||||
price
|
||||
}
|
||||
|
||||
unsafe fn lit_length_price(lit_length: u32, opt: &OptState, opt_level: c_int) -> u32 {
|
||||
if opt.priceType == ZOP_PREDEF {
|
||||
return weight(lit_length, opt_level);
|
||||
}
|
||||
if lit_length as usize == ZSTD_BLOCKSIZE_MAX {
|
||||
return BITCOST_MULTIPLIER
|
||||
+ unsafe { lit_length_price((ZSTD_BLOCKSIZE_MAX - 1) as u32, opt, opt_level) };
|
||||
}
|
||||
let code = ll_code(lit_length);
|
||||
(LL_BITS[code] as u32 * BITCOST_MULTIPLIER) + opt.litLengthSumBasePrice
|
||||
- weight(unsafe { *opt.litLengthFreq.add(code) }, opt_level)
|
||||
}
|
||||
|
||||
unsafe fn get_match_price(
|
||||
off_base: u32,
|
||||
match_length: u32,
|
||||
opt: &OptState,
|
||||
opt_level: c_int,
|
||||
) -> u32 {
|
||||
let off_code = ZSTD_highbit32(off_base);
|
||||
let ml_base = match_length - MINMATCH as u32;
|
||||
if opt.priceType == ZOP_PREDEF {
|
||||
return weight(ml_base, opt_level) + (16u32 + off_code).wrapping_mul(BITCOST_MULTIPLIER);
|
||||
}
|
||||
let mut price = off_code * BITCOST_MULTIPLIER + opt.offCodeSumBasePrice
|
||||
- weight(
|
||||
unsafe { *opt.offCodeFreq.add(off_code as usize) },
|
||||
opt_level,
|
||||
);
|
||||
if opt_level < 2 && off_code >= 20 {
|
||||
price += (off_code - 19) * 2 * BITCOST_MULTIPLIER;
|
||||
}
|
||||
let code = ml_code(ml_base);
|
||||
price += ML_BITS[code] as u32 * BITCOST_MULTIPLIER + opt.matchLengthSumBasePrice
|
||||
- weight(unsafe { *opt.matchLengthFreq.add(code) }, opt_level);
|
||||
price + BITCOST_MULTIPLIER / 5
|
||||
}
|
||||
|
||||
unsafe fn update_stats(
|
||||
opt: &mut OptState,
|
||||
lit_length: u32,
|
||||
literals: *const u8,
|
||||
off_base: u32,
|
||||
match_length: u32,
|
||||
) {
|
||||
if compressed_literals(opt) {
|
||||
for index in 0..lit_length as usize {
|
||||
let literal = unsafe { *literals.add(index) } as usize;
|
||||
let frequency = unsafe { opt.litFreq.add(literal) };
|
||||
unsafe { *frequency = (*frequency).wrapping_add(ZSTD_LITFREQ_ADD) };
|
||||
}
|
||||
opt.litSum = opt
|
||||
.litSum
|
||||
.wrapping_add(lit_length.wrapping_mul(ZSTD_LITFREQ_ADD));
|
||||
}
|
||||
let ll = ll_code(lit_length);
|
||||
unsafe { *opt.litLengthFreq.add(ll) = (*opt.litLengthFreq.add(ll)).wrapping_add(1) };
|
||||
opt.litLengthSum = opt.litLengthSum.wrapping_add(1);
|
||||
let off = ZSTD_highbit32(off_base) as usize;
|
||||
unsafe { *opt.offCodeFreq.add(off) = (*opt.offCodeFreq.add(off)).wrapping_add(1) };
|
||||
opt.offCodeSum = opt.offCodeSum.wrapping_add(1);
|
||||
let ml = ml_code(match_length - MINMATCH as u32);
|
||||
unsafe { *opt.matchLengthFreq.add(ml) = (*opt.matchLengthFreq.add(ml)).wrapping_add(1) };
|
||||
opt.matchLengthSum = opt.matchLengthSum.wrapping_add(1);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn offset_to_offbase(offset: u32) -> u32 {
|
||||
offset.wrapping_add(OFFSET_OFFBASE)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn repcode_to_offbase(rep_code: u32) -> u32 {
|
||||
rep_code
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn new_rep(mut rep: [u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> [u32; ZSTD_REP_NUM] {
|
||||
if off_base > OFFSET_OFFBASE {
|
||||
rep[2] = rep[1];
|
||||
rep[1] = rep[0];
|
||||
rep[0] = off_base - OFFSET_OFFBASE;
|
||||
} else {
|
||||
let rep_code = off_base - 1 + u32::from(ll0);
|
||||
if rep_code != 0 {
|
||||
let current = if rep_code == ZSTD_REP_NUM as u32 {
|
||||
rep[0].wrapping_sub(1)
|
||||
} else {
|
||||
rep[rep_code as usize]
|
||||
};
|
||||
if rep_code >= 2 {
|
||||
rep[2] = rep[1];
|
||||
}
|
||||
rep[1] = rep[0];
|
||||
rep[0] = current;
|
||||
}
|
||||
}
|
||||
rep
|
||||
}
|
||||
|
||||
unsafe fn store_seq(
|
||||
seq_store: *mut SeqStore_t,
|
||||
lit_length: usize,
|
||||
literals: *const u8,
|
||||
off_base: u32,
|
||||
match_length: usize,
|
||||
) {
|
||||
let store = unsafe { &mut *seq_store };
|
||||
let sequence = store.sequences;
|
||||
let index =
|
||||
ptr_diff(sequence.cast::<u8>(), store.sequencesStart.cast::<u8>()) / size_of::<SeqDef>();
|
||||
if lit_length != 0 {
|
||||
unsafe { ptr::copy_nonoverlapping(literals, store.lit, lit_length) };
|
||||
}
|
||||
store.lit = store.lit.wrapping_add(lit_length);
|
||||
if lit_length > u16::MAX as usize {
|
||||
store.longLengthType = 1;
|
||||
store.longLengthPos = index as u32;
|
||||
}
|
||||
unsafe {
|
||||
(*sequence).litLength = lit_length as u16;
|
||||
(*sequence).offBase = off_base;
|
||||
}
|
||||
let ml_base = match_length - MINMATCH;
|
||||
if ml_base > u16::MAX as usize {
|
||||
store.longLengthType = 2;
|
||||
store.longLengthPos = index as u32;
|
||||
}
|
||||
unsafe { (*sequence).mlBase = ml_base as u16 };
|
||||
store.sequences = store.sequences.wrapping_add(1);
|
||||
}
|
||||
|
||||
unsafe fn skip_raw_seq_store_bytes(store: &mut RawSeqStore, nb_bytes: usize) {
|
||||
let mut current = (store.posInSequence + nb_bytes) as u32;
|
||||
while current != 0 && store.pos < store.size {
|
||||
let sequence = unsafe { store.seq.add(store.pos).read() };
|
||||
let sequence_size = sequence.litLength.wrapping_add(sequence.matchLength);
|
||||
if current >= sequence_size {
|
||||
current -= sequence_size;
|
||||
store.pos += 1;
|
||||
} else {
|
||||
store.posInSequence = current as usize;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if current == 0 || store.pos == store.size {
|
||||
store.posInSequence = 0;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct OptLdm {
|
||||
seq_store: RawSeqStore,
|
||||
start_pos: u32,
|
||||
end_pos: u32,
|
||||
offset: u32,
|
||||
}
|
||||
|
||||
unsafe fn ldm_next_match(opt_ldm: &mut OptLdm, current_pos: u32, block_remaining: u32) {
|
||||
if opt_ldm.seq_store.size == 0 || opt_ldm.seq_store.pos >= opt_ldm.seq_store.size {
|
||||
opt_ldm.start_pos = u32::MAX;
|
||||
opt_ldm.end_pos = u32::MAX;
|
||||
return;
|
||||
}
|
||||
let sequence = unsafe { opt_ldm.seq_store.seq.add(opt_ldm.seq_store.pos).read() };
|
||||
let block_end = current_pos.wrapping_add(block_remaining);
|
||||
let literals_remaining =
|
||||
(sequence.litLength as usize).saturating_sub(opt_ldm.seq_store.posInSequence) as u32;
|
||||
let match_remaining = if literals_remaining == 0 {
|
||||
sequence
|
||||
.matchLength
|
||||
.wrapping_sub(opt_ldm.seq_store.posInSequence as u32 - sequence.litLength)
|
||||
} else {
|
||||
sequence.matchLength
|
||||
};
|
||||
if literals_remaining >= block_remaining {
|
||||
opt_ldm.start_pos = u32::MAX;
|
||||
opt_ldm.end_pos = u32::MAX;
|
||||
unsafe { skip_raw_seq_store_bytes(&mut opt_ldm.seq_store, block_remaining as usize) };
|
||||
return;
|
||||
}
|
||||
opt_ldm.start_pos = current_pos.wrapping_add(literals_remaining);
|
||||
opt_ldm.end_pos = opt_ldm.start_pos.wrapping_add(match_remaining);
|
||||
opt_ldm.offset = sequence.offset;
|
||||
if opt_ldm.end_pos > block_end {
|
||||
opt_ldm.end_pos = block_end;
|
||||
unsafe {
|
||||
skip_raw_seq_store_bytes(
|
||||
&mut opt_ldm.seq_store,
|
||||
block_end.wrapping_sub(current_pos) as usize,
|
||||
)
|
||||
};
|
||||
} else {
|
||||
unsafe {
|
||||
skip_raw_seq_store_bytes(
|
||||
&mut opt_ldm.seq_store,
|
||||
(literals_remaining + match_remaining) as usize,
|
||||
)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn ldm_maybe_add_match(
|
||||
matches: *mut ZSTD_match_t,
|
||||
nb_matches: &mut u32,
|
||||
opt_ldm: &OptLdm,
|
||||
current_pos: u32,
|
||||
min_match: u32,
|
||||
) {
|
||||
let position_diff = current_pos.wrapping_sub(opt_ldm.start_pos);
|
||||
let candidate_length = opt_ldm
|
||||
.end_pos
|
||||
.wrapping_sub(opt_ldm.start_pos)
|
||||
.wrapping_sub(position_diff);
|
||||
if current_pos < opt_ldm.start_pos
|
||||
|| current_pos >= opt_ldm.end_pos
|
||||
|| candidate_length < min_match
|
||||
{
|
||||
return;
|
||||
}
|
||||
if *nb_matches == 0
|
||||
|| (candidate_length > unsafe { (*matches.add(*nb_matches as usize - 1)).len }
|
||||
&& *nb_matches < ZSTD_OPT_NUM)
|
||||
{
|
||||
let slot = unsafe { &mut *matches.add(*nb_matches as usize) };
|
||||
slot.len = candidate_length;
|
||||
slot.off = offset_to_offbase(opt_ldm.offset);
|
||||
*nb_matches += 1;
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn ldm_process_match(
|
||||
opt_ldm: &mut OptLdm,
|
||||
matches: *mut ZSTD_match_t,
|
||||
nb_matches: &mut u32,
|
||||
current_pos: u32,
|
||||
remaining: u32,
|
||||
min_match: u32,
|
||||
) {
|
||||
if opt_ldm.seq_store.size == 0 || opt_ldm.seq_store.pos >= opt_ldm.seq_store.size {
|
||||
return;
|
||||
}
|
||||
if current_pos >= opt_ldm.end_pos {
|
||||
if current_pos > opt_ldm.end_pos {
|
||||
unsafe {
|
||||
skip_raw_seq_store_bytes(
|
||||
&mut opt_ldm.seq_store,
|
||||
current_pos.wrapping_sub(opt_ldm.end_pos) as usize,
|
||||
)
|
||||
};
|
||||
}
|
||||
unsafe { ldm_next_match(opt_ldm, current_pos, remaining) };
|
||||
}
|
||||
unsafe { ldm_maybe_add_match(matches, nb_matches, opt_ldm, current_pos, min_match) };
|
||||
}
|
||||
|
||||
unsafe fn insert_and_find_first_index_hash3(
|
||||
state: &ZSTD_RustOptState,
|
||||
next_to_update3: &mut u32,
|
||||
ip: *const u8,
|
||||
) -> u32 {
|
||||
let target = index_from(state.base, ip);
|
||||
let hash = unsafe { hash3_ptr(ip, state.hash_log3) };
|
||||
let mut index = *next_to_update3;
|
||||
while index < target {
|
||||
let position = state.base.wrapping_add(index as usize);
|
||||
let position_hash = unsafe { hash3_ptr(position, state.hash_log3) };
|
||||
unsafe { table_set(state.hash_table3, position_hash, index) };
|
||||
index = index.wrapping_add(1);
|
||||
}
|
||||
*next_to_update3 = target;
|
||||
unsafe { table_get(state.hash_table3, hash) }
|
||||
}
|
||||
|
||||
unsafe fn insert_bt_and_get_all_matches(
|
||||
state: &ZSTD_RustOptState,
|
||||
next_to_update3: &mut u32,
|
||||
ip: *const u8,
|
||||
input_limit: *const u8,
|
||||
dict_mode: c_int,
|
||||
rep: &[u32; ZSTD_REP_NUM],
|
||||
ll0: u32,
|
||||
length_to_beat: u32,
|
||||
mls: u32,
|
||||
) -> u32 {
|
||||
let sufficient_len = min(state.target_length, ZSTD_OPT_NUM - 1);
|
||||
let current = index_from(state.base, ip);
|
||||
let min_match = if mls == 3 { 3 } else { 4 };
|
||||
let hash = unsafe { hash_ptr(ip, state.hash_log, mls) };
|
||||
let mut match_index = unsafe { table_get(state.hash_table, hash) };
|
||||
let bt_log = state.chain_log.wrapping_sub(1);
|
||||
let bt_mask = (1u32 << bt_log).wrapping_sub(1);
|
||||
let dict_limit = state.dict_limit;
|
||||
let dict_end = state.dict_base.wrapping_add(dict_limit as usize);
|
||||
let prefix_start = state.base.wrapping_add(dict_limit as usize);
|
||||
let bt_low = current.saturating_sub(bt_mask);
|
||||
let window_low = lowest_match_index(state, current);
|
||||
let match_low = if window_low == 0 { 1 } else { window_low };
|
||||
let mut smaller = unsafe { state.chain_table.add(2 * (current & bt_mask) as usize) };
|
||||
let mut larger = unsafe { smaller.add(1) };
|
||||
let mut dummy = 0u32;
|
||||
let mut match_end_index = current.wrapping_add(HASH_READ_SIZE as u32 + 1);
|
||||
let mut common_smaller = 0usize;
|
||||
let mut common_larger = 0usize;
|
||||
let mut best_length = length_to_beat.wrapping_sub(1) as usize;
|
||||
let mut match_count = 0u32;
|
||||
let mut nb_compares = 1u32 << state.search_log;
|
||||
|
||||
let dms = if dict_mode == DICT_MATCH_STATE {
|
||||
debug_assert!(!state.dict_match_state.is_null());
|
||||
unsafe { &*state.dict_match_state }
|
||||
} else {
|
||||
state
|
||||
};
|
||||
let dms_base = dms.base;
|
||||
let dms_end = dms.next_src;
|
||||
let dms_high_limit = if dict_mode == DICT_MATCH_STATE {
|
||||
index_from(dms_base, dms_end)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let dms_low_limit = if dict_mode == DICT_MATCH_STATE {
|
||||
dms.low_limit
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let dms_index_delta = if dict_mode == DICT_MATCH_STATE {
|
||||
window_low.wrapping_sub(dms_high_limit)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let dms_hash_log = if dict_mode == DICT_MATCH_STATE {
|
||||
dms.hash_log
|
||||
} else {
|
||||
state.hash_log
|
||||
};
|
||||
let dms_bt_log = if dict_mode == DICT_MATCH_STATE {
|
||||
dms.chain_log.wrapping_sub(1)
|
||||
} else {
|
||||
bt_log
|
||||
};
|
||||
let dms_bt_mask = if dict_mode == DICT_MATCH_STATE {
|
||||
(1u32 << dms_bt_log).wrapping_sub(1)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let dms_bt_low = if dict_mode == DICT_MATCH_STATE
|
||||
&& dms_bt_mask < dms_high_limit.wrapping_sub(dms_low_limit)
|
||||
{
|
||||
dms_high_limit.wrapping_sub(dms_bt_mask)
|
||||
} else {
|
||||
dms_low_limit
|
||||
};
|
||||
|
||||
for rep_code in ll0..(ZSTD_REP_NUM as u32 + ll0) {
|
||||
let rep_offset = if rep_code == ZSTD_REP_NUM as u32 {
|
||||
rep[0].wrapping_sub(1)
|
||||
} else {
|
||||
rep[rep_code as usize]
|
||||
};
|
||||
let rep_index = current.wrapping_sub(rep_offset);
|
||||
let mut rep_len = 0usize;
|
||||
if rep_offset.wrapping_sub(1) < current.wrapping_sub(dict_limit) {
|
||||
if rep_index >= window_low
|
||||
&& unsafe { read_min_match(ip, min_match) }
|
||||
== unsafe { read_min_match(ip.wrapping_sub(rep_offset as usize), min_match) }
|
||||
{
|
||||
rep_len = unsafe {
|
||||
count(
|
||||
ip.wrapping_add(min_match as usize),
|
||||
ip.wrapping_sub(rep_offset as usize)
|
||||
.wrapping_add(min_match as usize),
|
||||
input_limit,
|
||||
) + min_match as usize
|
||||
};
|
||||
}
|
||||
} else {
|
||||
let rep_match = if dict_mode == DICT_MATCH_STATE {
|
||||
dms_base.wrapping_add(rep_index.wrapping_sub(dms_index_delta) as usize)
|
||||
} else {
|
||||
state.dict_base.wrapping_add(rep_index as usize)
|
||||
};
|
||||
if dict_mode == DICT_EXT
|
||||
&& rep_offset.wrapping_sub(1) < current.wrapping_sub(window_low)
|
||||
&& index_overlap_check(dict_limit, rep_index)
|
||||
&& unsafe { read_min_match(ip, min_match) }
|
||||
== unsafe { read_min_match(rep_match, min_match) }
|
||||
{
|
||||
rep_len = unsafe {
|
||||
count_2segments(
|
||||
ip.wrapping_add(min_match as usize),
|
||||
rep_match.wrapping_add(min_match as usize),
|
||||
input_limit,
|
||||
dict_end,
|
||||
prefix_start,
|
||||
) + min_match as usize
|
||||
};
|
||||
}
|
||||
if dict_mode == DICT_MATCH_STATE
|
||||
&& rep_offset.wrapping_sub(1)
|
||||
< current.wrapping_sub(dms_low_limit.wrapping_add(dms_index_delta))
|
||||
&& index_overlap_check(dict_limit, rep_index)
|
||||
&& unsafe { read_min_match(ip, min_match) }
|
||||
== unsafe { read_min_match(rep_match, min_match) }
|
||||
{
|
||||
rep_len = unsafe {
|
||||
count_2segments(
|
||||
ip.wrapping_add(min_match as usize),
|
||||
rep_match.wrapping_add(min_match as usize),
|
||||
input_limit,
|
||||
dms_end,
|
||||
prefix_start,
|
||||
) + min_match as usize
|
||||
};
|
||||
}
|
||||
}
|
||||
if rep_len > best_length {
|
||||
best_length = rep_len;
|
||||
let slot = unsafe { &mut *state.opt_match_table().add(match_count as usize) };
|
||||
slot.off = repcode_to_offbase(rep_code - ll0 + 1);
|
||||
slot.len = rep_len as u32;
|
||||
match_count += 1;
|
||||
if rep_len > sufficient_len as usize || ip.wrapping_add(rep_len) == input_limit {
|
||||
return match_count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if mls == 3 && best_length < mls as usize {
|
||||
let match_index3 = unsafe { insert_and_find_first_index_hash3(state, next_to_update3, ip) };
|
||||
if match_index3 >= match_low && current.wrapping_sub(match_index3) < (1 << 18) {
|
||||
let match_ptr = if dict_mode == DICT_EXT && match_index3 < dict_limit {
|
||||
state.dict_base.wrapping_add(match_index3 as usize)
|
||||
} else {
|
||||
state.base.wrapping_add(match_index3 as usize)
|
||||
};
|
||||
let match_length = if dict_mode == DICT_EXT && match_index3 < dict_limit {
|
||||
unsafe { count_2segments(ip, match_ptr, input_limit, dict_end, prefix_start) }
|
||||
} else {
|
||||
unsafe { count(ip, match_ptr, input_limit) }
|
||||
};
|
||||
if match_length >= mls as usize {
|
||||
let slot = unsafe { &mut *state.opt_match_table() };
|
||||
slot.off = offset_to_offbase(current.wrapping_sub(match_index3));
|
||||
slot.len = match_length as u32;
|
||||
if match_length > sufficient_len as usize
|
||||
|| ip.wrapping_add(match_length) == input_limit
|
||||
{
|
||||
unsafe { *state.next_to_update = current.wrapping_add(1) };
|
||||
return 1;
|
||||
}
|
||||
match_count = 1;
|
||||
best_length = match_length;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unsafe { table_set(state.hash_table, hash, current) };
|
||||
while nb_compares != 0 && match_index >= match_low {
|
||||
nb_compares -= 1;
|
||||
let next_ptr = unsafe { state.chain_table.add(2 * (match_index & bt_mask) as usize) };
|
||||
let mut match_length = min(common_smaller, common_larger);
|
||||
let mut match_ptr: *const u8;
|
||||
if dict_mode != DICT_EXT || match_index.wrapping_add(match_length as u32) >= dict_limit {
|
||||
match_ptr = state.base.wrapping_add(match_index as usize);
|
||||
match_length += unsafe {
|
||||
count(
|
||||
ip.wrapping_add(match_length),
|
||||
match_ptr.wrapping_add(match_length),
|
||||
input_limit,
|
||||
)
|
||||
};
|
||||
} else {
|
||||
match_ptr = state.dict_base.wrapping_add(match_index as usize);
|
||||
match_length += unsafe {
|
||||
count_2segments(
|
||||
ip.wrapping_add(match_length),
|
||||
match_ptr.wrapping_add(match_length),
|
||||
input_limit,
|
||||
dict_end,
|
||||
prefix_start,
|
||||
)
|
||||
};
|
||||
if match_index.wrapping_add(match_length as u32) >= dict_limit {
|
||||
match_ptr = state.base.wrapping_add(match_index as usize);
|
||||
}
|
||||
}
|
||||
|
||||
if match_length > best_length {
|
||||
if match_length > match_end_index.wrapping_sub(match_index) as usize {
|
||||
match_end_index = match_index.wrapping_add(match_length as u32);
|
||||
}
|
||||
best_length = match_length;
|
||||
let slot = unsafe { &mut *state.opt_match_table().add(match_count as usize) };
|
||||
slot.off = offset_to_offbase(current.wrapping_sub(match_index));
|
||||
slot.len = match_length as u32;
|
||||
match_count += 1;
|
||||
if match_length > ZSTD_OPT_NUM as usize || ip.wrapping_add(match_length) == input_limit
|
||||
{
|
||||
if dict_mode == DICT_MATCH_STATE {
|
||||
nb_compares = 0;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let next_byte = unsafe { *match_ptr.add(match_length) };
|
||||
let input_byte = unsafe { *ip.add(match_length) };
|
||||
if next_byte < input_byte {
|
||||
unsafe { *smaller = match_index };
|
||||
common_smaller = match_length;
|
||||
if match_index <= bt_low {
|
||||
smaller = &mut dummy;
|
||||
break;
|
||||
}
|
||||
smaller = unsafe { next_ptr.add(1) };
|
||||
match_index = unsafe { *next_ptr.add(1) };
|
||||
} else {
|
||||
unsafe { *larger = match_index };
|
||||
common_larger = match_length;
|
||||
if match_index <= bt_low {
|
||||
larger = &mut dummy;
|
||||
break;
|
||||
}
|
||||
larger = next_ptr;
|
||||
match_index = unsafe { *next_ptr };
|
||||
}
|
||||
}
|
||||
unsafe {
|
||||
*smaller = 0;
|
||||
*larger = 0;
|
||||
}
|
||||
|
||||
if dict_mode == DICT_MATCH_STATE && nb_compares != 0 {
|
||||
let dict_hash = unsafe { hash_ptr(ip, dms_hash_log, mls) };
|
||||
let mut dict_match_index = unsafe { table_get(dms.hash_table, dict_hash) };
|
||||
common_smaller = 0;
|
||||
common_larger = 0;
|
||||
while nb_compares != 0 && dict_match_index > dms_low_limit {
|
||||
nb_compares -= 1;
|
||||
let next_ptr = unsafe {
|
||||
dms.chain_table
|
||||
.add(2 * (dict_match_index & dms_bt_mask) as usize)
|
||||
};
|
||||
let mut match_length = min(common_smaller, common_larger);
|
||||
let mut match_ptr = dms_base.wrapping_add(dict_match_index as usize);
|
||||
match_length += unsafe {
|
||||
count_2segments(
|
||||
ip.wrapping_add(match_length),
|
||||
match_ptr.wrapping_add(match_length),
|
||||
input_limit,
|
||||
dms_end,
|
||||
prefix_start,
|
||||
)
|
||||
};
|
||||
if dict_match_index.wrapping_add(match_length as u32) >= dms_high_limit {
|
||||
match_ptr = state
|
||||
.base
|
||||
.wrapping_add(dict_match_index.wrapping_add(dms_index_delta) as usize);
|
||||
}
|
||||
if match_length > best_length {
|
||||
let prefix_index = dict_match_index.wrapping_add(dms_index_delta);
|
||||
if match_length > match_end_index.wrapping_sub(prefix_index) as usize {
|
||||
match_end_index = prefix_index.wrapping_add(match_length as u32);
|
||||
}
|
||||
best_length = match_length;
|
||||
let slot = unsafe { &mut *state.opt_match_table().add(match_count as usize) };
|
||||
slot.off = offset_to_offbase(current.wrapping_sub(prefix_index));
|
||||
slot.len = match_length as u32;
|
||||
match_count += 1;
|
||||
if match_length > ZSTD_OPT_NUM as usize
|
||||
|| ip.wrapping_add(match_length) == input_limit
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
if dict_match_index <= dms_bt_low {
|
||||
break;
|
||||
}
|
||||
if unsafe { *match_ptr.add(match_length) } < unsafe { *ip.add(match_length) } {
|
||||
common_smaller = match_length;
|
||||
dict_match_index = unsafe { *next_ptr.add(1) };
|
||||
} else {
|
||||
common_larger = match_length;
|
||||
dict_match_index = unsafe { *next_ptr };
|
||||
}
|
||||
}
|
||||
}
|
||||
unsafe { *state.next_to_update = match_end_index.wrapping_sub(8) };
|
||||
match_count
|
||||
}
|
||||
|
||||
impl ZSTD_RustOptState {
|
||||
#[inline]
|
||||
unsafe fn opt_match_table(&self) -> *mut ZSTD_match_t {
|
||||
unsafe { (*self.opt).matchTable }
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn get_all_matches(
|
||||
state: &mut ZSTD_RustOptState,
|
||||
next_to_update3: &mut u32,
|
||||
ip: *const u8,
|
||||
input_limit: *const u8,
|
||||
rep: &[u32; ZSTD_REP_NUM],
|
||||
ll0: u32,
|
||||
length_to_beat: u32,
|
||||
dict_mode: c_int,
|
||||
mls: u32,
|
||||
) -> u32 {
|
||||
debug_assert!((3..=6).contains(&mls));
|
||||
if (ip as usize)
|
||||
< state
|
||||
.base
|
||||
.wrapping_add(unsafe { *state.next_to_update } as usize) as usize
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
unsafe {
|
||||
ZSTD_rust_opt_updateTreeInternal(
|
||||
(state as *mut ZSTD_RustOptState).cast::<OptTreePrefix>(),
|
||||
ip.cast::<c_void>(),
|
||||
input_limit.cast::<c_void>(),
|
||||
mls,
|
||||
c_int::from(dict_mode == DICT_EXT),
|
||||
);
|
||||
insert_bt_and_get_all_matches(
|
||||
state,
|
||||
next_to_update3,
|
||||
ip,
|
||||
input_limit,
|
||||
dict_mode,
|
||||
rep,
|
||||
ll0,
|
||||
length_to_beat,
|
||||
mls,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn resolve_shortest_path(
|
||||
state: &mut ZSTD_RustOptState,
|
||||
seq_store: *mut SeqStore_t,
|
||||
rep: &mut [u32; ZSTD_REP_NUM],
|
||||
istart: *const u8,
|
||||
iend: *const u8,
|
||||
anchor: &mut *const u8,
|
||||
ip: &mut *const u8,
|
||||
mut cur: u32,
|
||||
last_pos: u32,
|
||||
last_stretch: ZSTD_optimal_t,
|
||||
opt_level: c_int,
|
||||
) {
|
||||
let opt_state = unsafe { &mut *state.opt };
|
||||
let opt = opt_state.priceTable;
|
||||
if last_stretch.mlen == 0 {
|
||||
*ip = ip.wrapping_add(last_pos as usize);
|
||||
return;
|
||||
}
|
||||
|
||||
if last_stretch.litlen == 0 {
|
||||
let previous = unsafe { (*opt.add(cur as usize)).rep };
|
||||
*rep = new_rep(previous, last_stretch.off, unsafe {
|
||||
(*opt.add(cur as usize)).litlen == 0
|
||||
});
|
||||
} else {
|
||||
*rep = last_stretch.rep;
|
||||
cur = cur.wrapping_sub(last_stretch.litlen);
|
||||
}
|
||||
|
||||
let store_end = cur + 2;
|
||||
let mut store_start = store_end;
|
||||
let mut stretch_pos = cur;
|
||||
unsafe {
|
||||
*opt.add(store_end as usize) = last_stretch;
|
||||
}
|
||||
|
||||
loop {
|
||||
let next_stretch = unsafe { *opt.add(stretch_pos as usize) };
|
||||
unsafe { (*opt.add(store_start as usize)).litlen = next_stretch.litlen };
|
||||
if next_stretch.mlen == 0 {
|
||||
break;
|
||||
}
|
||||
store_start -= 1;
|
||||
unsafe { *opt.add(store_start as usize) = next_stretch };
|
||||
stretch_pos = stretch_pos.wrapping_sub(next_stretch.litlen + next_stretch.mlen);
|
||||
}
|
||||
|
||||
let mut store_pos = store_start;
|
||||
while store_pos <= store_end {
|
||||
let current = unsafe { *opt.add(store_pos as usize) };
|
||||
let lit_length = current.litlen;
|
||||
let match_length = current.mlen;
|
||||
if match_length == 0 {
|
||||
*ip = anchor.wrapping_add(lit_length as usize);
|
||||
break;
|
||||
}
|
||||
unsafe {
|
||||
update_stats(opt_state, lit_length, *anchor, current.off, match_length);
|
||||
store_seq(
|
||||
seq_store,
|
||||
lit_length as usize,
|
||||
*anchor,
|
||||
current.off,
|
||||
match_length as usize,
|
||||
);
|
||||
}
|
||||
*anchor = anchor.wrapping_add((lit_length + match_length) as usize);
|
||||
*ip = *anchor;
|
||||
store_pos += 1;
|
||||
}
|
||||
unsafe { set_base_prices(opt_state, opt_level) };
|
||||
let _ = istart;
|
||||
let _ = iend;
|
||||
}
|
||||
|
||||
unsafe fn compress_block_opt_generic(
|
||||
state: &mut ZSTD_RustOptState,
|
||||
seq_store: *mut SeqStore_t,
|
||||
reps: *mut u32,
|
||||
src: *const u8,
|
||||
src_size: usize,
|
||||
opt_level: c_int,
|
||||
dict_mode: c_int,
|
||||
) -> usize {
|
||||
let istart = src;
|
||||
let mut ip = istart;
|
||||
let mut anchor = istart;
|
||||
let iend = src.wrapping_add(src_size);
|
||||
let ilimit = if src_size >= HASH_READ_SIZE {
|
||||
iend.wrapping_sub(HASH_READ_SIZE)
|
||||
} else {
|
||||
istart
|
||||
};
|
||||
let prefix_start = state.base.wrapping_add(state.dict_limit as usize);
|
||||
let sufficient_len = min(state.target_length, ZSTD_OPT_NUM - 1);
|
||||
let min_match = if state.min_match == 3 { 3 } else { 4 };
|
||||
let mut next_to_update3 = unsafe { *state.next_to_update };
|
||||
let opt_state = unsafe { &mut *state.opt };
|
||||
let matches = opt_state.matchTable;
|
||||
let opt = opt_state.priceTable;
|
||||
let mut last_stretch = ZSTD_optimal_t::default();
|
||||
let mut opt_ldm = OptLdm {
|
||||
seq_store: if state.ldm_seq_store.is_null() {
|
||||
RawSeqStore::default()
|
||||
} else {
|
||||
unsafe { *state.ldm_seq_store }
|
||||
},
|
||||
start_pos: 0,
|
||||
end_pos: 0,
|
||||
offset: 0,
|
||||
};
|
||||
|
||||
unsafe { ldm_next_match(&mut opt_ldm, 0, src_size as u32) };
|
||||
unsafe { rescale_freqs(state, src, src_size, opt_level) };
|
||||
if ip == prefix_start {
|
||||
ip = ip.wrapping_add(1);
|
||||
}
|
||||
|
||||
while (ip as usize) < ilimit as usize {
|
||||
let mut last_pos;
|
||||
let mut found_immediate = false;
|
||||
let litlen = ptr_diff(ip, anchor) as u32;
|
||||
let ll0 = u32::from(litlen == 0);
|
||||
let mut nb_matches = unsafe {
|
||||
get_all_matches(
|
||||
state,
|
||||
&mut next_to_update3,
|
||||
ip,
|
||||
iend,
|
||||
&[*reps, *reps.add(1), *reps.add(2)],
|
||||
ll0,
|
||||
min_match,
|
||||
dict_mode,
|
||||
min_match,
|
||||
)
|
||||
};
|
||||
unsafe {
|
||||
ldm_process_match(
|
||||
&mut opt_ldm,
|
||||
matches,
|
||||
&mut nb_matches,
|
||||
ptr_diff(ip, istart) as u32,
|
||||
ptr_diff(iend, ip) as u32,
|
||||
min_match,
|
||||
)
|
||||
};
|
||||
if nb_matches == 0 {
|
||||
ip = ip.wrapping_add(1);
|
||||
continue;
|
||||
}
|
||||
|
||||
unsafe {
|
||||
(*opt).mlen = 0;
|
||||
(*opt).litlen = litlen;
|
||||
(*opt).price = lit_length_price(litlen, opt_state, opt_level) as c_int;
|
||||
(*opt).rep = [*reps, *reps.add(1), *reps.add(2)];
|
||||
}
|
||||
|
||||
let max_match = unsafe { *matches.add(nb_matches as usize - 1) };
|
||||
if max_match.len > sufficient_len {
|
||||
last_stretch.litlen = 0;
|
||||
last_stretch.mlen = max_match.len;
|
||||
last_stretch.off = max_match.off;
|
||||
last_pos = max_match.len;
|
||||
found_immediate = true;
|
||||
} else {
|
||||
let mut pos = 1u32;
|
||||
while pos < min_match {
|
||||
unsafe {
|
||||
(*opt.add(pos as usize)).price = ZSTD_MAX_PRICE;
|
||||
(*opt.add(pos as usize)).mlen = 0;
|
||||
(*opt.add(pos as usize)).litlen = litlen + pos;
|
||||
}
|
||||
pos += 1;
|
||||
}
|
||||
for match_index in 0..nb_matches as usize {
|
||||
let candidate = unsafe { *matches.add(match_index) };
|
||||
while pos <= candidate.len {
|
||||
let match_price =
|
||||
unsafe { get_match_price(candidate.off, pos, opt_state, opt_level) }
|
||||
as c_int;
|
||||
let sequence_price = unsafe { (*opt).price } + match_price;
|
||||
unsafe {
|
||||
(*opt.add(pos as usize)).mlen = pos;
|
||||
(*opt.add(pos as usize)).off = candidate.off;
|
||||
(*opt.add(pos as usize)).litlen = 0;
|
||||
(*opt.add(pos as usize)).price =
|
||||
sequence_price + lit_length_price(0, opt_state, opt_level) as c_int;
|
||||
}
|
||||
pos += 1;
|
||||
}
|
||||
}
|
||||
last_pos = pos - 1;
|
||||
unsafe { (*opt.add(pos as usize)).price = ZSTD_MAX_PRICE };
|
||||
}
|
||||
|
||||
if found_immediate {
|
||||
let mut next_rep = unsafe { [*reps, *reps.add(1), *reps.add(2)] };
|
||||
unsafe {
|
||||
resolve_shortest_path(
|
||||
state,
|
||||
seq_store,
|
||||
&mut next_rep,
|
||||
istart,
|
||||
iend,
|
||||
&mut anchor,
|
||||
&mut ip,
|
||||
0,
|
||||
last_pos,
|
||||
last_stretch,
|
||||
opt_level,
|
||||
)
|
||||
};
|
||||
unsafe {
|
||||
*reps = next_rep[0];
|
||||
*reps.add(1) = next_rep[1];
|
||||
*reps.add(2) = next_rep[2];
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut cur = 1u32;
|
||||
let mut resolved_early = false;
|
||||
while cur <= last_pos {
|
||||
let inr = ip.wrapping_add(cur as usize);
|
||||
let literal_length = unsafe { (*opt.add((cur - 1) as usize)).litlen } + 1;
|
||||
let price = unsafe { (*opt.add((cur - 1) as usize)).price }
|
||||
+ unsafe {
|
||||
raw_literals_cost(ip.wrapping_add((cur - 1) as usize), 1, opt_state, opt_level)
|
||||
} as c_int
|
||||
+ (unsafe { lit_length_price(literal_length, opt_state, opt_level) } as c_int
|
||||
- unsafe { lit_length_price(literal_length - 1, opt_state, opt_level) }
|
||||
as c_int);
|
||||
if price <= unsafe { (*opt.add(cur as usize)).price } {
|
||||
let previous_match = unsafe { *opt.add(cur as usize) };
|
||||
let previous = unsafe { *opt.add((cur - 1) as usize) };
|
||||
unsafe { *opt.add(cur as usize) = previous };
|
||||
unsafe {
|
||||
(*opt.add(cur as usize)).litlen = literal_length;
|
||||
(*opt.add(cur as usize)).price = price;
|
||||
}
|
||||
if opt_level >= 1
|
||||
&& previous_match.litlen == 0
|
||||
&& (unsafe { lit_length_price(1, opt_state, opt_level) } as c_int
|
||||
- unsafe { lit_length_price(0, opt_state, opt_level) } as c_int)
|
||||
< 0
|
||||
&& (inr as usize) < (iend as usize)
|
||||
{
|
||||
let with_one = previous_match.price
|
||||
+ unsafe { raw_literals_cost(inr, 1, opt_state, opt_level) } as c_int
|
||||
+ (unsafe { lit_length_price(1, opt_state, opt_level) } as c_int
|
||||
- unsafe { lit_length_price(0, opt_state, opt_level) } as c_int);
|
||||
let with_more = price
|
||||
+ unsafe { raw_literals_cost(inr, 1, opt_state, opt_level) } as c_int
|
||||
+ (unsafe { lit_length_price(literal_length + 1, opt_state, opt_level) }
|
||||
as c_int
|
||||
- unsafe { lit_length_price(literal_length, opt_state, opt_level) }
|
||||
as c_int);
|
||||
if with_one < with_more
|
||||
&& with_one < unsafe { (*opt.add((cur + 1) as usize)).price }
|
||||
{
|
||||
let previous_index = cur - previous_match.mlen;
|
||||
let reps = new_rep(
|
||||
unsafe { (*opt.add(previous_index as usize)).rep },
|
||||
previous_match.off,
|
||||
unsafe { (*opt.add(previous_index as usize)).litlen == 0 },
|
||||
);
|
||||
unsafe {
|
||||
(*opt.add((cur + 1) as usize)).rep = reps;
|
||||
(*opt.add((cur + 1) as usize)).mlen = previous_match.mlen;
|
||||
(*opt.add((cur + 1) as usize)).off = previous_match.off;
|
||||
(*opt.add((cur + 1) as usize)).litlen = 1;
|
||||
(*opt.add((cur + 1) as usize)).price = with_one;
|
||||
}
|
||||
if last_pos < cur + 1 {
|
||||
last_pos = cur + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let selected = unsafe { *opt.add(cur as usize) };
|
||||
if selected.litlen == 0 {
|
||||
let previous_index = cur - selected.mlen;
|
||||
let reps = new_rep(
|
||||
unsafe { (*opt.add(previous_index as usize)).rep },
|
||||
selected.off,
|
||||
unsafe { (*opt.add(previous_index as usize)).litlen == 0 },
|
||||
);
|
||||
unsafe { (*opt.add(cur as usize)).rep = reps };
|
||||
}
|
||||
if (inr as usize) > (ilimit as usize) {
|
||||
cur += 1;
|
||||
continue;
|
||||
}
|
||||
if cur == last_pos {
|
||||
break;
|
||||
}
|
||||
if opt_level == 0
|
||||
&& unsafe { (*opt.add((cur + 1) as usize)).price }
|
||||
<= unsafe { (*opt.add(cur as usize)).price } + (BITCOST_MULTIPLIER / 2) as c_int
|
||||
{
|
||||
cur += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let previous_price = unsafe { (*opt.add(cur as usize)).price };
|
||||
let base_price =
|
||||
previous_price + unsafe { lit_length_price(0, opt_state, opt_level) } as c_int;
|
||||
let current_rep = unsafe { (*opt.add(cur as usize)).rep };
|
||||
let mut candidates = unsafe {
|
||||
get_all_matches(
|
||||
state,
|
||||
&mut next_to_update3,
|
||||
inr,
|
||||
iend,
|
||||
¤t_rep,
|
||||
u32::from(selected.litlen == 0),
|
||||
min_match,
|
||||
dict_mode,
|
||||
min_match,
|
||||
)
|
||||
};
|
||||
unsafe {
|
||||
ldm_process_match(
|
||||
&mut opt_ldm,
|
||||
matches,
|
||||
&mut candidates,
|
||||
ptr_diff(inr, istart) as u32,
|
||||
ptr_diff(iend, inr) as u32,
|
||||
min_match,
|
||||
)
|
||||
};
|
||||
if candidates == 0 {
|
||||
cur += 1;
|
||||
continue;
|
||||
}
|
||||
let longest = unsafe { (*matches.add(candidates as usize - 1)).len };
|
||||
if longest > sufficient_len
|
||||
|| cur + longest >= ZSTD_OPT_NUM
|
||||
|| ptr_diff(iend, inr) <= longest as usize
|
||||
{
|
||||
last_stretch.mlen = longest;
|
||||
last_stretch.off = unsafe { (*matches.add(candidates as usize - 1)).off };
|
||||
last_stretch.litlen = 0;
|
||||
last_pos = cur + longest;
|
||||
let mut path_rep = unsafe { [*reps, *reps.add(1), *reps.add(2)] };
|
||||
unsafe {
|
||||
resolve_shortest_path(
|
||||
state,
|
||||
seq_store,
|
||||
&mut path_rep,
|
||||
istart,
|
||||
iend,
|
||||
&mut anchor,
|
||||
&mut ip,
|
||||
cur,
|
||||
last_pos,
|
||||
last_stretch,
|
||||
opt_level,
|
||||
)
|
||||
};
|
||||
unsafe {
|
||||
*reps = path_rep[0];
|
||||
*reps.add(1) = path_rep[1];
|
||||
*reps.add(2) = path_rep[2];
|
||||
}
|
||||
resolved_early = true;
|
||||
cur = last_pos;
|
||||
break;
|
||||
}
|
||||
|
||||
for match_index in 0..candidates as usize {
|
||||
let candidate = unsafe { *matches.add(match_index) };
|
||||
let start_ml = if match_index > 0 {
|
||||
unsafe { (*matches.add(match_index - 1)).len + 1 }
|
||||
} else {
|
||||
min_match
|
||||
};
|
||||
let mut match_length = candidate.len;
|
||||
loop {
|
||||
let position = cur + match_length;
|
||||
let match_price = unsafe {
|
||||
get_match_price(candidate.off, match_length, opt_state, opt_level)
|
||||
} as c_int;
|
||||
let candidate_price = base_price + match_price;
|
||||
if position > last_pos
|
||||
|| candidate_price < unsafe { (*opt.add(position as usize)).price }
|
||||
{
|
||||
while last_pos < position {
|
||||
last_pos += 1;
|
||||
unsafe {
|
||||
(*opt.add(last_pos as usize)).price = ZSTD_MAX_PRICE;
|
||||
(*opt.add(last_pos as usize)).litlen = 1;
|
||||
}
|
||||
}
|
||||
unsafe {
|
||||
(*opt.add(position as usize)).mlen = match_length;
|
||||
(*opt.add(position as usize)).off = candidate.off;
|
||||
(*opt.add(position as usize)).litlen = 0;
|
||||
(*opt.add(position as usize)).price = candidate_price;
|
||||
}
|
||||
} else if opt_level == 0 {
|
||||
break;
|
||||
}
|
||||
if match_length == start_ml {
|
||||
break;
|
||||
}
|
||||
match_length -= 1;
|
||||
}
|
||||
}
|
||||
unsafe { (*opt.add((last_pos + 1) as usize)).price = ZSTD_MAX_PRICE };
|
||||
cur += 1;
|
||||
}
|
||||
|
||||
if cur == last_pos && !resolved_early {
|
||||
/* The previous loop reached its end without an early path. */
|
||||
last_stretch = unsafe { *opt.add(last_pos as usize) };
|
||||
let path_rep = unsafe { [*reps, *reps.add(1), *reps.add(2)] };
|
||||
let mut next_rep = path_rep;
|
||||
let path_cur = last_pos.wrapping_sub(last_stretch.mlen);
|
||||
unsafe {
|
||||
resolve_shortest_path(
|
||||
state,
|
||||
seq_store,
|
||||
&mut next_rep,
|
||||
istart,
|
||||
iend,
|
||||
&mut anchor,
|
||||
&mut ip,
|
||||
path_cur,
|
||||
last_pos,
|
||||
last_stretch,
|
||||
opt_level,
|
||||
)
|
||||
};
|
||||
unsafe {
|
||||
*reps = next_rep[0];
|
||||
*reps.add(1) = next_rep[1];
|
||||
*reps.add(2) = next_rep[2];
|
||||
}
|
||||
}
|
||||
}
|
||||
ptr_diff(iend, anchor)
|
||||
}
|
||||
|
||||
unsafe fn init_stats_ultra(
|
||||
state: &mut ZSTD_RustOptState,
|
||||
seq_store: *mut SeqStore_t,
|
||||
reps: *const u32,
|
||||
src: *const u8,
|
||||
src_size: usize,
|
||||
) {
|
||||
let mut temporary_rep = [unsafe { *reps }, unsafe { *reps.add(1) }, unsafe {
|
||||
*reps.add(2)
|
||||
}];
|
||||
unsafe {
|
||||
compress_block_opt_generic(
|
||||
state,
|
||||
seq_store,
|
||||
temporary_rep.as_mut_ptr(),
|
||||
src,
|
||||
src_size,
|
||||
2,
|
||||
DICT_NO_DICT,
|
||||
);
|
||||
}
|
||||
let store = unsafe { &mut *seq_store };
|
||||
store.sequences = store.sequencesStart;
|
||||
store.lit = store.litStart;
|
||||
store.longLengthType = 0;
|
||||
unsafe {
|
||||
*state.window_base = (*state.window_base).wrapping_sub(src_size);
|
||||
*state.window_dict_limit = (*state.window_dict_limit).wrapping_add(src_size as u32);
|
||||
*state.window_low_limit = *state.window_dict_limit;
|
||||
*state.next_to_update = *state.window_dict_limit;
|
||||
}
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_opt_updateTree(
|
||||
state: *mut ZSTD_RustOptState,
|
||||
ip: *const c_void,
|
||||
iend: *const c_void,
|
||||
mls: u32,
|
||||
dict_mode: c_int,
|
||||
) {
|
||||
let state = unsafe { &mut *state };
|
||||
unsafe {
|
||||
ZSTD_rust_opt_updateTreeInternal(
|
||||
(state as *mut ZSTD_RustOptState).cast::<OptTreePrefix>(),
|
||||
ip,
|
||||
iend,
|
||||
mls,
|
||||
c_int::from(dict_mode == DICT_EXT),
|
||||
)
|
||||
};
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_compressBlock_opt(
|
||||
state: *mut ZSTD_RustOptState,
|
||||
seq_store: *mut c_void,
|
||||
reps: *mut u32,
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
opt_level: c_int,
|
||||
dict_mode: c_int,
|
||||
) -> usize {
|
||||
let state = unsafe { &mut *state };
|
||||
unsafe {
|
||||
compress_block_opt_generic(
|
||||
state,
|
||||
seq_store.cast::<SeqStore_t>(),
|
||||
reps,
|
||||
src.cast::<u8>(),
|
||||
src_size,
|
||||
opt_level,
|
||||
dict_mode,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_initStats_ultra(
|
||||
state: *mut ZSTD_RustOptState,
|
||||
seq_store: *mut c_void,
|
||||
reps: *const u32,
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
) {
|
||||
let state = unsafe { &mut *state };
|
||||
unsafe {
|
||||
init_stats_ultra(
|
||||
state,
|
||||
seq_store.cast::<SeqStore_t>(),
|
||||
reps,
|
||||
src.cast::<u8>(),
|
||||
src_size,
|
||||
)
|
||||
};
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn ZSTD_rust_compressBlock_btultra2(
|
||||
state: *mut ZSTD_RustOptState,
|
||||
seq_store: *mut c_void,
|
||||
reps: *mut u32,
|
||||
src: *const c_void,
|
||||
src_size: usize,
|
||||
) -> usize {
|
||||
let state = unsafe { &mut *state };
|
||||
let store = unsafe { &*seq_store.cast::<SeqStore_t>() };
|
||||
let current = index_from(state.base, src.cast::<u8>());
|
||||
let should_seed = unsafe { (*state.opt).litLengthSum == 0 }
|
||||
&& store.sequences == store.sequencesStart
|
||||
&& unsafe { *state.window_dict_limit == *state.window_low_limit }
|
||||
&& current == unsafe { *state.window_dict_limit }
|
||||
&& src_size > ZSTD_PREDEF_THRESHOLD;
|
||||
if should_seed {
|
||||
unsafe { init_stats_ultra(state, seq_store.cast(), reps, src.cast(), src_size) };
|
||||
}
|
||||
unsafe {
|
||||
compress_block_opt_generic(
|
||||
state,
|
||||
seq_store.cast::<SeqStore_t>(),
|
||||
reps,
|
||||
src.cast::<u8>(),
|
||||
src_size,
|
||||
2,
|
||||
DICT_NO_DICT,
|
||||
)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user