diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 9b636445a..33576d8a7 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -5,19 +5,20 @@ * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. + * You may select, at your option, one or both of these licenses. */ +/* The optimal parser is implemented in rust/src/zstd_opt.rs. Keep the + * private ZSTD_MatchState_t and entropy-table layouts on the C side: this + * translation unit only projects the leaves Rust needs and preserves the + * established C entry points. */ #include "zstd_compress_internal.h" -#include "hist.h" #include "zstd_opt.h" #if !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \ || !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \ || !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR) -/* The Rust tree updater receives this leaf view instead of the private, - * configuration-dependent ZSTD_MatchState_t layout. */ typedef struct { U32* hashTable; U32* chainTable; @@ -32,1461 +33,211 @@ typedef struct { U32 searchLog; U32 windowLog; int useCPredict; -} ZSTD_RustOptTreeState; + U32* hashTable3; + U32 hashLog3; + const BYTE* nextSrc; + U32 minMatch; + U32 targetLength; + optState_t* opt; + const void* dictMatchState; + const RawSeqStore_t* ldmSeqStore; + const BYTE** windowBase; + U32* windowDictLimit; + U32* windowLowLimit; + const size_t* hufCTable; + int hufRepeatValid; + const U32* fseLitLengthCTable; + const U32* fseMatchLengthCTable; + const U32* fseOffCodeCTable; +} ZSTD_RustOptState; -static ZSTD_RustOptTreeState ZSTD_rustOptTreeState( +typedef char ZSTD_rust_opt_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1]; +typedef char ZSTD_rust_opt_match_layout[(sizeof(ZSTD_match_t) == 8) ? 1 : -1]; +typedef char ZSTD_rust_opt_rawseq_layout[(sizeof(rawSeq) == 12) ? 1 : -1]; +typedef char ZSTD_rust_opt_seqstore_layout[ + (offsetof(SeqStore_t, longLengthPos) == 9 * sizeof(size_t) + 4) ? 1 : -1]; +typedef char ZSTD_rust_optimal_layout[ + (sizeof(ZSTD_optimal_t) == 4 * sizeof(U32) + sizeof(U32) * ZSTD_REP_NUM) ? 1 : -1]; +typedef char ZSTD_rust_opt_rep_count[(ZSTD_REP_NUM == 3) ? 1 : -1]; + +static void ZSTD_rustOptState_init( + ZSTD_RustOptState* const out, + ZSTD_MatchState_t* const ms, + const ZSTD_RustOptState* const dms) +{ + out->hashTable = ms->hashTable; + out->chainTable = ms->chainTable; + out->base = ms->window.base; + out->dictBase = ms->window.dictBase; + out->dictLimit = ms->window.dictLimit; + out->lowLimit = ms->window.lowLimit; + out->loadedDictEnd = ms->loadedDictEnd; + out->nextToUpdate = &ms->nextToUpdate; + out->hashLog = ms->cParams.hashLog; + out->chainLog = ms->cParams.chainLog; + out->searchLog = ms->cParams.searchLog; +#ifdef ZSTD_C_PREDICT + out->useCPredict = 1; +#else + out->useCPredict = 0; +#endif + out->windowLog = ms->cParams.windowLog; + out->hashTable3 = ms->hashTable3; + out->hashLog3 = ms->hashLog3; + out->nextSrc = ms->window.nextSrc; + out->minMatch = ms->cParams.minMatch; + out->targetLength = ms->cParams.targetLength; + out->opt = &ms->opt; + out->dictMatchState = dms; + out->ldmSeqStore = ms->ldmSeqStore; + out->windowBase = &ms->window.base; + out->windowDictLimit = &ms->window.dictLimit; + out->windowLowLimit = &ms->window.lowLimit; + out->hufCTable = NULL; + out->hufRepeatValid = 0; + out->fseLitLengthCTable = NULL; + out->fseMatchLengthCTable = NULL; + out->fseOffCodeCTable = NULL; +} + +static void ZSTD_rustOptState_projectCosts( + ZSTD_RustOptState* const out, ZSTD_MatchState_t* const ms) { - ZSTD_RustOptTreeState state; - state.hashTable = ms->hashTable; - state.chainTable = ms->chainTable; - state.base = ms->window.base; - state.dictBase = ms->window.dictBase; - state.dictLimit = ms->window.dictLimit; - state.lowLimit = ms->window.lowLimit; - state.loadedDictEnd = ms->loadedDictEnd; - state.nextToUpdate = &ms->nextToUpdate; - state.hashLog = ms->cParams.hashLog; - state.chainLog = ms->cParams.chainLog; - state.searchLog = ms->cParams.searchLog; - state.windowLog = ms->cParams.windowLog; -#ifdef ZSTD_C_PREDICT - state.useCPredict = 1; -#else - state.useCPredict = 0; -#endif - return state; + const ZSTD_entropyCTables_t* const costs = ms->opt.symbolCosts; + out->hufCTable = costs ? costs->huf.CTable : NULL; + out->hufRepeatValid = costs && costs->huf.repeatMode == HUF_repeat_valid; + out->fseLitLengthCTable = costs ? costs->fse.litlengthCTable : NULL; + out->fseMatchLengthCTable = costs ? costs->fse.matchlengthCTable : NULL; + out->fseOffCodeCTable = costs ? costs->fse.offcodeCTable : NULL; } -void ZSTD_rust_opt_updateTreeInternal( - ZSTD_RustOptTreeState* state, +static void ZSTD_rustOptState_init_with_dict( + ZSTD_RustOptState* const out, + ZSTD_MatchState_t* const ms) +{ + ZSTD_rustOptState_init(out, ms, NULL); +} + +static void ZSTD_rustOptState_init_with_costs( + ZSTD_RustOptState* const out, + ZSTD_MatchState_t* const ms, + const ZSTD_RustOptState* const dms) +{ + ZSTD_rustOptState_init(out, ms, dms); + ZSTD_rustOptState_projectCosts(out, ms); +} + +void ZSTD_rust_opt_updateTree( + ZSTD_RustOptState* state, const void* ip, const void* iend, - U32 mls, int extDict); + U32 mls, int dictMode); -#define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */ -#define ZSTD_MAX_PRICE (1<<30) +size_t ZSTD_rust_compressBlock_opt( + ZSTD_RustOptState* state, void* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, int optLevel, int dictMode); -#define ZSTD_PREDEF_THRESHOLD 8 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ +void ZSTD_rust_initStats_ultra( + ZSTD_RustOptState* state, void* seqStore, const U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize); +size_t ZSTD_rust_compressBlock_btultra2( + ZSTD_RustOptState* state, void* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize); -/*-************************************* -* Price functions for optimal parser -***************************************/ +enum { + ZSTD_rust_dict_noDict = 0, + ZSTD_rust_dict_extDict = 1, + ZSTD_rust_dict_dictMatchState = 2, +}; -#if 0 /* approximation at bit level (for tests) */ -# define BITCOST_ACCURACY 0 -# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) -# define WEIGHT(stat, opt) ((void)(opt), ZSTD_bitWeight(stat)) -#elif 0 /* fractional bit accuracy (for tests) */ -# define BITCOST_ACCURACY 8 -# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) -# define WEIGHT(stat,opt) ((void)(opt), ZSTD_fracWeight(stat)) -#else /* opt==approx, ultra==accurate */ -# define BITCOST_ACCURACY 8 -# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) -# define WEIGHT(stat,opt) ((opt) ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat)) -#endif - -/* ZSTD_bitWeight() : - * provide estimated "cost" of a stat in full bits only */ -MEM_STATIC U32 ZSTD_bitWeight(U32 stat) +void ZSTD_updateTree(ZSTD_MatchState_t* const ms, const BYTE* const ip, + const BYTE* const iend) { - return (ZSTD_highbit32(stat+1) * BITCOST_MULTIPLIER); + ZSTD_RustOptState state; + ZSTD_rustOptState_init_with_dict(&state, ms); + ZSTD_rust_opt_updateTree(&state, ip, iend, ms->cParams.minMatch, + ZSTD_rust_dict_noDict); } -/* ZSTD_fracWeight() : - * provide fractional-bit "cost" of a stat, - * using linear interpolation approximation */ -MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat) -{ - U32 const stat = rawStat + 1; - U32 const hb = ZSTD_highbit32(stat); - U32 const BWeight = hb * BITCOST_MULTIPLIER; - /* Fweight was meant for "Fractional weight" - * but it's effectively a value between 1 and 2 - * using fixed point arithmetic */ - U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb; - U32 const weight = BWeight + FWeight; - assert(hb + BITCOST_ACCURACY < 31); - return weight; -} - -#if (DEBUGLEVEL>=2) -/* debugging function, - * @return price in bytes as fractional value - * for debug messages only */ -MEM_STATIC double ZSTD_fCost(int price) -{ - return (double)price / (BITCOST_MULTIPLIER*8); -} -#endif - -static int ZSTD_compressedLiterals(optState_t const* const optPtr) -{ - return optPtr->literalCompressionMode != ZSTD_ps_disable; -} - -static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) -{ - if (ZSTD_compressedLiterals(optPtr)) - optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel); - optPtr->litLengthSumBasePrice = WEIGHT(optPtr->litLengthSum, optLevel); - optPtr->matchLengthSumBasePrice = WEIGHT(optPtr->matchLengthSum, optLevel); - optPtr->offCodeSumBasePrice = WEIGHT(optPtr->offCodeSum, optLevel); -} - - -static U32 sum_u32(const unsigned table[], size_t nbElts) -{ - size_t n; - U32 total = 0; - for (n=0; n0); - unsigned const newStat = base + (table[s] >> shift); - sum += newStat; - table[s] = newStat; - } - return sum; -} - -/* ZSTD_scaleStats() : - * reduce all elt frequencies in table if sum too large - * return the resulting sum of elements */ -static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget) -{ - U32 const prevsum = sum_u32(table, lastEltIndex+1); - U32 const factor = prevsum >> logTarget; - DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget); - assert(logTarget < 30); - if (factor <= 1) return prevsum; - return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor), base_1guaranteed); -} - -/* ZSTD_rescaleFreqs() : - * if first block (detected by optPtr->litLengthSum == 0) : init statistics - * take hints from dictionary if there is one - * and init from zero if there is none, - * using src for literals stats, and baseline stats for sequence symbols - * otherwise downscale existing stats, to be used as seed for next block. - */ -static void -ZSTD_rescaleFreqs(optState_t* const optPtr, - const BYTE* const src, size_t const srcSize, - int const optLevel) -{ - int const compressedLiterals = ZSTD_compressedLiterals(optPtr); - DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize); - optPtr->priceType = zop_dynamic; - - if (optPtr->litLengthSum == 0) { /* no literals stats collected -> first block assumed -> init */ - - /* heuristic: use pre-defined stats for too small inputs */ - if (srcSize <= ZSTD_PREDEF_THRESHOLD) { - DEBUGLOG(5, "srcSize <= %i : use predefined stats", ZSTD_PREDEF_THRESHOLD); - optPtr->priceType = zop_predef; - } - - assert(optPtr->symbolCosts != NULL); - if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { - - /* huffman stats covering the full value set : table presumed generated by dictionary */ - optPtr->priceType = zop_dynamic; - - if (compressedLiterals) { - /* generate literals statistics from huffman table */ - unsigned lit; - assert(optPtr->litFreq != NULL); - optPtr->litSum = 0; - for (lit=0; lit<=MaxLit; lit++) { - U32 const scaleLog = 11; /* scale to 2K */ - U32 const bitCost = HUF_getNbBitsFromCTable(optPtr->symbolCosts->huf.CTable, lit); - assert(bitCost <= scaleLog); - optPtr->litFreq[lit] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; - optPtr->litSum += optPtr->litFreq[lit]; - } } - - { unsigned ll; - FSE_CState_t llstate; - FSE_initCState(&llstate, optPtr->symbolCosts->fse.litlengthCTable); - optPtr->litLengthSum = 0; - for (ll=0; ll<=MaxLL; ll++) { - U32 const scaleLog = 10; /* scale to 1K */ - U32 const bitCost = FSE_getMaxNbBits(llstate.symbolTT, ll); - assert(bitCost < scaleLog); - optPtr->litLengthFreq[ll] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; - optPtr->litLengthSum += optPtr->litLengthFreq[ll]; - } } - - { unsigned ml; - FSE_CState_t mlstate; - FSE_initCState(&mlstate, optPtr->symbolCosts->fse.matchlengthCTable); - optPtr->matchLengthSum = 0; - for (ml=0; ml<=MaxML; ml++) { - U32 const scaleLog = 10; - U32 const bitCost = FSE_getMaxNbBits(mlstate.symbolTT, ml); - assert(bitCost < scaleLog); - optPtr->matchLengthFreq[ml] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; - optPtr->matchLengthSum += optPtr->matchLengthFreq[ml]; - } } - - { unsigned of; - FSE_CState_t ofstate; - FSE_initCState(&ofstate, optPtr->symbolCosts->fse.offcodeCTable); - optPtr->offCodeSum = 0; - for (of=0; of<=MaxOff; of++) { - U32 const scaleLog = 10; - U32 const bitCost = FSE_getMaxNbBits(ofstate.symbolTT, of); - assert(bitCost < scaleLog); - optPtr->offCodeFreq[of] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; - optPtr->offCodeSum += optPtr->offCodeFreq[of]; - } } - - } else { /* first block, no dictionary */ - - assert(optPtr->litFreq != NULL); - if (compressedLiterals) { - /* base initial cost of literals on direct frequency within src */ - unsigned lit = MaxLit; - HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */ - optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8, base_0possible); - } - - { unsigned const baseLLfreqs[MaxLL+1] = { - 4, 2, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1 - }; - ZSTD_memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs)); - optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1); - } - - { unsigned ml; - for (ml=0; ml<=MaxML; ml++) - optPtr->matchLengthFreq[ml] = 1; - } - optPtr->matchLengthSum = MaxML+1; - - { unsigned const baseOFCfreqs[MaxOff+1] = { - 6, 2, 1, 1, 2, 3, 4, 4, - 4, 3, 2, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1 - }; - ZSTD_memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs)); - optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1); - } - - } - - } else { /* new block : scale down accumulated statistics */ - - if (compressedLiterals) - optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12); - optPtr->litLengthSum = ZSTD_scaleStats(optPtr->litLengthFreq, MaxLL, 11); - optPtr->matchLengthSum = ZSTD_scaleStats(optPtr->matchLengthFreq, MaxML, 11); - optPtr->offCodeSum = ZSTD_scaleStats(optPtr->offCodeFreq, MaxOff, 11); - } - - ZSTD_setBasePrices(optPtr, optLevel); -} - -/* ZSTD_rawLiteralsCost() : - * price of literals (only) in specified segment (which length can be 0). - * does not include price of literalLength symbol */ -static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength, - const optState_t* const optPtr, - int optLevel) -{ - DEBUGLOG(8, "ZSTD_rawLiteralsCost (%u literals)", litLength); - if (litLength == 0) return 0; - - if (!ZSTD_compressedLiterals(optPtr)) - return (litLength << 3) * BITCOST_MULTIPLIER; /* Uncompressed - 8 bytes per literal. */ - - if (optPtr->priceType == zop_predef) - return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */ - - /* dynamic statistics */ - { U32 price = optPtr->litSumBasePrice * litLength; - U32 const litPriceMax = optPtr->litSumBasePrice - BITCOST_MULTIPLIER; - U32 u; - assert(optPtr->litSumBasePrice >= BITCOST_MULTIPLIER); - for (u=0; u < litLength; u++) { - U32 litPrice = WEIGHT(optPtr->litFreq[literals[u]], optLevel); - if (UNLIKELY(litPrice > litPriceMax)) litPrice = litPriceMax; - price -= litPrice; - } - return price; - } -} - -/* ZSTD_litLengthPrice() : - * cost of literalLength symbol */ -static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr, int optLevel) -{ - assert(litLength <= ZSTD_BLOCKSIZE_MAX); - if (optPtr->priceType == zop_predef) - return WEIGHT(litLength, optLevel); - - /* ZSTD_LLcode() can't compute litLength price for sizes >= ZSTD_BLOCKSIZE_MAX - * because it isn't representable in the zstd format. - * So instead just pretend it would cost 1 bit more than ZSTD_BLOCKSIZE_MAX - 1. - * In such a case, the block would be all literals. - */ - if (litLength == ZSTD_BLOCKSIZE_MAX) - return BITCOST_MULTIPLIER + ZSTD_litLengthPrice(ZSTD_BLOCKSIZE_MAX - 1, optPtr, optLevel); - - /* dynamic statistics */ - { U32 const llCode = ZSTD_LLcode(litLength); - return (LL_bits[llCode] * BITCOST_MULTIPLIER) - + optPtr->litLengthSumBasePrice - - WEIGHT(optPtr->litLengthFreq[llCode], optLevel); - } -} - -/* ZSTD_getMatchPrice() : - * Provides the cost of the match part (offset + matchLength) of a sequence. - * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence. - * @offBase : sumtype, representing an offset or a repcode, and using numeric representation of ZSTD_storeSeq() - * @optLevel: when <2, favors small offset for decompression speed (improved cache efficiency) - */ -FORCE_INLINE_TEMPLATE U32 -ZSTD_getMatchPrice(U32 const offBase, - U32 const matchLength, - const optState_t* const optPtr, - int const optLevel) -{ - U32 price; - U32 const offCode = ZSTD_highbit32(offBase); - U32 const mlBase = matchLength - MINMATCH; - assert(matchLength >= MINMATCH); - - if (optPtr->priceType == zop_predef) /* fixed scheme, does not use statistics */ - return WEIGHT(mlBase, optLevel) - + ((16 + offCode) * BITCOST_MULTIPLIER); /* emulated offset cost */ - - /* dynamic statistics */ - price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel)); - if ((optLevel<2) /*static*/ && offCode >= 20) - price += (offCode-19)*2 * BITCOST_MULTIPLIER; /* handicap for long distance offsets, favor decompression speed */ - - /* match Length */ - { U32 const mlCode = ZSTD_MLcode(mlBase); - price += (ML_bits[mlCode] * BITCOST_MULTIPLIER) + (optPtr->matchLengthSumBasePrice - WEIGHT(optPtr->matchLengthFreq[mlCode], optLevel)); - } - - price += BITCOST_MULTIPLIER / 5; /* heuristic : make matches a bit more costly to favor less sequences -> faster decompression speed */ - - DEBUGLOG(8, "ZSTD_getMatchPrice(ml:%u) = %u", matchLength, price); - return price; -} - -/* ZSTD_updateStats() : - * assumption : literals + litLength <= iend */ -static void ZSTD_updateStats(optState_t* const optPtr, - U32 litLength, const BYTE* literals, - U32 offBase, U32 matchLength) -{ - /* literals */ - if (ZSTD_compressedLiterals(optPtr)) { - U32 u; - for (u=0; u < litLength; u++) - optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; - optPtr->litSum += litLength*ZSTD_LITFREQ_ADD; - } - - /* literal Length */ - { U32 const llCode = ZSTD_LLcode(litLength); - optPtr->litLengthFreq[llCode]++; - optPtr->litLengthSum++; - } - - /* offset code : follows storeSeq() numeric representation */ - { U32 const offCode = ZSTD_highbit32(offBase); - assert(offCode <= MaxOff); - optPtr->offCodeFreq[offCode]++; - optPtr->offCodeSum++; - } - - /* match Length */ - { U32 const mlBase = matchLength - MINMATCH; - U32 const mlCode = ZSTD_MLcode(mlBase); - optPtr->matchLengthFreq[mlCode]++; - optPtr->matchLengthSum++; - } -} - - -/* ZSTD_readMINMATCH() : - * function safe only for comparisons - * assumption : memPtr must be at least 4 bytes before end of buffer */ -MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length) -{ - switch (length) - { - default : - case 4 : return MEM_read32(memPtr); - case 3 : if (MEM_isLittleEndian()) - return MEM_read32(memPtr)<<8; - else - return MEM_read32(memPtr)>>8; - } -} - - -/* Update hashTable3 up to ip (excluded) - Assumption : always within prefix (i.e. not within extDict) */ -static -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -U32 ZSTD_insertAndFindFirstIndexHash3 (const ZSTD_MatchState_t* ms, - U32* nextToUpdate3, - const BYTE* const ip) -{ - U32* const hashTable3 = ms->hashTable3; - U32 const hashLog3 = ms->hashLog3; - const BYTE* const base = ms->window.base; - U32 idx = *nextToUpdate3; - U32 const target = (U32)(ip - base); - size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3); - assert(hashLog3 > 0); - - while(idx < target) { - hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; - idx++; - } - - *nextToUpdate3 = target; - return hashTable3[hash3]; -} - - -/*-************************************* -* Binary Tree search -***************************************/ -FORCE_INLINE_TEMPLATE -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -void ZSTD_updateTree_internal( - ZSTD_MatchState_t* ms, - const BYTE* const ip, const BYTE* const iend, - const U32 mls, const ZSTD_dictMode_e dictMode) -{ - ZSTD_RustOptTreeState state = ZSTD_rustOptTreeState(ms); - ZSTD_rust_opt_updateTreeInternal(&state, ip, iend, mls, - dictMode == ZSTD_extDict); -} - -void ZSTD_updateTree(ZSTD_MatchState_t* ms, const BYTE* ip, const BYTE* iend) { - ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict); -} - -FORCE_INLINE_TEMPLATE -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -U32 -ZSTD_insertBtAndGetAllMatches ( - ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */ - ZSTD_MatchState_t* ms, - U32* nextToUpdate3, - const BYTE* const ip, const BYTE* const iLimit, - const ZSTD_dictMode_e dictMode, - const U32 rep[ZSTD_REP_NUM], - const U32 ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ - const U32 lengthToBeat, - const U32 mls /* template */) -{ - const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); - const BYTE* const base = ms->window.base; - U32 const curr = (U32)(ip-base); - U32 const hashLog = cParams->hashLog; - U32 const minMatch = (mls==3) ? 3 : 4; - U32* const hashTable = ms->hashTable; - size_t const h = ZSTD_hashPtr(ip, hashLog, mls); - U32 matchIndex = hashTable[h]; - U32* const bt = ms->chainTable; - U32 const btLog = cParams->chainLog - 1; - U32 const btMask= (1U << btLog) - 1; - size_t commonLengthSmaller=0, commonLengthLarger=0; - const BYTE* const dictBase = ms->window.dictBase; - U32 const dictLimit = ms->window.dictLimit; - const BYTE* const dictEnd = dictBase + dictLimit; - const BYTE* const prefixStart = base + dictLimit; - U32 const btLow = (btMask >= curr) ? 0 : curr - btMask; - U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog); - U32 const matchLow = windowLow ? windowLow : 1; - U32* smallerPtr = bt + 2*(curr&btMask); - U32* largerPtr = bt + 2*(curr&btMask) + 1; - U32 matchEndIdx = curr+8+1; /* farthest referenced position of any match => detects repetitive patterns */ - U32 dummy32; /* to be nullified at the end */ - U32 mnum = 0; - U32 nbCompares = 1U << cParams->searchLog; - - const ZSTD_MatchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL; - const ZSTD_compressionParameters* const dmsCParams = - dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL; - const BYTE* const dmsBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL; - const BYTE* const dmsEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL; - U32 const dmsHighLimit = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0; - U32 const dmsLowLimit = dictMode == ZSTD_dictMatchState ? dms->window.lowLimit : 0; - U32 const dmsIndexDelta = dictMode == ZSTD_dictMatchState ? windowLow - dmsHighLimit : 0; - U32 const dmsHashLog = dictMode == ZSTD_dictMatchState ? dmsCParams->hashLog : hashLog; - U32 const dmsBtLog = dictMode == ZSTD_dictMatchState ? dmsCParams->chainLog - 1 : btLog; - U32 const dmsBtMask = dictMode == ZSTD_dictMatchState ? (1U << dmsBtLog) - 1 : 0; - U32 const dmsBtLow = dictMode == ZSTD_dictMatchState && dmsBtMask < dmsHighLimit - dmsLowLimit ? dmsHighLimit - dmsBtMask : dmsLowLimit; - - size_t bestLength = lengthToBeat-1; - DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", curr); - - /* check repCode */ - assert(ll0 <= 1); /* necessarily 1 or 0 */ - { U32 const lastR = ZSTD_REP_NUM + ll0; - U32 repCode; - for (repCode = ll0; repCode < lastR; repCode++) { - U32 const repOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; - U32 const repIndex = curr - repOffset; - U32 repLen = 0; - assert(curr >= dictLimit); - if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < curr-dictLimit) { /* equivalent to `curr > repIndex >= dictLimit` */ - /* We must validate the repcode offset because when we're using a dictionary the - * valid offset range shrinks when the dictionary goes out of bounds. - */ - if ((repIndex >= windowLow) & (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch))) { - repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch; - } - } else { /* repIndex < dictLimit || repIndex >= curr */ - const BYTE* const repMatch = dictMode == ZSTD_dictMatchState ? - dmsBase + repIndex - dmsIndexDelta : - dictBase + repIndex; - assert(curr >= windowLow); - if ( dictMode == ZSTD_extDict - && ( ((repOffset-1) /*intentional overflow*/ < curr - windowLow) /* equivalent to `curr > repIndex >= windowLow` */ - & (ZSTD_index_overlap_check(dictLimit, repIndex)) ) - && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { - repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch; - } - if (dictMode == ZSTD_dictMatchState - && ( ((repOffset-1) /*intentional overflow*/ < curr - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `curr > repIndex >= dmsLowLimit` */ - & (ZSTD_index_overlap_check(dictLimit, repIndex)) ) - && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { - repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch; - } } - /* save longer solution */ - if (repLen > bestLength) { - DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u", - repCode, ll0, repOffset, repLen); - bestLength = repLen; - matches[mnum].off = REPCODE_TO_OFFBASE(repCode - ll0 + 1); /* expect value between 1 and 3 */ - matches[mnum].len = (U32)repLen; - mnum++; - if ( (repLen > sufficient_len) - | (ip+repLen == iLimit) ) { /* best possible */ - return mnum; - } } } } - - /* HC3 match finder */ - if ((mls == 3) /*static*/ && (bestLength < mls)) { - U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, nextToUpdate3, ip); - if ((matchIndex3 >= matchLow) - & (curr - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { - size_t mlen; - if ((dictMode == ZSTD_noDict) /*static*/ || (dictMode == ZSTD_dictMatchState) /*static*/ || (matchIndex3 >= dictLimit)) { - const BYTE* const match = base + matchIndex3; - mlen = ZSTD_count(ip, match, iLimit); - } else { - const BYTE* const match = dictBase + matchIndex3; - mlen = ZSTD_count_2segments(ip, match, iLimit, dictEnd, prefixStart); - } - - /* save best solution */ - if (mlen >= mls /* == 3 > bestLength */) { - DEBUGLOG(8, "found small match with hlog3, of length %u", - (U32)mlen); - bestLength = mlen; - assert(curr > matchIndex3); - assert(mnum==0); /* no prior solution */ - matches[0].off = OFFSET_TO_OFFBASE(curr - matchIndex3); - matches[0].len = (U32)mlen; - mnum = 1; - if ( (mlen > sufficient_len) | - (ip+mlen == iLimit) ) { /* best possible length */ - ms->nextToUpdate = curr+1; /* skip insertion */ - return 1; - } } } - /* no dictMatchState lookup: dicts don't have a populated HC3 table */ - } /* if (mls == 3) */ - - hashTable[h] = curr; /* Update Hash Table */ - - for (; nbCompares && (matchIndex >= matchLow); --nbCompares) { - U32* const nextPtr = bt + 2*(matchIndex & btMask); - const BYTE* match; - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - assert(curr > matchIndex); - - if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) { - assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */ - match = base + matchIndex; - if (matchIndex >= dictLimit) assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ - matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit); - } else { - match = dictBase + matchIndex; - assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ - matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); - if (matchIndex+matchLength >= dictLimit) - match = base + matchIndex; /* prepare for match[matchLength] read */ - } - - if (matchLength > bestLength) { - DEBUGLOG(8, "found match of length %u at distance %u (offBase=%u)", - (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex)); - assert(matchEndIdx > matchIndex); - if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + (U32)matchLength; - bestLength = matchLength; - matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex); - matches[mnum].len = (U32)matchLength; - mnum++; - if ( (matchLength > ZSTD_OPT_NUM) - | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { - if (dictMode == ZSTD_dictMatchState) nbCompares = 0; /* break should also skip searching dms */ - break; /* drop, to preserve bt consistency (miss a little bit of compression) */ - } } - - if (match[matchLength] < ip[matchLength]) { - /* match smaller than current */ - *smallerPtr = matchIndex; /* update smaller idx */ - commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ - if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ - smallerPtr = nextPtr+1; /* new candidate => larger than match, which was smaller than current */ - matchIndex = nextPtr[1]; /* new matchIndex, larger than previous, closer to current */ - } else { - *largerPtr = matchIndex; - commonLengthLarger = matchLength; - if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ - largerPtr = nextPtr; - matchIndex = nextPtr[0]; - } } - - *smallerPtr = *largerPtr = 0; - - assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ - if (dictMode == ZSTD_dictMatchState && nbCompares) { - size_t const dmsH = ZSTD_hashPtr(ip, dmsHashLog, mls); - U32 dictMatchIndex = dms->hashTable[dmsH]; - const U32* const dmsBt = dms->chainTable; - commonLengthSmaller = commonLengthLarger = 0; - for (; nbCompares && (dictMatchIndex > dmsLowLimit); --nbCompares) { - const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & dmsBtMask); - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ - const BYTE* match = dmsBase + dictMatchIndex; - matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dmsEnd, prefixStart); - if (dictMatchIndex+matchLength >= dmsHighLimit) - match = base + dictMatchIndex + dmsIndexDelta; /* to prepare for next usage of match[matchLength] */ - - if (matchLength > bestLength) { - matchIndex = dictMatchIndex + dmsIndexDelta; - DEBUGLOG(8, "found dms match of length %u at distance %u (offBase=%u)", - (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex)); - if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + (U32)matchLength; - bestLength = matchLength; - matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex); - matches[mnum].len = (U32)matchLength; - mnum++; - if ( (matchLength > ZSTD_OPT_NUM) - | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { - break; /* drop, to guarantee consistency (miss a little bit of compression) */ - } } - - if (dictMatchIndex <= dmsBtLow) { break; } /* beyond tree size, stop the search */ - if (match[matchLength] < ip[matchLength]) { - commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ - dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ - } else { - /* match is larger than current */ - commonLengthLarger = matchLength; - dictMatchIndex = nextPtr[0]; - } } } /* if (dictMode == ZSTD_dictMatchState) */ - - assert(matchEndIdx > curr+8); - ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ - return mnum; -} - -typedef U32 (*ZSTD_getAllMatchesFn)( - ZSTD_match_t*, - ZSTD_MatchState_t*, - U32*, - const BYTE*, - const BYTE*, - const U32 rep[ZSTD_REP_NUM], - U32 const ll0, - U32 const lengthToBeat); - -FORCE_INLINE_TEMPLATE -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -U32 ZSTD_btGetAllMatches_internal( - ZSTD_match_t* matches, - ZSTD_MatchState_t* ms, - U32* nextToUpdate3, - const BYTE* ip, - const BYTE* const iHighLimit, - const U32 rep[ZSTD_REP_NUM], - U32 const ll0, - U32 const lengthToBeat, - const ZSTD_dictMode_e dictMode, - const U32 mls) -{ - assert(BOUNDED(3, ms->cParams.minMatch, 6) == mls); - DEBUGLOG(8, "ZSTD_BtGetAllMatches(dictMode=%d, mls=%u)", (int)dictMode, mls); - if (ip < ms->window.base + ms->nextToUpdate) - return 0; /* skipped area */ - ZSTD_updateTree_internal(ms, ip, iHighLimit, mls, dictMode); - return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, mls); -} - -#define ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls) ZSTD_btGetAllMatches_##dictMode##_##mls - -#define GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, mls) \ - static U32 ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls)( \ - ZSTD_match_t* matches, \ - ZSTD_MatchState_t* ms, \ - U32* nextToUpdate3, \ - const BYTE* ip, \ - const BYTE* const iHighLimit, \ - const U32 rep[ZSTD_REP_NUM], \ - U32 const ll0, \ - U32 const lengthToBeat) \ - { \ - return ZSTD_btGetAllMatches_internal( \ - matches, ms, nextToUpdate3, ip, iHighLimit, \ - rep, ll0, lengthToBeat, ZSTD_##dictMode, mls); \ - } - -#define GEN_ZSTD_BT_GET_ALL_MATCHES(dictMode) \ - GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 3) \ - GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 4) \ - GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 5) \ - GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 6) - -GEN_ZSTD_BT_GET_ALL_MATCHES(noDict) -GEN_ZSTD_BT_GET_ALL_MATCHES(extDict) -GEN_ZSTD_BT_GET_ALL_MATCHES(dictMatchState) - -#define ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMode) \ - { \ - ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 3), \ - ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 4), \ - ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 5), \ - ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 6) \ - } - -static ZSTD_getAllMatchesFn -ZSTD_selectBtGetAllMatches(ZSTD_MatchState_t const* ms, ZSTD_dictMode_e const dictMode) -{ - ZSTD_getAllMatchesFn const getAllMatchesFns[3][4] = { - ZSTD_BT_GET_ALL_MATCHES_ARRAY(noDict), - ZSTD_BT_GET_ALL_MATCHES_ARRAY(extDict), - ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMatchState) - }; - U32 const mls = BOUNDED(3, ms->cParams.minMatch, 6); - assert((U32)dictMode < 3); - assert(mls - 3 < 4); - return getAllMatchesFns[(int)dictMode][mls - 3]; -} - -/************************* -* LDM helper functions * -*************************/ - -/* Struct containing info needed to make decision about ldm inclusion */ -typedef struct { - RawSeqStore_t seqStore; /* External match candidates store for this block */ - U32 startPosInBlock; /* Start position of the current match candidate */ - U32 endPosInBlock; /* End position of the current match candidate */ - U32 offset; /* Offset of the match candidate */ -} ZSTD_optLdm_t; - -/* ZSTD_optLdm_skipRawSeqStoreBytes(): - * Moves forward in @rawSeqStore by @nbBytes, - * which will update the fields 'pos' and 'posInSequence'. - */ -static void ZSTD_optLdm_skipRawSeqStoreBytes(RawSeqStore_t* rawSeqStore, size_t nbBytes) -{ - U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); - while (currPos && rawSeqStore->pos < rawSeqStore->size) { - rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; - if (currPos >= currSeq.litLength + currSeq.matchLength) { - currPos -= currSeq.litLength + currSeq.matchLength; - rawSeqStore->pos++; - } else { - rawSeqStore->posInSequence = currPos; - break; - } - } - if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) { - rawSeqStore->posInSequence = 0; - } -} - -/* ZSTD_opt_getNextMatchAndUpdateSeqStore(): - * Calculates the beginning and end of the next match in the current block. - * Updates 'pos' and 'posInSequence' of the ldmSeqStore. - */ -static void -ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock, - U32 blockBytesRemaining) -{ - rawSeq currSeq; - U32 currBlockEndPos; - U32 literalsBytesRemaining; - U32 matchBytesRemaining; - - /* Setting match end position to MAX to ensure we never use an LDM during this block */ - if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { - optLdm->startPosInBlock = UINT_MAX; - optLdm->endPosInBlock = UINT_MAX; - return; - } - /* Calculate appropriate bytes left in matchLength and litLength - * after adjusting based on ldmSeqStore->posInSequence */ - currSeq = optLdm->seqStore.seq[optLdm->seqStore.pos]; - assert(optLdm->seqStore.posInSequence <= currSeq.litLength + currSeq.matchLength); - currBlockEndPos = currPosInBlock + blockBytesRemaining; - literalsBytesRemaining = (optLdm->seqStore.posInSequence < currSeq.litLength) ? - currSeq.litLength - (U32)optLdm->seqStore.posInSequence : - 0; - matchBytesRemaining = (literalsBytesRemaining == 0) ? - currSeq.matchLength - ((U32)optLdm->seqStore.posInSequence - currSeq.litLength) : - currSeq.matchLength; - - /* If there are more literal bytes than bytes remaining in block, no ldm is possible */ - if (literalsBytesRemaining >= blockBytesRemaining) { - optLdm->startPosInBlock = UINT_MAX; - optLdm->endPosInBlock = UINT_MAX; - ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, blockBytesRemaining); - return; - } - - /* Matches may be < minMatch by this process. In that case, we will reject them - when we are deciding whether or not to add the ldm */ - optLdm->startPosInBlock = currPosInBlock + literalsBytesRemaining; - optLdm->endPosInBlock = optLdm->startPosInBlock + matchBytesRemaining; - optLdm->offset = currSeq.offset; - - if (optLdm->endPosInBlock > currBlockEndPos) { - /* Match ends after the block ends, we can't use the whole match */ - optLdm->endPosInBlock = currBlockEndPos; - ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, currBlockEndPos - currPosInBlock); - } else { - /* Consume nb of bytes equal to size of sequence left */ - ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, literalsBytesRemaining + matchBytesRemaining); - } -} - -/* ZSTD_optLdm_maybeAddMatch(): - * Adds a match if it's long enough, - * based on it's 'matchStartPosInBlock' and 'matchEndPosInBlock', - * into 'matches'. Maintains the correct ordering of 'matches'. - */ -static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, - const ZSTD_optLdm_t* optLdm, U32 currPosInBlock, - U32 minMatch) -{ - U32 const posDiff = currPosInBlock - optLdm->startPosInBlock; - /* Note: ZSTD_match_t actually contains offBase and matchLength (before subtracting MINMATCH) */ - U32 const candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff; - - /* Ensure that current block position is not outside of the match */ - if (currPosInBlock < optLdm->startPosInBlock - || currPosInBlock >= optLdm->endPosInBlock - || candidateMatchLength < minMatch) { - return; - } - - if (*nbMatches == 0 || ((candidateMatchLength > matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM)) { - U32 const candidateOffBase = OFFSET_TO_OFFBASE(optLdm->offset); - DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offBase: %u matchLength %u) at block position=%u", - candidateOffBase, candidateMatchLength, currPosInBlock); - matches[*nbMatches].len = candidateMatchLength; - matches[*nbMatches].off = candidateOffBase; - (*nbMatches)++; - } -} - -/* ZSTD_optLdm_processMatchCandidate(): - * Wrapper function to update ldm seq store and call ldm functions as necessary. - */ -static void -ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, - ZSTD_match_t* matches, U32* nbMatches, - U32 currPosInBlock, U32 remainingBytes, - U32 minMatch) -{ - if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { - return; - } - - if (currPosInBlock >= optLdm->endPosInBlock) { - if (currPosInBlock > optLdm->endPosInBlock) { - /* The position at which ZSTD_optLdm_processMatchCandidate() is called is not necessarily - * at the end of a match from the ldm seq store, and will often be some bytes - * over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots" - */ - U32 const posOvershoot = currPosInBlock - optLdm->endPosInBlock; - ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, posOvershoot); - } - ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes); - } - ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock, minMatch); -} - - -/*-******************************* -* Optimal parser -*********************************/ - -#if 0 /* debug */ - -static void -listStats(const U32* table, int lastEltID) -{ - int const nbElts = lastEltID + 1; - int enb; - for (enb=0; enb < nbElts; enb++) { - (void)table; - /* RAWLOG(2, "%3i:%3i, ", enb, table[enb]); */ - RAWLOG(2, "%4i,", table[enb]); - } - RAWLOG(2, " \n"); -} - -#endif - -#define LIT_PRICE(_p) (int)ZSTD_rawLiteralsCost(_p, 1, optStatePtr, optLevel) -#define LL_PRICE(_l) (int)ZSTD_litLengthPrice(_l, optStatePtr, optLevel) -#define LL_INCPRICE(_l) (LL_PRICE(_l) - LL_PRICE(_l-1)) - -FORCE_INLINE_TEMPLATE -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -size_t -ZSTD_compressBlock_opt_generic(ZSTD_MatchState_t* ms, - SeqStore_t* seqStore, - U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize, - const int optLevel, - const ZSTD_dictMode_e dictMode) -{ - optState_t* const optStatePtr = &ms->opt; - const BYTE* const istart = (const BYTE*)src; - const BYTE* ip = istart; - const BYTE* anchor = istart; - const BYTE* const iend = istart + srcSize; - const BYTE* const ilimit = iend - 8; - const BYTE* const base = ms->window.base; - const BYTE* const prefixStart = base + ms->window.dictLimit; - const ZSTD_compressionParameters* const cParams = &ms->cParams; - - ZSTD_getAllMatchesFn getAllMatches = ZSTD_selectBtGetAllMatches(ms, dictMode); - - U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); - U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4; - U32 nextToUpdate3 = ms->nextToUpdate; - - ZSTD_optimal_t* const opt = optStatePtr->priceTable; - ZSTD_match_t* const matches = optStatePtr->matchTable; - ZSTD_optimal_t lastStretch; - ZSTD_optLdm_t optLdm; - - ZSTD_memset(&lastStretch, 0, sizeof(ZSTD_optimal_t)); - - optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore; - optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0; - ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip)); - - /* init */ - DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", - (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); - assert(optLevel <= 2); - ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel); - ip += (ip==prefixStart); - - /* Match Loop */ - while (ip < ilimit) { - U32 cur, last_pos = 0; - - /* find first match */ - { U32 const litlen = (U32)(ip - anchor); - U32 const ll0 = !litlen; - U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, ip, iend, rep, ll0, minMatch); - ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, - (U32)(ip-istart), (U32)(iend-ip), - minMatch); - if (!nbMatches) { - DEBUGLOG(8, "no match found at cPos %u", (unsigned)(ip-istart)); - ip++; - continue; - } - - /* Match found: let's store this solution, and eventually find more candidates. - * During this forward pass, @opt is used to store stretches, - * defined as "a match followed by N literals". - * Note how this is different from a Sequence, which is "N literals followed by a match". - * Storing stretches allows us to store different match predecessors - * for each literal position part of a literals run. */ - - /* initialize opt[0] */ - opt[0].mlen = 0; /* there are only literals so far */ - opt[0].litlen = litlen; - /* No need to include the actual price of the literals before the first match - * because it is static for the duration of the forward pass, and is included - * in every subsequent price. But, we include the literal length because - * the cost variation of litlen depends on the value of litlen. - */ - opt[0].price = LL_PRICE(litlen); - ZSTD_STATIC_ASSERT(sizeof(opt[0].rep[0]) == sizeof(rep[0])); - ZSTD_memcpy(&opt[0].rep, rep, sizeof(opt[0].rep)); - - /* large match -> immediate encoding */ - { U32 const maxML = matches[nbMatches-1].len; - U32 const maxOffBase = matches[nbMatches-1].off; - DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffBase=%u at cPos=%u => start new series", - nbMatches, maxML, maxOffBase, (U32)(ip-prefixStart)); - - if (maxML > sufficient_len) { - lastStretch.litlen = 0; - lastStretch.mlen = maxML; - lastStretch.off = maxOffBase; - DEBUGLOG(6, "large match (%u>%u) => immediate encoding", - maxML, sufficient_len); - cur = 0; - last_pos = maxML; - goto _shortestPath; - } } - - /* set prices for first matches starting position == 0 */ - assert(opt[0].price >= 0); - { U32 pos; - U32 matchNb; - for (pos = 1; pos < minMatch; pos++) { - opt[pos].price = ZSTD_MAX_PRICE; - opt[pos].mlen = 0; - opt[pos].litlen = litlen + pos; - } - for (matchNb = 0; matchNb < nbMatches; matchNb++) { - U32 const offBase = matches[matchNb].off; - U32 const end = matches[matchNb].len; - for ( ; pos <= end ; pos++ ) { - int const matchPrice = (int)ZSTD_getMatchPrice(offBase, pos, optStatePtr, optLevel); - int const sequencePrice = opt[0].price + matchPrice; - DEBUGLOG(7, "rPos:%u => set initial price : %.2f", - pos, ZSTD_fCost(sequencePrice)); - opt[pos].mlen = pos; - opt[pos].off = offBase; - opt[pos].litlen = 0; /* end of match */ - opt[pos].price = sequencePrice + LL_PRICE(0); - } - } - last_pos = pos-1; - opt[pos].price = ZSTD_MAX_PRICE; - } - } - - /* check further positions */ - for (cur = 1; cur <= last_pos; cur++) { - const BYTE* const inr = ip + cur; - assert(cur <= ZSTD_OPT_NUM); - DEBUGLOG(7, "cPos:%i==rPos:%u", (int)(inr-istart), cur); - - /* Fix current position with one literal if cheaper */ - { U32 const litlen = opt[cur-1].litlen + 1; - int const price = opt[cur-1].price - + LIT_PRICE(ip+cur-1) - + LL_INCPRICE(litlen); - assert(price < 1000000000); /* overflow check */ - if (price <= opt[cur].price) { - ZSTD_optimal_t const prevMatch = opt[cur]; - DEBUGLOG(7, "cPos:%i==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)", - (int)(inr-istart), cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen, - opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]); - opt[cur] = opt[cur-1]; - opt[cur].litlen = litlen; - opt[cur].price = price; - if ( (optLevel >= 1) /* additional check only for higher modes */ - && (prevMatch.litlen == 0) /* replace a match */ - && (LL_INCPRICE(1) < 0) /* ll1 is cheaper than ll0 */ - && LIKELY(ip + cur < iend) - ) { - /* check next position, in case it would be cheaper */ - int with1literal = prevMatch.price + LIT_PRICE(ip+cur) + LL_INCPRICE(1); - int withMoreLiterals = price + LIT_PRICE(ip+cur) + LL_INCPRICE(litlen+1); - DEBUGLOG(7, "then at next rPos %u : match+1lit %.2f vs %ulits %.2f", - cur+1, ZSTD_fCost(with1literal), litlen+1, ZSTD_fCost(withMoreLiterals)); - if ( (with1literal < withMoreLiterals) - && (with1literal < opt[cur+1].price) ) { - /* update offset history - before it disappears */ - U32 const prev = cur - prevMatch.mlen; - Repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, prevMatch.off, opt[prev].litlen==0); - assert(cur >= prevMatch.mlen); - DEBUGLOG(7, "==> match+1lit is cheaper (%.2f < %.2f) (hist:%u,%u,%u) !", - ZSTD_fCost(with1literal), ZSTD_fCost(withMoreLiterals), - newReps.rep[0], newReps.rep[1], newReps.rep[2] ); - opt[cur+1] = prevMatch; /* mlen & offbase */ - ZSTD_memcpy(opt[cur+1].rep, &newReps, sizeof(Repcodes_t)); - opt[cur+1].litlen = 1; - opt[cur+1].price = with1literal; - if (last_pos < cur+1) last_pos = cur+1; - } - } - } else { - DEBUGLOG(7, "cPos:%i==rPos:%u : literal would cost more (%.2f>%.2f)", - (int)(inr-istart), cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price)); - } - } - - /* Offset history is not updated during match comparison. - * Do it here, now that the match is selected and confirmed. - */ - ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(Repcodes_t)); - assert(cur >= opt[cur].mlen); - if (opt[cur].litlen == 0) { - /* just finished a match => alter offset history */ - U32 const prev = cur - opt[cur].mlen; - Repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[prev].litlen==0); - ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(Repcodes_t)); - } - - /* last match must start at a minimum distance of 8 from oend */ - if (inr > ilimit) continue; - - if (cur == last_pos) break; - - if ( (optLevel==0) /*static_test*/ - && (opt[cur+1].price <= opt[cur].price + (BITCOST_MULTIPLIER/2)) ) { - DEBUGLOG(7, "skip current position : next rPos(%u) price is cheaper", cur+1); - continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */ - } - - assert(opt[cur].price >= 0); - { U32 const ll0 = (opt[cur].litlen == 0); - int const previousPrice = opt[cur].price; - int const basePrice = previousPrice + LL_PRICE(0); - U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, inr, iend, opt[cur].rep, ll0, minMatch); - U32 matchNb; - - ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, - (U32)(inr-istart), (U32)(iend-inr), - minMatch); - - if (!nbMatches) { - DEBUGLOG(7, "rPos:%u : no match found", cur); - continue; - } - - { U32 const longestML = matches[nbMatches-1].len; - DEBUGLOG(7, "cPos:%i==rPos:%u, found %u matches, of longest ML=%u", - (int)(inr-istart), cur, nbMatches, longestML); - - if ( (longestML > sufficient_len) - || (cur + longestML >= ZSTD_OPT_NUM) - || (ip + cur + longestML >= iend) ) { - lastStretch.mlen = longestML; - lastStretch.off = matches[nbMatches-1].off; - lastStretch.litlen = 0; - last_pos = cur + longestML; - goto _shortestPath; - } } - - /* set prices using matches found at position == cur */ - for (matchNb = 0; matchNb < nbMatches; matchNb++) { - U32 const offset = matches[matchNb].off; - U32 const lastML = matches[matchNb].len; - U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch; - U32 mlen; - - DEBUGLOG(7, "testing match %u => offBase=%4u, mlen=%2u, llen=%2u", - matchNb, matches[matchNb].off, lastML, opt[cur].litlen); - - for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */ - U32 const pos = cur + mlen; - int const price = basePrice + (int)ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); - - if ((pos > last_pos) || (price < opt[pos].price)) { - DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)", - pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); - while (last_pos < pos) { - /* fill empty positions, for future comparisons */ - last_pos++; - opt[last_pos].price = ZSTD_MAX_PRICE; - opt[last_pos].litlen = !0; /* just needs to be != 0, to mean "not an end of match" */ - } - opt[pos].mlen = mlen; - opt[pos].off = offset; - opt[pos].litlen = 0; - opt[pos].price = price; - } else { - DEBUGLOG(7, "rPos:%u (ml=%2u) => new price is worse (%.2f>=%.2f)", - pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); - if (optLevel==0) break; /* early update abort; gets ~+10% speed for about -0.01 ratio loss */ - } - } } } - opt[last_pos+1].price = ZSTD_MAX_PRICE; - } /* for (cur = 1; cur <= last_pos; cur++) */ - - lastStretch = opt[last_pos]; - assert(cur >= lastStretch.mlen); - cur = last_pos - lastStretch.mlen; - -_shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ - assert(opt[0].mlen == 0); - assert(last_pos >= lastStretch.mlen); - assert(cur == last_pos - lastStretch.mlen); - - if (lastStretch.mlen==0) { - /* no solution : all matches have been converted into literals */ - assert(lastStretch.litlen == (ip - anchor) + last_pos); - ip += last_pos; - continue; - } - assert(lastStretch.off > 0); - - /* Update offset history */ - if (lastStretch.litlen == 0) { - /* finishing on a match : update offset history */ - Repcodes_t const reps = ZSTD_newRep(opt[cur].rep, lastStretch.off, opt[cur].litlen==0); - ZSTD_memcpy(rep, &reps, sizeof(Repcodes_t)); - } else { - ZSTD_memcpy(rep, lastStretch.rep, sizeof(Repcodes_t)); - assert(cur >= lastStretch.litlen); - cur -= lastStretch.litlen; - } - - /* Let's write the shortest path solution. - * It is stored in @opt in reverse order, - * starting from @storeEnd (==cur+2), - * effectively partially @opt overwriting. - * Content is changed too: - * - So far, @opt stored stretches, aka a match followed by literals - * - Now, it will store sequences, aka literals followed by a match - */ - { U32 const storeEnd = cur + 2; - U32 storeStart = storeEnd; - U32 stretchPos = cur; - - DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)", - last_pos, cur); (void)last_pos; - assert(storeEnd < ZSTD_OPT_SIZE); - DEBUGLOG(6, "last stretch copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", - storeEnd, lastStretch.litlen, lastStretch.mlen, lastStretch.off); - if (lastStretch.litlen > 0) { - /* last "sequence" is unfinished: just a bunch of literals */ - opt[storeEnd].litlen = lastStretch.litlen; - opt[storeEnd].mlen = 0; - storeStart = storeEnd-1; - opt[storeStart] = lastStretch; - } { - opt[storeEnd] = lastStretch; /* note: litlen will be fixed */ - storeStart = storeEnd; - } - while (1) { - ZSTD_optimal_t nextStretch = opt[stretchPos]; - opt[storeStart].litlen = nextStretch.litlen; - DEBUGLOG(6, "selected sequence (llen=%u,mlen=%u,ofc=%u)", - opt[storeStart].litlen, opt[storeStart].mlen, opt[storeStart].off); - if (nextStretch.mlen == 0) { - /* reaching beginning of segment */ - break; - } - storeStart--; - opt[storeStart] = nextStretch; /* note: litlen will be fixed */ - assert(nextStretch.litlen + nextStretch.mlen <= stretchPos); - stretchPos -= nextStretch.litlen + nextStretch.mlen; - } - - /* save sequences */ - DEBUGLOG(6, "sending selected sequences into seqStore"); - { U32 storePos; - for (storePos=storeStart; storePos <= storeEnd; storePos++) { - U32 const llen = opt[storePos].litlen; - U32 const mlen = opt[storePos].mlen; - U32 const offBase = opt[storePos].off; - U32 const advance = llen + mlen; - DEBUGLOG(6, "considering seq starting at %i, llen=%u, mlen=%u", - (int)(anchor - istart), (unsigned)llen, (unsigned)mlen); - - if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */ - assert(storePos == storeEnd); /* must be last sequence */ - ip = anchor + llen; /* last "sequence" is a bunch of literals => don't progress anchor */ - continue; /* will finish */ - } - - assert(anchor + llen <= iend); - ZSTD_updateStats(optStatePtr, llen, anchor, offBase, mlen); - ZSTD_storeSeq(seqStore, llen, anchor, iend, offBase, mlen); - anchor += advance; - ip = anchor; - } } - DEBUGLOG(7, "new offset history : %u, %u, %u", rep[0], rep[1], rep[2]); - - /* update all costs */ - ZSTD_setBasePrices(optStatePtr, optLevel); - } - } /* while (ip < ilimit) */ - - /* Return the last literals size */ - return (size_t)(iend - anchor); -} -#endif /* build exclusions */ - -#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR -static size_t ZSTD_compressBlock_opt0( - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode) -{ - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /* optLevel */, dictMode); -} -#endif - -#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR -static size_t ZSTD_compressBlock_opt2( - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode) -{ - return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /* optLevel */, dictMode); -} -#endif - #ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR size_t ZSTD_compressBlock_btopt( - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) + ZSTD_MatchState_t* const ms, SeqStore_t* const seqStore, + U32 rep[ZSTD_REP_NUM], const void* const src, size_t const srcSize) { - DEBUGLOG(5, "ZSTD_compressBlock_btopt"); - return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_noDict); -} -#endif - - - - -#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR -/* ZSTD_initStats_ultra(): - * make a first compression pass, just to seed stats with more accurate starting values. - * only works on first block, with no dictionary and no ldm. - * this function cannot error out, its narrow contract must be respected. - */ -static -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -void ZSTD_initStats_ultra(ZSTD_MatchState_t* ms, - SeqStore_t* seqStore, - U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) -{ - U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */ - ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep)); - - DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize); - assert(ms->opt.litLengthSum == 0); /* first block */ - assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */ - assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */ - assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */ - - ZSTD_compressBlock_opt2(ms, seqStore, tmpRep, src, srcSize, ZSTD_noDict); /* generate stats into ms->opt*/ - - /* invalidate first scan from history, only keep entropy stats */ - ZSTD_resetSeqStore(seqStore); - ms->window.base -= srcSize; - ms->window.dictLimit += (U32)srcSize; - ms->window.lowLimit = ms->window.dictLimit; - ms->nextToUpdate = ms->window.dictLimit; - + ZSTD_RustOptState state; + ZSTD_rustOptState_init_with_costs(&state, ms, NULL); + return ZSTD_rust_compressBlock_opt(&state, seqStore, rep, src, srcSize, + 0, ZSTD_rust_dict_noDict); } -size_t ZSTD_compressBlock_btultra( - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) -{ - DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize); - return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict); -} - -size_t ZSTD_compressBlock_btultra2( - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) -{ - U32 const curr = (U32)((const BYTE*)src - ms->window.base); - DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize); - - /* 2-passes strategy: - * this strategy makes a first pass over first block to collect statistics - * in order to seed next round's statistics with it. - * After 1st pass, function forgets history, and starts a new block. - * Consequently, this can only work if no data has been previously loaded in tables, - * aka, no dictionary, no prefix, no ldm preprocessing. - * The compression ratio gain is generally small (~0.5% on first block), - * the cost is 2x cpu time on first block. */ - assert(srcSize <= ZSTD_BLOCKSIZE_MAX); - if ( (ms->opt.litLengthSum==0) /* first block */ - && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */ - && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */ - && (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */ - && (srcSize > ZSTD_PREDEF_THRESHOLD) /* input large enough to not employ default stats */ - ) { - ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize); - } - - return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict); -} -#endif - -#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR size_t ZSTD_compressBlock_btopt_dictMatchState( - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) + ZSTD_MatchState_t* const ms, SeqStore_t* const seqStore, + U32 rep[ZSTD_REP_NUM], const void* const src, size_t const srcSize) { - return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState); + ZSTD_RustOptState state; + ZSTD_RustOptState dictState; + ZSTD_MatchState_t dictCopy = *ms->dictMatchState; + ZSTD_rustOptState_init(&dictState, &dictCopy, NULL); + ZSTD_rustOptState_init_with_costs(&state, ms, &dictState); + return ZSTD_rust_compressBlock_opt(&state, seqStore, rep, src, srcSize, + 0, ZSTD_rust_dict_dictMatchState); } size_t ZSTD_compressBlock_btopt_extDict( - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) + ZSTD_MatchState_t* const ms, SeqStore_t* const seqStore, + U32 rep[ZSTD_REP_NUM], const void* const src, size_t const srcSize) { - return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_extDict); + ZSTD_RustOptState state; + ZSTD_rustOptState_init_with_costs(&state, ms, NULL); + return ZSTD_rust_compressBlock_opt(&state, seqStore, rep, src, srcSize, + 0, ZSTD_rust_dict_extDict); } #endif #ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR -size_t ZSTD_compressBlock_btultra_dictMatchState( - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) +size_t ZSTD_compressBlock_btultra( + ZSTD_MatchState_t* const ms, SeqStore_t* const seqStore, + U32 rep[ZSTD_REP_NUM], const void* const src, size_t const srcSize) { - return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState); + ZSTD_RustOptState state; + ZSTD_rustOptState_init_with_costs(&state, ms, NULL); + return ZSTD_rust_compressBlock_opt(&state, seqStore, rep, src, srcSize, + 2, ZSTD_rust_dict_noDict); +} + +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_MatchState_t* const ms, SeqStore_t* const seqStore, + U32 rep[ZSTD_REP_NUM], const void* const src, size_t const srcSize) +{ + ZSTD_RustOptState state; + ZSTD_RustOptState dictState; + ZSTD_MatchState_t dictCopy = *ms->dictMatchState; + ZSTD_rustOptState_init(&dictState, &dictCopy, NULL); + ZSTD_rustOptState_init_with_costs(&state, ms, &dictState); + return ZSTD_rust_compressBlock_opt(&state, seqStore, rep, src, srcSize, + 2, ZSTD_rust_dict_dictMatchState); } size_t ZSTD_compressBlock_btultra_extDict( - ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize) + ZSTD_MatchState_t* const ms, SeqStore_t* const seqStore, + U32 rep[ZSTD_REP_NUM], const void* const src, size_t const srcSize) { - return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_extDict); + ZSTD_RustOptState state; + ZSTD_rustOptState_init_with_costs(&state, ms, NULL); + return ZSTD_rust_compressBlock_opt(&state, seqStore, rep, src, srcSize, + 2, ZSTD_rust_dict_extDict); +} + +size_t ZSTD_compressBlock_btultra2( + ZSTD_MatchState_t* const ms, SeqStore_t* const seqStore, + U32 rep[ZSTD_REP_NUM], const void* const src, size_t const srcSize) +{ + ZSTD_RustOptState state; + ZSTD_rustOptState_init_with_costs(&state, ms, NULL); + return ZSTD_rust_compressBlock_btultra2(&state, seqStore, rep, src, + srcSize); } #endif -/* note : no btultra2 variant for extDict nor dictMatchState, - * because btultra2 is not meant to work with dictionaries - * and is only specific for the first block (no prefix) */ +#endif /* any optimal parser entry point is enabled */ diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 698f7fef3..437e25ed6 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -55,6 +55,8 @@ pub mod zstd_lazy; #[cfg(feature = "compression")] pub mod zstd_ldm; #[cfg(feature = "compression")] +pub mod zstd_opt; +#[cfg(feature = "compression")] pub mod zstd_opt_tree; #[cfg(feature = "compression")] pub mod zstd_presplit; diff --git a/rust/src/zstd_opt.rs b/rust/src/zstd_opt.rs new file mode 100644 index 000000000..5a68a6622 --- /dev/null +++ b/rust/src/zstd_opt.rs @@ -0,0 +1,1836 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(clippy::missing_safety_doc)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::not_unsafe_ptr_arg_deref)] + +//! Optimal block parsing for the binary-tree compression strategies. +//! +//! The compression context remains private to C. `zstd_opt.c` projects the +//! fields used by this module into `ZSTD_RustOptState`; all buffers and tables +//! remain owned by the caller. This keeps the ABI independent of the private +//! `ZSTD_MatchState_t` layout while moving the price model, match enumeration, +//! optimal parse, and sequence emission into Rust. + +use crate::bits::ZSTD_highbit32; +use crate::common::{LL_BITS, MAX_LIT, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS}; +use crate::mem::{ + MEM_64bits, MEM_isLittleEndian, MEM_read16, MEM_read32, MEM_readLE32, MEM_readLE64, MEM_readST, +}; +use std::cmp::min; +use std::ffi::c_void; +use std::mem::size_of; +use std::os::raw::{c_int, c_uint}; +use std::ptr; + +const ZSTD_REP_NUM: usize = 3; +const HASH_READ_SIZE: usize = 8; +const ZSTD_OPT_NUM: u32 = 1 << 12; +const ZSTD_BLOCKSIZE_MAX: usize = 1 << 17; +const ZSTD_LITFREQ_ADD: u32 = 2; +const ZSTD_MAX_PRICE: i32 = 1 << 30; +const ZSTD_PREDEF_THRESHOLD: usize = 8; +const BITCOST_ACCURACY: u32 = 8; +const BITCOST_MULTIPLIER: u32 = 1 << BITCOST_ACCURACY; +const OFFSET_OFFBASE: u32 = ZSTD_REP_NUM as u32; +const PRIME3BYTES: u32 = 506_832_829; + +const ZOP_DYNAMIC: c_int = 0; +const ZOP_PREDEF: c_int = 1; +const ZSTD_PS_DISABLE: c_int = 2; + +const DICT_NO_DICT: c_int = 0; +const DICT_EXT: c_int = 1; +const DICT_MATCH_STATE: c_int = 2; + +const LL_CODE: [u8; 64] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, + 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, +]; + +#[allow(dead_code)] +const ML_CODE: [u8; 149] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 38, 38, + 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 41, 41, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, +]; + +#[repr(C)] +#[derive(Clone, Copy)] +struct SeqDef { + offBase: u32, + litLength: u16, + mlBase: u16, +} + +#[repr(C)] +struct SeqStore_t { + sequencesStart: *mut SeqDef, + sequences: *mut SeqDef, + litStart: *mut u8, + lit: *mut u8, + llCode: *mut u8, + mlCode: *mut u8, + ofCode: *mut u8, + maxNbSeq: usize, + maxNbLit: usize, + longLengthType: c_int, + longLengthPos: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct ZSTD_match_t { + off: u32, + len: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct ZSTD_optimal_t { + price: c_int, + off: u32, + mlen: u32, + litlen: u32, + rep: [u32; ZSTD_REP_NUM], +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct RawSeq { + offset: u32, + litLength: u32, + matchLength: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct RawSeqStore { + seq: *const RawSeq, + pos: usize, + posInSequence: usize, + size: usize, + capacity: usize, +} + +impl Default for RawSeqStore { + fn default() -> Self { + Self { + seq: ptr::null(), + pos: 0, + posInSequence: 0, + size: 0, + capacity: 0, + } + } +} + +#[repr(C)] +struct OptState { + litFreq: *mut c_uint, + litLengthFreq: *mut c_uint, + matchLengthFreq: *mut c_uint, + offCodeFreq: *mut c_uint, + matchTable: *mut ZSTD_match_t, + priceTable: *mut ZSTD_optimal_t, + litSum: u32, + litLengthSum: u32, + matchLengthSum: u32, + offCodeSum: u32, + litSumBasePrice: u32, + litLengthSumBasePrice: u32, + matchLengthSumBasePrice: u32, + offCodeSumBasePrice: u32, + priceType: c_int, + symbolCosts: *const c_void, + literalCompressionMode: c_int, +} + +/* The first fields intentionally match zstd_opt_tree's public leaf view. */ +#[repr(C)] +struct OptTreePrefix { + hash_table: *mut u32, + chain_table: *mut u32, + base: *const u8, + dict_base: *const u8, + dict_limit: u32, + low_limit: u32, + loaded_dict_end: u32, + next_to_update: *mut u32, + hash_log: u32, + chain_log: u32, + search_log: u32, + window_log: u32, + use_c_predict: c_int, +} + +/// Private-state projection populated by `lib/compress/zstd_opt.c`. +#[repr(C)] +pub struct ZSTD_RustOptState { + hash_table: *mut u32, + chain_table: *mut u32, + base: *const u8, + dict_base: *const u8, + dict_limit: u32, + low_limit: u32, + loaded_dict_end: u32, + next_to_update: *mut u32, + hash_log: u32, + chain_log: u32, + search_log: u32, + window_log: u32, + use_c_predict: c_int, + hash_table3: *mut u32, + hash_log3: u32, + next_src: *const u8, + min_match: u32, + target_length: u32, + opt: *mut OptState, + dict_match_state: *const ZSTD_RustOptState, + ldm_seq_store: *const RawSeqStore, + window_base: *mut *const u8, + window_dict_limit: *mut u32, + window_low_limit: *mut u32, + huf_ctable: *const usize, + huf_repeat_valid: c_int, + fse_litlength_ctable: *const u32, + fse_matchlength_ctable: *const u32, + fse_offcode_ctable: *const u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct FseSymbolCompressionTransform { + deltaFindState: c_int, + deltaNbBits: u32, +} + +extern "C" { + fn ZSTD_rust_opt_updateTreeInternal( + state: *mut OptTreePrefix, + ip: *const c_void, + iend: *const c_void, + mls: u32, + extDict: c_int, + ); + fn HUF_getNbBitsFromCTable(ctable: *const usize, symbolValue: u32) -> u32; +} + +#[inline] +fn ptr_diff(left: *const u8, right: *const u8) -> usize { + (left as usize).wrapping_sub(right as usize) +} + +#[inline] +fn index_from(base: *const u8, value: *const u8) -> u32 { + ptr_diff(value, base) as u32 +} + +#[inline] +unsafe fn count(mut input: *const u8, mut matched: *const u8, input_limit: *const u8) -> usize { + let input_start = input; + let word_size = size_of::(); + while ptr_diff(input_limit, input) >= word_size { + let difference = + unsafe { MEM_readST(matched.cast::()) ^ MEM_readST(input.cast::()) }; + if difference != 0 { + let common = if MEM_isLittleEndian() { + difference.trailing_zeros() + } else { + difference.leading_zeros() + } as usize + / 8; + return ptr_diff(input, input_start) + common; + } + input = input.wrapping_add(word_size); + matched = matched.wrapping_add(word_size); + } + if MEM_64bits() + && ptr_diff(input_limit, input) >= 4 + && unsafe { MEM_read32(matched.cast::()) == MEM_read32(input.cast::()) } + { + input = input.wrapping_add(4); + matched = matched.wrapping_add(4); + } + if ptr_diff(input_limit, input) >= 2 + && unsafe { MEM_read16(matched.cast::()) == MEM_read16(input.cast::()) } + { + input = input.wrapping_add(2); + matched = matched.wrapping_add(2); + } + if ptr_diff(input, input_limit) != 0 && unsafe { *matched == *input } { + input = input.wrapping_add(1); + } + ptr_diff(input, input_start) +} + +unsafe fn count_2segments( + input: *const u8, + matched: *const u8, + input_end: *const u8, + match_end: *const u8, + input_start: *const u8, +) -> usize { + let match_remaining = ptr_diff(match_end, matched); + let input_remaining = ptr_diff(input_end, input); + let first_end = input.wrapping_add(min(match_remaining, input_remaining)); + let first_count = unsafe { count(input, matched, first_end) }; + if matched.wrapping_add(first_count) != match_end { + return first_count; + } + first_count + unsafe { count(input.wrapping_add(first_count), input_start, input_end) } +} + +#[inline] +unsafe fn hash_ptr(input: *const u8, hbits: u32, mls: u32) -> usize { + let hash_shift32 = |value: u32| { + if hbits == 0 { + 0 + } else { + (value >> (32 - hbits)) as usize + } + }; + let hash_shift64 = |value: u64| { + if hbits == 0 { + 0 + } else { + (value >> (64 - hbits)) as usize + } + }; + match mls { + 5 => hash_shift64( + unsafe { MEM_readLE64(input.cast::()) } + .wrapping_shl(24) + .wrapping_mul(889_523_592_379), + ), + 6 => hash_shift64( + unsafe { MEM_readLE64(input.cast::()) } + .wrapping_shl(16) + .wrapping_mul(227_718_039_650_203), + ), + 7 => hash_shift64( + unsafe { MEM_readLE64(input.cast::()) } + .wrapping_shl(8) + .wrapping_mul(58_295_818_150_454_627), + ), + 8 => hash_shift64( + unsafe { MEM_readLE64(input.cast::()) }.wrapping_mul(0xCF1B_BCDC_B7A5_6463), + ), + _ => hash_shift32( + unsafe { MEM_readLE32(input.cast::()) }.wrapping_mul(2_654_435_761), + ), + } +} + +#[inline] +unsafe fn hash3_ptr(input: *const u8, hbits: u32) -> usize { + if hbits == 0 { + 0 + } else { + let value = unsafe { MEM_readLE32(input.cast::()) }; + (value.wrapping_shl(8).wrapping_mul(PRIME3BYTES) >> (32 - hbits)) as usize + } +} + +#[inline] +unsafe fn table_get(table: *const u32, index: usize) -> u32 { + unsafe { *table.add(index) } +} + +#[inline] +unsafe fn table_set(table: *mut u32, index: usize, value: u32) { + unsafe { *table.add(index) = value }; +} + +#[inline] +fn lowest_match_index(state: &ZSTD_RustOptState, current: u32) -> u32 { + let max_distance = 1u32.wrapping_shl(state.window_log); + let within_window = if current.wrapping_sub(state.low_limit) > max_distance { + current.wrapping_sub(max_distance) + } else { + state.low_limit + }; + if state.loaded_dict_end != 0 { + state.low_limit + } else { + within_window + } +} + +#[inline] +fn index_overlap_check(prefix_lowest_index: u32, rep_index: u32) -> bool { + prefix_lowest_index.wrapping_sub(1).wrapping_sub(rep_index) >= 3 +} + +#[inline] +unsafe fn read_min_match(input: *const u8, length: u32) -> u32 { + let value = unsafe { MEM_read32(input.cast::()) }; + if length == 3 && MEM_isLittleEndian() { + value << 8 + } else if length == 3 { + value >> 8 + } else { + value + } +} + +#[inline] +fn ll_code(value: u32) -> usize { + if value > 63 { + (ZSTD_highbit32(value) + 19) as usize + } else { + LL_CODE[value as usize] as usize + } +} + +#[inline] +fn ml_code(value: u32) -> usize { + match value { + 0..=31 => value as usize, + 32..=33 => 32, + 34..=35 => 33, + 36..=37 => 34, + 38..=39 => 35, + 40..=43 => 36, + 44..=47 => 37, + 48..=51 => 38, + 52..=55 => 39, + 56..=71 => 40, + 72..=87 => 41, + 88..=127 => 42, + _ => (ZSTD_highbit32(value) + 36) as usize, + } +} + +#[inline] +fn bit_weight(stat: u32) -> u32 { + ZSTD_highbit32(stat.wrapping_add(1)) * BITCOST_MULTIPLIER +} + +#[inline] +fn frac_weight(raw_stat: u32) -> u32 { + let stat = raw_stat.wrapping_add(1); + let high_bit = ZSTD_highbit32(stat); + let base = high_bit * BITCOST_MULTIPLIER; + base + (stat << BITCOST_ACCURACY >> high_bit) +} + +#[inline] +fn weight(stat: u32, opt_level: c_int) -> u32 { + if opt_level >= 2 { + frac_weight(stat) + } else { + bit_weight(stat) + } +} + +unsafe fn sum_u32(table: *const c_uint, count: usize) -> u32 { + let mut total: u32 = 0; + for index in 0..count { + total = total.wrapping_add(unsafe { *table.add(index) }); + } + total +} + +unsafe fn downscale_stats( + table: *mut c_uint, + last_index: usize, + shift: u32, + base_one: bool, +) -> u32 { + let mut sum: u32 = 0; + for index in 0..=last_index { + let old = unsafe { *table.add(index) }; + let value = u32::from(base_one) + (old >> shift); + unsafe { *table.add(index) = value }; + sum = sum.wrapping_add(value); + } + sum +} + +unsafe fn scale_stats(table: *mut c_uint, last_index: usize, log_target: u32) -> u32 { + let previous = unsafe { sum_u32(table, last_index + 1) }; + let factor = previous >> log_target; + if factor <= 1 { + previous + } else { + unsafe { downscale_stats(table, last_index, ZSTD_highbit32(factor), true) } + } +} + +unsafe fn fse_max_nb_bits(table: *const u32, symbol: usize) -> u32 { + let table_log = unsafe { MEM_read16(table.cast::()) } as usize; + let transform_offset = 1 + if table_log == 0 { + 1 + } else { + 1 << (table_log - 1) + }; + let transform = unsafe { + table + .add(transform_offset) + .cast::() + .add(symbol) + .read() + }; + (transform.deltaNbBits.wrapping_add(0xFFFF)) >> 16 +} + +#[inline] +fn compressed_literals(opt: &OptState) -> bool { + opt.literalCompressionMode != ZSTD_PS_DISABLE +} + +unsafe fn set_base_prices(opt: &mut OptState, opt_level: c_int) { + if compressed_literals(opt) { + opt.litSumBasePrice = weight(opt.litSum, opt_level); + } + opt.litLengthSumBasePrice = weight(opt.litLengthSum, opt_level); + opt.matchLengthSumBasePrice = weight(opt.matchLengthSum, opt_level); + opt.offCodeSumBasePrice = weight(opt.offCodeSum, opt_level); +} + +unsafe fn rescale_freqs( + state: &ZSTD_RustOptState, + src: *const u8, + src_size: usize, + opt_level: c_int, +) { + let opt = unsafe { &mut *state.opt }; + let compressed = compressed_literals(opt); + opt.priceType = ZOP_DYNAMIC; + if opt.litLengthSum == 0 { + if src_size <= ZSTD_PREDEF_THRESHOLD { + opt.priceType = ZOP_PREDEF; + } + + if state.huf_repeat_valid == 0 { + /* No valid dictionary table: seed from the first source block. */ + if compressed { + for index in 0..=MAX_LIT { + unsafe { *opt.litFreq.add(index) = 0 }; + } + for index in 0..src_size { + let byte = unsafe { *src.add(index) } as usize; + unsafe { *opt.litFreq.add(byte) = (*opt.litFreq.add(byte)).wrapping_add(1) }; + } + opt.litSum = unsafe { downscale_stats(opt.litFreq, MAX_LIT, 8, false) }; + } + let base_ll: [u32; MAX_LL + 1] = [ + 4, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + ]; + unsafe { + ptr::copy_nonoverlapping(base_ll.as_ptr(), opt.litLengthFreq, base_ll.len()); + } + opt.litLengthSum = unsafe { sum_u32(base_ll.as_ptr(), base_ll.len()) }; + for index in 0..=MAX_ML { + unsafe { *opt.matchLengthFreq.add(index) = 1 }; + } + opt.matchLengthSum = (MAX_ML + 1) as u32; + let base_of: [u32; MAX_OFF + 1] = [ + 6, 2, 1, 1, 2, 3, 4, 4, 4, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, + ]; + unsafe { + ptr::copy_nonoverlapping(base_of.as_ptr(), opt.offCodeFreq, base_of.len()); + } + opt.offCodeSum = unsafe { sum_u32(base_of.as_ptr(), base_of.len()) }; + } else { + /* Dictionary-seeded tables are projected as leaf pointers by C. */ + if compressed && !state.huf_ctable.is_null() { + opt.litSum = 0; + for lit in 0..=MAX_LIT { + let bit_cost = unsafe { HUF_getNbBitsFromCTable(state.huf_ctable, lit as u32) }; + let value = if bit_cost != 0 { + 1u32 << (11 - bit_cost) + } else { + 1 + }; + unsafe { *opt.litFreq.add(lit) = value }; + opt.litSum = opt.litSum.wrapping_add(value); + } + } + if !state.fse_litlength_ctable.is_null() { + opt.litLengthSum = 0; + for symbol in 0..=MAX_LL { + let bits = unsafe { fse_max_nb_bits(state.fse_litlength_ctable, symbol) }; + let value = if bits != 0 { 1u32 << (10 - bits) } else { 1 }; + unsafe { *opt.litLengthFreq.add(symbol) = value }; + opt.litLengthSum = opt.litLengthSum.wrapping_add(value); + } + } + if !state.fse_matchlength_ctable.is_null() { + opt.matchLengthSum = 0; + for symbol in 0..=MAX_ML { + let bits = unsafe { fse_max_nb_bits(state.fse_matchlength_ctable, symbol) }; + let value = if bits != 0 { 1u32 << (10 - bits) } else { 1 }; + unsafe { *opt.matchLengthFreq.add(symbol) = value }; + opt.matchLengthSum = opt.matchLengthSum.wrapping_add(value); + } + } + if !state.fse_offcode_ctable.is_null() { + opt.offCodeSum = 0; + for symbol in 0..=MAX_OFF { + let bits = unsafe { fse_max_nb_bits(state.fse_offcode_ctable, symbol) }; + let value = if bits != 0 { 1u32 << (10 - bits) } else { 1 }; + unsafe { *opt.offCodeFreq.add(symbol) = value }; + opt.offCodeSum = opt.offCodeSum.wrapping_add(value); + } + } + } + } else { + if compressed { + opt.litSum = unsafe { scale_stats(opt.litFreq, MAX_LIT, 12) }; + } + opt.litLengthSum = unsafe { scale_stats(opt.litLengthFreq, MAX_LL, 11) }; + opt.matchLengthSum = unsafe { scale_stats(opt.matchLengthFreq, MAX_ML, 11) }; + opt.offCodeSum = unsafe { scale_stats(opt.offCodeFreq, MAX_OFF, 11) }; + } + unsafe { set_base_prices(opt, opt_level) }; +} + +unsafe fn raw_literals_cost( + literals: *const u8, + lit_length: u32, + opt: &OptState, + opt_level: c_int, +) -> u32 { + if lit_length == 0 { + return 0; + } + if !compressed_literals(opt) { + return lit_length.wrapping_mul(8).wrapping_mul(BITCOST_MULTIPLIER); + } + if opt.priceType == ZOP_PREDEF { + return lit_length.wrapping_mul(6).wrapping_mul(BITCOST_MULTIPLIER); + } + let mut price = opt.litSumBasePrice.wrapping_mul(lit_length); + let max_price = opt.litSumBasePrice.wrapping_sub(BITCOST_MULTIPLIER); + for index in 0..lit_length as usize { + let literal = unsafe { *literals.add(index) } as usize; + let frequency = unsafe { *opt.litFreq.add(literal) }; + price = price.wrapping_sub(min(weight(frequency, opt_level), max_price)); + } + price +} + +unsafe fn lit_length_price(lit_length: u32, opt: &OptState, opt_level: c_int) -> u32 { + if opt.priceType == ZOP_PREDEF { + return weight(lit_length, opt_level); + } + if lit_length as usize == ZSTD_BLOCKSIZE_MAX { + return BITCOST_MULTIPLIER + + unsafe { lit_length_price((ZSTD_BLOCKSIZE_MAX - 1) as u32, opt, opt_level) }; + } + let code = ll_code(lit_length); + (LL_BITS[code] as u32 * BITCOST_MULTIPLIER) + opt.litLengthSumBasePrice + - weight(unsafe { *opt.litLengthFreq.add(code) }, opt_level) +} + +unsafe fn get_match_price( + off_base: u32, + match_length: u32, + opt: &OptState, + opt_level: c_int, +) -> u32 { + let off_code = ZSTD_highbit32(off_base); + let ml_base = match_length - MINMATCH as u32; + if opt.priceType == ZOP_PREDEF { + return weight(ml_base, opt_level) + (16u32 + off_code).wrapping_mul(BITCOST_MULTIPLIER); + } + let mut price = off_code * BITCOST_MULTIPLIER + opt.offCodeSumBasePrice + - weight( + unsafe { *opt.offCodeFreq.add(off_code as usize) }, + opt_level, + ); + if opt_level < 2 && off_code >= 20 { + price += (off_code - 19) * 2 * BITCOST_MULTIPLIER; + } + let code = ml_code(ml_base); + price += ML_BITS[code] as u32 * BITCOST_MULTIPLIER + opt.matchLengthSumBasePrice + - weight(unsafe { *opt.matchLengthFreq.add(code) }, opt_level); + price + BITCOST_MULTIPLIER / 5 +} + +unsafe fn update_stats( + opt: &mut OptState, + lit_length: u32, + literals: *const u8, + off_base: u32, + match_length: u32, +) { + if compressed_literals(opt) { + for index in 0..lit_length as usize { + let literal = unsafe { *literals.add(index) } as usize; + let frequency = unsafe { opt.litFreq.add(literal) }; + unsafe { *frequency = (*frequency).wrapping_add(ZSTD_LITFREQ_ADD) }; + } + opt.litSum = opt + .litSum + .wrapping_add(lit_length.wrapping_mul(ZSTD_LITFREQ_ADD)); + } + let ll = ll_code(lit_length); + unsafe { *opt.litLengthFreq.add(ll) = (*opt.litLengthFreq.add(ll)).wrapping_add(1) }; + opt.litLengthSum = opt.litLengthSum.wrapping_add(1); + let off = ZSTD_highbit32(off_base) as usize; + unsafe { *opt.offCodeFreq.add(off) = (*opt.offCodeFreq.add(off)).wrapping_add(1) }; + opt.offCodeSum = opt.offCodeSum.wrapping_add(1); + let ml = ml_code(match_length - MINMATCH as u32); + unsafe { *opt.matchLengthFreq.add(ml) = (*opt.matchLengthFreq.add(ml)).wrapping_add(1) }; + opt.matchLengthSum = opt.matchLengthSum.wrapping_add(1); +} + +#[inline] +fn offset_to_offbase(offset: u32) -> u32 { + offset.wrapping_add(OFFSET_OFFBASE) +} + +#[inline] +fn repcode_to_offbase(rep_code: u32) -> u32 { + rep_code +} + +#[inline] +fn new_rep(mut rep: [u32; ZSTD_REP_NUM], off_base: u32, ll0: bool) -> [u32; ZSTD_REP_NUM] { + if off_base > OFFSET_OFFBASE { + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = off_base - OFFSET_OFFBASE; + } else { + let rep_code = off_base - 1 + u32::from(ll0); + if rep_code != 0 { + let current = if rep_code == ZSTD_REP_NUM as u32 { + rep[0].wrapping_sub(1) + } else { + rep[rep_code as usize] + }; + if rep_code >= 2 { + rep[2] = rep[1]; + } + rep[1] = rep[0]; + rep[0] = current; + } + } + rep +} + +unsafe fn store_seq( + seq_store: *mut SeqStore_t, + lit_length: usize, + literals: *const u8, + off_base: u32, + match_length: usize, +) { + let store = unsafe { &mut *seq_store }; + let sequence = store.sequences; + let index = + ptr_diff(sequence.cast::(), store.sequencesStart.cast::()) / size_of::(); + if lit_length != 0 { + unsafe { ptr::copy_nonoverlapping(literals, store.lit, lit_length) }; + } + store.lit = store.lit.wrapping_add(lit_length); + if lit_length > u16::MAX as usize { + store.longLengthType = 1; + store.longLengthPos = index as u32; + } + unsafe { + (*sequence).litLength = lit_length as u16; + (*sequence).offBase = off_base; + } + let ml_base = match_length - MINMATCH; + if ml_base > u16::MAX as usize { + store.longLengthType = 2; + store.longLengthPos = index as u32; + } + unsafe { (*sequence).mlBase = ml_base as u16 }; + store.sequences = store.sequences.wrapping_add(1); +} + +unsafe fn skip_raw_seq_store_bytes(store: &mut RawSeqStore, nb_bytes: usize) { + let mut current = (store.posInSequence + nb_bytes) as u32; + while current != 0 && store.pos < store.size { + let sequence = unsafe { store.seq.add(store.pos).read() }; + let sequence_size = sequence.litLength.wrapping_add(sequence.matchLength); + if current >= sequence_size { + current -= sequence_size; + store.pos += 1; + } else { + store.posInSequence = current as usize; + break; + } + } + if current == 0 || store.pos == store.size { + store.posInSequence = 0; + } +} + +#[derive(Clone, Copy)] +struct OptLdm { + seq_store: RawSeqStore, + start_pos: u32, + end_pos: u32, + offset: u32, +} + +unsafe fn ldm_next_match(opt_ldm: &mut OptLdm, current_pos: u32, block_remaining: u32) { + if opt_ldm.seq_store.size == 0 || opt_ldm.seq_store.pos >= opt_ldm.seq_store.size { + opt_ldm.start_pos = u32::MAX; + opt_ldm.end_pos = u32::MAX; + return; + } + let sequence = unsafe { opt_ldm.seq_store.seq.add(opt_ldm.seq_store.pos).read() }; + let block_end = current_pos.wrapping_add(block_remaining); + let literals_remaining = + (sequence.litLength as usize).saturating_sub(opt_ldm.seq_store.posInSequence) as u32; + let match_remaining = if literals_remaining == 0 { + sequence + .matchLength + .wrapping_sub(opt_ldm.seq_store.posInSequence as u32 - sequence.litLength) + } else { + sequence.matchLength + }; + if literals_remaining >= block_remaining { + opt_ldm.start_pos = u32::MAX; + opt_ldm.end_pos = u32::MAX; + unsafe { skip_raw_seq_store_bytes(&mut opt_ldm.seq_store, block_remaining as usize) }; + return; + } + opt_ldm.start_pos = current_pos.wrapping_add(literals_remaining); + opt_ldm.end_pos = opt_ldm.start_pos.wrapping_add(match_remaining); + opt_ldm.offset = sequence.offset; + if opt_ldm.end_pos > block_end { + opt_ldm.end_pos = block_end; + unsafe { + skip_raw_seq_store_bytes( + &mut opt_ldm.seq_store, + block_end.wrapping_sub(current_pos) as usize, + ) + }; + } else { + unsafe { + skip_raw_seq_store_bytes( + &mut opt_ldm.seq_store, + (literals_remaining + match_remaining) as usize, + ) + }; + } +} + +unsafe fn ldm_maybe_add_match( + matches: *mut ZSTD_match_t, + nb_matches: &mut u32, + opt_ldm: &OptLdm, + current_pos: u32, + min_match: u32, +) { + let position_diff = current_pos.wrapping_sub(opt_ldm.start_pos); + let candidate_length = opt_ldm + .end_pos + .wrapping_sub(opt_ldm.start_pos) + .wrapping_sub(position_diff); + if current_pos < opt_ldm.start_pos + || current_pos >= opt_ldm.end_pos + || candidate_length < min_match + { + return; + } + if *nb_matches == 0 + || (candidate_length > unsafe { (*matches.add(*nb_matches as usize - 1)).len } + && *nb_matches < ZSTD_OPT_NUM) + { + let slot = unsafe { &mut *matches.add(*nb_matches as usize) }; + slot.len = candidate_length; + slot.off = offset_to_offbase(opt_ldm.offset); + *nb_matches += 1; + } +} + +unsafe fn ldm_process_match( + opt_ldm: &mut OptLdm, + matches: *mut ZSTD_match_t, + nb_matches: &mut u32, + current_pos: u32, + remaining: u32, + min_match: u32, +) { + if opt_ldm.seq_store.size == 0 || opt_ldm.seq_store.pos >= opt_ldm.seq_store.size { + return; + } + if current_pos >= opt_ldm.end_pos { + if current_pos > opt_ldm.end_pos { + unsafe { + skip_raw_seq_store_bytes( + &mut opt_ldm.seq_store, + current_pos.wrapping_sub(opt_ldm.end_pos) as usize, + ) + }; + } + unsafe { ldm_next_match(opt_ldm, current_pos, remaining) }; + } + unsafe { ldm_maybe_add_match(matches, nb_matches, opt_ldm, current_pos, min_match) }; +} + +unsafe fn insert_and_find_first_index_hash3( + state: &ZSTD_RustOptState, + next_to_update3: &mut u32, + ip: *const u8, +) -> u32 { + let target = index_from(state.base, ip); + let hash = unsafe { hash3_ptr(ip, state.hash_log3) }; + let mut index = *next_to_update3; + while index < target { + let position = state.base.wrapping_add(index as usize); + let position_hash = unsafe { hash3_ptr(position, state.hash_log3) }; + unsafe { table_set(state.hash_table3, position_hash, index) }; + index = index.wrapping_add(1); + } + *next_to_update3 = target; + unsafe { table_get(state.hash_table3, hash) } +} + +unsafe fn insert_bt_and_get_all_matches( + state: &ZSTD_RustOptState, + next_to_update3: &mut u32, + ip: *const u8, + input_limit: *const u8, + dict_mode: c_int, + rep: &[u32; ZSTD_REP_NUM], + ll0: u32, + length_to_beat: u32, + mls: u32, +) -> u32 { + let sufficient_len = min(state.target_length, ZSTD_OPT_NUM - 1); + let current = index_from(state.base, ip); + let min_match = if mls == 3 { 3 } else { 4 }; + let hash = unsafe { hash_ptr(ip, state.hash_log, mls) }; + let mut match_index = unsafe { table_get(state.hash_table, hash) }; + let bt_log = state.chain_log.wrapping_sub(1); + let bt_mask = (1u32 << bt_log).wrapping_sub(1); + let dict_limit = state.dict_limit; + let dict_end = state.dict_base.wrapping_add(dict_limit as usize); + let prefix_start = state.base.wrapping_add(dict_limit as usize); + let bt_low = current.saturating_sub(bt_mask); + let window_low = lowest_match_index(state, current); + let match_low = if window_low == 0 { 1 } else { window_low }; + let mut smaller = unsafe { state.chain_table.add(2 * (current & bt_mask) as usize) }; + let mut larger = unsafe { smaller.add(1) }; + let mut dummy = 0u32; + let mut match_end_index = current.wrapping_add(HASH_READ_SIZE as u32 + 1); + let mut common_smaller = 0usize; + let mut common_larger = 0usize; + let mut best_length = length_to_beat.wrapping_sub(1) as usize; + let mut match_count = 0u32; + let mut nb_compares = 1u32 << state.search_log; + + let dms = if dict_mode == DICT_MATCH_STATE { + debug_assert!(!state.dict_match_state.is_null()); + unsafe { &*state.dict_match_state } + } else { + state + }; + let dms_base = dms.base; + let dms_end = dms.next_src; + let dms_high_limit = if dict_mode == DICT_MATCH_STATE { + index_from(dms_base, dms_end) + } else { + 0 + }; + let dms_low_limit = if dict_mode == DICT_MATCH_STATE { + dms.low_limit + } else { + 0 + }; + let dms_index_delta = if dict_mode == DICT_MATCH_STATE { + window_low.wrapping_sub(dms_high_limit) + } else { + 0 + }; + let dms_hash_log = if dict_mode == DICT_MATCH_STATE { + dms.hash_log + } else { + state.hash_log + }; + let dms_bt_log = if dict_mode == DICT_MATCH_STATE { + dms.chain_log.wrapping_sub(1) + } else { + bt_log + }; + let dms_bt_mask = if dict_mode == DICT_MATCH_STATE { + (1u32 << dms_bt_log).wrapping_sub(1) + } else { + 0 + }; + let dms_bt_low = if dict_mode == DICT_MATCH_STATE + && dms_bt_mask < dms_high_limit.wrapping_sub(dms_low_limit) + { + dms_high_limit.wrapping_sub(dms_bt_mask) + } else { + dms_low_limit + }; + + for rep_code in ll0..(ZSTD_REP_NUM as u32 + ll0) { + let rep_offset = if rep_code == ZSTD_REP_NUM as u32 { + rep[0].wrapping_sub(1) + } else { + rep[rep_code as usize] + }; + let rep_index = current.wrapping_sub(rep_offset); + let mut rep_len = 0usize; + if rep_offset.wrapping_sub(1) < current.wrapping_sub(dict_limit) { + if rep_index >= window_low + && unsafe { read_min_match(ip, min_match) } + == unsafe { read_min_match(ip.wrapping_sub(rep_offset as usize), min_match) } + { + rep_len = unsafe { + count( + ip.wrapping_add(min_match as usize), + ip.wrapping_sub(rep_offset as usize) + .wrapping_add(min_match as usize), + input_limit, + ) + min_match as usize + }; + } + } else { + let rep_match = if dict_mode == DICT_MATCH_STATE { + dms_base.wrapping_add(rep_index.wrapping_sub(dms_index_delta) as usize) + } else { + state.dict_base.wrapping_add(rep_index as usize) + }; + if dict_mode == DICT_EXT + && rep_offset.wrapping_sub(1) < current.wrapping_sub(window_low) + && index_overlap_check(dict_limit, rep_index) + && unsafe { read_min_match(ip, min_match) } + == unsafe { read_min_match(rep_match, min_match) } + { + rep_len = unsafe { + count_2segments( + ip.wrapping_add(min_match as usize), + rep_match.wrapping_add(min_match as usize), + input_limit, + dict_end, + prefix_start, + ) + min_match as usize + }; + } + if dict_mode == DICT_MATCH_STATE + && rep_offset.wrapping_sub(1) + < current.wrapping_sub(dms_low_limit.wrapping_add(dms_index_delta)) + && index_overlap_check(dict_limit, rep_index) + && unsafe { read_min_match(ip, min_match) } + == unsafe { read_min_match(rep_match, min_match) } + { + rep_len = unsafe { + count_2segments( + ip.wrapping_add(min_match as usize), + rep_match.wrapping_add(min_match as usize), + input_limit, + dms_end, + prefix_start, + ) + min_match as usize + }; + } + } + if rep_len > best_length { + best_length = rep_len; + let slot = unsafe { &mut *state.opt_match_table().add(match_count as usize) }; + slot.off = repcode_to_offbase(rep_code - ll0 + 1); + slot.len = rep_len as u32; + match_count += 1; + if rep_len > sufficient_len as usize || ip.wrapping_add(rep_len) == input_limit { + return match_count; + } + } + } + + if mls == 3 && best_length < mls as usize { + let match_index3 = unsafe { insert_and_find_first_index_hash3(state, next_to_update3, ip) }; + if match_index3 >= match_low && current.wrapping_sub(match_index3) < (1 << 18) { + let match_ptr = if dict_mode == DICT_EXT && match_index3 < dict_limit { + state.dict_base.wrapping_add(match_index3 as usize) + } else { + state.base.wrapping_add(match_index3 as usize) + }; + let match_length = if dict_mode == DICT_EXT && match_index3 < dict_limit { + unsafe { count_2segments(ip, match_ptr, input_limit, dict_end, prefix_start) } + } else { + unsafe { count(ip, match_ptr, input_limit) } + }; + if match_length >= mls as usize { + let slot = unsafe { &mut *state.opt_match_table() }; + slot.off = offset_to_offbase(current.wrapping_sub(match_index3)); + slot.len = match_length as u32; + if match_length > sufficient_len as usize + || ip.wrapping_add(match_length) == input_limit + { + unsafe { *state.next_to_update = current.wrapping_add(1) }; + return 1; + } + match_count = 1; + best_length = match_length; + } + } + } + + unsafe { table_set(state.hash_table, hash, current) }; + while nb_compares != 0 && match_index >= match_low { + nb_compares -= 1; + let next_ptr = unsafe { state.chain_table.add(2 * (match_index & bt_mask) as usize) }; + let mut match_length = min(common_smaller, common_larger); + let mut match_ptr: *const u8; + if dict_mode != DICT_EXT || match_index.wrapping_add(match_length as u32) >= dict_limit { + match_ptr = state.base.wrapping_add(match_index as usize); + match_length += unsafe { + count( + ip.wrapping_add(match_length), + match_ptr.wrapping_add(match_length), + input_limit, + ) + }; + } else { + match_ptr = state.dict_base.wrapping_add(match_index as usize); + match_length += unsafe { + count_2segments( + ip.wrapping_add(match_length), + match_ptr.wrapping_add(match_length), + input_limit, + dict_end, + prefix_start, + ) + }; + if match_index.wrapping_add(match_length as u32) >= dict_limit { + match_ptr = state.base.wrapping_add(match_index as usize); + } + } + + if match_length > best_length { + if match_length > match_end_index.wrapping_sub(match_index) as usize { + match_end_index = match_index.wrapping_add(match_length as u32); + } + best_length = match_length; + let slot = unsafe { &mut *state.opt_match_table().add(match_count as usize) }; + slot.off = offset_to_offbase(current.wrapping_sub(match_index)); + slot.len = match_length as u32; + match_count += 1; + if match_length > ZSTD_OPT_NUM as usize || ip.wrapping_add(match_length) == input_limit + { + if dict_mode == DICT_MATCH_STATE { + nb_compares = 0; + } + break; + } + } + + let next_byte = unsafe { *match_ptr.add(match_length) }; + let input_byte = unsafe { *ip.add(match_length) }; + if next_byte < input_byte { + unsafe { *smaller = match_index }; + common_smaller = match_length; + if match_index <= bt_low { + smaller = &mut dummy; + break; + } + smaller = unsafe { next_ptr.add(1) }; + match_index = unsafe { *next_ptr.add(1) }; + } else { + unsafe { *larger = match_index }; + common_larger = match_length; + if match_index <= bt_low { + larger = &mut dummy; + break; + } + larger = next_ptr; + match_index = unsafe { *next_ptr }; + } + } + unsafe { + *smaller = 0; + *larger = 0; + } + + if dict_mode == DICT_MATCH_STATE && nb_compares != 0 { + let dict_hash = unsafe { hash_ptr(ip, dms_hash_log, mls) }; + let mut dict_match_index = unsafe { table_get(dms.hash_table, dict_hash) }; + common_smaller = 0; + common_larger = 0; + while nb_compares != 0 && dict_match_index > dms_low_limit { + nb_compares -= 1; + let next_ptr = unsafe { + dms.chain_table + .add(2 * (dict_match_index & dms_bt_mask) as usize) + }; + let mut match_length = min(common_smaller, common_larger); + let mut match_ptr = dms_base.wrapping_add(dict_match_index as usize); + match_length += unsafe { + count_2segments( + ip.wrapping_add(match_length), + match_ptr.wrapping_add(match_length), + input_limit, + dms_end, + prefix_start, + ) + }; + if dict_match_index.wrapping_add(match_length as u32) >= dms_high_limit { + match_ptr = state + .base + .wrapping_add(dict_match_index.wrapping_add(dms_index_delta) as usize); + } + if match_length > best_length { + let prefix_index = dict_match_index.wrapping_add(dms_index_delta); + if match_length > match_end_index.wrapping_sub(prefix_index) as usize { + match_end_index = prefix_index.wrapping_add(match_length as u32); + } + best_length = match_length; + let slot = unsafe { &mut *state.opt_match_table().add(match_count as usize) }; + slot.off = offset_to_offbase(current.wrapping_sub(prefix_index)); + slot.len = match_length as u32; + match_count += 1; + if match_length > ZSTD_OPT_NUM as usize + || ip.wrapping_add(match_length) == input_limit + { + break; + } + } + if dict_match_index <= dms_bt_low { + break; + } + if unsafe { *match_ptr.add(match_length) } < unsafe { *ip.add(match_length) } { + common_smaller = match_length; + dict_match_index = unsafe { *next_ptr.add(1) }; + } else { + common_larger = match_length; + dict_match_index = unsafe { *next_ptr }; + } + } + } + unsafe { *state.next_to_update = match_end_index.wrapping_sub(8) }; + match_count +} + +impl ZSTD_RustOptState { + #[inline] + unsafe fn opt_match_table(&self) -> *mut ZSTD_match_t { + unsafe { (*self.opt).matchTable } + } +} + +unsafe fn get_all_matches( + state: &mut ZSTD_RustOptState, + next_to_update3: &mut u32, + ip: *const u8, + input_limit: *const u8, + rep: &[u32; ZSTD_REP_NUM], + ll0: u32, + length_to_beat: u32, + dict_mode: c_int, + mls: u32, +) -> u32 { + debug_assert!((3..=6).contains(&mls)); + if (ip as usize) + < state + .base + .wrapping_add(unsafe { *state.next_to_update } as usize) as usize + { + return 0; + } + unsafe { + ZSTD_rust_opt_updateTreeInternal( + (state as *mut ZSTD_RustOptState).cast::(), + ip.cast::(), + input_limit.cast::(), + mls, + c_int::from(dict_mode == DICT_EXT), + ); + insert_bt_and_get_all_matches( + state, + next_to_update3, + ip, + input_limit, + dict_mode, + rep, + ll0, + length_to_beat, + mls, + ) + } +} + +unsafe fn resolve_shortest_path( + state: &mut ZSTD_RustOptState, + seq_store: *mut SeqStore_t, + rep: &mut [u32; ZSTD_REP_NUM], + istart: *const u8, + iend: *const u8, + anchor: &mut *const u8, + ip: &mut *const u8, + mut cur: u32, + last_pos: u32, + last_stretch: ZSTD_optimal_t, + opt_level: c_int, +) { + let opt_state = unsafe { &mut *state.opt }; + let opt = opt_state.priceTable; + if last_stretch.mlen == 0 { + *ip = ip.wrapping_add(last_pos as usize); + return; + } + + if last_stretch.litlen == 0 { + let previous = unsafe { (*opt.add(cur as usize)).rep }; + *rep = new_rep(previous, last_stretch.off, unsafe { + (*opt.add(cur as usize)).litlen == 0 + }); + } else { + *rep = last_stretch.rep; + cur = cur.wrapping_sub(last_stretch.litlen); + } + + let store_end = cur + 2; + let mut store_start = store_end; + let mut stretch_pos = cur; + unsafe { + *opt.add(store_end as usize) = last_stretch; + } + + loop { + let next_stretch = unsafe { *opt.add(stretch_pos as usize) }; + unsafe { (*opt.add(store_start as usize)).litlen = next_stretch.litlen }; + if next_stretch.mlen == 0 { + break; + } + store_start -= 1; + unsafe { *opt.add(store_start as usize) = next_stretch }; + stretch_pos = stretch_pos.wrapping_sub(next_stretch.litlen + next_stretch.mlen); + } + + let mut store_pos = store_start; + while store_pos <= store_end { + let current = unsafe { *opt.add(store_pos as usize) }; + let lit_length = current.litlen; + let match_length = current.mlen; + if match_length == 0 { + *ip = anchor.wrapping_add(lit_length as usize); + break; + } + unsafe { + update_stats(opt_state, lit_length, *anchor, current.off, match_length); + store_seq( + seq_store, + lit_length as usize, + *anchor, + current.off, + match_length as usize, + ); + } + *anchor = anchor.wrapping_add((lit_length + match_length) as usize); + *ip = *anchor; + store_pos += 1; + } + unsafe { set_base_prices(opt_state, opt_level) }; + let _ = istart; + let _ = iend; +} + +unsafe fn compress_block_opt_generic( + state: &mut ZSTD_RustOptState, + seq_store: *mut SeqStore_t, + reps: *mut u32, + src: *const u8, + src_size: usize, + opt_level: c_int, + dict_mode: c_int, +) -> usize { + let istart = src; + let mut ip = istart; + let mut anchor = istart; + let iend = src.wrapping_add(src_size); + let ilimit = if src_size >= HASH_READ_SIZE { + iend.wrapping_sub(HASH_READ_SIZE) + } else { + istart + }; + let prefix_start = state.base.wrapping_add(state.dict_limit as usize); + let sufficient_len = min(state.target_length, ZSTD_OPT_NUM - 1); + let min_match = if state.min_match == 3 { 3 } else { 4 }; + let mut next_to_update3 = unsafe { *state.next_to_update }; + let opt_state = unsafe { &mut *state.opt }; + let matches = opt_state.matchTable; + let opt = opt_state.priceTable; + let mut last_stretch = ZSTD_optimal_t::default(); + let mut opt_ldm = OptLdm { + seq_store: if state.ldm_seq_store.is_null() { + RawSeqStore::default() + } else { + unsafe { *state.ldm_seq_store } + }, + start_pos: 0, + end_pos: 0, + offset: 0, + }; + + unsafe { ldm_next_match(&mut opt_ldm, 0, src_size as u32) }; + unsafe { rescale_freqs(state, src, src_size, opt_level) }; + if ip == prefix_start { + ip = ip.wrapping_add(1); + } + + while (ip as usize) < ilimit as usize { + let mut last_pos; + let mut found_immediate = false; + let litlen = ptr_diff(ip, anchor) as u32; + let ll0 = u32::from(litlen == 0); + let mut nb_matches = unsafe { + get_all_matches( + state, + &mut next_to_update3, + ip, + iend, + &[*reps, *reps.add(1), *reps.add(2)], + ll0, + min_match, + dict_mode, + min_match, + ) + }; + unsafe { + ldm_process_match( + &mut opt_ldm, + matches, + &mut nb_matches, + ptr_diff(ip, istart) as u32, + ptr_diff(iend, ip) as u32, + min_match, + ) + }; + if nb_matches == 0 { + ip = ip.wrapping_add(1); + continue; + } + + unsafe { + (*opt).mlen = 0; + (*opt).litlen = litlen; + (*opt).price = lit_length_price(litlen, opt_state, opt_level) as c_int; + (*opt).rep = [*reps, *reps.add(1), *reps.add(2)]; + } + + let max_match = unsafe { *matches.add(nb_matches as usize - 1) }; + if max_match.len > sufficient_len { + last_stretch.litlen = 0; + last_stretch.mlen = max_match.len; + last_stretch.off = max_match.off; + last_pos = max_match.len; + found_immediate = true; + } else { + let mut pos = 1u32; + while pos < min_match { + unsafe { + (*opt.add(pos as usize)).price = ZSTD_MAX_PRICE; + (*opt.add(pos as usize)).mlen = 0; + (*opt.add(pos as usize)).litlen = litlen + pos; + } + pos += 1; + } + for match_index in 0..nb_matches as usize { + let candidate = unsafe { *matches.add(match_index) }; + while pos <= candidate.len { + let match_price = + unsafe { get_match_price(candidate.off, pos, opt_state, opt_level) } + as c_int; + let sequence_price = unsafe { (*opt).price } + match_price; + unsafe { + (*opt.add(pos as usize)).mlen = pos; + (*opt.add(pos as usize)).off = candidate.off; + (*opt.add(pos as usize)).litlen = 0; + (*opt.add(pos as usize)).price = + sequence_price + lit_length_price(0, opt_state, opt_level) as c_int; + } + pos += 1; + } + } + last_pos = pos - 1; + unsafe { (*opt.add(pos as usize)).price = ZSTD_MAX_PRICE }; + } + + if found_immediate { + let mut next_rep = unsafe { [*reps, *reps.add(1), *reps.add(2)] }; + unsafe { + resolve_shortest_path( + state, + seq_store, + &mut next_rep, + istart, + iend, + &mut anchor, + &mut ip, + 0, + last_pos, + last_stretch, + opt_level, + ) + }; + unsafe { + *reps = next_rep[0]; + *reps.add(1) = next_rep[1]; + *reps.add(2) = next_rep[2]; + } + continue; + } + + let mut cur = 1u32; + let mut resolved_early = false; + while cur <= last_pos { + let inr = ip.wrapping_add(cur as usize); + let literal_length = unsafe { (*opt.add((cur - 1) as usize)).litlen } + 1; + let price = unsafe { (*opt.add((cur - 1) as usize)).price } + + unsafe { + raw_literals_cost(ip.wrapping_add((cur - 1) as usize), 1, opt_state, opt_level) + } as c_int + + (unsafe { lit_length_price(literal_length, opt_state, opt_level) } as c_int + - unsafe { lit_length_price(literal_length - 1, opt_state, opt_level) } + as c_int); + if price <= unsafe { (*opt.add(cur as usize)).price } { + let previous_match = unsafe { *opt.add(cur as usize) }; + let previous = unsafe { *opt.add((cur - 1) as usize) }; + unsafe { *opt.add(cur as usize) = previous }; + unsafe { + (*opt.add(cur as usize)).litlen = literal_length; + (*opt.add(cur as usize)).price = price; + } + if opt_level >= 1 + && previous_match.litlen == 0 + && (unsafe { lit_length_price(1, opt_state, opt_level) } as c_int + - unsafe { lit_length_price(0, opt_state, opt_level) } as c_int) + < 0 + && (inr as usize) < (iend as usize) + { + let with_one = previous_match.price + + unsafe { raw_literals_cost(inr, 1, opt_state, opt_level) } as c_int + + (unsafe { lit_length_price(1, opt_state, opt_level) } as c_int + - unsafe { lit_length_price(0, opt_state, opt_level) } as c_int); + let with_more = price + + unsafe { raw_literals_cost(inr, 1, opt_state, opt_level) } as c_int + + (unsafe { lit_length_price(literal_length + 1, opt_state, opt_level) } + as c_int + - unsafe { lit_length_price(literal_length, opt_state, opt_level) } + as c_int); + if with_one < with_more + && with_one < unsafe { (*opt.add((cur + 1) as usize)).price } + { + let previous_index = cur - previous_match.mlen; + let reps = new_rep( + unsafe { (*opt.add(previous_index as usize)).rep }, + previous_match.off, + unsafe { (*opt.add(previous_index as usize)).litlen == 0 }, + ); + unsafe { + (*opt.add((cur + 1) as usize)).rep = reps; + (*opt.add((cur + 1) as usize)).mlen = previous_match.mlen; + (*opt.add((cur + 1) as usize)).off = previous_match.off; + (*opt.add((cur + 1) as usize)).litlen = 1; + (*opt.add((cur + 1) as usize)).price = with_one; + } + if last_pos < cur + 1 { + last_pos = cur + 1; + } + } + } + } + + let selected = unsafe { *opt.add(cur as usize) }; + if selected.litlen == 0 { + let previous_index = cur - selected.mlen; + let reps = new_rep( + unsafe { (*opt.add(previous_index as usize)).rep }, + selected.off, + unsafe { (*opt.add(previous_index as usize)).litlen == 0 }, + ); + unsafe { (*opt.add(cur as usize)).rep = reps }; + } + if (inr as usize) > (ilimit as usize) { + cur += 1; + continue; + } + if cur == last_pos { + break; + } + if opt_level == 0 + && unsafe { (*opt.add((cur + 1) as usize)).price } + <= unsafe { (*opt.add(cur as usize)).price } + (BITCOST_MULTIPLIER / 2) as c_int + { + cur += 1; + continue; + } + + let previous_price = unsafe { (*opt.add(cur as usize)).price }; + let base_price = + previous_price + unsafe { lit_length_price(0, opt_state, opt_level) } as c_int; + let current_rep = unsafe { (*opt.add(cur as usize)).rep }; + let mut candidates = unsafe { + get_all_matches( + state, + &mut next_to_update3, + inr, + iend, + ¤t_rep, + u32::from(selected.litlen == 0), + min_match, + dict_mode, + min_match, + ) + }; + unsafe { + ldm_process_match( + &mut opt_ldm, + matches, + &mut candidates, + ptr_diff(inr, istart) as u32, + ptr_diff(iend, inr) as u32, + min_match, + ) + }; + if candidates == 0 { + cur += 1; + continue; + } + let longest = unsafe { (*matches.add(candidates as usize - 1)).len }; + if longest > sufficient_len + || cur + longest >= ZSTD_OPT_NUM + || ptr_diff(iend, inr) <= longest as usize + { + last_stretch.mlen = longest; + last_stretch.off = unsafe { (*matches.add(candidates as usize - 1)).off }; + last_stretch.litlen = 0; + last_pos = cur + longest; + let mut path_rep = unsafe { [*reps, *reps.add(1), *reps.add(2)] }; + unsafe { + resolve_shortest_path( + state, + seq_store, + &mut path_rep, + istart, + iend, + &mut anchor, + &mut ip, + cur, + last_pos, + last_stretch, + opt_level, + ) + }; + unsafe { + *reps = path_rep[0]; + *reps.add(1) = path_rep[1]; + *reps.add(2) = path_rep[2]; + } + resolved_early = true; + cur = last_pos; + break; + } + + for match_index in 0..candidates as usize { + let candidate = unsafe { *matches.add(match_index) }; + let start_ml = if match_index > 0 { + unsafe { (*matches.add(match_index - 1)).len + 1 } + } else { + min_match + }; + let mut match_length = candidate.len; + loop { + let position = cur + match_length; + let match_price = unsafe { + get_match_price(candidate.off, match_length, opt_state, opt_level) + } as c_int; + let candidate_price = base_price + match_price; + if position > last_pos + || candidate_price < unsafe { (*opt.add(position as usize)).price } + { + while last_pos < position { + last_pos += 1; + unsafe { + (*opt.add(last_pos as usize)).price = ZSTD_MAX_PRICE; + (*opt.add(last_pos as usize)).litlen = 1; + } + } + unsafe { + (*opt.add(position as usize)).mlen = match_length; + (*opt.add(position as usize)).off = candidate.off; + (*opt.add(position as usize)).litlen = 0; + (*opt.add(position as usize)).price = candidate_price; + } + } else if opt_level == 0 { + break; + } + if match_length == start_ml { + break; + } + match_length -= 1; + } + } + unsafe { (*opt.add((last_pos + 1) as usize)).price = ZSTD_MAX_PRICE }; + cur += 1; + } + + if cur == last_pos && !resolved_early { + /* The previous loop reached its end without an early path. */ + last_stretch = unsafe { *opt.add(last_pos as usize) }; + let path_rep = unsafe { [*reps, *reps.add(1), *reps.add(2)] }; + let mut next_rep = path_rep; + let path_cur = last_pos.wrapping_sub(last_stretch.mlen); + unsafe { + resolve_shortest_path( + state, + seq_store, + &mut next_rep, + istart, + iend, + &mut anchor, + &mut ip, + path_cur, + last_pos, + last_stretch, + opt_level, + ) + }; + unsafe { + *reps = next_rep[0]; + *reps.add(1) = next_rep[1]; + *reps.add(2) = next_rep[2]; + } + } + } + ptr_diff(iend, anchor) +} + +unsafe fn init_stats_ultra( + state: &mut ZSTD_RustOptState, + seq_store: *mut SeqStore_t, + reps: *const u32, + src: *const u8, + src_size: usize, +) { + let mut temporary_rep = [unsafe { *reps }, unsafe { *reps.add(1) }, unsafe { + *reps.add(2) + }]; + unsafe { + compress_block_opt_generic( + state, + seq_store, + temporary_rep.as_mut_ptr(), + src, + src_size, + 2, + DICT_NO_DICT, + ); + } + let store = unsafe { &mut *seq_store }; + store.sequences = store.sequencesStart; + store.lit = store.litStart; + store.longLengthType = 0; + unsafe { + *state.window_base = (*state.window_base).wrapping_sub(src_size); + *state.window_dict_limit = (*state.window_dict_limit).wrapping_add(src_size as u32); + *state.window_low_limit = *state.window_dict_limit; + *state.next_to_update = *state.window_dict_limit; + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_opt_updateTree( + state: *mut ZSTD_RustOptState, + ip: *const c_void, + iend: *const c_void, + mls: u32, + dict_mode: c_int, +) { + let state = unsafe { &mut *state }; + unsafe { + ZSTD_rust_opt_updateTreeInternal( + (state as *mut ZSTD_RustOptState).cast::(), + ip, + iend, + mls, + c_int::from(dict_mode == DICT_EXT), + ) + }; +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_compressBlock_opt( + state: *mut ZSTD_RustOptState, + seq_store: *mut c_void, + reps: *mut u32, + src: *const c_void, + src_size: usize, + opt_level: c_int, + dict_mode: c_int, +) -> usize { + let state = unsafe { &mut *state }; + unsafe { + compress_block_opt_generic( + state, + seq_store.cast::(), + reps, + src.cast::(), + src_size, + opt_level, + dict_mode, + ) + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_initStats_ultra( + state: *mut ZSTD_RustOptState, + seq_store: *mut c_void, + reps: *const u32, + src: *const c_void, + src_size: usize, +) { + let state = unsafe { &mut *state }; + unsafe { + init_stats_ultra( + state, + seq_store.cast::(), + reps, + src.cast::(), + src_size, + ) + }; +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_compressBlock_btultra2( + state: *mut ZSTD_RustOptState, + seq_store: *mut c_void, + reps: *mut u32, + src: *const c_void, + src_size: usize, +) -> usize { + let state = unsafe { &mut *state }; + let store = unsafe { &*seq_store.cast::() }; + let current = index_from(state.base, src.cast::()); + let should_seed = unsafe { (*state.opt).litLengthSum == 0 } + && store.sequences == store.sequencesStart + && unsafe { *state.window_dict_limit == *state.window_low_limit } + && current == unsafe { *state.window_dict_limit } + && src_size > ZSTD_PREDEF_THRESHOLD; + if should_seed { + unsafe { init_stats_ultra(state, seq_store.cast(), reps, src.cast(), src_size) }; + } + unsafe { + compress_block_opt_generic( + state, + seq_store.cast::(), + reps, + src.cast::(), + src_size, + 2, + DICT_NO_DICT, + ) + } +}