diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index ea0007232..7d8f7e9f0 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -1,1464 +1,3 @@ -/* ****************************************************************** - * Huffman encoder, part of New Generation Entropy library - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * You can contact the author at : - * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy - * - Public forum : https://groups.google.com/forum/#!forum/lz4c - * - * This source code is licensed under both the BSD-style license (found in the - * LICENSE file in the root directory of this source tree) and the GPLv2 (found - * in the COPYING file in the root directory of this source tree). - * You may select, at your option, one of the above-listed licenses. -****************************************************************** */ - -/* ************************************************************** -* Compiler specifics -****************************************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - - -/* ************************************************************** -* Includes -****************************************************************/ -#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ -#include "../common/compiler.h" -#include "../common/bitstream.h" -#include "hist.h" -#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ -#include "../common/fse.h" /* header compression */ +/* Huffman compression is implemented in rust/src/huf_compress.rs. */ +#define FSE_STATIC_LINKING_ONLY #include "../common/huf.h" -#include "../common/error_private.h" -#include "../common/bits.h" /* ZSTD_highbit32 */ - - -/* ************************************************************** -* Error Management -****************************************************************/ -#define HUF_isError ERR_isError -#define HUF_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ - - -/* ************************************************************** -* Required declarations -****************************************************************/ -typedef struct nodeElt_s { - U32 count; - U16 parent; - BYTE byte; - BYTE nbBits; -} nodeElt; - - -/* ************************************************************** -* Debug Traces -****************************************************************/ - -#if DEBUGLEVEL >= 2 - -static size_t showU32(const U32* arr, size_t size) -{ - size_t u; - for (u=0; u= add) { - assert(add < align); - assert(((size_t)aligned & mask) == 0); - *workspaceSizePtr -= add; - return aligned; - } else { - *workspaceSizePtr = 0; - return NULL; - } -} - - -/* HUF_compressWeights() : - * Same as FSE_compress(), but dedicated to huff0's weights compression. - * The use case needs much less stack memory. - * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. - */ -#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 - -typedef struct { - FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; - U32 scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)]; - unsigned count[HUF_TABLELOG_MAX+1]; - S16 norm[HUF_TABLELOG_MAX+1]; -} HUF_CompressWeightsWksp; - -static size_t -HUF_compressWeights(void* dst, size_t dstSize, - const void* weightTable, size_t wtSize, - void* workspace, size_t workspaceSize) -{ - BYTE* const ostart = (BYTE*) dst; - BYTE* op = ostart; - BYTE* const oend = ostart + dstSize; - - unsigned maxSymbolValue = HUF_TABLELOG_MAX; - U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; - HUF_CompressWeightsWksp* wksp = (HUF_CompressWeightsWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32)); - - if (workspaceSize < sizeof(HUF_CompressWeightsWksp)) return ERROR(GENERIC); - - /* init conditions */ - if (wtSize <= 1) return 0; /* Not compressible */ - - /* Scan input and build symbol stats */ - { unsigned const maxCount = HIST_count_simple(wksp->count, &maxSymbolValue, weightTable, wtSize); /* never fails */ - if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */ - if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ - } - - tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); - CHECK_F( FSE_normalizeCount(wksp->norm, tableLog, wksp->count, wtSize, maxSymbolValue, /* useLowProbCount */ 0) ); - - /* Write table description header */ - { CHECK_V_F(hSize, FSE_writeNCount(op, (size_t)(oend-op), wksp->norm, maxSymbolValue, tableLog) ); - op += hSize; - } - - /* Compress */ - CHECK_F( FSE_buildCTable_wksp(wksp->CTable, wksp->norm, maxSymbolValue, tableLog, wksp->scratchBuffer, sizeof(wksp->scratchBuffer)) ); - { CHECK_V_F(cSize, FSE_compress_usingCTable(op, (size_t)(oend - op), weightTable, wtSize, wksp->CTable) ); - if (cSize == 0) return 0; /* not enough space for compressed data */ - op += cSize; - } - - return (size_t)(op-ostart); -} - -static size_t HUF_getNbBits(HUF_CElt elt) -{ - return elt & 0xFF; -} - -static size_t HUF_getNbBitsFast(HUF_CElt elt) -{ - return elt; -} - -static size_t HUF_getValue(HUF_CElt elt) -{ - return elt & ~(size_t)0xFF; -} - -static size_t HUF_getValueFast(HUF_CElt elt) -{ - return elt; -} - -static void HUF_setNbBits(HUF_CElt* elt, size_t nbBits) -{ - assert(nbBits <= HUF_TABLELOG_ABSOLUTEMAX); - *elt = nbBits; -} - -static void HUF_setValue(HUF_CElt* elt, size_t value) -{ - size_t const nbBits = HUF_getNbBits(*elt); - if (nbBits > 0) { - assert((value >> nbBits) == 0); - *elt |= value << (sizeof(HUF_CElt) * 8 - nbBits); - } -} - -HUF_CTableHeader HUF_readCTableHeader(HUF_CElt const* ctable) -{ - HUF_CTableHeader header; - ZSTD_memcpy(&header, ctable, sizeof(header)); - return header; -} - -static void HUF_writeCTableHeader(HUF_CElt* ctable, U32 tableLog, U32 maxSymbolValue) -{ - HUF_CTableHeader header; - HUF_STATIC_ASSERT(sizeof(ctable[0]) == sizeof(header)); - ZSTD_memset(&header, 0, sizeof(header)); - assert(tableLog < 256); - header.tableLog = (BYTE)tableLog; - assert(maxSymbolValue < 256); - header.maxSymbolValue = (BYTE)maxSymbolValue; - ZSTD_memcpy(ctable, &header, sizeof(header)); -} - -typedef struct { - HUF_CompressWeightsWksp wksp; - BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */ - BYTE huffWeight[HUF_SYMBOLVALUE_MAX]; -} HUF_WriteCTableWksp; - -size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, - const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, - void* workspace, size_t workspaceSize) -{ - HUF_CElt const* const ct = CTable + 1; - BYTE* op = (BYTE*)dst; - U32 n; - HUF_WriteCTableWksp* wksp = (HUF_WriteCTableWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32)); - - HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE >= sizeof(HUF_WriteCTableWksp)); - - assert(HUF_readCTableHeader(CTable).maxSymbolValue == maxSymbolValue); - assert(HUF_readCTableHeader(CTable).tableLog == huffLog); - - /* check conditions */ - if (workspaceSize < sizeof(HUF_WriteCTableWksp)) return ERROR(GENERIC); - if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); - - /* convert to weight */ - wksp->bitsToWeight[0] = 0; - for (n=1; nbitsToWeight[n] = (BYTE)(huffLog + 1 - n); - for (n=0; nhuffWeight[n] = wksp->bitsToWeight[HUF_getNbBits(ct[n])]; - - /* attempt weights compression by FSE */ - if (maxDstSize < 1) return ERROR(dstSize_tooSmall); - { CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, wksp->huffWeight, maxSymbolValue, &wksp->wksp, sizeof(wksp->wksp)) ); - if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */ - op[0] = (BYTE)hSize; - return hSize+1; - } } - - /* write raw values as 4-bits (max : 15) */ - if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ - if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ - op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1)); - wksp->huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ - for (n=0; nhuffWeight[n] << 4) + wksp->huffWeight[n+1]); - return ((maxSymbolValue+1)/2) + 1; -} - - -size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* hasZeroWeights) -{ - BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */ - U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ - U32 tableLog = 0; - U32 nbSymbols = 0; - HUF_CElt* const ct = CTable + 1; - - /* get symbol weights */ - CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize)); - *hasZeroWeights = (rankVal[0] > 0); - - /* check result */ - if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); - if (nbSymbols > *maxSymbolValuePtr+1) return ERROR(maxSymbolValue_tooSmall); - - *maxSymbolValuePtr = nbSymbols - 1; - - HUF_writeCTableHeader(CTable, tableLog, *maxSymbolValuePtr); - - /* Prepare base value per rank */ - { U32 n, nextRankStart = 0; - for (n=1; n<=tableLog; n++) { - U32 curr = nextRankStart; - nextRankStart += (rankVal[n] << (n-1)); - rankVal[n] = curr; - } } - - /* fill nbBits */ - { U32 n; for (n=0; nn=tableLog+1 */ - U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; - { U32 n; for (n=0; n0; n--) { /* start at n=tablelog <-> w=1 */ - valPerRank[n] = min; /* get starting value within each rank */ - min += nbPerRank[n]; - min >>= 1; - } } - /* assign value within rank, symbol order */ - { U32 n; for (n=0; n HUF_readCTableHeader(CTable).maxSymbolValue) - return 0; - return (U32)HUF_getNbBits(ct[symbolValue]); -} - - -/** - * HUF_setMaxHeight(): - * Try to enforce @targetNbBits on the Huffman tree described in @huffNode. - * - * It attempts to convert all nodes with nbBits > @targetNbBits - * to employ @targetNbBits instead. Then it adjusts the tree - * so that it remains a valid canonical Huffman tree. - * - * @pre The sum of the ranks of each symbol == 2^largestBits, - * where largestBits == huffNode[lastNonNull].nbBits. - * @post The sum of the ranks of each symbol == 2^largestBits, - * where largestBits is the return value (expected <= targetNbBits). - * - * @param huffNode The Huffman tree modified in place to enforce targetNbBits. - * It's presumed sorted, from most frequent to rarest symbol. - * @param lastNonNull The symbol with the lowest count in the Huffman tree. - * @param targetNbBits The allowed number of bits, which the Huffman tree - * may not respect. After this function the Huffman tree will - * respect targetNbBits. - * @return The maximum number of bits of the Huffman tree after adjustment. - */ -static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 targetNbBits) -{ - const U32 largestBits = huffNode[lastNonNull].nbBits; - /* early exit : no elt > targetNbBits, so the tree is already valid. */ - if (largestBits <= targetNbBits) return largestBits; - - DEBUGLOG(5, "HUF_setMaxHeight (targetNbBits = %u)", targetNbBits); - - /* there are several too large elements (at least >= 2) */ - { int totalCost = 0; - const U32 baseCost = 1 << (largestBits - targetNbBits); - int n = (int)lastNonNull; - - /* Adjust any ranks > targetNbBits to targetNbBits. - * Compute totalCost, which is how far the sum of the ranks is - * we are over 2^largestBits after adjust the offending ranks. - */ - while (huffNode[n].nbBits > targetNbBits) { - totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); - huffNode[n].nbBits = (BYTE)targetNbBits; - n--; - } - /* n stops at huffNode[n].nbBits <= targetNbBits */ - assert(huffNode[n].nbBits <= targetNbBits); - /* n end at index of smallest symbol using < targetNbBits */ - while (huffNode[n].nbBits == targetNbBits) --n; - - /* renorm totalCost from 2^largestBits to 2^targetNbBits - * note : totalCost is necessarily a multiple of baseCost */ - assert(((U32)totalCost & (baseCost - 1)) == 0); - totalCost >>= (largestBits - targetNbBits); - assert(totalCost > 0); - - /* repay normalized cost */ - { U32 const noSymbol = 0xF0F0F0F0; - U32 rankLast[HUF_TABLELOG_MAX+2]; - - /* Get pos of last (smallest = lowest cum. count) symbol per rank */ - ZSTD_memset(rankLast, 0xF0, sizeof(rankLast)); - { U32 currentNbBits = targetNbBits; - int pos; - for (pos=n ; pos >= 0; pos--) { - if (huffNode[pos].nbBits >= currentNbBits) continue; - currentNbBits = huffNode[pos].nbBits; /* < targetNbBits */ - rankLast[targetNbBits-currentNbBits] = (U32)pos; - } } - - while (totalCost > 0) { - /* Try to reduce the next power of 2 above totalCost because we - * gain back half the rank. - */ - U32 nBitsToDecrease = ZSTD_highbit32((U32)totalCost) + 1; - for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { - U32 const highPos = rankLast[nBitsToDecrease]; - U32 const lowPos = rankLast[nBitsToDecrease-1]; - if (highPos == noSymbol) continue; - /* Decrease highPos if no symbols of lowPos or if it is - * not cheaper to remove 2 lowPos than highPos. - */ - if (lowPos == noSymbol) break; - { U32 const highTotal = huffNode[highPos].count; - U32 const lowTotal = 2 * huffNode[lowPos].count; - if (highTotal <= lowTotal) break; - } } - /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ - assert(rankLast[nBitsToDecrease] != noSymbol || nBitsToDecrease == 1); - /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ - while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) - nBitsToDecrease++; - assert(rankLast[nBitsToDecrease] != noSymbol); - /* Increase the number of bits to gain back half the rank cost. */ - totalCost -= 1 << (nBitsToDecrease-1); - huffNode[rankLast[nBitsToDecrease]].nbBits++; - - /* Fix up the new rank. - * If the new rank was empty, this symbol is now its smallest. - * Otherwise, this symbol will be the largest in the new rank so no adjustment. - */ - if (rankLast[nBitsToDecrease-1] == noSymbol) - rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; - /* Fix up the old rank. - * If the symbol was at position 0, meaning it was the highest weight symbol in the tree, - * it must be the only symbol in its rank, so the old rank now has no symbols. - * Otherwise, since the Huffman nodes are sorted by count, the previous position is now - * the smallest node in the rank. If the previous position belongs to a different rank, - * then the rank is now empty. - */ - if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ - rankLast[nBitsToDecrease] = noSymbol; - else { - rankLast[nBitsToDecrease]--; - if (huffNode[rankLast[nBitsToDecrease]].nbBits != targetNbBits-nBitsToDecrease) - rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ - } - } /* while (totalCost > 0) */ - - /* If we've removed too much weight, then we have to add it back. - * To avoid overshooting again, we only adjust the smallest rank. - * We take the largest nodes from the lowest rank 0 and move them - * to rank 1. There's guaranteed to be enough rank 0 symbols because - * TODO. - */ - while (totalCost < 0) { /* Sometimes, cost correction overshoot */ - /* special case : no rank 1 symbol (using targetNbBits-1); - * let's create one from largest rank 0 (using targetNbBits). - */ - if (rankLast[1] == noSymbol) { - while (huffNode[n].nbBits == targetNbBits) n--; - huffNode[n+1].nbBits--; - assert(n >= 0); - rankLast[1] = (U32)(n+1); - totalCost++; - continue; - } - huffNode[ rankLast[1] + 1 ].nbBits--; - rankLast[1]++; - totalCost ++; - } - } /* repay normalized cost */ - } /* there are several too large elements (at least >= 2) */ - - return targetNbBits; -} - -typedef struct { - U16 base; - U16 curr; -} rankPos; - -typedef nodeElt huffNodeTable[2 * (HUF_SYMBOLVALUE_MAX + 1)]; - -/* Number of buckets available for HUF_sort() */ -#define RANK_POSITION_TABLE_SIZE 192 - -typedef struct { - huffNodeTable huffNodeTbl; - rankPos rankPosition[RANK_POSITION_TABLE_SIZE]; -} HUF_buildCTable_wksp_tables; - -/* RANK_POSITION_DISTINCT_COUNT_CUTOFF == Cutoff point in HUF_sort() buckets for which we use log2 bucketing. - * Strategy is to use as many buckets as possible for representing distinct - * counts while using the remainder to represent all "large" counts. - * - * To satisfy this requirement for 192 buckets, we can do the following: - * Let buckets 0-166 represent distinct counts of [0, 166] - * Let buckets 166 to 192 represent all remaining counts up to RANK_POSITION_MAX_COUNT_LOG using log2 bucketing. - */ -#define RANK_POSITION_MAX_COUNT_LOG 32 -#define RANK_POSITION_LOG_BUCKETS_BEGIN ((RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */) -#define RANK_POSITION_DISTINCT_COUNT_CUTOFF (RANK_POSITION_LOG_BUCKETS_BEGIN + ZSTD_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */) - -/* Return the appropriate bucket index for a given count. See definition of - * RANK_POSITION_DISTINCT_COUNT_CUTOFF for explanation of bucketing strategy. - */ -static U32 HUF_getIndex(U32 const count) { - return (count < RANK_POSITION_DISTINCT_COUNT_CUTOFF) - ? count - : ZSTD_highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN; -} - -/* Helper swap function for HUF_quickSortPartition() */ -static void HUF_swapNodes(nodeElt* a, nodeElt* b) { - nodeElt tmp = *a; - *a = *b; - *b = tmp; -} - -/* Returns 0 if the huffNode array is not sorted by descending count */ -MEM_STATIC int HUF_isSorted(nodeElt huffNode[], U32 const maxSymbolValue1) { - U32 i; - for (i = 1; i < maxSymbolValue1; ++i) { - if (huffNode[i].count > huffNode[i-1].count) { - return 0; - } - } - return 1; -} - -/* Insertion sort by descending order */ -HINT_INLINE void HUF_insertionSort(nodeElt huffNode[], int const low, int const high) { - int i; - int const size = high-low+1; - huffNode += low; - for (i = 1; i < size; ++i) { - nodeElt const key = huffNode[i]; - int j = i - 1; - while (j >= 0 && huffNode[j].count < key.count) { - huffNode[j + 1] = huffNode[j]; - j--; - } - huffNode[j + 1] = key; - } -} - -/* Pivot helper function for quicksort. */ -static int HUF_quickSortPartition(nodeElt arr[], int const low, int const high) { - /* Simply select rightmost element as pivot. "Better" selectors like - * median-of-three don't experimentally appear to have any benefit. - */ - U32 const pivot = arr[high].count; - int i = low - 1; - int j = low; - for ( ; j < high; j++) { - if (arr[j].count > pivot) { - i++; - HUF_swapNodes(&arr[i], &arr[j]); - } - } - HUF_swapNodes(&arr[i + 1], &arr[high]); - return i + 1; -} - -/* Classic quicksort by descending with partially iterative calls - * to reduce worst case callstack size. - */ -static void HUF_simpleQuickSort(nodeElt arr[], int low, int high) { - int const kInsertionSortThreshold = 8; - if (high - low < kInsertionSortThreshold) { - HUF_insertionSort(arr, low, high); - return; - } - while (low < high) { - int const idx = HUF_quickSortPartition(arr, low, high); - if (idx - low < high - idx) { - HUF_simpleQuickSort(arr, low, idx - 1); - low = idx + 1; - } else { - HUF_simpleQuickSort(arr, idx + 1, high); - high = idx - 1; - } - } -} - -/** - * HUF_sort(): - * Sorts the symbols [0, maxSymbolValue] by count[symbol] in decreasing order. - * This is a typical bucket sorting strategy that uses either quicksort or insertion sort to sort each bucket. - * - * @param[out] huffNode Sorted symbols by decreasing count. Only members `.count` and `.byte` are filled. - * Must have (maxSymbolValue + 1) entries. - * @param[in] count Histogram of the symbols. - * @param[in] maxSymbolValue Maximum symbol value. - * @param rankPosition This is a scratch workspace. Must have RANK_POSITION_TABLE_SIZE entries. - */ -static void HUF_sort(nodeElt huffNode[], const unsigned count[], U32 const maxSymbolValue, rankPos rankPosition[]) { - U32 n; - U32 const maxSymbolValue1 = maxSymbolValue+1; - - /* Compute base and set curr to base. - * For symbol s let lowerRank = HUF_getIndex(count[n]) and rank = lowerRank + 1. - * See HUF_getIndex to see bucketing strategy. - * We attribute each symbol to lowerRank's base value, because we want to know where - * each rank begins in the output, so for rank R we want to count ranks R+1 and above. - */ - ZSTD_memset(rankPosition, 0, sizeof(*rankPosition) * RANK_POSITION_TABLE_SIZE); - for (n = 0; n < maxSymbolValue1; ++n) { - U32 lowerRank = HUF_getIndex(count[n]); - assert(lowerRank < RANK_POSITION_TABLE_SIZE - 1); - rankPosition[lowerRank].base++; - } - - assert(rankPosition[RANK_POSITION_TABLE_SIZE - 1].base == 0); - /* Set up the rankPosition table */ - for (n = RANK_POSITION_TABLE_SIZE - 1; n > 0; --n) { - rankPosition[n-1].base += rankPosition[n].base; - rankPosition[n-1].curr = rankPosition[n-1].base; - } - - /* Insert each symbol into their appropriate bucket, setting up rankPosition table. */ - for (n = 0; n < maxSymbolValue1; ++n) { - U32 const c = count[n]; - U32 const r = HUF_getIndex(c) + 1; - U32 const pos = rankPosition[r].curr++; - assert(pos < maxSymbolValue1); - huffNode[pos].count = c; - huffNode[pos].byte = (BYTE)n; - } - - /* Sort each bucket. */ - for (n = RANK_POSITION_DISTINCT_COUNT_CUTOFF; n < RANK_POSITION_TABLE_SIZE - 1; ++n) { - int const bucketSize = rankPosition[n].curr - rankPosition[n].base; - U32 const bucketStartIdx = rankPosition[n].base; - if (bucketSize > 1) { - assert(bucketStartIdx < maxSymbolValue1); - HUF_simpleQuickSort(huffNode + bucketStartIdx, 0, bucketSize-1); - } - } - - assert(HUF_isSorted(huffNode, maxSymbolValue1)); -} - - -/** HUF_buildCTable_wksp() : - * Same as HUF_buildCTable(), but using externally allocated scratch buffer. - * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as sizeof(HUF_buildCTable_wksp_tables). - */ -#define STARTNODE (HUF_SYMBOLVALUE_MAX+1) - -/* HUF_buildTree(): - * Takes the huffNode array sorted by HUF_sort() and builds an unlimited-depth Huffman tree. - * - * @param huffNode The array sorted by HUF_sort(). Builds the Huffman tree in this array. - * @param maxSymbolValue The maximum symbol value. - * @return The smallest node in the Huffman tree (by count). - */ -static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue) -{ - nodeElt* const huffNode0 = huffNode - 1; - int nonNullRank; - int lowS, lowN; - int nodeNb = STARTNODE; - int n, nodeRoot; - DEBUGLOG(5, "HUF_buildTree (alphabet size = %u)", maxSymbolValue + 1); - /* init for parents */ - nonNullRank = (int)maxSymbolValue; - while(huffNode[nonNullRank].count == 0) nonNullRank--; - lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; - huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; - huffNode[lowS].parent = huffNode[lowS-1].parent = (U16)nodeNb; - nodeNb++; lowS-=2; - for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); - huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */ - - /* create parents */ - while (nodeNb <= nodeRoot) { - int const n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; - int const n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; - huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; - huffNode[n1].parent = huffNode[n2].parent = (U16)nodeNb; - nodeNb++; - } - - /* distribute weights (unlimited tree height) */ - huffNode[nodeRoot].nbBits = 0; - for (n=nodeRoot-1; n>=STARTNODE; n--) - huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; - for (n=0; n<=nonNullRank; n++) - huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; - - DEBUGLOG(6, "Initial distribution of bits completed (%zu sorted symbols)", showHNodeBits(huffNode, maxSymbolValue+1)); - - return nonNullRank; -} - -/** - * HUF_buildCTableFromTree(): - * Build the CTable given the Huffman tree in huffNode. - * - * @param[out] CTable The output Huffman CTable. - * @param huffNode The Huffman tree. - * @param nonNullRank The last and smallest node in the Huffman tree. - * @param maxSymbolValue The maximum symbol value. - * @param maxNbBits The exact maximum number of bits used in the Huffman tree. - */ -static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, int nonNullRank, U32 maxSymbolValue, U32 maxNbBits) -{ - HUF_CElt* const ct = CTable + 1; - /* fill result into ctable (val, nbBits) */ - int n; - U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; - U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; - int const alphabetSize = (int)(maxSymbolValue + 1); - for (n=0; n<=nonNullRank; n++) - nbPerRank[huffNode[n].nbBits]++; - /* determine starting value per rank */ - { U16 min = 0; - for (n=(int)maxNbBits; n>0; n--) { - valPerRank[n] = min; /* get starting value within each rank */ - min += nbPerRank[n]; - min >>= 1; - } } - for (n=0; nhuffNodeTbl; - nodeElt* const huffNode = huffNode0+1; - int nonNullRank; - - HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE == sizeof(HUF_buildCTable_wksp_tables)); - - DEBUGLOG(5, "HUF_buildCTable_wksp (alphabet size = %u)", maxSymbolValue+1); - - /* safety checks */ - if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) - return ERROR(workSpace_tooSmall); - if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; - if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) - return ERROR(maxSymbolValue_tooLarge); - ZSTD_memset(huffNode0, 0, sizeof(huffNodeTable)); - - /* sort, decreasing order */ - HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition); - DEBUGLOG(6, "sorted symbols completed (%zu symbols)", showHNodeSymbols(huffNode, maxSymbolValue+1)); - - /* build tree */ - nonNullRank = HUF_buildTree(huffNode, maxSymbolValue); - - /* determine and enforce maxTableLog */ - maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits); - if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ - - HUF_buildCTableFromTree(CTable, huffNode, nonNullRank, maxSymbolValue, maxNbBits); - - return maxNbBits; -} - -size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) -{ - HUF_CElt const* ct = CTable + 1; - size_t nbBits = 0; - int s; - for (s = 0; s <= (int)maxSymbolValue; ++s) { - nbBits += HUF_getNbBits(ct[s]) * count[s]; - } - return nbBits >> 3; -} - -int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { - HUF_CTableHeader header = HUF_readCTableHeader(CTable); - HUF_CElt const* ct = CTable + 1; - int bad = 0; - int s; - - assert(header.tableLog <= HUF_TABLELOG_ABSOLUTEMAX); - - if (header.maxSymbolValue < maxSymbolValue) - return 0; - - for (s = 0; s <= (int)maxSymbolValue; ++s) { - bad |= (count[s] != 0) & (HUF_getNbBits(ct[s]) == 0); - } - return !bad; -} - -size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } - -/** HUF_CStream_t: - * Huffman uses its own BIT_CStream_t implementation. - * There are three major differences from BIT_CStream_t: - * 1. HUF_addBits() takes a HUF_CElt (size_t) which is - * the pair (nbBits, value) in the format: - * format: - * - Bits [0, 4) = nbBits - * - Bits [4, 64 - nbBits) = 0 - * - Bits [64 - nbBits, 64) = value - * 2. The bitContainer is built from the upper bits and - * right shifted. E.g. to add a new value of N bits - * you right shift the bitContainer by N, then or in - * the new value into the N upper bits. - * 3. The bitstream has two bit containers. You can add - * bits to the second container and merge them into - * the first container. - */ - -#define HUF_BITS_IN_CONTAINER (sizeof(size_t) * 8) - -typedef struct { - size_t bitContainer[2]; - size_t bitPos[2]; - - BYTE* startPtr; - BYTE* ptr; - BYTE* endPtr; -} HUF_CStream_t; - -/**! HUF_initCStream(): - * Initializes the bitstream. - * @returns 0 or an error code. - */ -static size_t HUF_initCStream(HUF_CStream_t* bitC, - void* startPtr, size_t dstCapacity) -{ - ZSTD_memset(bitC, 0, sizeof(*bitC)); - bitC->startPtr = (BYTE*)startPtr; - bitC->ptr = bitC->startPtr; - bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer[0]); - if (dstCapacity <= sizeof(bitC->bitContainer[0])) return ERROR(dstSize_tooSmall); - return 0; -} - -/*! HUF_addBits(): - * Adds the symbol stored in HUF_CElt elt to the bitstream. - * - * @param elt The element we're adding. This is a (nbBits, value) pair. - * See the HUF_CStream_t docs for the format. - * @param idx Insert into the bitstream at this idx. - * @param kFast This is a template parameter. If the bitstream is guaranteed - * to have at least 4 unused bits after this call it may be 1, - * otherwise it must be 0. HUF_addBits() is faster when fast is set. - */ -FORCE_INLINE_TEMPLATE void HUF_addBits(HUF_CStream_t* bitC, HUF_CElt elt, int idx, int kFast) -{ - assert(idx <= 1); - assert(HUF_getNbBits(elt) <= HUF_TABLELOG_ABSOLUTEMAX); - /* This is efficient on x86-64 with BMI2 because shrx - * only reads the low 6 bits of the register. The compiler - * knows this and elides the mask. When fast is set, - * every operation can use the same value loaded from elt. - */ - bitC->bitContainer[idx] >>= HUF_getNbBits(elt); - bitC->bitContainer[idx] |= kFast ? HUF_getValueFast(elt) : HUF_getValue(elt); - /* We only read the low 8 bits of bitC->bitPos[idx] so it - * doesn't matter that the high bits have noise from the value. - */ - bitC->bitPos[idx] += HUF_getNbBitsFast(elt); - assert((bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER); - /* The last 4-bits of elt are dirty if fast is set, - * so we must not be overwriting bits that have already been - * inserted into the bit container. - */ -#if DEBUGLEVEL >= 1 - { - size_t const nbBits = HUF_getNbBits(elt); - size_t const dirtyBits = nbBits == 0 ? 0 : ZSTD_highbit32((U32)nbBits) + 1; - (void)dirtyBits; - /* Middle bits are 0. */ - assert(((elt >> dirtyBits) << (dirtyBits + nbBits)) == 0); - /* We didn't overwrite any bits in the bit container. */ - assert(!kFast || (bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER); - (void)dirtyBits; - } -#endif -} - -FORCE_INLINE_TEMPLATE void HUF_zeroIndex1(HUF_CStream_t* bitC) -{ - bitC->bitContainer[1] = 0; - bitC->bitPos[1] = 0; -} - -/*! HUF_mergeIndex1() : - * Merges the bit container @ index 1 into the bit container @ index 0 - * and zeros the bit container @ index 1. - */ -FORCE_INLINE_TEMPLATE void HUF_mergeIndex1(HUF_CStream_t* bitC) -{ - assert((bitC->bitPos[1] & 0xFF) < HUF_BITS_IN_CONTAINER); - bitC->bitContainer[0] >>= (bitC->bitPos[1] & 0xFF); - bitC->bitContainer[0] |= bitC->bitContainer[1]; - bitC->bitPos[0] += bitC->bitPos[1]; - assert((bitC->bitPos[0] & 0xFF) <= HUF_BITS_IN_CONTAINER); -} - -/*! HUF_flushBits() : -* Flushes the bits in the bit container @ index 0. -* -* @post bitPos will be < 8. -* @param kFast If kFast is set then we must know a-priori that -* the bit container will not overflow. -*/ -FORCE_INLINE_TEMPLATE void HUF_flushBits(HUF_CStream_t* bitC, int kFast) -{ - /* The upper bits of bitPos are noisy, so we must mask by 0xFF. */ - size_t const nbBits = bitC->bitPos[0] & 0xFF; - size_t const nbBytes = nbBits >> 3; - /* The top nbBits bits of bitContainer are the ones we need. */ - size_t const bitContainer = bitC->bitContainer[0] >> (HUF_BITS_IN_CONTAINER - nbBits); - /* Mask bitPos to account for the bytes we consumed. */ - bitC->bitPos[0] &= 7; - assert(nbBits > 0); - assert(nbBits <= sizeof(bitC->bitContainer[0]) * 8); - assert(bitC->ptr <= bitC->endPtr); - MEM_writeLEST(bitC->ptr, bitContainer); - bitC->ptr += nbBytes; - assert(!kFast || bitC->ptr <= bitC->endPtr); - if (!kFast && bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; - /* bitContainer doesn't need to be modified because the leftover - * bits are already the top bitPos bits. And we don't care about - * noise in the lower values. - */ -} - -/*! HUF_endMark() - * @returns The Huffman stream end mark: A 1-bit value = 1. - */ -static HUF_CElt HUF_endMark(void) -{ - HUF_CElt endMark; - HUF_setNbBits(&endMark, 1); - HUF_setValue(&endMark, 1); - return endMark; -} - -/*! HUF_closeCStream() : - * @return Size of CStream, in bytes, - * or 0 if it could not fit into dstBuffer */ -static size_t HUF_closeCStream(HUF_CStream_t* bitC) -{ - HUF_addBits(bitC, HUF_endMark(), /* idx */ 0, /* kFast */ 0); - HUF_flushBits(bitC, /* kFast */ 0); - { - size_t const nbBits = bitC->bitPos[0] & 0xFF; - if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ - return (size_t)(bitC->ptr - bitC->startPtr) + (nbBits > 0); - } -} - -FORCE_INLINE_TEMPLATE void -HUF_encodeSymbol(HUF_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable, int idx, int fast) -{ - HUF_addBits(bitCPtr, CTable[symbol], idx, fast); -} - -FORCE_INLINE_TEMPLATE void -HUF_compress1X_usingCTable_internal_body_loop(HUF_CStream_t* bitC, - const BYTE* ip, size_t srcSize, - const HUF_CElt* ct, - int kUnroll, int kFastFlush, int kLastFast) -{ - /* Join to kUnroll */ - int n = (int)srcSize; - int rem = n % kUnroll; - if (rem > 0) { - for (; rem > 0; --rem) { - HUF_encodeSymbol(bitC, ip[--n], ct, 0, /* fast */ 0); - } - HUF_flushBits(bitC, kFastFlush); - } - assert(n % kUnroll == 0); - - /* Join to 2 * kUnroll */ - if (n % (2 * kUnroll)) { - int u; - for (u = 1; u < kUnroll; ++u) { - HUF_encodeSymbol(bitC, ip[n - u], ct, 0, 1); - } - HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, 0, kLastFast); - HUF_flushBits(bitC, kFastFlush); - n -= kUnroll; - } - assert(n % (2 * kUnroll) == 0); - - for (; n>0; n-= 2 * kUnroll) { - /* Encode kUnroll symbols into the bitstream @ index 0. */ - int u; - for (u = 1; u < kUnroll; ++u) { - HUF_encodeSymbol(bitC, ip[n - u], ct, /* idx */ 0, /* fast */ 1); - } - HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, /* idx */ 0, /* fast */ kLastFast); - HUF_flushBits(bitC, kFastFlush); - /* Encode kUnroll symbols into the bitstream @ index 1. - * This allows us to start filling the bit container - * without any data dependencies. - */ - HUF_zeroIndex1(bitC); - for (u = 1; u < kUnroll; ++u) { - HUF_encodeSymbol(bitC, ip[n - kUnroll - u], ct, /* idx */ 1, /* fast */ 1); - } - HUF_encodeSymbol(bitC, ip[n - kUnroll - kUnroll], ct, /* idx */ 1, /* fast */ kLastFast); - /* Merge bitstream @ index 1 into the bitstream @ index 0 */ - HUF_mergeIndex1(bitC); - HUF_flushBits(bitC, kFastFlush); - } - assert(n == 0); - -} - -/** - * Returns a tight upper bound on the output space needed by Huffman - * with 8 bytes buffer to handle over-writes. If the output is at least - * this large we don't need to do bounds checks during Huffman encoding. - */ -static size_t HUF_tightCompressBound(size_t srcSize, size_t tableLog) -{ - return ((srcSize * tableLog) >> 3) + 8; -} - - -FORCE_INLINE_TEMPLATE size_t -HUF_compress1X_usingCTable_internal_body(void* dst, size_t dstSize, - const void* src, size_t srcSize, - const HUF_CElt* CTable) -{ - U32 const tableLog = HUF_readCTableHeader(CTable).tableLog; - HUF_CElt const* ct = CTable + 1; - const BYTE* ip = (const BYTE*) src; - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstSize; - HUF_CStream_t bitC; - - /* init */ - if (dstSize < 8) return 0; /* not enough space to compress */ - { BYTE* op = ostart; - size_t const initErr = HUF_initCStream(&bitC, op, (size_t)(oend-op)); - if (HUF_isError(initErr)) return 0; } - - if (dstSize < HUF_tightCompressBound(srcSize, (size_t)tableLog) || tableLog > 11) - HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ MEM_32bits() ? 2 : 4, /* kFast */ 0, /* kLastFast */ 0); - else { - if (MEM_32bits()) { - switch (tableLog) { - case 11: - HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 0); - break; - case 10: ZSTD_FALLTHROUGH; - case 9: ZSTD_FALLTHROUGH; - case 8: - HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 1); - break; - case 7: ZSTD_FALLTHROUGH; - default: - HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 3, /* kFastFlush */ 1, /* kLastFast */ 1); - break; - } - } else { - switch (tableLog) { - case 11: - HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 0); - break; - case 10: - HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 1); - break; - case 9: - HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 6, /* kFastFlush */ 1, /* kLastFast */ 0); - break; - case 8: - HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 7, /* kFastFlush */ 1, /* kLastFast */ 0); - break; - case 7: - HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 8, /* kFastFlush */ 1, /* kLastFast */ 0); - break; - case 6: ZSTD_FALLTHROUGH; - default: - HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 9, /* kFastFlush */ 1, /* kLastFast */ 1); - break; - } - } - } - assert(bitC.ptr <= bitC.endPtr); - - return HUF_closeCStream(&bitC); -} - -#if DYNAMIC_BMI2 - -static BMI2_TARGET_ATTRIBUTE size_t -HUF_compress1X_usingCTable_internal_bmi2(void* dst, size_t dstSize, - const void* src, size_t srcSize, - const HUF_CElt* CTable) -{ - return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); -} - -static size_t -HUF_compress1X_usingCTable_internal_default(void* dst, size_t dstSize, - const void* src, size_t srcSize, - const HUF_CElt* CTable) -{ - return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); -} - -static size_t -HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, - const void* src, size_t srcSize, - const HUF_CElt* CTable, const int flags) -{ - if (flags & HUF_flags_bmi2) { - return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable); - } - return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable); -} - -#else - -static size_t -HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, - const void* src, size_t srcSize, - const HUF_CElt* CTable, const int flags) -{ - (void)flags; - return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); -} - -#endif - -size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags) -{ - return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags); -} - -static size_t -HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, - const void* src, size_t srcSize, - const HUF_CElt* CTable, int flags) -{ - size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ - const BYTE* ip = (const BYTE*) src; - const BYTE* const iend = ip + srcSize; - BYTE* const ostart = (BYTE*) dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - - if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ - if (srcSize < 12) return 0; /* no saving possible : too small input */ - op += 6; /* jumpTable */ - - assert(op <= oend); - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); - if (cSize == 0 || cSize > 65535) return 0; - MEM_writeLE16(ostart, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - assert(op <= oend); - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); - if (cSize == 0 || cSize > 65535) return 0; - MEM_writeLE16(ostart+2, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - assert(op <= oend); - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); - if (cSize == 0 || cSize > 65535) return 0; - MEM_writeLE16(ostart+4, (U16)cSize); - op += cSize; - } - - ip += segmentSize; - assert(op <= oend); - assert(ip <= iend); - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, flags) ); - if (cSize == 0 || cSize > 65535) return 0; - op += cSize; - } - - return (size_t)(op-ostart); -} - -size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags) -{ - return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags); -} - -typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e; - -static size_t HUF_compressCTable_internal( - BYTE* const ostart, BYTE* op, BYTE* const oend, - const void* src, size_t srcSize, - HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int flags) -{ - size_t const cSize = (nbStreams==HUF_singleStream) ? - HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags) : - HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags); - if (HUF_isError(cSize)) { return cSize; } - if (cSize==0) { return 0; } /* uncompressible */ - op += cSize; - /* check compressibility */ - assert(op >= ostart); - if ((size_t)(op-ostart) >= srcSize-1) { return 0; } - return (size_t)(op-ostart); -} - -typedef struct { - unsigned count[HUF_SYMBOLVALUE_MAX + 1]; - HUF_CElt CTable[HUF_CTABLE_SIZE_ST(HUF_SYMBOLVALUE_MAX)]; - union { - HUF_buildCTable_wksp_tables buildCTable_wksp; - HUF_WriteCTableWksp writeCTable_wksp; - U32 hist_wksp[HIST_WKSP_SIZE_U32]; - } wksps; -} HUF_compress_tables_t; - -#define SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE 4096 -#define SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO 10 /* Must be >= 2 */ - -unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue) -{ - unsigned cardinality = 0; - unsigned i; - - for (i = 0; i < maxSymbolValue + 1; i++) { - if (count[i] != 0) cardinality += 1; - } - - return cardinality; -} - -unsigned HUF_minTableLog(unsigned symbolCardinality) -{ - U32 minBitsSymbols = ZSTD_highbit32(symbolCardinality) + 1; - return minBitsSymbols; -} - -unsigned HUF_optimalTableLog( - unsigned maxTableLog, - size_t srcSize, - unsigned maxSymbolValue, - void* workSpace, size_t wkspSize, - HUF_CElt* table, - const unsigned* count, - int flags) -{ - assert(srcSize > 1); /* Not supported, RLE should be used instead */ - assert(wkspSize >= sizeof(HUF_buildCTable_wksp_tables)); - - if (!(flags & HUF_flags_optimalDepth)) { - /* cheap evaluation, based on FSE */ - return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); - } - - { BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp); - size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp); - size_t hSize, newSize; - const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue); - const unsigned minTableLog = HUF_minTableLog(symbolCardinality); - size_t optSize = ((size_t) ~0) - 1; - unsigned optLog = maxTableLog, optLogGuess; - - DEBUGLOG(6, "HUF_optimalTableLog: probing huf depth (srcSize=%zu)", srcSize); - - /* Search until size increases */ - for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) { - DEBUGLOG(7, "checking for huffLog=%u", optLogGuess); - - { size_t maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize); - if (ERR_isError(maxBits)) continue; - - if (maxBits < optLogGuess && optLogGuess > minTableLog) break; - - hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize); - } - - if (ERR_isError(hSize)) continue; - - newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize; - - if (newSize > optSize + 1) { - break; - } - - if (newSize < optSize) { - optSize = newSize; - optLog = optLogGuess; - } - } - assert(optLog <= HUF_TABLELOG_MAX); - return optLog; - } -} - -/* HUF_compress_internal() : - * `workSpace_align4` must be aligned on 4-bytes boundaries, - * and occupies the same space as a table of HUF_WORKSPACE_SIZE_U64 unsigned */ -static size_t -HUF_compress_internal (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - HUF_nbStreams_e nbStreams, - void* workSpace, size_t wkspSize, - HUF_CElt* oldHufTable, HUF_repeat* repeat, int flags) -{ - HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t)); - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart; - - DEBUGLOG(5, "HUF_compress_internal (srcSize=%zu)", srcSize); - HUF_STATIC_ASSERT(sizeof(*table) + HUF_WORKSPACE_MAX_ALIGNMENT <= HUF_WORKSPACE_SIZE); - - /* checks & inits */ - if (wkspSize < sizeof(*table)) return ERROR(workSpace_tooSmall); - if (!srcSize) return 0; /* Uncompressed */ - if (!dstSize) return 0; /* cannot fit anything within dst budget */ - if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */ - if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); - if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); - if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX; - if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; - - /* Heuristic : If old table is valid, use it for small inputs */ - if ((flags & HUF_flags_preferRepeat) && repeat && *repeat == HUF_repeat_valid) { - return HUF_compressCTable_internal(ostart, op, oend, - src, srcSize, - nbStreams, oldHufTable, flags); - } - - /* If uncompressible data is suspected, do a smaller sampling first */ - DEBUG_STATIC_ASSERT(SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO >= 2); - if ((flags & HUF_flags_suspectUncompressible) && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) { - size_t largestTotal = 0; - DEBUGLOG(5, "input suspected incompressible : sampling to check"); - { unsigned maxSymbolValueBegin = maxSymbolValue; - CHECK_V_F(largestBegin, HIST_count_simple (table->count, &maxSymbolValueBegin, (const BYTE*)src, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) ); - largestTotal += largestBegin; - } - { unsigned maxSymbolValueEnd = maxSymbolValue; - CHECK_V_F(largestEnd, HIST_count_simple (table->count, &maxSymbolValueEnd, (const BYTE*)src + srcSize - SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) ); - largestTotal += largestEnd; - } - if (largestTotal <= ((2 * SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) >> 7)+4) return 0; /* heuristic : probably not compressible enough */ - } - - /* Scan input and build symbol stats */ - { CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, table->wksps.hist_wksp, sizeof(table->wksps.hist_wksp)) ); - if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ - if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */ - } - DEBUGLOG(6, "histogram detail completed (%zu symbols)", showU32(table->count, maxSymbolValue+1)); - - /* Check validity of previous table */ - if ( repeat - && *repeat == HUF_repeat_check - && !HUF_validateCTable(oldHufTable, table->count, maxSymbolValue)) { - *repeat = HUF_repeat_none; - } - /* Heuristic : use existing table for small inputs */ - if ((flags & HUF_flags_preferRepeat) && repeat && *repeat != HUF_repeat_none) { - return HUF_compressCTable_internal(ostart, op, oend, - src, srcSize, - nbStreams, oldHufTable, flags); - } - - /* Build Huffman Tree */ - huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, flags); - { size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count, - maxSymbolValue, huffLog, - &table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp)); - CHECK_F(maxBits); - huffLog = (U32)maxBits; - DEBUGLOG(6, "bit distribution completed (%zu symbols)", showCTableBits(table->CTable + 1, maxSymbolValue+1)); - } - - /* Write table description header */ - { CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, table->CTable, maxSymbolValue, huffLog, - &table->wksps.writeCTable_wksp, sizeof(table->wksps.writeCTable_wksp)) ); - /* Check if using previous huffman table is beneficial */ - if (repeat && *repeat != HUF_repeat_none) { - size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, table->count, maxSymbolValue); - size_t const newSize = HUF_estimateCompressedSize(table->CTable, table->count, maxSymbolValue); - if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) { - return HUF_compressCTable_internal(ostart, op, oend, - src, srcSize, - nbStreams, oldHufTable, flags); - } } - - /* Use the new huffman table */ - if (hSize + 12ul >= srcSize) { return 0; } - op += hSize; - if (repeat) { *repeat = HUF_repeat_none; } - if (oldHufTable) - ZSTD_memcpy(oldHufTable, table->CTable, sizeof(table->CTable)); /* Save new table */ - } - return HUF_compressCTable_internal(ostart, op, oend, - src, srcSize, - nbStreams, table->CTable, flags); -} - -size_t HUF_compress1X_repeat (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - void* workSpace, size_t wkspSize, - HUF_CElt* hufTable, HUF_repeat* repeat, int flags) -{ - DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize); - return HUF_compress_internal(dst, dstSize, src, srcSize, - maxSymbolValue, huffLog, HUF_singleStream, - workSpace, wkspSize, hufTable, - repeat, flags); -} - -/* HUF_compress4X_repeat(): - * compress input using 4 streams. - * consider skipping quickly - * reuse an existing huffman compression table */ -size_t HUF_compress4X_repeat (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - void* workSpace, size_t wkspSize, - HUF_CElt* hufTable, HUF_repeat* repeat, int flags) -{ - DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize); - return HUF_compress_internal(dst, dstSize, src, srcSize, - maxSymbolValue, huffLog, HUF_fourStreams, - workSpace, wkspSize, - hufTable, repeat, flags); -} diff --git a/rust/README.md b/rust/README.md index a2a5b6785..6f4e5cdc1 100644 --- a/rust/README.md +++ b/rust/README.md @@ -22,6 +22,8 @@ zstd ABI: - `fse_decompress` builds FSE decoding tables and decodes FSE streams. - `fse_compress` normalizes counts, writes FSE headers, builds compression tables, and encodes FSE streams. + - `huf_compress` builds Huffman compression tables, writes table headers, + and encodes one- and four-stream Huffman payloads. - `huf_decompress` builds Huffman decoding tables and decodes X1 and X2 Huffman streams. - Compression primitives diff --git a/rust/src/huf_compress.rs b/rust/src/huf_compress.rs new file mode 100644 index 000000000..97b36ab01 --- /dev/null +++ b/rust/src/huf_compress.rs @@ -0,0 +1,1546 @@ +#![allow(non_snake_case)] + +//! Huffman compression tables, headers, and single/four-stream encoders. +//! +//! This is the Rust implementation of `lib/compress/huf_compress.c`. The +//! externally supplied workspaces remain part of the C ABI: their alignment +//! and size checks are preserved even though the translation keeps temporary +//! state in fixed Rust arrays. + +use crate::bits::ZSTD_highbit32; +use crate::entropy_common::{HUF_readStats, HUF_TABLELOG_MAX}; +use crate::errors::{ERR_isError, ZstdErrorCode, ERROR}; +use crate::fse_compress::{ + FSE_buildCTable_wksp, FSE_compress_usingCTable, FSE_normalizeCount, FSE_optimalTableLog, + FSE_optimalTableLog_internal, FSE_writeNCount, +}; +use crate::hist::{HIST_count_simple, HIST_count_wksp, HIST_WKSP_SIZE_U32}; +use crate::mem::MEM_writeLE16; +use std::mem::size_of; +use std::os::raw::{c_int, c_short, c_uint, c_void}; + +const HUF_BLOCKSIZE_MAX: usize = 128 * 1024; +const HUF_TABLELOG_DEFAULT: u32 = 11; +const HUF_TABLELOG_ABSOLUTEMAX: u32 = 12; +const HUF_SYMBOLVALUE_MAX: u32 = 255; +const HUF_CTABLE_SIZE_ST: usize = HUF_SYMBOLVALUE_MAX as usize + 2; +const HUF_CTABLE_WORKSPACE_SIZE: usize = (4 * (HUF_SYMBOLVALUE_MAX as usize + 1) + 192) * 4; +const HUF_WORKSPACE_SIZE: usize = (8 << 10) + 512; +const HUF_WORKSPACE_MAX_ALIGNMENT: usize = 8; +const MAX_FSE_TABLELOG_FOR_HUFF_HEADER: u32 = 6; +const HUF_FLAGS_OPTIMAL_DEPTH: c_int = 1 << 1; +const HUF_FLAGS_PREFER_REPEAT: c_int = 1 << 2; +const HUF_FLAGS_SUSPECT_UNCOMPRESSIBLE: c_int = 1 << 3; +const HUF_REPEAT_NONE: c_int = 0; +const HUF_REPEAT_CHECK: c_int = 1; +const HUF_REPEAT_VALID: c_int = 2; +const SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE: usize = 4096; +const SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO: usize = 10; +const STARTNODE: usize = HUF_SYMBOLVALUE_MAX as usize + 1; +const RANK_POSITION_TABLE_SIZE: usize = 192; +const RANK_POSITION_LOG_BUCKETS_BEGIN: u32 = (RANK_POSITION_TABLE_SIZE as u32 - 1) - 32 - 1; +const RANK_POSITION_DISTINCT_COUNT_CUTOFF: u32 = RANK_POSITION_LOG_BUCKETS_BEGIN + 7; + +#[repr(C)] +#[derive(Copy, Clone)] +pub struct HUF_CTableHeader { + pub tableLog: u8, + pub maxSymbolValue: u8, + pub unused: [u8; size_of::() - 2], +} + +#[repr(C)] +#[derive(Copy, Clone, Default)] +struct NodeElt { + count: u32, + parent: u16, + byte: u8, + nb_bits: u8, +} + +#[repr(C)] +#[derive(Copy, Clone, Default)] +struct RankPos { + base: u16, + curr: u16, +} + +#[inline] +unsafe fn ctable_read(table: *const usize, index: usize) -> usize { + table.add(index).read_unaligned() +} + +#[inline] +unsafe fn ctable_write(table: *mut usize, index: usize, value: usize) { + table.add(index).write_unaligned(value); +} + +#[inline] +unsafe fn huf_read_ctable_header(table: *const usize) -> HUF_CTableHeader { + let mut header = HUF_CTableHeader { + tableLog: 0, + maxSymbolValue: 0, + unused: [0; size_of::() - 2], + }; + std::ptr::copy_nonoverlapping( + table.cast::(), + (&mut header as *mut HUF_CTableHeader).cast::(), + size_of::(), + ); + header +} + +#[inline] +unsafe fn huf_write_ctable_header(table: *mut usize, table_log: u32, max_symbol_value: u32) { + let header = HUF_CTableHeader { + tableLog: table_log as u8, + maxSymbolValue: max_symbol_value as u8, + unused: [0; size_of::() - 2], + }; + std::ptr::copy_nonoverlapping( + (&header as *const HUF_CTableHeader).cast::(), + table.cast::(), + size_of::(), + ); +} + +#[inline] +fn huf_get_nb_bits(elt: usize) -> usize { + elt & 0xff +} + +#[inline] +fn huf_get_value(elt: usize) -> usize { + elt & !0xffusize +} + +#[inline] +unsafe fn huf_set_nb_bits(table: *mut usize, index: usize, nb_bits: usize) { + debug_assert!(nb_bits <= HUF_TABLELOG_ABSOLUTEMAX as usize); + ctable_write(table, index, nb_bits); +} + +#[inline] +unsafe fn huf_set_value(table: *mut usize, index: usize, value: usize) { + let elt = ctable_read(table, index); + let nb_bits = huf_get_nb_bits(elt); + if nb_bits != 0 { + debug_assert!(value >> nb_bits == 0); + ctable_write( + table, + index, + elt | (value << (usize::BITS as usize - nb_bits)), + ); + } +} + +#[inline] +fn huf_aligned_workspace_size( + workspace: *mut c_void, + workspace_size: usize, + align: usize, +) -> usize { + debug_assert!(align.is_power_of_two()); + debug_assert!(align <= HUF_WORKSPACE_MAX_ALIGNMENT); + let add = (align - ((workspace as usize) & (align - 1))) & (align - 1); + workspace_size.saturating_sub(add) +} + +#[no_mangle] +pub unsafe extern "C" fn HUF_readCTableHeader(ctable: *const usize) -> HUF_CTableHeader { + huf_read_ctable_header(ctable) +} + +unsafe fn huf_compress_weights( + dst: *mut u8, + dst_size: usize, + weight_table: *const u8, + wt_size: usize, +) -> usize { + if wt_size <= 1 { + return 0; + } + + let mut count = [0u32; HUF_TABLELOG_MAX as usize + 1]; + let mut max_symbol_value = HUF_TABLELOG_MAX; + let max_count = HIST_count_simple( + count.as_mut_ptr(), + &mut max_symbol_value, + weight_table.cast::(), + wt_size, + ); + if max_count as usize == wt_size { + return 1; + } + if max_count == 1 { + return 0; + } + + let table_log = + FSE_optimalTableLog(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, wt_size, max_symbol_value); + let mut norm = [0i16; HUF_TABLELOG_MAX as usize + 1]; + let result = FSE_normalizeCount( + norm.as_mut_ptr().cast::(), + table_log, + count.as_ptr(), + wt_size, + max_symbol_value, + 0, + ); + if ERR_isError(result) { + return result; + } + + let h_size = FSE_writeNCount( + dst.cast::(), + dst_size, + norm.as_ptr().cast::(), + max_symbol_value, + table_log, + ); + if ERR_isError(h_size) { + return h_size; + } + if h_size > dst_size { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + + // FSE_CTABLE_SIZE_U32(6, HUF_TABLELOG_MAX) and the corresponding build + // workspace macro from fse.h. + let mut ctable = [0u32; 59]; + let mut scratch = [0u32; 41]; + let result = FSE_buildCTable_wksp( + ctable.as_mut_ptr(), + norm.as_ptr().cast::(), + max_symbol_value, + table_log, + scratch.as_mut_ptr().cast::(), + size_of::<[u32; 41]>(), + ); + if ERR_isError(result) { + return result; + } + + let c_size = FSE_compress_usingCTable( + dst.add(h_size).cast::(), + dst_size - h_size, + weight_table.cast::(), + wt_size, + ctable.as_ptr(), + ); + if ERR_isError(c_size) { + return c_size; + } + if c_size == 0 { + return 0; + } + h_size + c_size +} + +#[no_mangle] +pub unsafe extern "C" fn HUF_writeCTable_wksp( + dst: *mut c_void, + max_dst_size: usize, + ctable: *const usize, + max_symbol_value: c_uint, + huff_log: c_uint, + workspace: *mut c_void, + workspace_size: usize, +) -> usize { + let header = huf_read_ctable_header(ctable); + debug_assert_eq!(u32::from(header.maxSymbolValue), max_symbol_value); + debug_assert_eq!(u32::from(header.tableLog), huff_log); + + if huf_aligned_workspace_size(workspace, workspace_size, size_of::()) + < HUF_CTABLE_WORKSPACE_SIZE + { + return ERROR(ZstdErrorCode::Generic); + } + if max_symbol_value > HUF_SYMBOLVALUE_MAX { + return ERROR(ZstdErrorCode::MaxSymbolValueTooLarge); + } + if max_dst_size < 1 { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + + let mut bits_to_weight = [0u8; HUF_TABLELOG_MAX as usize + 1]; + for (n, weight) in bits_to_weight + .iter_mut() + .enumerate() + .take(huff_log as usize + 1) + .skip(1) + { + *weight = (huff_log as usize + 1 - n) as u8; + } + let mut huff_weight = [0u8; HUF_SYMBOLVALUE_MAX as usize + 1]; + for (n, weight) in huff_weight + .iter_mut() + .enumerate() + .take(max_symbol_value as usize) + { + let bits = huf_get_nb_bits(ctable_read(ctable, n + 1)); + if bits > huff_log as usize { + return ERROR(ZstdErrorCode::TableLogTooLarge); + } + *weight = bits_to_weight[bits]; + } + + let output = dst.cast::(); + let h_size = huf_compress_weights( + output.add(1), + max_dst_size - 1, + huff_weight.as_ptr(), + max_symbol_value as usize, + ); + if ERR_isError(h_size) { + return h_size; + } + if h_size > 1 && h_size < max_symbol_value as usize / 2 { + *output = h_size as u8; + return h_size + 1; + } + + if max_symbol_value > 256 - 128 { + return ERROR(ZstdErrorCode::Generic); + } + let raw_size = (max_symbol_value as usize).div_ceil(2) + 1; + if raw_size > max_dst_size { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + *output = (128 + (max_symbol_value - 1)) as u8; + huff_weight[max_symbol_value as usize] = 0; + for n in (0..max_symbol_value as usize).step_by(2) { + *output.add(n / 2 + 1) = (huff_weight[n] << 4).wrapping_add(huff_weight[n + 1]); + } + raw_size +} + +#[no_mangle] +pub unsafe extern "C" fn HUF_readCTable( + ctable: *mut usize, + max_symbol_value_ptr: *mut c_uint, + src: *const c_void, + src_size: usize, + has_zero_weights: *mut c_uint, +) -> usize { + let mut huff_weight = [0u8; HUF_SYMBOLVALUE_MAX as usize + 1]; + let mut rank_val = [0u32; HUF_TABLELOG_ABSOLUTEMAX as usize + 1]; + let mut table_log = 0u32; + let mut nb_symbols = 0u32; + let read_size = HUF_readStats( + huff_weight.as_mut_ptr(), + huff_weight.len(), + rank_val.as_mut_ptr(), + &mut nb_symbols, + &mut table_log, + src, + src_size, + ); + if ERR_isError(read_size) { + return read_size; + } + *has_zero_weights = u32::from(rank_val[0] > 0); + if table_log > HUF_TABLELOG_MAX { + return ERROR(ZstdErrorCode::TableLogTooLarge); + } + if nb_symbols > *max_symbol_value_ptr + 1 { + return ERROR(ZstdErrorCode::MaxSymbolValueTooSmall); + } + + *max_symbol_value_ptr = nb_symbols - 1; + huf_write_ctable_header(ctable, table_log, *max_symbol_value_ptr); + + let mut next_rank_start = 0u32; + for (rank, rank_value) in rank_val + .iter_mut() + .enumerate() + .take(table_log as usize + 1) + .skip(1) + { + let current = next_rank_start; + next_rank_start = next_rank_start.wrapping_add(*rank_value << (rank - 1)); + *rank_value = current; + } + + for (symbol, weight) in huff_weight + .iter() + .copied() + .take(nb_symbols as usize) + .enumerate() + { + let weight = weight as u32; + let bits = if weight == 0 { + 0 + } else { + table_log + 1 - weight + }; + huf_set_nb_bits(ctable, symbol + 1, bits as usize); + } + + let mut nb_per_rank = [0u16; HUF_TABLELOG_MAX as usize + 2]; + let mut val_per_rank = [0u16; HUF_TABLELOG_MAX as usize + 2]; + for symbol in 0..nb_symbols as usize { + let bits = huf_get_nb_bits(ctable_read(ctable, symbol + 1)); + nb_per_rank[bits] = nb_per_rank[bits].wrapping_add(1); + } + val_per_rank[table_log as usize + 1] = 0; + let mut min = 0u16; + for rank in (1..=table_log as usize).rev() { + val_per_rank[rank] = min; + min = min.wrapping_add(nb_per_rank[rank]); + min >>= 1; + } + for symbol in 0..nb_symbols as usize { + let bits = huf_get_nb_bits(ctable_read(ctable, symbol + 1)); + let value = val_per_rank[bits]; + huf_set_value(ctable, symbol + 1, value as usize); + val_per_rank[bits] = val_per_rank[bits].wrapping_add(1); + } + read_size +} + +#[no_mangle] +pub unsafe extern "C" fn HUF_getNbBitsFromCTable(ctable: *const usize, symbol_value: u32) -> u32 { + if symbol_value > HUF_SYMBOLVALUE_MAX { + return 0; + } + if symbol_value > u32::from(huf_read_ctable_header(ctable).maxSymbolValue) { + return 0; + } + huf_get_nb_bits(ctable_read(ctable, symbol_value as usize + 1)) as u32 +} + +#[inline] +fn huf_get_index(count: u32) -> usize { + if count < RANK_POSITION_DISTINCT_COUNT_CUTOFF { + count as usize + } else { + (ZSTD_highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN) as usize + } +} + +fn huf_insertion_sort(nodes: &mut [NodeElt], low: i32, high: i32) { + let size = high - low + 1; + if size <= 1 { + return; + } + for offset in 1..size { + let key = nodes[(low + offset) as usize]; + let mut index = offset - 1; + while index >= 0 && nodes[(low + index) as usize].count < key.count { + nodes[(low + index + 1) as usize] = nodes[(low + index) as usize]; + index -= 1; + } + nodes[(low + index + 1) as usize] = key; + } +} + +fn huf_quick_sort_partition(nodes: &mut [NodeElt], low: i32, high: i32) -> i32 { + let pivot = nodes[high as usize].count; + let mut i = low - 1; + for j in low..high { + if nodes[j as usize].count > pivot { + i += 1; + nodes.swap(i as usize, j as usize); + } + } + nodes.swap((i + 1) as usize, high as usize); + i + 1 +} + +fn huf_simple_quick_sort(nodes: &mut [NodeElt], mut low: i32, mut high: i32) { + const INSERTION_SORT_THRESHOLD: i32 = 8; + if high - low < INSERTION_SORT_THRESHOLD { + huf_insertion_sort(nodes, low, high); + return; + } + while low < high { + let index = huf_quick_sort_partition(nodes, low, high); + if index - low < high - index { + huf_simple_quick_sort(nodes, low, index - 1); + low = index + 1; + } else { + huf_simple_quick_sort(nodes, index + 1, high); + high = index - 1; + } + } +} + +fn huf_sort( + nodes: &mut [NodeElt; 2 * (HUF_SYMBOLVALUE_MAX as usize + 1)], + count: *const c_uint, + max_symbol_value: u32, + rank_position: &mut [RankPos; RANK_POSITION_TABLE_SIZE], +) { + let max_symbol_value1 = max_symbol_value as usize + 1; + *rank_position = [RankPos::default(); RANK_POSITION_TABLE_SIZE]; + for symbol in 0..max_symbol_value1 { + let lower_rank = huf_get_index(unsafe { *count.add(symbol) }); + debug_assert!(lower_rank < RANK_POSITION_TABLE_SIZE - 1); + rank_position[lower_rank].base = rank_position[lower_rank].base.wrapping_add(1); + } + + for rank in (1..RANK_POSITION_TABLE_SIZE).rev() { + rank_position[rank - 1].base = rank_position[rank - 1] + .base + .wrapping_add(rank_position[rank].base); + rank_position[rank - 1].curr = rank_position[rank - 1].base; + } + + for symbol in 0..max_symbol_value1 { + let current_count = unsafe { *count.add(symbol) }; + let rank = huf_get_index(current_count) + 1; + let position = rank_position[rank].curr as usize; + debug_assert!(position < max_symbol_value1); + nodes[position + 1].count = current_count; + nodes[position + 1].byte = symbol as u8; + rank_position[rank].curr = rank_position[rank].curr.wrapping_add(1); + } + + for position in rank_position + .iter() + .take(RANK_POSITION_TABLE_SIZE - 1) + .skip(RANK_POSITION_DISTINCT_COUNT_CUTOFF as usize) + { + let bucket_size = i32::from(position.curr) - i32::from(position.base); + let bucket_start = position.base as usize; + if bucket_size > 1 { + huf_simple_quick_sort( + &mut nodes[bucket_start + 1..bucket_start + 1 + bucket_size as usize], + 0, + bucket_size - 1, + ); + } + } +} + +#[inline] +fn huff_node_index(index: usize) -> usize { + index + 1 +} + +#[inline] +fn huff_node_index_signed(index: i32) -> usize { + debug_assert!(index >= -1); + (index + 1) as usize +} + +fn huf_build_tree( + nodes: &mut [NodeElt; 2 * (HUF_SYMBOLVALUE_MAX as usize + 1)], + max_symbol_value: u32, +) -> Option { + let mut non_null_rank = max_symbol_value as i32; + while non_null_rank >= 0 && nodes[huff_node_index(non_null_rank as usize)].count == 0 { + non_null_rank -= 1; + } + if non_null_rank < 1 { + return None; + } + + let mut low_s = non_null_rank; + let node_root = STARTNODE as i32 + low_s - 1; + let mut low_n = STARTNODE as i32; + let mut node_nb = STARTNODE as i32; + let first_count = nodes[huff_node_index_signed(low_s)].count; + let second_count = nodes[huff_node_index_signed(low_s - 1)].count; + nodes[huff_node_index(node_nb as usize)].count = first_count.wrapping_add(second_count); + nodes[huff_node_index_signed(low_s)].parent = node_nb as u16; + nodes[huff_node_index_signed(low_s - 1)].parent = node_nb as u16; + node_nb += 1; + low_s -= 2; + + for node in node_nb..=node_root { + nodes[huff_node_index(node as usize)].count = 1 << 30; + } + nodes[0].count = 1 << 31; + + while node_nb <= node_root { + let n1 = if nodes[huff_node_index_signed(low_s)].count + < nodes[huff_node_index_signed(low_n)].count + { + let node = low_s; + low_s -= 1; + node + } else { + let node = low_n; + low_n += 1; + node + }; + let n2 = if nodes[huff_node_index_signed(low_s)].count + < nodes[huff_node_index_signed(low_n)].count + { + let node = low_s; + low_s -= 1; + node + } else { + let node = low_n; + low_n += 1; + node + }; + nodes[huff_node_index(node_nb as usize)].count = nodes[huff_node_index_signed(n1)] + .count + .wrapping_add(nodes[huff_node_index_signed(n2)].count); + nodes[huff_node_index_signed(n1)].parent = node_nb as u16; + nodes[huff_node_index_signed(n2)].parent = node_nb as u16; + node_nb += 1; + } + + nodes[huff_node_index(node_root as usize)].nb_bits = 0; + for node in (STARTNODE as i32..node_root).rev() { + let parent = nodes[huff_node_index(node as usize)].parent as usize; + nodes[huff_node_index(node as usize)].nb_bits = + nodes[huff_node_index(parent)].nb_bits.wrapping_add(1); + } + for node in 0..=non_null_rank as usize { + let parent = nodes[huff_node_index(node)].parent as usize; + nodes[huff_node_index(node)].nb_bits = + nodes[huff_node_index(parent)].nb_bits.wrapping_add(1); + } + Some(non_null_rank as usize) +} + +fn huf_set_max_height( + nodes: &mut [NodeElt; 2 * (HUF_SYMBOLVALUE_MAX as usize + 1)], + last_non_null: usize, + target_nb_bits: u32, +) -> u32 { + let largest_bits = u32::from(nodes[huff_node_index(last_non_null)].nb_bits); + if largest_bits <= target_nb_bits { + return largest_bits; + } + + let mut total_cost = 0i32; + let base_cost = 1i32 << (largest_bits - target_nb_bits); + let mut n = last_non_null as i32; + while u32::from(nodes[huff_node_index(n as usize)].nb_bits) > target_nb_bits { + let bits = u32::from(nodes[huff_node_index(n as usize)].nb_bits); + total_cost += base_cost - (1i32 << (largest_bits - bits)); + nodes[huff_node_index(n as usize)].nb_bits = target_nb_bits as u8; + n -= 1; + } + while n >= 0 && u32::from(nodes[huff_node_index(n as usize)].nb_bits) == target_nb_bits { + n -= 1; + } + if n < 0 { + return target_nb_bits; + } + + total_cost >>= largest_bits - target_nb_bits; + const NO_SYMBOL: u32 = 0xf0f0_f0f0; + let mut rank_last = [NO_SYMBOL; HUF_TABLELOG_MAX as usize + 2]; + let mut current_nb_bits = target_nb_bits; + for position in (0..=n as usize).rev() { + let bits = u32::from(nodes[huff_node_index(position)].nb_bits); + if bits >= current_nb_bits { + continue; + } + current_nb_bits = bits; + rank_last[(target_nb_bits - current_nb_bits) as usize] = position as u32; + } + + while total_cost > 0 { + let mut bits_to_decrease = ZSTD_highbit32(total_cost as u32) + 1; + while bits_to_decrease > 1 { + let high_position = rank_last[bits_to_decrease as usize]; + let low_position = rank_last[(bits_to_decrease - 1) as usize]; + if high_position == NO_SYMBOL { + bits_to_decrease -= 1; + continue; + } + if low_position == NO_SYMBOL { + break; + } + let high_total = nodes[huff_node_index(high_position as usize)].count; + let low_total = nodes[huff_node_index(low_position as usize)] + .count + .wrapping_mul(2); + if high_total <= low_total { + break; + } + bits_to_decrease -= 1; + } + while bits_to_decrease <= HUF_TABLELOG_MAX + && rank_last[bits_to_decrease as usize] == NO_SYMBOL + { + bits_to_decrease += 1; + } + if bits_to_decrease > HUF_TABLELOG_MAX + 1 { + return target_nb_bits; + } + let position = rank_last[bits_to_decrease as usize] as usize; + total_cost -= 1i32 << (bits_to_decrease - 1); + nodes[huff_node_index(position)].nb_bits = + nodes[huff_node_index(position)].nb_bits.wrapping_add(1); + + if rank_last[(bits_to_decrease - 1) as usize] == NO_SYMBOL { + rank_last[(bits_to_decrease - 1) as usize] = rank_last[bits_to_decrease as usize]; + } + if rank_last[bits_to_decrease as usize] == 0 { + rank_last[bits_to_decrease as usize] = NO_SYMBOL; + } else { + rank_last[bits_to_decrease as usize] -= 1; + let previous = rank_last[bits_to_decrease as usize] as usize; + if u32::from(nodes[huff_node_index(previous)].nb_bits) + != target_nb_bits - bits_to_decrease + { + rank_last[bits_to_decrease as usize] = NO_SYMBOL; + } + } + } + + while total_cost < 0 { + if rank_last[1] == NO_SYMBOL { + while n >= 0 && u32::from(nodes[huff_node_index(n as usize)].nb_bits) == target_nb_bits + { + n -= 1; + } + if n < 0 { + return target_nb_bits; + } + let position = n as usize + 1; + nodes[huff_node_index(position)].nb_bits = + nodes[huff_node_index(position)].nb_bits.wrapping_sub(1); + rank_last[1] = position as u32; + total_cost += 1; + continue; + } + let position = rank_last[1] as usize + 1; + nodes[huff_node_index(position)].nb_bits = + nodes[huff_node_index(position)].nb_bits.wrapping_sub(1); + rank_last[1] = rank_last[1].wrapping_add(1); + total_cost += 1; + } + target_nb_bits +} + +unsafe fn huf_build_ctable_from_tree( + ctable: *mut usize, + nodes: &[NodeElt; 2 * (HUF_SYMBOLVALUE_MAX as usize + 1)], + non_null_rank: usize, + max_symbol_value: u32, + max_nb_bits: u32, +) { + let mut nb_per_rank = [0u16; HUF_TABLELOG_MAX as usize + 1]; + let mut val_per_rank = [0u16; HUF_TABLELOG_MAX as usize + 1]; + for node in 0..=non_null_rank { + let bits = nodes[huff_node_index(node)].nb_bits as usize; + nb_per_rank[bits] = nb_per_rank[bits].wrapping_add(1); + } + let mut min = 0u16; + for rank in (1..=max_nb_bits as usize).rev() { + val_per_rank[rank] = min; + min = min.wrapping_add(nb_per_rank[rank]); + min >>= 1; + } + for node in 0..=max_symbol_value as usize { + let symbol = nodes[huff_node_index(node)].byte as usize; + huf_set_nb_bits( + ctable, + symbol + 1, + nodes[huff_node_index(node)].nb_bits as usize, + ); + } + for symbol in 0..=max_symbol_value as usize { + let bits = huf_get_nb_bits(ctable_read(ctable, symbol + 1)); + let value = val_per_rank[bits]; + huf_set_value(ctable, symbol + 1, value as usize); + val_per_rank[bits] = val_per_rank[bits].wrapping_add(1); + } + huf_write_ctable_header(ctable, max_nb_bits, max_symbol_value); +} + +#[no_mangle] +pub unsafe extern "C" fn HUF_buildCTable_wksp( + ctable: *mut usize, + count: *const c_uint, + max_symbol_value: u32, + mut max_nb_bits: u32, + workspace: *mut c_void, + workspace_size: usize, +) -> usize { + if huf_aligned_workspace_size(workspace, workspace_size, size_of::()) + < HUF_CTABLE_WORKSPACE_SIZE + { + return ERROR(ZstdErrorCode::WorkSpaceTooSmall); + } + if max_nb_bits == 0 { + max_nb_bits = HUF_TABLELOG_DEFAULT; + } + if max_symbol_value > HUF_SYMBOLVALUE_MAX { + return ERROR(ZstdErrorCode::MaxSymbolValueTooLarge); + } + + let mut nodes = [NodeElt::default(); 2 * (HUF_SYMBOLVALUE_MAX as usize + 1)]; + let mut rank_position = [RankPos::default(); RANK_POSITION_TABLE_SIZE]; + huf_sort(&mut nodes, count, max_symbol_value, &mut rank_position); + let non_null_rank = match huf_build_tree(&mut nodes, max_symbol_value) { + Some(rank) => rank, + None => return ERROR(ZstdErrorCode::Generic), + }; + max_nb_bits = huf_set_max_height(&mut nodes, non_null_rank, max_nb_bits); + if max_nb_bits > HUF_TABLELOG_MAX { + return ERROR(ZstdErrorCode::Generic); + } + huf_build_ctable_from_tree(ctable, &nodes, non_null_rank, max_symbol_value, max_nb_bits); + max_nb_bits as usize +} + +#[no_mangle] +pub unsafe extern "C" fn HUF_estimateCompressedSize( + ctable: *const usize, + count: *const c_uint, + max_symbol_value: c_uint, +) -> usize { + let mut bits = 0usize; + for symbol in 0..=max_symbol_value as usize { + bits = bits.wrapping_add( + huf_get_nb_bits(ctable_read(ctable, symbol + 1)) + .wrapping_mul(*count.add(symbol) as usize), + ); + } + bits >> 3 +} + +#[no_mangle] +pub unsafe extern "C" fn HUF_validateCTable( + ctable: *const usize, + count: *const c_uint, + max_symbol_value: c_uint, +) -> c_int { + let header = huf_read_ctable_header(ctable); + if u32::from(header.maxSymbolValue) < max_symbol_value { + return 0; + } + for symbol in 0..=max_symbol_value as usize { + if *count.add(symbol) != 0 && huf_get_nb_bits(ctable_read(ctable, symbol + 1)) == 0 { + return 0; + } + } + 1 +} + +#[no_mangle] +pub extern "C" fn HUF_compressBound(size: usize) -> usize { + 129usize + .wrapping_add(size) + .wrapping_add(size >> 8) + .wrapping_add(8) +} + +struct HufCStream { + bit_container: usize, + bit_pos: usize, + dst: *mut u8, + ptr: usize, + end: usize, +} + +impl HufCStream { + unsafe fn init(dst: *mut u8, dst_capacity: usize) -> Option { + let container_size = size_of::(); + if dst_capacity <= container_size { + return None; + } + Some(Self { + bit_container: 0, + bit_pos: 0, + dst, + ptr: 0, + end: dst_capacity - container_size, + }) + } + + #[inline] + fn add_bits(&mut self, elt: usize) { + let nb_bits = huf_get_nb_bits(elt); + self.bit_container >>= nb_bits; + self.bit_container |= huf_get_value(elt); + self.bit_pos = self.bit_pos.wrapping_add(nb_bits); + } + + unsafe fn flush_bits(&mut self) { + let nb_bits = self.bit_pos & 0xff; + if nb_bits == 0 { + return; + } + let nb_bytes = nb_bits >> 3; + let bit_container = self.bit_container >> (usize::BITS as usize - nb_bits); + for byte in 0..nb_bytes { + // The C encoder writes a full native word at `ptr`, then clamps the + // logical cursor. Only materialize the bytes that are logically + // part of the stream; `ptr <= end` guarantees those are in bounds. + *self.dst.add(self.ptr + byte) = (bit_container >> (8 * byte)) as u8; + } + self.ptr = self.ptr.saturating_add(nb_bytes); + if self.ptr > self.end { + self.ptr = self.end; + } + self.bit_pos &= 7; + } + + unsafe fn close(mut self) -> usize { + let end_mark = 1usize | (1usize << (usize::BITS as usize - 1)); + self.add_bits(end_mark); + self.flush_bits(); + if self.ptr >= self.end { + return 0; + } + if self.bit_pos & 0xff != 0 { + let partial = self.bit_container >> (usize::BITS as usize - (self.bit_pos & 0xff)); + *self.dst.add(self.ptr) = partial as u8; + } + self.ptr + usize::from((self.bit_pos & 0xff) > 0) + } +} + +unsafe fn huf_compress1x_using_ctable_internal( + dst: *mut u8, + dst_size: usize, + src: *const u8, + src_size: usize, + ctable: *const usize, +) -> usize { + if dst_size < 8 { + return 0; + } + let mut stream = match HufCStream::init(dst, dst_size) { + Some(stream) => stream, + None => return 0, + }; + for index in (0..src_size).rev() { + let symbol = *src.add(index) as usize; + stream.add_bits(ctable_read(ctable, symbol + 1)); + // Keeping the pending tail below one byte makes the portable Rust + // stream independent of the C implementation's BMI2-oriented unroll + // schedule while retaining the exact HUF bitstream representation. + stream.flush_bits(); + } + stream.close() +} + +#[no_mangle] +pub unsafe extern "C" fn HUF_compress1X_usingCTable( + dst: *mut c_void, + dst_size: usize, + src: *const c_void, + src_size: usize, + ctable: *const usize, + _flags: c_int, +) -> usize { + huf_compress1x_using_ctable_internal(dst.cast(), dst_size, src.cast(), src_size, ctable) +} + +unsafe fn huf_compress4x_using_ctable_internal( + dst: *mut u8, + dst_size: usize, + src: *const u8, + src_size: usize, + ctable: *const usize, +) -> usize { + if dst_size < 6 + 1 + 1 + 1 + 8 || src_size < 12 { + return 0; + } + let segment_size = src_size.div_ceil(4); + let mut input_offset = 0usize; + let mut output_offset = 6usize; + for segment in 0..4 { + let input_size = if segment < 3 { + segment_size + } else { + src_size - input_offset + }; + let c_size = huf_compress1x_using_ctable_internal( + dst.add(output_offset), + dst_size - output_offset, + src.add(input_offset), + input_size, + ctable, + ); + if c_size == 0 || c_size > u16::MAX as usize { + return 0; + } + if segment < 3 { + MEM_writeLE16(dst.add(segment * 2).cast::(), c_size as u16); + } + output_offset += c_size; + input_offset += input_size; + } + output_offset +} + +#[no_mangle] +pub unsafe extern "C" fn HUF_compress4X_usingCTable( + dst: *mut c_void, + dst_size: usize, + src: *const c_void, + src_size: usize, + ctable: *const usize, + _flags: c_int, +) -> usize { + huf_compress4x_using_ctable_internal(dst.cast(), dst_size, src.cast(), src_size, ctable) +} + +#[allow(clippy::too_many_arguments)] +unsafe fn huf_compress_ctable_internal( + dst: *mut u8, + dst_size: usize, + prefix_size: usize, + src: *const u8, + src_size: usize, + four_streams: bool, + ctable: *const usize, + flags: c_int, +) -> usize { + if prefix_size > dst_size { + return 0; + } + let c_size = if four_streams { + HUF_compress4X_usingCTable( + dst.add(prefix_size).cast::(), + dst_size - prefix_size, + src.cast::(), + src_size, + ctable, + flags, + ) + } else { + HUF_compress1X_usingCTable( + dst.add(prefix_size).cast::(), + dst_size - prefix_size, + src.cast::(), + src_size, + ctable, + flags, + ) + }; + if ERR_isError(c_size) { + return c_size; + } + let total_size = prefix_size.wrapping_add(c_size); + if c_size == 0 || total_size >= src_size.saturating_sub(1) { + return 0; + } + total_size +} + +#[no_mangle] +pub unsafe extern "C" fn HUF_cardinality(count: *const c_uint, max_symbol_value: c_uint) -> c_uint { + let mut cardinality = 0u32; + for symbol in 0..=max_symbol_value as usize { + if *count.add(symbol) != 0 { + cardinality += 1; + } + } + cardinality +} + +#[no_mangle] +pub extern "C" fn HUF_minTableLog(symbol_cardinality: c_uint) -> c_uint { + ZSTD_highbit32(symbol_cardinality) + 1 +} + +#[no_mangle] +pub unsafe extern "C" fn HUF_optimalTableLog( + max_table_log: c_uint, + src_size: usize, + max_symbol_value: c_uint, + workspace: *mut c_void, + workspace_size: usize, + table: *mut usize, + count: *const c_uint, + flags: c_int, +) -> c_uint { + if flags & HUF_FLAGS_OPTIMAL_DEPTH == 0 { + return FSE_optimalTableLog_internal(max_table_log, src_size, max_symbol_value, 1); + } + + let symbol_cardinality = HUF_cardinality(count, max_symbol_value); + if symbol_cardinality == 0 { + return max_table_log; + } + let min_table_log = HUF_minTableLog(symbol_cardinality); + let mut opt_size = usize::MAX - 1; + let mut opt_log = max_table_log; + let mut probe_dst = [0u8; HUF_WORKSPACE_SIZE]; + let write_workspace_size = workspace_size.saturating_sub(748); + + for guess in min_table_log..=max_table_log { + let max_bits = HUF_buildCTable_wksp( + table, + count, + max_symbol_value, + guess, + workspace, + workspace_size, + ); + if ERR_isError(max_bits) { + continue; + } + if max_bits < guess as usize && guess > min_table_log { + break; + } + let h_size = HUF_writeCTable_wksp( + probe_dst.as_mut_ptr().cast::(), + write_workspace_size.min(probe_dst.len()), + table, + max_symbol_value, + max_bits as u32, + workspace, + workspace_size, + ); + if ERR_isError(h_size) { + continue; + } + let new_size = + HUF_estimateCompressedSize(table, count, max_symbol_value).wrapping_add(h_size); + if new_size > opt_size.wrapping_add(1) { + break; + } + if new_size < opt_size { + opt_size = new_size; + opt_log = guess; + } + } + opt_log +} + +#[inline] +fn huf_compress_tables_size() -> usize { + (HUF_SYMBOLVALUE_MAX as usize + 1) * size_of::() + + HUF_CTABLE_SIZE_ST * size_of::() + + HUF_CTABLE_WORKSPACE_SIZE +} + +#[allow(clippy::too_many_arguments)] +unsafe fn huf_compress_internal( + dst: *mut u8, + dst_size: usize, + src: *const u8, + src_size: usize, + mut max_symbol_value: u32, + mut huff_log: u32, + four_streams: bool, + workspace: *mut c_void, + workspace_size: usize, + old_huf_table: *mut usize, + repeat: *mut c_int, + flags: c_int, +) -> usize { + if huf_aligned_workspace_size(workspace, workspace_size, size_of::()) + < huf_compress_tables_size() + { + return ERROR(ZstdErrorCode::WorkSpaceTooSmall); + } + if src_size == 0 || dst_size == 0 { + return 0; + } + if src_size > HUF_BLOCKSIZE_MAX { + return ERROR(ZstdErrorCode::SrcSizeWrong); + } + if huff_log > HUF_TABLELOG_MAX { + return ERROR(ZstdErrorCode::TableLogTooLarge); + } + if max_symbol_value > HUF_SYMBOLVALUE_MAX { + return ERROR(ZstdErrorCode::MaxSymbolValueTooLarge); + } + if max_symbol_value == 0 { + max_symbol_value = HUF_SYMBOLVALUE_MAX; + } + if huff_log == 0 { + huff_log = HUF_TABLELOG_DEFAULT; + } + + if flags & HUF_FLAGS_PREFER_REPEAT != 0 + && !repeat.is_null() + && *repeat == HUF_REPEAT_VALID + && !old_huf_table.is_null() + { + return huf_compress_ctable_internal( + dst, + dst_size, + 0, + src, + src_size, + four_streams, + old_huf_table, + flags, + ); + } + + let mut count = [0u32; HUF_SYMBOLVALUE_MAX as usize + 1]; + if flags & HUF_FLAGS_SUSPECT_UNCOMPRESSIBLE != 0 + && src_size >= SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO + { + let mut largest_total = 0usize; + let mut sample_max = max_symbol_value; + largest_total += HIST_count_simple( + count.as_mut_ptr(), + &mut sample_max, + src.cast::(), + SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE, + ) as usize; + sample_max = max_symbol_value; + largest_total += HIST_count_simple( + count.as_mut_ptr(), + &mut sample_max, + src.add(src_size - SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) + .cast::(), + SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE, + ) as usize; + if largest_total <= ((2 * SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) >> 7).wrapping_add(4) { + return 0; + } + } + + let mut hist_workspace = [0u32; HIST_WKSP_SIZE_U32]; + let largest = HIST_count_wksp( + count.as_mut_ptr(), + &mut max_symbol_value, + src.cast::(), + src_size, + hist_workspace.as_mut_ptr().cast::(), + size_of::<[u32; HIST_WKSP_SIZE_U32]>(), + ); + if ERR_isError(largest) { + return largest; + } + if largest == src_size { + *dst = *src; + return 1; + } + if largest <= (src_size >> 7).wrapping_add(4) { + return 0; + } + + if !repeat.is_null() + && *repeat == HUF_REPEAT_CHECK + && (old_huf_table.is_null() + || HUF_validateCTable(old_huf_table, count.as_ptr(), max_symbol_value) == 0) + { + *repeat = HUF_REPEAT_NONE; + } + if flags & HUF_FLAGS_PREFER_REPEAT != 0 + && !repeat.is_null() + && *repeat != HUF_REPEAT_NONE + && !old_huf_table.is_null() + { + return huf_compress_ctable_internal( + dst, + dst_size, + 0, + src, + src_size, + four_streams, + old_huf_table, + flags, + ); + } + + let mut ctable = [0usize; HUF_CTABLE_SIZE_ST]; + huff_log = HUF_optimalTableLog( + huff_log, + src_size, + max_symbol_value, + workspace, + HUF_CTABLE_WORKSPACE_SIZE, + ctable.as_mut_ptr(), + count.as_ptr(), + flags, + ); + let max_bits = HUF_buildCTable_wksp( + ctable.as_mut_ptr(), + count.as_ptr(), + max_symbol_value, + huff_log, + workspace, + HUF_CTABLE_WORKSPACE_SIZE, + ); + if ERR_isError(max_bits) { + return max_bits; + } + huff_log = max_bits as u32; + + let header_size = HUF_writeCTable_wksp( + dst.cast::(), + dst_size, + ctable.as_ptr(), + max_symbol_value, + huff_log, + workspace, + HUF_CTABLE_WORKSPACE_SIZE, + ); + if ERR_isError(header_size) { + return header_size; + } + if !repeat.is_null() && *repeat != HUF_REPEAT_NONE && !old_huf_table.is_null() { + let old_size = HUF_estimateCompressedSize(old_huf_table, count.as_ptr(), max_symbol_value); + let new_size = + HUF_estimateCompressedSize(ctable.as_ptr(), count.as_ptr(), max_symbol_value); + if old_size <= header_size.wrapping_add(new_size) + || header_size.wrapping_add(12) >= src_size + { + return huf_compress_ctable_internal( + dst, + dst_size, + 0, + src, + src_size, + four_streams, + old_huf_table, + flags, + ); + } + } + if header_size.wrapping_add(12) >= src_size { + return 0; + } + if !repeat.is_null() { + *repeat = HUF_REPEAT_NONE; + } + if !old_huf_table.is_null() { + std::ptr::copy_nonoverlapping( + ctable.as_ptr().cast::(), + old_huf_table.cast::(), + size_of::<[usize; HUF_CTABLE_SIZE_ST]>(), + ); + } + huf_compress_ctable_internal( + dst, + dst_size, + header_size, + src, + src_size, + four_streams, + ctable.as_ptr(), + flags, + ) +} + +#[no_mangle] +pub unsafe extern "C" fn HUF_compress1X_repeat( + dst: *mut c_void, + dst_size: usize, + src: *const c_void, + src_size: usize, + max_symbol_value: c_uint, + huff_log: c_uint, + workspace: *mut c_void, + workspace_size: usize, + huf_table: *mut usize, + repeat: *mut c_int, + flags: c_int, +) -> usize { + huf_compress_internal( + dst.cast(), + dst_size, + src.cast(), + src_size, + max_symbol_value, + huff_log, + false, + workspace, + workspace_size, + huf_table, + repeat, + flags, + ) +} + +#[no_mangle] +pub unsafe extern "C" fn HUF_compress4X_repeat( + dst: *mut c_void, + dst_size: usize, + src: *const c_void, + src_size: usize, + max_symbol_value: c_uint, + huff_log: c_uint, + workspace: *mut c_void, + workspace_size: usize, + huf_table: *mut usize, + repeat: *mut c_int, + flags: c_int, +) -> usize { + huf_compress_internal( + dst.cast(), + dst_size, + src.cast(), + src_size, + max_symbol_value, + huff_log, + true, + workspace, + workspace_size, + huf_table, + repeat, + flags, + ) +} + +#[cfg(all(test, feature = "decompression"))] +mod tests { + use super::*; + use crate::huf_decompress::{ + HUF_decompress1X_usingDTable, HUF_decompress4X_usingDTable, HUF_readDTableX1_wksp, + }; + + const WORKSPACE_SIZE: usize = HUF_WORKSPACE_SIZE; + + fn source() -> Vec { + let mut source = Vec::with_capacity(4096); + for index in 0..4096 { + source.push(match index % 17 { + 0..=8 => b'a', + 9..=12 => b'b', + 13..=14 => b'c', + 15 => b'd', + _ => b'e', + }); + } + source + } + + unsafe fn make_table( + source: &[u8], + table: &mut [usize; HUF_CTABLE_SIZE_ST], + workspace: &mut [u64; WORKSPACE_SIZE / size_of::()], + ) -> u32 { + let mut count = [0u32; HUF_SYMBOLVALUE_MAX as usize + 1]; + for byte in source { + count[*byte as usize] += 1; + } + let mut max_symbol = HUF_SYMBOLVALUE_MAX; + while count[max_symbol as usize] == 0 { + max_symbol -= 1; + } + let bits = HUF_buildCTable_wksp( + table.as_mut_ptr(), + count.as_ptr(), + max_symbol, + 0, + workspace.as_mut_ptr().cast::(), + std::mem::size_of_val(workspace), + ); + assert!(!ERR_isError(bits)); + bits as u32 + } + + unsafe fn dtable_from_header( + header: &[u8], + workspace: &mut [u64; WORKSPACE_SIZE / size_of::()], + ) -> [u32; 1 + (1 << 11)] { + let mut dtable = [0u32; 1 + (1 << 11)]; + dtable[0] = 11 * 0x0100_0001; + let read = HUF_readDTableX1_wksp( + dtable.as_mut_ptr(), + header.as_ptr().cast::(), + header.len(), + workspace.as_mut_ptr().cast::(), + std::mem::size_of_val(workspace), + 0, + ); + assert!(!ERR_isError(read)); + dtable + } + + #[test] + fn tables_and_single_stream_round_trip() { + let source = source(); + let mut workspace = [0u64; WORKSPACE_SIZE / size_of::()]; + let mut table = [0usize; HUF_CTABLE_SIZE_ST]; + let bits = unsafe { make_table(&source, &mut table, &mut workspace) }; + assert!((1..=HUF_TABLELOG_MAX).contains(&bits)); + + let max_symbol = unsafe { HUF_readCTableHeader(table.as_ptr()).maxSymbolValue as u32 }; + let mut header = [0u8; 512]; + let header_size = unsafe { + HUF_writeCTable_wksp( + header.as_mut_ptr().cast::(), + header.len(), + table.as_ptr(), + max_symbol, + bits, + workspace.as_mut_ptr().cast::(), + std::mem::size_of_val(&workspace), + ) + }; + assert!(!ERR_isError(header_size)); + let dtable = unsafe { dtable_from_header(&header[..header_size], &mut workspace) }; + + let mut compressed = vec![0u8; source.len() + 64]; + let c_size = unsafe { + HUF_compress1X_usingCTable( + compressed.as_mut_ptr().cast::(), + compressed.len(), + source.as_ptr().cast::(), + source.len(), + table.as_ptr(), + 0, + ) + }; + assert!(c_size > 0); + let mut decoded = vec![0u8; source.len()]; + let d_size = unsafe { + HUF_decompress1X_usingDTable( + decoded.as_mut_ptr().cast::(), + decoded.len(), + compressed.as_ptr().cast::(), + c_size, + dtable.as_ptr(), + 0, + ) + }; + assert_eq!(d_size, source.len()); + assert_eq!(decoded, source); + } + + #[test] + fn four_stream_and_repeat_round_trip() { + let source = source(); + let mut workspace = [0u64; WORKSPACE_SIZE / size_of::()]; + let mut old_table = [0usize; HUF_CTABLE_SIZE_ST]; + let mut repeat = HUF_REPEAT_NONE; + let mut compressed = vec![0u8; source.len() + 256]; + let c_size = unsafe { + HUF_compress4X_repeat( + compressed.as_mut_ptr().cast::(), + compressed.len(), + source.as_ptr().cast::(), + source.len(), + HUF_SYMBOLVALUE_MAX, + HUF_TABLELOG_DEFAULT, + workspace.as_mut_ptr().cast::(), + std::mem::size_of_val(&workspace), + old_table.as_mut_ptr(), + &mut repeat, + HUF_FLAGS_OPTIMAL_DEPTH, + ) + }; + assert!(c_size > 0); + let mut dtable = [0u32; 1 + (1 << 11)]; + dtable[0] = 11 * 0x0100_0001; + let header_size = unsafe { + HUF_readDTableX1_wksp( + dtable.as_mut_ptr(), + compressed.as_ptr().cast::(), + c_size, + workspace.as_mut_ptr().cast::(), + std::mem::size_of_val(&workspace), + 0, + ) + }; + assert!(!ERR_isError(header_size)); + let mut decoded = vec![0u8; source.len()]; + let d_size = unsafe { + HUF_decompress4X_usingDTable( + decoded.as_mut_ptr().cast::(), + decoded.len(), + compressed.as_ptr().add(header_size).cast::(), + c_size - header_size, + dtable.as_ptr(), + 0, + ) + }; + assert_eq!(d_size, source.len()); + assert_eq!(decoded, source); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index dc3325213..48c86c8ed 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -12,6 +12,8 @@ pub mod fse_compress; pub mod fse_decompress; #[cfg(feature = "compression")] pub mod hist; +#[cfg(feature = "compression")] +pub mod huf_compress; #[cfg(feature = "decompression")] pub mod huf_decompress; pub mod mem;