diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 05448a420..b56b20ed6 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -88,6 +88,67 @@ size_t ZSTD_rust_params_estimateCDictSizeFromCParams( size_t dictSize, ZSTD_compressionParameters cParams, int dictLoadMethod, const ZSTD_rustCDictSizing* sizing); +/* Sequence statistics and seqStore entropy compression live in Rust + * (rust/src/zstd_compress_stats.rs), which also exports ZSTD_seqToCodes() + * under its original name. The shims below extract the sequence store, the + * entropy-table leaves, and the only two ZSTD_CCtx_params fields these paths + * read (the strategy and the literals-compression switch), so the private + * parameter structure layout never crosses the language boundary. The + * shared leaf layouts are pinned by the compile-time asserts that follow. */ +size_t ZSTD_rust_entropyCompressSeqStore_internal( + void* dst, size_t dstCapacity, + const void* literals, size_t litSize, + const SeqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + int strategy, int disableLiteralCompression, + void* entropyWorkspace, size_t entropyWkspSize, + int bmi2); +size_t ZSTD_rust_entropyCompressSeqStore( + const SeqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + int strategy, int disableLiteralCompression, + void* dst, size_t dstCapacity, + size_t srcSize, + void* entropyWorkspace, size_t entropyWkspSize, + int bmi2); +size_t ZSTD_rust_buildBlockEntropyStats( + const SeqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + int strategy, int disableLiteralCompression, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize); +size_t ZSTD_rust_copyBlockSequences( + SeqCollector* seqCollector, const SeqStore_t* seqStore, + const U32 prevRepcodes[ZSTD_REP_NUM]); + +typedef char ZSTD_rust_stats_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1]; +typedef char ZSTD_rust_stats_seqstore_long_length_pos[ + (offsetof(SeqStore_t, longLengthPos) == 9 * sizeof(size_t) + 4) ? 1 : -1]; +typedef char ZSTD_rust_stats_seqstore_layout[ + (sizeof(SeqStore_t) == 9 * sizeof(size_t) + 8) ? 1 : -1]; +typedef char ZSTD_rust_stats_huf_layout[ + (sizeof(ZSTD_hufCTables_t) == 258 * sizeof(size_t)) ? 1 : -1]; +typedef char ZSTD_rust_stats_fse_layout[(sizeof(ZSTD_fseCTables_t) == 3552) ? 1 : -1]; +typedef char ZSTD_rust_stats_entropy_fse_offset[ + (offsetof(ZSTD_entropyCTables_t, fse) == sizeof(ZSTD_hufCTables_t)) ? 1 : -1]; +typedef char ZSTD_rust_stats_huf_metadata_buffer[ + (offsetof(ZSTD_hufCTablesMetadata_t, hufDesBuffer) == 4 + && ZSTD_MAX_HUF_HEADER_SIZE == 128) ? 1 : -1]; +typedef char ZSTD_rust_stats_fse_metadata_buffer[ + (offsetof(ZSTD_fseCTablesMetadata_t, fseTablesBuffer) == 12 + && ZSTD_MAX_FSE_HEADERS_SIZE == 133) ? 1 : -1]; +typedef char ZSTD_rust_stats_entropy_metadata_layout[ + (offsetof(ZSTD_entropyCTablesMetadata_t, fseMetadata) + == sizeof(ZSTD_hufCTablesMetadata_t)) ? 1 : -1]; +typedef char ZSTD_rust_stats_sequence_layout[(sizeof(ZSTD_Sequence) == 16) ? 1 : -1]; +typedef char ZSTD_rust_stats_seqcollector_layout[ + (offsetof(SeqCollector, seqStart) == sizeof(size_t) + && offsetof(SeqCollector, seqIndex) == 2 * sizeof(size_t) + && sizeof(SeqCollector) == 4 * sizeof(size_t)) ? 1 : -1]; + #if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) # define ZSTD_RUST_ASAN_REDZONE_SIZE ((size_t)ZSTD_CWKSP_ASAN_REDZONE_SIZE) #else @@ -2561,33 +2622,7 @@ static void ZSTD_reduceIndex (ZSTD_MatchState_t* ms, ZSTD_CCtx_params const* par /* See doc/zstd_compression_format.md for detailed format description */ -int ZSTD_seqToCodes(const SeqStore_t* seqStorePtr) -{ - const SeqDef* const sequences = seqStorePtr->sequencesStart; - BYTE* const llCodeTable = seqStorePtr->llCode; - BYTE* const ofCodeTable = seqStorePtr->ofCode; - BYTE* const mlCodeTable = seqStorePtr->mlCode; - U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); - U32 u; - int longOffsets = 0; - assert(nbSeq <= seqStorePtr->maxNbSeq); - for (u=0; u= STREAM_ACCUMULATOR_MIN)); - if (MEM_32bits() && ofCode >= STREAM_ACCUMULATOR_MIN) - longOffsets = 1; - } - if (seqStorePtr->longLengthType==ZSTD_llt_literalLength) - llCodeTable[seqStorePtr->longLengthPos] = MaxLL; - if (seqStorePtr->longLengthType==ZSTD_llt_matchLength) - mlCodeTable[seqStorePtr->longLengthPos] = MaxML; - return longOffsets; -} +/* ZSTD_seqToCodes() lives in rust/src/zstd_compress_stats.rs. */ /* ZSTD_useTargetCBlockSize(): * Returns if target compressed block size param is being used. @@ -2611,150 +2646,11 @@ static int ZSTD_blockSplitterEnabled(ZSTD_CCtx_params* cctxParams) return (cctxParams->postBlockSplitter == ZSTD_ps_enable); } -/* Type returned by ZSTD_buildSequencesStatistics containing finalized symbol encoding types - * and size of the sequences statistics - */ -typedef struct { - U32 LLtype; - U32 Offtype; - U32 MLtype; - size_t size; - size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ - int longOffsets; -} ZSTD_symbolEncodingTypeStats_t; +/* ZSTD_buildSequencesStatistics() and the ZSTD_entropyCompressSeqStore*() + * implementations live in rust/src/zstd_compress_stats.rs. The wrappers + * below keep the original signatures and extract the only two + * ZSTD_CCtx_params fields those paths read. */ -/* ZSTD_buildSequencesStatistics(): - * Returns a ZSTD_symbolEncodingTypeStats_t, or a zstd error code in the `size` field. - * Modifies `nextEntropy` to have the appropriate values as a side effect. - * nbSeq must be greater than 0. - * - * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32) - */ -static ZSTD_symbolEncodingTypeStats_t -ZSTD_buildSequencesStatistics( - const SeqStore_t* seqStorePtr, size_t nbSeq, - const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy, - BYTE* dst, const BYTE* const dstEnd, - ZSTD_strategy strategy, unsigned* countWorkspace, - void* entropyWorkspace, size_t entropyWkspSize) -{ - BYTE* const ostart = dst; - const BYTE* const oend = dstEnd; - BYTE* op = ostart; - FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable; - FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable; - FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable; - const BYTE* const ofCodeTable = seqStorePtr->ofCode; - const BYTE* const llCodeTable = seqStorePtr->llCode; - const BYTE* const mlCodeTable = seqStorePtr->mlCode; - ZSTD_symbolEncodingTypeStats_t stats; - - stats.lastCountSize = 0; - /* convert length/distances into codes */ - stats.longOffsets = ZSTD_seqToCodes(seqStorePtr); - assert(op <= oend); - assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */ - /* build CTable for Literal Lengths */ - { unsigned max = MaxLL; - size_t const mostFrequent = HIST_countFast_wksp(countWorkspace, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ - DEBUGLOG(5, "Building LL table"); - nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; - stats.LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, - countWorkspace, max, mostFrequent, nbSeq, - LLFSELog, prevEntropy->litlengthCTable, - LL_defaultNorm, LL_defaultNormLog, - ZSTD_defaultAllowed, strategy); - assert(set_basic < set_compressed && set_rle < set_compressed); - assert(!(stats.LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable( - op, (size_t)(oend - op), - CTable_LitLength, LLFSELog, (SymbolEncodingType_e)stats.LLtype, - countWorkspace, max, llCodeTable, nbSeq, - LL_defaultNorm, LL_defaultNormLog, MaxLL, - prevEntropy->litlengthCTable, - sizeof(prevEntropy->litlengthCTable), - entropyWorkspace, entropyWkspSize); - if (ZSTD_isError(countSize)) { - DEBUGLOG(3, "ZSTD_buildCTable for LitLens failed"); - stats.size = countSize; - return stats; - } - if (stats.LLtype == set_compressed) - stats.lastCountSize = countSize; - op += countSize; - assert(op <= oend); - } } - /* build CTable for Offsets */ - { unsigned max = MaxOff; - size_t const mostFrequent = HIST_countFast_wksp( - countWorkspace, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ - /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ - ZSTD_DefaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; - DEBUGLOG(5, "Building OF table"); - nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; - stats.Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, - countWorkspace, max, mostFrequent, nbSeq, - OffFSELog, prevEntropy->offcodeCTable, - OF_defaultNorm, OF_defaultNormLog, - defaultPolicy, strategy); - assert(!(stats.Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable( - op, (size_t)(oend - op), - CTable_OffsetBits, OffFSELog, (SymbolEncodingType_e)stats.Offtype, - countWorkspace, max, ofCodeTable, nbSeq, - OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, - prevEntropy->offcodeCTable, - sizeof(prevEntropy->offcodeCTable), - entropyWorkspace, entropyWkspSize); - if (ZSTD_isError(countSize)) { - DEBUGLOG(3, "ZSTD_buildCTable for Offsets failed"); - stats.size = countSize; - return stats; - } - if (stats.Offtype == set_compressed) - stats.lastCountSize = countSize; - op += countSize; - assert(op <= oend); - } } - /* build CTable for MatchLengths */ - { unsigned max = MaxML; - size_t const mostFrequent = HIST_countFast_wksp( - countWorkspace, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ - DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); - nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; - stats.MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, - countWorkspace, max, mostFrequent, nbSeq, - MLFSELog, prevEntropy->matchlengthCTable, - ML_defaultNorm, ML_defaultNormLog, - ZSTD_defaultAllowed, strategy); - assert(!(stats.MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable( - op, (size_t)(oend - op), - CTable_MatchLength, MLFSELog, (SymbolEncodingType_e)stats.MLtype, - countWorkspace, max, mlCodeTable, nbSeq, - ML_defaultNorm, ML_defaultNormLog, MaxML, - prevEntropy->matchlengthCTable, - sizeof(prevEntropy->matchlengthCTable), - entropyWorkspace, entropyWkspSize); - if (ZSTD_isError(countSize)) { - DEBUGLOG(3, "ZSTD_buildCTable for MatchLengths failed"); - stats.size = countSize; - return stats; - } - if (stats.MLtype == set_compressed) - stats.lastCountSize = countSize; - op += countSize; - assert(op <= oend); - } } - stats.size = (size_t)(op-ostart); - return stats; -} - -/* ZSTD_entropyCompressSeqStore_internal(): - * compresses both literals and sequences - * Returns compressed size of block, or a zstd error. - */ -#define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20 MEM_STATIC size_t ZSTD_entropyCompressSeqStore_internal( void* dst, size_t dstCapacity, @@ -2766,150 +2662,13 @@ ZSTD_entropyCompressSeqStore_internal( void* entropyWorkspace, size_t entropyWkspSize, const int bmi2) { - ZSTD_strategy const strategy = cctxParams->cParams.strategy; - unsigned* count = (unsigned*)entropyWorkspace; - FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable; - FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable; - FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable; - const SeqDef* const sequences = seqStorePtr->sequencesStart; - const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); - const BYTE* const ofCodeTable = seqStorePtr->ofCode; - const BYTE* const llCodeTable = seqStorePtr->llCode; - const BYTE* const mlCodeTable = seqStorePtr->mlCode; - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstCapacity; - BYTE* op = ostart; - size_t lastCountSize; - int longOffsets = 0; - - entropyWorkspace = count + (MaxSeq + 1); - entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); - - DEBUGLOG(5, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu, dstCapacity=%zu)", nbSeq, dstCapacity); - ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= HUF_WORKSPACE_SIZE); - - /* Compress literals */ - { size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); - /* Base suspicion of uncompressibility on ratio of literals to sequences */ - int const suspectUncompressible = (numSequences == 0) || (litSize / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO); - - size_t const cSize = ZSTD_compressLiterals( - op, dstCapacity, - literals, litSize, - entropyWorkspace, entropyWkspSize, - &prevEntropy->huf, &nextEntropy->huf, - cctxParams->cParams.strategy, - ZSTD_literalsCompressionIsDisabled(cctxParams), - suspectUncompressible, bmi2); - FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed"); - assert(cSize <= dstCapacity); - op += cSize; - } - - /* Sequences Header */ - RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, - dstSize_tooSmall, "Can't fit seq hdr in output buf!"); - if (nbSeq < 128) { - *op++ = (BYTE)nbSeq; - } else if (nbSeq < LONGNBSEQ) { - op[0] = (BYTE)((nbSeq>>8) + 0x80); - op[1] = (BYTE)nbSeq; - op+=2; - } else { - op[0]=0xFF; - MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)); - op+=3; - } - assert(op <= oend); - if (nbSeq==0) { - /* Copy the old tables over as if we repeated them */ - ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); - return (size_t)(op - ostart); - } - { BYTE* const seqHead = op++; - /* build stats for sequences */ - const ZSTD_symbolEncodingTypeStats_t stats = - ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, - &prevEntropy->fse, &nextEntropy->fse, - op, oend, - strategy, count, - entropyWorkspace, entropyWkspSize); - FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); - *seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2)); - lastCountSize = stats.lastCountSize; - op += stats.size; - longOffsets = stats.longOffsets; - } - - { size_t const bitstreamSize = ZSTD_encodeSequences( - op, (size_t)(oend - op), - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, - longOffsets, bmi2); - FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); - op += bitstreamSize; - assert(op <= oend); - /* zstd versions <= 1.3.4 mistakenly report corruption when - * FSE_readNCount() receives a buffer < 4 bytes. - * Fixed by https://github.com/facebook/zstd/pull/1146. - * This can happen when the last set_compressed table present is 2 - * bytes and the bitstream is only one byte. - * In this exceedingly rare case, we will simply emit an uncompressed - * block, since it isn't worth optimizing. - */ - if (lastCountSize && (lastCountSize + bitstreamSize) < 4) { - /* lastCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ - assert(lastCountSize + bitstreamSize == 3); - DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " - "emitting an uncompressed block."); - return 0; - } - } - - DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart)); - return (size_t)(op - ostart); -} - -static size_t -ZSTD_entropyCompressSeqStore_wExtLitBuffer( - void* dst, size_t dstCapacity, - const void* literals, size_t litSize, - size_t blockSize, - const SeqStore_t* seqStorePtr, - const ZSTD_entropyCTables_t* prevEntropy, - ZSTD_entropyCTables_t* nextEntropy, - const ZSTD_CCtx_params* cctxParams, - void* entropyWorkspace, size_t entropyWkspSize, - int bmi2) -{ - size_t const cSize = ZSTD_entropyCompressSeqStore_internal( - dst, dstCapacity, - literals, litSize, - seqStorePtr, prevEntropy, nextEntropy, cctxParams, - entropyWorkspace, entropyWkspSize, bmi2); - if (cSize == 0) return 0; - /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. - * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. - */ - if ((cSize == ERROR(dstSize_tooSmall)) & (blockSize <= dstCapacity)) { - DEBUGLOG(4, "not enough dstCapacity (%zu) for ZSTD_entropyCompressSeqStore_internal()=> do not compress block", dstCapacity); - return 0; /* block not compressed */ - } - FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed"); - - /* Check compressibility */ - { size_t const maxCSize = blockSize - ZSTD_minGain(blockSize, cctxParams->cParams.strategy); - if (cSize >= maxCSize) return 0; /* block not compressed */ - } - DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize); - /* libzstd decoder before > v1.5.4 is not compatible with compressed blocks of size ZSTD_BLOCKSIZE_MAX exactly. - * This restriction is indirectly already fulfilled by respecting ZSTD_minGain() condition above. - */ - assert(cSize < ZSTD_BLOCKSIZE_MAX); - return cSize; + return ZSTD_rust_entropyCompressSeqStore_internal( + dst, dstCapacity, + literals, litSize, + seqStorePtr, prevEntropy, nextEntropy, + (int)cctxParams->cParams.strategy, + ZSTD_literalsCompressionIsDisabled(cctxParams), + entropyWorkspace, entropyWkspSize, bmi2); } static size_t @@ -2923,13 +2682,12 @@ ZSTD_entropyCompressSeqStore( void* entropyWorkspace, size_t entropyWkspSize, int bmi2) { - return ZSTD_entropyCompressSeqStore_wExtLitBuffer( + return ZSTD_rust_entropyCompressSeqStore( + seqStorePtr, prevEntropy, nextEntropy, + (int)cctxParams->cParams.strategy, + ZSTD_literalsCompressionIsDisabled(cctxParams), dst, dstCapacity, - seqStorePtr->litStart, (size_t)(seqStorePtr->lit - seqStorePtr->litStart), srcSize, - seqStorePtr, - prevEntropy, nextEntropy, - cctxParams, entropyWorkspace, entropyWkspSize, bmi2); } @@ -3299,87 +3057,8 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) static size_t ZSTD_copyBlockSequences(SeqCollector* seqCollector, const SeqStore_t* seqStore, const U32 prevRepcodes[ZSTD_REP_NUM]) { - const SeqDef* inSeqs = seqStore->sequencesStart; - const size_t nbInSequences = (size_t)(seqStore->sequences - inSeqs); - const size_t nbInLiterals = (size_t)(seqStore->lit - seqStore->litStart); - - ZSTD_Sequence* outSeqs = seqCollector->seqIndex == 0 ? seqCollector->seqStart : seqCollector->seqStart + seqCollector->seqIndex; - const size_t nbOutSequences = nbInSequences + 1; - size_t nbOutLiterals = 0; - Repcodes_t repcodes; - size_t i; - - /* Bounds check that we have enough space for every input sequence - * and the block delimiter - */ - assert(seqCollector->seqIndex <= seqCollector->maxSequences); - RETURN_ERROR_IF( - nbOutSequences > (size_t)(seqCollector->maxSequences - seqCollector->seqIndex), - dstSize_tooSmall, - "Not enough space to copy sequences"); - - ZSTD_memcpy(&repcodes, prevRepcodes, sizeof(repcodes)); - for (i = 0; i < nbInSequences; ++i) { - U32 rawOffset; - outSeqs[i].litLength = inSeqs[i].litLength; - outSeqs[i].matchLength = inSeqs[i].mlBase + MINMATCH; - outSeqs[i].rep = 0; - - /* Handle the possible single length >= 64K - * There can only be one because we add MINMATCH to every match length, - * and blocks are at most 128K. - */ - if (i == seqStore->longLengthPos) { - if (seqStore->longLengthType == ZSTD_llt_literalLength) { - outSeqs[i].litLength += 0x10000; - } else if (seqStore->longLengthType == ZSTD_llt_matchLength) { - outSeqs[i].matchLength += 0x10000; - } - } - - /* Determine the raw offset given the offBase, which may be a repcode. */ - if (OFFBASE_IS_REPCODE(inSeqs[i].offBase)) { - const U32 repcode = OFFBASE_TO_REPCODE(inSeqs[i].offBase); - assert(repcode > 0); - outSeqs[i].rep = repcode; - if (outSeqs[i].litLength != 0) { - rawOffset = repcodes.rep[repcode - 1]; - } else { - if (repcode == 3) { - assert(repcodes.rep[0] > 1); - rawOffset = repcodes.rep[0] - 1; - } else { - rawOffset = repcodes.rep[repcode]; - } - } - } else { - rawOffset = OFFBASE_TO_OFFSET(inSeqs[i].offBase); - } - outSeqs[i].offset = rawOffset; - - /* Update repcode history for the sequence */ - ZSTD_updateRep(repcodes.rep, - inSeqs[i].offBase, - inSeqs[i].litLength == 0); - - nbOutLiterals += outSeqs[i].litLength; - } - /* Insert last literals (if any exist) in the block as a sequence with ml == off == 0. - * If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker - * for the block boundary, according to the API. - */ - assert(nbInLiterals >= nbOutLiterals); - { - const size_t lastLLSize = nbInLiterals - nbOutLiterals; - outSeqs[nbInSequences].litLength = (U32)lastLLSize; - outSeqs[nbInSequences].matchLength = 0; - outSeqs[nbInSequences].offset = 0; - assert(nbOutSequences == nbInSequences + 1); - } - seqCollector->seqIndex += nbOutSequences; - assert(seqCollector->seqIndex <= seqCollector->maxSequences); - - return 0; + /* The implementation lives in rust/src/zstd_compress_stats.rs. */ + return ZSTD_rust_copyBlockSequences(seqCollector, seqStore, prevRepcodes); } /* ZSTD_sequenceBound() lives in rust/src/zstd_compress_api.rs. */ @@ -3475,178 +3154,14 @@ writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock) DEBUGLOG(5, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock); } -/** ZSTD_buildBlockEntropyStats_literals() : - * Builds entropy for the literals. - * Stores literals block type (raw, rle, compressed, repeat) and - * huffman description table to hufMetadata. - * Requires ENTROPY_WORKSPACE_SIZE workspace - * @return : size of huffman description table, or an error code - */ -static size_t -ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize, - const ZSTD_hufCTables_t* prevHuf, - ZSTD_hufCTables_t* nextHuf, - ZSTD_hufCTablesMetadata_t* hufMetadata, - const int literalsCompressionIsDisabled, - void* workspace, size_t wkspSize, - int hufFlags) -{ - BYTE* const wkspStart = (BYTE*)workspace; - BYTE* const wkspEnd = wkspStart + wkspSize; - BYTE* const countWkspStart = wkspStart; - unsigned* const countWksp = (unsigned*)workspace; - const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned); - BYTE* const nodeWksp = countWkspStart + countWkspSize; - const size_t nodeWkspSize = (size_t)(wkspEnd - nodeWksp); - unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; - unsigned huffLog = LitHufLog; - HUF_repeat repeat = prevHuf->repeatMode; - DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_literals (srcSize=%zu)", srcSize); - - /* Prepare nextEntropy assuming reusing the existing table */ - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - - if (literalsCompressionIsDisabled) { - DEBUGLOG(5, "set_basic - disabled"); - hufMetadata->hType = set_basic; - return 0; - } - - /* small ? don't even attempt compression (speed opt) */ -#ifndef COMPRESS_LITERALS_SIZE_MIN -# define COMPRESS_LITERALS_SIZE_MIN 63 /* heuristic */ -#endif - { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; - if (srcSize <= minLitSize) { - DEBUGLOG(5, "set_basic - too small"); - hufMetadata->hType = set_basic; - return 0; - } } - - /* Scan input and build symbol stats */ - { size_t const largest = - HIST_count_wksp (countWksp, &maxSymbolValue, - (const BYTE*)src, srcSize, - workspace, wkspSize); - FORWARD_IF_ERROR(largest, "HIST_count_wksp failed"); - if (largest == srcSize) { - /* only one literal symbol */ - DEBUGLOG(5, "set_rle"); - hufMetadata->hType = set_rle; - return 0; - } - if (largest <= (srcSize >> 7)+4) { - /* heuristic: likely not compressible */ - DEBUGLOG(5, "set_basic - no gain"); - hufMetadata->hType = set_basic; - return 0; - } } - - /* Validate the previous Huffman table */ - if (repeat == HUF_repeat_check - && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) { - repeat = HUF_repeat_none; - } - - /* Build Huffman Tree */ - ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); - huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, hufFlags); - assert(huffLog <= LitHufLog); - { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, - maxSymbolValue, huffLog, - nodeWksp, nodeWkspSize); - FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp"); - huffLog = (U32)maxBits; - } - { /* Build and write the CTable */ - size_t const newCSize = HUF_estimateCompressedSize( - (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue); - size_t const hSize = HUF_writeCTable_wksp( - hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer), - (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog, - nodeWksp, nodeWkspSize); - /* Check against repeating the previous CTable */ - if (repeat != HUF_repeat_none) { - size_t const oldCSize = HUF_estimateCompressedSize( - (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); - if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) { - DEBUGLOG(5, "set_repeat - smaller"); - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - hufMetadata->hType = set_repeat; - return 0; - } } - if (newCSize + hSize >= srcSize) { - DEBUGLOG(5, "set_basic - no gains"); - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - hufMetadata->hType = set_basic; - return 0; - } - DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize); - hufMetadata->hType = set_compressed; - nextHuf->repeatMode = HUF_repeat_check; - return hSize; - } -} - - -/* ZSTD_buildDummySequencesStatistics(): - * Returns a ZSTD_symbolEncodingTypeStats_t with all encoding types as set_basic, - * and updates nextEntropy to the appropriate repeatMode. - */ -static ZSTD_symbolEncodingTypeStats_t -ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) -{ - ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0, 0}; - nextEntropy->litlength_repeatMode = FSE_repeat_none; - nextEntropy->offcode_repeatMode = FSE_repeat_none; - nextEntropy->matchlength_repeatMode = FSE_repeat_none; - return stats; -} - -/** ZSTD_buildBlockEntropyStats_sequences() : - * Builds entropy for the sequences. - * Stores symbol compression modes and fse table to fseMetadata. - * Requires ENTROPY_WORKSPACE_SIZE wksp. - * @return : size of fse tables or error code */ -static size_t -ZSTD_buildBlockEntropyStats_sequences( - const SeqStore_t* seqStorePtr, - const ZSTD_fseCTables_t* prevEntropy, - ZSTD_fseCTables_t* nextEntropy, - const ZSTD_CCtx_params* cctxParams, - ZSTD_fseCTablesMetadata_t* fseMetadata, - void* workspace, size_t wkspSize) -{ - ZSTD_strategy const strategy = cctxParams->cParams.strategy; - size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); - BYTE* const ostart = fseMetadata->fseTablesBuffer; - BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer); - BYTE* op = ostart; - unsigned* countWorkspace = (unsigned*)workspace; - unsigned* entropyWorkspace = countWorkspace + (MaxSeq + 1); - size_t entropyWorkspaceSize = wkspSize - (MaxSeq + 1) * sizeof(*countWorkspace); - ZSTD_symbolEncodingTypeStats_t stats; - - DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_sequences (nbSeq=%zu)", nbSeq); - stats = nbSeq != 0 ? ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, - prevEntropy, nextEntropy, op, oend, - strategy, countWorkspace, - entropyWorkspace, entropyWorkspaceSize) - : ZSTD_buildDummySequencesStatistics(nextEntropy); - FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); - fseMetadata->llType = (SymbolEncodingType_e) stats.LLtype; - fseMetadata->ofType = (SymbolEncodingType_e) stats.Offtype; - fseMetadata->mlType = (SymbolEncodingType_e) stats.MLtype; - fseMetadata->lastCountSize = stats.lastCountSize; - return stats.size; -} - - /** ZSTD_buildBlockEntropyStats() : * Builds entropy for the block. * Requires workspace size ENTROPY_WORKSPACE_SIZE * @return : 0 on success, or an error code * Note : also employed in superblock + * + * The implementation, together with its literals/sequences/dummy helpers, + * lives in rust/src/zstd_compress_stats.rs. */ size_t ZSTD_buildBlockEntropyStats( const SeqStore_t* seqStorePtr, @@ -3656,26 +3171,12 @@ size_t ZSTD_buildBlockEntropyStats( ZSTD_entropyCTablesMetadata_t* entropyMetadata, void* workspace, size_t wkspSize) { - size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart); - int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD); - int const hufFlags = huf_useOptDepth ? HUF_flags_optimalDepth : 0; - - entropyMetadata->hufMetadata.hufDesSize = - ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize, - &prevEntropy->huf, &nextEntropy->huf, - &entropyMetadata->hufMetadata, - ZSTD_literalsCompressionIsDisabled(cctxParams), - workspace, wkspSize, hufFlags); - - FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed"); - entropyMetadata->fseMetadata.fseTablesSize = - ZSTD_buildBlockEntropyStats_sequences(seqStorePtr, - &prevEntropy->fse, &nextEntropy->fse, - cctxParams, - &entropyMetadata->fseMetadata, - workspace, wkspSize); - FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildBlockEntropyStats_sequences failed"); - return 0; + return ZSTD_rust_buildBlockEntropyStats( + seqStorePtr, prevEntropy, nextEntropy, + (int)cctxParams->cParams.strategy, + ZSTD_literalsCompressionIsDisabled(cctxParams), + entropyMetadata, + workspace, wkspSize); } /* Returns the size estimate for the literals section (header + content) of a block */ diff --git a/lib/compress/zstd_compress_superblock.c b/lib/compress/zstd_compress_superblock.c index 8c7722c51..cebd404e2 100644 --- a/lib/compress/zstd_compress_superblock.c +++ b/lib/compress/zstd_compress_superblock.c @@ -29,7 +29,7 @@ size_t ZSTD_rust_compressSuperBlock( const SeqStore_t* seqStore, const ZSTD_compressedBlockState_t* prevCBlock, ZSTD_compressedBlockState_t* nextCBlock, - const ZSTD_CCtx_params* cctxParams, + int strategy, int disableLiteralCompression, void* workspace, size_t wkspSize, int bmi2, U32 windowLog, size_t targetCBlockSize, void* dst, size_t dstCapacity, @@ -45,7 +45,8 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, &zc->seqStore, zc->blockState.prevCBlock, zc->blockState.nextCBlock, - &zc->appliedParams, + (int)zc->appliedParams.cParams.strategy, + ZSTD_literalsCompressionIsDisabled(&zc->appliedParams), zc->tmpWorkspace, zc->tmpWkspSize, zc->bmi2, zc->appliedParams.cParams.windowLog, zc->appliedParams.targetCBlockSize, diff --git a/rust/README.md b/rust/README.md index 5eea53a91..b2e01e4fb 100644 --- a/rust/README.md +++ b/rust/README.md @@ -31,6 +31,13 @@ zstd ABI: - `zstd_presplit` chooses split points for full compression blocks. - `zstd_compress_literals` emits raw, RLE, and Huffman literal sections while preserving the compressor's Huffman-table repeat state. + - `zstd_compress_stats` converts stored sequences into symbol codes, + selects each block's symbol encoding types, compresses a seqStore's + literals and sequences into a compressed-block body, builds the block + entropy statistics shared with the superblock writer and the block + splitter, and exports collected sequences in the public `ZSTD_Sequence` + format. Its C shims extract the sequence store, the entropy-table + leaves, and the two `ZSTD_CCtx_params` scalars these paths read. - `zstd_compress_frame` serializes frame headers, skippable frames, and the last empty block; it takes scalar frame parameters so the C-owned `ZSTD_CCtx_params` layout never crosses the language boundary. diff --git a/rust/src/lib.rs b/rust/src/lib.rs index a7f47a257..570209abf 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -32,6 +32,8 @@ pub mod zstd_compress_params; #[cfg(feature = "compression")] pub mod zstd_compress_sequences; #[cfg(feature = "compression")] +pub mod zstd_compress_stats; +#[cfg(feature = "compression")] pub mod zstd_compress_superblock; #[cfg(feature = "decompression")] pub mod zstd_ddict; diff --git a/rust/src/zstd_compress_literals.rs b/rust/src/zstd_compress_literals.rs index cf2047f2d..dd8549a92 100644 --- a/rust/src/zstd_compress_literals.rs +++ b/rust/src/zstd_compress_literals.rs @@ -65,7 +65,7 @@ unsafe fn copy_bytes(dst: *mut u8, src: *const u8, size: usize) { } #[inline] -unsafe fn copy_huf_tables(dst: *mut ZSTD_hufCTables_t, src: *const ZSTD_hufCTables_t) { +pub(crate) unsafe fn copy_huf_tables(dst: *mut ZSTD_hufCTables_t, src: *const ZSTD_hufCTables_t) { // Copy as bytes, not as a Rust struct: C's memcpy preserves the four tail // padding bytes that this layout has on 64-bit targets. Some C callers // copy or compare the complete entropy state, so leaving those bytes from @@ -192,8 +192,10 @@ fn min_literals_to_compress(strategy: c_int, huf_repeat: c_int) -> usize { } } +/// C's `ZSTD_minGain()`: minimum compression gain required to emit a +/// compressed block or compressed literals section. #[inline] -fn min_gain(src_size: usize, strategy: c_int) -> usize { +pub(crate) fn min_gain(src_size: usize, strategy: c_int) -> usize { let min_log = if strategy >= ZSTD_BTULTRA { strategy.saturating_sub(1) as u32 } else { diff --git a/rust/src/zstd_compress_stats.rs b/rust/src/zstd_compress_stats.rs new file mode 100644 index 000000000..d6121a1ab --- /dev/null +++ b/rust/src/zstd_compress_stats.rs @@ -0,0 +1,1549 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] + +//! Sequence statistics and seqStore entropy compression. +//! +//! This module translates the block entropy-statistics layer of +//! `zstd_compress.c`: `ZSTD_seqToCodes()`, `ZSTD_buildSequencesStatistics()`, +//! `ZSTD_entropyCompressSeqStore*()`, `ZSTD_buildBlockEntropyStats*()`, and +//! `ZSTD_copyBlockSequences()`. The `ZSTD_CCtx` and `ZSTD_CCtx_params` +//! layouts stay private to C: the C shims extract the sequence store, the +//! entropy-table leaves, and the two parameter scalars these paths read +//! (the compression strategy and the literals-compression switch). Block +//! dispatch, block splitting, and the block-size estimation heuristics also +//! remain in C for a later slice. + +use crate::bits::ZSTD_highbit32; +use crate::common::{ + DEFAULT_MAX_OFF, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, LL_FSE_LOG, LONGNBSEQ, MAX_LL, MAX_ML, + MAX_OFF, MAX_SEQ, MINMATCH, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, ML_FSE_LOG, OFF_FSE_LOG, + OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, ZSTD_MAX_FSE_HEADERS_SIZE, ZSTD_MAX_HUF_HEADER_SIZE, + ZSTD_REP_NUM, +}; +use crate::errors::{ERR_isError, ZstdErrorCode, ERROR}; +use crate::hist::{HIST_countFast_wksp, HIST_count_wksp}; +use crate::huf_compress::{ + HUF_buildCTable_wksp, HUF_estimateCompressedSize, HUF_optimalTableLog, HUF_validateCTable, + HUF_writeCTable_wksp, +}; +use crate::mem::{MEM_32bits, MEM_writeLE16}; +use crate::zstd_compress_literals::{ + copy_huf_tables, min_gain, ZSTD_compressLiterals, ZSTD_hufCTables_t, +}; +use crate::zstd_compress_sequences::{ + SeqDef, ZSTD_buildCTable, ZSTD_encodeSequences, ZSTD_selectEncodingType, +}; +use std::ffi::c_void; +use std::mem::{size_of, size_of_val}; +use std::os::raw::{c_int, c_uint}; +use std::ptr; + +const SET_BASIC: c_int = 0; +const SET_RLE: c_int = 1; +const SET_COMPRESSED: c_int = 2; +const SET_REPEAT: c_int = 3; + +const FSE_REPEAT_NONE: c_int = 0; + +const HUF_REPEAT_NONE: c_int = 0; +const HUF_REPEAT_CHECK: c_int = 1; +const HUF_REPEAT_VALID: c_int = 2; + +const HUF_FLAGS_OPTIMAL_DEPTH: c_int = 1 << 1; +const HUF_OPTIMAL_DEPTH_THRESHOLD: c_int = 8; /* ZSTD_btultra */ + +const ZSTD_DEFAULT_DISALLOWED: c_int = 0; +const ZSTD_DEFAULT_ALLOWED: c_int = 1; + +const HUF_SYMBOLVALUE_MAX: c_uint = 255; +const LIT_HUF_LOG: c_uint = 11; +const COMPRESS_LITERALS_SIZE_MIN: usize = 63; +const SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO: usize = 20; + +const STREAM_ACCUMULATOR_MIN_32: u32 = 25; +const STREAM_ACCUMULATOR_MIN_64: u32 = 57; + +const LL_DELTA_CODE: u32 = 19; +const ML_DELTA_CODE: u32 = 36; + +pub(crate) const ZSTD_LLT_LITERAL_LENGTH: c_int = 1; +pub(crate) const ZSTD_LLT_MATCH_LENGTH: c_int = 2; + +/// `FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)` from the C headers. +pub const OFF_CTABLE_SIZE: usize = 1 + (1 << (OFF_FSE_LOG - 1)) + ((MAX_OFF + 1) * 2); +/// `FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)` from the C headers. +pub const ML_CTABLE_SIZE: usize = 1 + (1 << (ML_FSE_LOG - 1)) + ((MAX_ML + 1) * 2); +/// `FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)` from the C headers. +pub const LL_CTABLE_SIZE: usize = 1 + (1 << (LL_FSE_LOG - 1)) + ((MAX_LL + 1) * 2); + +/// ABI-compatible `SeqStore_t` leaf layout from `zstd_compress_internal.h`. +/// +/// The C context itself remains opaque; its shims pass the `seqStore` member +/// directly, so this small, stable hot-path structure is the only sequence +/// storage representation crossing into Rust. +#[repr(C)] +pub struct SeqStore_t { + pub sequencesStart: *mut SeqDef, + pub sequences: *mut SeqDef, + pub litStart: *mut u8, + pub lit: *mut u8, + pub llCode: *mut u8, + pub mlCode: *mut u8, + pub ofCode: *mut u8, + pub maxNbSeq: usize, + pub maxNbLit: usize, + pub longLengthType: c_int, + pub longLengthPos: u32, +} + +/// C's `ZSTD_fseCTables_t`. The table element type is `FSE_CTable`, an +/// `unsigned`, and the table lengths are the header macros expanded above. +#[repr(C)] +pub struct ZSTD_fseCTables_t { + pub offcodeCTable: [u32; OFF_CTABLE_SIZE], + pub matchlengthCTable: [u32; ML_CTABLE_SIZE], + pub litlengthCTable: [u32; LL_CTABLE_SIZE], + pub offcode_repeatMode: c_int, + pub matchlength_repeatMode: c_int, + pub litlength_repeatMode: c_int, +} + +/// C's `ZSTD_entropyCTables_t`. +#[repr(C)] +pub struct ZSTD_entropyCTables_t { + pub huf: ZSTD_hufCTables_t, + pub fse: ZSTD_fseCTables_t, +} + +/// C's `ZSTD_compressedBlockState_t`. +#[repr(C)] +pub struct ZSTD_compressedBlockState_t { + pub entropy: ZSTD_entropyCTables_t, + pub rep: [u32; ZSTD_REP_NUM], +} + +/// C's `ZSTD_hufCTablesMetadata_t`. +#[repr(C)] +pub struct ZSTD_hufCTablesMetadata_t { + pub hType: c_int, + pub hufDesBuffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE], + pub hufDesSize: usize, +} + +/// C's `ZSTD_fseCTablesMetadata_t`. +#[repr(C)] +pub struct ZSTD_fseCTablesMetadata_t { + pub llType: c_int, + pub ofType: c_int, + pub mlType: c_int, + pub fseTablesBuffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE], + pub fseTablesSize: usize, + pub lastCountSize: usize, +} + +/// C's `ZSTD_entropyCTablesMetadata_t`. +#[repr(C)] +pub struct ZSTD_entropyCTablesMetadata_t { + pub hufMetadata: ZSTD_hufCTablesMetadata_t, + pub fseMetadata: ZSTD_fseCTablesMetadata_t, +} + +/// Public `ZSTD_Sequence` from `zstd.h`. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ZSTD_Sequence { + pub offset: u32, + pub litLength: u32, + pub matchLength: u32, + pub rep: u32, +} + +/// C's `SeqCollector` leaf from `zstd_compress_internal.h`. +#[repr(C)] +pub struct SeqCollector { + pub collectSequences: c_int, + pub seqStart: *mut ZSTD_Sequence, + pub seqIndex: usize, + pub maxSequences: usize, +} + +/// Result of `build_sequences_statistics()`; mirrors C's +/// `ZSTD_symbolEncodingTypeStats_t`, which no longer crosses the boundary. +pub(crate) struct SymbolEncodingTypeStats { + pub ll_type: u32, + pub off_type: u32, + pub ml_type: u32, + pub size: usize, + /// Accounts for a bug in zstd 1.3.4; see + /// `entropy_compress_seq_store_internal()`. + pub last_count_size: usize, + pub long_offsets: c_int, +} + +/// C's `ZSTD_updateRep()` repeat-offset history update. +#[inline] +pub(crate) fn update_rep(reps: &mut [u32; 3], off_base: u32, literal_length_is_zero: bool) { + if off_base > 3 { + reps[2] = reps[1]; + reps[1] = reps[0]; + reps[0] = off_base - 3; + return; + } + let rep_code = off_base - 1 + u32::from(literal_length_is_zero); + if rep_code == 0 { + return; + } + let current_offset = if rep_code == 3 { + reps[0].wrapping_sub(1) + } else { + reps[rep_code as usize] + }; + reps[2] = if rep_code >= 2 { reps[1] } else { reps[2] }; + reps[1] = reps[0]; + reps[0] = current_offset; +} + +#[inline] +fn ll_code(lit_length: u32) -> u32 { + const LL_CODE: [u8; 64] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, + 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + ]; + if lit_length > 63 { + ZSTD_highbit32(lit_length) + LL_DELTA_CODE + } else { + LL_CODE[lit_length as usize] as u32 + } +} + +#[inline] +fn ml_code(ml_base: u32) -> u32 { + const ML_CODE: [u8; 128] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, 40, 40, 40, 40, 40, 40, 40, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 41, 41, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + ]; + if ml_base > 127 { + ZSTD_highbit32(ml_base) + ML_DELTA_CODE + } else { + ML_CODE[ml_base as usize] as u32 + } +} + +#[inline] +unsafe fn copy_fse_tables(dst: *mut ZSTD_fseCTables_t, src: *const ZSTD_fseCTables_t) { + unsafe { + ptr::copy_nonoverlapping( + src.cast::(), + dst.cast::(), + size_of::(), + ) + }; +} + +/// Converts the stored lengths and distances into symbol codes. +/// +/// Retains its original exported name: the C dictionary builder and the +/// decodecorpus test tool link against it directly. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_seqToCodes(seq_store: *const SeqStore_t) -> c_int { + let store = unsafe { &*seq_store }; + let sequences = store.sequencesStart; + let nb_seq = unsafe { store.sequences.offset_from(sequences) } as usize; + let mut long_offsets = 0; + debug_assert!(nb_seq <= store.maxNbSeq); + + for index in 0..nb_seq { + let sequence = unsafe { sequences.add(index).read() }; + let of_code = ZSTD_highbit32(sequence.offBase); + unsafe { + *store.llCode.add(index) = ll_code(sequence.litLength as u32) as u8; + *store.ofCode.add(index) = of_code as u8; + *store.mlCode.add(index) = ml_code(sequence.mlBase as u32) as u8; + } + debug_assert!(MEM_32bits() || of_code < STREAM_ACCUMULATOR_MIN_64); + if MEM_32bits() && of_code >= STREAM_ACCUMULATOR_MIN_32 { + long_offsets = 1; + } + } + if store.longLengthType == ZSTD_LLT_LITERAL_LENGTH { + unsafe { *store.llCode.add(store.longLengthPos as usize) = MAX_LL as u8 }; + } + if store.longLengthType == ZSTD_LLT_MATCH_LENGTH { + unsafe { *store.mlCode.add(store.longLengthPos as usize) = MAX_ML as u8 }; + } + long_offsets +} + +/// C's `ZSTD_buildSequencesStatistics()`. +/// +/// Returns the statistics with a zstd error code in the `size` field on +/// failure, and updates `next_entropy` as a side effect. `nb_seq` must be +/// greater than zero, and the entropy workspace must hold at least +/// `ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1) * sizeof(U32)` bytes. +#[allow(clippy::too_many_arguments)] +pub(crate) unsafe fn build_sequences_statistics( + seq_store: *const SeqStore_t, + nb_seq: usize, + prev_entropy: *const ZSTD_fseCTables_t, + next_entropy: *mut ZSTD_fseCTables_t, + dst: *mut u8, + dst_capacity: usize, + strategy: c_int, + count_workspace: *mut c_uint, + entropy_workspace: *mut c_void, + entropy_wksp_size: usize, +) -> SymbolEncodingTypeStats { + let store = unsafe { &*seq_store }; + let mut stats = SymbolEncodingTypeStats { + ll_type: 0, + off_type: 0, + ml_type: 0, + size: 0, + last_count_size: 0, + long_offsets: 0, + }; + let mut written = 0usize; + + /* convert length/distances into codes */ + stats.long_offsets = unsafe { ZSTD_seqToCodes(seq_store) }; + debug_assert!(nb_seq != 0); /* ZSTD_selectEncodingType() divides by nb_seq */ + + /* build CTable for Literal Lengths */ + { + let mut max = MAX_LL as c_uint; + let most_frequent = unsafe { + HIST_countFast_wksp( + count_workspace, + &mut max, + store.llCode.cast::(), + nb_seq, + entropy_workspace, + entropy_wksp_size, + ) + }; /* can't fail */ + unsafe { (*next_entropy).litlength_repeatMode = (*prev_entropy).litlength_repeatMode }; + stats.ll_type = unsafe { + ZSTD_selectEncodingType( + ptr::addr_of_mut!((*next_entropy).litlength_repeatMode), + count_workspace, + max, + most_frequent, + nb_seq, + LL_FSE_LOG as c_uint, + ptr::addr_of!((*prev_entropy).litlengthCTable).cast::(), + LL_DEFAULT_NORM.as_ptr(), + LL_DEFAULT_NORM_LOG, + ZSTD_DEFAULT_ALLOWED, + strategy, + ) + } as u32; + let count_size = unsafe { + ZSTD_buildCTable( + dst.add(written).cast::(), + dst_capacity - written, + ptr::addr_of_mut!((*next_entropy).litlengthCTable).cast::(), + LL_FSE_LOG as u32, + stats.ll_type as c_int, + count_workspace, + max, + store.llCode, + nb_seq, + LL_DEFAULT_NORM.as_ptr(), + LL_DEFAULT_NORM_LOG, + MAX_LL as u32, + ptr::addr_of!((*prev_entropy).litlengthCTable).cast::(), + size_of::<[u32; LL_CTABLE_SIZE]>(), + entropy_workspace, + entropy_wksp_size, + ) + }; + if ERR_isError(count_size) { + stats.size = count_size; + return stats; + } + if stats.ll_type == SET_COMPRESSED as u32 { + stats.last_count_size = count_size; + } + written += count_size; + } + /* build CTable for Offsets */ + { + let mut max = MAX_OFF as c_uint; + let most_frequent = unsafe { + HIST_countFast_wksp( + count_workspace, + &mut max, + store.ofCode.cast::(), + nb_seq, + entropy_workspace, + entropy_wksp_size, + ) + }; /* can't fail */ + /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ + let default_policy = if max <= DEFAULT_MAX_OFF as c_uint { + ZSTD_DEFAULT_ALLOWED + } else { + ZSTD_DEFAULT_DISALLOWED + }; + unsafe { (*next_entropy).offcode_repeatMode = (*prev_entropy).offcode_repeatMode }; + stats.off_type = unsafe { + ZSTD_selectEncodingType( + ptr::addr_of_mut!((*next_entropy).offcode_repeatMode), + count_workspace, + max, + most_frequent, + nb_seq, + OFF_FSE_LOG as c_uint, + ptr::addr_of!((*prev_entropy).offcodeCTable).cast::(), + OF_DEFAULT_NORM.as_ptr(), + OF_DEFAULT_NORM_LOG, + default_policy, + strategy, + ) + } as u32; + let count_size = unsafe { + ZSTD_buildCTable( + dst.add(written).cast::(), + dst_capacity - written, + ptr::addr_of_mut!((*next_entropy).offcodeCTable).cast::(), + OFF_FSE_LOG as u32, + stats.off_type as c_int, + count_workspace, + max, + store.ofCode, + nb_seq, + OF_DEFAULT_NORM.as_ptr(), + OF_DEFAULT_NORM_LOG, + DEFAULT_MAX_OFF as u32, + ptr::addr_of!((*prev_entropy).offcodeCTable).cast::(), + size_of::<[u32; OFF_CTABLE_SIZE]>(), + entropy_workspace, + entropy_wksp_size, + ) + }; + if ERR_isError(count_size) { + stats.size = count_size; + return stats; + } + if stats.off_type == SET_COMPRESSED as u32 { + stats.last_count_size = count_size; + } + written += count_size; + } + /* build CTable for MatchLengths */ + { + let mut max = MAX_ML as c_uint; + let most_frequent = unsafe { + HIST_countFast_wksp( + count_workspace, + &mut max, + store.mlCode.cast::(), + nb_seq, + entropy_workspace, + entropy_wksp_size, + ) + }; /* can't fail */ + unsafe { (*next_entropy).matchlength_repeatMode = (*prev_entropy).matchlength_repeatMode }; + stats.ml_type = unsafe { + ZSTD_selectEncodingType( + ptr::addr_of_mut!((*next_entropy).matchlength_repeatMode), + count_workspace, + max, + most_frequent, + nb_seq, + ML_FSE_LOG as c_uint, + ptr::addr_of!((*prev_entropy).matchlengthCTable).cast::(), + ML_DEFAULT_NORM.as_ptr(), + ML_DEFAULT_NORM_LOG, + ZSTD_DEFAULT_ALLOWED, + strategy, + ) + } as u32; + let count_size = unsafe { + ZSTD_buildCTable( + dst.add(written).cast::(), + dst_capacity - written, + ptr::addr_of_mut!((*next_entropy).matchlengthCTable).cast::(), + ML_FSE_LOG as u32, + stats.ml_type as c_int, + count_workspace, + max, + store.mlCode, + nb_seq, + ML_DEFAULT_NORM.as_ptr(), + ML_DEFAULT_NORM_LOG, + MAX_ML as u32, + ptr::addr_of!((*prev_entropy).matchlengthCTable).cast::(), + size_of::<[u32; ML_CTABLE_SIZE]>(), + entropy_workspace, + entropy_wksp_size, + ) + }; + if ERR_isError(count_size) { + stats.size = count_size; + return stats; + } + if stats.ml_type == SET_COMPRESSED as u32 { + stats.last_count_size = count_size; + } + written += count_size; + } + stats.size = written; + stats +} + +/// C's `ZSTD_entropyCompressSeqStore_internal()`: compresses literals and +/// sequences into one compressed-block body. +#[allow(clippy::too_many_arguments)] +unsafe fn entropy_compress_seq_store_internal( + dst: *mut c_void, + dst_capacity: usize, + literals: *const c_void, + lit_size: usize, + seq_store: *const SeqStore_t, + prev_entropy: *const ZSTD_entropyCTables_t, + next_entropy: *mut ZSTD_entropyCTables_t, + strategy: c_int, + disable_literal_compression: c_int, + entropy_workspace: *mut c_void, + entropy_wksp_size: usize, + bmi2: c_int, +) -> usize { + let count = entropy_workspace.cast::(); + let entropy_workspace = unsafe { count.add(MAX_SEQ + 1) }.cast::(); + let entropy_wksp_size = entropy_wksp_size.wrapping_sub((MAX_SEQ + 1) * size_of::()); + let store = unsafe { &*seq_store }; + let sequences = store.sequencesStart; + let nb_seq = unsafe { store.sequences.offset_from(sequences) } as usize; + let ostart = dst.cast::(); + let mut written = 0usize; + let last_count_size; + let long_offsets; + + /* Compress literals */ + { + /* Base suspicion of uncompressibility on ratio of literals to sequences */ + let suspect_uncompressible = + c_int::from(nb_seq == 0 || lit_size / nb_seq >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO); + let c_size = unsafe { + ZSTD_compressLiterals( + dst, + dst_capacity, + literals, + lit_size, + entropy_workspace, + entropy_wksp_size, + ptr::addr_of!((*prev_entropy).huf), + ptr::addr_of_mut!((*next_entropy).huf), + strategy, + disable_literal_compression, + suspect_uncompressible, + bmi2, + ) + }; + if ERR_isError(c_size) { + return c_size; + } + debug_assert!(c_size <= dst_capacity); + written += c_size; + } + + /* Sequences Header */ + if (dst_capacity as isize).wrapping_sub(written as isize) < 3 /* max nbSeq Size */ + 1 + /* seqHead */ + { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + unsafe { + if nb_seq < 128 { + *ostart.add(written) = nb_seq as u8; + written += 1; + } else if nb_seq < LONGNBSEQ as usize { + *ostart.add(written) = ((nb_seq >> 8) + 0x80) as u8; + *ostart.add(written + 1) = nb_seq as u8; + written += 2; + } else { + *ostart.add(written) = 0xFF; + MEM_writeLE16( + ostart.add(written + 1).cast::(), + (nb_seq - LONGNBSEQ as usize) as u16, + ); + written += 3; + } + } + if nb_seq == 0 { + /* Copy the old tables over as if we repeated them */ + unsafe { + copy_fse_tables( + ptr::addr_of_mut!((*next_entropy).fse), + ptr::addr_of!((*prev_entropy).fse), + ) + }; + return written; + } + { + let seq_head = written; + written += 1; + /* build stats for sequences */ + let stats = unsafe { + build_sequences_statistics( + seq_store, + nb_seq, + ptr::addr_of!((*prev_entropy).fse), + ptr::addr_of_mut!((*next_entropy).fse), + ostart.add(written), + dst_capacity - written, + strategy, + count, + entropy_workspace, + entropy_wksp_size, + ) + }; + if ERR_isError(stats.size) { + return stats.size; + } + unsafe { + *ostart.add(seq_head) = + ((stats.ll_type << 6) + (stats.off_type << 4) + (stats.ml_type << 2)) as u8; + } + last_count_size = stats.last_count_size; + written += stats.size; + long_offsets = stats.long_offsets; + } + + { + let bitstream_size = unsafe { + ZSTD_encodeSequences( + ostart.add(written).cast::(), + dst_capacity - written, + ptr::addr_of!((*next_entropy).fse.matchlengthCTable).cast::(), + store.mlCode, + ptr::addr_of!((*next_entropy).fse.offcodeCTable).cast::(), + store.ofCode, + ptr::addr_of!((*next_entropy).fse.litlengthCTable).cast::(), + store.llCode, + sequences, + nb_seq, + long_offsets, + bmi2, + ) + }; + if ERR_isError(bitstream_size) { + return bitstream_size; + } + written += bitstream_size; + /* zstd versions <= 1.3.4 mistakenly report corruption when + * FSE_readNCount() receives a buffer < 4 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1146. + * This can happen when the last set_compressed table present is 2 + * bytes and the bitstream is only one byte. + * In this exceedingly rare case, we will simply emit an uncompressed + * block, since it isn't worth optimizing. + */ + if last_count_size != 0 && last_count_size + bitstream_size < 4 { + /* last_count_size >= 2 && bitstream_size > 0 ==> last_count_size == 3 */ + debug_assert!(last_count_size + bitstream_size == 3); + return 0; + } + } + + written +} + +/// C's `ZSTD_entropyCompressSeqStore_wExtLitBuffer()`: adds the +/// incompressibility fallbacks around the internal entropy pass. +#[allow(clippy::too_many_arguments)] +unsafe fn entropy_compress_seq_store_w_ext_lit_buffer( + dst: *mut c_void, + dst_capacity: usize, + literals: *const c_void, + lit_size: usize, + block_size: usize, + seq_store: *const SeqStore_t, + prev_entropy: *const ZSTD_entropyCTables_t, + next_entropy: *mut ZSTD_entropyCTables_t, + strategy: c_int, + disable_literal_compression: c_int, + entropy_workspace: *mut c_void, + entropy_wksp_size: usize, + bmi2: c_int, +) -> usize { + let c_size = unsafe { + entropy_compress_seq_store_internal( + dst, + dst_capacity, + literals, + lit_size, + seq_store, + prev_entropy, + next_entropy, + strategy, + disable_literal_compression, + entropy_workspace, + entropy_wksp_size, + bmi2, + ) + }; + if c_size == 0 { + return 0; + } + /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. + * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. + */ + if c_size == ERROR(ZstdErrorCode::DstSizeTooSmall) && block_size <= dst_capacity { + return 0; /* block not compressed */ + } + if ERR_isError(c_size) { + return c_size; + } + + /* Check compressibility */ + let max_c_size = block_size.wrapping_sub(min_gain(block_size, strategy)); + if c_size >= max_c_size { + return 0; /* block not compressed */ + } + c_size +} + +/// C ABI wrapper for the external-literal-buffer entropy pass; the C shim +/// keeps the `ZSTD_entropyCompressSeqStore_internal()` name and extracts the +/// two `ZSTD_CCtx_params` scalars. +#[allow(clippy::too_many_arguments)] +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_entropyCompressSeqStore_internal( + dst: *mut c_void, + dst_capacity: usize, + literals: *const c_void, + lit_size: usize, + seq_store: *const SeqStore_t, + prev_entropy: *const ZSTD_entropyCTables_t, + next_entropy: *mut ZSTD_entropyCTables_t, + strategy: c_int, + disable_literal_compression: c_int, + entropy_workspace: *mut c_void, + entropy_wksp_size: usize, + bmi2: c_int, +) -> usize { + unsafe { + entropy_compress_seq_store_internal( + dst, + dst_capacity, + literals, + lit_size, + seq_store, + prev_entropy, + next_entropy, + strategy, + disable_literal_compression, + entropy_workspace, + entropy_wksp_size, + bmi2, + ) + } +} + +/// C ABI implementation behind the `ZSTD_entropyCompressSeqStore()` shim. +#[allow(clippy::too_many_arguments)] +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_entropyCompressSeqStore( + seq_store: *const SeqStore_t, + prev_entropy: *const ZSTD_entropyCTables_t, + next_entropy: *mut ZSTD_entropyCTables_t, + strategy: c_int, + disable_literal_compression: c_int, + dst: *mut c_void, + dst_capacity: usize, + src_size: usize, + entropy_workspace: *mut c_void, + entropy_wksp_size: usize, + bmi2: c_int, +) -> usize { + let store = unsafe { &*seq_store }; + let lit_size = unsafe { store.lit.offset_from(store.litStart) } as usize; + unsafe { + entropy_compress_seq_store_w_ext_lit_buffer( + dst, + dst_capacity, + store.litStart.cast::(), + lit_size, + src_size, + seq_store, + prev_entropy, + next_entropy, + strategy, + disable_literal_compression, + entropy_workspace, + entropy_wksp_size, + bmi2, + ) + } +} + +/// C's `ZSTD_buildBlockEntropyStats_literals()`. +/// +/// Stores the literals block type and Huffman description table into +/// `huf_metadata` and returns the description size, or an error code. +#[allow(clippy::too_many_arguments)] +unsafe fn build_block_entropy_stats_literals( + src: *const c_void, + src_size: usize, + prev_huf: *const ZSTD_hufCTables_t, + next_huf: *mut ZSTD_hufCTables_t, + huf_metadata: *mut ZSTD_hufCTablesMetadata_t, + literals_compression_is_disabled: c_int, + workspace: *mut c_void, + wksp_size: usize, + huf_flags: c_int, +) -> usize { + let count_wksp = workspace.cast::(); + let count_wksp_size = (HUF_SYMBOLVALUE_MAX as usize + 1) * size_of::(); + let node_wksp = unsafe { workspace.cast::().add(count_wksp_size) }.cast::(); + let node_wksp_size = wksp_size.wrapping_sub(count_wksp_size); + let mut max_symbol_value = HUF_SYMBOLVALUE_MAX; + let mut huff_log = LIT_HUF_LOG; + let mut repeat = unsafe { (*prev_huf).repeatMode }; + + /* Prepare nextEntropy assuming reusing the existing table */ + unsafe { copy_huf_tables(next_huf, prev_huf) }; + + if literals_compression_is_disabled != 0 { + unsafe { (*huf_metadata).hType = SET_BASIC }; + return 0; + } + + /* small ? don't even attempt compression (speed opt) */ + { + let min_lit_size = if repeat == HUF_REPEAT_VALID { + 6 + } else { + COMPRESS_LITERALS_SIZE_MIN + }; + if src_size <= min_lit_size { + unsafe { (*huf_metadata).hType = SET_BASIC }; + return 0; + } + } + + /* Scan input and build symbol stats */ + { + let largest = unsafe { + HIST_count_wksp( + count_wksp, + &mut max_symbol_value, + src, + src_size, + workspace, + wksp_size, + ) + }; + if ERR_isError(largest) { + return largest; + } + if largest == src_size { + /* only one literal symbol */ + unsafe { (*huf_metadata).hType = SET_RLE }; + return 0; + } + if largest <= (src_size >> 7) + 4 { + /* heuristic: likely not compressible */ + unsafe { (*huf_metadata).hType = SET_BASIC }; + return 0; + } + } + + /* Validate the previous Huffman table */ + if repeat == HUF_REPEAT_CHECK + && unsafe { + HUF_validateCTable( + ptr::addr_of!((*prev_huf).CTable).cast::(), + count_wksp, + max_symbol_value, + ) + } == 0 + { + repeat = HUF_REPEAT_NONE; + } + + /* Build Huffman Tree */ + unsafe { + ptr::write_bytes( + ptr::addr_of_mut!((*next_huf).CTable).cast::(), + 0, + size_of_val(&(*next_huf).CTable), + ); + } + huff_log = unsafe { + HUF_optimalTableLog( + huff_log, + src_size, + max_symbol_value, + node_wksp, + node_wksp_size, + ptr::addr_of_mut!((*next_huf).CTable).cast::(), + count_wksp, + huf_flags, + ) + }; + debug_assert!(huff_log <= LIT_HUF_LOG); + { + let max_bits = unsafe { + HUF_buildCTable_wksp( + ptr::addr_of_mut!((*next_huf).CTable).cast::(), + count_wksp, + max_symbol_value, + huff_log, + node_wksp, + node_wksp_size, + ) + }; + if ERR_isError(max_bits) { + return max_bits; + } + huff_log = max_bits as c_uint; + } + { + /* Build and write the CTable */ + let new_c_size = unsafe { + HUF_estimateCompressedSize( + ptr::addr_of!((*next_huf).CTable).cast::(), + count_wksp, + max_symbol_value, + ) + }; + let h_size = unsafe { + HUF_writeCTable_wksp( + ptr::addr_of_mut!((*huf_metadata).hufDesBuffer).cast::(), + size_of::<[u8; ZSTD_MAX_HUF_HEADER_SIZE]>(), + ptr::addr_of!((*next_huf).CTable).cast::(), + max_symbol_value, + huff_log, + node_wksp, + node_wksp_size, + ) + }; + /* Check against repeating the previous CTable */ + if repeat != HUF_REPEAT_NONE { + let old_c_size = unsafe { + HUF_estimateCompressedSize( + ptr::addr_of!((*prev_huf).CTable).cast::(), + count_wksp, + max_symbol_value, + ) + }; + if old_c_size < src_size + && (old_c_size <= h_size.wrapping_add(new_c_size) + || h_size.wrapping_add(12) >= src_size) + { + unsafe { + copy_huf_tables(next_huf, prev_huf); + (*huf_metadata).hType = SET_REPEAT; + } + return 0; + } + } + if new_c_size.wrapping_add(h_size) >= src_size { + unsafe { + copy_huf_tables(next_huf, prev_huf); + (*huf_metadata).hType = SET_BASIC; + } + return 0; + } + unsafe { + (*huf_metadata).hType = SET_COMPRESSED; + (*next_huf).repeatMode = HUF_REPEAT_CHECK; + } + h_size + } +} + +/// C's `ZSTD_buildDummySequencesStatistics()`: marks every encoding type as +/// basic and resets the FSE repeat modes. +unsafe fn build_dummy_sequences_statistics( + next_entropy: *mut ZSTD_fseCTables_t, +) -> SymbolEncodingTypeStats { + unsafe { + (*next_entropy).litlength_repeatMode = FSE_REPEAT_NONE; + (*next_entropy).offcode_repeatMode = FSE_REPEAT_NONE; + (*next_entropy).matchlength_repeatMode = FSE_REPEAT_NONE; + } + SymbolEncodingTypeStats { + ll_type: SET_BASIC as u32, + off_type: SET_BASIC as u32, + ml_type: SET_BASIC as u32, + size: 0, + last_count_size: 0, + long_offsets: 0, + } +} + +/// C's `ZSTD_buildBlockEntropyStats_sequences()`. +#[allow(clippy::too_many_arguments)] +unsafe fn build_block_entropy_stats_sequences( + seq_store: *const SeqStore_t, + prev_entropy: *const ZSTD_fseCTables_t, + next_entropy: *mut ZSTD_fseCTables_t, + strategy: c_int, + fse_metadata: *mut ZSTD_fseCTablesMetadata_t, + workspace: *mut c_void, + wksp_size: usize, +) -> usize { + let store = unsafe { &*seq_store }; + let nb_seq = unsafe { store.sequences.offset_from(store.sequencesStart) } as usize; + let count_workspace = workspace.cast::(); + let entropy_workspace = unsafe { count_workspace.add(MAX_SEQ + 1) }.cast::(); + let entropy_workspace_size = wksp_size.wrapping_sub((MAX_SEQ + 1) * size_of::()); + + let stats = if nb_seq != 0 { + unsafe { + build_sequences_statistics( + seq_store, + nb_seq, + prev_entropy, + next_entropy, + ptr::addr_of_mut!((*fse_metadata).fseTablesBuffer).cast::(), + ZSTD_MAX_FSE_HEADERS_SIZE, + strategy, + count_workspace, + entropy_workspace, + entropy_workspace_size, + ) + } + } else { + unsafe { build_dummy_sequences_statistics(next_entropy) } + }; + if ERR_isError(stats.size) { + return stats.size; + } + unsafe { + (*fse_metadata).llType = stats.ll_type as c_int; + (*fse_metadata).ofType = stats.off_type as c_int; + (*fse_metadata).mlType = stats.ml_type as c_int; + (*fse_metadata).lastCountSize = stats.last_count_size; + } + stats.size +} + +/// C's `ZSTD_buildBlockEntropyStats()`: builds entropy statistics for one +/// block. Requires an `ENTROPY_WORKSPACE_SIZE` workspace. +#[allow(clippy::too_many_arguments)] +pub(crate) unsafe fn build_block_entropy_stats( + seq_store: *const SeqStore_t, + prev_entropy: *const ZSTD_entropyCTables_t, + next_entropy: *mut ZSTD_entropyCTables_t, + strategy: c_int, + disable_literal_compression: c_int, + entropy_metadata: *mut ZSTD_entropyCTablesMetadata_t, + workspace: *mut c_void, + wksp_size: usize, +) -> usize { + let store = unsafe { &*seq_store }; + let lit_size = unsafe { store.lit.offset_from(store.litStart) } as usize; + let huf_flags = if strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD { + HUF_FLAGS_OPTIMAL_DEPTH + } else { + 0 + }; + + let huf_des_size = unsafe { + build_block_entropy_stats_literals( + store.litStart.cast::(), + lit_size, + ptr::addr_of!((*prev_entropy).huf), + ptr::addr_of_mut!((*next_entropy).huf), + ptr::addr_of_mut!((*entropy_metadata).hufMetadata), + disable_literal_compression, + workspace, + wksp_size, + huf_flags, + ) + }; + unsafe { (*entropy_metadata).hufMetadata.hufDesSize = huf_des_size }; + if ERR_isError(huf_des_size) { + return huf_des_size; + } + + let fse_tables_size = unsafe { + build_block_entropy_stats_sequences( + seq_store, + ptr::addr_of!((*prev_entropy).fse), + ptr::addr_of_mut!((*next_entropy).fse), + strategy, + ptr::addr_of_mut!((*entropy_metadata).fseMetadata), + workspace, + wksp_size, + ) + }; + unsafe { (*entropy_metadata).fseMetadata.fseTablesSize = fse_tables_size }; + if ERR_isError(fse_tables_size) { + return fse_tables_size; + } + 0 +} + +/// C ABI wrapper behind the `ZSTD_buildBlockEntropyStats()` shim, which is +/// still exported from C for the block splitter's size estimation. +#[allow(clippy::too_many_arguments)] +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_buildBlockEntropyStats( + seq_store: *const SeqStore_t, + prev_entropy: *const ZSTD_entropyCTables_t, + next_entropy: *mut ZSTD_entropyCTables_t, + strategy: c_int, + disable_literal_compression: c_int, + entropy_metadata: *mut ZSTD_entropyCTablesMetadata_t, + workspace: *mut c_void, + wksp_size: usize, +) -> usize { + unsafe { + build_block_entropy_stats( + seq_store, + prev_entropy, + next_entropy, + strategy, + disable_literal_compression, + entropy_metadata, + workspace, + wksp_size, + ) + } +} + +/// C's `ZSTD_copyBlockSequences()`: exports a block's sequences in the public +/// `ZSTD_Sequence` format, appending the last-literals block delimiter. +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_copyBlockSequences( + seq_collector: *mut SeqCollector, + seq_store: *const SeqStore_t, + prev_repcodes: *const u32, +) -> usize { + let collector = unsafe { &mut *seq_collector }; + let store = unsafe { &*seq_store }; + let in_seqs = store.sequencesStart; + let nb_in_sequences = unsafe { store.sequences.offset_from(in_seqs) } as usize; + let nb_in_literals = unsafe { store.lit.offset_from(store.litStart) } as usize; + let out_seqs = if collector.seqIndex == 0 { + collector.seqStart + } else { + unsafe { collector.seqStart.add(collector.seqIndex) } + }; + let nb_out_sequences = nb_in_sequences + 1; + let mut nb_out_literals = 0usize; + + /* Bounds check that we have enough space for every input sequence + * and the block delimiter + */ + debug_assert!(collector.seqIndex <= collector.maxSequences); + if nb_out_sequences > collector.maxSequences.wrapping_sub(collector.seqIndex) { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + + let mut repcodes = [0u32; ZSTD_REP_NUM]; + unsafe { ptr::copy_nonoverlapping(prev_repcodes, repcodes.as_mut_ptr(), ZSTD_REP_NUM) }; + for index in 0..nb_in_sequences { + let in_seq = unsafe { in_seqs.add(index).read() }; + let out = unsafe { out_seqs.add(index) }; + let mut lit_length = in_seq.litLength as u32; + let mut match_length = in_seq.mlBase as u32 + MINMATCH as u32; + let mut rep = 0u32; + + /* Handle the possible single length >= 64K + * There can only be one because we add MINMATCH to every match length, + * and blocks are at most 128K. + */ + if index == store.longLengthPos as usize { + if store.longLengthType == ZSTD_LLT_LITERAL_LENGTH { + lit_length += 0x10000; + } else if store.longLengthType == ZSTD_LLT_MATCH_LENGTH { + match_length += 0x10000; + } + } + + /* Determine the raw offset given the offBase, which may be a repcode. */ + let raw_offset = if (1..=ZSTD_REP_NUM as u32).contains(&in_seq.offBase) { + let repcode = in_seq.offBase; + debug_assert!(repcode > 0); + rep = repcode; + if lit_length != 0 { + repcodes[repcode as usize - 1] + } else if repcode == 3 { + debug_assert!(repcodes[0] > 1); + repcodes[0].wrapping_sub(1) + } else { + repcodes[repcode as usize] + } + } else { + in_seq.offBase.wrapping_sub(ZSTD_REP_NUM as u32) + }; + unsafe { + (*out).litLength = lit_length; + (*out).matchLength = match_length; + (*out).offset = raw_offset; + (*out).rep = rep; + } + + /* Update repcode history for the sequence */ + update_rep(&mut repcodes, in_seq.offBase, in_seq.litLength == 0); + + nb_out_literals += lit_length as usize; + } + /* Insert last literals (if any exist) in the block as a sequence with ml == off == 0. + * If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker + * for the block boundary, according to the API. + */ + debug_assert!(nb_in_literals >= nb_out_literals); + { + let last_ll_size = nb_in_literals.wrapping_sub(nb_out_literals); + unsafe { + let last = out_seqs.add(nb_in_sequences); + (*last).litLength = last_ll_size as u32; + (*last).matchLength = 0; + (*last).offset = 0; + } + } + collector.seqIndex += nb_out_sequences; + debug_assert!(collector.seqIndex <= collector.maxSequences); + + 0 +} + +#[cfg(test)] +mod tests { + use super::*; + use std::mem::{align_of, offset_of}; + + const HUF_CTABLE_SIZE_ST: usize = HUF_SYMBOLVALUE_MAX as usize + 2; + const ENTROPY_WORKSPACE_SIZE: usize = (8 << 10) + 512 + (MAX_SEQ + 2) * size_of::(); + + fn empty_huf_tables(repeat: c_int) -> ZSTD_hufCTables_t { + ZSTD_hufCTables_t { + CTable: [0; HUF_CTABLE_SIZE_ST], + repeatMode: repeat, + } + } + + fn empty_fse_tables(repeat: c_int) -> ZSTD_fseCTables_t { + ZSTD_fseCTables_t { + offcodeCTable: [0; OFF_CTABLE_SIZE], + matchlengthCTable: [0; ML_CTABLE_SIZE], + litlengthCTable: [0; LL_CTABLE_SIZE], + offcode_repeatMode: repeat, + matchlength_repeatMode: repeat, + litlength_repeatMode: repeat, + } + } + + struct TestSeqStore { + sequences: Vec, + literals: Vec, + ll_codes: Vec, + ml_codes: Vec, + of_codes: Vec, + } + + impl TestSeqStore { + fn new(sequences: Vec, literals: Vec) -> Self { + let nb_seq = sequences.len(); + TestSeqStore { + sequences, + literals, + ll_codes: vec![0; nb_seq], + ml_codes: vec![0; nb_seq], + of_codes: vec![0; nb_seq], + } + } + + fn seq_store(&mut self) -> SeqStore_t { + let nb_seq = self.sequences.len(); + let nb_lit = self.literals.len(); + SeqStore_t { + sequencesStart: self.sequences.as_mut_ptr(), + sequences: unsafe { self.sequences.as_mut_ptr().add(nb_seq) }, + litStart: self.literals.as_mut_ptr(), + lit: unsafe { self.literals.as_mut_ptr().add(nb_lit) }, + llCode: self.ll_codes.as_mut_ptr(), + mlCode: self.ml_codes.as_mut_ptr(), + ofCode: self.of_codes.as_mut_ptr(), + maxNbSeq: nb_seq, + maxNbLit: nb_lit, + longLengthType: 0, + longLengthPos: 0, + } + } + } + + #[test] + fn c_leaf_layouts_match_supported_abis() { + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), align_of::()); + assert_eq!(offset_of!(SeqStore_t, sequencesStart), 0); + assert_eq!( + offset_of!(SeqStore_t, longLengthPos), + 9 * size_of::() + 4 + ); + assert_eq!(size_of::(), 9 * size_of::() + 8); + assert_eq!(size_of::(), 3552); + assert_eq!(offset_of!(ZSTD_compressedBlockState_t, entropy), 0); + assert_eq!( + offset_of!(ZSTD_compressedBlockState_t, rep), + size_of::() + ); + assert_eq!(size_of::(), 16); + assert_eq!(offset_of!(SeqCollector, seqStart), size_of::()); + assert_eq!(offset_of!(SeqCollector, seqIndex), 2 * size_of::()); + assert_eq!(size_of::(), 4 * size_of::()); + if size_of::() == 8 { + assert_eq!(size_of::(), 144); + assert_eq!(size_of::(), 168); + assert_eq!(size_of::(), 312); + assert_eq!(size_of::(), 5616); + assert_eq!(size_of::(), 5632); + } else { + assert_eq!(size_of::(), 136); + assert_eq!(size_of::(), 156); + assert_eq!(size_of::(), 292); + assert_eq!(size_of::(), 4584); + assert_eq!(size_of::(), 4596); + } + } + + #[test] + fn repcode_updates_match_the_c_sum_type_rules() { + let mut reps = [1, 4, 8]; + update_rep(&mut reps, 10, false); + assert_eq!(reps, [7, 1, 4]); + update_rep(&mut reps, 1, true); + assert_eq!(reps, [1, 7, 4]); + update_rep(&mut reps, 2, false); + assert_eq!(reps, [7, 1, 4]); + update_rep(&mut reps, 3, false); + assert_eq!(reps, [4, 7, 1]); + } + + #[test] + fn seq_to_codes_matches_the_reference_tables() { + let mut store = TestSeqStore::new( + vec![ + SeqDef { + offBase: 4, + litLength: 0, + mlBase: 0, + }, + SeqDef { + offBase: 1, + litLength: 63, + mlBase: 127, + }, + SeqDef { + offBase: 1 << 20, + litLength: 64, + mlBase: 128, + }, + ], + vec![0; 127], + ); + let seq_store = store.seq_store(); + let long_offsets = unsafe { ZSTD_seqToCodes(&seq_store) }; + assert_eq!(long_offsets, 0); + assert_eq!(store.ll_codes, [0, 24, 25]); + assert_eq!(store.ml_codes, [0, 42, 43]); + assert_eq!(store.of_codes, [2, 0, 20]); + } + + #[test] + fn rle_sequences_produce_one_byte_table_headers() { + // Five identical sequences select set_rle for all three symbol kinds; + // each table header is then exactly the one RLE symbol byte. + let mut store = TestSeqStore::new( + vec![ + SeqDef { + offBase: 8, + litLength: 2, + mlBase: 3, + }; + 5 + ], + vec![b'x'; 10], + ); + let seq_store = store.seq_store(); + let prev = empty_fse_tables(FSE_REPEAT_NONE); + let mut next = empty_fse_tables(FSE_REPEAT_NONE); + let mut metadata = + unsafe { std::mem::MaybeUninit::::zeroed().assume_init() }; + let mut workspace = vec![0u64; ENTROPY_WORKSPACE_SIZE / size_of::()]; + + let size = unsafe { + build_block_entropy_stats_sequences( + &seq_store, + &prev, + &mut next, + 1, + &mut metadata, + workspace.as_mut_ptr().cast::(), + ENTROPY_WORKSPACE_SIZE, + ) + }; + assert_eq!(size, 3); + assert_eq!(metadata.llType, SET_RLE); + assert_eq!(metadata.ofType, SET_RLE); + assert_eq!(metadata.mlType, SET_RLE); + assert_eq!(metadata.lastCountSize, 0); + assert_eq!(&metadata.fseTablesBuffer[..3], &[2, 3, 3]); + assert_eq!(next.litlength_repeatMode, FSE_REPEAT_NONE); + assert_eq!(next.offcode_repeatMode, FSE_REPEAT_NONE); + assert_eq!(next.matchlength_repeatMode, FSE_REPEAT_NONE); + } + + #[test] + fn empty_seq_store_emits_raw_literals_and_repeats_tables() { + let mut store = TestSeqStore::new(Vec::new(), b"hello".to_vec()); + let seq_store = store.seq_store(); + let prev = ZSTD_entropyCTables_t { + huf: empty_huf_tables(HUF_REPEAT_NONE), + fse: empty_fse_tables(2), + }; + let mut next = ZSTD_entropyCTables_t { + huf: empty_huf_tables(HUF_REPEAT_NONE), + fse: empty_fse_tables(FSE_REPEAT_NONE), + }; + let mut workspace = vec![0u64; ENTROPY_WORKSPACE_SIZE / size_of::()]; + let mut output = [0xA5u8; 32]; + + let size = unsafe { + ZSTD_rust_entropyCompressSeqStore_internal( + output.as_mut_ptr().cast::(), + output.len(), + seq_store.litStart.cast::(), + 5, + &seq_store, + &prev, + &mut next, + 1, + 0, + workspace.as_mut_ptr().cast::(), + ENTROPY_WORKSPACE_SIZE, + 0, + ) + }; + // Raw literals header (5 << 3), five literal bytes, nbSeq == 0. + assert_eq!(size, 7); + assert_eq!(&output[..7], &[0x28, b'h', b'e', b'l', b'l', b'o', 0x00]); + // With no sequences, the previous FSE tables are copied unchanged. + assert_eq!(next.fse.offcode_repeatMode, 2); + assert_eq!(next.fse.matchlength_repeatMode, 2); + assert_eq!(next.fse.litlength_repeatMode, 2); + } + + #[test] + fn literal_stats_pick_rle_and_compressed_types() { + let mut workspace = vec![0u64; ENTROPY_WORKSPACE_SIZE / size_of::()]; + let prev = empty_huf_tables(HUF_REPEAT_NONE); + let mut next = empty_huf_tables(HUF_REPEAT_NONE); + let mut metadata = + unsafe { std::mem::MaybeUninit::::zeroed().assume_init() }; + + let rle = [7u8; 128]; + let size = unsafe { + build_block_entropy_stats_literals( + rle.as_ptr().cast::(), + rle.len(), + &prev, + &mut next, + &mut metadata, + 0, + workspace.as_mut_ptr().cast::(), + ENTROPY_WORKSPACE_SIZE, + 0, + ) + }; + assert_eq!(size, 0); + assert_eq!(metadata.hType, SET_RLE); + + let mut compressible = [0u8; 1024]; + for (index, byte) in compressible.iter_mut().enumerate() { + *byte = (index % 4) as u8; + } + let size = unsafe { + build_block_entropy_stats_literals( + compressible.as_ptr().cast::(), + compressible.len(), + &prev, + &mut next, + &mut metadata, + 0, + workspace.as_mut_ptr().cast::(), + ENTROPY_WORKSPACE_SIZE, + 0, + ) + }; + assert!(!ERR_isError(size)); + assert!(size > 0); + assert_eq!(metadata.hType, SET_COMPRESSED); + assert_eq!(metadata.hufDesSize, 0); // caller stores the returned size + assert_eq!(next.repeatMode, HUF_REPEAT_CHECK); + } + + #[test] + fn copied_sequences_resolve_repcodes_and_append_the_delimiter() { + let mut store = TestSeqStore::new( + vec![ + SeqDef { + offBase: 3 + 100, // raw offset 100 + litLength: 4, + mlBase: 5, + }, + SeqDef { + offBase: 1, // repcode 1 with literals: rep[0] == 100 + litLength: 2, + mlBase: 7, + }, + SeqDef { + offBase: 3, // repcode 3 without literals: rep[0] - 1 == 99 + litLength: 0, + mlBase: 9, + }, + ], + vec![0; 10], + ); + let seq_store = store.seq_store(); + let mut out = [ZSTD_Sequence { + offset: 0xAA, + litLength: 0xAA, + matchLength: 0xAA, + rep: 0xAA, + }; 8]; + let mut collector = SeqCollector { + collectSequences: 1, + seqStart: out.as_mut_ptr(), + seqIndex: 1, + maxSequences: 8, + }; + let reps = [1u32, 4, 8]; + + let result = + unsafe { ZSTD_rust_copyBlockSequences(&mut collector, &seq_store, reps.as_ptr()) }; + assert_eq!(result, 0); + assert_eq!(collector.seqIndex, 5); + let expect = [ + (100, 4, 5 + 3, 0), + (100, 2, 7 + 3, 1), + (99, 0, 9 + 3, 3), + (0, 10 - 6, 0, 0xAA), // delimiter: last literals; rep untouched + ]; + for (index, &(offset, lit, matched, rep)) in expect.iter().enumerate() { + assert_eq!(out[1 + index].offset, offset, "offset {index}"); + assert_eq!(out[1 + index].litLength, lit, "litLength {index}"); + assert_eq!(out[1 + index].matchLength, matched, "matchLength {index}"); + assert_eq!(out[1 + index].rep, rep, "rep {index}"); + } + + collector.maxSequences = 5; + let result = + unsafe { ZSTD_rust_copyBlockSequences(&mut collector, &seq_store, reps.as_ptr()) }; + assert_eq!(result, ERROR(ZstdErrorCode::DstSizeTooSmall)); + } +} diff --git a/rust/src/zstd_compress_superblock.rs b/rust/src/zstd_compress_superblock.rs index 904ef0e8b..6d1007f81 100644 --- a/rust/src/zstd_compress_superblock.rs +++ b/rust/src/zstd_compress_superblock.rs @@ -6,13 +6,13 @@ //! This is the Rust implementation of `zstd_compress_superblock.c`. Its C //! entry-point shim only extracts fields from the opaque `ZSTD_CCtx`; all //! sequence partitioning, literal and sequence section writing, entropy -//! fallback, and repcode repair remain here. The leaf layouts below are -//! intentionally kept C-shaped and checked for both supported pointer widths. +//! fallback, and repcode repair remain here. The C-shaped leaf layouts are +//! shared with `zstd_compress_stats`, which also owns the block entropy +//! statistics this module consumes. use crate::common::{ LL_BITS, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS, ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, - ZSTD_MAX_FSE_HEADERS_SIZE, ZSTD_MAX_HUF_HEADER_SIZE, }; use crate::errors::{ERR_isError, ZstdErrorCode, ERROR}; use crate::hist::{HIST_countFast_wksp, HIST_count_wksp}; @@ -23,9 +23,17 @@ use crate::mem::{MEM_32bits, MEM_writeLE16, MEM_writeLE24, MEM_writeLE32}; use crate::zstd_compress_literals::{ ZSTD_compressRleLiteralsBlock, ZSTD_hufCTables_t, ZSTD_noCompressLiterals, }; +use crate::zstd_compress_sequences::{ + SeqDef, ZSTD_crossEntropyCost, ZSTD_encodeSequences, ZSTD_fseBitCost, +}; +use crate::zstd_compress_stats::{ + build_block_entropy_stats, update_rep, SeqStore_t, ZSTD_compressedBlockState_t, + ZSTD_entropyCTablesMetadata_t, ZSTD_entropyCTables_t, ZSTD_fseCTablesMetadata_t, + ZSTD_fseCTables_t, ZSTD_hufCTablesMetadata_t, +}; use std::ffi::c_void; use std::mem::MaybeUninit; -use std::os::raw::{c_int, c_short, c_uint}; +use std::os::raw::c_int; use std::ptr; const SET_BASIC: c_int = 0; @@ -42,121 +50,6 @@ const LONG_NB_SEQ: usize = 0x7f00; const DEFAULT_MAX_OFF: u32 = 28; const BYTE_SCALE: usize = 256; -const OFF_CTABLE_SIZE: usize = 1 + (1 << 7) + ((MAX_OFF + 1) * 2); -const ML_CTABLE_SIZE: usize = 1 + (1 << 8) + ((MAX_ML + 1) * 2); -const LL_CTABLE_SIZE: usize = 1 + (1 << 8) + ((MAX_LL + 1) * 2); - -/// ABI-compatible `SeqDef` from `zstd_compress_internal.h`. -#[repr(C)] -#[derive(Clone, Copy, Debug, Default)] -struct SeqDef { - offBase: u32, - litLength: u16, - mlBase: u16, -} - -/// ABI-compatible `SeqStore_t` leaf layout. -/// -/// The C context itself remains opaque. The C shim passes its `seqStore` -/// member directly, so this small, stable hot-path structure is the only -/// sequence storage representation crossing into Rust. -#[repr(C)] -struct SeqStore_t { - sequencesStart: *mut SeqDef, - sequences: *mut SeqDef, - litStart: *mut u8, - lit: *mut u8, - llCode: *mut u8, - mlCode: *mut u8, - ofCode: *mut u8, - maxNbSeq: usize, - maxNbLit: usize, - longLengthType: c_int, - longLengthPos: u32, -} - -/// C's `ZSTD_fseCTables_t`. The table element type is `FSE_CTable`, an -/// `unsigned`, and the table lengths are the header macros expanded above. -#[repr(C)] -struct ZSTD_fseCTables_t { - offcodeCTable: [u32; OFF_CTABLE_SIZE], - matchlengthCTable: [u32; ML_CTABLE_SIZE], - litlengthCTable: [u32; LL_CTABLE_SIZE], - offcode_repeatMode: c_int, - matchlength_repeatMode: c_int, - litlength_repeatMode: c_int, -} - -#[repr(C)] -struct ZSTD_entropyCTables_t { - huf: ZSTD_hufCTables_t, - fse: ZSTD_fseCTables_t, -} - -#[repr(C)] -struct ZSTD_compressedBlockState_t { - entropy: ZSTD_entropyCTables_t, - rep: [u32; 3], -} - -#[repr(C)] -struct ZSTD_hufCTablesMetadata_t { - hType: c_int, - hufDesBuffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE], - hufDesSize: usize, -} - -#[repr(C)] -struct ZSTD_fseCTablesMetadata_t { - llType: c_int, - ofType: c_int, - mlType: c_int, - fseTablesBuffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE], - fseTablesSize: usize, - lastCountSize: usize, -} - -#[repr(C)] -struct ZSTD_entropyCTablesMetadata_t { - hufMetadata: ZSTD_hufCTablesMetadata_t, - fseMetadata: ZSTD_fseCTablesMetadata_t, -} - -unsafe extern "C" { - fn ZSTD_buildBlockEntropyStats( - seq_store: *const SeqStore_t, - prev_entropy: *const ZSTD_entropyCTables_t, - next_entropy: *mut ZSTD_entropyCTables_t, - cctx_params: *const c_void, - entropy_metadata: *mut ZSTD_entropyCTablesMetadata_t, - workspace: *mut c_void, - wksp_size: usize, - ) -> usize; - - fn ZSTD_encodeSequences( - dst: *mut c_void, - dst_capacity: usize, - ctable_match_length: *const u32, - ml_code_table: *const u8, - ctable_offset_bits: *const u32, - of_code_table: *const u8, - ctable_lit_length: *const u32, - ll_code_table: *const u8, - sequences: *const SeqDef, - nb_seq: usize, - long_offsets: c_int, - bmi2: c_int, - ) -> usize; - - fn ZSTD_fseBitCost(ctable: *const u32, count: *const u32, max: c_uint) -> usize; - fn ZSTD_crossEntropyCost( - norm: *const c_short, - accuracy_log: c_uint, - count: *const c_uint, - max: c_uint, - ) -> usize; -} - #[inline] fn remaining_capacity(capacity: usize, written: usize) -> Result { capacity @@ -838,28 +731,6 @@ unsafe fn size_block_sequences( nb_seq } -#[inline] -fn update_rep(reps: &mut [u32; 3], off_base: u32, literal_length_is_zero: bool) { - if off_base > 3 { - reps[2] = reps[1]; - reps[1] = reps[0]; - reps[0] = off_base - 3; - return; - } - let rep_code = off_base - 1 + u32::from(literal_length_is_zero); - if rep_code == 0 { - return; - } - let current_offset = if rep_code == 3 { - reps[0].wrapping_sub(1) - } else { - reps[rep_code as usize] - }; - reps[2] = if rep_code >= 2 { reps[1] } else { reps[2] }; - reps[1] = reps[0]; - reps[0] = current_offset; -} - #[allow(clippy::too_many_arguments, clippy::manual_checked_ops)] unsafe fn compress_subblock_multi( seq_store: *const SeqStore_t, @@ -1098,16 +969,18 @@ unsafe fn compress_subblock_multi( /// C ABI implementation called by the declaration-only C superblock shim. /// -/// `cctx_params` deliberately remains opaque: only its two required scalar -/// fields are read by the shim, while the existing C entropy builder receives -/// the original pointer unchanged. +/// `ZSTD_CCtx_params` deliberately remains opaque: the C shim extracts the +/// two scalar fields this path reads (the strategy and the +/// literals-compression switch) and the crate-internal entropy statistics +/// builder receives them directly. #[allow(clippy::too_many_arguments)] #[no_mangle] pub unsafe extern "C" fn ZSTD_rust_compressSuperBlock( seq_store: *const c_void, prev_cblock: *const c_void, next_cblock: *mut c_void, - cctx_params: *const c_void, + strategy: c_int, + disable_literal_compression: c_int, workspace: *mut c_void, wksp_size: usize, bmi2: c_int, @@ -1122,16 +995,17 @@ pub unsafe extern "C" fn ZSTD_rust_compressSuperBlock( let seq_store = seq_store.cast::(); let prev_cblock = prev_cblock.cast::(); let next_cblock = next_cblock.cast::(); - // C only writes the used prefixes of its two metadata byte buffers. Start - // with initialized storage so treating the completed C struct as a Rust - // value never exposes uninitialized array elements. + // The statistics builder only writes the used prefixes of the two + // metadata byte buffers. Start with initialized storage so treating the + // completed struct as a Rust value never exposes uninitialized elements. let mut entropy_metadata = MaybeUninit::::zeroed(); let entropy_result = unsafe { - ZSTD_buildBlockEntropyStats( + build_block_entropy_stats( seq_store, ptr::addr_of!((*prev_cblock).entropy), ptr::addr_of_mut!((*next_cblock).entropy), - cctx_params, + strategy, + disable_literal_compression, entropy_metadata.as_mut_ptr(), workspace, wksp_size, @@ -1163,50 +1037,34 @@ pub unsafe extern "C" fn ZSTD_rust_compressSuperBlock( #[cfg(test)] mod tests { + // The shared C leaf layouts and the repcode update rules are asserted in + // `zstd_compress_stats`, which owns those definitions. use super::*; - use std::mem::{align_of, offset_of, size_of}; #[test] - fn c_leaf_layouts_match_supported_abis() { - assert_eq!(size_of::(), 8); - assert_eq!(align_of::(), align_of::()); - assert_eq!(offset_of!(SeqStore_t, sequencesStart), 0); - assert_eq!( - offset_of!(SeqStore_t, longLengthPos), - 9 * size_of::() + 4 - ); - assert_eq!(size_of::(), 9 * size_of::() + 8); - assert_eq!(size_of::(), 3552); - assert_eq!(offset_of!(ZSTD_compressedBlockState_t, entropy), 0); - assert_eq!( - offset_of!(ZSTD_compressedBlockState_t, rep), - size_of::() - ); - if size_of::() == 8 { - assert_eq!(size_of::(), 144); - assert_eq!(size_of::(), 168); - assert_eq!(size_of::(), 312); - assert_eq!(size_of::(), 5616); - assert_eq!(size_of::(), 5632); - } else { - assert_eq!(size_of::(), 136); - assert_eq!(size_of::(), 156); - assert_eq!(size_of::(), 292); - assert_eq!(size_of::(), 4584); - assert_eq!(size_of::(), 4596); - } - } + fn subblock_sizing_respects_the_budget() { + let sequences = [SeqDef { + offBase: 4, + litLength: 10, + mlBase: 10, + }; 8]; - #[test] - fn repcode_updates_match_the_c_sum_type_rules() { - let mut reps = [1, 4, 8]; - update_rep(&mut reps, 10, false); - assert_eq!(reps, [7, 1, 4]); - update_rep(&mut reps, 1, true); - assert_eq!(reps, [1, 7, 4]); - update_rep(&mut reps, 2, false); - assert_eq!(reps, [7, 1, 4]); - update_rep(&mut reps, 3, false); - assert_eq!(reps, [4, 7, 1]); + // A generous budget keeps every sequence in one sub-block. + let all = unsafe { + size_block_sequences( + sequences.as_ptr(), + sequences.len(), + usize::MAX, + 256, + 256, + true, + ) + }; + assert_eq!(all, sequences.len()); + + // The first sequence always survives, even over budget. + let first = + unsafe { size_block_sequences(sequences.as_ptr(), sequences.len(), 0, 256, 256, true) }; + assert_eq!(first, 1); } }