diff --git a/lib/compress/zstd_compress_superblock.c b/lib/compress/zstd_compress_superblock.c index 6f57345be..8c7722c51 100644 --- a/lib/compress/zstd_compress_superblock.c +++ b/lib/compress/zstd_compress_superblock.c @@ -8,681 +8,48 @@ * You may select, at your option, one of the above-listed licenses. */ - /*-************************************* - * Dependencies - ***************************************/ +/* The implementation lives in rust/src/zstd_compress_superblock.rs. Keep the + * context extraction here: ZSTD_CCtx is intentionally opaque to Rust, while + * the small sequence and entropy leaf layouts are asserted in both sources. */ #include "zstd_compress_superblock.h" +#include "zstd_compress_internal.h" -#include "../common/zstd_internal.h" /* ZSTD_getSequenceLength */ -#include "hist.h" /* HIST_countFast_wksp */ -#include "zstd_compress_internal.h" /* ZSTD_[huf|fse|entropy]CTablesMetadata_t */ -#include "zstd_compress_sequences.h" -#include "zstd_compress_literals.h" +typedef char ZSTD_rust_superblock_seqdef_layout[(sizeof(SeqDef) == 8) ? 1 : -1]; +typedef char ZSTD_rust_superblock_fse_layout[(sizeof(ZSTD_fseCTables_t) == 3552) ? 1 : -1]; +typedef char ZSTD_rust_superblock_entropy_offset[ + (offsetof(ZSTD_compressedBlockState_t, entropy) == 0) ? 1 : -1]; +typedef char ZSTD_rust_superblock_rep_offset[ + (offsetof(ZSTD_compressedBlockState_t, rep) == sizeof(ZSTD_entropyCTables_t)) ? 1 : -1]; +typedef char ZSTD_rust_superblock_seqstore_long_length_pos[ + (offsetof(SeqStore_t, longLengthPos) == 9 * sizeof(size_t) + 4) ? 1 : -1]; +typedef char ZSTD_rust_superblock_seqstore_layout[ + (sizeof(SeqStore_t) == 9 * sizeof(size_t) + 8) ? 1 : -1]; -/** ZSTD_compressSubBlock_literal() : - * Compresses literals section for a sub-block. - * When we have to write the Huffman table we will sometimes choose a header - * size larger than necessary. This is because we have to pick the header size - * before we know the table size + compressed size, so we have a bound on the - * table size. If we guessed incorrectly, we fall back to uncompressed literals. - * - * We write the header when writeEntropy=1 and set entropyWritten=1 when we succeeded - * in writing the header, otherwise it is set to 0. - * - * hufMetadata->hType has literals block type info. - * If it is set_basic, all sub-blocks literals section will be Raw_Literals_Block. - * If it is set_rle, all sub-blocks literals section will be RLE_Literals_Block. - * If it is set_compressed, first sub-block's literals section will be Compressed_Literals_Block - * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block - * and the following sub-blocks' literals sections will be Treeless_Literals_Block. - * @return : compressed size of literals section of a sub-block - * Or 0 if unable to compress. - * Or error code */ -static size_t -ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, - const ZSTD_hufCTablesMetadata_t* hufMetadata, - const BYTE* literals, size_t litSize, - void* dst, size_t dstSize, - const int bmi2, int writeEntropy, int* entropyWritten) -{ - size_t const header = writeEntropy ? 200 : 0; - size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header)); - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstSize; - BYTE* op = ostart + lhSize; - U32 const singleStream = lhSize == 3; - SymbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat; - size_t cLitSize = 0; - - DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy); - - *entropyWritten = 0; - if (litSize == 0 || hufMetadata->hType == set_basic) { - DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal"); - return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); - } else if (hufMetadata->hType == set_rle) { - DEBUGLOG(5, "ZSTD_compressSubBlock_literal using rle literal"); - return ZSTD_compressRleLiteralsBlock(dst, dstSize, literals, litSize); - } - - assert(litSize > 0); - assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat); - - if (writeEntropy && hufMetadata->hType == set_compressed) { - ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize); - op += hufMetadata->hufDesSize; - cLitSize += hufMetadata->hufDesSize; - DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize); - } - - { int const flags = bmi2 ? HUF_flags_bmi2 : 0; - const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, (size_t)(oend-op), literals, litSize, hufTable, flags) - : HUF_compress4X_usingCTable(op, (size_t)(oend-op), literals, litSize, hufTable, flags); - op += cSize; - cLitSize += cSize; - if (cSize == 0 || ERR_isError(cSize)) { - DEBUGLOG(5, "Failed to write entropy tables %s", ZSTD_getErrorName(cSize)); - return 0; - } - /* If we expand and we aren't writing a header then emit uncompressed */ - if (!writeEntropy && cLitSize >= litSize) { - DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal because uncompressible"); - return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); - } - /* If we are writing headers then allow expansion that doesn't change our header size. */ - if (lhSize < (size_t)(3 + (cLitSize >= 1 KB) + (cLitSize >= 16 KB))) { - assert(cLitSize > litSize); - DEBUGLOG(5, "Literals expanded beyond allowed header size"); - return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); - } - DEBUGLOG(5, "ZSTD_compressSubBlock_literal (cSize=%zu)", cSize); - } - - /* Build header */ - switch(lhSize) - { - case 3: /* 2 - 2 - 10 - 10 */ - { U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14); - MEM_writeLE24(ostart, lhc); - break; - } - case 4: /* 2 - 2 - 14 - 14 */ - { U32 const lhc = hType + (2 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<18); - MEM_writeLE32(ostart, lhc); - break; - } - case 5: /* 2 - 2 - 18 - 18 */ - { U32 const lhc = hType + (3 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<22); - MEM_writeLE32(ostart, lhc); - ostart[4] = (BYTE)(cLitSize >> 10); - break; - } - default: /* not possible : lhSize is {3,4,5} */ - assert(0); - } - *entropyWritten = 1; - DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)litSize, (U32)(op-ostart)); - return (size_t)(op-ostart); -} - -static size_t -ZSTD_seqDecompressedSize(SeqStore_t const* seqStore, - const SeqDef* sequences, size_t nbSeqs, - size_t litSize, int lastSubBlock) -{ - size_t matchLengthSum = 0; - size_t litLengthSum = 0; - size_t n; - for (n=0; nllType, fseMetadata->ofType, and fseMetadata->mlType have - * symbol compression modes for the super-block. - * The first successfully compressed block will have these in its header. - * We set entropyWritten=1 when we succeed in compressing the sequences. - * The following sub-blocks will always have repeat mode. - * @return : compressed size of sequences section of a sub-block - * Or 0 if it is unable to compress - * Or error code. */ -static size_t -ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables, - const ZSTD_fseCTablesMetadata_t* fseMetadata, - const SeqDef* sequences, size_t nbSeq, - const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, - const ZSTD_CCtx_params* cctxParams, - void* dst, size_t dstCapacity, - const int bmi2, int writeEntropy, int* entropyWritten) -{ - const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstCapacity; - BYTE* op = ostart; - BYTE* seqHead; - - DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (nbSeq=%zu, writeEntropy=%d, longOffsets=%d)", nbSeq, writeEntropy, longOffsets); - - *entropyWritten = 0; - /* Sequences Header */ - RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, - dstSize_tooSmall, ""); - if (nbSeq < 128) - *op++ = (BYTE)nbSeq; - else if (nbSeq < LONGNBSEQ) - op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; - else - op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; - if (nbSeq==0) { - return (size_t)(op - ostart); - } - - /* seqHead : flags for FSE encoding type */ - seqHead = op++; - - DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (seqHeadSize=%u)", (unsigned)(op-ostart)); - - if (writeEntropy) { - const U32 LLtype = fseMetadata->llType; - const U32 Offtype = fseMetadata->ofType; - const U32 MLtype = fseMetadata->mlType; - DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize); - *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); - ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize); - op += fseMetadata->fseTablesSize; - } else { - const U32 repeat = set_repeat; - *seqHead = (BYTE)((repeat<<6) + (repeat<<4) + (repeat<<2)); - } - - { size_t const bitstreamSize = ZSTD_encodeSequences( - op, (size_t)(oend - op), - fseTables->matchlengthCTable, mlCode, - fseTables->offcodeCTable, ofCode, - fseTables->litlengthCTable, llCode, - sequences, nbSeq, - longOffsets, bmi2); - FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); - op += bitstreamSize; - /* zstd versions <= 1.3.4 mistakenly report corruption when - * FSE_readNCount() receives a buffer < 4 bytes. - * Fixed by https://github.com/facebook/zstd/pull/1146. - * This can happen when the last set_compressed table present is 2 - * bytes and the bitstream is only one byte. - * In this exceedingly rare case, we will simply emit an uncompressed - * block, since it isn't worth optimizing. - */ -#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - if (writeEntropy && fseMetadata->lastCountSize && fseMetadata->lastCountSize + bitstreamSize < 4) { - /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ - assert(fseMetadata->lastCountSize + bitstreamSize == 3); - DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " - "emitting an uncompressed block."); - return 0; - } -#endif - DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (bitstreamSize=%zu)", bitstreamSize); - } - - /* zstd versions <= 1.4.0 mistakenly report error when - * sequences section body size is less than 3 bytes. - * Fixed by https://github.com/facebook/zstd/pull/1664. - * This can happen when the previous sequences section block is compressed - * with rle mode and the current block's sequences section is compressed - * with repeat mode where sequences section body size can be 1 byte. - */ -#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - if (op-seqHead < 4) { - DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.4.0 by emitting " - "an uncompressed block when sequences are < 4 bytes"); - return 0; - } -#endif - - *entropyWritten = 1; - return (size_t)(op - ostart); -} - -/** ZSTD_compressSubBlock() : - * Compresses a single sub-block. - * @return : compressed size of the sub-block - * Or 0 if it failed to compress. */ -static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy, - const ZSTD_entropyCTablesMetadata_t* entropyMetadata, - const SeqDef* sequences, size_t nbSeq, - const BYTE* literals, size_t litSize, - const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, - const ZSTD_CCtx_params* cctxParams, - void* dst, size_t dstCapacity, - const int bmi2, - int writeLitEntropy, int writeSeqEntropy, - int* litEntropyWritten, int* seqEntropyWritten, - U32 lastBlock) -{ - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstCapacity; - BYTE* op = ostart + ZSTD_blockHeaderSize; - DEBUGLOG(5, "ZSTD_compressSubBlock (litSize=%zu, nbSeq=%zu, writeLitEntropy=%d, writeSeqEntropy=%d, lastBlock=%d)", - litSize, nbSeq, writeLitEntropy, writeSeqEntropy, lastBlock); - { size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable, - &entropyMetadata->hufMetadata, literals, litSize, - op, (size_t)(oend-op), - bmi2, writeLitEntropy, litEntropyWritten); - FORWARD_IF_ERROR(cLitSize, "ZSTD_compressSubBlock_literal failed"); - if (cLitSize == 0) return 0; - op += cLitSize; - } - { size_t cSeqSize = ZSTD_compressSubBlock_sequences(&entropy->fse, - &entropyMetadata->fseMetadata, - sequences, nbSeq, - llCode, mlCode, ofCode, - cctxParams, - op, (size_t)(oend-op), - bmi2, writeSeqEntropy, seqEntropyWritten); - FORWARD_IF_ERROR(cSeqSize, "ZSTD_compressSubBlock_sequences failed"); - if (cSeqSize == 0) return 0; - op += cSeqSize; - } - /* Write block header */ - { size_t cSize = (size_t)(op-ostart) - ZSTD_blockHeaderSize; - U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); - MEM_writeLE24(ostart, cBlockHeader24); - } - return (size_t)(op-ostart); -} - -static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t litSize, - const ZSTD_hufCTables_t* huf, - const ZSTD_hufCTablesMetadata_t* hufMetadata, - void* workspace, size_t wkspSize, - int writeEntropy) -{ - unsigned* const countWksp = (unsigned*)workspace; - unsigned maxSymbolValue = 255; - size_t literalSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ - - if (hufMetadata->hType == set_basic) return litSize; - else if (hufMetadata->hType == set_rle) return 1; - else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { - size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize); - if (ZSTD_isError(largest)) return litSize; - { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue); - if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; - return cLitSizeEstimate + literalSectionHeaderSize; - } } - assert(0); /* impossible */ - return 0; -} - -static size_t ZSTD_estimateSubBlockSize_symbolType(SymbolEncodingType_e type, - const BYTE* codeTable, unsigned maxCode, - size_t nbSeq, const FSE_CTable* fseCTable, - const U8* additionalBits, - short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, - void* workspace, size_t wkspSize) -{ - unsigned* const countWksp = (unsigned*)workspace; - const BYTE* ctp = codeTable; - const BYTE* const ctStart = ctp; - const BYTE* const ctEnd = ctStart + nbSeq; - size_t cSymbolTypeSizeEstimateInBits = 0; - unsigned max = maxCode; - - HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ - if (type == set_basic) { - /* We selected this encoding type, so it must be valid. */ - assert(max <= defaultMax); - cSymbolTypeSizeEstimateInBits = max <= defaultMax - ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max) - : ERROR(GENERIC); - } else if (type == set_rle) { - cSymbolTypeSizeEstimateInBits = 0; - } else if (type == set_compressed || type == set_repeat) { - cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); - } - if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) return nbSeq * 10; - while (ctp < ctEnd) { - if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; - else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */ - ctp++; - } - return cSymbolTypeSizeEstimateInBits / 8; -} - -static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable, - const BYTE* llCodeTable, - const BYTE* mlCodeTable, - size_t nbSeq, - const ZSTD_fseCTables_t* fseTables, - const ZSTD_fseCTablesMetadata_t* fseMetadata, - void* workspace, size_t wkspSize, - int writeEntropy) -{ - size_t const sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ - size_t cSeqSizeEstimate = 0; - if (nbSeq == 0) return sequencesSectionHeaderSize; - cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff, - nbSeq, fseTables->offcodeCTable, NULL, - OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, - workspace, wkspSize); - cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL, - nbSeq, fseTables->litlengthCTable, LL_bits, - LL_defaultNorm, LL_defaultNormLog, MaxLL, - workspace, wkspSize); - cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML, - nbSeq, fseTables->matchlengthCTable, ML_bits, - ML_defaultNorm, ML_defaultNormLog, MaxML, - workspace, wkspSize); - if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; - return cSeqSizeEstimate + sequencesSectionHeaderSize; -} - -typedef struct { - size_t estLitSize; - size_t estBlockSize; -} EstimatedBlockSize; -static EstimatedBlockSize ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize, - const BYTE* ofCodeTable, - const BYTE* llCodeTable, - const BYTE* mlCodeTable, - size_t nbSeq, - const ZSTD_entropyCTables_t* entropy, - const ZSTD_entropyCTablesMetadata_t* entropyMetadata, - void* workspace, size_t wkspSize, - int writeLitEntropy, int writeSeqEntropy) -{ - EstimatedBlockSize ebs; - ebs.estLitSize = ZSTD_estimateSubBlockSize_literal(literals, litSize, - &entropy->huf, &entropyMetadata->hufMetadata, - workspace, wkspSize, writeLitEntropy); - ebs.estBlockSize = ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, - nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, - workspace, wkspSize, writeSeqEntropy); - ebs.estBlockSize += ebs.estLitSize + ZSTD_blockHeaderSize; - return ebs; -} - -static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMetadata) -{ - if (fseMetadata->llType == set_compressed || fseMetadata->llType == set_rle) - return 1; - if (fseMetadata->mlType == set_compressed || fseMetadata->mlType == set_rle) - return 1; - if (fseMetadata->ofType == set_compressed || fseMetadata->ofType == set_rle) - return 1; - return 0; -} - -static size_t countLiterals(SeqStore_t const* seqStore, const SeqDef* sp, size_t seqCount) -{ - size_t n, total = 0; - assert(sp != NULL); - for (n=0; n %zu bytes", seqCount, (const void*)sp, total); - return total; -} - -#define BYTESCALE 256 - -static size_t sizeBlockSequences(const SeqDef* sp, size_t nbSeqs, - size_t targetBudget, size_t avgLitCost, size_t avgSeqCost, - int firstSubBlock) -{ - size_t n, budget = 0, inSize=0; - /* entropy headers */ - size_t const headerSize = (size_t)firstSubBlock * 120 * BYTESCALE; /* generous estimate */ - assert(firstSubBlock==0 || firstSubBlock==1); - budget += headerSize; - - /* first sequence => at least one sequence*/ - budget += sp[0].litLength * avgLitCost + avgSeqCost; - if (budget > targetBudget) return 1; - inSize = sp[0].litLength + (sp[0].mlBase+MINMATCH); - - /* loop over sequences */ - for (n=1; n targetBudget) - /* though continue to expand until the sub-block is deemed compressible */ - && (budget < inSize * BYTESCALE) ) - break; - } - - return n; -} - -/** ZSTD_compressSubBlock_multi() : - * Breaks super-block into multiple sub-blocks and compresses them. - * Entropy will be written into the first block. - * The following blocks use repeat_mode to compress. - * Sub-blocks are all compressed, except the last one when beneficial. - * @return : compressed size of the super block (which features multiple ZSTD blocks) - * or 0 if it failed to compress. */ -static size_t ZSTD_compressSubBlock_multi(const SeqStore_t* seqStorePtr, - const ZSTD_compressedBlockState_t* prevCBlock, - ZSTD_compressedBlockState_t* nextCBlock, - const ZSTD_entropyCTablesMetadata_t* entropyMetadata, - const ZSTD_CCtx_params* cctxParams, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const int bmi2, U32 lastBlock, - void* workspace, size_t wkspSize) -{ - const SeqDef* const sstart = seqStorePtr->sequencesStart; - const SeqDef* const send = seqStorePtr->sequences; - const SeqDef* sp = sstart; /* tracks progresses within seqStorePtr->sequences */ - size_t const nbSeqs = (size_t)(send - sstart); - const BYTE* const lstart = seqStorePtr->litStart; - const BYTE* const lend = seqStorePtr->lit; - const BYTE* lp = lstart; - size_t const nbLiterals = (size_t)(lend - lstart); - BYTE const* ip = (BYTE const*)src; - BYTE const* const iend = ip + srcSize; - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + dstCapacity; - BYTE* op = ostart; - const BYTE* llCodePtr = seqStorePtr->llCode; - const BYTE* mlCodePtr = seqStorePtr->mlCode; - const BYTE* ofCodePtr = seqStorePtr->ofCode; - size_t const minTarget = ZSTD_TARGETCBLOCKSIZE_MIN; /* enforce minimum size, to reduce undesirable side effects */ - size_t const targetCBlockSize = MAX(minTarget, cctxParams->targetCBlockSize); - int writeLitEntropy = (entropyMetadata->hufMetadata.hType == set_compressed); - int writeSeqEntropy = 1; - - DEBUGLOG(5, "ZSTD_compressSubBlock_multi (srcSize=%u, litSize=%u, nbSeq=%u)", - (unsigned)srcSize, (unsigned)(lend-lstart), (unsigned)(send-sstart)); - - /* let's start by a general estimation for the full block */ - if (nbSeqs > 0) { - EstimatedBlockSize const ebs = - ZSTD_estimateSubBlockSize(lp, nbLiterals, - ofCodePtr, llCodePtr, mlCodePtr, nbSeqs, - &nextCBlock->entropy, entropyMetadata, - workspace, wkspSize, - writeLitEntropy, writeSeqEntropy); - /* quick estimation */ - size_t const avgLitCost = nbLiterals ? (ebs.estLitSize * BYTESCALE) / nbLiterals : BYTESCALE; - size_t const avgSeqCost = ((ebs.estBlockSize - ebs.estLitSize) * BYTESCALE) / nbSeqs; - const size_t nbSubBlocks = MAX((ebs.estBlockSize + (targetCBlockSize/2)) / targetCBlockSize, 1); - size_t n, avgBlockBudget, blockBudgetSupp=0; - avgBlockBudget = (ebs.estBlockSize * BYTESCALE) / nbSubBlocks; - DEBUGLOG(5, "estimated fullblock size=%u bytes ; avgLitCost=%.2f ; avgSeqCost=%.2f ; targetCBlockSize=%u, nbSubBlocks=%u ; avgBlockBudget=%.0f bytes", - (unsigned)ebs.estBlockSize, (double)avgLitCost/BYTESCALE, (double)avgSeqCost/BYTESCALE, - (unsigned)targetCBlockSize, (unsigned)nbSubBlocks, (double)avgBlockBudget/BYTESCALE); - /* simplification: if estimates states that the full superblock doesn't compress, just bail out immediately - * this will result in the production of a single uncompressed block covering @srcSize.*/ - if (ebs.estBlockSize > srcSize) return 0; - - /* compress and write sub-blocks */ - assert(nbSubBlocks>0); - for (n=0; n < nbSubBlocks-1; n++) { - /* determine nb of sequences for current sub-block + nbLiterals from next sequence */ - size_t const seqCount = sizeBlockSequences(sp, (size_t)(send-sp), - avgBlockBudget + blockBudgetSupp, avgLitCost, avgSeqCost, n==0); - /* if reached last sequence : break to last sub-block (simplification) */ - assert(seqCount <= (size_t)(send-sp)); - if (sp + seqCount == send) break; - assert(seqCount > 0); - /* compress sub-block */ - { int litEntropyWritten = 0; - int seqEntropyWritten = 0; - size_t litSize = countLiterals(seqStorePtr, sp, seqCount); - const size_t decompressedSize = - ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, 0); - size_t const cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, - sp, seqCount, - lp, litSize, - llCodePtr, mlCodePtr, ofCodePtr, - cctxParams, - op, (size_t)(oend-op), - bmi2, writeLitEntropy, writeSeqEntropy, - &litEntropyWritten, &seqEntropyWritten, - 0); - FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed"); - - /* check compressibility, update state components */ - if (cSize > 0 && cSize < decompressedSize) { - DEBUGLOG(5, "Committed sub-block compressing %u bytes => %u bytes", - (unsigned)decompressedSize, (unsigned)cSize); - assert(ip + decompressedSize <= iend); - ip += decompressedSize; - lp += litSize; - op += cSize; - llCodePtr += seqCount; - mlCodePtr += seqCount; - ofCodePtr += seqCount; - /* Entropy only needs to be written once */ - if (litEntropyWritten) { - writeLitEntropy = 0; - } - if (seqEntropyWritten) { - writeSeqEntropy = 0; - } - sp += seqCount; - blockBudgetSupp = 0; - } } - /* otherwise : do not compress yet, coalesce current sub-block with following one */ - } - } /* if (nbSeqs > 0) */ - - /* write last block */ - DEBUGLOG(5, "Generate last sub-block: %u sequences remaining", (unsigned)(send - sp)); - { int litEntropyWritten = 0; - int seqEntropyWritten = 0; - size_t litSize = (size_t)(lend - lp); - size_t seqCount = (size_t)(send - sp); - const size_t decompressedSize = - ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, 1); - size_t const cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, - sp, seqCount, - lp, litSize, - llCodePtr, mlCodePtr, ofCodePtr, - cctxParams, - op, (size_t)(oend-op), - bmi2, writeLitEntropy, writeSeqEntropy, - &litEntropyWritten, &seqEntropyWritten, - lastBlock); - FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed"); - - /* update pointers, the nb of literals borrowed from next sequence must be preserved */ - if (cSize > 0 && cSize < decompressedSize) { - DEBUGLOG(5, "Last sub-block compressed %u bytes => %u bytes", - (unsigned)decompressedSize, (unsigned)cSize); - assert(ip + decompressedSize <= iend); - ip += decompressedSize; - lp += litSize; - op += cSize; - llCodePtr += seqCount; - mlCodePtr += seqCount; - ofCodePtr += seqCount; - /* Entropy only needs to be written once */ - if (litEntropyWritten) { - writeLitEntropy = 0; - } - if (seqEntropyWritten) { - writeSeqEntropy = 0; - } - sp += seqCount; - } - } - - - if (writeLitEntropy) { - DEBUGLOG(5, "Literal entropy tables were never written"); - ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf)); - } - if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) { - /* If we haven't written our entropy tables, then we've violated our contract and - * must emit an uncompressed block. - */ - DEBUGLOG(5, "Sequence entropy tables were never written => cancel, emit an uncompressed block"); - return 0; - } - - if (ip < iend) { - /* some data left : last part of the block sent uncompressed */ - size_t const rSize = (size_t)((iend - ip)); - size_t const cSize = ZSTD_noCompressBlock(op, (size_t)(oend - op), ip, rSize, lastBlock); - DEBUGLOG(5, "Generate last uncompressed sub-block of %u bytes", (unsigned)(rSize)); - FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); - assert(cSize != 0); - op += cSize; - /* We have to regenerate the repcodes because we've skipped some sequences */ - if (sp < send) { - const SeqDef* seq; - Repcodes_t rep; - ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep)); - for (seq = sstart; seq < sp; ++seq) { - ZSTD_updateRep(rep.rep, seq->offBase, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0); - } - ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep)); - } - } - - DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed all subBlocks: total compressed size = %u", - (unsigned)(op-ostart)); - return (size_t)(op-ostart); -} +size_t ZSTD_rust_compressSuperBlock( + const SeqStore_t* seqStore, + const ZSTD_compressedBlockState_t* prevCBlock, + ZSTD_compressedBlockState_t* nextCBlock, + const ZSTD_CCtx_params* cctxParams, + void* workspace, size_t wkspSize, + int bmi2, U32 windowLog, size_t targetCBlockSize, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastBlock); size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned lastBlock) { - ZSTD_entropyCTablesMetadata_t entropyMetadata; - - FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore, - &zc->blockState.prevCBlock->entropy, - &zc->blockState.nextCBlock->entropy, - &zc->appliedParams, - &entropyMetadata, - zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */), ""); - - return ZSTD_compressSubBlock_multi(&zc->seqStore, + return ZSTD_rust_compressSuperBlock( + &zc->seqStore, zc->blockState.prevCBlock, zc->blockState.nextCBlock, - &entropyMetadata, &zc->appliedParams, + zc->tmpWorkspace, zc->tmpWkspSize, + zc->bmi2, zc->appliedParams.cParams.windowLog, + zc->appliedParams.targetCBlockSize, dst, dstCapacity, src, srcSize, - zc->bmi2, lastBlock, - zc->tmpWorkspace, zc->tmpWkspSize /* statically allocated in resetCCtx */); + lastBlock); } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index ba3e3f272..75df03764 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -25,6 +25,8 @@ pub mod zstd_common; pub mod zstd_compress_literals; #[cfg(feature = "compression")] pub mod zstd_compress_sequences; +#[cfg(feature = "compression")] +pub mod zstd_compress_superblock; #[cfg(feature = "decompression")] pub mod zstd_ddict; #[cfg(feature = "compression")] diff --git a/rust/src/zstd_compress_superblock.rs b/rust/src/zstd_compress_superblock.rs new file mode 100644 index 000000000..904ef0e8b --- /dev/null +++ b/rust/src/zstd_compress_superblock.rs @@ -0,0 +1,1212 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] + +//! Target-size super-block compression. +//! +//! This is the Rust implementation of `zstd_compress_superblock.c`. Its C +//! entry-point shim only extracts fields from the opaque `ZSTD_CCtx`; all +//! sequence partitioning, literal and sequence section writing, entropy +//! fallback, and repcode repair remain here. The leaf layouts below are +//! intentionally kept C-shaped and checked for both supported pointer widths. + +use crate::common::{ + LL_BITS, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, MAX_LL, MAX_ML, MAX_OFF, MINMATCH, ML_BITS, + ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, + ZSTD_MAX_FSE_HEADERS_SIZE, ZSTD_MAX_HUF_HEADER_SIZE, +}; +use crate::errors::{ERR_isError, ZstdErrorCode, ERROR}; +use crate::hist::{HIST_countFast_wksp, HIST_count_wksp}; +use crate::huf_compress::{ + HUF_compress1X_usingCTable, HUF_compress4X_usingCTable, HUF_estimateCompressedSize, +}; +use crate::mem::{MEM_32bits, MEM_writeLE16, MEM_writeLE24, MEM_writeLE32}; +use crate::zstd_compress_literals::{ + ZSTD_compressRleLiteralsBlock, ZSTD_hufCTables_t, ZSTD_noCompressLiterals, +}; +use std::ffi::c_void; +use std::mem::MaybeUninit; +use std::os::raw::{c_int, c_short, c_uint}; +use std::ptr; + +const SET_BASIC: c_int = 0; +const SET_RLE: c_int = 1; +const SET_COMPRESSED: c_int = 2; +const SET_REPEAT: c_int = 3; + +const HUF_FLAGS_BMI2: c_int = 1; +const ZSTD_BLOCK_HEADER_SIZE: usize = 3; +const ZSTD_TARGET_CBLOCK_SIZE_MIN: usize = 1340; +const STREAM_ACCUMULATOR_MIN_32: u32 = 25; +const STREAM_ACCUMULATOR_MIN_64: u32 = 57; +const LONG_NB_SEQ: usize = 0x7f00; +const DEFAULT_MAX_OFF: u32 = 28; +const BYTE_SCALE: usize = 256; + +const OFF_CTABLE_SIZE: usize = 1 + (1 << 7) + ((MAX_OFF + 1) * 2); +const ML_CTABLE_SIZE: usize = 1 + (1 << 8) + ((MAX_ML + 1) * 2); +const LL_CTABLE_SIZE: usize = 1 + (1 << 8) + ((MAX_LL + 1) * 2); + +/// ABI-compatible `SeqDef` from `zstd_compress_internal.h`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +struct SeqDef { + offBase: u32, + litLength: u16, + mlBase: u16, +} + +/// ABI-compatible `SeqStore_t` leaf layout. +/// +/// The C context itself remains opaque. The C shim passes its `seqStore` +/// member directly, so this small, stable hot-path structure is the only +/// sequence storage representation crossing into Rust. +#[repr(C)] +struct SeqStore_t { + sequencesStart: *mut SeqDef, + sequences: *mut SeqDef, + litStart: *mut u8, + lit: *mut u8, + llCode: *mut u8, + mlCode: *mut u8, + ofCode: *mut u8, + maxNbSeq: usize, + maxNbLit: usize, + longLengthType: c_int, + longLengthPos: u32, +} + +/// C's `ZSTD_fseCTables_t`. The table element type is `FSE_CTable`, an +/// `unsigned`, and the table lengths are the header macros expanded above. +#[repr(C)] +struct ZSTD_fseCTables_t { + offcodeCTable: [u32; OFF_CTABLE_SIZE], + matchlengthCTable: [u32; ML_CTABLE_SIZE], + litlengthCTable: [u32; LL_CTABLE_SIZE], + offcode_repeatMode: c_int, + matchlength_repeatMode: c_int, + litlength_repeatMode: c_int, +} + +#[repr(C)] +struct ZSTD_entropyCTables_t { + huf: ZSTD_hufCTables_t, + fse: ZSTD_fseCTables_t, +} + +#[repr(C)] +struct ZSTD_compressedBlockState_t { + entropy: ZSTD_entropyCTables_t, + rep: [u32; 3], +} + +#[repr(C)] +struct ZSTD_hufCTablesMetadata_t { + hType: c_int, + hufDesBuffer: [u8; ZSTD_MAX_HUF_HEADER_SIZE], + hufDesSize: usize, +} + +#[repr(C)] +struct ZSTD_fseCTablesMetadata_t { + llType: c_int, + ofType: c_int, + mlType: c_int, + fseTablesBuffer: [u8; ZSTD_MAX_FSE_HEADERS_SIZE], + fseTablesSize: usize, + lastCountSize: usize, +} + +#[repr(C)] +struct ZSTD_entropyCTablesMetadata_t { + hufMetadata: ZSTD_hufCTablesMetadata_t, + fseMetadata: ZSTD_fseCTablesMetadata_t, +} + +unsafe extern "C" { + fn ZSTD_buildBlockEntropyStats( + seq_store: *const SeqStore_t, + prev_entropy: *const ZSTD_entropyCTables_t, + next_entropy: *mut ZSTD_entropyCTables_t, + cctx_params: *const c_void, + entropy_metadata: *mut ZSTD_entropyCTablesMetadata_t, + workspace: *mut c_void, + wksp_size: usize, + ) -> usize; + + fn ZSTD_encodeSequences( + dst: *mut c_void, + dst_capacity: usize, + ctable_match_length: *const u32, + ml_code_table: *const u8, + ctable_offset_bits: *const u32, + of_code_table: *const u8, + ctable_lit_length: *const u32, + ll_code_table: *const u8, + sequences: *const SeqDef, + nb_seq: usize, + long_offsets: c_int, + bmi2: c_int, + ) -> usize; + + fn ZSTD_fseBitCost(ctable: *const u32, count: *const u32, max: c_uint) -> usize; + fn ZSTD_crossEntropyCost( + norm: *const c_short, + accuracy_log: c_uint, + count: *const c_uint, + max: c_uint, + ) -> usize; +} + +#[inline] +fn remaining_capacity(capacity: usize, written: usize) -> Result { + capacity + .checked_sub(written) + .ok_or_else(|| ERROR(ZstdErrorCode::DstSizeTooSmall)) +} + +#[inline] +unsafe fn copy_bytes(dst: *mut u8, src: *const u8, size: usize) { + if size != 0 { + unsafe { ptr::copy_nonoverlapping(src, dst, size) }; + } +} + +#[inline] +unsafe fn sequence_length(seq_store: *const SeqStore_t, sequence: *const SeqDef) -> (usize, usize) { + let sequence_value = unsafe { sequence.read() }; + let mut lit_length = sequence_value.litLength as usize; + let mut match_length = sequence_value.mlBase as usize + MINMATCH; + let sequence_index = unsafe { sequence.offset_from((*seq_store).sequencesStart) as usize }; + if unsafe { (*seq_store).longLengthPos } == sequence_index as u32 { + match unsafe { (*seq_store).longLengthType } { + 1 => lit_length += 1 << 16, + 2 => match_length += 1 << 16, + _ => {} + } + } + (lit_length, match_length) +} + +#[inline] +unsafe fn no_compress_block( + dst: *mut u8, + dst_capacity: usize, + src: *const u8, + src_size: usize, + last_block: u32, +) -> usize { + let needed = match src_size.checked_add(ZSTD_BLOCK_HEADER_SIZE) { + Some(value) => value, + None => return ERROR(ZstdErrorCode::DstSizeTooSmall), + }; + if needed > dst_capacity { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + let block_header = last_block.wrapping_add((src_size as u32).wrapping_shl(3)); + unsafe { + MEM_writeLE24(dst.cast::(), block_header); + copy_bytes(dst.add(ZSTD_BLOCK_HEADER_SIZE), src, src_size); + } + needed +} + +#[allow(clippy::too_many_arguments)] +unsafe fn compress_subblock_literal( + huf_table: *const usize, + huf_metadata: *const ZSTD_hufCTablesMetadata_t, + literals: *const u8, + lit_size: usize, + dst: *mut u8, + dst_size: usize, + bmi2: c_int, + write_entropy: bool, + entropy_written: &mut bool, +) -> usize { + let header_slack = if write_entropy { 200 } else { 0 }; + let literal_header_size = 3 + + usize::from(lit_size >= 1024 - header_slack) + + usize::from(lit_size >= 16 * 1024 - header_slack); + let metadata = unsafe { &*huf_metadata }; + *entropy_written = false; + + if lit_size == 0 || metadata.hType == SET_BASIC { + return unsafe { + ZSTD_noCompressLiterals( + dst.cast::(), + dst_size, + literals.cast::(), + lit_size, + ) + }; + } + if metadata.hType == SET_RLE { + return unsafe { + ZSTD_compressRleLiteralsBlock( + dst.cast::(), + dst_size, + literals.cast::(), + lit_size, + ) + }; + } + + debug_assert!(metadata.hType == SET_COMPRESSED || metadata.hType == SET_REPEAT); + if literal_header_size > dst_size { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + + let h_type = if write_entropy { + metadata.hType + } else { + SET_REPEAT + }; + let mut written = literal_header_size; + let mut compressed_literals_size = 0usize; + if write_entropy && metadata.hType == SET_COMPRESSED { + let remaining = match remaining_capacity(dst_size, written) { + Ok(value) => value, + Err(error) => return error, + }; + if metadata.hufDesSize > remaining { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + unsafe { + copy_bytes( + dst.add(written), + metadata.hufDesBuffer.as_ptr(), + metadata.hufDesSize, + ); + } + written += metadata.hufDesSize; + compressed_literals_size += metadata.hufDesSize; + } + + let flags = if bmi2 != 0 { HUF_FLAGS_BMI2 } else { 0 }; + let remaining = match remaining_capacity(dst_size, written) { + Ok(value) => value, + Err(error) => return error, + }; + let compressed_size = if literal_header_size == 3 { + unsafe { + HUF_compress1X_usingCTable( + dst.add(written).cast::(), + remaining, + literals.cast::(), + lit_size, + huf_table, + flags, + ) + } + } else { + unsafe { + HUF_compress4X_usingCTable( + dst.add(written).cast::(), + remaining, + literals.cast::(), + lit_size, + huf_table, + flags, + ) + } + }; + compressed_literals_size = compressed_literals_size.wrapping_add(compressed_size); + if compressed_size == 0 || ERR_isError(compressed_size) { + return 0; + } + written += compressed_size; + + if !write_entropy && compressed_literals_size >= lit_size { + return unsafe { + ZSTD_noCompressLiterals( + dst.cast::(), + dst_size, + literals.cast::(), + lit_size, + ) + }; + } + let encoded_header_size = 3 + + usize::from(compressed_literals_size >= 1024) + + usize::from(compressed_literals_size >= 16 * 1024); + if literal_header_size < encoded_header_size { + return unsafe { + ZSTD_noCompressLiterals( + dst.cast::(), + dst_size, + literals.cast::(), + lit_size, + ) + }; + } + + unsafe { + match literal_header_size { + 3 => { + let header = (h_type as u32) + .wrapping_add(((literal_header_size != 3) as u32) << 2) + .wrapping_add((lit_size as u32).wrapping_shl(4)) + .wrapping_add((compressed_literals_size as u32).wrapping_shl(14)); + MEM_writeLE24(dst.cast::(), header); + } + 4 => { + let header = (h_type as u32) + .wrapping_add(2 << 2) + .wrapping_add((lit_size as u32).wrapping_shl(4)) + .wrapping_add((compressed_literals_size as u32).wrapping_shl(18)); + MEM_writeLE32(dst.cast::(), header); + } + 5 => { + let header = (h_type as u32) + .wrapping_add(3 << 2) + .wrapping_add((lit_size as u32).wrapping_shl(4)) + .wrapping_add((compressed_literals_size as u32).wrapping_shl(22)); + MEM_writeLE32(dst.cast::(), header); + *dst.add(4) = (compressed_literals_size >> 10) as u8; + } + _ => unreachable!("literal header size is three through five bytes"), + } + } + *entropy_written = true; + written +} + +#[allow(clippy::too_many_arguments)] +unsafe fn compress_subblock_sequences( + fse_tables: *const ZSTD_fseCTables_t, + fse_metadata: *const ZSTD_fseCTablesMetadata_t, + sequences: *const SeqDef, + nb_seq: usize, + ll_code: *const u8, + ml_code: *const u8, + of_code: *const u8, + window_log: u32, + dst: *mut u8, + dst_capacity: usize, + bmi2: c_int, + write_entropy: bool, + entropy_written: &mut bool, +) -> usize { + *entropy_written = false; + if dst_capacity < 4 { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + + let mut written = unsafe { + if nb_seq < 128 { + *dst = nb_seq as u8; + 1 + } else if nb_seq < LONG_NB_SEQ { + *dst = ((nb_seq >> 8) + 0x80) as u8; + *dst.add(1) = nb_seq as u8; + 2 + } else { + *dst = 0xff; + MEM_writeLE16(dst.add(1).cast::(), (nb_seq - LONG_NB_SEQ) as u16); + 3 + } + }; + if nb_seq == 0 { + return written; + } + + let sequence_header = written; + written += 1; + let metadata = unsafe { &*fse_metadata }; + if write_entropy { + let remaining = match remaining_capacity(dst_capacity, written) { + Ok(value) => value, + Err(error) => return error, + }; + if metadata.fseTablesSize > remaining { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + unsafe { + *dst.add(sequence_header) = ((metadata.llType as u8) << 6) + .wrapping_add((metadata.ofType as u8) << 4) + .wrapping_add((metadata.mlType as u8) << 2); + copy_bytes( + dst.add(written), + metadata.fseTablesBuffer.as_ptr(), + metadata.fseTablesSize, + ); + } + written += metadata.fseTablesSize; + } else { + unsafe { *dst.add(sequence_header) = 0b1111_1100 }; + } + + let accumulator_min = if MEM_32bits() { + STREAM_ACCUMULATOR_MIN_32 + } else { + STREAM_ACCUMULATOR_MIN_64 + }; + let bitstream_size = unsafe { + ZSTD_encodeSequences( + dst.add(written).cast::(), + dst_capacity - written, + (*fse_tables).matchlengthCTable.as_ptr(), + ml_code, + (*fse_tables).offcodeCTable.as_ptr(), + of_code, + (*fse_tables).litlengthCTable.as_ptr(), + ll_code, + sequences, + nb_seq, + c_int::from(window_log > accumulator_min), + bmi2, + ) + }; + if ERR_isError(bitstream_size) { + return bitstream_size; + } + written += bitstream_size; + + if write_entropy + && metadata.lastCountSize != 0 + && metadata.lastCountSize.saturating_add(bitstream_size) < 4 + { + return 0; + } + if written - sequence_header < 4 { + return 0; + } + *entropy_written = true; + written +} + +#[allow(clippy::too_many_arguments)] +unsafe fn compress_subblock( + entropy: *const ZSTD_entropyCTables_t, + entropy_metadata: *const ZSTD_entropyCTablesMetadata_t, + sequences: *const SeqDef, + nb_seq: usize, + literals: *const u8, + lit_size: usize, + ll_code: *const u8, + ml_code: *const u8, + of_code: *const u8, + window_log: u32, + dst: *mut u8, + dst_capacity: usize, + bmi2: c_int, + write_lit_entropy: bool, + write_seq_entropy: bool, + lit_entropy_written: &mut bool, + seq_entropy_written: &mut bool, + last_block: u32, +) -> usize { + if dst_capacity < ZSTD_BLOCK_HEADER_SIZE { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + let mut written = ZSTD_BLOCK_HEADER_SIZE; + let literal_size = unsafe { + compress_subblock_literal( + (*entropy).huf.CTable.as_ptr(), + ptr::addr_of!((*entropy_metadata).hufMetadata), + literals, + lit_size, + dst.add(written), + dst_capacity - written, + bmi2, + write_lit_entropy, + lit_entropy_written, + ) + }; + if ERR_isError(literal_size) { + return literal_size; + } + if literal_size == 0 { + return 0; + } + written += literal_size; + + let sequence_size = unsafe { + compress_subblock_sequences( + ptr::addr_of!((*entropy).fse), + ptr::addr_of!((*entropy_metadata).fseMetadata), + sequences, + nb_seq, + ll_code, + ml_code, + of_code, + window_log, + dst.add(written), + dst_capacity - written, + bmi2, + write_seq_entropy, + seq_entropy_written, + ) + }; + if ERR_isError(sequence_size) { + return sequence_size; + } + if sequence_size == 0 { + return 0; + } + written += sequence_size; + + let compressed_size = written - ZSTD_BLOCK_HEADER_SIZE; + let block_header = last_block + .wrapping_add(2 << 1) + .wrapping_add((compressed_size as u32).wrapping_shl(3)); + unsafe { MEM_writeLE24(dst.cast::(), block_header) }; + written +} + +unsafe fn estimate_subblock_literal( + literals: *const u8, + lit_size: usize, + huf: *const ZSTD_hufCTables_t, + huf_metadata: *const ZSTD_hufCTablesMetadata_t, + workspace: *mut c_void, + wksp_size: usize, + write_entropy: bool, +) -> usize { + let metadata = unsafe { &*huf_metadata }; + if metadata.hType == SET_BASIC { + return lit_size; + } + if metadata.hType == SET_RLE { + return 1; + } + + let mut max_symbol_value = 255u32; + let largest = unsafe { + HIST_count_wksp( + workspace.cast::(), + &mut max_symbol_value, + literals.cast::(), + lit_size, + workspace, + wksp_size, + ) + }; + if ERR_isError(largest) { + return lit_size; + } + let mut estimate = unsafe { + HUF_estimateCompressedSize( + (*huf).CTable.as_ptr(), + workspace.cast::(), + max_symbol_value, + ) + }; + if write_entropy { + estimate = estimate.wrapping_add(metadata.hufDesSize); + } + estimate.wrapping_add(3) +} + +#[allow(clippy::too_many_arguments)] +unsafe fn estimate_subblock_symbol_type( + encoding_type: c_int, + code_table: *const u8, + max_code: u32, + nb_seq: usize, + fse_ctable: *const u32, + additional_bits: Option<&[u8]>, + default_norm: *const i16, + default_norm_log: u32, + default_max: u32, + workspace: *mut c_void, + wksp_size: usize, +) -> usize { + let mut max = max_code; + unsafe { + let _ = HIST_countFast_wksp( + workspace.cast::(), + &mut max, + code_table.cast::(), + nb_seq, + workspace, + wksp_size, + ); + } + + let mut size_in_bits = match encoding_type { + SET_BASIC if max <= default_max => unsafe { + ZSTD_crossEntropyCost(default_norm, default_norm_log, workspace.cast::(), max) + }, + SET_BASIC => ERROR(ZstdErrorCode::Generic), + SET_RLE => 0, + SET_COMPRESSED | SET_REPEAT => unsafe { + ZSTD_fseBitCost(fse_ctable, workspace.cast::(), max) + }, + _ => ERROR(ZstdErrorCode::Generic), + }; + if ERR_isError(size_in_bits) { + return nb_seq.wrapping_mul(10); + } + for index in 0..nb_seq { + let code = unsafe { *code_table.add(index) }; + size_in_bits = size_in_bits.wrapping_add(match additional_bits { + Some(bits) => bits[code as usize] as usize, + None => code as usize, + }); + } + size_in_bits / 8 +} + +#[allow(clippy::too_many_arguments)] +unsafe fn estimate_subblock_sequences( + of_code_table: *const u8, + ll_code_table: *const u8, + ml_code_table: *const u8, + nb_seq: usize, + fse_tables: *const ZSTD_fseCTables_t, + fse_metadata: *const ZSTD_fseCTablesMetadata_t, + workspace: *mut c_void, + wksp_size: usize, + write_entropy: bool, +) -> usize { + if nb_seq == 0 { + return 3; + } + let metadata = unsafe { &*fse_metadata }; + let tables = unsafe { &*fse_tables }; + let mut estimate = unsafe { + estimate_subblock_symbol_type( + metadata.ofType, + of_code_table, + MAX_OFF as u32, + nb_seq, + tables.offcodeCTable.as_ptr(), + None, + OF_DEFAULT_NORM.as_ptr(), + OF_DEFAULT_NORM_LOG, + DEFAULT_MAX_OFF, + workspace, + wksp_size, + ) + }; + estimate = estimate.wrapping_add(unsafe { + estimate_subblock_symbol_type( + metadata.llType, + ll_code_table, + MAX_LL as u32, + nb_seq, + tables.litlengthCTable.as_ptr(), + Some(&LL_BITS), + LL_DEFAULT_NORM.as_ptr(), + LL_DEFAULT_NORM_LOG, + MAX_LL as u32, + workspace, + wksp_size, + ) + }); + estimate = estimate.wrapping_add(unsafe { + estimate_subblock_symbol_type( + metadata.mlType, + ml_code_table, + MAX_ML as u32, + nb_seq, + tables.matchlengthCTable.as_ptr(), + Some(&ML_BITS), + ML_DEFAULT_NORM.as_ptr(), + ML_DEFAULT_NORM_LOG, + MAX_ML as u32, + workspace, + wksp_size, + ) + }); + if write_entropy { + estimate = estimate.wrapping_add(metadata.fseTablesSize); + } + estimate.wrapping_add(3) +} + +struct EstimatedBlockSize { + est_lit_size: usize, + est_block_size: usize, +} + +#[allow(clippy::too_many_arguments)] +unsafe fn estimate_subblock_size( + literals: *const u8, + lit_size: usize, + of_code_table: *const u8, + ll_code_table: *const u8, + ml_code_table: *const u8, + nb_seq: usize, + entropy: *const ZSTD_entropyCTables_t, + entropy_metadata: *const ZSTD_entropyCTablesMetadata_t, + workspace: *mut c_void, + wksp_size: usize, + write_lit_entropy: bool, + write_seq_entropy: bool, +) -> EstimatedBlockSize { + let est_lit_size = unsafe { + estimate_subblock_literal( + literals, + lit_size, + ptr::addr_of!((*entropy).huf), + ptr::addr_of!((*entropy_metadata).hufMetadata), + workspace, + wksp_size, + write_lit_entropy, + ) + }; + let est_sequences = unsafe { + estimate_subblock_sequences( + of_code_table, + ll_code_table, + ml_code_table, + nb_seq, + ptr::addr_of!((*entropy).fse), + ptr::addr_of!((*entropy_metadata).fseMetadata), + workspace, + wksp_size, + write_seq_entropy, + ) + }; + EstimatedBlockSize { + est_lit_size, + est_block_size: est_sequences + .wrapping_add(est_lit_size) + .wrapping_add(ZSTD_BLOCK_HEADER_SIZE), + } +} + +#[inline] +unsafe fn need_sequence_entropy_tables(metadata: *const ZSTD_fseCTablesMetadata_t) -> bool { + let metadata = unsafe { &*metadata }; + [metadata.llType, metadata.mlType, metadata.ofType] + .iter() + .any(|&encoding_type| encoding_type == SET_COMPRESSED || encoding_type == SET_RLE) +} + +unsafe fn count_literals( + seq_store: *const SeqStore_t, + sequences: *const SeqDef, + count: usize, +) -> usize { + let mut total = 0usize; + for index in 0..count { + total = total.wrapping_add(unsafe { sequence_length(seq_store, sequences.add(index)).0 }); + } + total +} + +unsafe fn sequence_decompressed_size( + seq_store: *const SeqStore_t, + sequences: *const SeqDef, + count: usize, + lit_size: usize, +) -> usize { + let mut match_length_sum = 0usize; + for index in 0..count { + match_length_sum = match_length_sum + .wrapping_add(unsafe { sequence_length(seq_store, sequences.add(index)).1 }); + } + match_length_sum.wrapping_add(lit_size) +} + +unsafe fn size_block_sequences( + sequences: *const SeqDef, + nb_seq: usize, + target_budget: usize, + avg_lit_cost: usize, + avg_seq_cost: usize, + first_subblock: bool, +) -> usize { + debug_assert!(nb_seq > 0); + let mut budget = if first_subblock { + 120usize.wrapping_mul(BYTE_SCALE) + } else { + 0 + }; + let first = unsafe { sequences.read() }; + budget = budget + .wrapping_add((first.litLength as usize).wrapping_mul(avg_lit_cost)) + .wrapping_add(avg_seq_cost); + if budget > target_budget { + return 1; + } + let mut in_size = first.litLength as usize + first.mlBase as usize + MINMATCH; + for index in 1..nb_seq { + let sequence = unsafe { sequences.add(index).read() }; + budget = budget + .wrapping_add((sequence.litLength as usize).wrapping_mul(avg_lit_cost)) + .wrapping_add(avg_seq_cost); + in_size = in_size + .wrapping_add(sequence.litLength as usize) + .wrapping_add(sequence.mlBase as usize + MINMATCH); + if budget > target_budget && budget < in_size.wrapping_mul(BYTE_SCALE) { + return index; + } + } + nb_seq +} + +#[inline] +fn update_rep(reps: &mut [u32; 3], off_base: u32, literal_length_is_zero: bool) { + if off_base > 3 { + reps[2] = reps[1]; + reps[1] = reps[0]; + reps[0] = off_base - 3; + return; + } + let rep_code = off_base - 1 + u32::from(literal_length_is_zero); + if rep_code == 0 { + return; + } + let current_offset = if rep_code == 3 { + reps[0].wrapping_sub(1) + } else { + reps[rep_code as usize] + }; + reps[2] = if rep_code >= 2 { reps[1] } else { reps[2] }; + reps[1] = reps[0]; + reps[0] = current_offset; +} + +#[allow(clippy::too_many_arguments, clippy::manual_checked_ops)] +unsafe fn compress_subblock_multi( + seq_store: *const SeqStore_t, + prev_cblock: *const ZSTD_compressedBlockState_t, + next_cblock: *mut ZSTD_compressedBlockState_t, + entropy_metadata: *const ZSTD_entropyCTablesMetadata_t, + window_log: u32, + target_cblock_size: usize, + dst: *mut u8, + dst_capacity: usize, + src: *const u8, + src_size: usize, + bmi2: c_int, + last_block: u32, + workspace: *mut c_void, + wksp_size: usize, +) -> usize { + let sequence_start = unsafe { (*seq_store).sequencesStart }; + let sequence_end = unsafe { (*seq_store).sequences }; + let nb_sequences = unsafe { sequence_end.offset_from(sequence_start) as usize }; + let literals_start = unsafe { (*seq_store).litStart }; + let literals_end = unsafe { (*seq_store).lit }; + let nb_literals = unsafe { literals_end.offset_from(literals_start) as usize }; + let mut sequence_index = 0usize; + let mut literal_offset = 0usize; + let mut source_offset = 0usize; + let mut output_offset = 0usize; + let mut write_lit_entropy = unsafe { (*entropy_metadata).hufMetadata.hType == SET_COMPRESSED }; + let mut write_seq_entropy = true; + let target_cblock_size = target_cblock_size.max(ZSTD_TARGET_CBLOCK_SIZE_MIN); + + if nb_sequences > 0 { + let estimate = unsafe { + estimate_subblock_size( + literals_start, + nb_literals, + (*seq_store).ofCode, + (*seq_store).llCode, + (*seq_store).mlCode, + nb_sequences, + ptr::addr_of!((*next_cblock).entropy), + entropy_metadata, + workspace, + wksp_size, + write_lit_entropy, + write_seq_entropy, + ) + }; + if estimate.est_block_size > src_size { + return 0; + } + let avg_lit_cost = if nb_literals != 0 { + estimate.est_lit_size.wrapping_mul(BYTE_SCALE) / nb_literals + } else { + BYTE_SCALE + }; + let avg_seq_cost = estimate + .est_block_size + .wrapping_sub(estimate.est_lit_size) + .wrapping_mul(BYTE_SCALE) + / nb_sequences; + let nb_subblocks = ((estimate.est_block_size.wrapping_add(target_cblock_size / 2)) + / target_cblock_size) + .max(1); + let avg_block_budget = estimate.est_block_size.wrapping_mul(BYTE_SCALE) / nb_subblocks; + let block_budget_supp = 0usize; + + for subblock_index in 0..nb_subblocks.saturating_sub(1) { + let remaining_sequences = nb_sequences - sequence_index; + let sequence_count = unsafe { + size_block_sequences( + sequence_start.add(sequence_index), + remaining_sequences, + avg_block_budget.wrapping_add(block_budget_supp), + avg_lit_cost, + avg_seq_cost, + subblock_index == 0, + ) + }; + if sequence_index + sequence_count == nb_sequences { + break; + } + let literal_size = unsafe { + count_literals( + seq_store, + sequence_start.add(sequence_index), + sequence_count, + ) + }; + let decompressed_size = unsafe { + sequence_decompressed_size( + seq_store, + sequence_start.add(sequence_index), + sequence_count, + literal_size, + ) + }; + let mut lit_entropy_written = false; + let mut seq_entropy_written = false; + let compressed_size = unsafe { + compress_subblock( + ptr::addr_of!((*next_cblock).entropy), + entropy_metadata, + sequence_start.add(sequence_index), + sequence_count, + literals_start.add(literal_offset), + literal_size, + (*seq_store).llCode.add(sequence_index), + (*seq_store).mlCode.add(sequence_index), + (*seq_store).ofCode.add(sequence_index), + window_log, + dst.add(output_offset), + dst_capacity - output_offset, + bmi2, + write_lit_entropy, + write_seq_entropy, + &mut lit_entropy_written, + &mut seq_entropy_written, + 0, + ) + }; + if ERR_isError(compressed_size) { + return compressed_size; + } + if compressed_size != 0 && compressed_size < decompressed_size { + source_offset = source_offset.wrapping_add(decompressed_size); + literal_offset = literal_offset.wrapping_add(literal_size); + output_offset = output_offset.wrapping_add(compressed_size); + sequence_index += sequence_count; + if lit_entropy_written { + write_lit_entropy = false; + } + if seq_entropy_written { + write_seq_entropy = false; + } + } + } + } + + let literal_size = nb_literals - literal_offset; + let sequence_count = nb_sequences - sequence_index; + let decompressed_size = unsafe { + sequence_decompressed_size( + seq_store, + sequence_start.add(sequence_index), + sequence_count, + literal_size, + ) + }; + let mut lit_entropy_written = false; + let mut seq_entropy_written = false; + let compressed_size = unsafe { + compress_subblock( + ptr::addr_of!((*next_cblock).entropy), + entropy_metadata, + sequence_start.add(sequence_index), + sequence_count, + literals_start.add(literal_offset), + literal_size, + (*seq_store).llCode.add(sequence_index), + (*seq_store).mlCode.add(sequence_index), + (*seq_store).ofCode.add(sequence_index), + window_log, + dst.add(output_offset), + dst_capacity - output_offset, + bmi2, + write_lit_entropy, + write_seq_entropy, + &mut lit_entropy_written, + &mut seq_entropy_written, + last_block, + ) + }; + if ERR_isError(compressed_size) { + return compressed_size; + } + if compressed_size != 0 && compressed_size < decompressed_size { + source_offset = source_offset.wrapping_add(decompressed_size); + output_offset = output_offset.wrapping_add(compressed_size); + sequence_index += sequence_count; + if lit_entropy_written { + write_lit_entropy = false; + } + if seq_entropy_written { + write_seq_entropy = false; + } + } + + if write_lit_entropy { + unsafe { + ptr::copy_nonoverlapping( + ptr::addr_of!((*prev_cblock).entropy.huf), + ptr::addr_of_mut!((*next_cblock).entropy.huf), + 1, + ); + } + } + if write_seq_entropy + && unsafe { need_sequence_entropy_tables(ptr::addr_of!((*entropy_metadata).fseMetadata)) } + { + return 0; + } + + if source_offset < src_size { + let raw_size = src_size - source_offset; + let remaining = match remaining_capacity(dst_capacity, output_offset) { + Ok(value) => value, + Err(error) => return error, + }; + let raw_compressed_size = unsafe { + no_compress_block( + dst.add(output_offset), + remaining, + src.add(source_offset), + raw_size, + last_block, + ) + }; + if ERR_isError(raw_compressed_size) { + return raw_compressed_size; + } + output_offset += raw_compressed_size; + if sequence_index < nb_sequences { + let mut reps = unsafe { (*prev_cblock).rep }; + for index in 0..sequence_index { + let sequence = unsafe { sequence_start.add(index).read() }; + let lit_length_is_zero = + unsafe { sequence_length(seq_store, sequence_start.add(index)).0 == 0 }; + update_rep(&mut reps, sequence.offBase, lit_length_is_zero); + } + unsafe { (*next_cblock).rep = reps }; + } + } + output_offset +} + +/// C ABI implementation called by the declaration-only C superblock shim. +/// +/// `cctx_params` deliberately remains opaque: only its two required scalar +/// fields are read by the shim, while the existing C entropy builder receives +/// the original pointer unchanged. +#[allow(clippy::too_many_arguments)] +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_compressSuperBlock( + seq_store: *const c_void, + prev_cblock: *const c_void, + next_cblock: *mut c_void, + cctx_params: *const c_void, + workspace: *mut c_void, + wksp_size: usize, + bmi2: c_int, + window_log: u32, + target_cblock_size: usize, + dst: *mut c_void, + dst_capacity: usize, + src: *const c_void, + src_size: usize, + last_block: u32, +) -> usize { + let seq_store = seq_store.cast::(); + let prev_cblock = prev_cblock.cast::(); + let next_cblock = next_cblock.cast::(); + // C only writes the used prefixes of its two metadata byte buffers. Start + // with initialized storage so treating the completed C struct as a Rust + // value never exposes uninitialized array elements. + let mut entropy_metadata = MaybeUninit::::zeroed(); + let entropy_result = unsafe { + ZSTD_buildBlockEntropyStats( + seq_store, + ptr::addr_of!((*prev_cblock).entropy), + ptr::addr_of_mut!((*next_cblock).entropy), + cctx_params, + entropy_metadata.as_mut_ptr(), + workspace, + wksp_size, + ) + }; + if ERR_isError(entropy_result) { + return entropy_result; + } + let entropy_metadata = unsafe { entropy_metadata.assume_init() }; + unsafe { + compress_subblock_multi( + seq_store, + prev_cblock, + next_cblock, + &entropy_metadata, + window_log, + target_cblock_size, + dst.cast::(), + dst_capacity, + src.cast::(), + src_size, + bmi2, + last_block, + workspace, + wksp_size, + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::mem::{align_of, offset_of, size_of}; + + #[test] + fn c_leaf_layouts_match_supported_abis() { + assert_eq!(size_of::(), 8); + assert_eq!(align_of::(), align_of::()); + assert_eq!(offset_of!(SeqStore_t, sequencesStart), 0); + assert_eq!( + offset_of!(SeqStore_t, longLengthPos), + 9 * size_of::() + 4 + ); + assert_eq!(size_of::(), 9 * size_of::() + 8); + assert_eq!(size_of::(), 3552); + assert_eq!(offset_of!(ZSTD_compressedBlockState_t, entropy), 0); + assert_eq!( + offset_of!(ZSTD_compressedBlockState_t, rep), + size_of::() + ); + if size_of::() == 8 { + assert_eq!(size_of::(), 144); + assert_eq!(size_of::(), 168); + assert_eq!(size_of::(), 312); + assert_eq!(size_of::(), 5616); + assert_eq!(size_of::(), 5632); + } else { + assert_eq!(size_of::(), 136); + assert_eq!(size_of::(), 156); + assert_eq!(size_of::(), 292); + assert_eq!(size_of::(), 4584); + assert_eq!(size_of::(), 4596); + } + } + + #[test] + fn repcode_updates_match_the_c_sum_type_rules() { + let mut reps = [1, 4, 8]; + update_rep(&mut reps, 10, false); + assert_eq!(reps, [7, 1, 4]); + update_rep(&mut reps, 1, true); + assert_eq!(reps, [1, 7, 4]); + update_rep(&mut reps, 2, false); + assert_eq!(reps, [7, 1, 4]); + update_rep(&mut reps, 3, false); + assert_eq!(reps, [4, 7, 1]); + } +}