diff --git a/lib/decompress/zstd_decompress_block.c b/lib/decompress/zstd_decompress_block.c index 862785a49..d8b0c9ca4 100644 --- a/lib/decompress/zstd_decompress_block.c +++ b/lib/decompress/zstd_decompress_block.c @@ -8,2202 +8,152 @@ * You may select, at your option, one of the above-listed licenses. */ -/* zstd_decompress_block : - * this module takes care of decompressing _compressed_ block */ - -/*-******************************************************* -* Dependencies -*********************************************************/ -#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ -#include "../common/compiler.h" /* prefetch */ -#include "../common/cpu.h" /* bmi2 */ -#include "../common/mem.h" /* low level memory routines */ -#define FSE_STATIC_LINKING_ONLY -#include "../common/fse.h" -#include "../common/huf.h" -#include "../common/zstd_internal.h" -#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ -#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ -#include "zstd_decompress_block.h" -#include "../common/bits.h" /* ZSTD_highbit32 */ - -/*_******************************************************* -* Macros -**********************************************************/ - -/* These two optional macros force the use one way or another of the two - * ZSTD_decompressSequences implementations. You can't force in both directions - * at the same time. +/* + * Compressed-block decoding is implemented in + * rust/src/zstd_decompress_block.rs. Keep `ZSTD_DCtx` C-owned: it has + * optional build-dependent fields, while this small view contains only the + * leaves read or written by the Rust decoder. No decoder algorithm remains + * in this translation unit. */ -#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ - defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) -#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!" -#endif +#include "../common/zstd_deps.h" +#include "../common/zstd_internal.h" +#include "zstd_decompress_internal.h" +#include "zstd_decompress_block.h" +typedef char ZSTD_rust_block_seq_symbol_layout[(sizeof(ZSTD_seqSymbol) == 8) ? 1 : -1]; +typedef char ZSTD_rust_block_entropy_rep_offset[ + (offsetof(ZSTD_entropyDTables_t, rep) == 26652) ? 1 : -1]; +typedef char ZSTD_rust_block_entropy_workspace_offset[ + (offsetof(ZSTD_entropyDTables_t, workspace) == 26664) ? 1 : -1]; -/*_******************************************************* -* Memory operations -**********************************************************/ -static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } +typedef struct { + const ZSTD_seqSymbol** lltPtr; + const ZSTD_seqSymbol** mltPtr; + const ZSTD_seqSymbol** oftPtr; + const HUF_DTable** hufPtr; + ZSTD_entropyDTables_t* entropy; + U32* workspace; + size_t workspaceSize; + const void** previousDstEnd; + const void** prefixStart; + const void** virtualStart; + const void** dictEnd; + size_t blockSizeMax; + int* isFrameDecompression; + U32* litEntropy; + U32* fseEntropy; + int bmi2; + int* ddictIsCold; + int disableHufAsm; + const BYTE** litPtr; + size_t* litSize; + size_t* rleSize; + BYTE** litBuffer; + const BYTE** litBufferEnd; + ZSTD_litLocation_e* litBufferLocation; + BYTE* litExtraBuffer; + size_t litExtraBufferSize; +} ZSTD_rustBlockCtx; - -/*-************************************************************* - * Block decoding - ***************************************************************/ - -static size_t ZSTD_blockSizeMax(ZSTD_DCtx const* dctx) +static ZSTD_rustBlockCtx ZSTD_rust_block_context(ZSTD_DCtx* dctx) { - size_t const blockSizeMax = dctx->isFrameDecompression ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX; - assert(blockSizeMax <= ZSTD_BLOCKSIZE_MAX); - return blockSizeMax; + ZSTD_rustBlockCtx ctx; + ctx.lltPtr = &dctx->LLTptr; + ctx.mltPtr = &dctx->MLTptr; + ctx.oftPtr = &dctx->OFTptr; + ctx.hufPtr = &dctx->HUFptr; + ctx.entropy = &dctx->entropy; + ctx.workspace = dctx->workspace; + ctx.workspaceSize = sizeof(dctx->workspace); + ctx.previousDstEnd = &dctx->previousDstEnd; + ctx.prefixStart = &dctx->prefixStart; + ctx.virtualStart = &dctx->virtualStart; + ctx.dictEnd = &dctx->dictEnd; + ctx.blockSizeMax = dctx->fParams.blockSizeMax; + ctx.isFrameDecompression = &dctx->isFrameDecompression; + ctx.litEntropy = &dctx->litEntropy; + ctx.fseEntropy = &dctx->fseEntropy; + ctx.bmi2 = ZSTD_DCtx_get_bmi2(dctx); + ctx.ddictIsCold = &dctx->ddictIsCold; + ctx.disableHufAsm = dctx->disableHufAsm; + ctx.litPtr = &dctx->litPtr; + ctx.litSize = &dctx->litSize; + ctx.rleSize = &dctx->rleSize; + ctx.litBuffer = &dctx->litBuffer; + ctx.litBufferEnd = &dctx->litBufferEnd; + ctx.litBufferLocation = &dctx->litBufferLocation; + ctx.litExtraBuffer = dctx->litExtraBuffer; + ctx.litExtraBufferSize = ZSTD_LITBUFFEREXTRASIZE; + return ctx; } -/*! ZSTD_getcBlockSize() : - * Provides the size of compressed block from block header `src` */ -size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, - blockProperties_t* bpPtr) -{ - RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, ""); +size_t ZSTD_rust_decodeLiteralsBlock_wrapper( + ZSTD_rustBlockCtx* ctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity); +size_t ZSTD_rust_decodeSeqHeaders( + ZSTD_rustBlockCtx* ctx, int* nbSeqPtr, + const void* src, size_t srcSize); +size_t ZSTD_rust_decompressBlock_internal( + ZSTD_rustBlockCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, int streaming); +void ZSTD_rust_checkContinuity( + ZSTD_rustBlockCtx* ctx, const void* dst, size_t dstSize); +size_t ZSTD_rust_decompressBlock_deprecated( + ZSTD_rustBlockCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); - { U32 const cBlockHeader = MEM_readLE24(src); - U32 const cSize = cBlockHeader >> 3; - bpPtr->lastBlock = cBlockHeader & 1; - bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); - bpPtr->origSize = cSize; /* only useful for RLE */ - if (bpPtr->blockType == bt_rle) return 1; - RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, ""); - return cSize; - } -} - -/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */ -static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize, - const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately) -{ - size_t const blockSizeMax = ZSTD_blockSizeMax(dctx); - assert(litSize <= blockSizeMax); - assert(dctx->isFrameDecompression || streaming == not_streaming); - assert(expectedWriteSize <= blockSizeMax); - if (streaming == not_streaming && dstCapacity > blockSizeMax + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH) { - /* If we aren't streaming, we can just put the literals after the output - * of the current block. We don't need to worry about overwriting the - * extDict of our window, because it doesn't exist. - * So if we have space after the end of the block, just put it there. - */ - dctx->litBuffer = (BYTE*)dst + blockSizeMax + WILDCOPY_OVERLENGTH; - dctx->litBufferEnd = dctx->litBuffer + litSize; - dctx->litBufferLocation = ZSTD_in_dst; - } else if (litSize <= ZSTD_LITBUFFEREXTRASIZE) { - /* Literals fit entirely within the extra buffer, put them there to avoid - * having to split the literals. - */ - dctx->litBuffer = dctx->litExtraBuffer; - dctx->litBufferEnd = dctx->litBuffer + litSize; - dctx->litBufferLocation = ZSTD_not_in_dst; - } else { - assert(blockSizeMax > ZSTD_LITBUFFEREXTRASIZE); - /* Literals must be split between the output block and the extra lit - * buffer. We fill the extra lit buffer with the tail of the literals, - * and put the rest of the literals at the end of the block, with - * WILDCOPY_OVERLENGTH of buffer room to allow for overreads. - * This MUST not write more than our maxBlockSize beyond dst, because in - * streaming mode, that could overwrite part of our extDict window. - */ - if (splitImmediately) { - /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */ - dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; - dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE; - } else { - /* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */ - dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize; - dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize; - } - dctx->litBufferLocation = ZSTD_split; - assert(dctx->litBufferEnd <= (BYTE*)dst + expectedWriteSize); - } -} - -/*! ZSTD_decodeLiteralsBlock() : - * Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored - * in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current - * block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being - * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write. - * - * @return : nb of bytes read from src (< srcSize ) - * note : symbol not declared but exposed for fullbench */ -static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, - const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */ - void* dst, size_t dstCapacity, const streaming_operation streaming) -{ - DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); - RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, ""); - - { const BYTE* const istart = (const BYTE*) src; - SymbolEncodingType_e const litEncType = (SymbolEncodingType_e)(istart[0] & 3); - size_t const blockSizeMax = ZSTD_blockSizeMax(dctx); - - switch(litEncType) - { - case set_repeat: - DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); - RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, ""); - ZSTD_FALLTHROUGH; - - case set_compressed: - RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3"); - { size_t lhSize, litSize, litCSize; - U32 singleStream=0; - U32 const lhlCode = (istart[0] >> 2) & 3; - U32 const lhc = MEM_readLE32(istart); - size_t hufSuccess; - size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); - int const flags = 0 - | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0) - | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0); - switch(lhlCode) - { - case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ - /* 2 - 2 - 10 - 10 */ - singleStream = !lhlCode; - lhSize = 3; - litSize = (lhc >> 4) & 0x3FF; - litCSize = (lhc >> 14) & 0x3FF; - break; - case 2: - /* 2 - 2 - 14 - 14 */ - lhSize = 4; - litSize = (lhc >> 4) & 0x3FFF; - litCSize = lhc >> 18; - break; - case 3: - /* 2 - 2 - 18 - 18 */ - lhSize = 5; - litSize = (lhc >> 4) & 0x3FFFF; - litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); - break; - } - RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); - RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); - if (!singleStream) - RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong, - "Not enough literals (%zu) for the 4-streams mode (min %u)", - litSize, MIN_LITERALS_FOR_4_STREAMS); - RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); - RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, ""); - ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0); - - /* prefetch huffman table if cold */ - if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { - PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable)); - } - - if (litEncType==set_repeat) { - if (singleStream) { - hufSuccess = HUF_decompress1X_usingDTable( - dctx->litBuffer, litSize, istart+lhSize, litCSize, - dctx->HUFptr, flags); - } else { - assert(litSize >= MIN_LITERALS_FOR_4_STREAMS); - hufSuccess = HUF_decompress4X_usingDTable( - dctx->litBuffer, litSize, istart+lhSize, litCSize, - dctx->HUFptr, flags); - } - } else { - if (singleStream) { -#if defined(HUF_FORCE_DECOMPRESS_X2) - hufSuccess = HUF_decompress1X_DCtx_wksp( - dctx->entropy.hufTable, dctx->litBuffer, litSize, - istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace), flags); -#else - hufSuccess = HUF_decompress1X1_DCtx_wksp( - dctx->entropy.hufTable, dctx->litBuffer, litSize, - istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace), flags); -#endif - } else { - hufSuccess = HUF_decompress4X_hufOnly_wksp( - dctx->entropy.hufTable, dctx->litBuffer, litSize, - istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace), flags); - } - } - if (dctx->litBufferLocation == ZSTD_split) - { - assert(litSize > ZSTD_LITBUFFEREXTRASIZE); - ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); - ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE); - dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; - dctx->litBufferEnd -= WILDCOPY_OVERLENGTH; - assert(dctx->litBufferEnd <= (BYTE*)dst + blockSizeMax); - } - - RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, ""); - - dctx->litPtr = dctx->litBuffer; - dctx->litSize = litSize; - dctx->litEntropy = 1; - if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; - return litCSize + lhSize; - } - - case set_basic: - { size_t litSize, lhSize; - U32 const lhlCode = ((istart[0]) >> 2) & 3; - size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); - switch(lhlCode) - { - case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ - lhSize = 1; - litSize = istart[0] >> 3; - break; - case 1: - lhSize = 2; - litSize = MEM_readLE16(istart) >> 4; - break; - case 3: - lhSize = 3; - RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3"); - litSize = MEM_readLE24(istart) >> 4; - break; - } - - RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); - RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); - RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); - ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); - if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ - RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, ""); - if (dctx->litBufferLocation == ZSTD_split) - { - ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE); - ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); - } - else - { - ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize); - } - dctx->litPtr = dctx->litBuffer; - dctx->litSize = litSize; - return lhSize+litSize; - } - /* direct reference into compressed stream */ - dctx->litPtr = istart+lhSize; - dctx->litSize = litSize; - dctx->litBufferEnd = dctx->litPtr + litSize; - dctx->litBufferLocation = ZSTD_not_in_dst; - return lhSize+litSize; - } - - case set_rle: - { U32 const lhlCode = ((istart[0]) >> 2) & 3; - size_t litSize, lhSize; - size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); - switch(lhlCode) - { - case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ - lhSize = 1; - litSize = istart[0] >> 3; - break; - case 1: - lhSize = 2; - RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3"); - litSize = MEM_readLE16(istart) >> 4; - break; - case 3: - lhSize = 3; - RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4"); - litSize = MEM_readLE24(istart) >> 4; - break; - } - RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); - RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); - RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); - ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); - if (dctx->litBufferLocation == ZSTD_split) - { - ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE); - ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE); - } - else - { - ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize); - } - dctx->litPtr = dctx->litBuffer; - dctx->litSize = litSize; - return lhSize+1; - } - default: - RETURN_ERROR(corruption_detected, "impossible"); - } - } -} - -/* Hidden declaration for fullbench */ +/* Hidden declaration for fullbench. */ size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, - const void* src, size_t srcSize, - void* dst, size_t dstCapacity); + const void* src, size_t srcSize, + void* dst, size_t dstCapacity); size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, - const void* src, size_t srcSize, - void* dst, size_t dstCapacity) + const void* src, size_t srcSize, + void* dst, size_t dstCapacity) { - dctx->isFrameDecompression = 0; - return ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, not_streaming); -} - -/* Default FSE distribution tables. - * These are pre-calculated FSE decoding tables using default distributions as defined in specification : - * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions - * They were generated programmatically with following method : - * - start from default distributions, present in /lib/common/zstd_internal.h - * - generate tables normally, using ZSTD_buildFSETable() - * - printout the content of tables - * - prettify output, report below, test with fuzzer to ensure it's correct */ - -/* Default FSE distribution table for Literal Lengths */ -static const ZSTD_seqSymbol LL_defaultDTable[(1<tableLog = 0; - DTableH->fastMode = 0; - - cell->nbBits = 0; - cell->nextState = 0; - assert(nbAddBits < 255); - cell->nbAdditionalBits = nbAddBits; - cell->baseValue = baseValue; -} - - -/* ZSTD_buildFSETable() : - * generate FSE decoding table for one symbol (ll, ml or off) - * cannot fail if input is valid => - * all inputs are presumed validated at this stage */ -FORCE_INLINE_TEMPLATE -void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, - const short* normalizedCounter, unsigned maxSymbolValue, - const U32* baseValue, const U8* nbAdditionalBits, - unsigned tableLog, void* wksp, size_t wkspSize) -{ - ZSTD_seqSymbol* const tableDecode = dt+1; - U32 const maxSV1 = maxSymbolValue + 1; - U32 const tableSize = 1 << tableLog; - - U16* symbolNext = (U16*)wksp; - BYTE* spread = (BYTE*)(symbolNext + MaxSeq + 1); - U32 highThreshold = tableSize - 1; - - - /* Sanity Checks */ - assert(maxSymbolValue <= MaxSeq); - assert(tableLog <= MaxFSELog); - assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE); - (void)wkspSize; - /* Init, lay down lowprob symbols */ - { ZSTD_seqSymbol_header DTableH; - DTableH.tableLog = tableLog; - DTableH.fastMode = 1; - { S16 const largeLimit= (S16)(1 << (tableLog-1)); - U32 s; - for (s=0; s= largeLimit) DTableH.fastMode=0; - assert(normalizedCounter[s]>=0); - symbolNext[s] = (U16)normalizedCounter[s]; - } } } - ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); - } - - /* Spread symbols */ - assert(tableSize <= 512); - /* Specialized symbol spreading for the case when there are - * no low probability (-1 count) symbols. When compressing - * small blocks we avoid low probability symbols to hit this - * case, since header decoding speed matters more. - */ - if (highThreshold == tableSize - 1) { - size_t const tableMask = tableSize-1; - size_t const step = FSE_TABLESTEP(tableSize); - /* First lay down the symbols in order. - * We use a uint64_t to lay down 8 bytes at a time. This reduces branch - * misses since small blocks generally have small table logs, so nearly - * all symbols have counts <= 8. We ensure we have 8 bytes at the end of - * our buffer to handle the over-write. - */ - { - U64 const add = 0x0101010101010101ull; - size_t pos = 0; - U64 sv = 0; - U32 s; - for (s=0; s=0); - pos += (size_t)n; - } - } - /* Now we spread those positions across the table. - * The benefit of doing it in two stages is that we avoid the - * variable size inner loop, which caused lots of branch misses. - * Now we can run through all the positions without any branch misses. - * We unroll the loop twice, since that is what empirically worked best. - */ - { - size_t position = 0; - size_t s; - size_t const unroll = 2; - assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ - for (s = 0; s < (size_t)tableSize; s += unroll) { - size_t u; - for (u = 0; u < unroll; ++u) { - size_t const uPosition = (position + (u * step)) & tableMask; - tableDecode[uPosition].baseValue = spread[s + u]; - } - position = (position + (unroll * step)) & tableMask; - } - assert(position == 0); - } - } else { - U32 const tableMask = tableSize-1; - U32 const step = FSE_TABLESTEP(tableSize); - U32 s, position = 0; - for (s=0; s highThreshold)) position = (position + step) & tableMask; /* lowprob area */ - } } - assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ - } - - /* Build Decoding table */ - { - U32 u; - for (u=0; u max, corruption_detected, ""); - { U32 const symbol = *(const BYTE*)src; - U32 const baseline = baseValue[symbol]; - U8 const nbBits = nbAdditionalBits[symbol]; - ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); - } - *DTablePtr = DTableSpace; - return 1; - case set_basic : - *DTablePtr = defaultTable; - return 0; - case set_repeat: - RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, ""); - /* prefetch FSE table if used */ - if (ddictIsCold && (nbSeq > 24 /* heuristic */)) { - const void* const pStart = *DTablePtr; - size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog)); - PREFETCH_AREA(pStart, pSize); - } - return 0; - case set_compressed : - { unsigned tableLog; - S16 norm[MaxSeq+1]; - size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); - RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, ""); - RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, ""); - ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize, bmi2); - *DTablePtr = DTableSpace; - return headerSize; - } - default : - assert(0); - RETURN_ERROR(GENERIC, "impossible"); - } + ZSTD_rustBlockCtx ctx = ZSTD_rust_block_context(dctx); + return ZSTD_rust_decodeLiteralsBlock_wrapper( + &ctx, src, srcSize, dst, dstCapacity); } size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, const void* src, size_t srcSize) { - const BYTE* const istart = (const BYTE*)src; - const BYTE* const iend = istart + srcSize; - const BYTE* ip = istart; - int nbSeq; - DEBUGLOG(5, "ZSTD_decodeSeqHeaders"); - - /* check */ - RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong, ""); - - /* SeqHead */ - nbSeq = *ip++; - if (nbSeq > 0x7F) { - if (nbSeq == 0xFF) { - RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, ""); - nbSeq = MEM_readLE16(ip) + LONGNBSEQ; - ip+=2; - } else { - RETURN_ERROR_IF(ip >= iend, srcSize_wrong, ""); - nbSeq = ((nbSeq-0x80)<<8) + *ip++; - } - } - *nbSeqPtr = nbSeq; - - if (nbSeq == 0) { - /* No sequence : section ends immediately */ - RETURN_ERROR_IF(ip != iend, corruption_detected, - "extraneous data present in the Sequences section"); - return (size_t)(ip - istart); - } - - /* FSE table descriptors */ - RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */ - RETURN_ERROR_IF(*ip & 3, corruption_detected, ""); /* The last field, Reserved, must be all-zeroes. */ - { SymbolEncodingType_e const LLtype = (SymbolEncodingType_e)(*ip >> 6); - SymbolEncodingType_e const OFtype = (SymbolEncodingType_e)((*ip >> 4) & 3); - SymbolEncodingType_e const MLtype = (SymbolEncodingType_e)((*ip >> 2) & 3); - ip++; - - /* Build DTables */ - { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, - LLtype, MaxLL, LLFSELog, - ip, iend-ip, - LL_base, LL_bits, - LL_defaultDTable, dctx->fseEntropy, - dctx->ddictIsCold, nbSeq, - dctx->workspace, sizeof(dctx->workspace), - ZSTD_DCtx_get_bmi2(dctx)); - RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed"); - ip += llhSize; - } - - { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, - OFtype, MaxOff, OffFSELog, - ip, iend-ip, - OF_base, OF_bits, - OF_defaultDTable, dctx->fseEntropy, - dctx->ddictIsCold, nbSeq, - dctx->workspace, sizeof(dctx->workspace), - ZSTD_DCtx_get_bmi2(dctx)); - RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed"); - ip += ofhSize; - } - - { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, - MLtype, MaxML, MLFSELog, - ip, iend-ip, - ML_base, ML_bits, - ML_defaultDTable, dctx->fseEntropy, - dctx->ddictIsCold, nbSeq, - dctx->workspace, sizeof(dctx->workspace), - ZSTD_DCtx_get_bmi2(dctx)); - RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed"); - ip += mlhSize; - } - } - - return ip-istart; + ZSTD_rustBlockCtx ctx = ZSTD_rust_block_context(dctx); + return ZSTD_rust_decodeSeqHeaders(&ctx, nbSeqPtr, src, srcSize); } - -typedef struct { - size_t litLength; - size_t matchLength; - size_t offset; -} seq_t; - -typedef struct { - size_t state; - const ZSTD_seqSymbol* table; -} ZSTD_fseState; - -typedef struct { - BIT_DStream_t DStream; - ZSTD_fseState stateLL; - ZSTD_fseState stateOffb; - ZSTD_fseState stateML; - size_t prevOffset[ZSTD_REP_NUM]; -} seqState_t; - -/*! ZSTD_overlapCopy8() : - * Copies 8 bytes from ip to op and updates op and ip where ip <= op. - * If the offset is < 8 then the offset is spread to at least 8 bytes. - * - * Precondition: *ip <= *op - * Postcondition: *op - *op >= 8 - */ -HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { - assert(*ip <= *op); - if (offset < 8) { - /* close range match, overlap */ - static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ - static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ - int const sub2 = dec64table[offset]; - (*op)[0] = (*ip)[0]; - (*op)[1] = (*ip)[1]; - (*op)[2] = (*ip)[2]; - (*op)[3] = (*ip)[3]; - *ip += dec32table[offset]; - ZSTD_copy4(*op+4, *ip); - *ip -= sub2; - } else { - ZSTD_copy8(*op, *ip); - } - *ip += 8; - *op += 8; - assert(*op - *ip >= 8); -} - -/*! ZSTD_safecopy() : - * Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer - * and write up to 16 bytes past oend_w (op >= oend_w is allowed). - * This function is only called in the uncommon case where the sequence is near the end of the block. It - * should be fast for a single long sequence, but can be slow for several short sequences. - * - * @param ovtype controls the overlap detection - * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. - * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. - * The src buffer must be before the dst buffer. - */ -static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { - ptrdiff_t const diff = op - ip; - BYTE* const oend = op + length; - - assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) || - (ovtype == ZSTD_overlap_src_before_dst && diff >= 0)); - - if (length < 8) { - /* Handle short lengths. */ - while (op < oend) *op++ = *ip++; - return; - } - if (ovtype == ZSTD_overlap_src_before_dst) { - /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ - assert(length >= 8); - ZSTD_overlapCopy8(&op, &ip, diff); - length -= 8; - assert(op - ip >= 8); - assert(op <= oend); - } - - if (oend <= oend_w) { - /* No risk of overwrite. */ - ZSTD_wildcopy(op, ip, length, ovtype); - return; - } - if (op <= oend_w) { - /* Wildcopy until we get close to the end. */ - assert(oend > oend_w); - ZSTD_wildcopy(op, ip, oend_w - op, ovtype); - ip += oend_w - op; - op += oend_w - op; - } - /* Handle the leftovers. */ - while (op < oend) *op++ = *ip++; -} - -/* ZSTD_safecopyDstBeforeSrc(): - * This version allows overlap with dst before src, or handles the non-overlap case with dst after src - * Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */ -static void ZSTD_safecopyDstBeforeSrc(BYTE* op, const BYTE* ip, ptrdiff_t length) { - ptrdiff_t const diff = op - ip; - BYTE* const oend = op + length; - - if (length < 8 || diff > -8) { - /* Handle short lengths, close overlaps, and dst not before src. */ - while (op < oend) *op++ = *ip++; - return; - } - - if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) { - ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap); - ip += oend - WILDCOPY_OVERLENGTH - op; - op += oend - WILDCOPY_OVERLENGTH - op; - } - - /* Handle the leftovers. */ - while (op < oend) *op++ = *ip++; -} - -/* ZSTD_execSequenceEnd(): - * This version handles cases that are near the end of the output buffer. It requires - * more careful checks to make sure there is no overflow. By separating out these hard - * and unlikely cases, we can speed up the common cases. - * - * NOTE: This function needs to be fast for a single long sequence, but doesn't need - * to be optimized for many small sequences, since those fall into ZSTD_execSequence(). - */ -FORCE_NOINLINE -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -size_t ZSTD_execSequenceEnd(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const streaming_operation streaming) { - BYTE* const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - const BYTE* const iLitEnd = *litPtr + sequence.litLength; - const BYTE* match = oLitEnd - sequence.offset; - BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; - - /* bounds checks : careful of address space overflow in 32-bit mode */ - RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); - RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); - assert(op < op + sequenceLength); - assert(oLitEnd < op + sequenceLength); - - /* copy literals */ - ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap); - op = oLitEnd; - *litPtr = iLitEnd; - - /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { - /* offset beyond prefix */ - RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); - match = dictEnd - (prefixStart - match); - if (match + sequence.matchLength <= dictEnd) { - ZSTD_memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currentPrefixSegment */ - { size_t const length1 = dictEnd - match; - ZSTD_memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = prefixStart; - } - } - ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); - return sequenceLength; + ZSTD_rustBlockCtx ctx = ZSTD_rust_block_context(dctx); + return ZSTD_rust_decompressBlock_internal(&ctx, dst, dstCapacity, src, srcSize, + (int)streaming); } -/* ZSTD_execSequenceEndSplitLitBuffer(): - * This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case. - */ -FORCE_NOINLINE -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op, - BYTE* const oend, const BYTE* const oend_w, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) -{ - BYTE* const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - const BYTE* const iLitEnd = *litPtr + sequence.litLength; - const BYTE* match = oLitEnd - sequence.offset; - - - /* bounds checks : careful of address space overflow in 32-bit mode */ - RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); - RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); - assert(op < op + sequenceLength); - assert(oLitEnd < op + sequenceLength); - - /* copy literals */ - RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer"); - ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength); - op = oLitEnd; - *litPtr = iLitEnd; - - /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { - /* offset beyond prefix */ - RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); - match = dictEnd - (prefixStart - match); - if (match + sequence.matchLength <= dictEnd) { - ZSTD_memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currentPrefixSegment */ - { size_t const length1 = dictEnd - match; - ZSTD_memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = prefixStart; - } - } - ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); - return sequenceLength; -} - -HINT_INLINE -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -size_t ZSTD_execSequence(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) -{ - BYTE* const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */ - const BYTE* const iLitEnd = *litPtr + sequence.litLength; - const BYTE* match = oLitEnd - sequence.offset; - - assert(op != NULL /* Precondition */); - assert(oend_w < oend /* No underflow */); - -#if defined(__aarch64__) - /* prefetch sequence starting from match that will be used for copy later */ - PREFETCH_L1(match); -#endif - /* Handle edge cases in a slow path: - * - Read beyond end of literals - * - Match end is within WILDCOPY_OVERLIMIT of oend - * - 32-bit mode and the match length overflows - */ - if (UNLIKELY( - iLitEnd > litLimit || - oMatchEnd > oend_w || - (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) - return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); - - /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ - assert(op <= oLitEnd /* No overflow */); - assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); - assert(oMatchEnd <= oend /* No underflow */); - assert(iLitEnd <= litLimit /* Literal length is in bounds */); - assert(oLitEnd <= oend_w /* Can wildcopy literals */); - assert(oMatchEnd <= oend_w /* Can wildcopy matches */); - - /* Copy Literals: - * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. - * We likely don't need the full 32-byte wildcopy. - */ - assert(WILDCOPY_OVERLENGTH >= 16); - ZSTD_copy16(op, (*litPtr)); - if (UNLIKELY(sequence.litLength > 16)) { - ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap); - } - op = oLitEnd; - *litPtr = iLitEnd; /* update for next sequence */ - - /* Copy Match */ - if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { - /* offset beyond prefix -> go into extDict */ - RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); - match = dictEnd + (match - prefixStart); - if (match + sequence.matchLength <= dictEnd) { - ZSTD_memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currentPrefixSegment */ - { size_t const length1 = dictEnd - match; - ZSTD_memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = prefixStart; - } - } - /* Match within prefix of 1 or more bytes */ - assert(op <= oMatchEnd); - assert(oMatchEnd <= oend_w); - assert(match >= prefixStart); - assert(sequence.matchLength >= 1); - - /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy - * without overlap checking. - */ - if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { - /* We bet on a full wildcopy for matches, since we expect matches to be - * longer than literals (in general). In silesia, ~10% of matches are longer - * than 16 bytes. - */ - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); - return sequenceLength; - } - assert(sequence.offset < WILDCOPY_VECLEN); - - /* Copy 8 bytes and spread the offset to be >= 8. */ - ZSTD_overlapCopy8(&op, &match, sequence.offset); - - /* If the match length is > 8 bytes, then continue with the wildcopy. */ - if (sequence.matchLength > 8) { - assert(op < oMatchEnd); - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst); - } - return sequenceLength; -} - -HINT_INLINE -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR -size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op, - BYTE* const oend, const BYTE* const oend_w, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) -{ - BYTE* const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - const BYTE* const iLitEnd = *litPtr + sequence.litLength; - const BYTE* match = oLitEnd - sequence.offset; - - assert(op != NULL /* Precondition */); - assert(oend_w < oend /* No underflow */); - /* Handle edge cases in a slow path: - * - Read beyond end of literals - * - Match end is within WILDCOPY_OVERLIMIT of oend - * - 32-bit mode and the match length overflows - */ - if (UNLIKELY( - iLitEnd > litLimit || - oMatchEnd > oend_w || - (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) - return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); - - /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ - assert(op <= oLitEnd /* No overflow */); - assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); - assert(oMatchEnd <= oend /* No underflow */); - assert(iLitEnd <= litLimit /* Literal length is in bounds */); - assert(oLitEnd <= oend_w /* Can wildcopy literals */); - assert(oMatchEnd <= oend_w /* Can wildcopy matches */); - - /* Copy Literals: - * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. - * We likely don't need the full 32-byte wildcopy. - */ - assert(WILDCOPY_OVERLENGTH >= 16); - ZSTD_copy16(op, (*litPtr)); - if (UNLIKELY(sequence.litLength > 16)) { - ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap); - } - op = oLitEnd; - *litPtr = iLitEnd; /* update for next sequence */ - - /* Copy Match */ - if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { - /* offset beyond prefix -> go into extDict */ - RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); - match = dictEnd + (match - prefixStart); - if (match + sequence.matchLength <= dictEnd) { - ZSTD_memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currentPrefixSegment */ - { size_t const length1 = dictEnd - match; - ZSTD_memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = prefixStart; - } } - /* Match within prefix of 1 or more bytes */ - assert(op <= oMatchEnd); - assert(oMatchEnd <= oend_w); - assert(match >= prefixStart); - assert(sequence.matchLength >= 1); - - /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy - * without overlap checking. - */ - if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { - /* We bet on a full wildcopy for matches, since we expect matches to be - * longer than literals (in general). In silesia, ~10% of matches are longer - * than 16 bytes. - */ - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); - return sequenceLength; - } - assert(sequence.offset < WILDCOPY_VECLEN); - - /* Copy 8 bytes and spread the offset to be >= 8. */ - ZSTD_overlapCopy8(&op, &match, sequence.offset); - - /* If the match length is > 8 bytes, then continue with the wildcopy. */ - if (sequence.matchLength > 8) { - assert(op < oMatchEnd); - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); - } - return sequenceLength; -} - - -static void -ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) -{ - const void* ptr = dt; - const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr; - DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); - DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits", - (U32)DStatePtr->state, DTableH->tableLog); - BIT_reloadDStream(bitD); - DStatePtr->table = dt + 1; -} - -FORCE_INLINE_TEMPLATE void -ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits) -{ - size_t const lowBits = BIT_readBits(bitD, nbBits); - DStatePtr->state = nextState + lowBits; -} - -/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum - * offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32 - * bits before reloading. This value is the maximum number of bytes we read - * after reloading when we are decoding long offsets. - */ -#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \ - (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \ - ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \ - : 0) - -typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; - -/** - * ZSTD_decodeSequence(): - * @p longOffsets : tells the decoder to reload more bit while decoding large offsets - * only used in 32-bit mode - * @return : Sequence (litL + matchL + offset) - */ -FORCE_INLINE_TEMPLATE seq_t -ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const int isLastSeq) -{ - seq_t seq; - /* - * ZSTD_seqSymbol is a 64 bits wide structure. - * It can be loaded in one operation - * and its fields extracted by simply shifting or bit-extracting on aarch64. - * GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh - * operations that cause performance drop. This can be avoided by using this - * ZSTD_memcpy hack. - */ -#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__)) - ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS; - ZSTD_seqSymbol* const llDInfo = &llDInfoS; - ZSTD_seqSymbol* const mlDInfo = &mlDInfoS; - ZSTD_seqSymbol* const ofDInfo = &ofDInfoS; - ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol)); - ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol)); - ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol)); -#else - const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state; - const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state; - const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state; -#endif - seq.matchLength = mlDInfo->baseValue; - seq.litLength = llDInfo->baseValue; - { U32 const ofBase = ofDInfo->baseValue; - BYTE const llBits = llDInfo->nbAdditionalBits; - BYTE const mlBits = mlDInfo->nbAdditionalBits; - BYTE const ofBits = ofDInfo->nbAdditionalBits; - BYTE const totalBits = llBits+mlBits+ofBits; - - U16 const llNext = llDInfo->nextState; - U16 const mlNext = mlDInfo->nextState; - U16 const ofNext = ofDInfo->nextState; - U32 const llnbBits = llDInfo->nbBits; - U32 const mlnbBits = mlDInfo->nbBits; - U32 const ofnbBits = ofDInfo->nbBits; - - assert(llBits <= MaxLLBits); - assert(mlBits <= MaxMLBits); - assert(ofBits <= MaxOff); - /* - * As gcc has better branch and block analyzers, sometimes it is only - * valuable to mark likeliness for clang, it gives around 3-4% of - * performance. - */ - - /* sequence */ - { size_t offset; - if (ofBits > 1) { - ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); - ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); - ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32); - ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits); - if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { - /* Always read extra bits, this keeps the logic simple, - * avoids branches, and avoids accidentally reading 0 bits. - */ - U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32; - offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); - BIT_reloadDStream(&seqState->DStream); - offset += BIT_readBitsFast(&seqState->DStream, extraBits); - } else { - offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); - } - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; - } else { - U32 const ll0 = (llDInfo->baseValue == 0); - if (LIKELY((ofBits == 0))) { - offset = seqState->prevOffset[ll0]; - seqState->prevOffset[1] = seqState->prevOffset[!ll0]; - seqState->prevOffset[0] = offset; - } else { - offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); - { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; - temp -= !temp; /* 0 is not valid: input corrupted => force offset to -1 => corruption detected at execSequence */ - if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset = temp; - } } } - seq.offset = offset; - } - - if (mlBits > 0) - seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); - - if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) - BIT_reloadDStream(&seqState->DStream); - if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) - BIT_reloadDStream(&seqState->DStream); - /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ - ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); - - if (llBits > 0) - seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); - - if (MEM_32bits()) - BIT_reloadDStream(&seqState->DStream); - - DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", - (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); - - if (!isLastSeq) { - /* don't update FSE state for last Sequence */ - ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */ - BIT_reloadDStream(&seqState->DStream); - } - } - - return seq; -} - -#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) -#if DEBUGLEVEL >= 1 -static int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) -{ - size_t const windowSize = dctx->fParams.windowSize; - /* No dictionary used. */ - if (dctx->dictContentEndForFuzzing == NULL) return 0; - /* Dictionary is our prefix. */ - if (prefixStart == dctx->dictContentBeginForFuzzing) return 1; - /* Dictionary is not our ext-dict. */ - if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0; - /* Dictionary is not within our window size. */ - if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0; - /* Dictionary is active. */ - return 1; -} -#endif - -static void ZSTD_assertValidSequence( - ZSTD_DCtx const* dctx, - BYTE const* op, BYTE const* oend, - seq_t const seq, - BYTE const* prefixStart, BYTE const* virtualStart) -{ -#if DEBUGLEVEL >= 1 - if (dctx->isFrameDecompression) { - size_t const windowSize = dctx->fParams.windowSize; - size_t const sequenceSize = seq.litLength + seq.matchLength; - BYTE const* const oLitEnd = op + seq.litLength; - DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", - (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); - assert(op <= oend); - assert((size_t)(oend - op) >= sequenceSize); - assert(sequenceSize <= ZSTD_blockSizeMax(dctx)); - if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { - size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); - /* Offset must be within the dictionary. */ - assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); - assert(seq.offset <= windowSize + dictSize); - } else { - /* Offset must be within our window. */ - assert(seq.offset <= windowSize); - } - } -#else - (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart; -#endif -} -#endif - -#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG - - -FORCE_INLINE_TEMPLATE size_t -DONT_VECTORIZE -ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - const BYTE* ip = (const BYTE*)seqStart; - const BYTE* const iend = ip + seqSize; - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, maxDstSize); - BYTE* op = ostart; - const BYTE* litPtr = dctx->litPtr; - const BYTE* litBufferEnd = dctx->litBufferEnd; - const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); - const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); - const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer (%i seqs)", nbSeq); - - /* Literals are split between internal buffer & output buffer */ - if (nbSeq) { - seqState_t seqState; - dctx->fseEntropy = 1; - { U32 i; for (i=0; ientropy.rep[i]; } - RETURN_ERROR_IF( - ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), - corruption_detected, ""); - ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); - ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); - ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); - assert(dst != NULL); - - ZSTD_STATIC_ASSERT( - BIT_DStream_unfinished < BIT_DStream_completed && - BIT_DStream_endOfBuffer < BIT_DStream_completed && - BIT_DStream_completed < BIT_DStream_overflow); - - /* decompress without overrunning litPtr begins */ - { seq_t sequence = {0,0,0}; /* some static analyzer believe that @sequence is not initialized (it necessarily is, since for(;;) loop as at least one iteration) */ - /* Align the decompression loop to 32 + 16 bytes. - * - * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression - * speed swings based on the alignment of the decompression loop. This - * performance swing is caused by parts of the decompression loop falling - * out of the DSB. The entire decompression loop should fit in the DSB, - * when it can't we get much worse performance. You can measure if you've - * hit the good case or the bad case with this perf command for some - * compressed file test.zst: - * - * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ - * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst - * - * If you see most cycles served out of the MITE you've hit the bad case. - * If you see most cycles served out of the DSB you've hit the good case. - * If it is pretty even then you may be in an okay case. - * - * This issue has been reproduced on the following CPUs: - * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 - * Use Instruments->Counters to get DSB/MITE cycles. - * I never got performance swings, but I was able to - * go from the good case of mostly DSB to half of the - * cycles served from MITE. - * - Coffeelake: Intel i9-9900k - * - Coffeelake: Intel i7-9700k - * - * I haven't been able to reproduce the instability or DSB misses on any - * of the following CPUS: - * - Haswell - * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH - * - Skylake - * - * Alignment is done for each of the three major decompression loops: - * - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer - * - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer - * - ZSTD_decompressSequences_body - * Alignment choices are made to minimize large swings on bad cases and influence on performance - * from changes external to this code, rather than to overoptimize on the current commit. - * - * If you are seeing performance stability this script can help test. - * It tests on 4 commits in zstd where I saw performance change. - * - * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 - */ -#if defined(__GNUC__) && defined(__x86_64__) - __asm__(".p2align 6"); -# if __GNUC__ >= 7 - /* good for gcc-7, gcc-9, and gcc-11 */ - __asm__("nop"); - __asm__(".p2align 5"); - __asm__("nop"); - __asm__(".p2align 4"); -# if __GNUC__ == 8 || __GNUC__ == 10 - /* good for gcc-8 and gcc-10 */ - __asm__("nop"); - __asm__(".p2align 3"); -# endif -# endif -#endif - - /* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */ - for ( ; nbSeq; nbSeq--) { - sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); - if (litPtr + sequence.litLength > dctx->litBufferEnd) break; - { size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); -#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); -#endif - if (UNLIKELY(ZSTD_isError(oneSeqSize))) - return oneSeqSize; - DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); - op += oneSeqSize; - } } - DEBUGLOG(6, "reached: (litPtr + sequence.litLength > dctx->litBufferEnd)"); - - /* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */ - if (nbSeq > 0) { - const size_t leftoverLit = dctx->litBufferEnd - litPtr; - DEBUGLOG(6, "There are %i sequences left, and %zu/%zu literals left in buffer", nbSeq, leftoverLit, sequence.litLength); - if (leftoverLit) { - RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); - ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); - sequence.litLength -= leftoverLit; - op += leftoverLit; - } - litPtr = dctx->litExtraBuffer; - litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; - dctx->litBufferLocation = ZSTD_not_in_dst; - { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); -#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); -#endif - if (UNLIKELY(ZSTD_isError(oneSeqSize))) - return oneSeqSize; - DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); - op += oneSeqSize; - } - nbSeq--; - } - } - - if (nbSeq > 0) { - /* there is remaining lit from extra buffer */ - -#if defined(__GNUC__) && defined(__x86_64__) - __asm__(".p2align 6"); - __asm__("nop"); -# if __GNUC__ != 7 - /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */ - __asm__(".p2align 4"); - __asm__("nop"); - __asm__(".p2align 3"); -# elif __GNUC__ >= 11 - __asm__(".p2align 3"); -# else - __asm__(".p2align 5"); - __asm__("nop"); - __asm__(".p2align 3"); -# endif -#endif - - for ( ; nbSeq ; nbSeq--) { - seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); - size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); -#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); -#endif - if (UNLIKELY(ZSTD_isError(oneSeqSize))) - return oneSeqSize; - DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); - op += oneSeqSize; - } - } - - /* check if reached exact end */ - DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq); - RETURN_ERROR_IF(nbSeq, corruption_detected, ""); - DEBUGLOG(5, "bitStream : start=%p, ptr=%p, bitsConsumed=%u", seqState.DStream.start, seqState.DStream.ptr, seqState.DStream.bitsConsumed); - RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); - /* save reps for next block */ - { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } - } - - /* last literal segment */ - if (dctx->litBufferLocation == ZSTD_split) { - /* split hasn't been reached yet, first get dst then copy litExtraBuffer */ - size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); - DEBUGLOG(6, "copy last literals from segment : %u", (U32)lastLLSize); - RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); - if (op != NULL) { - ZSTD_memmove(op, litPtr, lastLLSize); - op += lastLLSize; - } - litPtr = dctx->litExtraBuffer; - litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; - dctx->litBufferLocation = ZSTD_not_in_dst; - } - /* copy last literals from internal buffer */ - { size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); - DEBUGLOG(6, "copy last literals from internal buffer : %u", (U32)lastLLSize); - RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); - if (op != NULL) { - ZSTD_memcpy(op, litPtr, lastLLSize); - op += lastLLSize; - } } - - DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); - return (size_t)(op - ostart); -} - -FORCE_INLINE_TEMPLATE size_t -DONT_VECTORIZE -ZSTD_decompressSequences_body(ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - const BYTE* ip = (const BYTE*)seqStart; - const BYTE* const iend = ip + seqSize; - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ZSTD_maybeNullPtrAdd(ostart, maxDstSize) : dctx->litBuffer; - BYTE* op = ostart; - const BYTE* litPtr = dctx->litPtr; - const BYTE* const litEnd = litPtr + dctx->litSize; - const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart); - const BYTE* const vBase = (const BYTE*)(dctx->virtualStart); - const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd); - DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq); - - /* Regen sequences */ - if (nbSeq) { - seqState_t seqState; - dctx->fseEntropy = 1; - { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } - RETURN_ERROR_IF( - ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)), - corruption_detected, ""); - ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); - ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); - ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); - assert(dst != NULL); - -#if defined(__GNUC__) && defined(__x86_64__) - __asm__(".p2align 6"); - __asm__("nop"); -# if __GNUC__ >= 7 - __asm__(".p2align 5"); - __asm__("nop"); - __asm__(".p2align 3"); -# else - __asm__(".p2align 4"); - __asm__("nop"); - __asm__(".p2align 3"); -# endif -#endif - - for ( ; nbSeq ; nbSeq--) { - seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); - size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); -#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); -#endif - if (UNLIKELY(ZSTD_isError(oneSeqSize))) - return oneSeqSize; - DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); - op += oneSeqSize; - } - - /* check if reached exact end */ - assert(nbSeq == 0); - RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); - /* save reps for next block */ - { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } - } - - /* last literal segment */ - { size_t const lastLLSize = (size_t)(litEnd - litPtr); - DEBUGLOG(6, "copy last literals : %u", (U32)lastLLSize); - RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); - if (op != NULL) { - ZSTD_memcpy(op, litPtr, lastLLSize); - op += lastLLSize; - } } - - DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); - return (size_t)(op - ostart); -} - -static size_t -ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); -} - -static size_t -ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); -} -#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ - -#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT - -FORCE_INLINE_TEMPLATE - -size_t ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence, - const BYTE* const prefixStart, const BYTE* const dictEnd) -{ - prefetchPos += sequence.litLength; - { const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart; - /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. - * No consequence though : memory address is only used for prefetching, not for dereferencing */ - const BYTE* const match = ZSTD_wrappedPtrSub(ZSTD_wrappedPtrAdd(matchBase, prefetchPos), sequence.offset); - PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ - } - return prefetchPos + sequence.matchLength; -} - -/* This decoding function employs prefetching - * to reduce latency impact of cache misses. - * It's generally employed when block contains a significant portion of long-distance matches - * or when coupled with a "cold" dictionary */ -FORCE_INLINE_TEMPLATE size_t -ZSTD_decompressSequencesLong_body( - ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - const BYTE* ip = (const BYTE*)seqStart; - const BYTE* const iend = ip + seqSize; - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ZSTD_maybeNullPtrAdd(ostart, maxDstSize); - BYTE* op = ostart; - const BYTE* litPtr = dctx->litPtr; - const BYTE* litBufferEnd = dctx->litBufferEnd; - const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); - const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); - const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - - /* Regen sequences */ - if (nbSeq) { -#define STORED_SEQS 8 -#define STORED_SEQS_MASK (STORED_SEQS-1) -#define ADVANCED_SEQS STORED_SEQS - seq_t sequences[STORED_SEQS]; - int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); - seqState_t seqState; - int seqNb; - size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */ - - dctx->fseEntropy = 1; - { int i; for (i=0; ientropy.rep[i]; } - assert(dst != NULL); - assert(iend >= ip); - RETURN_ERROR_IF( - ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), - corruption_detected, ""); - ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); - ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); - ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); - - /* prepare in advance */ - for (seqNb=0; seqNblitBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd) { - /* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */ - const size_t leftoverLit = dctx->litBufferEnd - litPtr; - if (leftoverLit) - { - RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); - ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); - sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit; - op += leftoverLit; - } - litPtr = dctx->litExtraBuffer; - litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; - dctx->litBufferLocation = ZSTD_not_in_dst; - { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); -#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); -#endif - if (ZSTD_isError(oneSeqSize)) return oneSeqSize; - - prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); - sequences[seqNb & STORED_SEQS_MASK] = sequence; - op += oneSeqSize; - } } - else - { - /* lit buffer is either wholly contained in first or second split, or not split at all*/ - size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? - ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : - ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); -#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); -#endif - if (ZSTD_isError(oneSeqSize)) return oneSeqSize; - - prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); - sequences[seqNb & STORED_SEQS_MASK] = sequence; - op += oneSeqSize; - } - } - RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); - - /* finish queue */ - seqNb -= seqAdvance; - for ( ; seqNblitBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd) { - const size_t leftoverLit = dctx->litBufferEnd - litPtr; - if (leftoverLit) { - RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); - ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); - sequence->litLength -= leftoverLit; - op += leftoverLit; - } - litPtr = dctx->litExtraBuffer; - litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; - dctx->litBufferLocation = ZSTD_not_in_dst; - { size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); -#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); -#endif - if (ZSTD_isError(oneSeqSize)) return oneSeqSize; - op += oneSeqSize; - } - } - else - { - size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? - ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : - ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); -#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) - assert(!ZSTD_isError(oneSeqSize)); - ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); -#endif - if (ZSTD_isError(oneSeqSize)) return oneSeqSize; - op += oneSeqSize; - } - } - - /* save reps for next block */ - { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } - } - - /* last literal segment */ - if (dctx->litBufferLocation == ZSTD_split) { /* first deplete literal buffer in dst, then copy litExtraBuffer */ - size_t const lastLLSize = litBufferEnd - litPtr; - RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); - if (op != NULL) { - ZSTD_memmove(op, litPtr, lastLLSize); - op += lastLLSize; - } - litPtr = dctx->litExtraBuffer; - litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; - } - { size_t const lastLLSize = litBufferEnd - litPtr; - RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); - if (op != NULL) { - ZSTD_memmove(op, litPtr, lastLLSize); - op += lastLLSize; - } - } - - return (size_t)(op - ostart); -} - -static size_t -ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); -} -#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ - - - -#if DYNAMIC_BMI2 - -#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG -static BMI2_TARGET_ATTRIBUTE size_t -DONT_VECTORIZE -ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); -} -static BMI2_TARGET_ATTRIBUTE size_t -DONT_VECTORIZE -ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); -} -#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ - -#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT -static BMI2_TARGET_ATTRIBUTE size_t -ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); -} -#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ - -#endif /* DYNAMIC_BMI2 */ - -#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG -static size_t -ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - DEBUGLOG(5, "ZSTD_decompressSequences"); -#if DYNAMIC_BMI2 - if (ZSTD_DCtx_get_bmi2(dctx)) { - return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); - } -#endif - return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); -} -static size_t -ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer"); -#if DYNAMIC_BMI2 - if (ZSTD_DCtx_get_bmi2(dctx)) { - return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); - } -#endif - return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); -} -#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ - - -#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT -/* ZSTD_decompressSequencesLong() : - * decompression function triggered when a minimum share of offsets is considered "long", - * aka out of cache. - * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance". - * This function will try to mitigate main memory latency through the use of prefetching */ -static size_t -ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - DEBUGLOG(5, "ZSTD_decompressSequencesLong"); -#if DYNAMIC_BMI2 - if (ZSTD_DCtx_get_bmi2(dctx)) { - return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); - } -#endif - return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); -} -#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ - - -/** - * @returns The total size of the history referenceable by zstd, including - * both the prefix and the extDict. At @p op any offset larger than this - * is invalid. - */ -static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart) -{ - return (size_t)(op - virtualStart); -} - -typedef struct { - unsigned longOffsetShare; - unsigned maxNbAdditionalBits; -} ZSTD_OffsetInfo; - -/* ZSTD_getOffsetInfo() : - * condition : offTable must be valid - * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) - * compared to maximum possible of (1< 22) info.longOffsetShare += 1; - } - - assert(tableLog <= OffFSELog); - info.longOffsetShare <<= (OffFSELog - tableLog); /* scale to OffFSELog */ - } - - return info; -} - -/** - * @returns The maximum offset we can decode in one read of our bitstream, without - * reloading more bits in the middle of the offset bits read. Any offsets larger - * than this must use the long offset decoder. - */ -static size_t ZSTD_maxShortOffset(void) -{ - if (MEM_64bits()) { - /* We can decode any offset without reloading bits. - * This might change if the max window size grows. - */ - ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); - return (size_t)-1; - } else { - /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1. - * This offBase would require STREAM_ACCUMULATOR_MIN extra bits. - * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset. - */ - size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1; - size_t const maxOffset = maxOffbase - ZSTD_REP_NUM; - assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN); - return maxOffset; - } -} - -size_t -ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, const streaming_operation streaming) -{ /* blockType == blockCompressed */ - const BYTE* ip = (const BYTE*)src; - DEBUGLOG(5, "ZSTD_decompressBlock_internal (cSize : %u)", (unsigned)srcSize); - - /* Note : the wording of the specification - * allows compressed block to be sized exactly ZSTD_blockSizeMax(dctx). - * This generally does not happen, as it makes little sense, - * since an uncompressed block would feature same size and have no decompression cost. - * Also, note that decoder from reference libzstd before < v1.5.4 - * would consider this edge case as an error. - * As a consequence, avoid generating compressed blocks of size ZSTD_blockSizeMax(dctx) - * for broader compatibility with the deployed ecosystem of zstd decoders */ - RETURN_ERROR_IF(srcSize > ZSTD_blockSizeMax(dctx), srcSize_wrong, ""); - - /* Decode literals section */ - { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming); - DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize); - if (ZSTD_isError(litCSize)) return litCSize; - ip += litCSize; - srcSize -= litCSize; - } - - /* Build Decoding Tables */ - { - /* Compute the maximum block size, which must also work when !frame and fParams are unset. - * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t. - */ - size_t const blockSizeMax = MIN(dstCapacity, ZSTD_blockSizeMax(dctx)); - size_t const totalHistorySize = ZSTD_totalHistorySize(ZSTD_maybeNullPtrAdd((BYTE*)dst, blockSizeMax), (BYTE const*)dctx->virtualStart); - /* isLongOffset must be true if there are long offsets. - * Offsets are long if they are larger than ZSTD_maxShortOffset(). - * We don't expect that to be the case in 64-bit mode. - * - * We check here to see if our history is large enough to allow long offsets. - * If it isn't, then we can't possible have (valid) long offsets. If the offset - * is invalid, then it is okay to read it incorrectly. - * - * If isLongOffsets is true, then we will later check our decoding table to see - * if it is even possible to generate long offsets. - */ - ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset())); - /* These macros control at build-time which decompressor implementation - * we use. If neither is defined, we do some inspection and dispatch at - * runtime. - */ -#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ - !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) - int usePrefetchDecoder = dctx->ddictIsCold; -#else - /* Set to 1 to avoid computing offset info if we don't need to. - * Otherwise this value is ignored. - */ - int usePrefetchDecoder = 1; -#endif - int nbSeq; - size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); - if (ZSTD_isError(seqHSize)) return seqHSize; - ip += seqHSize; - srcSize -= seqHSize; - - RETURN_ERROR_IF((dst == NULL || dstCapacity == 0) && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); - RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (size_t)(1 << 20), dstSize_tooSmall, - "invalid dst"); - - /* If we could potentially have long offsets, or we might want to use the prefetch decoder, - * compute information about the share of long offsets, and the maximum nbAdditionalBits. - * NOTE: could probably use a larger nbSeq limit - */ - if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) { - ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq); - if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) { - /* If isLongOffset, but the maximum number of additional bits that we see in our table is small - * enough, then we know it is impossible to have too long an offset in this block, so we can - * use the regular offset decoder. - */ - isLongOffset = ZSTD_lo_isRegularOffset; - } - if (!usePrefetchDecoder) { - U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ - usePrefetchDecoder = (info.longOffsetShare >= minShare); - } - } - - dctx->ddictIsCold = 0; - -#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ - !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) - if (usePrefetchDecoder) { -#else - (void)usePrefetchDecoder; - { -#endif -#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT - return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); -#endif - } - -#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG - /* else */ - if (dctx->litBufferLocation == ZSTD_split) - return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); - else - return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); -#endif - } -} - - -ZSTD_ALLOW_POINTER_OVERFLOW_ATTR void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) { - if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */ - dctx->dictEnd = dctx->previousDstEnd; - dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); - dctx->prefixStart = dst; - dctx->previousDstEnd = dst; - } + ZSTD_rustBlockCtx ctx = ZSTD_rust_block_context(dctx); + ZSTD_rust_checkContinuity(&ctx, dst, dstSize); } - size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, - const void* src, size_t srcSize) + const void* src, size_t srcSize) { - size_t dSize; - dctx->isFrameDecompression = 0; - ZSTD_checkContinuity(dctx, dst, dstCapacity); - dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, not_streaming); - FORWARD_IF_ERROR(dSize, ""); - dctx->previousDstEnd = (char*)dst + dSize; - return dSize; + ZSTD_rustBlockCtx ctx = ZSTD_rust_block_context(dctx); + return ZSTD_rust_decompressBlock_deprecated(&ctx, + dst, dstCapacity, src, srcSize); } - -/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated() */ +/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated(). */ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, - const void* src, size_t srcSize) + const void* src, size_t srcSize) { return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize); } diff --git a/rust/README.md b/rust/README.md index 2ac7d048e..5e6d19c44 100644 --- a/rust/README.md +++ b/rust/README.md @@ -40,11 +40,14 @@ zstd ABI: - `pool` implements the bounded worker pool used by multithreaded compression. - Dictionary support - `zstd_ddict` owns, loads, copies, and references decode dictionaries. +- Block decompression + - `zstd_decompress_block` decodes literal and sequence sections, maintains + FSE/Huffman repeat state, and executes compressed-block sequences. -The optimal block matcher, general decompression, dictionary-building, legacy, -and CLI translation units are still C. They must move before the rewrite is -complete. Keeping that boundary explicit prevents a passing hybrid build from -being mistaken for the final all-Rust result. +The optimal block matcher, high-level frame decompression, dictionary-building, +legacy, and CLI translation units are still C. They must move before the +rewrite is complete. Keeping that boundary explicit prevents a passing hybrid +build from being mistaken for the final all-Rust result. ## Compatibility boundary diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 84a3ece17..08f1079ab 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -31,6 +31,8 @@ pub mod zstd_compress_sequences; pub mod zstd_compress_superblock; #[cfg(feature = "decompression")] pub mod zstd_ddict; +#[cfg(feature = "decompression")] +pub mod zstd_decompress_block; #[cfg(feature = "compression")] pub mod zstd_double_fast; #[cfg(feature = "compression")] diff --git a/rust/src/zstd_decompress_block.rs b/rust/src/zstd_decompress_block.rs new file mode 100644 index 000000000..8ea71ff5c --- /dev/null +++ b/rust/src/zstd_decompress_block.rs @@ -0,0 +1,1700 @@ +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(clippy::missing_safety_doc)] +#![allow(clippy::too_many_arguments)] + +//! Compressed-block decoding. +//! +//! The C decoder context deliberately remains opaque. The companion C shim +//! extracts only the pointers and scalar values this translation needs into +//! [`ZSTD_rustBlockCtx`], so optional C context fields cannot silently change +//! this Rust module's ABI. Literal parsing, FSE table construction, sequence +//! decoding, and block execution live here. + +use crate::bitstream::{ + BIT_DStream_t, BIT_endOfDStream, BIT_initDStream, BIT_readBits, BIT_readBitsFast, + BIT_reloadDStream, +}; +use crate::common::{ + DEFAULT_MAX_OFF, LL_BITS, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, MAX_FSE_LOG, MAX_LL, MAX_ML, + MAX_OFF, MIN_CBLOCK_SIZE, MIN_LITERALS_FOR_4_STREAMS, MIN_SEQUENCES_SIZE, ML_BITS, + ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, ZSTD_REP_NUM, +}; +use crate::entropy_common::FSE_readNCount; +use crate::errors::{ERR_isError, ZstdErrorCode, ERROR}; +#[cfg(not(feature = "huf-force-decompress-x2"))] +use crate::huf_decompress::HUF_decompress1X1_DCtx_wksp; +#[cfg(feature = "huf-force-decompress-x2")] +use crate::huf_decompress::HUF_decompress1X_DCtx_wksp; +use crate::huf_decompress::{ + HUF_decompress1X_usingDTable, HUF_decompress4X_hufOnly_wksp, HUF_decompress4X_usingDTable, +}; +use crate::mem::{MEM_32bits, MEM_64bits, MEM_readLE16, MEM_readLE24, U32}; +use std::cmp::min; +use std::ffi::c_void; +use std::mem::MaybeUninit; +use std::os::raw::{c_int, c_short, c_uint}; +use std::ptr; +use std::sync::OnceLock; + +const ZSTD_BLOCKSIZE_MAX: usize = 128 << 10; +const ZSTD_BLOCK_HEADER_SIZE: usize = 3; +const WILDCOPY_OVERLENGTH: usize = 32; +const HUF_FLAGS_BMI2: c_int = 1 << 0; +const HUF_FLAGS_DISABLE_ASM: c_int = 1 << 4; +const SET_BASIC: c_int = 0; +const SET_RLE: c_int = 1; +const SET_COMPRESSED: c_int = 2; +const SET_REPEAT: c_int = 3; +const NOT_STREAMING: c_int = 0; +const ZSTD_NOT_IN_DST: c_int = 0; +const ZSTD_IN_DST: c_int = 1; +const ZSTD_SPLIT: c_int = 2; +const LONG_NB_SEQ: usize = 0x7f00; +const LL_FSE_LOG: u32 = 9; +const OFF_FSE_LOG: u32 = 8; +const ML_FSE_LOG: u32 = 9; +const ZSTD_HUFFDTABLE_CAPACITY_LOG: usize = 12; +const HUF_DTABLE_SIZE: usize = 1 + (1 << ZSTD_HUFFDTABLE_CAPACITY_LOG); +const ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32: usize = 157; + +const LL_BASE: [u32; MAX_LL + 1] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 22, 24, 28, 32, 40, 48, 64, + 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000, +]; +const OF_BASE: [u32; MAX_OFF + 1] = [ + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD, + 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD, +]; +const OF_BITS: [u8; MAX_OFF + 1] = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 26, 27, 28, 29, 30, 31, +]; +const ML_BASE: [u32; MAX_ML + 1] = [ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, + 0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003, +]; + +/// `ZSTD_seqSymbol` from `zstd_decompress_internal.h`. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +pub struct ZSTD_seqSymbol { + next_state: u16, + nb_additional_bits: u8, + nb_bits: u8, + base_value: u32, +} + +/// The first entry of a sequence table is overlaid as this C header. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +struct ZSTD_seqSymbol_header { + fast_mode: u32, + table_log: u32, +} + +#[repr(C)] +struct ZSTD_entropyDTables_t { + ll_table: [ZSTD_seqSymbol; 1 + (1 << LL_FSE_LOG)], + of_table: [ZSTD_seqSymbol; 1 + (1 << OFF_FSE_LOG)], + ml_table: [ZSTD_seqSymbol; 1 + (1 << ML_FSE_LOG)], + huf_table: [u32; HUF_DTABLE_SIZE], + rep: [u32; ZSTD_REP_NUM], + workspace: [u32; ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32], +} + +/// C-owned leaves of `ZSTD_DCtx_s` used by this translation unit. +/// +/// Every pointer is produced by `zstd_decompress_block.c` under the active C +/// configuration. This avoids assuming offsets for optional context members +/// such as `DYNAMIC_BMI2`, fuzzing bounds, and tracing state. +#[repr(C)] +pub struct ZSTD_rustBlockCtx { + llt_ptr: *mut *const ZSTD_seqSymbol, + mlt_ptr: *mut *const ZSTD_seqSymbol, + oft_ptr: *mut *const ZSTD_seqSymbol, + huf_ptr: *mut *const u32, + entropy: *mut ZSTD_entropyDTables_t, + workspace: *mut u32, + workspace_size: usize, + previous_dst_end: *mut *const u8, + prefix_start: *mut *const u8, + virtual_start: *mut *const u8, + dict_end: *mut *const u8, + block_size_max: usize, + is_frame_decompression: *mut c_int, + lit_entropy: *mut u32, + fse_entropy: *mut u32, + bmi2: c_int, + ddict_is_cold: *mut c_int, + disable_huf_asm: c_int, + lit_ptr: *mut *const u8, + lit_size: *mut usize, + rle_size: *mut usize, + lit_buffer: *mut *mut u8, + lit_buffer_end: *mut *const u8, + lit_buffer_location: *mut c_int, + lit_extra_buffer: *mut u8, + lit_extra_buffer_size: usize, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default)] +pub struct ZSTD_rustSeq { + lit_length: usize, + match_length: usize, + offset: usize, +} + +#[repr(C)] +struct block_properties_t { + block_type: c_int, + last_block: u32, + orig_size: u32, +} + +#[derive(Clone, Copy)] +struct ZSTD_fseState { + state: usize, + table: *const ZSTD_seqSymbol, +} + +struct seq_state_t { + dstream: BIT_DStream_t, + state_ll: ZSTD_fseState, + state_off: ZSTD_fseState, + state_ml: ZSTD_fseState, + prev_offset: [usize; ZSTD_REP_NUM], +} + +#[inline] +unsafe fn entropy(ctx: *mut ZSTD_rustBlockCtx) -> *mut ZSTD_entropyDTables_t { + unsafe { (*ctx).entropy } +} + +#[inline] +unsafe fn address_distance(end: *const u8, start: *const u8) -> usize { + (end as usize).wrapping_sub(start as usize) +} + +#[inline] +unsafe fn ptr_add(ptr: *mut u8, amount: usize) -> *mut u8 { + if ptr.is_null() { + debug_assert_eq!(amount, 0); + ptr + } else { + unsafe { ptr.add(amount) } + } +} + +#[inline] +unsafe fn const_ptr_add(ptr: *const u8, amount: usize) -> *const u8 { + if ptr.is_null() { + debug_assert_eq!(amount, 0); + ptr + } else { + unsafe { ptr.add(amount) } + } +} + +#[inline] +unsafe fn seq_header(table: *const ZSTD_seqSymbol) -> ZSTD_seqSymbol_header { + unsafe { table.cast::().read_unaligned() } +} + +#[inline] +unsafe fn set_seq_header(table: *mut ZSTD_seqSymbol, header: ZSTD_seqSymbol_header) { + unsafe { + table + .cast::() + .write_unaligned(header) + }; +} + +#[inline] +unsafe fn copy_bytes(dst: *mut u8, src: *const u8, len: usize) { + if len != 0 { + unsafe { ptr::copy(src, dst, len) }; + } +} + +#[inline] +unsafe fn fill_bytes(dst: *mut u8, value: u8, len: usize) { + if len != 0 { + unsafe { ptr::write_bytes(dst, value, len) }; + } +} + +/// Copy a match using forward byte semantics, which is required for repeated +/// short-offset matches (unlike `memmove`, which would not expand overlap). +#[inline] +unsafe fn copy_match(mut dst: *mut u8, mut src: *const u8, len: usize) { + for _ in 0..len { + unsafe { dst.write(src.read()) }; + dst = unsafe { dst.add(1) }; + src = unsafe { src.add(1) }; + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_getcBlockSize( + src: *const c_void, + src_size: usize, + bp_ptr: *mut c_void, +) -> usize { + if src_size < ZSTD_BLOCK_HEADER_SIZE { + return ERROR(ZstdErrorCode::SrcSizeWrong); + } + let header = unsafe { MEM_readLE24(src) }; + let bp = bp_ptr.cast::(); + unsafe { + (*bp).last_block = header & 1; + (*bp).block_type = ((header >> 1) & 3) as c_int; + (*bp).orig_size = header >> 3; + } + if unsafe { (*bp).block_type } == 1 { + return 1; + } + if unsafe { (*bp).block_type } == 3 { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + (header >> 3) as usize +} + +#[inline] +unsafe fn block_size_max(ctx: *const ZSTD_rustBlockCtx) -> usize { + let value = unsafe { + if *(*ctx).is_frame_decompression != 0 { + (*ctx).block_size_max + } else { + ZSTD_BLOCKSIZE_MAX + } + }; + debug_assert!(value <= ZSTD_BLOCKSIZE_MAX); + value +} + +unsafe fn allocate_literals_buffer( + ctx: *mut ZSTD_rustBlockCtx, + dst: *mut u8, + dst_capacity: usize, + lit_size: usize, + streaming: c_int, + expected_write_size: usize, + split_immediately: bool, +) { + let block_max = unsafe { block_size_max(ctx) }; + debug_assert!(lit_size <= block_max); + debug_assert!(*unsafe { (*ctx).is_frame_decompression } != 0 || streaming == NOT_STREAMING); + if streaming == NOT_STREAMING + && dst_capacity + > block_max + .saturating_add(WILDCOPY_OVERLENGTH) + .saturating_add(lit_size) + .saturating_add(WILDCOPY_OVERLENGTH) + { + let buffer = unsafe { ptr_add(dst, block_max + WILDCOPY_OVERLENGTH) }; + unsafe { + *(*ctx).lit_buffer = buffer; + *(*ctx).lit_buffer_end = buffer.add(lit_size); + *(*ctx).lit_buffer_location = ZSTD_IN_DST; + } + } else if lit_size <= unsafe { (*ctx).lit_extra_buffer_size } { + let buffer = unsafe { (*ctx).lit_extra_buffer }; + unsafe { + *(*ctx).lit_buffer = buffer; + *(*ctx).lit_buffer_end = buffer.add(lit_size); + *(*ctx).lit_buffer_location = ZSTD_NOT_IN_DST; + } + } else { + let extra = unsafe { (*ctx).lit_extra_buffer_size }; + debug_assert!(block_max > extra); + let buffer = if split_immediately { + unsafe { + ptr_add( + dst, + expected_write_size - lit_size + extra - WILDCOPY_OVERLENGTH, + ) + } + } else { + unsafe { ptr_add(dst, expected_write_size - lit_size) } + }; + unsafe { + *(*ctx).lit_buffer = buffer; + *(*ctx).lit_buffer_end = if split_immediately { + buffer.add(lit_size - extra) + } else { + ptr_add(dst, expected_write_size).cast_const() + }; + *(*ctx).lit_buffer_location = ZSTD_SPLIT; + } + } +} + +unsafe fn decode_literals_block( + ctx: *mut ZSTD_rustBlockCtx, + src: *const u8, + src_size: usize, + dst: *mut u8, + dst_capacity: usize, + streaming: c_int, +) -> usize { + if src_size < MIN_CBLOCK_SIZE { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + let lit_type = unsafe { *src & 3 } as c_int; + let block_max = unsafe { block_size_max(ctx) }; + match lit_type { + SET_REPEAT | SET_COMPRESSED => { + if lit_type == SET_REPEAT && unsafe { *(*ctx).lit_entropy } == 0 { + return ERROR(ZstdErrorCode::DictionaryCorrupted); + } + if src_size < 5 { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + let lhl_code = unsafe { (*src >> 2) & 3 }; + let header = unsafe { crate::mem::MEM_readLE32(src.cast()) }; + let (header_size, lit_size, lit_c_size, single_stream) = match lhl_code { + 0 | 1 => ( + 3usize, + ((header >> 4) & 0x3ff) as usize, + ((header >> 14) & 0x3ff) as usize, + lhl_code == 0, + ), + 2 => ( + 4usize, + ((header >> 4) & 0x3fff) as usize, + (header >> 18) as usize, + false, + ), + _ => ( + 5usize, + ((header >> 4) & 0x3ffff) as usize, + ((header >> 22) as usize).wrapping_add((unsafe { *src.add(4) } as usize) << 10), + false, + ), + }; + if (lit_size != 0 && dst.is_null()) + || lit_size > block_max + || (!single_stream && lit_size < MIN_LITERALS_FOR_4_STREAMS) + || header_size.checked_add(lit_c_size).is_none() + || header_size + lit_c_size > src_size + || min(block_max, dst_capacity) < lit_size + { + return if lit_size != 0 && dst.is_null() || min(block_max, dst_capacity) < lit_size + { + ERROR(ZstdErrorCode::DstSizeTooSmall) + } else { + ERROR(ZstdErrorCode::CorruptionDetected) + }; + } + unsafe { + allocate_literals_buffer( + ctx, + dst, + dst_capacity, + lit_size, + streaming, + min(block_max, dst_capacity), + false, + ); + } + let flags = (if unsafe { (*ctx).bmi2 } != 0 { + HUF_FLAGS_BMI2 + } else { + 0 + }) | (if unsafe { (*ctx).disable_huf_asm } != 0 { + HUF_FLAGS_DISABLE_ASM + } else { + 0 + }); + let lit_buffer = unsafe { *(*ctx).lit_buffer }; + let huf_result = if lit_type == SET_REPEAT { + if single_stream { + unsafe { + HUF_decompress1X_usingDTable( + lit_buffer.cast(), + lit_size, + src.add(header_size).cast(), + lit_c_size, + *(*ctx).huf_ptr, + flags, + ) + } + } else { + unsafe { + HUF_decompress4X_usingDTable( + lit_buffer.cast(), + lit_size, + src.add(header_size).cast(), + lit_c_size, + *(*ctx).huf_ptr, + flags, + ) + } + } + } else if single_stream { + #[cfg(feature = "huf-force-decompress-x2")] + { + unsafe { + HUF_decompress1X_DCtx_wksp( + (*entropy(ctx)).huf_table.as_mut_ptr(), + lit_buffer.cast(), + lit_size, + src.add(header_size).cast(), + lit_c_size, + (*ctx).workspace.cast(), + (*ctx).workspace_size, + flags, + ) + } + } + #[cfg(not(feature = "huf-force-decompress-x2"))] + { + unsafe { + HUF_decompress1X1_DCtx_wksp( + (*entropy(ctx)).huf_table.as_mut_ptr(), + lit_buffer.cast(), + lit_size, + src.add(header_size).cast(), + lit_c_size, + (*ctx).workspace.cast(), + (*ctx).workspace_size, + flags, + ) + } + } + } else { + unsafe { + HUF_decompress4X_hufOnly_wksp( + (*entropy(ctx)).huf_table.as_mut_ptr(), + lit_buffer.cast(), + lit_size, + src.add(header_size).cast(), + lit_c_size, + (*ctx).workspace.cast(), + (*ctx).workspace_size, + flags, + ) + } + }; + if ERR_isError(huf_result) { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + if unsafe { *(*ctx).lit_buffer_location } == ZSTD_SPLIT { + let extra = unsafe { (*ctx).lit_extra_buffer_size }; + unsafe { + copy_bytes( + (*ctx).lit_extra_buffer, + (*(*ctx).lit_buffer_end).sub(extra), + extra, + ); + ptr::copy( + *(*ctx).lit_buffer, + (*(*ctx).lit_buffer).add(extra - WILDCOPY_OVERLENGTH), + lit_size - extra, + ); + *(*ctx).lit_buffer = (*(*ctx).lit_buffer).add(extra - WILDCOPY_OVERLENGTH); + *(*ctx).lit_buffer_end = (*(*ctx).lit_buffer_end).sub(WILDCOPY_OVERLENGTH); + } + } + unsafe { + *(*ctx).lit_ptr = *(*ctx).lit_buffer; + *(*ctx).lit_size = lit_size; + *(*ctx).lit_entropy = 1; + if lit_type == SET_COMPRESSED { + *(*ctx).huf_ptr = (*entropy(ctx)).huf_table.as_ptr(); + } + } + header_size + lit_c_size + } + SET_BASIC => { + let lhl_code = unsafe { (*src >> 2) & 3 }; + let (header_size, lit_size) = match lhl_code { + 0 | 2 => (1usize, (unsafe { *src } >> 3) as usize), + 1 => (2usize, (unsafe { MEM_readLE16(src.cast()) } >> 4) as usize), + _ => { + if src_size < 3 { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + (3usize, (unsafe { MEM_readLE24(src.cast()) } >> 4) as usize) + } + }; + if (lit_size != 0 && dst.is_null()) + || lit_size > block_max + || min(block_max, dst_capacity) < lit_size + { + return if lit_size != 0 && dst.is_null() || min(block_max, dst_capacity) < lit_size + { + ERROR(ZstdErrorCode::DstSizeTooSmall) + } else { + ERROR(ZstdErrorCode::CorruptionDetected) + }; + } + unsafe { + allocate_literals_buffer( + ctx, + dst, + dst_capacity, + lit_size, + streaming, + min(block_max, dst_capacity), + true, + ); + } + if header_size + .checked_add(lit_size) + .and_then(|size| size.checked_add(WILDCOPY_OVERLENGTH)) + .is_none_or(|size| size > src_size) + { + if header_size + .checked_add(lit_size) + .is_none_or(|size| size > src_size) + { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + unsafe { + if *(*ctx).lit_buffer_location == ZSTD_SPLIT { + let extra = (*ctx).lit_extra_buffer_size; + copy_bytes(*(*ctx).lit_buffer, src.add(header_size), lit_size - extra); + copy_bytes( + (*ctx).lit_extra_buffer, + src.add(header_size + lit_size - extra), + extra, + ); + } else { + copy_bytes(*(*ctx).lit_buffer, src.add(header_size), lit_size); + } + *(*ctx).lit_ptr = *(*ctx).lit_buffer; + *(*ctx).lit_size = lit_size; + } + return header_size + lit_size; + } + unsafe { + *(*ctx).lit_ptr = src.add(header_size); + *(*ctx).lit_size = lit_size; + *(*ctx).lit_buffer_end = src.add(header_size + lit_size); + *(*ctx).lit_buffer_location = ZSTD_NOT_IN_DST; + } + header_size + lit_size + } + SET_RLE => { + let lhl_code = unsafe { (*src >> 2) & 3 }; + let (header_size, lit_size) = match lhl_code { + 0 | 2 => (1usize, (unsafe { *src } >> 3) as usize), + 1 => { + if src_size < 3 { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + (2usize, (unsafe { MEM_readLE16(src.cast()) } >> 4) as usize) + } + _ => { + if src_size < 4 { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + (3usize, (unsafe { MEM_readLE24(src.cast()) } >> 4) as usize) + } + }; + if (lit_size != 0 && dst.is_null()) + || lit_size > block_max + || min(block_max, dst_capacity) < lit_size + { + return if lit_size != 0 && dst.is_null() || min(block_max, dst_capacity) < lit_size + { + ERROR(ZstdErrorCode::DstSizeTooSmall) + } else { + ERROR(ZstdErrorCode::CorruptionDetected) + }; + } + unsafe { + allocate_literals_buffer( + ctx, + dst, + dst_capacity, + lit_size, + streaming, + min(block_max, dst_capacity), + true, + ); + let value = *src.add(header_size); + if *(*ctx).lit_buffer_location == ZSTD_SPLIT { + let extra = (*ctx).lit_extra_buffer_size; + fill_bytes(*(*ctx).lit_buffer, value, lit_size - extra); + fill_bytes((*ctx).lit_extra_buffer, value, extra); + } else { + fill_bytes(*(*ctx).lit_buffer, value, lit_size); + } + *(*ctx).lit_ptr = *(*ctx).lit_buffer; + *(*ctx).lit_size = lit_size; + } + header_size + 1 + } + _ => ERROR(ZstdErrorCode::CorruptionDetected), + } +} + +unsafe fn build_seq_table_rle(table: *mut ZSTD_seqSymbol, base_value: u32, nb_add_bits: u8) { + unsafe { + set_seq_header( + table, + ZSTD_seqSymbol_header { + fast_mode: 0, + table_log: 0, + }, + ); + *table.add(1) = ZSTD_seqSymbol { + next_state: 0, + nb_additional_bits: nb_add_bits, + nb_bits: 0, + base_value, + }; + } +} + +unsafe fn build_fse_table_body( + table: *mut ZSTD_seqSymbol, + normalized_counter: *const i16, + max_symbol_value: u32, + base_value: *const u32, + nb_additional_bits: *const u8, + table_log: u32, + workspace: *mut u32, + workspace_size: usize, +) { + debug_assert!(max_symbol_value as usize <= MAX_ML); + debug_assert!(table_log as usize <= MAX_FSE_LOG); + debug_assert!( + workspace_size >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 * std::mem::size_of::() + ); + let table_decode = unsafe { table.add(1) }; + let table_size = 1usize << table_log; + let symbol_next = workspace.cast::(); + let spread = unsafe { symbol_next.add(MAX_ML + 1).cast::() }; + let mut high_threshold = table_size - 1; + let mut fast_mode = 1u32; + let large_limit = 1i16 << (table_log - 1); + for symbol in 0..=max_symbol_value as usize { + let count = unsafe { *normalized_counter.add(symbol) }; + if count == -1 { + unsafe { + (*table_decode.add(high_threshold)).base_value = symbol as u32; + *symbol_next.add(symbol) = 1; + } + high_threshold = high_threshold.wrapping_sub(1); + } else { + if count >= large_limit { + fast_mode = 0; + } + debug_assert!(count >= 0); + unsafe { *symbol_next.add(symbol) = count as u16 }; + } + } + unsafe { + set_seq_header( + table, + ZSTD_seqSymbol_header { + fast_mode, + table_log, + }, + ); + } + let table_mask = table_size - 1; + let step = (table_size >> 1) + (table_size >> 3) + 3; + if high_threshold == table_size - 1 { + let mut pos = 0usize; + for symbol in 0..=max_symbol_value as usize { + let count = unsafe { *normalized_counter.add(symbol) }; + debug_assert!(count >= 0); + for index in 0..count as usize { + unsafe { *spread.add(pos + index) = symbol as u8 }; + } + pos += count as usize; + } + let mut position = 0usize; + for symbol_index in 0..table_size { + unsafe { (*table_decode.add(position)).base_value = *spread.add(symbol_index) as u32 }; + position = (position + step) & table_mask; + } + debug_assert_eq!(position, 0); + } else { + let mut position = 0usize; + for symbol in 0..=max_symbol_value as usize { + let count = unsafe { *normalized_counter.add(symbol) }; + for _ in 0..count.max(0) as usize { + unsafe { (*table_decode.add(position)).base_value = symbol as u32 }; + position = (position + step) & table_mask; + while position > high_threshold { + position = (position + step) & table_mask; + } + } + } + debug_assert_eq!(position, 0); + } + for index in 0..table_size { + let symbol = unsafe { (*table_decode.add(index)).base_value as usize }; + let next_state = unsafe { *symbol_next.add(symbol) } as u32; + unsafe { *symbol_next.add(symbol) = next_state.wrapping_add(1) as u16 }; + let nb_bits = table_log - crate::bits::ZSTD_highbit32(next_state); + unsafe { + let entry = &mut *table_decode.add(index); + entry.nb_bits = nb_bits as u8; + entry.next_state = ((next_state << nb_bits) - table_size as u32) as u16; + entry.nb_additional_bits = *nb_additional_bits.add(symbol); + entry.base_value = *base_value.add(symbol); + } + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_buildFSETable_body( + table: *mut ZSTD_seqSymbol, + normalized_counter: *const c_short, + max_symbol_value: c_uint, + base_value: *const U32, + nb_additional_bits: *const u8, + table_log: c_uint, + workspace: *mut c_void, + workspace_size: usize, +) { + unsafe { + build_fse_table_body( + table, + normalized_counter, + max_symbol_value, + base_value, + nb_additional_bits, + table_log, + workspace.cast(), + workspace_size, + ); + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_buildFSETable( + table: *mut ZSTD_seqSymbol, + normalized_counter: *const c_short, + max_symbol_value: c_uint, + base_value: *const U32, + nb_additional_bits: *const u8, + table_log: c_uint, + workspace: *mut c_void, + workspace_size: usize, + _bmi2: c_int, +) { + unsafe { + build_fse_table_body( + table, + normalized_counter, + max_symbol_value, + base_value, + nb_additional_bits, + table_log, + workspace.cast(), + workspace_size, + ); + } +} + +fn build_default_table( + normalized: &[i16], + base: &[u32], + bits: &[u8], + max_symbol_value: u32, + table_log: u32, +) -> [ZSTD_seqSymbol; N] { + let mut table = [ZSTD_seqSymbol::default(); N]; + let mut workspace = [0u32; ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; + unsafe { + build_fse_table_body( + table.as_mut_ptr(), + normalized.as_ptr(), + max_symbol_value, + base.as_ptr(), + bits.as_ptr(), + table_log, + workspace.as_mut_ptr(), + std::mem::size_of_val(&workspace), + ); + } + table +} + +static LL_DEFAULT_TABLE: OnceLock<[ZSTD_seqSymbol; 1 + (1 << LL_FSE_LOG)]> = OnceLock::new(); +static OF_DEFAULT_TABLE: OnceLock<[ZSTD_seqSymbol; 1 + (1 << OFF_FSE_LOG)]> = OnceLock::new(); +static ML_DEFAULT_TABLE: OnceLock<[ZSTD_seqSymbol; 1 + (1 << ML_FSE_LOG)]> = OnceLock::new(); + +#[inline] +fn ll_default_table() -> *const ZSTD_seqSymbol { + LL_DEFAULT_TABLE + .get_or_init(|| { + build_default_table( + &LL_DEFAULT_NORM, + &LL_BASE, + &LL_BITS, + MAX_LL as u32, + LL_DEFAULT_NORM_LOG, + ) + }) + .as_ptr() +} + +#[inline] +fn of_default_table() -> *const ZSTD_seqSymbol { + OF_DEFAULT_TABLE + .get_or_init(|| { + build_default_table( + &OF_DEFAULT_NORM, + &OF_BASE, + &OF_BITS, + DEFAULT_MAX_OFF as u32, + OF_DEFAULT_NORM_LOG, + ) + }) + .as_ptr() +} + +#[inline] +fn ml_default_table() -> *const ZSTD_seqSymbol { + ML_DEFAULT_TABLE + .get_or_init(|| { + build_default_table( + &ML_DEFAULT_NORM, + &ML_BASE, + &ML_BITS, + MAX_ML as u32, + ML_DEFAULT_NORM_LOG, + ) + }) + .as_ptr() +} + +unsafe fn build_seq_table( + table_space: *mut ZSTD_seqSymbol, + table_ptr: *mut *const ZSTD_seqSymbol, + table_type: c_int, + mut max_symbol: u32, + max_log: u32, + src: *const u8, + src_size: usize, + base: &[u32], + bits: &[u8], + default_table: *const ZSTD_seqSymbol, + repeat_table_available: u32, + _ddict_is_cold: c_int, + _nb_seq: c_int, + workspace: *mut u32, + workspace_size: usize, + bmi2: c_int, +) -> usize { + match table_type { + SET_RLE => { + if src_size == 0 { + return ERROR(ZstdErrorCode::SrcSizeWrong); + } + let symbol = unsafe { *src } as usize; + if symbol > max_symbol as usize { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + unsafe { build_seq_table_rle(table_space, base[symbol], bits[symbol]) }; + unsafe { *table_ptr = table_space }; + 1 + } + SET_BASIC => { + unsafe { *table_ptr = default_table }; + 0 + } + SET_REPEAT => { + if repeat_table_available == 0 { + ERROR(ZstdErrorCode::CorruptionDetected) + } else { + 0 + } + } + SET_COMPRESSED => { + let mut table_log = 0u32; + let mut norm = [0i16; MAX_ML + 1]; + let header_size = unsafe { + FSE_readNCount( + norm.as_mut_ptr(), + &mut max_symbol, + &mut table_log, + src.cast(), + src_size, + ) + }; + if ERR_isError(header_size) || table_log > max_log { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + unsafe { + build_fse_table_body( + table_space, + norm.as_ptr(), + max_symbol, + base.as_ptr(), + bits.as_ptr(), + table_log, + workspace, + workspace_size, + ); + *table_ptr = table_space; + } + let _ = bmi2; + header_size + } + _ => ERROR(ZstdErrorCode::Generic), + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_decodeSeqHeaders( + ctx: *mut ZSTD_rustBlockCtx, + nb_seq_ptr: *mut c_int, + src: *const c_void, + src_size: usize, +) -> usize { + if src_size < MIN_SEQUENCES_SIZE { + return ERROR(ZstdErrorCode::SrcSizeWrong); + } + let start = src.cast::(); + let end = unsafe { start.add(src_size) }; + let mut ip = start; + let mut nb_seq = unsafe { *ip } as usize; + ip = unsafe { ip.add(1) }; + if nb_seq > 0x7f { + if nb_seq == 0xff { + if (ip as usize).wrapping_add(2) > end as usize { + return ERROR(ZstdErrorCode::SrcSizeWrong); + } + nb_seq = unsafe { MEM_readLE16(ip.cast()) as usize } + LONG_NB_SEQ; + ip = unsafe { ip.add(2) }; + } else { + if ip >= end { + return ERROR(ZstdErrorCode::SrcSizeWrong); + } + nb_seq = ((nb_seq - 0x80) << 8) + unsafe { *ip } as usize; + ip = unsafe { ip.add(1) }; + } + } + unsafe { *nb_seq_ptr = nb_seq as c_int }; + if nb_seq == 0 { + return if ip == end { + unsafe { ip.offset_from(start) as usize } + } else { + ERROR(ZstdErrorCode::CorruptionDetected) + }; + } + if ip >= end || unsafe { *ip & 3 } != 0 { + return if ip >= end { + ERROR(ZstdErrorCode::SrcSizeWrong) + } else { + ERROR(ZstdErrorCode::CorruptionDetected) + }; + } + let descriptor = unsafe { *ip }; + ip = unsafe { ip.add(1) }; + let ll_type = (descriptor >> 6) as c_int; + let of_type = ((descriptor >> 4) & 3) as c_int; + let ml_type = ((descriptor >> 2) & 3) as c_int; + let entropy = unsafe { entropy(ctx) }; + let repeat = unsafe { *(*ctx).fse_entropy }; + let cold = unsafe { *(*ctx).ddict_is_cold }; + let ll_size = unsafe { + build_seq_table( + (*entropy).ll_table.as_mut_ptr(), + (*ctx).llt_ptr, + ll_type, + MAX_LL as u32, + LL_FSE_LOG, + ip, + address_distance(end, ip), + &LL_BASE, + &LL_BITS, + ll_default_table(), + repeat, + cold, + nb_seq as c_int, + (*ctx).workspace, + (*ctx).workspace_size, + (*ctx).bmi2, + ) + }; + if ERR_isError(ll_size) || ll_size > unsafe { address_distance(end, ip) } { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + ip = unsafe { ip.add(ll_size) }; + let of_size = unsafe { + build_seq_table( + (*entropy).of_table.as_mut_ptr(), + (*ctx).oft_ptr, + of_type, + MAX_OFF as u32, + OFF_FSE_LOG, + ip, + address_distance(end, ip), + &OF_BASE, + &OF_BITS, + of_default_table(), + repeat, + cold, + nb_seq as c_int, + (*ctx).workspace, + (*ctx).workspace_size, + (*ctx).bmi2, + ) + }; + if ERR_isError(of_size) || of_size > unsafe { address_distance(end, ip) } { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + ip = unsafe { ip.add(of_size) }; + let ml_size = unsafe { + build_seq_table( + (*entropy).ml_table.as_mut_ptr(), + (*ctx).mlt_ptr, + ml_type, + MAX_ML as u32, + ML_FSE_LOG, + ip, + address_distance(end, ip), + &ML_BASE, + &ML_BITS, + ml_default_table(), + repeat, + cold, + nb_seq as c_int, + (*ctx).workspace, + (*ctx).workspace_size, + (*ctx).bmi2, + ) + }; + if ERR_isError(ml_size) || ml_size > unsafe { address_distance(end, ip) } { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + ip = unsafe { ip.add(ml_size) }; + unsafe { ip.offset_from(start) as usize } +} + +#[inline] +unsafe fn init_fse_state( + state: &mut ZSTD_fseState, + dstream: *mut BIT_DStream_t, + table: *const ZSTD_seqSymbol, +) -> Result<(), usize> { + if table.is_null() { + return Err(ERROR(ZstdErrorCode::CorruptionDetected)); + } + let header = unsafe { seq_header(table) }; + if header.table_log > MAX_FSE_LOG as u32 { + return Err(ERROR(ZstdErrorCode::CorruptionDetected)); + } + state.state = unsafe { BIT_readBits(dstream, header.table_log) }; + let _ = unsafe { BIT_reloadDStream(dstream) }; + state.table = unsafe { table.add(1) }; + Ok(()) +} + +#[inline] +unsafe fn update_fse_state( + state: &mut ZSTD_fseState, + dstream: *mut BIT_DStream_t, + next_state: u16, + nb_bits: u8, +) { + let low_bits = unsafe { BIT_readBits(dstream, nb_bits as u32) }; + state.state = next_state as usize + low_bits; +} + +unsafe fn decode_sequence( + state: &mut seq_state_t, + long_offsets: bool, + is_last_sequence: bool, +) -> ZSTD_rustSeq { + let ll_info = unsafe { state.state_ll.table.add(state.state_ll.state).read() }; + let ml_info = unsafe { state.state_ml.table.add(state.state_ml.state).read() }; + let of_info = unsafe { state.state_off.table.add(state.state_off.state).read() }; + let ll_bits = ll_info.nb_additional_bits as u32; + let ml_bits = ml_info.nb_additional_bits as u32; + let of_bits = of_info.nb_additional_bits as u32; + let total_bits = ll_bits + ml_bits + of_bits; + let mut sequence = ZSTD_rustSeq { + lit_length: ll_info.base_value as usize, + match_length: ml_info.base_value as usize, + offset: 0, + }; + + let offset = if of_bits > 1 { + let value = if MEM_32bits() && long_offsets && of_bits >= 25 { + let upper = unsafe { BIT_readBitsFast(&mut state.dstream, of_bits - 5) } << 5; + let _ = unsafe { BIT_reloadDStream(&mut state.dstream) }; + upper + unsafe { BIT_readBitsFast(&mut state.dstream, 5) } + } else { + let value = unsafe { BIT_readBitsFast(&mut state.dstream, of_bits) }; + if MEM_32bits() { + let _ = unsafe { BIT_reloadDStream(&mut state.dstream) }; + } + value + }; + let value = (of_info.base_value as usize).wrapping_add(value); + state.prev_offset[2] = state.prev_offset[1]; + state.prev_offset[1] = state.prev_offset[0]; + state.prev_offset[0] = value; + value + } else { + let ll_zero = ll_info.base_value == 0; + if of_bits == 0 { + let value = state.prev_offset[usize::from(ll_zero)]; + state.prev_offset[1] = state.prev_offset[usize::from(!ll_zero)]; + state.prev_offset[0] = value; + value + } else { + let code = (of_info.base_value as usize) + .wrapping_add(usize::from(ll_zero)) + .wrapping_add(unsafe { BIT_readBitsFast(&mut state.dstream, 1) }); + let mut value = if code == 3 { + state.prev_offset[0].wrapping_sub(1) + } else if code < ZSTD_REP_NUM { + state.prev_offset[code] + } else { + usize::MAX + }; + value = value.wrapping_sub(usize::from(value == 0)); + if code != 1 { + state.prev_offset[2] = state.prev_offset[1]; + } + state.prev_offset[1] = state.prev_offset[0]; + state.prev_offset[0] = value; + value + } + }; + sequence.offset = offset; + if ml_bits != 0 { + sequence.match_length = sequence + .match_length + .wrapping_add(unsafe { BIT_readBitsFast(&mut state.dstream, ml_bits) }); + } + if MEM_32bits() && ml_bits + ll_bits >= 20 { + let _ = unsafe { BIT_reloadDStream(&mut state.dstream) }; + } + if MEM_64bits() && total_bits >= 30 { + let _ = unsafe { BIT_reloadDStream(&mut state.dstream) }; + } + if ll_bits != 0 { + sequence.lit_length = sequence + .lit_length + .wrapping_add(unsafe { BIT_readBitsFast(&mut state.dstream, ll_bits) }); + } + if MEM_32bits() { + let _ = unsafe { BIT_reloadDStream(&mut state.dstream) }; + } + if !is_last_sequence { + unsafe { + update_fse_state( + &mut state.state_ll, + &mut state.dstream, + ll_info.next_state, + ll_info.nb_bits, + ); + update_fse_state( + &mut state.state_ml, + &mut state.dstream, + ml_info.next_state, + ml_info.nb_bits, + ); + } + if MEM_32bits() { + let _ = unsafe { BIT_reloadDStream(&mut state.dstream) }; + } + unsafe { + update_fse_state( + &mut state.state_off, + &mut state.dstream, + of_info.next_state, + of_info.nb_bits, + ); + } + let _ = unsafe { BIT_reloadDStream(&mut state.dstream) }; + } + sequence +} + +unsafe fn exec_sequence( + op: *mut u8, + oend: *mut u8, + sequence: ZSTD_rustSeq, + lit_ptr: &mut *const u8, + lit_limit: *const u8, + prefix_start: *const u8, + virtual_start: *const u8, + dict_end: *const u8, + split_literals: bool, +) -> usize { + let sequence_length = match sequence.lit_length.checked_add(sequence.match_length) { + Some(length) => length, + None => return ERROR(ZstdErrorCode::DstSizeTooSmall), + }; + let output_capacity = unsafe { address_distance(oend.cast_const(), op.cast_const()) }; + if sequence_length > output_capacity { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + let literal_available = unsafe { address_distance(lit_limit, *lit_ptr) }; + if sequence.lit_length > literal_available { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + let lit_end = unsafe { ptr_add(op, sequence.lit_length) }; + if split_literals + && (op as usize) > (*lit_ptr as usize) + && (op as usize) < (*lit_ptr as usize).wrapping_add(sequence.lit_length) + { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + unsafe { copy_bytes(op, *lit_ptr, sequence.lit_length) }; + *lit_ptr = unsafe { const_ptr_add(*lit_ptr, sequence.lit_length) }; + if sequence.offset == 0 { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + let prefix_history = unsafe { address_distance(lit_end.cast_const(), prefix_start) }; + let mut output = lit_end; + let mut match_length = sequence.match_length; + if sequence.offset > prefix_history { + let virtual_history = unsafe { address_distance(lit_end.cast_const(), virtual_start) }; + if sequence.offset > virtual_history || dict_end.is_null() { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + let before_prefix = sequence.offset - prefix_history; + let match_ptr = unsafe { dict_end.sub(before_prefix) }; + let dict_available = unsafe { address_distance(dict_end, match_ptr) }; + let first_length = min(match_length, dict_available); + unsafe { copy_bytes(output, match_ptr, first_length) }; + output = unsafe { output.add(first_length) }; + match_length -= first_length; + if match_length != 0 { + if prefix_start.is_null() { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + unsafe { copy_match(output, prefix_start, match_length) }; + } + } else { + let match_ptr = unsafe { lit_end.sub(sequence.offset) }; + if match_ptr.is_null() { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + unsafe { copy_match(output, match_ptr, match_length) }; + } + sequence_length +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_execSequenceEnd( + op: *mut u8, + oend: *mut u8, + sequence: ZSTD_rustSeq, + lit_ptr: *mut *const u8, + lit_limit: *const u8, + prefix_start: *const u8, + virtual_start: *const u8, + dict_end: *const u8, +) -> usize { + unsafe { + exec_sequence( + op, + oend, + sequence, + &mut *lit_ptr, + lit_limit, + prefix_start, + virtual_start, + dict_end, + false, + ) + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_execSequenceEndSplitLitBuffer( + op: *mut u8, + oend: *mut u8, + _oend_w: *const u8, + sequence: ZSTD_rustSeq, + lit_ptr: *mut *const u8, + lit_limit: *const u8, + prefix_start: *const u8, + virtual_start: *const u8, + dict_end: *const u8, +) -> usize { + unsafe { + exec_sequence( + op, + oend, + sequence, + &mut *lit_ptr, + lit_limit, + prefix_start, + virtual_start, + dict_end, + true, + ) + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_execSequence( + op: *mut u8, + oend: *mut u8, + sequence: ZSTD_rustSeq, + lit_ptr: *mut *const u8, + lit_limit: *const u8, + prefix_start: *const u8, + virtual_start: *const u8, + dict_end: *const u8, +) -> usize { + unsafe { + exec_sequence( + op, + oend, + sequence, + &mut *lit_ptr, + lit_limit, + prefix_start, + virtual_start, + dict_end, + false, + ) + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_execSequenceSplitLitBuffer( + op: *mut u8, + oend: *mut u8, + _oend_w: *const u8, + sequence: ZSTD_rustSeq, + lit_ptr: *mut *const u8, + lit_limit: *const u8, + prefix_start: *const u8, + virtual_start: *const u8, + dict_end: *const u8, +) -> usize { + unsafe { + exec_sequence( + op, + oend, + sequence, + &mut *lit_ptr, + lit_limit, + prefix_start, + virtual_start, + dict_end, + true, + ) + } +} + +unsafe fn decompress_sequences( + ctx: *mut ZSTD_rustBlockCtx, + dst: *mut u8, + dst_capacity: usize, + sequence_start: *const u8, + sequence_size: usize, + nb_seq: c_int, + long_offsets: bool, +) -> usize { + if nb_seq < 0 { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + let mut location = unsafe { *(*ctx).lit_buffer_location }; + let mut op = dst; + let mut oend = if location == ZSTD_NOT_IN_DST || location == ZSTD_SPLIT { + unsafe { ptr_add(dst, dst_capacity) } + } else { + unsafe { *(*ctx).lit_buffer } + }; + let mut lit_ptr = unsafe { *(*ctx).lit_ptr }; + let mut lit_limit = if location == ZSTD_SPLIT { + unsafe { *(*ctx).lit_buffer_end } + } else { + unsafe { const_ptr_add(lit_ptr, *(*ctx).lit_size) } + }; + let prefix_start = unsafe { *(*ctx).prefix_start }; + let virtual_start = unsafe { *(*ctx).virtual_start }; + let dict_end = unsafe { *(*ctx).dict_end }; + if nb_seq != 0 { + if dst.is_null() { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + let mut state = seq_state_t { + dstream: unsafe { MaybeUninit::zeroed().assume_init() }, + state_ll: ZSTD_fseState { + state: 0, + table: ptr::null(), + }, + state_off: ZSTD_fseState { + state: 0, + table: ptr::null(), + }, + state_ml: ZSTD_fseState { + state: 0, + table: ptr::null(), + }, + prev_offset: unsafe { (*entropy(ctx)).rep.map(|value| value as usize) }, + }; + unsafe { *(*ctx).fse_entropy = 1 }; + let init = + unsafe { BIT_initDStream(&mut state.dstream, sequence_start.cast(), sequence_size) }; + if ERR_isError(init) { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + if let Err(error) = + unsafe { init_fse_state(&mut state.state_ll, &mut state.dstream, *(*ctx).llt_ptr) } + { + return error; + } + if let Err(error) = + unsafe { init_fse_state(&mut state.state_off, &mut state.dstream, *(*ctx).oft_ptr) } + { + return error; + } + if let Err(error) = + unsafe { init_fse_state(&mut state.state_ml, &mut state.dstream, *(*ctx).mlt_ptr) } + { + return error; + } + for remaining in (1..=nb_seq as usize).rev() { + let mut sequence = unsafe { decode_sequence(&mut state, long_offsets, remaining == 1) }; + if location == ZSTD_SPLIT + && sequence.lit_length > unsafe { address_distance(lit_limit, lit_ptr) } + { + let leftover = unsafe { address_distance(lit_limit, lit_ptr) }; + if leftover > unsafe { address_distance(oend.cast_const(), op.cast_const()) } { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + unsafe { copy_bytes(op, lit_ptr, leftover) }; + op = unsafe { op.add(leftover) }; + sequence.lit_length -= leftover; + lit_ptr = unsafe { (*ctx).lit_extra_buffer.cast_const() }; + lit_limit = unsafe { + const_ptr_add( + (*ctx).lit_extra_buffer.cast_const(), + (*ctx).lit_extra_buffer_size, + ) + }; + location = ZSTD_NOT_IN_DST; + unsafe { *(*ctx).lit_buffer_location = ZSTD_NOT_IN_DST }; + } + let decoded = unsafe { + exec_sequence( + op, + oend, + sequence, + &mut lit_ptr, + lit_limit, + prefix_start, + virtual_start, + dict_end, + location == ZSTD_SPLIT, + ) + }; + if ERR_isError(decoded) { + return decoded; + } + op = unsafe { op.add(decoded) }; + } + if unsafe { BIT_endOfDStream(&state.dstream) } == 0 { + return ERROR(ZstdErrorCode::CorruptionDetected); + } + unsafe { (*entropy(ctx)).rep = state.prev_offset.map(|value| value as u32) }; + } + if location == ZSTD_SPLIT { + let first_size = unsafe { address_distance(lit_limit, lit_ptr) }; + if first_size > unsafe { address_distance(oend.cast_const(), op.cast_const()) } { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + unsafe { copy_bytes(op, lit_ptr, first_size) }; + op = unsafe { op.add(first_size) }; + lit_ptr = unsafe { (*ctx).lit_extra_buffer.cast_const() }; + lit_limit = unsafe { + const_ptr_add( + (*ctx).lit_extra_buffer.cast_const(), + (*ctx).lit_extra_buffer_size, + ) + }; + unsafe { *(*ctx).lit_buffer_location = ZSTD_NOT_IN_DST }; + oend = unsafe { ptr_add(dst, dst_capacity) }; + } + let last_size = unsafe { address_distance(lit_limit, lit_ptr) }; + if last_size > unsafe { address_distance(oend.cast_const(), op.cast_const()) } { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + unsafe { copy_bytes(op, lit_ptr, last_size) }; + op = unsafe { op.add(last_size) }; + unsafe { address_distance(op.cast_const(), dst.cast_const()) } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_prefetchMatch( + prefetch_pos: usize, + sequence: ZSTD_rustSeq, + _prefix_start: *const u8, + _dict_end: *const u8, +) -> usize { + prefetch_pos + .wrapping_add(sequence.lit_length) + .wrapping_add(sequence.match_length) +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_decodeLiteralsBlock_wrapper( + ctx: *mut ZSTD_rustBlockCtx, + src: *const c_void, + src_size: usize, + dst: *mut c_void, + dst_capacity: usize, +) -> usize { + unsafe { + *(*ctx).is_frame_decompression = 0; + decode_literals_block( + ctx, + src.cast(), + src_size, + dst.cast(), + dst_capacity, + NOT_STREAMING, + ) + } +} + +#[inline] +fn max_short_offset() -> usize { + if MEM_64bits() { + usize::MAX + } else { + ((1usize << 26) - 1).wrapping_sub(ZSTD_REP_NUM) + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_decompressBlock_internal( + ctx: *mut ZSTD_rustBlockCtx, + dst: *mut c_void, + dst_capacity: usize, + src: *const c_void, + src_size: usize, + streaming: c_int, +) -> usize { + let block_max = unsafe { block_size_max(ctx) }; + if src_size > block_max { + return ERROR(ZstdErrorCode::SrcSizeWrong); + } + let mut input = src.cast::(); + let lit_size = + unsafe { decode_literals_block(ctx, input, src_size, dst.cast(), dst_capacity, streaming) }; + if ERR_isError(lit_size) || lit_size > src_size { + return lit_size; + } + input = unsafe { input.add(lit_size) }; + let remaining = src_size - lit_size; + let history_end = unsafe { ptr_add(dst.cast(), min(dst_capacity, block_max)) }; + let history_size = unsafe { address_distance(history_end.cast_const(), *(*ctx).virtual_start) }; + let long_offsets = MEM_32bits() && history_size > max_short_offset(); + let mut nb_seq = 0 as c_int; + let header_size = + unsafe { ZSTD_rust_decodeSeqHeaders(ctx, &mut nb_seq, input.cast(), remaining) }; + if ERR_isError(header_size) || header_size > remaining { + return header_size; + } + input = unsafe { input.add(header_size) }; + let sequence_size = remaining - header_size; + if (dst.is_null() || dst_capacity == 0) && nb_seq > 0 { + return ERROR(ZstdErrorCode::DstSizeTooSmall); + } + unsafe { *(*ctx).ddict_is_cold = 0 }; + unsafe { + decompress_sequences( + ctx, + dst.cast(), + dst_capacity, + input, + sequence_size, + nb_seq, + long_offsets, + ) + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_checkContinuity( + ctx: *mut ZSTD_rustBlockCtx, + dst: *const c_void, + dst_size: usize, +) { + unsafe { + if dst.cast::() != *(*ctx).previous_dst_end && dst_size != 0 { + *(*ctx).dict_end = *(*ctx).previous_dst_end; + let prefix = *(*ctx).prefix_start; + let previous = *(*ctx).previous_dst_end; + let distance = address_distance(previous, prefix); + *(*ctx).virtual_start = dst.cast::().wrapping_sub(distance); + *(*ctx).prefix_start = dst.cast(); + *(*ctx).previous_dst_end = dst.cast(); + } + } +} + +#[no_mangle] +pub unsafe extern "C" fn ZSTD_rust_decompressBlock_deprecated( + ctx: *mut ZSTD_rustBlockCtx, + dst: *mut c_void, + dst_capacity: usize, + src: *const c_void, + src_size: usize, +) -> usize { + unsafe { + *(*ctx).is_frame_decompression = 0; + ZSTD_rust_checkContinuity(ctx, dst.cast(), dst_capacity); + let result = ZSTD_rust_decompressBlock_internal( + ctx, + dst, + dst_capacity, + src, + src_size, + NOT_STREAMING, + ); + if !ERR_isError(result) { + *(*ctx).previous_dst_end = ptr_add(dst.cast(), result).cast_const(); + } + result + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn offset_code_22_and_23_keep_their_22_and_23_bit_bases() { + assert_eq!(OF_BASE[22], 0x3F_FFFD); + assert_eq!(OF_BASE[23], 0x7F_FFFD); + assert_eq!(OF_BASE[24], 0xFF_FFFD); + + let mut normalized = [0i16; MAX_OFF + 1]; + normalized[22] = 1 << 5; + let mut table = [ZSTD_seqSymbol::default(); 1 + (1 << 5)]; + let mut workspace = [0u32; ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; + unsafe { + build_fse_table_body( + table.as_mut_ptr(), + normalized.as_ptr(), + 22, + OF_BASE.as_ptr(), + OF_BITS.as_ptr(), + 5, + workspace.as_mut_ptr(), + std::mem::size_of_val(&workspace), + ); + } + assert!(table[1..] + .iter() + .all(|symbol| symbol.base_value == 0x3F_FFFD)); + } +}