ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(rust): port compressed block decoding

Browse Source 
Move literal decoding, sequence-table construction, FSE sequence decoding, and
sequence execution into Rust. The C shim retains ownership of the configured
decoder context and passes only the leaf state needed by the block codec.

Correct the two high offset-code bases while porting the tables. Their prior
values made valid large-window streams decode as corrupted data; the new unit
test fixes the exact values and a native zstream regression exercises them.
High-level frame and streaming context control remains C for now.

Test Plan:
- cargo test --all-targets
- cargo test --target i686-unknown-linux-gnu --all-targets
- cargo clippy && cargo clippy --benches && cargo clippy --tests
- cargo +nightly fmt
- make -B -C tests -j2 fuzzer zstreamtest invalidDictionaries poolTests
- ./tests/fuzzer -s5346 -i1 --no-big-tests
- ./tests/zstreamtest -i3000 -s334462
- ./tests/invalidDictionaries
- timeout 20s stdbuf -oL ./tests/poolTests

Refs: rust/README.md
This commit is contained in:
ddidderr
2026-07-11 08:07:40 +02:00
parent 7f16a07375
commit e5eebd4892
4 changed files with 1823 additions and 2168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/decompress/zstd_decompress_block.c
+114 -2164
View File
@@ -8,2202 +8,152 @@
* You may select, at your option, one of the above-listed licenses. * You may select, at your option, one of the above-listed licenses.
*/ */
/* zstd_decompress_block : /*
* this module takes care of decompressing _compressed_ block */ * Compressed-block decoding is implemented in
* rust/src/zstd_decompress_block.rs. Keep `ZSTD_DCtx` C-owned: it has
/*-******************************************************* * optional build-dependent fields, while this small view contains only the
* Dependencies * leaves read or written by the Rust decoder. No decoder algorithm remains
*********************************************************/ * in this translation unit.
#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
#include "../common/compiler.h" /* prefetch */
#include "../common/cpu.h" /* bmi2 */
#include "../common/mem.h" /* low level memory routines */
#define FSE_STATIC_LINKING_ONLY
#include "../common/fse.h"
#include "../common/huf.h"
#include "../common/zstd_internal.h"
#include "zstd_decompress_internal.h" /* ZSTD_DCtx */
#include "zstd_ddict.h" /* ZSTD_DDictDictContent */
#include "zstd_decompress_block.h"
#include "../common/bits.h" /* ZSTD_highbit32 */
/*_*******************************************************
* Macros
**********************************************************/
/* These two optional macros force the use one way or another of the two
* ZSTD_decompressSequences implementations. You can't force in both directions
* at the same time.
*/ */
#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ #include "../common/zstd_deps.h"
defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) #include "../common/zstd_internal.h"
#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!" #include "zstd_decompress_internal.h"
#endif #include "zstd_decompress_block.h"
typedef char ZSTD_rust_block_seq_symbol_layout[(sizeof(ZSTD_seqSymbol) == 8) ? 1 : -1];
typedef char ZSTD_rust_block_entropy_rep_offset[
(offsetof(ZSTD_entropyDTables_t, rep) == 26652) ? 1 : -1];
typedef char ZSTD_rust_block_entropy_workspace_offset[
(offsetof(ZSTD_entropyDTables_t, workspace) == 26664) ? 1 : -1];
/*_******************************************************* typedef struct {
* Memory operations const ZSTD_seqSymbol** lltPtr;
**********************************************************/ const ZSTD_seqSymbol** mltPtr;
static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } const ZSTD_seqSymbol** oftPtr;
const HUF_DTable** hufPtr;
ZSTD_entropyDTables_t* entropy;
U32* workspace;
size_t workspaceSize;
const void** previousDstEnd;
const void** prefixStart;
const void** virtualStart;
const void** dictEnd;
size_t blockSizeMax;
int* isFrameDecompression;
U32* litEntropy;
U32* fseEntropy;
int bmi2;
int* ddictIsCold;
int disableHufAsm;
const BYTE** litPtr;
size_t* litSize;
size_t* rleSize;
BYTE** litBuffer;
const BYTE** litBufferEnd;
ZSTD_litLocation_e* litBufferLocation;
BYTE* litExtraBuffer;
size_t litExtraBufferSize;
} ZSTD_rustBlockCtx;
static ZSTD_rustBlockCtx ZSTD_rust_block_context(ZSTD_DCtx* dctx)
/*-*************************************************************
* Block decoding
***************************************************************/
static size_t ZSTD_blockSizeMax(ZSTD_DCtx const* dctx)
{ {
size_t const blockSizeMax = dctx->isFrameDecompression ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX; ZSTD_rustBlockCtx ctx;
assert(blockSizeMax <= ZSTD_BLOCKSIZE_MAX); ctx.lltPtr = &dctx->LLTptr;
return blockSizeMax; ctx.mltPtr = &dctx->MLTptr;
ctx.oftPtr = &dctx->OFTptr;
ctx.hufPtr = &dctx->HUFptr;
ctx.entropy = &dctx->entropy;
ctx.workspace = dctx->workspace;
ctx.workspaceSize = sizeof(dctx->workspace);
ctx.previousDstEnd = &dctx->previousDstEnd;
ctx.prefixStart = &dctx->prefixStart;
ctx.virtualStart = &dctx->virtualStart;
ctx.dictEnd = &dctx->dictEnd;
ctx.blockSizeMax = dctx->fParams.blockSizeMax;
ctx.isFrameDecompression = &dctx->isFrameDecompression;
ctx.litEntropy = &dctx->litEntropy;
ctx.fseEntropy = &dctx->fseEntropy;
ctx.bmi2 = ZSTD_DCtx_get_bmi2(dctx);
ctx.ddictIsCold = &dctx->ddictIsCold;
ctx.disableHufAsm = dctx->disableHufAsm;
ctx.litPtr = &dctx->litPtr;
ctx.litSize = &dctx->litSize;
ctx.rleSize = &dctx->rleSize;
ctx.litBuffer = &dctx->litBuffer;
ctx.litBufferEnd = &dctx->litBufferEnd;
ctx.litBufferLocation = &dctx->litBufferLocation;
ctx.litExtraBuffer = dctx->litExtraBuffer;
ctx.litExtraBufferSize = ZSTD_LITBUFFEREXTRASIZE;
return ctx;
} }
/*! ZSTD_getcBlockSize() : size_t ZSTD_rust_decodeLiteralsBlock_wrapper(
* Provides the size of compressed block from block header `src` */ ZSTD_rustBlockCtx* ctx,
size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, const void* src, size_t srcSize,
blockProperties_t* bpPtr) void* dst, size_t dstCapacity);
{ size_t ZSTD_rust_decodeSeqHeaders(
RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, ""); ZSTD_rustBlockCtx* ctx, int* nbSeqPtr,
const void* src, size_t srcSize);
size_t ZSTD_rust_decompressBlock_internal(
ZSTD_rustBlockCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize, int streaming);
void ZSTD_rust_checkContinuity(
ZSTD_rustBlockCtx* ctx, const void* dst, size_t dstSize);
size_t ZSTD_rust_decompressBlock_deprecated(
ZSTD_rustBlockCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
{ U32 const cBlockHeader = MEM_readLE24(src); /* Hidden declaration for fullbench. */
U32 const cSize = cBlockHeader >> 3;
bpPtr->lastBlock = cBlockHeader & 1;
bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3);
bpPtr->origSize = cSize; /* only useful for RLE */
if (bpPtr->blockType == bt_rle) return 1;
RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, "");
return cSize;
}
}
/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */
static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize,
const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately)
{
size_t const blockSizeMax = ZSTD_blockSizeMax(dctx);
assert(litSize <= blockSizeMax);
assert(dctx->isFrameDecompression || streaming == not_streaming);
assert(expectedWriteSize <= blockSizeMax);
if (streaming == not_streaming && dstCapacity > blockSizeMax + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH) {
/* If we aren't streaming, we can just put the literals after the output
* of the current block. We don't need to worry about overwriting the
* extDict of our window, because it doesn't exist.
* So if we have space after the end of the block, just put it there.
*/
dctx->litBuffer = (BYTE*)dst + blockSizeMax + WILDCOPY_OVERLENGTH;
dctx->litBufferEnd = dctx->litBuffer + litSize;
dctx->litBufferLocation = ZSTD_in_dst;
} else if (litSize <= ZSTD_LITBUFFEREXTRASIZE) {
/* Literals fit entirely within the extra buffer, put them there to avoid
* having to split the literals.
*/
dctx->litBuffer = dctx->litExtraBuffer;
dctx->litBufferEnd = dctx->litBuffer + litSize;
dctx->litBufferLocation = ZSTD_not_in_dst;
} else {
assert(blockSizeMax > ZSTD_LITBUFFEREXTRASIZE);
/* Literals must be split between the output block and the extra lit
* buffer. We fill the extra lit buffer with the tail of the literals,
* and put the rest of the literals at the end of the block, with
* WILDCOPY_OVERLENGTH of buffer room to allow for overreads.
* This MUST not write more than our maxBlockSize beyond dst, because in
* streaming mode, that could overwrite part of our extDict window.
*/
if (splitImmediately) {
/* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */
dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH;
dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE;
} else {
/* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */
dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize;
dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize;
}
dctx->litBufferLocation = ZSTD_split;
assert(dctx->litBufferEnd <= (BYTE*)dst + expectedWriteSize);
}
}
/*! ZSTD_decodeLiteralsBlock() :
* Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored
* in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current
* block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being
* stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write.
*
* @return : nb of bytes read from src (< srcSize )
* note : symbol not declared but exposed for fullbench */
static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */
void* dst, size_t dstCapacity, const streaming_operation streaming)
{
DEBUGLOG(5, "ZSTD_decodeLiteralsBlock");
RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, "");
{ const BYTE* const istart = (const BYTE*) src;
SymbolEncodingType_e const litEncType = (SymbolEncodingType_e)(istart[0] & 3);
size_t const blockSizeMax = ZSTD_blockSizeMax(dctx);
switch(litEncType)
{
case set_repeat:
DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block");
RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, "");
ZSTD_FALLTHROUGH;
case set_compressed:
RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3");
{ size_t lhSize, litSize, litCSize;
U32 singleStream=0;
U32 const lhlCode = (istart[0] >> 2) & 3;
U32 const lhc = MEM_readLE32(istart);
size_t hufSuccess;
size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity);
int const flags = 0
| (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0)
| (dctx->disableHufAsm ? HUF_flags_disableAsm : 0);
switch(lhlCode)
{
case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */
/* 2 - 2 - 10 - 10 */
singleStream = !lhlCode;
lhSize = 3;
litSize = (lhc >> 4) & 0x3FF;
litCSize = (lhc >> 14) & 0x3FF;
break;
case 2:
/* 2 - 2 - 14 - 14 */
lhSize = 4;
litSize = (lhc >> 4) & 0x3FFF;
litCSize = lhc >> 18;
break;
case 3:
/* 2 - 2 - 18 - 18 */
lhSize = 5;
litSize = (lhc >> 4) & 0x3FFFF;
litCSize = (lhc >> 22) + ((size_t)istart[4] << 10);
break;
}
RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, "");
if (!singleStream)
RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong,
"Not enough literals (%zu) for the 4-streams mode (min %u)",
litSize, MIN_LITERALS_FOR_4_STREAMS);
RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, "");
RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, "");
ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0);
/* prefetch huffman table if cold */
if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) {
PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable));
}
if (litEncType==set_repeat) {
if (singleStream) {
hufSuccess = HUF_decompress1X_usingDTable(
dctx->litBuffer, litSize, istart+lhSize, litCSize,
dctx->HUFptr, flags);
} else {
assert(litSize >= MIN_LITERALS_FOR_4_STREAMS);
hufSuccess = HUF_decompress4X_usingDTable(
dctx->litBuffer, litSize, istart+lhSize, litCSize,
dctx->HUFptr, flags);
}
} else {
if (singleStream) {
#if defined(HUF_FORCE_DECOMPRESS_X2)
hufSuccess = HUF_decompress1X_DCtx_wksp(
dctx->entropy.hufTable, dctx->litBuffer, litSize,
istart+lhSize, litCSize, dctx->workspace,
sizeof(dctx->workspace), flags);
#else
hufSuccess = HUF_decompress1X1_DCtx_wksp(
dctx->entropy.hufTable, dctx->litBuffer, litSize,
istart+lhSize, litCSize, dctx->workspace,
sizeof(dctx->workspace), flags);
#endif
} else {
hufSuccess = HUF_decompress4X_hufOnly_wksp(
dctx->entropy.hufTable, dctx->litBuffer, litSize,
istart+lhSize, litCSize, dctx->workspace,
sizeof(dctx->workspace), flags);
}
}
if (dctx->litBufferLocation == ZSTD_split)
{
assert(litSize > ZSTD_LITBUFFEREXTRASIZE);
ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE);
ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE);
dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH;
dctx->litBufferEnd -= WILDCOPY_OVERLENGTH;
assert(dctx->litBufferEnd <= (BYTE*)dst + blockSizeMax);
}
RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, "");
dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize;
dctx->litEntropy = 1;
if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable;
return litCSize + lhSize;
}
case set_basic:
{ size_t litSize, lhSize;
U32 const lhlCode = ((istart[0]) >> 2) & 3;
size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity);
switch(lhlCode)
{
case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */
lhSize = 1;
litSize = istart[0] >> 3;
break;
case 1:
lhSize = 2;
litSize = MEM_readLE16(istart) >> 4;
break;
case 3:
lhSize = 3;
RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3");
litSize = MEM_readLE24(istart) >> 4;
break;
}
RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, "");
RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, "");
ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1);
if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */
RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, "");
if (dctx->litBufferLocation == ZSTD_split)
{
ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE);
ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE);
}
else
{
ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize);
}
dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize;
return lhSize+litSize;
}
/* direct reference into compressed stream */
dctx->litPtr = istart+lhSize;
dctx->litSize = litSize;
dctx->litBufferEnd = dctx->litPtr + litSize;
dctx->litBufferLocation = ZSTD_not_in_dst;
return lhSize+litSize;
}
case set_rle:
{ U32 const lhlCode = ((istart[0]) >> 2) & 3;
size_t litSize, lhSize;
size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity);
switch(lhlCode)
{
case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */
lhSize = 1;
litSize = istart[0] >> 3;
break;
case 1:
lhSize = 2;
RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3");
litSize = MEM_readLE16(istart) >> 4;
break;
case 3:
lhSize = 3;
RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4");
litSize = MEM_readLE24(istart) >> 4;
break;
}
RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, "");
RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, "");
ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1);
if (dctx->litBufferLocation == ZSTD_split)
{
ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE);
ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE);
}
else
{
ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize);
}
dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize;
return lhSize+1;
}
default:
RETURN_ERROR(corruption_detected, "impossible");
}
}
}
/* Hidden declaration for fullbench */
size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx,
const void* src, size_t srcSize, const void* src, size_t srcSize,
void* dst, size_t dstCapacity); void* dst, size_t dstCapacity);
size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx,
const void* src, size_t srcSize, const void* src, size_t srcSize,
void* dst, size_t dstCapacity) void* dst, size_t dstCapacity)
{ {
dctx->isFrameDecompression = 0; ZSTD_rustBlockCtx ctx = ZSTD_rust_block_context(dctx);
return ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, not_streaming); return ZSTD_rust_decodeLiteralsBlock_wrapper(
} &ctx, src, srcSize, dst, dstCapacity);
/* Default FSE distribution tables.
* These are pre-calculated FSE decoding tables using default distributions as defined in specification :
* https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions
* They were generated programmatically with following method :
* - start from default distributions, present in /lib/common/zstd_internal.h
* - generate tables normally, using ZSTD_buildFSETable()
* - printout the content of tables
* - prettify output, report below, test with fuzzer to ensure it's correct */
/* Default FSE distribution table for Literal Lengths */
static const ZSTD_seqSymbol LL_defaultDTable[(1<<LL_DEFAULTNORMLOG)+1] = {
{ 1, 1, 1, LL_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
/* nextState, nbAddBits, nbBits, baseVal */
{ 0, 0, 4, 0}, { 16, 0, 4, 0},
{ 32, 0, 5, 1}, { 0, 0, 5, 3},
{ 0, 0, 5, 4}, { 0, 0, 5, 6},
{ 0, 0, 5, 7}, { 0, 0, 5, 9},
{ 0, 0, 5, 10}, { 0, 0, 5, 12},
{ 0, 0, 6, 14}, { 0, 1, 5, 16},
{ 0, 1, 5, 20}, { 0, 1, 5, 22},
{ 0, 2, 5, 28}, { 0, 3, 5, 32},
{ 0, 4, 5, 48}, { 32, 6, 5, 64},
{ 0, 7, 5, 128}, { 0, 8, 6, 256},
{ 0, 10, 6, 1024}, { 0, 12, 6, 4096},
{ 32, 0, 4, 0}, { 0, 0, 4, 1},
{ 0, 0, 5, 2}, { 32, 0, 5, 4},
{ 0, 0, 5, 5}, { 32, 0, 5, 7},
{ 0, 0, 5, 8}, { 32, 0, 5, 10},
{ 0, 0, 5, 11}, { 0, 0, 6, 13},
{ 32, 1, 5, 16}, { 0, 1, 5, 18},
{ 32, 1, 5, 22}, { 0, 2, 5, 24},
{ 32, 3, 5, 32}, { 0, 3, 5, 40},
{ 0, 6, 4, 64}, { 16, 6, 4, 64},
{ 32, 7, 5, 128}, { 0, 9, 6, 512},
{ 0, 11, 6, 2048}, { 48, 0, 4, 0},
{ 16, 0, 4, 1}, { 32, 0, 5, 2},
{ 32, 0, 5, 3}, { 32, 0, 5, 5},
{ 32, 0, 5, 6}, { 32, 0, 5, 8},
{ 32, 0, 5, 9}, { 32, 0, 5, 11},
{ 32, 0, 5, 12}, { 0, 0, 6, 15},
{ 32, 1, 5, 18}, { 32, 1, 5, 20},
{ 32, 2, 5, 24}, { 32, 2, 5, 28},
{ 32, 3, 5, 40}, { 32, 4, 5, 48},
{ 0, 16, 6,65536}, { 0, 15, 6,32768},
{ 0, 14, 6,16384}, { 0, 13, 6, 8192},
}; /* LL_defaultDTable */
/* Default FSE distribution table for Offset Codes */
static const ZSTD_seqSymbol OF_defaultDTable[(1<<OF_DEFAULTNORMLOG)+1] = {
{ 1, 1, 1, OF_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
/* nextState, nbAddBits, nbBits, baseVal */
{ 0, 0, 5, 0}, { 0, 6, 4, 61},
{ 0, 9, 5, 509}, { 0, 15, 5,32765},
{ 0, 21, 5,2097149}, { 0, 3, 5, 5},
{ 0, 7, 4, 125}, { 0, 12, 5, 4093},
{ 0, 18, 5,262141}, { 0, 23, 5,8388605},
{ 0, 5, 5, 29}, { 0, 8, 4, 253},
{ 0, 14, 5,16381}, { 0, 20, 5,1048573},
{ 0, 2, 5, 1}, { 16, 7, 4, 125},
{ 0, 11, 5, 2045}, { 0, 17, 5,131069},
{ 0, 22, 5,4194301}, { 0, 4, 5, 13},
{ 16, 8, 4, 253}, { 0, 13, 5, 8189},
{ 0, 19, 5,524285}, { 0, 1, 5, 1},
{ 16, 6, 4, 61}, { 0, 10, 5, 1021},
{ 0, 16, 5,65533}, { 0, 28, 5,268435453},
{ 0, 27, 5,134217725}, { 0, 26, 5,67108861},
{ 0, 25, 5,33554429}, { 0, 24, 5,16777213},
}; /* OF_defaultDTable */
/* Default FSE distribution table for Match Lengths */
static const ZSTD_seqSymbol ML_defaultDTable[(1<<ML_DEFAULTNORMLOG)+1] = {
{ 1, 1, 1, ML_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
/* nextState, nbAddBits, nbBits, baseVal */
{ 0, 0, 6, 3}, { 0, 0, 4, 4},
{ 32, 0, 5, 5}, { 0, 0, 5, 6},
{ 0, 0, 5, 8}, { 0, 0, 5, 9},
{ 0, 0, 5, 11}, { 0, 0, 6, 13},
{ 0, 0, 6, 16}, { 0, 0, 6, 19},
{ 0, 0, 6, 22}, { 0, 0, 6, 25},
{ 0, 0, 6, 28}, { 0, 0, 6, 31},
{ 0, 0, 6, 34}, { 0, 1, 6, 37},
{ 0, 1, 6, 41}, { 0, 2, 6, 47},
{ 0, 3, 6, 59}, { 0, 4, 6, 83},
{ 0, 7, 6, 131}, { 0, 9, 6, 515},
{ 16, 0, 4, 4}, { 0, 0, 4, 5},
{ 32, 0, 5, 6}, { 0, 0, 5, 7},
{ 32, 0, 5, 9}, { 0, 0, 5, 10},
{ 0, 0, 6, 12}, { 0, 0, 6, 15},
{ 0, 0, 6, 18}, { 0, 0, 6, 21},
{ 0, 0, 6, 24}, { 0, 0, 6, 27},
{ 0, 0, 6, 30}, { 0, 0, 6, 33},
{ 0, 1, 6, 35}, { 0, 1, 6, 39},
{ 0, 2, 6, 43}, { 0, 3, 6, 51},
{ 0, 4, 6, 67}, { 0, 5, 6, 99},
{ 0, 8, 6, 259}, { 32, 0, 4, 4},
{ 48, 0, 4, 4}, { 16, 0, 4, 5},
{ 32, 0, 5, 7}, { 32, 0, 5, 8},
{ 32, 0, 5, 10}, { 32, 0, 5, 11},
{ 0, 0, 6, 14}, { 0, 0, 6, 17},
{ 0, 0, 6, 20}, { 0, 0, 6, 23},
{ 0, 0, 6, 26}, { 0, 0, 6, 29},
{ 0, 0, 6, 32}, { 0, 16, 6,65539},
{ 0, 15, 6,32771}, { 0, 14, 6,16387},
{ 0, 13, 6, 8195}, { 0, 12, 6, 4099},
{ 0, 11, 6, 2051}, { 0, 10, 6, 1027},
}; /* ML_defaultDTable */
static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U8 nbAddBits)
{
void* ptr = dt;
ZSTD_seqSymbol_header* const DTableH = (ZSTD_seqSymbol_header*)ptr;
ZSTD_seqSymbol* const cell = dt + 1;
DTableH->tableLog = 0;
DTableH->fastMode = 0;
cell->nbBits = 0;
cell->nextState = 0;
assert(nbAddBits < 255);
cell->nbAdditionalBits = nbAddBits;
cell->baseValue = baseValue;
}
/* ZSTD_buildFSETable() :
* generate FSE decoding table for one symbol (ll, ml or off)
* cannot fail if input is valid =>
* all inputs are presumed validated at this stage */
FORCE_INLINE_TEMPLATE
void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt,
const short* normalizedCounter, unsigned maxSymbolValue,
const U32* baseValue, const U8* nbAdditionalBits,
unsigned tableLog, void* wksp, size_t wkspSize)
{
ZSTD_seqSymbol* const tableDecode = dt+1;
U32 const maxSV1 = maxSymbolValue + 1;
U32 const tableSize = 1 << tableLog;
U16* symbolNext = (U16*)wksp;
BYTE* spread = (BYTE*)(symbolNext + MaxSeq + 1);
U32 highThreshold = tableSize - 1;
/* Sanity Checks */
assert(maxSymbolValue <= MaxSeq);
assert(tableLog <= MaxFSELog);
assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE);
(void)wkspSize;
/* Init, lay down lowprob symbols */
{ ZSTD_seqSymbol_header DTableH;
DTableH.tableLog = tableLog;
DTableH.fastMode = 1;
{ S16 const largeLimit= (S16)(1 << (tableLog-1));
U32 s;
for (s=0; s<maxSV1; s++) {
if (normalizedCounter[s]==-1) {
tableDecode[highThreshold--].baseValue = s;
symbolNext[s] = 1;
} else {
if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
assert(normalizedCounter[s]>=0);
symbolNext[s] = (U16)normalizedCounter[s];
} } }
ZSTD_memcpy(dt, &DTableH, sizeof(DTableH));
}
/* Spread symbols */
assert(tableSize <= 512);
/* Specialized symbol spreading for the case when there are
* no low probability (-1 count) symbols. When compressing
* small blocks we avoid low probability symbols to hit this
* case, since header decoding speed matters more.
*/
if (highThreshold == tableSize - 1) {
size_t const tableMask = tableSize-1;
size_t const step = FSE_TABLESTEP(tableSize);
/* First lay down the symbols in order.
* We use a uint64_t to lay down 8 bytes at a time. This reduces branch
* misses since small blocks generally have small table logs, so nearly
* all symbols have counts <= 8. We ensure we have 8 bytes at the end of
* our buffer to handle the over-write.
*/
{
U64 const add = 0x0101010101010101ull;
size_t pos = 0;
U64 sv = 0;
U32 s;
for (s=0; s<maxSV1; ++s, sv += add) {
int i;
int const n = normalizedCounter[s];
MEM_write64(spread + pos, sv);
for (i = 8; i < n; i += 8) {
MEM_write64(spread + pos + i, sv);
}
assert(n>=0);
pos += (size_t)n;
}
}
/* Now we spread those positions across the table.
* The benefit of doing it in two stages is that we avoid the
* variable size inner loop, which caused lots of branch misses.
* Now we can run through all the positions without any branch misses.
* We unroll the loop twice, since that is what empirically worked best.
*/
{
size_t position = 0;
size_t s;
size_t const unroll = 2;
assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
for (s = 0; s < (size_t)tableSize; s += unroll) {
size_t u;
for (u = 0; u < unroll; ++u) {
size_t const uPosition = (position + (u * step)) & tableMask;
tableDecode[uPosition].baseValue = spread[s + u];
}
position = (position + (unroll * step)) & tableMask;
}
assert(position == 0);
}
} else {
U32 const tableMask = tableSize-1;
U32 const step = FSE_TABLESTEP(tableSize);
U32 s, position = 0;
for (s=0; s<maxSV1; s++) {
int i;
int const n = normalizedCounter[s];
for (i=0; i<n; i++) {
tableDecode[position].baseValue = s;
position = (position + step) & tableMask;
while (UNLIKELY(position > highThreshold)) position = (position + step) & tableMask; /* lowprob area */
} }
assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
}
/* Build Decoding table */
{
U32 u;
for (u=0; u<tableSize; u++) {
U32 const symbol = tableDecode[u].baseValue;
U32 const nextState = symbolNext[symbol]++;
tableDecode[u].nbBits = (BYTE) (tableLog - ZSTD_highbit32(nextState) );
tableDecode[u].nextState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
assert(nbAdditionalBits[symbol] < 255);
tableDecode[u].nbAdditionalBits = nbAdditionalBits[symbol];
tableDecode[u].baseValue = baseValue[symbol];
}
}
}
/* Avoids the FORCE_INLINE of the _body() function. */
static void ZSTD_buildFSETable_body_default(ZSTD_seqSymbol* dt,
const short* normalizedCounter, unsigned maxSymbolValue,
const U32* baseValue, const U8* nbAdditionalBits,
unsigned tableLog, void* wksp, size_t wkspSize)
{
ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue,
baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
}
#if DYNAMIC_BMI2
BMI2_TARGET_ATTRIBUTE static void ZSTD_buildFSETable_body_bmi2(ZSTD_seqSymbol* dt,
const short* normalizedCounter, unsigned maxSymbolValue,
const U32* baseValue, const U8* nbAdditionalBits,
unsigned tableLog, void* wksp, size_t wkspSize)
{
ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue,
baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
}
#endif
void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
const short* normalizedCounter, unsigned maxSymbolValue,
const U32* baseValue, const U8* nbAdditionalBits,
unsigned tableLog, void* wksp, size_t wkspSize, int bmi2)
{
#if DYNAMIC_BMI2
if (bmi2) {
ZSTD_buildFSETable_body_bmi2(dt, normalizedCounter, maxSymbolValue,
baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
return;
}
#endif
(void)bmi2;
ZSTD_buildFSETable_body_default(dt, normalizedCounter, maxSymbolValue,
baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
}
/*! ZSTD_buildSeqTable() :
* @return : nb bytes read from src,
* or an error code if it fails */
static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymbol** DTablePtr,
SymbolEncodingType_e type, unsigned max, U32 maxLog,
const void* src, size_t srcSize,
const U32* baseValue, const U8* nbAdditionalBits,
const ZSTD_seqSymbol* defaultTable, U32 flagRepeatTable,
int ddictIsCold, int nbSeq, U32* wksp, size_t wkspSize,
int bmi2)
{
switch(type)
{
case set_rle :
RETURN_ERROR_IF(!srcSize, srcSize_wrong, "");
RETURN_ERROR_IF((*(const BYTE*)src) > max, corruption_detected, "");
{ U32 const symbol = *(const BYTE*)src;
U32 const baseline = baseValue[symbol];
U8 const nbBits = nbAdditionalBits[symbol];
ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits);
}
*DTablePtr = DTableSpace;
return 1;
case set_basic :
*DTablePtr = defaultTable;
return 0;
case set_repeat:
RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, "");
/* prefetch FSE table if used */
if (ddictIsCold && (nbSeq > 24 /* heuristic */)) {
const void* const pStart = *DTablePtr;
size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog));
PREFETCH_AREA(pStart, pSize);
}
return 0;
case set_compressed :
{ unsigned tableLog;
S16 norm[MaxSeq+1];
size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, "");
RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, "");
ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize, bmi2);
*DTablePtr = DTableSpace;
return headerSize;
}
default :
assert(0);
RETURN_ERROR(GENERIC, "impossible");
}
} }
size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
const void* src, size_t srcSize) const void* src, size_t srcSize)
{ {
const BYTE* const istart = (const BYTE*)src; ZSTD_rustBlockCtx ctx = ZSTD_rust_block_context(dctx);
const BYTE* const iend = istart + srcSize; return ZSTD_rust_decodeSeqHeaders(&ctx, nbSeqPtr, src, srcSize);
const BYTE* ip = istart;
int nbSeq;
DEBUGLOG(5, "ZSTD_decodeSeqHeaders");
/* check */
RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong, "");
/* SeqHead */
nbSeq = *ip++;
if (nbSeq > 0x7F) {
if (nbSeq == 0xFF) {
RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, "");
nbSeq = MEM_readLE16(ip) + LONGNBSEQ;
ip+=2;
} else {
RETURN_ERROR_IF(ip >= iend, srcSize_wrong, "");
nbSeq = ((nbSeq-0x80)<<8) + *ip++;
}
}
*nbSeqPtr = nbSeq;
if (nbSeq == 0) {
/* No sequence : section ends immediately */
RETURN_ERROR_IF(ip != iend, corruption_detected,
"extraneous data present in the Sequences section");
return (size_t)(ip - istart);
}
/* FSE table descriptors */
RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */
RETURN_ERROR_IF(*ip & 3, corruption_detected, ""); /* The last field, Reserved, must be all-zeroes. */
{ SymbolEncodingType_e const LLtype = (SymbolEncodingType_e)(*ip >> 6);
SymbolEncodingType_e const OFtype = (SymbolEncodingType_e)((*ip >> 4) & 3);
SymbolEncodingType_e const MLtype = (SymbolEncodingType_e)((*ip >> 2) & 3);
ip++;
/* Build DTables */
{ size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr,
LLtype, MaxLL, LLFSELog,
ip, iend-ip,
LL_base, LL_bits,
LL_defaultDTable, dctx->fseEntropy,
dctx->ddictIsCold, nbSeq,
dctx->workspace, sizeof(dctx->workspace),
ZSTD_DCtx_get_bmi2(dctx));
RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed");
ip += llhSize;
}
{ size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr,
OFtype, MaxOff, OffFSELog,
ip, iend-ip,
OF_base, OF_bits,
OF_defaultDTable, dctx->fseEntropy,
dctx->ddictIsCold, nbSeq,
dctx->workspace, sizeof(dctx->workspace),
ZSTD_DCtx_get_bmi2(dctx));
RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed");
ip += ofhSize;
}
{ size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr,
MLtype, MaxML, MLFSELog,
ip, iend-ip,
ML_base, ML_bits,
ML_defaultDTable, dctx->fseEntropy,
dctx->ddictIsCold, nbSeq,
dctx->workspace, sizeof(dctx->workspace),
ZSTD_DCtx_get_bmi2(dctx));
RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed");
ip += mlhSize;
}
}
return ip-istart;
} }
size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
typedef struct { void* dst, size_t dstCapacity,
size_t litLength; const void* src, size_t srcSize,
size_t matchLength; const streaming_operation streaming)
size_t offset;
} seq_t;
typedef struct {
size_t state;
const ZSTD_seqSymbol* table;
} ZSTD_fseState;
typedef struct {
BIT_DStream_t DStream;
ZSTD_fseState stateLL;
ZSTD_fseState stateOffb;
ZSTD_fseState stateML;
size_t prevOffset[ZSTD_REP_NUM];
} seqState_t;
/*! ZSTD_overlapCopy8() :
* Copies 8 bytes from ip to op and updates op and ip where ip <= op.
* If the offset is < 8 then the offset is spread to at least 8 bytes.
*
* Precondition: *ip <= *op
* Postcondition: *op - *op >= 8
*/
HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) {
assert(*ip <= *op);
if (offset < 8) {
/* close range match, overlap */
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
int const sub2 = dec64table[offset];
(*op)[0] = (*ip)[0];
(*op)[1] = (*ip)[1];
(*op)[2] = (*ip)[2];
(*op)[3] = (*ip)[3];
*ip += dec32table[offset];
ZSTD_copy4(*op+4, *ip);
*ip -= sub2;
} else {
ZSTD_copy8(*op, *ip);
}
*ip += 8;
*op += 8;
assert(*op - *ip >= 8);
}
/*! ZSTD_safecopy() :
* Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer
* and write up to 16 bytes past oend_w (op >= oend_w is allowed).
* This function is only called in the uncommon case where the sequence is near the end of the block. It
* should be fast for a single long sequence, but can be slow for several short sequences.
*
* @param ovtype controls the overlap detection
* - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.
* - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart.
* The src buffer must be before the dst buffer.
*/
static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) {
ptrdiff_t const diff = op - ip;
BYTE* const oend = op + length;
assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) ||
(ovtype == ZSTD_overlap_src_before_dst && diff >= 0));
if (length < 8) {
/* Handle short lengths. */
while (op < oend) *op++ = *ip++;
return;
}
if (ovtype == ZSTD_overlap_src_before_dst) {
/* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */
assert(length >= 8);
ZSTD_overlapCopy8(&op, &ip, diff);
length -= 8;
assert(op - ip >= 8);
assert(op <= oend);
}
if (oend <= oend_w) {
/* No risk of overwrite. */
ZSTD_wildcopy(op, ip, length, ovtype);
return;
}
if (op <= oend_w) {
/* Wildcopy until we get close to the end. */
assert(oend > oend_w);
ZSTD_wildcopy(op, ip, oend_w - op, ovtype);
ip += oend_w - op;
op += oend_w - op;
}
/* Handle the leftovers. */
while (op < oend) *op++ = *ip++;
}
/* ZSTD_safecopyDstBeforeSrc():
* This version allows overlap with dst before src, or handles the non-overlap case with dst after src
* Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */
static void ZSTD_safecopyDstBeforeSrc(BYTE* op, const BYTE* ip, ptrdiff_t length) {
ptrdiff_t const diff = op - ip;
BYTE* const oend = op + length;
if (length < 8 || diff > -8) {
/* Handle short lengths, close overlaps, and dst not before src. */
while (op < oend) *op++ = *ip++;
return;
}
if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) {
ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap);
ip += oend - WILDCOPY_OVERLENGTH - op;
op += oend - WILDCOPY_OVERLENGTH - op;
}
/* Handle the leftovers. */
while (op < oend) *op++ = *ip++;
}
/* ZSTD_execSequenceEnd():
* This version handles cases that are near the end of the output buffer. It requires
* more careful checks to make sure there is no overflow. By separating out these hard
* and unlikely cases, we can speed up the common cases.
*
* NOTE: This function needs to be fast for a single long sequence, but doesn't need
* to be optimized for many small sequences, since those fall into ZSTD_execSequence().
*/
FORCE_NOINLINE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_execSequenceEnd(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
{ {
BYTE* const oLitEnd = op + sequence.litLength; ZSTD_rustBlockCtx ctx = ZSTD_rust_block_context(dctx);
size_t const sequenceLength = sequence.litLength + sequence.matchLength; return ZSTD_rust_decompressBlock_internal(&ctx, dst, dstCapacity, src, srcSize,
const BYTE* const iLitEnd = *litPtr + sequence.litLength; (int)streaming);
const BYTE* match = oLitEnd - sequence.offset;
BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;
/* bounds checks : careful of address space overflow in 32-bit mode */
RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer");
RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer");
assert(op < op + sequenceLength);
assert(oLitEnd < op + sequenceLength);
/* copy literals */
ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap);
op = oLitEnd;
*litPtr = iLitEnd;
/* copy Match */
if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
/* offset beyond prefix */
RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, "");
match = dictEnd - (prefixStart - match);
if (match + sequence.matchLength <= dictEnd) {
ZSTD_memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength;
}
/* span extDict & currentPrefixSegment */
{ size_t const length1 = dictEnd - match;
ZSTD_memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = prefixStart;
}
}
ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst);
return sequenceLength;
} }
/* ZSTD_execSequenceEndSplitLitBuffer():
* This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case.
*/
FORCE_NOINLINE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op,
BYTE* const oend, const BYTE* const oend_w, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
const BYTE* match = oLitEnd - sequence.offset;
/* bounds checks : careful of address space overflow in 32-bit mode */
RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer");
RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer");
assert(op < op + sequenceLength);
assert(oLitEnd < op + sequenceLength);
/* copy literals */
RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer");
ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength);
op = oLitEnd;
*litPtr = iLitEnd;
/* copy Match */
if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
/* offset beyond prefix */
RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, "");
match = dictEnd - (prefixStart - match);
if (match + sequence.matchLength <= dictEnd) {
ZSTD_memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength;
}
/* span extDict & currentPrefixSegment */
{ size_t const length1 = dictEnd - match;
ZSTD_memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = prefixStart;
}
}
ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst);
return sequenceLength;
}
HINT_INLINE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
const BYTE* match = oLitEnd - sequence.offset;
assert(op != NULL /* Precondition */);
assert(oend_w < oend /* No underflow */);
#if defined(__aarch64__)
/* prefetch sequence starting from match that will be used for copy later */
PREFETCH_L1(match);
#endif
/* Handle edge cases in a slow path:
* - Read beyond end of literals
* - Match end is within WILDCOPY_OVERLIMIT of oend
* - 32-bit mode and the match length overflows
*/
if (UNLIKELY(
iLitEnd > litLimit ||
oMatchEnd > oend_w ||
(MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH)))
return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
/* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */
assert(op <= oLitEnd /* No overflow */);
assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */);
assert(oMatchEnd <= oend /* No underflow */);
assert(iLitEnd <= litLimit /* Literal length is in bounds */);
assert(oLitEnd <= oend_w /* Can wildcopy literals */);
assert(oMatchEnd <= oend_w /* Can wildcopy matches */);
/* Copy Literals:
* Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.
* We likely don't need the full 32-byte wildcopy.
*/
assert(WILDCOPY_OVERLENGTH >= 16);
ZSTD_copy16(op, (*litPtr));
if (UNLIKELY(sequence.litLength > 16)) {
ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap);
}
op = oLitEnd;
*litPtr = iLitEnd; /* update for next sequence */
/* Copy Match */
if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
/* offset beyond prefix -> go into extDict */
RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, "");
match = dictEnd + (match - prefixStart);
if (match + sequence.matchLength <= dictEnd) {
ZSTD_memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength;
}
/* span extDict & currentPrefixSegment */
{ size_t const length1 = dictEnd - match;
ZSTD_memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = prefixStart;
}
}
/* Match within prefix of 1 or more bytes */
assert(op <= oMatchEnd);
assert(oMatchEnd <= oend_w);
assert(match >= prefixStart);
assert(sequence.matchLength >= 1);
/* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy
* without overlap checking.
*/
if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) {
/* We bet on a full wildcopy for matches, since we expect matches to be
* longer than literals (in general). In silesia, ~10% of matches are longer
* than 16 bytes.
*/
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
return sequenceLength;
}
assert(sequence.offset < WILDCOPY_VECLEN);
/* Copy 8 bytes and spread the offset to be >= 8. */
ZSTD_overlapCopy8(&op, &match, sequence.offset);
/* If the match length is > 8 bytes, then continue with the wildcopy. */
if (sequence.matchLength > 8) {
assert(op < oMatchEnd);
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst);
}
return sequenceLength;
}
HINT_INLINE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op,
BYTE* const oend, const BYTE* const oend_w, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
const BYTE* match = oLitEnd - sequence.offset;
assert(op != NULL /* Precondition */);
assert(oend_w < oend /* No underflow */);
/* Handle edge cases in a slow path:
* - Read beyond end of literals
* - Match end is within WILDCOPY_OVERLIMIT of oend
* - 32-bit mode and the match length overflows
*/
if (UNLIKELY(
iLitEnd > litLimit ||
oMatchEnd > oend_w ||
(MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH)))
return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
/* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */
assert(op <= oLitEnd /* No overflow */);
assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */);
assert(oMatchEnd <= oend /* No underflow */);
assert(iLitEnd <= litLimit /* Literal length is in bounds */);
assert(oLitEnd <= oend_w /* Can wildcopy literals */);
assert(oMatchEnd <= oend_w /* Can wildcopy matches */);
/* Copy Literals:
* Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.
* We likely don't need the full 32-byte wildcopy.
*/
assert(WILDCOPY_OVERLENGTH >= 16);
ZSTD_copy16(op, (*litPtr));
if (UNLIKELY(sequence.litLength > 16)) {
ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap);
}
op = oLitEnd;
*litPtr = iLitEnd; /* update for next sequence */
/* Copy Match */
if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
/* offset beyond prefix -> go into extDict */
RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, "");
match = dictEnd + (match - prefixStart);
if (match + sequence.matchLength <= dictEnd) {
ZSTD_memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength;
}
/* span extDict & currentPrefixSegment */
{ size_t const length1 = dictEnd - match;
ZSTD_memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = prefixStart;
} }
/* Match within prefix of 1 or more bytes */
assert(op <= oMatchEnd);
assert(oMatchEnd <= oend_w);
assert(match >= prefixStart);
assert(sequence.matchLength >= 1);
/* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy
* without overlap checking.
*/
if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) {
/* We bet on a full wildcopy for matches, since we expect matches to be
* longer than literals (in general). In silesia, ~10% of matches are longer
* than 16 bytes.
*/
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
return sequenceLength;
}
assert(sequence.offset < WILDCOPY_VECLEN);
/* Copy 8 bytes and spread the offset to be >= 8. */
ZSTD_overlapCopy8(&op, &match, sequence.offset);
/* If the match length is > 8 bytes, then continue with the wildcopy. */
if (sequence.matchLength > 8) {
assert(op < oMatchEnd);
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst);
}
return sequenceLength;
}
static void
ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt)
{
const void* ptr = dt;
const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr;
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits",
(U32)DStatePtr->state, DTableH->tableLog);
BIT_reloadDStream(bitD);
DStatePtr->table = dt + 1;
}
FORCE_INLINE_TEMPLATE void
ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits)
{
size_t const lowBits = BIT_readBits(bitD, nbBits);
DStatePtr->state = nextState + lowBits;
}
/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum
* offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32
* bits before reloading. This value is the maximum number of bytes we read
* after reloading when we are decoding long offsets.
*/
#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \
(ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \
? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \
: 0)
typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e;
/**
* ZSTD_decodeSequence():
* @p longOffsets : tells the decoder to reload more bit while decoding large offsets
* only used in 32-bit mode
* @return : Sequence (litL + matchL + offset)
*/
FORCE_INLINE_TEMPLATE seq_t
ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const int isLastSeq)
{
seq_t seq;
/*
* ZSTD_seqSymbol is a 64 bits wide structure.
* It can be loaded in one operation
* and its fields extracted by simply shifting or bit-extracting on aarch64.
* GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh
* operations that cause performance drop. This can be avoided by using this
* ZSTD_memcpy hack.
*/
#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__))
ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS;
ZSTD_seqSymbol* const llDInfo = &llDInfoS;
ZSTD_seqSymbol* const mlDInfo = &mlDInfoS;
ZSTD_seqSymbol* const ofDInfo = &ofDInfoS;
ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol));
ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol));
ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol));
#else
const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state;
const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state;
const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state;
#endif
seq.matchLength = mlDInfo->baseValue;
seq.litLength = llDInfo->baseValue;
{ U32 const ofBase = ofDInfo->baseValue;
BYTE const llBits = llDInfo->nbAdditionalBits;
BYTE const mlBits = mlDInfo->nbAdditionalBits;
BYTE const ofBits = ofDInfo->nbAdditionalBits;
BYTE const totalBits = llBits+mlBits+ofBits;
U16 const llNext = llDInfo->nextState;
U16 const mlNext = mlDInfo->nextState;
U16 const ofNext = ofDInfo->nextState;
U32 const llnbBits = llDInfo->nbBits;
U32 const mlnbBits = mlDInfo->nbBits;
U32 const ofnbBits = ofDInfo->nbBits;
assert(llBits <= MaxLLBits);
assert(mlBits <= MaxMLBits);
assert(ofBits <= MaxOff);
/*
* As gcc has better branch and block analyzers, sometimes it is only
* valuable to mark likeliness for clang, it gives around 3-4% of
* performance.
*/
/* sequence */
{ size_t offset;
if (ofBits > 1) {
ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32);
ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits);
if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) {
/* Always read extra bits, this keeps the logic simple,
* avoids branches, and avoids accidentally reading 0 bits.
*/
U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32;
offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
BIT_reloadDStream(&seqState->DStream);
offset += BIT_readBitsFast(&seqState->DStream, extraBits);
} else {
offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
}
seqState->prevOffset[2] = seqState->prevOffset[1];
seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[0] = offset;
} else {
U32 const ll0 = (llDInfo->baseValue == 0);
if (LIKELY((ofBits == 0))) {
offset = seqState->prevOffset[ll0];
seqState->prevOffset[1] = seqState->prevOffset[!ll0];
seqState->prevOffset[0] = offset;
} else {
offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1);
{ size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
temp -= !temp; /* 0 is not valid: input corrupted => force offset to -1 => corruption detected at execSequence */
if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[0] = offset = temp;
} } }
seq.offset = offset;
}
if (mlBits > 0)
seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/);
if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32))
BIT_reloadDStream(&seqState->DStream);
if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog)))
BIT_reloadDStream(&seqState->DStream);
/* Ensure there are enough bits to read the rest of data in 64-bit mode. */
ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64);
if (llBits > 0)
seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/);
if (MEM_32bits())
BIT_reloadDStream(&seqState->DStream);
DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u",
(U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
if (!isLastSeq) {
/* don't update FSE state for last Sequence */
ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */
ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */
if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */
BIT_reloadDStream(&seqState->DStream);
}
}
return seq;
}
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
#if DEBUGLEVEL >= 1
static int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd)
{
size_t const windowSize = dctx->fParams.windowSize;
/* No dictionary used. */
if (dctx->dictContentEndForFuzzing == NULL) return 0;
/* Dictionary is our prefix. */
if (prefixStart == dctx->dictContentBeginForFuzzing) return 1;
/* Dictionary is not our ext-dict. */
if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0;
/* Dictionary is not within our window size. */
if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0;
/* Dictionary is active. */
return 1;
}
#endif
static void ZSTD_assertValidSequence(
ZSTD_DCtx const* dctx,
BYTE const* op, BYTE const* oend,
seq_t const seq,
BYTE const* prefixStart, BYTE const* virtualStart)
{
#if DEBUGLEVEL >= 1
if (dctx->isFrameDecompression) {
size_t const windowSize = dctx->fParams.windowSize;
size_t const sequenceSize = seq.litLength + seq.matchLength;
BYTE const* const oLitEnd = op + seq.litLength;
DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u",
(U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
assert(op <= oend);
assert((size_t)(oend - op) >= sequenceSize);
assert(sequenceSize <= ZSTD_blockSizeMax(dctx));
if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) {
size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing);
/* Offset must be within the dictionary. */
assert(seq.offset <= (size_t)(oLitEnd - virtualStart));
assert(seq.offset <= windowSize + dictSize);
} else {
/* Offset must be within our window. */
assert(seq.offset <= windowSize);
}
}
#else
(void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart;
#endif
}
#endif
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
FORCE_INLINE_TEMPLATE size_t
DONT_VECTORIZE
ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
const BYTE* ip = (const BYTE*)seqStart;
const BYTE* const iend = ip + seqSize;
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, maxDstSize);
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
const BYTE* litBufferEnd = dctx->litBufferEnd;
const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
const BYTE* const vBase = (const BYTE*) (dctx->virtualStart);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer (%i seqs)", nbSeq);
/* Literals are split between internal buffer & output buffer */
if (nbSeq) {
seqState_t seqState;
dctx->fseEntropy = 1;
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
RETURN_ERROR_IF(
ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
corruption_detected, "");
ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
assert(dst != NULL);
ZSTD_STATIC_ASSERT(
BIT_DStream_unfinished < BIT_DStream_completed &&
BIT_DStream_endOfBuffer < BIT_DStream_completed &&
BIT_DStream_completed < BIT_DStream_overflow);
/* decompress without overrunning litPtr begins */
{ seq_t sequence = {0,0,0}; /* some static analyzer believe that @sequence is not initialized (it necessarily is, since for(;;) loop as at least one iteration) */
/* Align the decompression loop to 32 + 16 bytes.
*
* zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression
* speed swings based on the alignment of the decompression loop. This
* performance swing is caused by parts of the decompression loop falling
* out of the DSB. The entire decompression loop should fit in the DSB,
* when it can't we get much worse performance. You can measure if you've
* hit the good case or the bad case with this perf command for some
* compressed file test.zst:
*
* perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \
* -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst
*
* If you see most cycles served out of the MITE you've hit the bad case.
* If you see most cycles served out of the DSB you've hit the good case.
* If it is pretty even then you may be in an okay case.
*
* This issue has been reproduced on the following CPUs:
* - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9
* Use Instruments->Counters to get DSB/MITE cycles.
* I never got performance swings, but I was able to
* go from the good case of mostly DSB to half of the
* cycles served from MITE.
* - Coffeelake: Intel i9-9900k
* - Coffeelake: Intel i7-9700k
*
* I haven't been able to reproduce the instability or DSB misses on any
* of the following CPUS:
* - Haswell
* - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH
* - Skylake
*
* Alignment is done for each of the three major decompression loops:
* - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer
* - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer
* - ZSTD_decompressSequences_body
* Alignment choices are made to minimize large swings on bad cases and influence on performance
* from changes external to this code, rather than to overoptimize on the current commit.
*
* If you are seeing performance stability this script can help test.
* It tests on 4 commits in zstd where I saw performance change.
*
* https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4
*/
#if defined(__GNUC__) && defined(__x86_64__)
__asm__(".p2align 6");
# if __GNUC__ >= 7
/* good for gcc-7, gcc-9, and gcc-11 */
__asm__("nop");
__asm__(".p2align 5");
__asm__("nop");
__asm__(".p2align 4");
# if __GNUC__ == 8 || __GNUC__ == 10
/* good for gcc-8 and gcc-10 */
__asm__("nop");
__asm__(".p2align 3");
# endif
# endif
#endif
/* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */
for ( ; nbSeq; nbSeq--) {
sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1);
if (litPtr + sequence.litLength > dctx->litBufferEnd) break;
{ size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
#endif
if (UNLIKELY(ZSTD_isError(oneSeqSize)))
return oneSeqSize;
DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
op += oneSeqSize;
} }
DEBUGLOG(6, "reached: (litPtr + sequence.litLength > dctx->litBufferEnd)");
/* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */
if (nbSeq > 0) {
const size_t leftoverLit = dctx->litBufferEnd - litPtr;
DEBUGLOG(6, "There are %i sequences left, and %zu/%zu literals left in buffer", nbSeq, leftoverLit, sequence.litLength);
if (leftoverLit) {
RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
sequence.litLength -= leftoverLit;
op += leftoverLit;
}
litPtr = dctx->litExtraBuffer;
litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
dctx->litBufferLocation = ZSTD_not_in_dst;
{ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
#endif
if (UNLIKELY(ZSTD_isError(oneSeqSize)))
return oneSeqSize;
DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
op += oneSeqSize;
}
nbSeq--;
}
}
if (nbSeq > 0) {
/* there is remaining lit from extra buffer */
#if defined(__GNUC__) && defined(__x86_64__)
__asm__(".p2align 6");
__asm__("nop");
# if __GNUC__ != 7
/* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */
__asm__(".p2align 4");
__asm__("nop");
__asm__(".p2align 3");
# elif __GNUC__ >= 11
__asm__(".p2align 3");
# else
__asm__(".p2align 5");
__asm__("nop");
__asm__(".p2align 3");
# endif
#endif
for ( ; nbSeq ; nbSeq--) {
seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1);
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
#endif
if (UNLIKELY(ZSTD_isError(oneSeqSize)))
return oneSeqSize;
DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
op += oneSeqSize;
}
}
/* check if reached exact end */
DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq);
RETURN_ERROR_IF(nbSeq, corruption_detected, "");
DEBUGLOG(5, "bitStream : start=%p, ptr=%p, bitsConsumed=%u", seqState.DStream.start, seqState.DStream.ptr, seqState.DStream.bitsConsumed);
RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, "");
/* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
}
/* last literal segment */
if (dctx->litBufferLocation == ZSTD_split) {
/* split hasn't been reached yet, first get dst then copy litExtraBuffer */
size_t const lastLLSize = (size_t)(litBufferEnd - litPtr);
DEBUGLOG(6, "copy last literals from segment : %u", (U32)lastLLSize);
RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
if (op != NULL) {
ZSTD_memmove(op, litPtr, lastLLSize);
op += lastLLSize;
}
litPtr = dctx->litExtraBuffer;
litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
dctx->litBufferLocation = ZSTD_not_in_dst;
}
/* copy last literals from internal buffer */
{ size_t const lastLLSize = (size_t)(litBufferEnd - litPtr);
DEBUGLOG(6, "copy last literals from internal buffer : %u", (U32)lastLLSize);
RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
if (op != NULL) {
ZSTD_memcpy(op, litPtr, lastLLSize);
op += lastLLSize;
} }
DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart));
return (size_t)(op - ostart);
}
FORCE_INLINE_TEMPLATE size_t
DONT_VECTORIZE
ZSTD_decompressSequences_body(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
const BYTE* ip = (const BYTE*)seqStart;
const BYTE* const iend = ip + seqSize;
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ZSTD_maybeNullPtrAdd(ostart, maxDstSize) : dctx->litBuffer;
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
const BYTE* const litEnd = litPtr + dctx->litSize;
const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart);
const BYTE* const vBase = (const BYTE*)(dctx->virtualStart);
const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd);
DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq);
/* Regen sequences */
if (nbSeq) {
seqState_t seqState;
dctx->fseEntropy = 1;
{ U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
RETURN_ERROR_IF(
ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)),
corruption_detected, "");
ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
assert(dst != NULL);
#if defined(__GNUC__) && defined(__x86_64__)
__asm__(".p2align 6");
__asm__("nop");
# if __GNUC__ >= 7
__asm__(".p2align 5");
__asm__("nop");
__asm__(".p2align 3");
# else
__asm__(".p2align 4");
__asm__("nop");
__asm__(".p2align 3");
# endif
#endif
for ( ; nbSeq ; nbSeq--) {
seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1);
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
#endif
if (UNLIKELY(ZSTD_isError(oneSeqSize)))
return oneSeqSize;
DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
op += oneSeqSize;
}
/* check if reached exact end */
assert(nbSeq == 0);
RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, "");
/* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
}
/* last literal segment */
{ size_t const lastLLSize = (size_t)(litEnd - litPtr);
DEBUGLOG(6, "copy last literals : %u", (U32)lastLLSize);
RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
if (op != NULL) {
ZSTD_memcpy(op, litPtr, lastLLSize);
op += lastLLSize;
} }
DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart));
return (size_t)(op - ostart);
}
static size_t
ZSTD_decompressSequences_default(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
static size_t
ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
FORCE_INLINE_TEMPLATE
size_t ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence,
const BYTE* const prefixStart, const BYTE* const dictEnd)
{
prefetchPos += sequence.litLength;
{ const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart;
/* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted.
* No consequence though : memory address is only used for prefetching, not for dereferencing */
const BYTE* const match = ZSTD_wrappedPtrSub(ZSTD_wrappedPtrAdd(matchBase, prefetchPos), sequence.offset);
PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */
}
return prefetchPos + sequence.matchLength;
}
/* This decoding function employs prefetching
* to reduce latency impact of cache misses.
* It's generally employed when block contains a significant portion of long-distance matches
* or when coupled with a "cold" dictionary */
FORCE_INLINE_TEMPLATE size_t
ZSTD_decompressSequencesLong_body(
ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
const BYTE* ip = (const BYTE*)seqStart;
const BYTE* const iend = ip + seqSize;
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ZSTD_maybeNullPtrAdd(ostart, maxDstSize);
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
const BYTE* litBufferEnd = dctx->litBufferEnd;
const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
/* Regen sequences */
if (nbSeq) {
#define STORED_SEQS 8
#define STORED_SEQS_MASK (STORED_SEQS-1)
#define ADVANCED_SEQS STORED_SEQS
seq_t sequences[STORED_SEQS];
int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS);
seqState_t seqState;
int seqNb;
size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */
dctx->fseEntropy = 1;
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
assert(dst != NULL);
assert(iend >= ip);
RETURN_ERROR_IF(
ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
corruption_detected, "");
ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
/* prepare in advance */
for (seqNb=0; seqNb<seqAdvance; seqNb++) {
seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, seqNb == nbSeq-1);
prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
sequences[seqNb] = sequence;
}
/* decompress without stomping litBuffer */
for (; seqNb < nbSeq; seqNb++) {
seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset, seqNb == nbSeq-1);
if (dctx->litBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd) {
/* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */
const size_t leftoverLit = dctx->litBufferEnd - litPtr;
if (leftoverLit)
{
RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit;
op += leftoverLit;
}
litPtr = dctx->litExtraBuffer;
litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
dctx->litBufferLocation = ZSTD_not_in_dst;
{ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart);
#endif
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
sequences[seqNb & STORED_SEQS_MASK] = sequence;
op += oneSeqSize;
} }
else
{
/* lit buffer is either wholly contained in first or second split, or not split at all*/
size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) :
ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart);
#endif
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
sequences[seqNb & STORED_SEQS_MASK] = sequence;
op += oneSeqSize;
}
}
RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, "");
/* finish queue */
seqNb -= seqAdvance;
for ( ; seqNb<nbSeq ; seqNb++) {
seq_t *sequence = &(sequences[seqNb&STORED_SEQS_MASK]);
if (dctx->litBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd) {
const size_t leftoverLit = dctx->litBufferEnd - litPtr;
if (leftoverLit) {
RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
sequence->litLength -= leftoverLit;
op += leftoverLit;
}
litPtr = dctx->litExtraBuffer;
litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
dctx->litBufferLocation = ZSTD_not_in_dst;
{ size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart);
#endif
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}
}
else
{
size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) :
ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart);
#endif
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}
}
/* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
}
/* last literal segment */
if (dctx->litBufferLocation == ZSTD_split) { /* first deplete literal buffer in dst, then copy litExtraBuffer */
size_t const lastLLSize = litBufferEnd - litPtr;
RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
if (op != NULL) {
ZSTD_memmove(op, litPtr, lastLLSize);
op += lastLLSize;
}
litPtr = dctx->litExtraBuffer;
litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
}
{ size_t const lastLLSize = litBufferEnd - litPtr;
RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
if (op != NULL) {
ZSTD_memmove(op, litPtr, lastLLSize);
op += lastLLSize;
}
}
return (size_t)(op - ostart);
}
static size_t
ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
#if DYNAMIC_BMI2
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
static BMI2_TARGET_ATTRIBUTE size_t
DONT_VECTORIZE
ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
static BMI2_TARGET_ATTRIBUTE size_t
DONT_VECTORIZE
ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
static BMI2_TARGET_ATTRIBUTE size_t
ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
#endif /* DYNAMIC_BMI2 */
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
static size_t
ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
DEBUGLOG(5, "ZSTD_decompressSequences");
#if DYNAMIC_BMI2
if (ZSTD_DCtx_get_bmi2(dctx)) {
return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif
return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
static size_t
ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer");
#if DYNAMIC_BMI2
if (ZSTD_DCtx_get_bmi2(dctx)) {
return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif
return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
/* ZSTD_decompressSequencesLong() :
* decompression function triggered when a minimum share of offsets is considered "long",
* aka out of cache.
* note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance".
* This function will try to mitigate main memory latency through the use of prefetching */
static size_t
ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
DEBUGLOG(5, "ZSTD_decompressSequencesLong");
#if DYNAMIC_BMI2
if (ZSTD_DCtx_get_bmi2(dctx)) {
return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif
return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
/**
* @returns The total size of the history referenceable by zstd, including
* both the prefix and the extDict. At @p op any offset larger than this
* is invalid.
*/
static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart)
{
return (size_t)(op - virtualStart);
}
typedef struct {
unsigned longOffsetShare;
unsigned maxNbAdditionalBits;
} ZSTD_OffsetInfo;
/* ZSTD_getOffsetInfo() :
* condition : offTable must be valid
* @return : "share" of long offsets (arbitrarily defined as > (1<<23))
* compared to maximum possible of (1<<OffFSELog),
* as well as the maximum number additional bits required.
*/
static ZSTD_OffsetInfo
ZSTD_getOffsetInfo(const ZSTD_seqSymbol* offTable, int nbSeq)
{
ZSTD_OffsetInfo info = {0, 0};
/* If nbSeq == 0, then the offTable is uninitialized, but we have
* no sequences, so both values should be 0.
*/
if (nbSeq != 0) {
const void* ptr = offTable;
U32 const tableLog = ((const ZSTD_seqSymbol_header*)ptr)[0].tableLog;
const ZSTD_seqSymbol* table = offTable + 1;
U32 const max = 1 << tableLog;
U32 u;
DEBUGLOG(5, "ZSTD_getLongOffsetsShare: (tableLog=%u)", tableLog);
assert(max <= (1 << OffFSELog)); /* max not too large */
for (u=0; u<max; u++) {
info.maxNbAdditionalBits = MAX(info.maxNbAdditionalBits, table[u].nbAdditionalBits);
if (table[u].nbAdditionalBits > 22) info.longOffsetShare += 1;
}
assert(tableLog <= OffFSELog);
info.longOffsetShare <<= (OffFSELog - tableLog); /* scale to OffFSELog */
}
return info;
}
/**
* @returns The maximum offset we can decode in one read of our bitstream, without
* reloading more bits in the middle of the offset bits read. Any offsets larger
* than this must use the long offset decoder.
*/
static size_t ZSTD_maxShortOffset(void)
{
if (MEM_64bits()) {
/* We can decode any offset without reloading bits.
* This might change if the max window size grows.
*/
ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
return (size_t)-1;
} else {
/* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1.
* This offBase would require STREAM_ACCUMULATOR_MIN extra bits.
* Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset.
*/
size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1;
size_t const maxOffset = maxOffbase - ZSTD_REP_NUM;
assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN);
return maxOffset;
}
}
size_t
ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const streaming_operation streaming)
{ /* blockType == blockCompressed */
const BYTE* ip = (const BYTE*)src;
DEBUGLOG(5, "ZSTD_decompressBlock_internal (cSize : %u)", (unsigned)srcSize);
/* Note : the wording of the specification
* allows compressed block to be sized exactly ZSTD_blockSizeMax(dctx).
* This generally does not happen, as it makes little sense,
* since an uncompressed block would feature same size and have no decompression cost.
* Also, note that decoder from reference libzstd before < v1.5.4
* would consider this edge case as an error.
* As a consequence, avoid generating compressed blocks of size ZSTD_blockSizeMax(dctx)
* for broader compatibility with the deployed ecosystem of zstd decoders */
RETURN_ERROR_IF(srcSize > ZSTD_blockSizeMax(dctx), srcSize_wrong, "");
/* Decode literals section */
{ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming);
DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize);
if (ZSTD_isError(litCSize)) return litCSize;
ip += litCSize;
srcSize -= litCSize;
}
/* Build Decoding Tables */
{
/* Compute the maximum block size, which must also work when !frame and fParams are unset.
* Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t.
*/
size_t const blockSizeMax = MIN(dstCapacity, ZSTD_blockSizeMax(dctx));
size_t const totalHistorySize = ZSTD_totalHistorySize(ZSTD_maybeNullPtrAdd((BYTE*)dst, blockSizeMax), (BYTE const*)dctx->virtualStart);
/* isLongOffset must be true if there are long offsets.
* Offsets are long if they are larger than ZSTD_maxShortOffset().
* We don't expect that to be the case in 64-bit mode.
*
* We check here to see if our history is large enough to allow long offsets.
* If it isn't, then we can't possible have (valid) long offsets. If the offset
* is invalid, then it is okay to read it incorrectly.
*
* If isLongOffsets is true, then we will later check our decoding table to see
* if it is even possible to generate long offsets.
*/
ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset()));
/* These macros control at build-time which decompressor implementation
* we use. If neither is defined, we do some inspection and dispatch at
* runtime.
*/
#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
!defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
int usePrefetchDecoder = dctx->ddictIsCold;
#else
/* Set to 1 to avoid computing offset info if we don't need to.
* Otherwise this value is ignored.
*/
int usePrefetchDecoder = 1;
#endif
int nbSeq;
size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize);
if (ZSTD_isError(seqHSize)) return seqHSize;
ip += seqHSize;
srcSize -= seqHSize;
RETURN_ERROR_IF((dst == NULL || dstCapacity == 0) && nbSeq > 0, dstSize_tooSmall, "NULL not handled");
RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (size_t)(1 << 20), dstSize_tooSmall,
"invalid dst");
/* If we could potentially have long offsets, or we might want to use the prefetch decoder,
* compute information about the share of long offsets, and the maximum nbAdditionalBits.
* NOTE: could probably use a larger nbSeq limit
*/
if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) {
ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq);
if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) {
/* If isLongOffset, but the maximum number of additional bits that we see in our table is small
* enough, then we know it is impossible to have too long an offset in this block, so we can
* use the regular offset decoder.
*/
isLongOffset = ZSTD_lo_isRegularOffset;
}
if (!usePrefetchDecoder) {
U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */
usePrefetchDecoder = (info.longOffsetShare >= minShare);
}
}
dctx->ddictIsCold = 0;
#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
!defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
if (usePrefetchDecoder) {
#else
(void)usePrefetchDecoder;
{
#endif
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset);
#endif
}
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
/* else */
if (dctx->litBufferLocation == ZSTD_split)
return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset);
else
return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset);
#endif
}
}
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize)
{ {
if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */ ZSTD_rustBlockCtx ctx = ZSTD_rust_block_context(dctx);
dctx->dictEnd = dctx->previousDstEnd; ZSTD_rust_checkContinuity(&ctx, dst, dstSize);
dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));
dctx->prefixStart = dst;
dctx->previousDstEnd = dst;
}
} }
size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize) const void* src, size_t srcSize)
{ {
size_t dSize; ZSTD_rustBlockCtx ctx = ZSTD_rust_block_context(dctx);
dctx->isFrameDecompression = 0; return ZSTD_rust_decompressBlock_deprecated(&ctx,
ZSTD_checkContinuity(dctx, dst, dstCapacity); dst, dstCapacity, src, srcSize);
dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, not_streaming);
FORWARD_IF_ERROR(dSize, "");
dctx->previousDstEnd = (char*)dst + dSize;
return dSize;
} }
/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated(). */
/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated() */
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize) const void* src, size_t srcSize)
{ {
return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize); return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize);
} }
rust/README.md
+7 -4
View File
@@ -40,11 +40,14 @@ zstd ABI:
- `pool` implements the bounded worker pool used by multithreaded compression. - `pool` implements the bounded worker pool used by multithreaded compression.
- Dictionary support - Dictionary support
- `zstd_ddict` owns, loads, copies, and references decode dictionaries. - `zstd_ddict` owns, loads, copies, and references decode dictionaries.
- Block decompression
- `zstd_decompress_block` decodes literal and sequence sections, maintains
FSE/Huffman repeat state, and executes compressed-block sequences.
The optimal block matcher, general decompression, dictionary-building, legacy, The optimal block matcher, high-level frame decompression, dictionary-building,
and CLI translation units are still C. They must move before the rewrite is legacy, and CLI translation units are still C. They must move before the
complete. Keeping that boundary explicit prevents a passing hybrid build from rewrite is complete. Keeping that boundary explicit prevents a passing hybrid
being mistaken for the final all-Rust result. build from being mistaken for the final all-Rust result.
## Compatibility boundary ## Compatibility boundary
rust/src/lib.rs
+2
View File
@@ -31,6 +31,8 @@ pub mod zstd_compress_sequences;
pub mod zstd_compress_superblock; pub mod zstd_compress_superblock;
#[cfg(feature = "decompression")] #[cfg(feature = "decompression")]
pub mod zstd_ddict; pub mod zstd_ddict;
#[cfg(feature = "decompression")]
pub mod zstd_decompress_block;
#[cfg(feature = "compression")] #[cfg(feature = "compression")]
pub mod zstd_double_fast; pub mod zstd_double_fast;
#[cfg(feature = "compression")] #[cfg(feature = "compression")]
rust/src/zstd_decompress_block.rs
+1700
View File
@@ -0,0 +1,1700 @@
#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::too_many_arguments)]
//! Compressed-block decoding.
//!
//! The C decoder context deliberately remains opaque. The companion C shim
//! extracts only the pointers and scalar values this translation needs into
//! [`ZSTD_rustBlockCtx`], so optional C context fields cannot silently change
//! this Rust module's ABI. Literal parsing, FSE table construction, sequence
//! decoding, and block execution live here.
use crate::bitstream::{
BIT_DStream_t, BIT_endOfDStream, BIT_initDStream, BIT_readBits, BIT_readBitsFast,
BIT_reloadDStream,
};
use crate::common::{
DEFAULT_MAX_OFF, LL_BITS, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, MAX_FSE_LOG, MAX_LL, MAX_ML,
MAX_OFF, MIN_CBLOCK_SIZE, MIN_LITERALS_FOR_4_STREAMS, MIN_SEQUENCES_SIZE, ML_BITS,
ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, ZSTD_REP_NUM,
};
use crate::entropy_common::FSE_readNCount;
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
#[cfg(not(feature = "huf-force-decompress-x2"))]
use crate::huf_decompress::HUF_decompress1X1_DCtx_wksp;
#[cfg(feature = "huf-force-decompress-x2")]
use crate::huf_decompress::HUF_decompress1X_DCtx_wksp;
use crate::huf_decompress::{
HUF_decompress1X_usingDTable, HUF_decompress4X_hufOnly_wksp, HUF_decompress4X_usingDTable,
};
use crate::mem::{MEM_32bits, MEM_64bits, MEM_readLE16, MEM_readLE24, U32};
use std::cmp::min;
use std::ffi::c_void;
use std::mem::MaybeUninit;
use std::os::raw::{c_int, c_short, c_uint};
use std::ptr;
use std::sync::OnceLock;
const ZSTD_BLOCKSIZE_MAX: usize = 128 << 10;
const ZSTD_BLOCK_HEADER_SIZE: usize = 3;
const WILDCOPY_OVERLENGTH: usize = 32;
const HUF_FLAGS_BMI2: c_int = 1 << 0;
const HUF_FLAGS_DISABLE_ASM: c_int = 1 << 4;
const SET_BASIC: c_int = 0;
const SET_RLE: c_int = 1;
const SET_COMPRESSED: c_int = 2;
const SET_REPEAT: c_int = 3;
const NOT_STREAMING: c_int = 0;
const ZSTD_NOT_IN_DST: c_int = 0;
const ZSTD_IN_DST: c_int = 1;
const ZSTD_SPLIT: c_int = 2;
const LONG_NB_SEQ: usize = 0x7f00;
const LL_FSE_LOG: u32 = 9;
const OFF_FSE_LOG: u32 = 8;
const ML_FSE_LOG: u32 = 9;
const ZSTD_HUFFDTABLE_CAPACITY_LOG: usize = 12;
const HUF_DTABLE_SIZE: usize = 1 + (1 << ZSTD_HUFFDTABLE_CAPACITY_LOG);
const ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32: usize = 157;
const LL_BASE: [u32; MAX_LL + 1] = [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 22, 24, 28, 32, 40, 48, 64,
0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000,
];
const OF_BASE: [u32; MAX_OFF + 1] = [
0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD,
0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD,
];
const OF_BITS: [u8; MAX_OFF + 1] = [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
26, 27, 28, 29, 30, 31,
];
const ML_BASE: [u32; MAX_ML + 1] = [
3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203,
0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003,
];
/// `ZSTD_seqSymbol` from `zstd_decompress_internal.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
pub struct ZSTD_seqSymbol {
next_state: u16,
nb_additional_bits: u8,
nb_bits: u8,
base_value: u32,
}
/// The first entry of a sequence table is overlaid as this C header.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
struct ZSTD_seqSymbol_header {
fast_mode: u32,
table_log: u32,
}
#[repr(C)]
struct ZSTD_entropyDTables_t {
ll_table: [ZSTD_seqSymbol; 1 + (1 << LL_FSE_LOG)],
of_table: [ZSTD_seqSymbol; 1 + (1 << OFF_FSE_LOG)],
ml_table: [ZSTD_seqSymbol; 1 + (1 << ML_FSE_LOG)],
huf_table: [u32; HUF_DTABLE_SIZE],
rep: [u32; ZSTD_REP_NUM],
workspace: [u32; ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32],
}
/// C-owned leaves of `ZSTD_DCtx_s` used by this translation unit.
///
/// Every pointer is produced by `zstd_decompress_block.c` under the active C
/// configuration. This avoids assuming offsets for optional context members
/// such as `DYNAMIC_BMI2`, fuzzing bounds, and tracing state.
#[repr(C)]
pub struct ZSTD_rustBlockCtx {
llt_ptr: *mut *const ZSTD_seqSymbol,
mlt_ptr: *mut *const ZSTD_seqSymbol,
oft_ptr: *mut *const ZSTD_seqSymbol,
huf_ptr: *mut *const u32,
entropy: *mut ZSTD_entropyDTables_t,
workspace: *mut u32,
workspace_size: usize,
previous_dst_end: *mut *const u8,
prefix_start: *mut *const u8,
virtual_start: *mut *const u8,
dict_end: *mut *const u8,
block_size_max: usize,
is_frame_decompression: *mut c_int,
lit_entropy: *mut u32,
fse_entropy: *mut u32,
bmi2: c_int,
ddict_is_cold: *mut c_int,
disable_huf_asm: c_int,
lit_ptr: *mut *const u8,
lit_size: *mut usize,
rle_size: *mut usize,
lit_buffer: *mut *mut u8,
lit_buffer_end: *mut *const u8,
lit_buffer_location: *mut c_int,
lit_extra_buffer: *mut u8,
lit_extra_buffer_size: usize,
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
pub struct ZSTD_rustSeq {
lit_length: usize,
match_length: usize,
offset: usize,
}
#[repr(C)]
struct block_properties_t {
block_type: c_int,
last_block: u32,
orig_size: u32,
}
#[derive(Clone, Copy)]
struct ZSTD_fseState {
state: usize,
table: *const ZSTD_seqSymbol,
}
struct seq_state_t {
dstream: BIT_DStream_t,
state_ll: ZSTD_fseState,
state_off: ZSTD_fseState,
state_ml: ZSTD_fseState,
prev_offset: [usize; ZSTD_REP_NUM],
}
#[inline]
unsafe fn entropy(ctx: *mut ZSTD_rustBlockCtx) -> *mut ZSTD_entropyDTables_t {
unsafe { (*ctx).entropy }
}
#[inline]
unsafe fn address_distance(end: *const u8, start: *const u8) -> usize {
(end as usize).wrapping_sub(start as usize)
}
#[inline]
unsafe fn ptr_add(ptr: *mut u8, amount: usize) -> *mut u8 {
if ptr.is_null() {
debug_assert_eq!(amount, 0);
ptr
} else {
unsafe { ptr.add(amount) }
}
}
#[inline]
unsafe fn const_ptr_add(ptr: *const u8, amount: usize) -> *const u8 {
if ptr.is_null() {
debug_assert_eq!(amount, 0);
ptr
} else {
unsafe { ptr.add(amount) }
}
}
#[inline]
unsafe fn seq_header(table: *const ZSTD_seqSymbol) -> ZSTD_seqSymbol_header {
unsafe { table.cast::<ZSTD_seqSymbol_header>().read_unaligned() }
}
#[inline]
unsafe fn set_seq_header(table: *mut ZSTD_seqSymbol, header: ZSTD_seqSymbol_header) {
unsafe {
table
.cast::<ZSTD_seqSymbol_header>()
.write_unaligned(header)
};
}
#[inline]
unsafe fn copy_bytes(dst: *mut u8, src: *const u8, len: usize) {
if len != 0 {
unsafe { ptr::copy(src, dst, len) };
}
}
#[inline]
unsafe fn fill_bytes(dst: *mut u8, value: u8, len: usize) {
if len != 0 {
unsafe { ptr::write_bytes(dst, value, len) };
}
}
/// Copy a match using forward byte semantics, which is required for repeated
/// short-offset matches (unlike `memmove`, which would not expand overlap).
#[inline]
unsafe fn copy_match(mut dst: *mut u8, mut src: *const u8, len: usize) {
for _ in 0..len {
unsafe { dst.write(src.read()) };
dst = unsafe { dst.add(1) };
src = unsafe { src.add(1) };
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_getcBlockSize(
src: *const c_void,
src_size: usize,
bp_ptr: *mut c_void,
) -> usize {
if src_size < ZSTD_BLOCK_HEADER_SIZE {
return ERROR(ZstdErrorCode::SrcSizeWrong);
}
let header = unsafe { MEM_readLE24(src) };
let bp = bp_ptr.cast::<block_properties_t>();
unsafe {
(*bp).last_block = header & 1;
(*bp).block_type = ((header >> 1) & 3) as c_int;
(*bp).orig_size = header >> 3;
}
if unsafe { (*bp).block_type } == 1 {
return 1;
}
if unsafe { (*bp).block_type } == 3 {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
(header >> 3) as usize
}
#[inline]
unsafe fn block_size_max(ctx: *const ZSTD_rustBlockCtx) -> usize {
let value = unsafe {
if *(*ctx).is_frame_decompression != 0 {
(*ctx).block_size_max
} else {
ZSTD_BLOCKSIZE_MAX
}
};
debug_assert!(value <= ZSTD_BLOCKSIZE_MAX);
value
}
unsafe fn allocate_literals_buffer(
ctx: *mut ZSTD_rustBlockCtx,
dst: *mut u8,
dst_capacity: usize,
lit_size: usize,
streaming: c_int,
expected_write_size: usize,
split_immediately: bool,
) {
let block_max = unsafe { block_size_max(ctx) };
debug_assert!(lit_size <= block_max);
debug_assert!(*unsafe { (*ctx).is_frame_decompression } != 0 || streaming == NOT_STREAMING);
if streaming == NOT_STREAMING
&& dst_capacity
> block_max
.saturating_add(WILDCOPY_OVERLENGTH)
.saturating_add(lit_size)
.saturating_add(WILDCOPY_OVERLENGTH)
{
let buffer = unsafe { ptr_add(dst, block_max + WILDCOPY_OVERLENGTH) };
unsafe {
*(*ctx).lit_buffer = buffer;
*(*ctx).lit_buffer_end = buffer.add(lit_size);
*(*ctx).lit_buffer_location = ZSTD_IN_DST;
}
} else if lit_size <= unsafe { (*ctx).lit_extra_buffer_size } {
let buffer = unsafe { (*ctx).lit_extra_buffer };
unsafe {
*(*ctx).lit_buffer = buffer;
*(*ctx).lit_buffer_end = buffer.add(lit_size);
*(*ctx).lit_buffer_location = ZSTD_NOT_IN_DST;
}
} else {
let extra = unsafe { (*ctx).lit_extra_buffer_size };
debug_assert!(block_max > extra);
let buffer = if split_immediately {
unsafe {
ptr_add(
dst,
expected_write_size - lit_size + extra - WILDCOPY_OVERLENGTH,
)
}
} else {
unsafe { ptr_add(dst, expected_write_size - lit_size) }
};
unsafe {
*(*ctx).lit_buffer = buffer;
*(*ctx).lit_buffer_end = if split_immediately {
buffer.add(lit_size - extra)
} else {
ptr_add(dst, expected_write_size).cast_const()
};
*(*ctx).lit_buffer_location = ZSTD_SPLIT;
}
}
}
unsafe fn decode_literals_block(
ctx: *mut ZSTD_rustBlockCtx,
src: *const u8,
src_size: usize,
dst: *mut u8,
dst_capacity: usize,
streaming: c_int,
) -> usize {
if src_size < MIN_CBLOCK_SIZE {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
let lit_type = unsafe { *src & 3 } as c_int;
let block_max = unsafe { block_size_max(ctx) };
match lit_type {
SET_REPEAT | SET_COMPRESSED => {
if lit_type == SET_REPEAT && unsafe { *(*ctx).lit_entropy } == 0 {
return ERROR(ZstdErrorCode::DictionaryCorrupted);
}
if src_size < 5 {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
let lhl_code = unsafe { (*src >> 2) & 3 };
let header = unsafe { crate::mem::MEM_readLE32(src.cast()) };
let (header_size, lit_size, lit_c_size, single_stream) = match lhl_code {
0 | 1 => (
3usize,
((header >> 4) & 0x3ff) as usize,
((header >> 14) & 0x3ff) as usize,
lhl_code == 0,
),
2 => (
4usize,
((header >> 4) & 0x3fff) as usize,
(header >> 18) as usize,
false,
),
_ => (
5usize,
((header >> 4) & 0x3ffff) as usize,
((header >> 22) as usize).wrapping_add((unsafe { *src.add(4) } as usize) << 10),
false,
),
};
if (lit_size != 0 && dst.is_null())
|| lit_size > block_max
|| (!single_stream && lit_size < MIN_LITERALS_FOR_4_STREAMS)
|| header_size.checked_add(lit_c_size).is_none()
|| header_size + lit_c_size > src_size
|| min(block_max, dst_capacity) < lit_size
{
return if lit_size != 0 && dst.is_null() || min(block_max, dst_capacity) < lit_size
{
ERROR(ZstdErrorCode::DstSizeTooSmall)
} else {
ERROR(ZstdErrorCode::CorruptionDetected)
};
}
unsafe {
allocate_literals_buffer(
ctx,
dst,
dst_capacity,
lit_size,
streaming,
min(block_max, dst_capacity),
false,
);
}
let flags = (if unsafe { (*ctx).bmi2 } != 0 {
HUF_FLAGS_BMI2
} else {
0
}) | (if unsafe { (*ctx).disable_huf_asm } != 0 {
HUF_FLAGS_DISABLE_ASM
} else {
0
});
let lit_buffer = unsafe { *(*ctx).lit_buffer };
let huf_result = if lit_type == SET_REPEAT {
if single_stream {
unsafe {
HUF_decompress1X_usingDTable(
lit_buffer.cast(),
lit_size,
src.add(header_size).cast(),
lit_c_size,
*(*ctx).huf_ptr,
flags,
)
}
} else {
unsafe {
HUF_decompress4X_usingDTable(
lit_buffer.cast(),
lit_size,
src.add(header_size).cast(),
lit_c_size,
*(*ctx).huf_ptr,
flags,
)
}
}
} else if single_stream {
#[cfg(feature = "huf-force-decompress-x2")]
{
unsafe {
HUF_decompress1X_DCtx_wksp(
(*entropy(ctx)).huf_table.as_mut_ptr(),
lit_buffer.cast(),
lit_size,
src.add(header_size).cast(),
lit_c_size,
(*ctx).workspace.cast(),
(*ctx).workspace_size,
flags,
)
}
}
#[cfg(not(feature = "huf-force-decompress-x2"))]
{
unsafe {
HUF_decompress1X1_DCtx_wksp(
(*entropy(ctx)).huf_table.as_mut_ptr(),
lit_buffer.cast(),
lit_size,
src.add(header_size).cast(),
lit_c_size,
(*ctx).workspace.cast(),
(*ctx).workspace_size,
flags,
)
}
}
} else {
unsafe {
HUF_decompress4X_hufOnly_wksp(
(*entropy(ctx)).huf_table.as_mut_ptr(),
lit_buffer.cast(),
lit_size,
src.add(header_size).cast(),
lit_c_size,
(*ctx).workspace.cast(),
(*ctx).workspace_size,
flags,
)
}
};
if ERR_isError(huf_result) {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
if unsafe { *(*ctx).lit_buffer_location } == ZSTD_SPLIT {
let extra = unsafe { (*ctx).lit_extra_buffer_size };
unsafe {
copy_bytes(
(*ctx).lit_extra_buffer,
(*(*ctx).lit_buffer_end).sub(extra),
extra,
);
ptr::copy(
*(*ctx).lit_buffer,
(*(*ctx).lit_buffer).add(extra - WILDCOPY_OVERLENGTH),
lit_size - extra,
);
*(*ctx).lit_buffer = (*(*ctx).lit_buffer).add(extra - WILDCOPY_OVERLENGTH);
*(*ctx).lit_buffer_end = (*(*ctx).lit_buffer_end).sub(WILDCOPY_OVERLENGTH);
}
}
unsafe {
*(*ctx).lit_ptr = *(*ctx).lit_buffer;
*(*ctx).lit_size = lit_size;
*(*ctx).lit_entropy = 1;
if lit_type == SET_COMPRESSED {
*(*ctx).huf_ptr = (*entropy(ctx)).huf_table.as_ptr();
}
}
header_size + lit_c_size
}
SET_BASIC => {
let lhl_code = unsafe { (*src >> 2) & 3 };
let (header_size, lit_size) = match lhl_code {
0 | 2 => (1usize, (unsafe { *src } >> 3) as usize),
1 => (2usize, (unsafe { MEM_readLE16(src.cast()) } >> 4) as usize),
_ => {
if src_size < 3 {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
(3usize, (unsafe { MEM_readLE24(src.cast()) } >> 4) as usize)
}
};
if (lit_size != 0 && dst.is_null())
|| lit_size > block_max
|| min(block_max, dst_capacity) < lit_size
{
return if lit_size != 0 && dst.is_null() || min(block_max, dst_capacity) < lit_size
{
ERROR(ZstdErrorCode::DstSizeTooSmall)
} else {
ERROR(ZstdErrorCode::CorruptionDetected)
};
}
unsafe {
allocate_literals_buffer(
ctx,
dst,
dst_capacity,
lit_size,
streaming,
min(block_max, dst_capacity),
true,
);
}
if header_size
.checked_add(lit_size)
.and_then(|size| size.checked_add(WILDCOPY_OVERLENGTH))
.is_none_or(|size| size > src_size)
{
if header_size
.checked_add(lit_size)
.is_none_or(|size| size > src_size)
{
return ERROR(ZstdErrorCode::CorruptionDetected);
}
unsafe {
if *(*ctx).lit_buffer_location == ZSTD_SPLIT {
let extra = (*ctx).lit_extra_buffer_size;
copy_bytes(*(*ctx).lit_buffer, src.add(header_size), lit_size - extra);
copy_bytes(
(*ctx).lit_extra_buffer,
src.add(header_size + lit_size - extra),
extra,
);
} else {
copy_bytes(*(*ctx).lit_buffer, src.add(header_size), lit_size);
}
*(*ctx).lit_ptr = *(*ctx).lit_buffer;
*(*ctx).lit_size = lit_size;
}
return header_size + lit_size;
}
unsafe {
*(*ctx).lit_ptr = src.add(header_size);
*(*ctx).lit_size = lit_size;
*(*ctx).lit_buffer_end = src.add(header_size + lit_size);
*(*ctx).lit_buffer_location = ZSTD_NOT_IN_DST;
}
header_size + lit_size
}
SET_RLE => {
let lhl_code = unsafe { (*src >> 2) & 3 };
let (header_size, lit_size) = match lhl_code {
0 | 2 => (1usize, (unsafe { *src } >> 3) as usize),
1 => {
if src_size < 3 {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
(2usize, (unsafe { MEM_readLE16(src.cast()) } >> 4) as usize)
}
_ => {
if src_size < 4 {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
(3usize, (unsafe { MEM_readLE24(src.cast()) } >> 4) as usize)
}
};
if (lit_size != 0 && dst.is_null())
|| lit_size > block_max
|| min(block_max, dst_capacity) < lit_size
{
return if lit_size != 0 && dst.is_null() || min(block_max, dst_capacity) < lit_size
{
ERROR(ZstdErrorCode::DstSizeTooSmall)
} else {
ERROR(ZstdErrorCode::CorruptionDetected)
};
}
unsafe {
allocate_literals_buffer(
ctx,
dst,
dst_capacity,
lit_size,
streaming,
min(block_max, dst_capacity),
true,
);
let value = *src.add(header_size);
if *(*ctx).lit_buffer_location == ZSTD_SPLIT {
let extra = (*ctx).lit_extra_buffer_size;
fill_bytes(*(*ctx).lit_buffer, value, lit_size - extra);
fill_bytes((*ctx).lit_extra_buffer, value, extra);
} else {
fill_bytes(*(*ctx).lit_buffer, value, lit_size);
}
*(*ctx).lit_ptr = *(*ctx).lit_buffer;
*(*ctx).lit_size = lit_size;
}
header_size + 1
}
_ => ERROR(ZstdErrorCode::CorruptionDetected),
}
}
unsafe fn build_seq_table_rle(table: *mut ZSTD_seqSymbol, base_value: u32, nb_add_bits: u8) {
unsafe {
set_seq_header(
table,
ZSTD_seqSymbol_header {
fast_mode: 0,
table_log: 0,
},
);
*table.add(1) = ZSTD_seqSymbol {
next_state: 0,
nb_additional_bits: nb_add_bits,
nb_bits: 0,
base_value,
};
}
}
unsafe fn build_fse_table_body(
table: *mut ZSTD_seqSymbol,
normalized_counter: *const i16,
max_symbol_value: u32,
base_value: *const u32,
nb_additional_bits: *const u8,
table_log: u32,
workspace: *mut u32,
workspace_size: usize,
) {
debug_assert!(max_symbol_value as usize <= MAX_ML);
debug_assert!(table_log as usize <= MAX_FSE_LOG);
debug_assert!(
workspace_size >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 * std::mem::size_of::<u32>()
);
let table_decode = unsafe { table.add(1) };
let table_size = 1usize << table_log;
let symbol_next = workspace.cast::<u16>();
let spread = unsafe { symbol_next.add(MAX_ML + 1).cast::<u8>() };
let mut high_threshold = table_size - 1;
let mut fast_mode = 1u32;
let large_limit = 1i16 << (table_log - 1);
for symbol in 0..=max_symbol_value as usize {
let count = unsafe { *normalized_counter.add(symbol) };
if count == -1 {
unsafe {
(*table_decode.add(high_threshold)).base_value = symbol as u32;
*symbol_next.add(symbol) = 1;
}
high_threshold = high_threshold.wrapping_sub(1);
} else {
if count >= large_limit {
fast_mode = 0;
}
debug_assert!(count >= 0);
unsafe { *symbol_next.add(symbol) = count as u16 };
}
}
unsafe {
set_seq_header(
table,
ZSTD_seqSymbol_header {
fast_mode,
table_log,
},
);
}
let table_mask = table_size - 1;
let step = (table_size >> 1) + (table_size >> 3) + 3;
if high_threshold == table_size - 1 {
let mut pos = 0usize;
for symbol in 0..=max_symbol_value as usize {
let count = unsafe { *normalized_counter.add(symbol) };
debug_assert!(count >= 0);
for index in 0..count as usize {
unsafe { *spread.add(pos + index) = symbol as u8 };
}
pos += count as usize;
}
let mut position = 0usize;
for symbol_index in 0..table_size {
unsafe { (*table_decode.add(position)).base_value = *spread.add(symbol_index) as u32 };
position = (position + step) & table_mask;
}
debug_assert_eq!(position, 0);
} else {
let mut position = 0usize;
for symbol in 0..=max_symbol_value as usize {
let count = unsafe { *normalized_counter.add(symbol) };
for _ in 0..count.max(0) as usize {
unsafe { (*table_decode.add(position)).base_value = symbol as u32 };
position = (position + step) & table_mask;
while position > high_threshold {
position = (position + step) & table_mask;
}
}
}
debug_assert_eq!(position, 0);
}
for index in 0..table_size {
let symbol = unsafe { (*table_decode.add(index)).base_value as usize };
let next_state = unsafe { *symbol_next.add(symbol) } as u32;
unsafe { *symbol_next.add(symbol) = next_state.wrapping_add(1) as u16 };
let nb_bits = table_log - crate::bits::ZSTD_highbit32(next_state);
unsafe {
let entry = &mut *table_decode.add(index);
entry.nb_bits = nb_bits as u8;
entry.next_state = ((next_state << nb_bits) - table_size as u32) as u16;
entry.nb_additional_bits = *nb_additional_bits.add(symbol);
entry.base_value = *base_value.add(symbol);
}
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_buildFSETable_body(
table: *mut ZSTD_seqSymbol,
normalized_counter: *const c_short,
max_symbol_value: c_uint,
base_value: *const U32,
nb_additional_bits: *const u8,
table_log: c_uint,
workspace: *mut c_void,
workspace_size: usize,
) {
unsafe {
build_fse_table_body(
table,
normalized_counter,
max_symbol_value,
base_value,
nb_additional_bits,
table_log,
workspace.cast(),
workspace_size,
);
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_buildFSETable(
table: *mut ZSTD_seqSymbol,
normalized_counter: *const c_short,
max_symbol_value: c_uint,
base_value: *const U32,
nb_additional_bits: *const u8,
table_log: c_uint,
workspace: *mut c_void,
workspace_size: usize,
_bmi2: c_int,
) {
unsafe {
build_fse_table_body(
table,
normalized_counter,
max_symbol_value,
base_value,
nb_additional_bits,
table_log,
workspace.cast(),
workspace_size,
);
}
}
fn build_default_table<const N: usize>(
normalized: &[i16],
base: &[u32],
bits: &[u8],
max_symbol_value: u32,
table_log: u32,
) -> [ZSTD_seqSymbol; N] {
let mut table = [ZSTD_seqSymbol::default(); N];
let mut workspace = [0u32; ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32];
unsafe {
build_fse_table_body(
table.as_mut_ptr(),
normalized.as_ptr(),
max_symbol_value,
base.as_ptr(),
bits.as_ptr(),
table_log,
workspace.as_mut_ptr(),
std::mem::size_of_val(&workspace),
);
}
table
}
static LL_DEFAULT_TABLE: OnceLock<[ZSTD_seqSymbol; 1 + (1 << LL_FSE_LOG)]> = OnceLock::new();
static OF_DEFAULT_TABLE: OnceLock<[ZSTD_seqSymbol; 1 + (1 << OFF_FSE_LOG)]> = OnceLock::new();
static ML_DEFAULT_TABLE: OnceLock<[ZSTD_seqSymbol; 1 + (1 << ML_FSE_LOG)]> = OnceLock::new();
#[inline]
fn ll_default_table() -> *const ZSTD_seqSymbol {
LL_DEFAULT_TABLE
.get_or_init(|| {
build_default_table(
&LL_DEFAULT_NORM,
&LL_BASE,
&LL_BITS,
MAX_LL as u32,
LL_DEFAULT_NORM_LOG,
)
})
.as_ptr()
}
#[inline]
fn of_default_table() -> *const ZSTD_seqSymbol {
OF_DEFAULT_TABLE
.get_or_init(|| {
build_default_table(
&OF_DEFAULT_NORM,
&OF_BASE,
&OF_BITS,
DEFAULT_MAX_OFF as u32,
OF_DEFAULT_NORM_LOG,
)
})
.as_ptr()
}
#[inline]
fn ml_default_table() -> *const ZSTD_seqSymbol {
ML_DEFAULT_TABLE
.get_or_init(|| {
build_default_table(
&ML_DEFAULT_NORM,
&ML_BASE,
&ML_BITS,
MAX_ML as u32,
ML_DEFAULT_NORM_LOG,
)
})
.as_ptr()
}
unsafe fn build_seq_table(
table_space: *mut ZSTD_seqSymbol,
table_ptr: *mut *const ZSTD_seqSymbol,
table_type: c_int,
mut max_symbol: u32,
max_log: u32,
src: *const u8,
src_size: usize,
base: &[u32],
bits: &[u8],
default_table: *const ZSTD_seqSymbol,
repeat_table_available: u32,
_ddict_is_cold: c_int,
_nb_seq: c_int,
workspace: *mut u32,
workspace_size: usize,
bmi2: c_int,
) -> usize {
match table_type {
SET_RLE => {
if src_size == 0 {
return ERROR(ZstdErrorCode::SrcSizeWrong);
}
let symbol = unsafe { *src } as usize;
if symbol > max_symbol as usize {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
unsafe { build_seq_table_rle(table_space, base[symbol], bits[symbol]) };
unsafe { *table_ptr = table_space };
1
}
SET_BASIC => {
unsafe { *table_ptr = default_table };
0
}
SET_REPEAT => {
if repeat_table_available == 0 {
ERROR(ZstdErrorCode::CorruptionDetected)
} else {
0
}
}
SET_COMPRESSED => {
let mut table_log = 0u32;
let mut norm = [0i16; MAX_ML + 1];
let header_size = unsafe {
FSE_readNCount(
norm.as_mut_ptr(),
&mut max_symbol,
&mut table_log,
src.cast(),
src_size,
)
};
if ERR_isError(header_size) || table_log > max_log {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
unsafe {
build_fse_table_body(
table_space,
norm.as_ptr(),
max_symbol,
base.as_ptr(),
bits.as_ptr(),
table_log,
workspace,
workspace_size,
);
*table_ptr = table_space;
}
let _ = bmi2;
header_size
}
_ => ERROR(ZstdErrorCode::Generic),
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_decodeSeqHeaders(
ctx: *mut ZSTD_rustBlockCtx,
nb_seq_ptr: *mut c_int,
src: *const c_void,
src_size: usize,
) -> usize {
if src_size < MIN_SEQUENCES_SIZE {
return ERROR(ZstdErrorCode::SrcSizeWrong);
}
let start = src.cast::<u8>();
let end = unsafe { start.add(src_size) };
let mut ip = start;
let mut nb_seq = unsafe { *ip } as usize;
ip = unsafe { ip.add(1) };
if nb_seq > 0x7f {
if nb_seq == 0xff {
if (ip as usize).wrapping_add(2) > end as usize {
return ERROR(ZstdErrorCode::SrcSizeWrong);
}
nb_seq = unsafe { MEM_readLE16(ip.cast()) as usize } + LONG_NB_SEQ;
ip = unsafe { ip.add(2) };
} else {
if ip >= end {
return ERROR(ZstdErrorCode::SrcSizeWrong);
}
nb_seq = ((nb_seq - 0x80) << 8) + unsafe { *ip } as usize;
ip = unsafe { ip.add(1) };
}
}
unsafe { *nb_seq_ptr = nb_seq as c_int };
if nb_seq == 0 {
return if ip == end {
unsafe { ip.offset_from(start) as usize }
} else {
ERROR(ZstdErrorCode::CorruptionDetected)
};
}
if ip >= end || unsafe { *ip & 3 } != 0 {
return if ip >= end {
ERROR(ZstdErrorCode::SrcSizeWrong)
} else {
ERROR(ZstdErrorCode::CorruptionDetected)
};
}
let descriptor = unsafe { *ip };
ip = unsafe { ip.add(1) };
let ll_type = (descriptor >> 6) as c_int;
let of_type = ((descriptor >> 4) & 3) as c_int;
let ml_type = ((descriptor >> 2) & 3) as c_int;
let entropy = unsafe { entropy(ctx) };
let repeat = unsafe { *(*ctx).fse_entropy };
let cold = unsafe { *(*ctx).ddict_is_cold };
let ll_size = unsafe {
build_seq_table(
(*entropy).ll_table.as_mut_ptr(),
(*ctx).llt_ptr,
ll_type,
MAX_LL as u32,
LL_FSE_LOG,
ip,
address_distance(end, ip),
&LL_BASE,
&LL_BITS,
ll_default_table(),
repeat,
cold,
nb_seq as c_int,
(*ctx).workspace,
(*ctx).workspace_size,
(*ctx).bmi2,
)
};
if ERR_isError(ll_size) || ll_size > unsafe { address_distance(end, ip) } {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
ip = unsafe { ip.add(ll_size) };
let of_size = unsafe {
build_seq_table(
(*entropy).of_table.as_mut_ptr(),
(*ctx).oft_ptr,
of_type,
MAX_OFF as u32,
OFF_FSE_LOG,
ip,
address_distance(end, ip),
&OF_BASE,
&OF_BITS,
of_default_table(),
repeat,
cold,
nb_seq as c_int,
(*ctx).workspace,
(*ctx).workspace_size,
(*ctx).bmi2,
)
};
if ERR_isError(of_size) || of_size > unsafe { address_distance(end, ip) } {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
ip = unsafe { ip.add(of_size) };
let ml_size = unsafe {
build_seq_table(
(*entropy).ml_table.as_mut_ptr(),
(*ctx).mlt_ptr,
ml_type,
MAX_ML as u32,
ML_FSE_LOG,
ip,
address_distance(end, ip),
&ML_BASE,
&ML_BITS,
ml_default_table(),
repeat,
cold,
nb_seq as c_int,
(*ctx).workspace,
(*ctx).workspace_size,
(*ctx).bmi2,
)
};
if ERR_isError(ml_size) || ml_size > unsafe { address_distance(end, ip) } {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
ip = unsafe { ip.add(ml_size) };
unsafe { ip.offset_from(start) as usize }
}
#[inline]
unsafe fn init_fse_state(
state: &mut ZSTD_fseState,
dstream: *mut BIT_DStream_t,
table: *const ZSTD_seqSymbol,
) -> Result<(), usize> {
if table.is_null() {
return Err(ERROR(ZstdErrorCode::CorruptionDetected));
}
let header = unsafe { seq_header(table) };
if header.table_log > MAX_FSE_LOG as u32 {
return Err(ERROR(ZstdErrorCode::CorruptionDetected));
}
state.state = unsafe { BIT_readBits(dstream, header.table_log) };
let _ = unsafe { BIT_reloadDStream(dstream) };
state.table = unsafe { table.add(1) };
Ok(())
}
#[inline]
unsafe fn update_fse_state(
state: &mut ZSTD_fseState,
dstream: *mut BIT_DStream_t,
next_state: u16,
nb_bits: u8,
) {
let low_bits = unsafe { BIT_readBits(dstream, nb_bits as u32) };
state.state = next_state as usize + low_bits;
}
unsafe fn decode_sequence(
state: &mut seq_state_t,
long_offsets: bool,
is_last_sequence: bool,
) -> ZSTD_rustSeq {
let ll_info = unsafe { state.state_ll.table.add(state.state_ll.state).read() };
let ml_info = unsafe { state.state_ml.table.add(state.state_ml.state).read() };
let of_info = unsafe { state.state_off.table.add(state.state_off.state).read() };
let ll_bits = ll_info.nb_additional_bits as u32;
let ml_bits = ml_info.nb_additional_bits as u32;
let of_bits = of_info.nb_additional_bits as u32;
let total_bits = ll_bits + ml_bits + of_bits;
let mut sequence = ZSTD_rustSeq {
lit_length: ll_info.base_value as usize,
match_length: ml_info.base_value as usize,
offset: 0,
};
let offset = if of_bits > 1 {
let value = if MEM_32bits() && long_offsets && of_bits >= 25 {
let upper = unsafe { BIT_readBitsFast(&mut state.dstream, of_bits - 5) } << 5;
let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
upper + unsafe { BIT_readBitsFast(&mut state.dstream, 5) }
} else {
let value = unsafe { BIT_readBitsFast(&mut state.dstream, of_bits) };
if MEM_32bits() {
let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
}
value
};
let value = (of_info.base_value as usize).wrapping_add(value);
state.prev_offset[2] = state.prev_offset[1];
state.prev_offset[1] = state.prev_offset[0];
state.prev_offset[0] = value;
value
} else {
let ll_zero = ll_info.base_value == 0;
if of_bits == 0 {
let value = state.prev_offset[usize::from(ll_zero)];
state.prev_offset[1] = state.prev_offset[usize::from(!ll_zero)];
state.prev_offset[0] = value;
value
} else {
let code = (of_info.base_value as usize)
.wrapping_add(usize::from(ll_zero))
.wrapping_add(unsafe { BIT_readBitsFast(&mut state.dstream, 1) });
let mut value = if code == 3 {
state.prev_offset[0].wrapping_sub(1)
} else if code < ZSTD_REP_NUM {
state.prev_offset[code]
} else {
usize::MAX
};
value = value.wrapping_sub(usize::from(value == 0));
if code != 1 {
state.prev_offset[2] = state.prev_offset[1];
}
state.prev_offset[1] = state.prev_offset[0];
state.prev_offset[0] = value;
value
}
};
sequence.offset = offset;
if ml_bits != 0 {
sequence.match_length = sequence
.match_length
.wrapping_add(unsafe { BIT_readBitsFast(&mut state.dstream, ml_bits) });
}
if MEM_32bits() && ml_bits + ll_bits >= 20 {
let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
}
if MEM_64bits() && total_bits >= 30 {
let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
}
if ll_bits != 0 {
sequence.lit_length = sequence
.lit_length
.wrapping_add(unsafe { BIT_readBitsFast(&mut state.dstream, ll_bits) });
}
if MEM_32bits() {
let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
}
if !is_last_sequence {
unsafe {
update_fse_state(
&mut state.state_ll,
&mut state.dstream,
ll_info.next_state,
ll_info.nb_bits,
);
update_fse_state(
&mut state.state_ml,
&mut state.dstream,
ml_info.next_state,
ml_info.nb_bits,
);
}
if MEM_32bits() {
let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
}
unsafe {
update_fse_state(
&mut state.state_off,
&mut state.dstream,
of_info.next_state,
of_info.nb_bits,
);
}
let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
}
sequence
}
unsafe fn exec_sequence(
op: *mut u8,
oend: *mut u8,
sequence: ZSTD_rustSeq,
lit_ptr: &mut *const u8,
lit_limit: *const u8,
prefix_start: *const u8,
virtual_start: *const u8,
dict_end: *const u8,
split_literals: bool,
) -> usize {
let sequence_length = match sequence.lit_length.checked_add(sequence.match_length) {
Some(length) => length,
None => return ERROR(ZstdErrorCode::DstSizeTooSmall),
};
let output_capacity = unsafe { address_distance(oend.cast_const(), op.cast_const()) };
if sequence_length > output_capacity {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
let literal_available = unsafe { address_distance(lit_limit, *lit_ptr) };
if sequence.lit_length > literal_available {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
let lit_end = unsafe { ptr_add(op, sequence.lit_length) };
if split_literals
&& (op as usize) > (*lit_ptr as usize)
&& (op as usize) < (*lit_ptr as usize).wrapping_add(sequence.lit_length)
{
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
unsafe { copy_bytes(op, *lit_ptr, sequence.lit_length) };
*lit_ptr = unsafe { const_ptr_add(*lit_ptr, sequence.lit_length) };
if sequence.offset == 0 {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
let prefix_history = unsafe { address_distance(lit_end.cast_const(), prefix_start) };
let mut output = lit_end;
let mut match_length = sequence.match_length;
if sequence.offset > prefix_history {
let virtual_history = unsafe { address_distance(lit_end.cast_const(), virtual_start) };
if sequence.offset > virtual_history || dict_end.is_null() {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
let before_prefix = sequence.offset - prefix_history;
let match_ptr = unsafe { dict_end.sub(before_prefix) };
let dict_available = unsafe { address_distance(dict_end, match_ptr) };
let first_length = min(match_length, dict_available);
unsafe { copy_bytes(output, match_ptr, first_length) };
output = unsafe { output.add(first_length) };
match_length -= first_length;
if match_length != 0 {
if prefix_start.is_null() {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
unsafe { copy_match(output, prefix_start, match_length) };
}
} else {
let match_ptr = unsafe { lit_end.sub(sequence.offset) };
if match_ptr.is_null() {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
unsafe { copy_match(output, match_ptr, match_length) };
}
sequence_length
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_execSequenceEnd(
op: *mut u8,
oend: *mut u8,
sequence: ZSTD_rustSeq,
lit_ptr: *mut *const u8,
lit_limit: *const u8,
prefix_start: *const u8,
virtual_start: *const u8,
dict_end: *const u8,
) -> usize {
unsafe {
exec_sequence(
op,
oend,
sequence,
&mut *lit_ptr,
lit_limit,
prefix_start,
virtual_start,
dict_end,
false,
)
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_execSequenceEndSplitLitBuffer(
op: *mut u8,
oend: *mut u8,
_oend_w: *const u8,
sequence: ZSTD_rustSeq,
lit_ptr: *mut *const u8,
lit_limit: *const u8,
prefix_start: *const u8,
virtual_start: *const u8,
dict_end: *const u8,
) -> usize {
unsafe {
exec_sequence(
op,
oend,
sequence,
&mut *lit_ptr,
lit_limit,
prefix_start,
virtual_start,
dict_end,
true,
)
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_execSequence(
op: *mut u8,
oend: *mut u8,
sequence: ZSTD_rustSeq,
lit_ptr: *mut *const u8,
lit_limit: *const u8,
prefix_start: *const u8,
virtual_start: *const u8,
dict_end: *const u8,
) -> usize {
unsafe {
exec_sequence(
op,
oend,
sequence,
&mut *lit_ptr,
lit_limit,
prefix_start,
virtual_start,
dict_end,
false,
)
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_execSequenceSplitLitBuffer(
op: *mut u8,
oend: *mut u8,
_oend_w: *const u8,
sequence: ZSTD_rustSeq,
lit_ptr: *mut *const u8,
lit_limit: *const u8,
prefix_start: *const u8,
virtual_start: *const u8,
dict_end: *const u8,
) -> usize {
unsafe {
exec_sequence(
op,
oend,
sequence,
&mut *lit_ptr,
lit_limit,
prefix_start,
virtual_start,
dict_end,
true,
)
}
}
unsafe fn decompress_sequences(
ctx: *mut ZSTD_rustBlockCtx,
dst: *mut u8,
dst_capacity: usize,
sequence_start: *const u8,
sequence_size: usize,
nb_seq: c_int,
long_offsets: bool,
) -> usize {
if nb_seq < 0 {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
let mut location = unsafe { *(*ctx).lit_buffer_location };
let mut op = dst;
let mut oend = if location == ZSTD_NOT_IN_DST || location == ZSTD_SPLIT {
unsafe { ptr_add(dst, dst_capacity) }
} else {
unsafe { *(*ctx).lit_buffer }
};
let mut lit_ptr = unsafe { *(*ctx).lit_ptr };
let mut lit_limit = if location == ZSTD_SPLIT {
unsafe { *(*ctx).lit_buffer_end }
} else {
unsafe { const_ptr_add(lit_ptr, *(*ctx).lit_size) }
};
let prefix_start = unsafe { *(*ctx).prefix_start };
let virtual_start = unsafe { *(*ctx).virtual_start };
let dict_end = unsafe { *(*ctx).dict_end };
if nb_seq != 0 {
if dst.is_null() {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
let mut state = seq_state_t {
dstream: unsafe { MaybeUninit::zeroed().assume_init() },
state_ll: ZSTD_fseState {
state: 0,
table: ptr::null(),
},
state_off: ZSTD_fseState {
state: 0,
table: ptr::null(),
},
state_ml: ZSTD_fseState {
state: 0,
table: ptr::null(),
},
prev_offset: unsafe { (*entropy(ctx)).rep.map(|value| value as usize) },
};
unsafe { *(*ctx).fse_entropy = 1 };
let init =
unsafe { BIT_initDStream(&mut state.dstream, sequence_start.cast(), sequence_size) };
if ERR_isError(init) {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
if let Err(error) =
unsafe { init_fse_state(&mut state.state_ll, &mut state.dstream, *(*ctx).llt_ptr) }
{
return error;
}
if let Err(error) =
unsafe { init_fse_state(&mut state.state_off, &mut state.dstream, *(*ctx).oft_ptr) }
{
return error;
}
if let Err(error) =
unsafe { init_fse_state(&mut state.state_ml, &mut state.dstream, *(*ctx).mlt_ptr) }
{
return error;
}
for remaining in (1..=nb_seq as usize).rev() {
let mut sequence = unsafe { decode_sequence(&mut state, long_offsets, remaining == 1) };
if location == ZSTD_SPLIT
&& sequence.lit_length > unsafe { address_distance(lit_limit, lit_ptr) }
{
let leftover = unsafe { address_distance(lit_limit, lit_ptr) };
if leftover > unsafe { address_distance(oend.cast_const(), op.cast_const()) } {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
unsafe { copy_bytes(op, lit_ptr, leftover) };
op = unsafe { op.add(leftover) };
sequence.lit_length -= leftover;
lit_ptr = unsafe { (*ctx).lit_extra_buffer.cast_const() };
lit_limit = unsafe {
const_ptr_add(
(*ctx).lit_extra_buffer.cast_const(),
(*ctx).lit_extra_buffer_size,
)
};
location = ZSTD_NOT_IN_DST;
unsafe { *(*ctx).lit_buffer_location = ZSTD_NOT_IN_DST };
}
let decoded = unsafe {
exec_sequence(
op,
oend,
sequence,
&mut lit_ptr,
lit_limit,
prefix_start,
virtual_start,
dict_end,
location == ZSTD_SPLIT,
)
};
if ERR_isError(decoded) {
return decoded;
}
op = unsafe { op.add(decoded) };
}
if unsafe { BIT_endOfDStream(&state.dstream) } == 0 {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
unsafe { (*entropy(ctx)).rep = state.prev_offset.map(|value| value as u32) };
}
if location == ZSTD_SPLIT {
let first_size = unsafe { address_distance(lit_limit, lit_ptr) };
if first_size > unsafe { address_distance(oend.cast_const(), op.cast_const()) } {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
unsafe { copy_bytes(op, lit_ptr, first_size) };
op = unsafe { op.add(first_size) };
lit_ptr = unsafe { (*ctx).lit_extra_buffer.cast_const() };
lit_limit = unsafe {
const_ptr_add(
(*ctx).lit_extra_buffer.cast_const(),
(*ctx).lit_extra_buffer_size,
)
};
unsafe { *(*ctx).lit_buffer_location = ZSTD_NOT_IN_DST };
oend = unsafe { ptr_add(dst, dst_capacity) };
}
let last_size = unsafe { address_distance(lit_limit, lit_ptr) };
if last_size > unsafe { address_distance(oend.cast_const(), op.cast_const()) } {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
unsafe { copy_bytes(op, lit_ptr, last_size) };
op = unsafe { op.add(last_size) };
unsafe { address_distance(op.cast_const(), dst.cast_const()) }
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_prefetchMatch(
prefetch_pos: usize,
sequence: ZSTD_rustSeq,
_prefix_start: *const u8,
_dict_end: *const u8,
) -> usize {
prefetch_pos
.wrapping_add(sequence.lit_length)
.wrapping_add(sequence.match_length)
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_decodeLiteralsBlock_wrapper(
ctx: *mut ZSTD_rustBlockCtx,
src: *const c_void,
src_size: usize,
dst: *mut c_void,
dst_capacity: usize,
) -> usize {
unsafe {
*(*ctx).is_frame_decompression = 0;
decode_literals_block(
ctx,
src.cast(),
src_size,
dst.cast(),
dst_capacity,
NOT_STREAMING,
)
}
}
#[inline]
fn max_short_offset() -> usize {
if MEM_64bits() {
usize::MAX
} else {
((1usize << 26) - 1).wrapping_sub(ZSTD_REP_NUM)
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_decompressBlock_internal(
ctx: *mut ZSTD_rustBlockCtx,
dst: *mut c_void,
dst_capacity: usize,
src: *const c_void,
src_size: usize,
streaming: c_int,
) -> usize {
let block_max = unsafe { block_size_max(ctx) };
if src_size > block_max {
return ERROR(ZstdErrorCode::SrcSizeWrong);
}
let mut input = src.cast::<u8>();
let lit_size =
unsafe { decode_literals_block(ctx, input, src_size, dst.cast(), dst_capacity, streaming) };
if ERR_isError(lit_size) || lit_size > src_size {
return lit_size;
}
input = unsafe { input.add(lit_size) };
let remaining = src_size - lit_size;
let history_end = unsafe { ptr_add(dst.cast(), min(dst_capacity, block_max)) };
let history_size = unsafe { address_distance(history_end.cast_const(), *(*ctx).virtual_start) };
let long_offsets = MEM_32bits() && history_size > max_short_offset();
let mut nb_seq = 0 as c_int;
let header_size =
unsafe { ZSTD_rust_decodeSeqHeaders(ctx, &mut nb_seq, input.cast(), remaining) };
if ERR_isError(header_size) || header_size > remaining {
return header_size;
}
input = unsafe { input.add(header_size) };
let sequence_size = remaining - header_size;
if (dst.is_null() || dst_capacity == 0) && nb_seq > 0 {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
unsafe { *(*ctx).ddict_is_cold = 0 };
unsafe {
decompress_sequences(
ctx,
dst.cast(),
dst_capacity,
input,
sequence_size,
nb_seq,
long_offsets,
)
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_checkContinuity(
ctx: *mut ZSTD_rustBlockCtx,
dst: *const c_void,
dst_size: usize,
) {
unsafe {
if dst.cast::<u8>() != *(*ctx).previous_dst_end && dst_size != 0 {
*(*ctx).dict_end = *(*ctx).previous_dst_end;
let prefix = *(*ctx).prefix_start;
let previous = *(*ctx).previous_dst_end;
let distance = address_distance(previous, prefix);
*(*ctx).virtual_start = dst.cast::<u8>().wrapping_sub(distance);
*(*ctx).prefix_start = dst.cast();
*(*ctx).previous_dst_end = dst.cast();
}
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_decompressBlock_deprecated(
ctx: *mut ZSTD_rustBlockCtx,
dst: *mut c_void,
dst_capacity: usize,
src: *const c_void,
src_size: usize,
) -> usize {
unsafe {
*(*ctx).is_frame_decompression = 0;
ZSTD_rust_checkContinuity(ctx, dst.cast(), dst_capacity);
let result = ZSTD_rust_decompressBlock_internal(
ctx,
dst,
dst_capacity,
src,
src_size,
NOT_STREAMING,
);
if !ERR_isError(result) {
*(*ctx).previous_dst_end = ptr_add(dst.cast(), result).cast_const();
}
result
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn offset_code_22_and_23_keep_their_22_and_23_bit_bases() {
assert_eq!(OF_BASE[22], 0x3F_FFFD);
assert_eq!(OF_BASE[23], 0x7F_FFFD);
assert_eq!(OF_BASE[24], 0xFF_FFFD);
let mut normalized = [0i16; MAX_OFF + 1];
normalized[22] = 1 << 5;
let mut table = [ZSTD_seqSymbol::default(); 1 + (1 << 5)];
let mut workspace = [0u32; ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32];
unsafe {
build_fse_table_body(
table.as_mut_ptr(),
normalized.as_ptr(),
22,
OF_BASE.as_ptr(),
OF_BITS.as_ptr(),
5,
workspace.as_mut_ptr(),
std::mem::size_of_val(&workspace),
);
}
assert!(table[1..]
.iter()
.all(|symbol| symbol.base_value == 0x3F_FFFD));
}
}