ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(rust): port compressed block decoding

Browse Source 
Move literal decoding, sequence-table construction, FSE sequence decoding, and
sequence execution into Rust. The C shim retains ownership of the configured
decoder context and passes only the leaf state needed by the block codec.

Correct the two high offset-code bases while porting the tables. Their prior
values made valid large-window streams decode as corrupted data; the new unit
test fixes the exact values and a native zstream regression exercises them.
High-level frame and streaming context control remains C for now.

Test Plan:
- cargo test --all-targets
- cargo test --target i686-unknown-linux-gnu --all-targets
- cargo clippy && cargo clippy --benches && cargo clippy --tests
- cargo +nightly fmt
- make -B -C tests -j2 fuzzer zstreamtest invalidDictionaries poolTests
- ./tests/fuzzer -s5346 -i1 --no-big-tests
- ./tests/zstreamtest -i3000 -s334462
- ./tests/invalidDictionaries
- timeout 20s stdbuf -oL ./tests/poolTests

Refs: rust/README.md
This commit is contained in:
ddidderr
2026-07-11 08:07:40 +02:00
parent 7f16a07375
commit e5eebd4892
4 changed files with 1823 additions and 2168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/decompress/zstd_decompress_block.c
+114 -2164
View File
@@ -8,2202 +8,152 @@
* You may select, at your option, one of the above-listed licenses.
*/
/* zstd_decompress_block :
* this module takes care of decompressing _compressed_ block */
/*-*******************************************************
* Dependencies
*********************************************************/
#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
#include "../common/compiler.h" /* prefetch */
#include "../common/cpu.h" /* bmi2 */
#include "../common/mem.h" /* low level memory routines */
#define FSE_STATIC_LINKING_ONLY
#include "../common/fse.h"
#include "../common/huf.h"
#include "../common/zstd_internal.h"
#include "zstd_decompress_internal.h" /* ZSTD_DCtx */
#include "zstd_ddict.h" /* ZSTD_DDictDictContent */
#include "zstd_decompress_block.h"
#include "../common/bits.h" /* ZSTD_highbit32 */
/*_*******************************************************
* Macros
**********************************************************/
/* These two optional macros force the use one way or another of the two
* ZSTD_decompressSequences implementations. You can't force in both directions
* at the same time.
/*
* Compressed-block decoding is implemented in
* rust/src/zstd_decompress_block.rs. Keep `ZSTD_DCtx` C-owned: it has
* optional build-dependent fields, while this small view contains only the
* leaves read or written by the Rust decoder. No decoder algorithm remains
* in this translation unit.
*/
#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!"
#endif
#include "../common/zstd_deps.h"
#include "../common/zstd_internal.h"
#include "zstd_decompress_internal.h"
#include "zstd_decompress_block.h"
typedef char ZSTD_rust_block_seq_symbol_layout[(sizeof(ZSTD_seqSymbol) == 8) ? 1 : -1];
typedef char ZSTD_rust_block_entropy_rep_offset[
(offsetof(ZSTD_entropyDTables_t, rep) == 26652) ? 1 : -1];
typedef char ZSTD_rust_block_entropy_workspace_offset[
(offsetof(ZSTD_entropyDTables_t, workspace) == 26664) ? 1 : -1];
/*_*******************************************************
* Memory operations
**********************************************************/
static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); }
typedef struct {
const ZSTD_seqSymbol** lltPtr;
const ZSTD_seqSymbol** mltPtr;
const ZSTD_seqSymbol** oftPtr;
const HUF_DTable** hufPtr;
ZSTD_entropyDTables_t* entropy;
U32* workspace;
size_t workspaceSize;
const void** previousDstEnd;
const void** prefixStart;
const void** virtualStart;
const void** dictEnd;
size_t blockSizeMax;
int* isFrameDecompression;
U32* litEntropy;
U32* fseEntropy;
int bmi2;
int* ddictIsCold;
int disableHufAsm;
const BYTE** litPtr;
size_t* litSize;
size_t* rleSize;
BYTE** litBuffer;
const BYTE** litBufferEnd;
ZSTD_litLocation_e* litBufferLocation;
BYTE* litExtraBuffer;
size_t litExtraBufferSize;
} ZSTD_rustBlockCtx;
/*-*************************************************************
* Block decoding
***************************************************************/
static size_t ZSTD_blockSizeMax(ZSTD_DCtx const* dctx)
static ZSTD_rustBlockCtx ZSTD_rust_block_context(ZSTD_DCtx* dctx)
{
size_t const blockSizeMax = dctx->isFrameDecompression ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX;
assert(blockSizeMax <= ZSTD_BLOCKSIZE_MAX);
return blockSizeMax;
ZSTD_rustBlockCtx ctx;
ctx.lltPtr = &dctx->LLTptr;
ctx.mltPtr = &dctx->MLTptr;
ctx.oftPtr = &dctx->OFTptr;
ctx.hufPtr = &dctx->HUFptr;
ctx.entropy = &dctx->entropy;
ctx.workspace = dctx->workspace;
ctx.workspaceSize = sizeof(dctx->workspace);
ctx.previousDstEnd = &dctx->previousDstEnd;
ctx.prefixStart = &dctx->prefixStart;
ctx.virtualStart = &dctx->virtualStart;
ctx.dictEnd = &dctx->dictEnd;
ctx.blockSizeMax = dctx->fParams.blockSizeMax;
ctx.isFrameDecompression = &dctx->isFrameDecompression;
ctx.litEntropy = &dctx->litEntropy;
ctx.fseEntropy = &dctx->fseEntropy;
ctx.bmi2 = ZSTD_DCtx_get_bmi2(dctx);
ctx.ddictIsCold = &dctx->ddictIsCold;
ctx.disableHufAsm = dctx->disableHufAsm;
ctx.litPtr = &dctx->litPtr;
ctx.litSize = &dctx->litSize;
ctx.rleSize = &dctx->rleSize;
ctx.litBuffer = &dctx->litBuffer;
ctx.litBufferEnd = &dctx->litBufferEnd;
ctx.litBufferLocation = &dctx->litBufferLocation;
ctx.litExtraBuffer = dctx->litExtraBuffer;
ctx.litExtraBufferSize = ZSTD_LITBUFFEREXTRASIZE;
return ctx;
}
/*! ZSTD_getcBlockSize() :
* Provides the size of compressed block from block header `src` */
size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
blockProperties_t* bpPtr)
{
RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, "");
size_t ZSTD_rust_decodeLiteralsBlock_wrapper(
ZSTD_rustBlockCtx* ctx,
const void* src, size_t srcSize,
void* dst, size_t dstCapacity);
size_t ZSTD_rust_decodeSeqHeaders(
ZSTD_rustBlockCtx* ctx, int* nbSeqPtr,
const void* src, size_t srcSize);
size_t ZSTD_rust_decompressBlock_internal(
ZSTD_rustBlockCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize, int streaming);
void ZSTD_rust_checkContinuity(
ZSTD_rustBlockCtx* ctx, const void* dst, size_t dstSize);
size_t ZSTD_rust_decompressBlock_deprecated(
ZSTD_rustBlockCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
{ U32 const cBlockHeader = MEM_readLE24(src);
U32 const cSize = cBlockHeader >> 3;
bpPtr->lastBlock = cBlockHeader & 1;
bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3);
bpPtr->origSize = cSize; /* only useful for RLE */
if (bpPtr->blockType == bt_rle) return 1;
RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, "");
return cSize;
}
}
/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */
static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize,
const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately)
{
size_t const blockSizeMax = ZSTD_blockSizeMax(dctx);
assert(litSize <= blockSizeMax);
assert(dctx->isFrameDecompression || streaming == not_streaming);
assert(expectedWriteSize <= blockSizeMax);
if (streaming == not_streaming && dstCapacity > blockSizeMax + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH) {
/* If we aren't streaming, we can just put the literals after the output
* of the current block. We don't need to worry about overwriting the
* extDict of our window, because it doesn't exist.
* So if we have space after the end of the block, just put it there.
*/
dctx->litBuffer = (BYTE*)dst + blockSizeMax + WILDCOPY_OVERLENGTH;
dctx->litBufferEnd = dctx->litBuffer + litSize;
dctx->litBufferLocation = ZSTD_in_dst;
} else if (litSize <= ZSTD_LITBUFFEREXTRASIZE) {
/* Literals fit entirely within the extra buffer, put them there to avoid
* having to split the literals.
*/
dctx->litBuffer = dctx->litExtraBuffer;
dctx->litBufferEnd = dctx->litBuffer + litSize;
dctx->litBufferLocation = ZSTD_not_in_dst;
} else {
assert(blockSizeMax > ZSTD_LITBUFFEREXTRASIZE);
/* Literals must be split between the output block and the extra lit
* buffer. We fill the extra lit buffer with the tail of the literals,
* and put the rest of the literals at the end of the block, with
* WILDCOPY_OVERLENGTH of buffer room to allow for overreads.
* This MUST not write more than our maxBlockSize beyond dst, because in
* streaming mode, that could overwrite part of our extDict window.
*/
if (splitImmediately) {
/* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */
dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH;
dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE;
} else {
/* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */
dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize;
dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize;
}
dctx->litBufferLocation = ZSTD_split;
assert(dctx->litBufferEnd <= (BYTE*)dst + expectedWriteSize);
}
}
/*! ZSTD_decodeLiteralsBlock() :
* Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored
* in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current
* block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being
* stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write.
*
* @return : nb of bytes read from src (< srcSize )
* note : symbol not declared but exposed for fullbench */
static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */
void* dst, size_t dstCapacity, const streaming_operation streaming)
{
DEBUGLOG(5, "ZSTD_decodeLiteralsBlock");
RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, "");
{ const BYTE* const istart = (const BYTE*) src;
SymbolEncodingType_e const litEncType = (SymbolEncodingType_e)(istart[0] & 3);
size_t const blockSizeMax = ZSTD_blockSizeMax(dctx);
switch(litEncType)
{
case set_repeat:
DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block");
RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, "");
ZSTD_FALLTHROUGH;
case set_compressed:
RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3");
{ size_t lhSize, litSize, litCSize;
U32 singleStream=0;
U32 const lhlCode = (istart[0] >> 2) & 3;
U32 const lhc = MEM_readLE32(istart);
size_t hufSuccess;
size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity);
int const flags = 0
| (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0)
| (dctx->disableHufAsm ? HUF_flags_disableAsm : 0);
switch(lhlCode)
{
case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */
/* 2 - 2 - 10 - 10 */
singleStream = !lhlCode;
lhSize = 3;
litSize = (lhc >> 4) & 0x3FF;
litCSize = (lhc >> 14) & 0x3FF;
break;
case 2:
/* 2 - 2 - 14 - 14 */
lhSize = 4;
litSize = (lhc >> 4) & 0x3FFF;
litCSize = lhc >> 18;
break;
case 3:
/* 2 - 2 - 18 - 18 */
lhSize = 5;
litSize = (lhc >> 4) & 0x3FFFF;
litCSize = (lhc >> 22) + ((size_t)istart[4] << 10);
break;
}
RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, "");
if (!singleStream)
RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong,
"Not enough literals (%zu) for the 4-streams mode (min %u)",
litSize, MIN_LITERALS_FOR_4_STREAMS);
RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, "");
RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, "");
ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0);
/* prefetch huffman table if cold */
if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) {
PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable));
}
if (litEncType==set_repeat) {
if (singleStream) {
hufSuccess = HUF_decompress1X_usingDTable(
dctx->litBuffer, litSize, istart+lhSize, litCSize,
dctx->HUFptr, flags);
} else {
assert(litSize >= MIN_LITERALS_FOR_4_STREAMS);
hufSuccess = HUF_decompress4X_usingDTable(
dctx->litBuffer, litSize, istart+lhSize, litCSize,
dctx->HUFptr, flags);
}
} else {
if (singleStream) {
#if defined(HUF_FORCE_DECOMPRESS_X2)
hufSuccess = HUF_decompress1X_DCtx_wksp(
dctx->entropy.hufTable, dctx->litBuffer, litSize,
istart+lhSize, litCSize, dctx->workspace,
sizeof(dctx->workspace), flags);
#else
hufSuccess = HUF_decompress1X1_DCtx_wksp(
dctx->entropy.hufTable, dctx->litBuffer, litSize,
istart+lhSize, litCSize, dctx->workspace,
sizeof(dctx->workspace), flags);
#endif
} else {
hufSuccess = HUF_decompress4X_hufOnly_wksp(
dctx->entropy.hufTable, dctx->litBuffer, litSize,
istart+lhSize, litCSize, dctx->workspace,
sizeof(dctx->workspace), flags);
}
}
if (dctx->litBufferLocation == ZSTD_split)
{
assert(litSize > ZSTD_LITBUFFEREXTRASIZE);
ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE);
ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE);
dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH;
dctx->litBufferEnd -= WILDCOPY_OVERLENGTH;
assert(dctx->litBufferEnd <= (BYTE*)dst + blockSizeMax);
}
RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, "");
dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize;
dctx->litEntropy = 1;
if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable;
return litCSize + lhSize;
}
case set_basic:
{ size_t litSize, lhSize;
U32 const lhlCode = ((istart[0]) >> 2) & 3;
size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity);
switch(lhlCode)
{
case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */
lhSize = 1;
litSize = istart[0] >> 3;
break;
case 1:
lhSize = 2;
litSize = MEM_readLE16(istart) >> 4;
break;
case 3:
lhSize = 3;
RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3");
litSize = MEM_readLE24(istart) >> 4;
break;
}
RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, "");
RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, "");
ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1);
if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */
RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, "");
if (dctx->litBufferLocation == ZSTD_split)
{
ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE);
ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE);
}
else
{
ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize);
}
dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize;
return lhSize+litSize;
}
/* direct reference into compressed stream */
dctx->litPtr = istart+lhSize;
dctx->litSize = litSize;
dctx->litBufferEnd = dctx->litPtr + litSize;
dctx->litBufferLocation = ZSTD_not_in_dst;
return lhSize+litSize;
}
case set_rle:
{ U32 const lhlCode = ((istart[0]) >> 2) & 3;
size_t litSize, lhSize;
size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity);
switch(lhlCode)
{
case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */
lhSize = 1;
litSize = istart[0] >> 3;
break;
case 1:
lhSize = 2;
RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3");
litSize = MEM_readLE16(istart) >> 4;
break;
case 3:
lhSize = 3;
RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4");
litSize = MEM_readLE24(istart) >> 4;
break;
}
RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, "");
RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, "");
ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1);
if (dctx->litBufferLocation == ZSTD_split)
{
ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE);
ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE);
}
else
{
ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize);
}
dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize;
return lhSize+1;
}
default:
RETURN_ERROR(corruption_detected, "impossible");
}
}
}
/* Hidden declaration for fullbench */
/* Hidden declaration for fullbench. */
size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx,
const void* src, size_t srcSize,
void* dst, size_t dstCapacity);
const void* src, size_t srcSize,
void* dst, size_t dstCapacity);
size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx,
const void* src, size_t srcSize,
void* dst, size_t dstCapacity)
const void* src, size_t srcSize,
void* dst, size_t dstCapacity)
{
dctx->isFrameDecompression = 0;
return ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, not_streaming);
}
/* Default FSE distribution tables.
* These are pre-calculated FSE decoding tables using default distributions as defined in specification :
* https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions
* They were generated programmatically with following method :
* - start from default distributions, present in /lib/common/zstd_internal.h
* - generate tables normally, using ZSTD_buildFSETable()
* - printout the content of tables
* - prettify output, report below, test with fuzzer to ensure it's correct */
/* Default FSE distribution table for Literal Lengths */
static const ZSTD_seqSymbol LL_defaultDTable[(1<<LL_DEFAULTNORMLOG)+1] = {
{ 1, 1, 1, LL_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
/* nextState, nbAddBits, nbBits, baseVal */
{ 0, 0, 4, 0}, { 16, 0, 4, 0},
{ 32, 0, 5, 1}, { 0, 0, 5, 3},
{ 0, 0, 5, 4}, { 0, 0, 5, 6},
{ 0, 0, 5, 7}, { 0, 0, 5, 9},
{ 0, 0, 5, 10}, { 0, 0, 5, 12},
{ 0, 0, 6, 14}, { 0, 1, 5, 16},
{ 0, 1, 5, 20}, { 0, 1, 5, 22},
{ 0, 2, 5, 28}, { 0, 3, 5, 32},
{ 0, 4, 5, 48}, { 32, 6, 5, 64},
{ 0, 7, 5, 128}, { 0, 8, 6, 256},
{ 0, 10, 6, 1024}, { 0, 12, 6, 4096},
{ 32, 0, 4, 0}, { 0, 0, 4, 1},
{ 0, 0, 5, 2}, { 32, 0, 5, 4},
{ 0, 0, 5, 5}, { 32, 0, 5, 7},
{ 0, 0, 5, 8}, { 32, 0, 5, 10},
{ 0, 0, 5, 11}, { 0, 0, 6, 13},
{ 32, 1, 5, 16}, { 0, 1, 5, 18},
{ 32, 1, 5, 22}, { 0, 2, 5, 24},
{ 32, 3, 5, 32}, { 0, 3, 5, 40},
{ 0, 6, 4, 64}, { 16, 6, 4, 64},
{ 32, 7, 5, 128}, { 0, 9, 6, 512},
{ 0, 11, 6, 2048}, { 48, 0, 4, 0},
{ 16, 0, 4, 1}, { 32, 0, 5, 2},
{ 32, 0, 5, 3}, { 32, 0, 5, 5},
{ 32, 0, 5, 6}, { 32, 0, 5, 8},
{ 32, 0, 5, 9}, { 32, 0, 5, 11},
{ 32, 0, 5, 12}, { 0, 0, 6, 15},
{ 32, 1, 5, 18}, { 32, 1, 5, 20},
{ 32, 2, 5, 24}, { 32, 2, 5, 28},
{ 32, 3, 5, 40}, { 32, 4, 5, 48},
{ 0, 16, 6,65536}, { 0, 15, 6,32768},
{ 0, 14, 6,16384}, { 0, 13, 6, 8192},
}; /* LL_defaultDTable */
/* Default FSE distribution table for Offset Codes */
static const ZSTD_seqSymbol OF_defaultDTable[(1<<OF_DEFAULTNORMLOG)+1] = {
{ 1, 1, 1, OF_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
/* nextState, nbAddBits, nbBits, baseVal */
{ 0, 0, 5, 0}, { 0, 6, 4, 61},
{ 0, 9, 5, 509}, { 0, 15, 5,32765},
{ 0, 21, 5,2097149}, { 0, 3, 5, 5},
{ 0, 7, 4, 125}, { 0, 12, 5, 4093},
{ 0, 18, 5,262141}, { 0, 23, 5,8388605},
{ 0, 5, 5, 29}, { 0, 8, 4, 253},
{ 0, 14, 5,16381}, { 0, 20, 5,1048573},
{ 0, 2, 5, 1}, { 16, 7, 4, 125},
{ 0, 11, 5, 2045}, { 0, 17, 5,131069},
{ 0, 22, 5,4194301}, { 0, 4, 5, 13},
{ 16, 8, 4, 253}, { 0, 13, 5, 8189},
{ 0, 19, 5,524285}, { 0, 1, 5, 1},
{ 16, 6, 4, 61}, { 0, 10, 5, 1021},
{ 0, 16, 5,65533}, { 0, 28, 5,268435453},
{ 0, 27, 5,134217725}, { 0, 26, 5,67108861},
{ 0, 25, 5,33554429}, { 0, 24, 5,16777213},
}; /* OF_defaultDTable */
/* Default FSE distribution table for Match Lengths */
static const ZSTD_seqSymbol ML_defaultDTable[(1<<ML_DEFAULTNORMLOG)+1] = {
{ 1, 1, 1, ML_DEFAULTNORMLOG}, /* header : fastMode, tableLog */
/* nextState, nbAddBits, nbBits, baseVal */
{ 0, 0, 6, 3}, { 0, 0, 4, 4},
{ 32, 0, 5, 5}, { 0, 0, 5, 6},
{ 0, 0, 5, 8}, { 0, 0, 5, 9},
{ 0, 0, 5, 11}, { 0, 0, 6, 13},
{ 0, 0, 6, 16}, { 0, 0, 6, 19},
{ 0, 0, 6, 22}, { 0, 0, 6, 25},
{ 0, 0, 6, 28}, { 0, 0, 6, 31},
{ 0, 0, 6, 34}, { 0, 1, 6, 37},
{ 0, 1, 6, 41}, { 0, 2, 6, 47},
{ 0, 3, 6, 59}, { 0, 4, 6, 83},
{ 0, 7, 6, 131}, { 0, 9, 6, 515},
{ 16, 0, 4, 4}, { 0, 0, 4, 5},
{ 32, 0, 5, 6}, { 0, 0, 5, 7},
{ 32, 0, 5, 9}, { 0, 0, 5, 10},
{ 0, 0, 6, 12}, { 0, 0, 6, 15},
{ 0, 0, 6, 18}, { 0, 0, 6, 21},
{ 0, 0, 6, 24}, { 0, 0, 6, 27},
{ 0, 0, 6, 30}, { 0, 0, 6, 33},
{ 0, 1, 6, 35}, { 0, 1, 6, 39},
{ 0, 2, 6, 43}, { 0, 3, 6, 51},
{ 0, 4, 6, 67}, { 0, 5, 6, 99},
{ 0, 8, 6, 259}, { 32, 0, 4, 4},
{ 48, 0, 4, 4}, { 16, 0, 4, 5},
{ 32, 0, 5, 7}, { 32, 0, 5, 8},
{ 32, 0, 5, 10}, { 32, 0, 5, 11},
{ 0, 0, 6, 14}, { 0, 0, 6, 17},
{ 0, 0, 6, 20}, { 0, 0, 6, 23},
{ 0, 0, 6, 26}, { 0, 0, 6, 29},
{ 0, 0, 6, 32}, { 0, 16, 6,65539},
{ 0, 15, 6,32771}, { 0, 14, 6,16387},
{ 0, 13, 6, 8195}, { 0, 12, 6, 4099},
{ 0, 11, 6, 2051}, { 0, 10, 6, 1027},
}; /* ML_defaultDTable */
static void ZSTD_buildSeqTable_rle(ZSTD_seqSymbol* dt, U32 baseValue, U8 nbAddBits)
{
void* ptr = dt;
ZSTD_seqSymbol_header* const DTableH = (ZSTD_seqSymbol_header*)ptr;
ZSTD_seqSymbol* const cell = dt + 1;
DTableH->tableLog = 0;
DTableH->fastMode = 0;
cell->nbBits = 0;
cell->nextState = 0;
assert(nbAddBits < 255);
cell->nbAdditionalBits = nbAddBits;
cell->baseValue = baseValue;
}
/* ZSTD_buildFSETable() :
* generate FSE decoding table for one symbol (ll, ml or off)
* cannot fail if input is valid =>
* all inputs are presumed validated at this stage */
FORCE_INLINE_TEMPLATE
void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt,
const short* normalizedCounter, unsigned maxSymbolValue,
const U32* baseValue, const U8* nbAdditionalBits,
unsigned tableLog, void* wksp, size_t wkspSize)
{
ZSTD_seqSymbol* const tableDecode = dt+1;
U32 const maxSV1 = maxSymbolValue + 1;
U32 const tableSize = 1 << tableLog;
U16* symbolNext = (U16*)wksp;
BYTE* spread = (BYTE*)(symbolNext + MaxSeq + 1);
U32 highThreshold = tableSize - 1;
/* Sanity Checks */
assert(maxSymbolValue <= MaxSeq);
assert(tableLog <= MaxFSELog);
assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE);
(void)wkspSize;
/* Init, lay down lowprob symbols */
{ ZSTD_seqSymbol_header DTableH;
DTableH.tableLog = tableLog;
DTableH.fastMode = 1;
{ S16 const largeLimit= (S16)(1 << (tableLog-1));
U32 s;
for (s=0; s<maxSV1; s++) {
if (normalizedCounter[s]==-1) {
tableDecode[highThreshold--].baseValue = s;
symbolNext[s] = 1;
} else {
if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
assert(normalizedCounter[s]>=0);
symbolNext[s] = (U16)normalizedCounter[s];
} } }
ZSTD_memcpy(dt, &DTableH, sizeof(DTableH));
}
/* Spread symbols */
assert(tableSize <= 512);
/* Specialized symbol spreading for the case when there are
* no low probability (-1 count) symbols. When compressing
* small blocks we avoid low probability symbols to hit this
* case, since header decoding speed matters more.
*/
if (highThreshold == tableSize - 1) {
size_t const tableMask = tableSize-1;
size_t const step = FSE_TABLESTEP(tableSize);
/* First lay down the symbols in order.
* We use a uint64_t to lay down 8 bytes at a time. This reduces branch
* misses since small blocks generally have small table logs, so nearly
* all symbols have counts <= 8. We ensure we have 8 bytes at the end of
* our buffer to handle the over-write.
*/
{
U64 const add = 0x0101010101010101ull;
size_t pos = 0;
U64 sv = 0;
U32 s;
for (s=0; s<maxSV1; ++s, sv += add) {
int i;
int const n = normalizedCounter[s];
MEM_write64(spread + pos, sv);
for (i = 8; i < n; i += 8) {
MEM_write64(spread + pos + i, sv);
}
assert(n>=0);
pos += (size_t)n;
}
}
/* Now we spread those positions across the table.
* The benefit of doing it in two stages is that we avoid the
* variable size inner loop, which caused lots of branch misses.
* Now we can run through all the positions without any branch misses.
* We unroll the loop twice, since that is what empirically worked best.
*/
{
size_t position = 0;
size_t s;
size_t const unroll = 2;
assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */
for (s = 0; s < (size_t)tableSize; s += unroll) {
size_t u;
for (u = 0; u < unroll; ++u) {
size_t const uPosition = (position + (u * step)) & tableMask;
tableDecode[uPosition].baseValue = spread[s + u];
}
position = (position + (unroll * step)) & tableMask;
}
assert(position == 0);
}
} else {
U32 const tableMask = tableSize-1;
U32 const step = FSE_TABLESTEP(tableSize);
U32 s, position = 0;
for (s=0; s<maxSV1; s++) {
int i;
int const n = normalizedCounter[s];
for (i=0; i<n; i++) {
tableDecode[position].baseValue = s;
position = (position + step) & tableMask;
while (UNLIKELY(position > highThreshold)) position = (position + step) & tableMask; /* lowprob area */
} }
assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
}
/* Build Decoding table */
{
U32 u;
for (u=0; u<tableSize; u++) {
U32 const symbol = tableDecode[u].baseValue;
U32 const nextState = symbolNext[symbol]++;
tableDecode[u].nbBits = (BYTE) (tableLog - ZSTD_highbit32(nextState) );
tableDecode[u].nextState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
assert(nbAdditionalBits[symbol] < 255);
tableDecode[u].nbAdditionalBits = nbAdditionalBits[symbol];
tableDecode[u].baseValue = baseValue[symbol];
}
}
}
/* Avoids the FORCE_INLINE of the _body() function. */
static void ZSTD_buildFSETable_body_default(ZSTD_seqSymbol* dt,
const short* normalizedCounter, unsigned maxSymbolValue,
const U32* baseValue, const U8* nbAdditionalBits,
unsigned tableLog, void* wksp, size_t wkspSize)
{
ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue,
baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
}
#if DYNAMIC_BMI2
BMI2_TARGET_ATTRIBUTE static void ZSTD_buildFSETable_body_bmi2(ZSTD_seqSymbol* dt,
const short* normalizedCounter, unsigned maxSymbolValue,
const U32* baseValue, const U8* nbAdditionalBits,
unsigned tableLog, void* wksp, size_t wkspSize)
{
ZSTD_buildFSETable_body(dt, normalizedCounter, maxSymbolValue,
baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
}
#endif
void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
const short* normalizedCounter, unsigned maxSymbolValue,
const U32* baseValue, const U8* nbAdditionalBits,
unsigned tableLog, void* wksp, size_t wkspSize, int bmi2)
{
#if DYNAMIC_BMI2
if (bmi2) {
ZSTD_buildFSETable_body_bmi2(dt, normalizedCounter, maxSymbolValue,
baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
return;
}
#endif
(void)bmi2;
ZSTD_buildFSETable_body_default(dt, normalizedCounter, maxSymbolValue,
baseValue, nbAdditionalBits, tableLog, wksp, wkspSize);
}
/*! ZSTD_buildSeqTable() :
* @return : nb bytes read from src,
* or an error code if it fails */
static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymbol** DTablePtr,
SymbolEncodingType_e type, unsigned max, U32 maxLog,
const void* src, size_t srcSize,
const U32* baseValue, const U8* nbAdditionalBits,
const ZSTD_seqSymbol* defaultTable, U32 flagRepeatTable,
int ddictIsCold, int nbSeq, U32* wksp, size_t wkspSize,
int bmi2)
{
switch(type)
{
case set_rle :
RETURN_ERROR_IF(!srcSize, srcSize_wrong, "");
RETURN_ERROR_IF((*(const BYTE*)src) > max, corruption_detected, "");
{ U32 const symbol = *(const BYTE*)src;
U32 const baseline = baseValue[symbol];
U8 const nbBits = nbAdditionalBits[symbol];
ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits);
}
*DTablePtr = DTableSpace;
return 1;
case set_basic :
*DTablePtr = defaultTable;
return 0;
case set_repeat:
RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, "");
/* prefetch FSE table if used */
if (ddictIsCold && (nbSeq > 24 /* heuristic */)) {
const void* const pStart = *DTablePtr;
size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog));
PREFETCH_AREA(pStart, pSize);
}
return 0;
case set_compressed :
{ unsigned tableLog;
S16 norm[MaxSeq+1];
size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize);
RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, "");
RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, "");
ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize, bmi2);
*DTablePtr = DTableSpace;
return headerSize;
}
default :
assert(0);
RETURN_ERROR(GENERIC, "impossible");
}
ZSTD_rustBlockCtx ctx = ZSTD_rust_block_context(dctx);
return ZSTD_rust_decodeLiteralsBlock_wrapper(
&ctx, src, srcSize, dst, dstCapacity);
}
size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
const void* src, size_t srcSize)
{
const BYTE* const istart = (const BYTE*)src;
const BYTE* const iend = istart + srcSize;
const BYTE* ip = istart;
int nbSeq;
DEBUGLOG(5, "ZSTD_decodeSeqHeaders");
/* check */
RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong, "");
/* SeqHead */
nbSeq = *ip++;
if (nbSeq > 0x7F) {
if (nbSeq == 0xFF) {
RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, "");
nbSeq = MEM_readLE16(ip) + LONGNBSEQ;
ip+=2;
} else {
RETURN_ERROR_IF(ip >= iend, srcSize_wrong, "");
nbSeq = ((nbSeq-0x80)<<8) + *ip++;
}
}
*nbSeqPtr = nbSeq;
if (nbSeq == 0) {
/* No sequence : section ends immediately */
RETURN_ERROR_IF(ip != iend, corruption_detected,
"extraneous data present in the Sequences section");
return (size_t)(ip - istart);
}
/* FSE table descriptors */
RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */
RETURN_ERROR_IF(*ip & 3, corruption_detected, ""); /* The last field, Reserved, must be all-zeroes. */
{ SymbolEncodingType_e const LLtype = (SymbolEncodingType_e)(*ip >> 6);
SymbolEncodingType_e const OFtype = (SymbolEncodingType_e)((*ip >> 4) & 3);
SymbolEncodingType_e const MLtype = (SymbolEncodingType_e)((*ip >> 2) & 3);
ip++;
/* Build DTables */
{ size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr,
LLtype, MaxLL, LLFSELog,
ip, iend-ip,
LL_base, LL_bits,
LL_defaultDTable, dctx->fseEntropy,
dctx->ddictIsCold, nbSeq,
dctx->workspace, sizeof(dctx->workspace),
ZSTD_DCtx_get_bmi2(dctx));
RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed");
ip += llhSize;
}
{ size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr,
OFtype, MaxOff, OffFSELog,
ip, iend-ip,
OF_base, OF_bits,
OF_defaultDTable, dctx->fseEntropy,
dctx->ddictIsCold, nbSeq,
dctx->workspace, sizeof(dctx->workspace),
ZSTD_DCtx_get_bmi2(dctx));
RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed");
ip += ofhSize;
}
{ size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr,
MLtype, MaxML, MLFSELog,
ip, iend-ip,
ML_base, ML_bits,
ML_defaultDTable, dctx->fseEntropy,
dctx->ddictIsCold, nbSeq,
dctx->workspace, sizeof(dctx->workspace),
ZSTD_DCtx_get_bmi2(dctx));
RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed");
ip += mlhSize;
}
}
return ip-istart;
ZSTD_rustBlockCtx ctx = ZSTD_rust_block_context(dctx);
return ZSTD_rust_decodeSeqHeaders(&ctx, nbSeqPtr, src, srcSize);
}
typedef struct {
size_t litLength;
size_t matchLength;
size_t offset;
} seq_t;
typedef struct {
size_t state;
const ZSTD_seqSymbol* table;
} ZSTD_fseState;
typedef struct {
BIT_DStream_t DStream;
ZSTD_fseState stateLL;
ZSTD_fseState stateOffb;
ZSTD_fseState stateML;
size_t prevOffset[ZSTD_REP_NUM];
} seqState_t;
/*! ZSTD_overlapCopy8() :
* Copies 8 bytes from ip to op and updates op and ip where ip <= op.
* If the offset is < 8 then the offset is spread to at least 8 bytes.
*
* Precondition: *ip <= *op
* Postcondition: *op - *op >= 8
*/
HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) {
assert(*ip <= *op);
if (offset < 8) {
/* close range match, overlap */
static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */
static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */
int const sub2 = dec64table[offset];
(*op)[0] = (*ip)[0];
(*op)[1] = (*ip)[1];
(*op)[2] = (*ip)[2];
(*op)[3] = (*ip)[3];
*ip += dec32table[offset];
ZSTD_copy4(*op+4, *ip);
*ip -= sub2;
} else {
ZSTD_copy8(*op, *ip);
}
*ip += 8;
*op += 8;
assert(*op - *ip >= 8);
}
/*! ZSTD_safecopy() :
* Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer
* and write up to 16 bytes past oend_w (op >= oend_w is allowed).
* This function is only called in the uncommon case where the sequence is near the end of the block. It
* should be fast for a single long sequence, but can be slow for several short sequences.
*
* @param ovtype controls the overlap detection
* - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart.
* - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart.
* The src buffer must be before the dst buffer.
*/
static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) {
ptrdiff_t const diff = op - ip;
BYTE* const oend = op + length;
assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) ||
(ovtype == ZSTD_overlap_src_before_dst && diff >= 0));
if (length < 8) {
/* Handle short lengths. */
while (op < oend) *op++ = *ip++;
return;
}
if (ovtype == ZSTD_overlap_src_before_dst) {
/* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */
assert(length >= 8);
ZSTD_overlapCopy8(&op, &ip, diff);
length -= 8;
assert(op - ip >= 8);
assert(op <= oend);
}
if (oend <= oend_w) {
/* No risk of overwrite. */
ZSTD_wildcopy(op, ip, length, ovtype);
return;
}
if (op <= oend_w) {
/* Wildcopy until we get close to the end. */
assert(oend > oend_w);
ZSTD_wildcopy(op, ip, oend_w - op, ovtype);
ip += oend_w - op;
op += oend_w - op;
}
/* Handle the leftovers. */
while (op < oend) *op++ = *ip++;
}
/* ZSTD_safecopyDstBeforeSrc():
* This version allows overlap with dst before src, or handles the non-overlap case with dst after src
* Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */
static void ZSTD_safecopyDstBeforeSrc(BYTE* op, const BYTE* ip, ptrdiff_t length) {
ptrdiff_t const diff = op - ip;
BYTE* const oend = op + length;
if (length < 8 || diff > -8) {
/* Handle short lengths, close overlaps, and dst not before src. */
while (op < oend) *op++ = *ip++;
return;
}
if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) {
ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap);
ip += oend - WILDCOPY_OVERLENGTH - op;
op += oend - WILDCOPY_OVERLENGTH - op;
}
/* Handle the leftovers. */
while (op < oend) *op++ = *ip++;
}
/* ZSTD_execSequenceEnd():
* This version handles cases that are near the end of the output buffer. It requires
* more careful checks to make sure there is no overflow. By separating out these hard
* and unlikely cases, we can speed up the common cases.
*
* NOTE: This function needs to be fast for a single long sequence, but doesn't need
* to be optimized for many small sequences, since those fall into ZSTD_execSequence().
*/
FORCE_NOINLINE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_execSequenceEnd(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const streaming_operation streaming)
{
BYTE* const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
const BYTE* match = oLitEnd - sequence.offset;
BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;
/* bounds checks : careful of address space overflow in 32-bit mode */
RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer");
RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer");
assert(op < op + sequenceLength);
assert(oLitEnd < op + sequenceLength);
/* copy literals */
ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap);
op = oLitEnd;
*litPtr = iLitEnd;
/* copy Match */
if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
/* offset beyond prefix */
RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, "");
match = dictEnd - (prefixStart - match);
if (match + sequence.matchLength <= dictEnd) {
ZSTD_memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength;
}
/* span extDict & currentPrefixSegment */
{ size_t const length1 = dictEnd - match;
ZSTD_memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = prefixStart;
}
}
ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst);
return sequenceLength;
ZSTD_rustBlockCtx ctx = ZSTD_rust_block_context(dctx);
return ZSTD_rust_decompressBlock_internal(&ctx, dst, dstCapacity, src, srcSize,
(int)streaming);
}
/* ZSTD_execSequenceEndSplitLitBuffer():
* This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case.
*/
FORCE_NOINLINE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op,
BYTE* const oend, const BYTE* const oend_w, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
const BYTE* match = oLitEnd - sequence.offset;
/* bounds checks : careful of address space overflow in 32-bit mode */
RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer");
RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer");
assert(op < op + sequenceLength);
assert(oLitEnd < op + sequenceLength);
/* copy literals */
RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer");
ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength);
op = oLitEnd;
*litPtr = iLitEnd;
/* copy Match */
if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
/* offset beyond prefix */
RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, "");
match = dictEnd - (prefixStart - match);
if (match + sequence.matchLength <= dictEnd) {
ZSTD_memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength;
}
/* span extDict & currentPrefixSegment */
{ size_t const length1 = dictEnd - match;
ZSTD_memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = prefixStart;
}
}
ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst);
return sequenceLength;
}
HINT_INLINE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
const BYTE* match = oLitEnd - sequence.offset;
assert(op != NULL /* Precondition */);
assert(oend_w < oend /* No underflow */);
#if defined(__aarch64__)
/* prefetch sequence starting from match that will be used for copy later */
PREFETCH_L1(match);
#endif
/* Handle edge cases in a slow path:
* - Read beyond end of literals
* - Match end is within WILDCOPY_OVERLIMIT of oend
* - 32-bit mode and the match length overflows
*/
if (UNLIKELY(
iLitEnd > litLimit ||
oMatchEnd > oend_w ||
(MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH)))
return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
/* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */
assert(op <= oLitEnd /* No overflow */);
assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */);
assert(oMatchEnd <= oend /* No underflow */);
assert(iLitEnd <= litLimit /* Literal length is in bounds */);
assert(oLitEnd <= oend_w /* Can wildcopy literals */);
assert(oMatchEnd <= oend_w /* Can wildcopy matches */);
/* Copy Literals:
* Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.
* We likely don't need the full 32-byte wildcopy.
*/
assert(WILDCOPY_OVERLENGTH >= 16);
ZSTD_copy16(op, (*litPtr));
if (UNLIKELY(sequence.litLength > 16)) {
ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap);
}
op = oLitEnd;
*litPtr = iLitEnd; /* update for next sequence */
/* Copy Match */
if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
/* offset beyond prefix -> go into extDict */
RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, "");
match = dictEnd + (match - prefixStart);
if (match + sequence.matchLength <= dictEnd) {
ZSTD_memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength;
}
/* span extDict & currentPrefixSegment */
{ size_t const length1 = dictEnd - match;
ZSTD_memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = prefixStart;
}
}
/* Match within prefix of 1 or more bytes */
assert(op <= oMatchEnd);
assert(oMatchEnd <= oend_w);
assert(match >= prefixStart);
assert(sequence.matchLength >= 1);
/* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy
* without overlap checking.
*/
if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) {
/* We bet on a full wildcopy for matches, since we expect matches to be
* longer than literals (in general). In silesia, ~10% of matches are longer
* than 16 bytes.
*/
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
return sequenceLength;
}
assert(sequence.offset < WILDCOPY_VECLEN);
/* Copy 8 bytes and spread the offset to be >= 8. */
ZSTD_overlapCopy8(&op, &match, sequence.offset);
/* If the match length is > 8 bytes, then continue with the wildcopy. */
if (sequence.matchLength > 8) {
assert(op < oMatchEnd);
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst);
}
return sequenceLength;
}
HINT_INLINE
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op,
BYTE* const oend, const BYTE* const oend_w, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
const BYTE* match = oLitEnd - sequence.offset;
assert(op != NULL /* Precondition */);
assert(oend_w < oend /* No underflow */);
/* Handle edge cases in a slow path:
* - Read beyond end of literals
* - Match end is within WILDCOPY_OVERLIMIT of oend
* - 32-bit mode and the match length overflows
*/
if (UNLIKELY(
iLitEnd > litLimit ||
oMatchEnd > oend_w ||
(MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH)))
return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
/* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */
assert(op <= oLitEnd /* No overflow */);
assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */);
assert(oMatchEnd <= oend /* No underflow */);
assert(iLitEnd <= litLimit /* Literal length is in bounds */);
assert(oLitEnd <= oend_w /* Can wildcopy literals */);
assert(oMatchEnd <= oend_w /* Can wildcopy matches */);
/* Copy Literals:
* Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.
* We likely don't need the full 32-byte wildcopy.
*/
assert(WILDCOPY_OVERLENGTH >= 16);
ZSTD_copy16(op, (*litPtr));
if (UNLIKELY(sequence.litLength > 16)) {
ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap);
}
op = oLitEnd;
*litPtr = iLitEnd; /* update for next sequence */
/* Copy Match */
if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
/* offset beyond prefix -> go into extDict */
RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, "");
match = dictEnd + (match - prefixStart);
if (match + sequence.matchLength <= dictEnd) {
ZSTD_memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength;
}
/* span extDict & currentPrefixSegment */
{ size_t const length1 = dictEnd - match;
ZSTD_memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = prefixStart;
} }
/* Match within prefix of 1 or more bytes */
assert(op <= oMatchEnd);
assert(oMatchEnd <= oend_w);
assert(match >= prefixStart);
assert(sequence.matchLength >= 1);
/* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy
* without overlap checking.
*/
if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) {
/* We bet on a full wildcopy for matches, since we expect matches to be
* longer than literals (in general). In silesia, ~10% of matches are longer
* than 16 bytes.
*/
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
return sequenceLength;
}
assert(sequence.offset < WILDCOPY_VECLEN);
/* Copy 8 bytes and spread the offset to be >= 8. */
ZSTD_overlapCopy8(&op, &match, sequence.offset);
/* If the match length is > 8 bytes, then continue with the wildcopy. */
if (sequence.matchLength > 8) {
assert(op < oMatchEnd);
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst);
}
return sequenceLength;
}
static void
ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt)
{
const void* ptr = dt;
const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr;
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits",
(U32)DStatePtr->state, DTableH->tableLog);
BIT_reloadDStream(bitD);
DStatePtr->table = dt + 1;
}
FORCE_INLINE_TEMPLATE void
ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits)
{
size_t const lowBits = BIT_readBits(bitD, nbBits);
DStatePtr->state = nextState + lowBits;
}
/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum
* offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32
* bits before reloading. This value is the maximum number of bytes we read
* after reloading when we are decoding long offsets.
*/
#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \
(ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \
? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \
: 0)
typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e;
/**
* ZSTD_decodeSequence():
* @p longOffsets : tells the decoder to reload more bit while decoding large offsets
* only used in 32-bit mode
* @return : Sequence (litL + matchL + offset)
*/
FORCE_INLINE_TEMPLATE seq_t
ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const int isLastSeq)
{
seq_t seq;
/*
* ZSTD_seqSymbol is a 64 bits wide structure.
* It can be loaded in one operation
* and its fields extracted by simply shifting or bit-extracting on aarch64.
* GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh
* operations that cause performance drop. This can be avoided by using this
* ZSTD_memcpy hack.
*/
#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__))
ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS;
ZSTD_seqSymbol* const llDInfo = &llDInfoS;
ZSTD_seqSymbol* const mlDInfo = &mlDInfoS;
ZSTD_seqSymbol* const ofDInfo = &ofDInfoS;
ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol));
ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol));
ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol));
#else
const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state;
const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state;
const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state;
#endif
seq.matchLength = mlDInfo->baseValue;
seq.litLength = llDInfo->baseValue;
{ U32 const ofBase = ofDInfo->baseValue;
BYTE const llBits = llDInfo->nbAdditionalBits;
BYTE const mlBits = mlDInfo->nbAdditionalBits;
BYTE const ofBits = ofDInfo->nbAdditionalBits;
BYTE const totalBits = llBits+mlBits+ofBits;
U16 const llNext = llDInfo->nextState;
U16 const mlNext = mlDInfo->nextState;
U16 const ofNext = ofDInfo->nextState;
U32 const llnbBits = llDInfo->nbBits;
U32 const mlnbBits = mlDInfo->nbBits;
U32 const ofnbBits = ofDInfo->nbBits;
assert(llBits <= MaxLLBits);
assert(mlBits <= MaxMLBits);
assert(ofBits <= MaxOff);
/*
* As gcc has better branch and block analyzers, sometimes it is only
* valuable to mark likeliness for clang, it gives around 3-4% of
* performance.
*/
/* sequence */
{ size_t offset;
if (ofBits > 1) {
ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32);
ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits);
if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) {
/* Always read extra bits, this keeps the logic simple,
* avoids branches, and avoids accidentally reading 0 bits.
*/
U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32;
offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
BIT_reloadDStream(&seqState->DStream);
offset += BIT_readBitsFast(&seqState->DStream, extraBits);
} else {
offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */
if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
}
seqState->prevOffset[2] = seqState->prevOffset[1];
seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[0] = offset;
} else {
U32 const ll0 = (llDInfo->baseValue == 0);
if (LIKELY((ofBits == 0))) {
offset = seqState->prevOffset[ll0];
seqState->prevOffset[1] = seqState->prevOffset[!ll0];
seqState->prevOffset[0] = offset;
} else {
offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1);
{ size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
temp -= !temp; /* 0 is not valid: input corrupted => force offset to -1 => corruption detected at execSequence */
if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[0] = offset = temp;
} } }
seq.offset = offset;
}
if (mlBits > 0)
seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/);
if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32))
BIT_reloadDStream(&seqState->DStream);
if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog)))
BIT_reloadDStream(&seqState->DStream);
/* Ensure there are enough bits to read the rest of data in 64-bit mode. */
ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64);
if (llBits > 0)
seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/);
if (MEM_32bits())
BIT_reloadDStream(&seqState->DStream);
DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u",
(U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
if (!isLastSeq) {
/* don't update FSE state for last Sequence */
ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */
ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */
if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */
ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */
BIT_reloadDStream(&seqState->DStream);
}
}
return seq;
}
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
#if DEBUGLEVEL >= 1
static int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd)
{
size_t const windowSize = dctx->fParams.windowSize;
/* No dictionary used. */
if (dctx->dictContentEndForFuzzing == NULL) return 0;
/* Dictionary is our prefix. */
if (prefixStart == dctx->dictContentBeginForFuzzing) return 1;
/* Dictionary is not our ext-dict. */
if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0;
/* Dictionary is not within our window size. */
if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0;
/* Dictionary is active. */
return 1;
}
#endif
static void ZSTD_assertValidSequence(
ZSTD_DCtx const* dctx,
BYTE const* op, BYTE const* oend,
seq_t const seq,
BYTE const* prefixStart, BYTE const* virtualStart)
{
#if DEBUGLEVEL >= 1
if (dctx->isFrameDecompression) {
size_t const windowSize = dctx->fParams.windowSize;
size_t const sequenceSize = seq.litLength + seq.matchLength;
BYTE const* const oLitEnd = op + seq.litLength;
DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u",
(U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
assert(op <= oend);
assert((size_t)(oend - op) >= sequenceSize);
assert(sequenceSize <= ZSTD_blockSizeMax(dctx));
if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) {
size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing);
/* Offset must be within the dictionary. */
assert(seq.offset <= (size_t)(oLitEnd - virtualStart));
assert(seq.offset <= windowSize + dictSize);
} else {
/* Offset must be within our window. */
assert(seq.offset <= windowSize);
}
}
#else
(void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart;
#endif
}
#endif
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
FORCE_INLINE_TEMPLATE size_t
DONT_VECTORIZE
ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
const BYTE* ip = (const BYTE*)seqStart;
const BYTE* const iend = ip + seqSize;
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, maxDstSize);
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
const BYTE* litBufferEnd = dctx->litBufferEnd;
const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
const BYTE* const vBase = (const BYTE*) (dctx->virtualStart);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer (%i seqs)", nbSeq);
/* Literals are split between internal buffer & output buffer */
if (nbSeq) {
seqState_t seqState;
dctx->fseEntropy = 1;
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
RETURN_ERROR_IF(
ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
corruption_detected, "");
ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
assert(dst != NULL);
ZSTD_STATIC_ASSERT(
BIT_DStream_unfinished < BIT_DStream_completed &&
BIT_DStream_endOfBuffer < BIT_DStream_completed &&
BIT_DStream_completed < BIT_DStream_overflow);
/* decompress without overrunning litPtr begins */
{ seq_t sequence = {0,0,0}; /* some static analyzer believe that @sequence is not initialized (it necessarily is, since for(;;) loop as at least one iteration) */
/* Align the decompression loop to 32 + 16 bytes.
*
* zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression
* speed swings based on the alignment of the decompression loop. This
* performance swing is caused by parts of the decompression loop falling
* out of the DSB. The entire decompression loop should fit in the DSB,
* when it can't we get much worse performance. You can measure if you've
* hit the good case or the bad case with this perf command for some
* compressed file test.zst:
*
* perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \
* -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst
*
* If you see most cycles served out of the MITE you've hit the bad case.
* If you see most cycles served out of the DSB you've hit the good case.
* If it is pretty even then you may be in an okay case.
*
* This issue has been reproduced on the following CPUs:
* - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9
* Use Instruments->Counters to get DSB/MITE cycles.
* I never got performance swings, but I was able to
* go from the good case of mostly DSB to half of the
* cycles served from MITE.
* - Coffeelake: Intel i9-9900k
* - Coffeelake: Intel i7-9700k
*
* I haven't been able to reproduce the instability or DSB misses on any
* of the following CPUS:
* - Haswell
* - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH
* - Skylake
*
* Alignment is done for each of the three major decompression loops:
* - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer
* - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer
* - ZSTD_decompressSequences_body
* Alignment choices are made to minimize large swings on bad cases and influence on performance
* from changes external to this code, rather than to overoptimize on the current commit.
*
* If you are seeing performance stability this script can help test.
* It tests on 4 commits in zstd where I saw performance change.
*
* https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4
*/
#if defined(__GNUC__) && defined(__x86_64__)
__asm__(".p2align 6");
# if __GNUC__ >= 7
/* good for gcc-7, gcc-9, and gcc-11 */
__asm__("nop");
__asm__(".p2align 5");
__asm__("nop");
__asm__(".p2align 4");
# if __GNUC__ == 8 || __GNUC__ == 10
/* good for gcc-8 and gcc-10 */
__asm__("nop");
__asm__(".p2align 3");
# endif
# endif
#endif
/* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */
for ( ; nbSeq; nbSeq--) {
sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1);
if (litPtr + sequence.litLength > dctx->litBufferEnd) break;
{ size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
#endif
if (UNLIKELY(ZSTD_isError(oneSeqSize)))
return oneSeqSize;
DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
op += oneSeqSize;
} }
DEBUGLOG(6, "reached: (litPtr + sequence.litLength > dctx->litBufferEnd)");
/* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */
if (nbSeq > 0) {
const size_t leftoverLit = dctx->litBufferEnd - litPtr;
DEBUGLOG(6, "There are %i sequences left, and %zu/%zu literals left in buffer", nbSeq, leftoverLit, sequence.litLength);
if (leftoverLit) {
RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
sequence.litLength -= leftoverLit;
op += leftoverLit;
}
litPtr = dctx->litExtraBuffer;
litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
dctx->litBufferLocation = ZSTD_not_in_dst;
{ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
#endif
if (UNLIKELY(ZSTD_isError(oneSeqSize)))
return oneSeqSize;
DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
op += oneSeqSize;
}
nbSeq--;
}
}
if (nbSeq > 0) {
/* there is remaining lit from extra buffer */
#if defined(__GNUC__) && defined(__x86_64__)
__asm__(".p2align 6");
__asm__("nop");
# if __GNUC__ != 7
/* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */
__asm__(".p2align 4");
__asm__("nop");
__asm__(".p2align 3");
# elif __GNUC__ >= 11
__asm__(".p2align 3");
# else
__asm__(".p2align 5");
__asm__("nop");
__asm__(".p2align 3");
# endif
#endif
for ( ; nbSeq ; nbSeq--) {
seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1);
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
#endif
if (UNLIKELY(ZSTD_isError(oneSeqSize)))
return oneSeqSize;
DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
op += oneSeqSize;
}
}
/* check if reached exact end */
DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq);
RETURN_ERROR_IF(nbSeq, corruption_detected, "");
DEBUGLOG(5, "bitStream : start=%p, ptr=%p, bitsConsumed=%u", seqState.DStream.start, seqState.DStream.ptr, seqState.DStream.bitsConsumed);
RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, "");
/* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
}
/* last literal segment */
if (dctx->litBufferLocation == ZSTD_split) {
/* split hasn't been reached yet, first get dst then copy litExtraBuffer */
size_t const lastLLSize = (size_t)(litBufferEnd - litPtr);
DEBUGLOG(6, "copy last literals from segment : %u", (U32)lastLLSize);
RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
if (op != NULL) {
ZSTD_memmove(op, litPtr, lastLLSize);
op += lastLLSize;
}
litPtr = dctx->litExtraBuffer;
litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
dctx->litBufferLocation = ZSTD_not_in_dst;
}
/* copy last literals from internal buffer */
{ size_t const lastLLSize = (size_t)(litBufferEnd - litPtr);
DEBUGLOG(6, "copy last literals from internal buffer : %u", (U32)lastLLSize);
RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
if (op != NULL) {
ZSTD_memcpy(op, litPtr, lastLLSize);
op += lastLLSize;
} }
DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart));
return (size_t)(op - ostart);
}
FORCE_INLINE_TEMPLATE size_t
DONT_VECTORIZE
ZSTD_decompressSequences_body(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
const BYTE* ip = (const BYTE*)seqStart;
const BYTE* const iend = ip + seqSize;
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ZSTD_maybeNullPtrAdd(ostart, maxDstSize) : dctx->litBuffer;
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
const BYTE* const litEnd = litPtr + dctx->litSize;
const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart);
const BYTE* const vBase = (const BYTE*)(dctx->virtualStart);
const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd);
DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq);
/* Regen sequences */
if (nbSeq) {
seqState_t seqState;
dctx->fseEntropy = 1;
{ U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
RETURN_ERROR_IF(
ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)),
corruption_detected, "");
ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
assert(dst != NULL);
#if defined(__GNUC__) && defined(__x86_64__)
__asm__(".p2align 6");
__asm__("nop");
# if __GNUC__ >= 7
__asm__(".p2align 5");
__asm__("nop");
__asm__(".p2align 3");
# else
__asm__(".p2align 4");
__asm__("nop");
__asm__(".p2align 3");
# endif
#endif
for ( ; nbSeq ; nbSeq--) {
seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1);
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
#endif
if (UNLIKELY(ZSTD_isError(oneSeqSize)))
return oneSeqSize;
DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
op += oneSeqSize;
}
/* check if reached exact end */
assert(nbSeq == 0);
RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, "");
/* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
}
/* last literal segment */
{ size_t const lastLLSize = (size_t)(litEnd - litPtr);
DEBUGLOG(6, "copy last literals : %u", (U32)lastLLSize);
RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
if (op != NULL) {
ZSTD_memcpy(op, litPtr, lastLLSize);
op += lastLLSize;
} }
DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart));
return (size_t)(op - ostart);
}
static size_t
ZSTD_decompressSequences_default(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
static size_t
ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
FORCE_INLINE_TEMPLATE
size_t ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence,
const BYTE* const prefixStart, const BYTE* const dictEnd)
{
prefetchPos += sequence.litLength;
{ const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart;
/* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted.
* No consequence though : memory address is only used for prefetching, not for dereferencing */
const BYTE* const match = ZSTD_wrappedPtrSub(ZSTD_wrappedPtrAdd(matchBase, prefetchPos), sequence.offset);
PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */
}
return prefetchPos + sequence.matchLength;
}
/* This decoding function employs prefetching
* to reduce latency impact of cache misses.
* It's generally employed when block contains a significant portion of long-distance matches
* or when coupled with a "cold" dictionary */
FORCE_INLINE_TEMPLATE size_t
ZSTD_decompressSequencesLong_body(
ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
const BYTE* ip = (const BYTE*)seqStart;
const BYTE* const iend = ip + seqSize;
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ZSTD_maybeNullPtrAdd(ostart, maxDstSize);
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
const BYTE* litBufferEnd = dctx->litBufferEnd;
const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
/* Regen sequences */
if (nbSeq) {
#define STORED_SEQS 8
#define STORED_SEQS_MASK (STORED_SEQS-1)
#define ADVANCED_SEQS STORED_SEQS
seq_t sequences[STORED_SEQS];
int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS);
seqState_t seqState;
int seqNb;
size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */
dctx->fseEntropy = 1;
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
assert(dst != NULL);
assert(iend >= ip);
RETURN_ERROR_IF(
ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)),
corruption_detected, "");
ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
/* prepare in advance */
for (seqNb=0; seqNb<seqAdvance; seqNb++) {
seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, seqNb == nbSeq-1);
prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
sequences[seqNb] = sequence;
}
/* decompress without stomping litBuffer */
for (; seqNb < nbSeq; seqNb++) {
seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset, seqNb == nbSeq-1);
if (dctx->litBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd) {
/* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */
const size_t leftoverLit = dctx->litBufferEnd - litPtr;
if (leftoverLit)
{
RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit;
op += leftoverLit;
}
litPtr = dctx->litExtraBuffer;
litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
dctx->litBufferLocation = ZSTD_not_in_dst;
{ size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart);
#endif
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
sequences[seqNb & STORED_SEQS_MASK] = sequence;
op += oneSeqSize;
} }
else
{
/* lit buffer is either wholly contained in first or second split, or not split at all*/
size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) :
ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart);
#endif
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
sequences[seqNb & STORED_SEQS_MASK] = sequence;
op += oneSeqSize;
}
}
RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, "");
/* finish queue */
seqNb -= seqAdvance;
for ( ; seqNb<nbSeq ; seqNb++) {
seq_t *sequence = &(sequences[seqNb&STORED_SEQS_MASK]);
if (dctx->litBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd) {
const size_t leftoverLit = dctx->litBufferEnd - litPtr;
if (leftoverLit) {
RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
sequence->litLength -= leftoverLit;
op += leftoverLit;
}
litPtr = dctx->litExtraBuffer;
litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
dctx->litBufferLocation = ZSTD_not_in_dst;
{ size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart);
#endif
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}
}
else
{
size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) :
ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart);
#endif
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}
}
/* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
}
/* last literal segment */
if (dctx->litBufferLocation == ZSTD_split) { /* first deplete literal buffer in dst, then copy litExtraBuffer */
size_t const lastLLSize = litBufferEnd - litPtr;
RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
if (op != NULL) {
ZSTD_memmove(op, litPtr, lastLLSize);
op += lastLLSize;
}
litPtr = dctx->litExtraBuffer;
litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
}
{ size_t const lastLLSize = litBufferEnd - litPtr;
RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
if (op != NULL) {
ZSTD_memmove(op, litPtr, lastLLSize);
op += lastLLSize;
}
}
return (size_t)(op - ostart);
}
static size_t
ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
#if DYNAMIC_BMI2
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
static BMI2_TARGET_ATTRIBUTE size_t
DONT_VECTORIZE
ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
static BMI2_TARGET_ATTRIBUTE size_t
DONT_VECTORIZE
ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
static BMI2_TARGET_ATTRIBUTE size_t
ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
#endif /* DYNAMIC_BMI2 */
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
static size_t
ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
DEBUGLOG(5, "ZSTD_decompressSequences");
#if DYNAMIC_BMI2
if (ZSTD_DCtx_get_bmi2(dctx)) {
return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif
return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
static size_t
ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer");
#if DYNAMIC_BMI2
if (ZSTD_DCtx_get_bmi2(dctx)) {
return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif
return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
/* ZSTD_decompressSequencesLong() :
* decompression function triggered when a minimum share of offsets is considered "long",
* aka out of cache.
* note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance".
* This function will try to mitigate main memory latency through the use of prefetching */
static size_t
ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset)
{
DEBUGLOG(5, "ZSTD_decompressSequencesLong");
#if DYNAMIC_BMI2
if (ZSTD_DCtx_get_bmi2(dctx)) {
return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif
return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset);
}
#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
/**
* @returns The total size of the history referenceable by zstd, including
* both the prefix and the extDict. At @p op any offset larger than this
* is invalid.
*/
static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart)
{
return (size_t)(op - virtualStart);
}
typedef struct {
unsigned longOffsetShare;
unsigned maxNbAdditionalBits;
} ZSTD_OffsetInfo;
/* ZSTD_getOffsetInfo() :
* condition : offTable must be valid
* @return : "share" of long offsets (arbitrarily defined as > (1<<23))
* compared to maximum possible of (1<<OffFSELog),
* as well as the maximum number additional bits required.
*/
static ZSTD_OffsetInfo
ZSTD_getOffsetInfo(const ZSTD_seqSymbol* offTable, int nbSeq)
{
ZSTD_OffsetInfo info = {0, 0};
/* If nbSeq == 0, then the offTable is uninitialized, but we have
* no sequences, so both values should be 0.
*/
if (nbSeq != 0) {
const void* ptr = offTable;
U32 const tableLog = ((const ZSTD_seqSymbol_header*)ptr)[0].tableLog;
const ZSTD_seqSymbol* table = offTable + 1;
U32 const max = 1 << tableLog;
U32 u;
DEBUGLOG(5, "ZSTD_getLongOffsetsShare: (tableLog=%u)", tableLog);
assert(max <= (1 << OffFSELog)); /* max not too large */
for (u=0; u<max; u++) {
info.maxNbAdditionalBits = MAX(info.maxNbAdditionalBits, table[u].nbAdditionalBits);
if (table[u].nbAdditionalBits > 22) info.longOffsetShare += 1;
}
assert(tableLog <= OffFSELog);
info.longOffsetShare <<= (OffFSELog - tableLog); /* scale to OffFSELog */
}
return info;
}
/**
* @returns The maximum offset we can decode in one read of our bitstream, without
* reloading more bits in the middle of the offset bits read. Any offsets larger
* than this must use the long offset decoder.
*/
static size_t ZSTD_maxShortOffset(void)
{
if (MEM_64bits()) {
/* We can decode any offset without reloading bits.
* This might change if the max window size grows.
*/
ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
return (size_t)-1;
} else {
/* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1.
* This offBase would require STREAM_ACCUMULATOR_MIN extra bits.
* Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset.
*/
size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1;
size_t const maxOffset = maxOffbase - ZSTD_REP_NUM;
assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN);
return maxOffset;
}
}
size_t
ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const streaming_operation streaming)
{ /* blockType == blockCompressed */
const BYTE* ip = (const BYTE*)src;
DEBUGLOG(5, "ZSTD_decompressBlock_internal (cSize : %u)", (unsigned)srcSize);
/* Note : the wording of the specification
* allows compressed block to be sized exactly ZSTD_blockSizeMax(dctx).
* This generally does not happen, as it makes little sense,
* since an uncompressed block would feature same size and have no decompression cost.
* Also, note that decoder from reference libzstd before < v1.5.4
* would consider this edge case as an error.
* As a consequence, avoid generating compressed blocks of size ZSTD_blockSizeMax(dctx)
* for broader compatibility with the deployed ecosystem of zstd decoders */
RETURN_ERROR_IF(srcSize > ZSTD_blockSizeMax(dctx), srcSize_wrong, "");
/* Decode literals section */
{ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming);
DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize);
if (ZSTD_isError(litCSize)) return litCSize;
ip += litCSize;
srcSize -= litCSize;
}
/* Build Decoding Tables */
{
/* Compute the maximum block size, which must also work when !frame and fParams are unset.
* Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t.
*/
size_t const blockSizeMax = MIN(dstCapacity, ZSTD_blockSizeMax(dctx));
size_t const totalHistorySize = ZSTD_totalHistorySize(ZSTD_maybeNullPtrAdd((BYTE*)dst, blockSizeMax), (BYTE const*)dctx->virtualStart);
/* isLongOffset must be true if there are long offsets.
* Offsets are long if they are larger than ZSTD_maxShortOffset().
* We don't expect that to be the case in 64-bit mode.
*
* We check here to see if our history is large enough to allow long offsets.
* If it isn't, then we can't possible have (valid) long offsets. If the offset
* is invalid, then it is okay to read it incorrectly.
*
* If isLongOffsets is true, then we will later check our decoding table to see
* if it is even possible to generate long offsets.
*/
ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset()));
/* These macros control at build-time which decompressor implementation
* we use. If neither is defined, we do some inspection and dispatch at
* runtime.
*/
#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
!defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
int usePrefetchDecoder = dctx->ddictIsCold;
#else
/* Set to 1 to avoid computing offset info if we don't need to.
* Otherwise this value is ignored.
*/
int usePrefetchDecoder = 1;
#endif
int nbSeq;
size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize);
if (ZSTD_isError(seqHSize)) return seqHSize;
ip += seqHSize;
srcSize -= seqHSize;
RETURN_ERROR_IF((dst == NULL || dstCapacity == 0) && nbSeq > 0, dstSize_tooSmall, "NULL not handled");
RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (size_t)(1 << 20), dstSize_tooSmall,
"invalid dst");
/* If we could potentially have long offsets, or we might want to use the prefetch decoder,
* compute information about the share of long offsets, and the maximum nbAdditionalBits.
* NOTE: could probably use a larger nbSeq limit
*/
if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) {
ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq);
if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) {
/* If isLongOffset, but the maximum number of additional bits that we see in our table is small
* enough, then we know it is impossible to have too long an offset in this block, so we can
* use the regular offset decoder.
*/
isLongOffset = ZSTD_lo_isRegularOffset;
}
if (!usePrefetchDecoder) {
U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */
usePrefetchDecoder = (info.longOffsetShare >= minShare);
}
}
dctx->ddictIsCold = 0;
#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
!defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
if (usePrefetchDecoder) {
#else
(void)usePrefetchDecoder;
{
#endif
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset);
#endif
}
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
/* else */
if (dctx->litBufferLocation == ZSTD_split)
return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset);
else
return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset);
#endif
}
}
ZSTD_ALLOW_POINTER_OVERFLOW_ATTR
void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize)
{
if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */
dctx->dictEnd = dctx->previousDstEnd;
dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart));
dctx->prefixStart = dst;
dctx->previousDstEnd = dst;
}
ZSTD_rustBlockCtx ctx = ZSTD_rust_block_context(dctx);
ZSTD_rust_checkContinuity(&ctx, dst, dstSize);
}
size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
const void* src, size_t srcSize)
{
size_t dSize;
dctx->isFrameDecompression = 0;
ZSTD_checkContinuity(dctx, dst, dstCapacity);
dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, not_streaming);
FORWARD_IF_ERROR(dSize, "");
dctx->previousDstEnd = (char*)dst + dSize;
return dSize;
ZSTD_rustBlockCtx ctx = ZSTD_rust_block_context(dctx);
return ZSTD_rust_decompressBlock_deprecated(&ctx,
dst, dstCapacity, src, srcSize);
}
/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated() */
/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated(). */
size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
const void* src, size_t srcSize)
{
return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize);
}
rust/README.md
+7 -4
View File
@@ -40,11 +40,14 @@ zstd ABI:
- `pool` implements the bounded worker pool used by multithreaded compression.
- Dictionary support
- `zstd_ddict` owns, loads, copies, and references decode dictionaries.
- Block decompression
- `zstd_decompress_block` decodes literal and sequence sections, maintains
FSE/Huffman repeat state, and executes compressed-block sequences.
The optimal block matcher, general decompression, dictionary-building, legacy,
and CLI translation units are still C. They must move before the rewrite is
complete. Keeping that boundary explicit prevents a passing hybrid build from
being mistaken for the final all-Rust result.
The optimal block matcher, high-level frame decompression, dictionary-building,
legacy, and CLI translation units are still C. They must move before the
rewrite is complete. Keeping that boundary explicit prevents a passing hybrid
build from being mistaken for the final all-Rust result.
## Compatibility boundary
rust/src/lib.rs
+2
View File
@@ -31,6 +31,8 @@ pub mod zstd_compress_sequences;
pub mod zstd_compress_superblock;
#[cfg(feature = "decompression")]
pub mod zstd_ddict;
#[cfg(feature = "decompression")]
pub mod zstd_decompress_block;
#[cfg(feature = "compression")]
pub mod zstd_double_fast;
#[cfg(feature = "compression")]
rust/src/zstd_decompress_block.rs
+1700
View File
@@ -0,0 +1,1700 @@
#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
#![allow(clippy::missing_safety_doc)]
#![allow(clippy::too_many_arguments)]
//! Compressed-block decoding.
//!
//! The C decoder context deliberately remains opaque. The companion C shim
//! extracts only the pointers and scalar values this translation needs into
//! [`ZSTD_rustBlockCtx`], so optional C context fields cannot silently change
//! this Rust module's ABI. Literal parsing, FSE table construction, sequence
//! decoding, and block execution live here.
use crate::bitstream::{
BIT_DStream_t, BIT_endOfDStream, BIT_initDStream, BIT_readBits, BIT_readBitsFast,
BIT_reloadDStream,
};
use crate::common::{
DEFAULT_MAX_OFF, LL_BITS, LL_DEFAULT_NORM, LL_DEFAULT_NORM_LOG, MAX_FSE_LOG, MAX_LL, MAX_ML,
MAX_OFF, MIN_CBLOCK_SIZE, MIN_LITERALS_FOR_4_STREAMS, MIN_SEQUENCES_SIZE, ML_BITS,
ML_DEFAULT_NORM, ML_DEFAULT_NORM_LOG, OF_DEFAULT_NORM, OF_DEFAULT_NORM_LOG, ZSTD_REP_NUM,
};
use crate::entropy_common::FSE_readNCount;
use crate::errors::{ERR_isError, ZstdErrorCode, ERROR};
#[cfg(not(feature = "huf-force-decompress-x2"))]
use crate::huf_decompress::HUF_decompress1X1_DCtx_wksp;
#[cfg(feature = "huf-force-decompress-x2")]
use crate::huf_decompress::HUF_decompress1X_DCtx_wksp;
use crate::huf_decompress::{
HUF_decompress1X_usingDTable, HUF_decompress4X_hufOnly_wksp, HUF_decompress4X_usingDTable,
};
use crate::mem::{MEM_32bits, MEM_64bits, MEM_readLE16, MEM_readLE24, U32};
use std::cmp::min;
use std::ffi::c_void;
use std::mem::MaybeUninit;
use std::os::raw::{c_int, c_short, c_uint};
use std::ptr;
use std::sync::OnceLock;
const ZSTD_BLOCKSIZE_MAX: usize = 128 << 10;
const ZSTD_BLOCK_HEADER_SIZE: usize = 3;
const WILDCOPY_OVERLENGTH: usize = 32;
const HUF_FLAGS_BMI2: c_int = 1 << 0;
const HUF_FLAGS_DISABLE_ASM: c_int = 1 << 4;
const SET_BASIC: c_int = 0;
const SET_RLE: c_int = 1;
const SET_COMPRESSED: c_int = 2;
const SET_REPEAT: c_int = 3;
const NOT_STREAMING: c_int = 0;
const ZSTD_NOT_IN_DST: c_int = 0;
const ZSTD_IN_DST: c_int = 1;
const ZSTD_SPLIT: c_int = 2;
const LONG_NB_SEQ: usize = 0x7f00;
const LL_FSE_LOG: u32 = 9;
const OFF_FSE_LOG: u32 = 8;
const ML_FSE_LOG: u32 = 9;
const ZSTD_HUFFDTABLE_CAPACITY_LOG: usize = 12;
const HUF_DTABLE_SIZE: usize = 1 + (1 << ZSTD_HUFFDTABLE_CAPACITY_LOG);
const ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32: usize = 157;
const LL_BASE: [u32; MAX_LL + 1] = [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 20, 22, 24, 28, 32, 40, 48, 64,
0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000,
];
const OF_BASE: [u32; MAX_OFF + 1] = [
0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, 0xFFFFFD, 0x1FFFFFD,
0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD,
];
const OF_BITS: [u8; MAX_OFF + 1] = [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
26, 27, 28, 29, 30, 31,
];
const ML_BASE: [u32; MAX_ML + 1] = [
3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27,
28, 29, 30, 31, 32, 33, 34, 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203,
0x403, 0x803, 0x1003, 0x2003, 0x4003, 0x8003, 0x10003,
];
/// `ZSTD_seqSymbol` from `zstd_decompress_internal.h`.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
pub struct ZSTD_seqSymbol {
next_state: u16,
nb_additional_bits: u8,
nb_bits: u8,
base_value: u32,
}
/// The first entry of a sequence table is overlaid as this C header.
#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
struct ZSTD_seqSymbol_header {
fast_mode: u32,
table_log: u32,
}
#[repr(C)]
struct ZSTD_entropyDTables_t {
ll_table: [ZSTD_seqSymbol; 1 + (1 << LL_FSE_LOG)],
of_table: [ZSTD_seqSymbol; 1 + (1 << OFF_FSE_LOG)],
ml_table: [ZSTD_seqSymbol; 1 + (1 << ML_FSE_LOG)],
huf_table: [u32; HUF_DTABLE_SIZE],
rep: [u32; ZSTD_REP_NUM],
workspace: [u32; ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32],
}
/// C-owned leaves of `ZSTD_DCtx_s` used by this translation unit.
///
/// Every pointer is produced by `zstd_decompress_block.c` under the active C
/// configuration. This avoids assuming offsets for optional context members
/// such as `DYNAMIC_BMI2`, fuzzing bounds, and tracing state.
#[repr(C)]
pub struct ZSTD_rustBlockCtx {
llt_ptr: *mut *const ZSTD_seqSymbol,
mlt_ptr: *mut *const ZSTD_seqSymbol,
oft_ptr: *mut *const ZSTD_seqSymbol,
huf_ptr: *mut *const u32,
entropy: *mut ZSTD_entropyDTables_t,
workspace: *mut u32,
workspace_size: usize,
previous_dst_end: *mut *const u8,
prefix_start: *mut *const u8,
virtual_start: *mut *const u8,
dict_end: *mut *const u8,
block_size_max: usize,
is_frame_decompression: *mut c_int,
lit_entropy: *mut u32,
fse_entropy: *mut u32,
bmi2: c_int,
ddict_is_cold: *mut c_int,
disable_huf_asm: c_int,
lit_ptr: *mut *const u8,
lit_size: *mut usize,
rle_size: *mut usize,
lit_buffer: *mut *mut u8,
lit_buffer_end: *mut *const u8,
lit_buffer_location: *mut c_int,
lit_extra_buffer: *mut u8,
lit_extra_buffer_size: usize,
}
#[repr(C)]
#[derive(Clone, Copy, Debug, Default)]
pub struct ZSTD_rustSeq {
lit_length: usize,
match_length: usize,
offset: usize,
}
#[repr(C)]
struct block_properties_t {
block_type: c_int,
last_block: u32,
orig_size: u32,
}
#[derive(Clone, Copy)]
struct ZSTD_fseState {
state: usize,
table: *const ZSTD_seqSymbol,
}
struct seq_state_t {
dstream: BIT_DStream_t,
state_ll: ZSTD_fseState,
state_off: ZSTD_fseState,
state_ml: ZSTD_fseState,
prev_offset: [usize; ZSTD_REP_NUM],
}
#[inline]
unsafe fn entropy(ctx: *mut ZSTD_rustBlockCtx) -> *mut ZSTD_entropyDTables_t {
unsafe { (*ctx).entropy }
}
#[inline]
unsafe fn address_distance(end: *const u8, start: *const u8) -> usize {
(end as usize).wrapping_sub(start as usize)
}
#[inline]
unsafe fn ptr_add(ptr: *mut u8, amount: usize) -> *mut u8 {
if ptr.is_null() {
debug_assert_eq!(amount, 0);
ptr
} else {
unsafe { ptr.add(amount) }
}
}
#[inline]
unsafe fn const_ptr_add(ptr: *const u8, amount: usize) -> *const u8 {
if ptr.is_null() {
debug_assert_eq!(amount, 0);
ptr
} else {
unsafe { ptr.add(amount) }
}
}
#[inline]
unsafe fn seq_header(table: *const ZSTD_seqSymbol) -> ZSTD_seqSymbol_header {
unsafe { table.cast::<ZSTD_seqSymbol_header>().read_unaligned() }
}
#[inline]
unsafe fn set_seq_header(table: *mut ZSTD_seqSymbol, header: ZSTD_seqSymbol_header) {
unsafe {
table
.cast::<ZSTD_seqSymbol_header>()
.write_unaligned(header)
};
}
#[inline]
unsafe fn copy_bytes(dst: *mut u8, src: *const u8, len: usize) {
if len != 0 {
unsafe { ptr::copy(src, dst, len) };
}
}
#[inline]
unsafe fn fill_bytes(dst: *mut u8, value: u8, len: usize) {
if len != 0 {
unsafe { ptr::write_bytes(dst, value, len) };
}
}
/// Copy a match using forward byte semantics, which is required for repeated
/// short-offset matches (unlike `memmove`, which would not expand overlap).
#[inline]
unsafe fn copy_match(mut dst: *mut u8, mut src: *const u8, len: usize) {
for _ in 0..len {
unsafe { dst.write(src.read()) };
dst = unsafe { dst.add(1) };
src = unsafe { src.add(1) };
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_getcBlockSize(
src: *const c_void,
src_size: usize,
bp_ptr: *mut c_void,
) -> usize {
if src_size < ZSTD_BLOCK_HEADER_SIZE {
return ERROR(ZstdErrorCode::SrcSizeWrong);
}
let header = unsafe { MEM_readLE24(src) };
let bp = bp_ptr.cast::<block_properties_t>();
unsafe {
(*bp).last_block = header & 1;
(*bp).block_type = ((header >> 1) & 3) as c_int;
(*bp).orig_size = header >> 3;
}
if unsafe { (*bp).block_type } == 1 {
return 1;
}
if unsafe { (*bp).block_type } == 3 {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
(header >> 3) as usize
}
#[inline]
unsafe fn block_size_max(ctx: *const ZSTD_rustBlockCtx) -> usize {
let value = unsafe {
if *(*ctx).is_frame_decompression != 0 {
(*ctx).block_size_max
} else {
ZSTD_BLOCKSIZE_MAX
}
};
debug_assert!(value <= ZSTD_BLOCKSIZE_MAX);
value
}
unsafe fn allocate_literals_buffer(
ctx: *mut ZSTD_rustBlockCtx,
dst: *mut u8,
dst_capacity: usize,
lit_size: usize,
streaming: c_int,
expected_write_size: usize,
split_immediately: bool,
) {
let block_max = unsafe { block_size_max(ctx) };
debug_assert!(lit_size <= block_max);
debug_assert!(*unsafe { (*ctx).is_frame_decompression } != 0 || streaming == NOT_STREAMING);
if streaming == NOT_STREAMING
&& dst_capacity
> block_max
.saturating_add(WILDCOPY_OVERLENGTH)
.saturating_add(lit_size)
.saturating_add(WILDCOPY_OVERLENGTH)
{
let buffer = unsafe { ptr_add(dst, block_max + WILDCOPY_OVERLENGTH) };
unsafe {
*(*ctx).lit_buffer = buffer;
*(*ctx).lit_buffer_end = buffer.add(lit_size);
*(*ctx).lit_buffer_location = ZSTD_IN_DST;
}
} else if lit_size <= unsafe { (*ctx).lit_extra_buffer_size } {
let buffer = unsafe { (*ctx).lit_extra_buffer };
unsafe {
*(*ctx).lit_buffer = buffer;
*(*ctx).lit_buffer_end = buffer.add(lit_size);
*(*ctx).lit_buffer_location = ZSTD_NOT_IN_DST;
}
} else {
let extra = unsafe { (*ctx).lit_extra_buffer_size };
debug_assert!(block_max > extra);
let buffer = if split_immediately {
unsafe {
ptr_add(
dst,
expected_write_size - lit_size + extra - WILDCOPY_OVERLENGTH,
)
}
} else {
unsafe { ptr_add(dst, expected_write_size - lit_size) }
};
unsafe {
*(*ctx).lit_buffer = buffer;
*(*ctx).lit_buffer_end = if split_immediately {
buffer.add(lit_size - extra)
} else {
ptr_add(dst, expected_write_size).cast_const()
};
*(*ctx).lit_buffer_location = ZSTD_SPLIT;
}
}
}
unsafe fn decode_literals_block(
ctx: *mut ZSTD_rustBlockCtx,
src: *const u8,
src_size: usize,
dst: *mut u8,
dst_capacity: usize,
streaming: c_int,
) -> usize {
if src_size < MIN_CBLOCK_SIZE {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
let lit_type = unsafe { *src & 3 } as c_int;
let block_max = unsafe { block_size_max(ctx) };
match lit_type {
SET_REPEAT | SET_COMPRESSED => {
if lit_type == SET_REPEAT && unsafe { *(*ctx).lit_entropy } == 0 {
return ERROR(ZstdErrorCode::DictionaryCorrupted);
}
if src_size < 5 {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
let lhl_code = unsafe { (*src >> 2) & 3 };
let header = unsafe { crate::mem::MEM_readLE32(src.cast()) };
let (header_size, lit_size, lit_c_size, single_stream) = match lhl_code {
0 | 1 => (
3usize,
((header >> 4) & 0x3ff) as usize,
((header >> 14) & 0x3ff) as usize,
lhl_code == 0,
),
2 => (
4usize,
((header >> 4) & 0x3fff) as usize,
(header >> 18) as usize,
false,
),
_ => (
5usize,
((header >> 4) & 0x3ffff) as usize,
((header >> 22) as usize).wrapping_add((unsafe { *src.add(4) } as usize) << 10),
false,
),
};
if (lit_size != 0 && dst.is_null())
|| lit_size > block_max
|| (!single_stream && lit_size < MIN_LITERALS_FOR_4_STREAMS)
|| header_size.checked_add(lit_c_size).is_none()
|| header_size + lit_c_size > src_size
|| min(block_max, dst_capacity) < lit_size
{
return if lit_size != 0 && dst.is_null() || min(block_max, dst_capacity) < lit_size
{
ERROR(ZstdErrorCode::DstSizeTooSmall)
} else {
ERROR(ZstdErrorCode::CorruptionDetected)
};
}
unsafe {
allocate_literals_buffer(
ctx,
dst,
dst_capacity,
lit_size,
streaming,
min(block_max, dst_capacity),
false,
);
}
let flags = (if unsafe { (*ctx).bmi2 } != 0 {
HUF_FLAGS_BMI2
} else {
0
}) | (if unsafe { (*ctx).disable_huf_asm } != 0 {
HUF_FLAGS_DISABLE_ASM
} else {
0
});
let lit_buffer = unsafe { *(*ctx).lit_buffer };
let huf_result = if lit_type == SET_REPEAT {
if single_stream {
unsafe {
HUF_decompress1X_usingDTable(
lit_buffer.cast(),
lit_size,
src.add(header_size).cast(),
lit_c_size,
*(*ctx).huf_ptr,
flags,
)
}
} else {
unsafe {
HUF_decompress4X_usingDTable(
lit_buffer.cast(),
lit_size,
src.add(header_size).cast(),
lit_c_size,
*(*ctx).huf_ptr,
flags,
)
}
}
} else if single_stream {
#[cfg(feature = "huf-force-decompress-x2")]
{
unsafe {
HUF_decompress1X_DCtx_wksp(
(*entropy(ctx)).huf_table.as_mut_ptr(),
lit_buffer.cast(),
lit_size,
src.add(header_size).cast(),
lit_c_size,
(*ctx).workspace.cast(),
(*ctx).workspace_size,
flags,
)
}
}
#[cfg(not(feature = "huf-force-decompress-x2"))]
{
unsafe {
HUF_decompress1X1_DCtx_wksp(
(*entropy(ctx)).huf_table.as_mut_ptr(),
lit_buffer.cast(),
lit_size,
src.add(header_size).cast(),
lit_c_size,
(*ctx).workspace.cast(),
(*ctx).workspace_size,
flags,
)
}
}
} else {
unsafe {
HUF_decompress4X_hufOnly_wksp(
(*entropy(ctx)).huf_table.as_mut_ptr(),
lit_buffer.cast(),
lit_size,
src.add(header_size).cast(),
lit_c_size,
(*ctx).workspace.cast(),
(*ctx).workspace_size,
flags,
)
}
};
if ERR_isError(huf_result) {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
if unsafe { *(*ctx).lit_buffer_location } == ZSTD_SPLIT {
let extra = unsafe { (*ctx).lit_extra_buffer_size };
unsafe {
copy_bytes(
(*ctx).lit_extra_buffer,
(*(*ctx).lit_buffer_end).sub(extra),
extra,
);
ptr::copy(
*(*ctx).lit_buffer,
(*(*ctx).lit_buffer).add(extra - WILDCOPY_OVERLENGTH),
lit_size - extra,
);
*(*ctx).lit_buffer = (*(*ctx).lit_buffer).add(extra - WILDCOPY_OVERLENGTH);
*(*ctx).lit_buffer_end = (*(*ctx).lit_buffer_end).sub(WILDCOPY_OVERLENGTH);
}
}
unsafe {
*(*ctx).lit_ptr = *(*ctx).lit_buffer;
*(*ctx).lit_size = lit_size;
*(*ctx).lit_entropy = 1;
if lit_type == SET_COMPRESSED {
*(*ctx).huf_ptr = (*entropy(ctx)).huf_table.as_ptr();
}
}
header_size + lit_c_size
}
SET_BASIC => {
let lhl_code = unsafe { (*src >> 2) & 3 };
let (header_size, lit_size) = match lhl_code {
0 | 2 => (1usize, (unsafe { *src } >> 3) as usize),
1 => (2usize, (unsafe { MEM_readLE16(src.cast()) } >> 4) as usize),
_ => {
if src_size < 3 {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
(3usize, (unsafe { MEM_readLE24(src.cast()) } >> 4) as usize)
}
};
if (lit_size != 0 && dst.is_null())
|| lit_size > block_max
|| min(block_max, dst_capacity) < lit_size
{
return if lit_size != 0 && dst.is_null() || min(block_max, dst_capacity) < lit_size
{
ERROR(ZstdErrorCode::DstSizeTooSmall)
} else {
ERROR(ZstdErrorCode::CorruptionDetected)
};
}
unsafe {
allocate_literals_buffer(
ctx,
dst,
dst_capacity,
lit_size,
streaming,
min(block_max, dst_capacity),
true,
);
}
if header_size
.checked_add(lit_size)
.and_then(|size| size.checked_add(WILDCOPY_OVERLENGTH))
.is_none_or(|size| size > src_size)
{
if header_size
.checked_add(lit_size)
.is_none_or(|size| size > src_size)
{
return ERROR(ZstdErrorCode::CorruptionDetected);
}
unsafe {
if *(*ctx).lit_buffer_location == ZSTD_SPLIT {
let extra = (*ctx).lit_extra_buffer_size;
copy_bytes(*(*ctx).lit_buffer, src.add(header_size), lit_size - extra);
copy_bytes(
(*ctx).lit_extra_buffer,
src.add(header_size + lit_size - extra),
extra,
);
} else {
copy_bytes(*(*ctx).lit_buffer, src.add(header_size), lit_size);
}
*(*ctx).lit_ptr = *(*ctx).lit_buffer;
*(*ctx).lit_size = lit_size;
}
return header_size + lit_size;
}
unsafe {
*(*ctx).lit_ptr = src.add(header_size);
*(*ctx).lit_size = lit_size;
*(*ctx).lit_buffer_end = src.add(header_size + lit_size);
*(*ctx).lit_buffer_location = ZSTD_NOT_IN_DST;
}
header_size + lit_size
}
SET_RLE => {
let lhl_code = unsafe { (*src >> 2) & 3 };
let (header_size, lit_size) = match lhl_code {
0 | 2 => (1usize, (unsafe { *src } >> 3) as usize),
1 => {
if src_size < 3 {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
(2usize, (unsafe { MEM_readLE16(src.cast()) } >> 4) as usize)
}
_ => {
if src_size < 4 {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
(3usize, (unsafe { MEM_readLE24(src.cast()) } >> 4) as usize)
}
};
if (lit_size != 0 && dst.is_null())
|| lit_size > block_max
|| min(block_max, dst_capacity) < lit_size
{
return if lit_size != 0 && dst.is_null() || min(block_max, dst_capacity) < lit_size
{
ERROR(ZstdErrorCode::DstSizeTooSmall)
} else {
ERROR(ZstdErrorCode::CorruptionDetected)
};
}
unsafe {
allocate_literals_buffer(
ctx,
dst,
dst_capacity,
lit_size,
streaming,
min(block_max, dst_capacity),
true,
);
let value = *src.add(header_size);
if *(*ctx).lit_buffer_location == ZSTD_SPLIT {
let extra = (*ctx).lit_extra_buffer_size;
fill_bytes(*(*ctx).lit_buffer, value, lit_size - extra);
fill_bytes((*ctx).lit_extra_buffer, value, extra);
} else {
fill_bytes(*(*ctx).lit_buffer, value, lit_size);
}
*(*ctx).lit_ptr = *(*ctx).lit_buffer;
*(*ctx).lit_size = lit_size;
}
header_size + 1
}
_ => ERROR(ZstdErrorCode::CorruptionDetected),
}
}
unsafe fn build_seq_table_rle(table: *mut ZSTD_seqSymbol, base_value: u32, nb_add_bits: u8) {
unsafe {
set_seq_header(
table,
ZSTD_seqSymbol_header {
fast_mode: 0,
table_log: 0,
},
);
*table.add(1) = ZSTD_seqSymbol {
next_state: 0,
nb_additional_bits: nb_add_bits,
nb_bits: 0,
base_value,
};
}
}
unsafe fn build_fse_table_body(
table: *mut ZSTD_seqSymbol,
normalized_counter: *const i16,
max_symbol_value: u32,
base_value: *const u32,
nb_additional_bits: *const u8,
table_log: u32,
workspace: *mut u32,
workspace_size: usize,
) {
debug_assert!(max_symbol_value as usize <= MAX_ML);
debug_assert!(table_log as usize <= MAX_FSE_LOG);
debug_assert!(
workspace_size >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 * std::mem::size_of::<u32>()
);
let table_decode = unsafe { table.add(1) };
let table_size = 1usize << table_log;
let symbol_next = workspace.cast::<u16>();
let spread = unsafe { symbol_next.add(MAX_ML + 1).cast::<u8>() };
let mut high_threshold = table_size - 1;
let mut fast_mode = 1u32;
let large_limit = 1i16 << (table_log - 1);
for symbol in 0..=max_symbol_value as usize {
let count = unsafe { *normalized_counter.add(symbol) };
if count == -1 {
unsafe {
(*table_decode.add(high_threshold)).base_value = symbol as u32;
*symbol_next.add(symbol) = 1;
}
high_threshold = high_threshold.wrapping_sub(1);
} else {
if count >= large_limit {
fast_mode = 0;
}
debug_assert!(count >= 0);
unsafe { *symbol_next.add(symbol) = count as u16 };
}
}
unsafe {
set_seq_header(
table,
ZSTD_seqSymbol_header {
fast_mode,
table_log,
},
);
}
let table_mask = table_size - 1;
let step = (table_size >> 1) + (table_size >> 3) + 3;
if high_threshold == table_size - 1 {
let mut pos = 0usize;
for symbol in 0..=max_symbol_value as usize {
let count = unsafe { *normalized_counter.add(symbol) };
debug_assert!(count >= 0);
for index in 0..count as usize {
unsafe { *spread.add(pos + index) = symbol as u8 };
}
pos += count as usize;
}
let mut position = 0usize;
for symbol_index in 0..table_size {
unsafe { (*table_decode.add(position)).base_value = *spread.add(symbol_index) as u32 };
position = (position + step) & table_mask;
}
debug_assert_eq!(position, 0);
} else {
let mut position = 0usize;
for symbol in 0..=max_symbol_value as usize {
let count = unsafe { *normalized_counter.add(symbol) };
for _ in 0..count.max(0) as usize {
unsafe { (*table_decode.add(position)).base_value = symbol as u32 };
position = (position + step) & table_mask;
while position > high_threshold {
position = (position + step) & table_mask;
}
}
}
debug_assert_eq!(position, 0);
}
for index in 0..table_size {
let symbol = unsafe { (*table_decode.add(index)).base_value as usize };
let next_state = unsafe { *symbol_next.add(symbol) } as u32;
unsafe { *symbol_next.add(symbol) = next_state.wrapping_add(1) as u16 };
let nb_bits = table_log - crate::bits::ZSTD_highbit32(next_state);
unsafe {
let entry = &mut *table_decode.add(index);
entry.nb_bits = nb_bits as u8;
entry.next_state = ((next_state << nb_bits) - table_size as u32) as u16;
entry.nb_additional_bits = *nb_additional_bits.add(symbol);
entry.base_value = *base_value.add(symbol);
}
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_buildFSETable_body(
table: *mut ZSTD_seqSymbol,
normalized_counter: *const c_short,
max_symbol_value: c_uint,
base_value: *const U32,
nb_additional_bits: *const u8,
table_log: c_uint,
workspace: *mut c_void,
workspace_size: usize,
) {
unsafe {
build_fse_table_body(
table,
normalized_counter,
max_symbol_value,
base_value,
nb_additional_bits,
table_log,
workspace.cast(),
workspace_size,
);
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_buildFSETable(
table: *mut ZSTD_seqSymbol,
normalized_counter: *const c_short,
max_symbol_value: c_uint,
base_value: *const U32,
nb_additional_bits: *const u8,
table_log: c_uint,
workspace: *mut c_void,
workspace_size: usize,
_bmi2: c_int,
) {
unsafe {
build_fse_table_body(
table,
normalized_counter,
max_symbol_value,
base_value,
nb_additional_bits,
table_log,
workspace.cast(),
workspace_size,
);
}
}
fn build_default_table<const N: usize>(
normalized: &[i16],
base: &[u32],
bits: &[u8],
max_symbol_value: u32,
table_log: u32,
) -> [ZSTD_seqSymbol; N] {
let mut table = [ZSTD_seqSymbol::default(); N];
let mut workspace = [0u32; ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32];
unsafe {
build_fse_table_body(
table.as_mut_ptr(),
normalized.as_ptr(),
max_symbol_value,
base.as_ptr(),
bits.as_ptr(),
table_log,
workspace.as_mut_ptr(),
std::mem::size_of_val(&workspace),
);
}
table
}
static LL_DEFAULT_TABLE: OnceLock<[ZSTD_seqSymbol; 1 + (1 << LL_FSE_LOG)]> = OnceLock::new();
static OF_DEFAULT_TABLE: OnceLock<[ZSTD_seqSymbol; 1 + (1 << OFF_FSE_LOG)]> = OnceLock::new();
static ML_DEFAULT_TABLE: OnceLock<[ZSTD_seqSymbol; 1 + (1 << ML_FSE_LOG)]> = OnceLock::new();
#[inline]
fn ll_default_table() -> *const ZSTD_seqSymbol {
LL_DEFAULT_TABLE
.get_or_init(|| {
build_default_table(
&LL_DEFAULT_NORM,
&LL_BASE,
&LL_BITS,
MAX_LL as u32,
LL_DEFAULT_NORM_LOG,
)
})
.as_ptr()
}
#[inline]
fn of_default_table() -> *const ZSTD_seqSymbol {
OF_DEFAULT_TABLE
.get_or_init(|| {
build_default_table(
&OF_DEFAULT_NORM,
&OF_BASE,
&OF_BITS,
DEFAULT_MAX_OFF as u32,
OF_DEFAULT_NORM_LOG,
)
})
.as_ptr()
}
#[inline]
fn ml_default_table() -> *const ZSTD_seqSymbol {
ML_DEFAULT_TABLE
.get_or_init(|| {
build_default_table(
&ML_DEFAULT_NORM,
&ML_BASE,
&ML_BITS,
MAX_ML as u32,
ML_DEFAULT_NORM_LOG,
)
})
.as_ptr()
}
unsafe fn build_seq_table(
table_space: *mut ZSTD_seqSymbol,
table_ptr: *mut *const ZSTD_seqSymbol,
table_type: c_int,
mut max_symbol: u32,
max_log: u32,
src: *const u8,
src_size: usize,
base: &[u32],
bits: &[u8],
default_table: *const ZSTD_seqSymbol,
repeat_table_available: u32,
_ddict_is_cold: c_int,
_nb_seq: c_int,
workspace: *mut u32,
workspace_size: usize,
bmi2: c_int,
) -> usize {
match table_type {
SET_RLE => {
if src_size == 0 {
return ERROR(ZstdErrorCode::SrcSizeWrong);
}
let symbol = unsafe { *src } as usize;
if symbol > max_symbol as usize {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
unsafe { build_seq_table_rle(table_space, base[symbol], bits[symbol]) };
unsafe { *table_ptr = table_space };
1
}
SET_BASIC => {
unsafe { *table_ptr = default_table };
0
}
SET_REPEAT => {
if repeat_table_available == 0 {
ERROR(ZstdErrorCode::CorruptionDetected)
} else {
0
}
}
SET_COMPRESSED => {
let mut table_log = 0u32;
let mut norm = [0i16; MAX_ML + 1];
let header_size = unsafe {
FSE_readNCount(
norm.as_mut_ptr(),
&mut max_symbol,
&mut table_log,
src.cast(),
src_size,
)
};
if ERR_isError(header_size) || table_log > max_log {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
unsafe {
build_fse_table_body(
table_space,
norm.as_ptr(),
max_symbol,
base.as_ptr(),
bits.as_ptr(),
table_log,
workspace,
workspace_size,
);
*table_ptr = table_space;
}
let _ = bmi2;
header_size
}
_ => ERROR(ZstdErrorCode::Generic),
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_decodeSeqHeaders(
ctx: *mut ZSTD_rustBlockCtx,
nb_seq_ptr: *mut c_int,
src: *const c_void,
src_size: usize,
) -> usize {
if src_size < MIN_SEQUENCES_SIZE {
return ERROR(ZstdErrorCode::SrcSizeWrong);
}
let start = src.cast::<u8>();
let end = unsafe { start.add(src_size) };
let mut ip = start;
let mut nb_seq = unsafe { *ip } as usize;
ip = unsafe { ip.add(1) };
if nb_seq > 0x7f {
if nb_seq == 0xff {
if (ip as usize).wrapping_add(2) > end as usize {
return ERROR(ZstdErrorCode::SrcSizeWrong);
}
nb_seq = unsafe { MEM_readLE16(ip.cast()) as usize } + LONG_NB_SEQ;
ip = unsafe { ip.add(2) };
} else {
if ip >= end {
return ERROR(ZstdErrorCode::SrcSizeWrong);
}
nb_seq = ((nb_seq - 0x80) << 8) + unsafe { *ip } as usize;
ip = unsafe { ip.add(1) };
}
}
unsafe { *nb_seq_ptr = nb_seq as c_int };
if nb_seq == 0 {
return if ip == end {
unsafe { ip.offset_from(start) as usize }
} else {
ERROR(ZstdErrorCode::CorruptionDetected)
};
}
if ip >= end || unsafe { *ip & 3 } != 0 {
return if ip >= end {
ERROR(ZstdErrorCode::SrcSizeWrong)
} else {
ERROR(ZstdErrorCode::CorruptionDetected)
};
}
let descriptor = unsafe { *ip };
ip = unsafe { ip.add(1) };
let ll_type = (descriptor >> 6) as c_int;
let of_type = ((descriptor >> 4) & 3) as c_int;
let ml_type = ((descriptor >> 2) & 3) as c_int;
let entropy = unsafe { entropy(ctx) };
let repeat = unsafe { *(*ctx).fse_entropy };
let cold = unsafe { *(*ctx).ddict_is_cold };
let ll_size = unsafe {
build_seq_table(
(*entropy).ll_table.as_mut_ptr(),
(*ctx).llt_ptr,
ll_type,
MAX_LL as u32,
LL_FSE_LOG,
ip,
address_distance(end, ip),
&LL_BASE,
&LL_BITS,
ll_default_table(),
repeat,
cold,
nb_seq as c_int,
(*ctx).workspace,
(*ctx).workspace_size,
(*ctx).bmi2,
)
};
if ERR_isError(ll_size) || ll_size > unsafe { address_distance(end, ip) } {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
ip = unsafe { ip.add(ll_size) };
let of_size = unsafe {
build_seq_table(
(*entropy).of_table.as_mut_ptr(),
(*ctx).oft_ptr,
of_type,
MAX_OFF as u32,
OFF_FSE_LOG,
ip,
address_distance(end, ip),
&OF_BASE,
&OF_BITS,
of_default_table(),
repeat,
cold,
nb_seq as c_int,
(*ctx).workspace,
(*ctx).workspace_size,
(*ctx).bmi2,
)
};
if ERR_isError(of_size) || of_size > unsafe { address_distance(end, ip) } {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
ip = unsafe { ip.add(of_size) };
let ml_size = unsafe {
build_seq_table(
(*entropy).ml_table.as_mut_ptr(),
(*ctx).mlt_ptr,
ml_type,
MAX_ML as u32,
ML_FSE_LOG,
ip,
address_distance(end, ip),
&ML_BASE,
&ML_BITS,
ml_default_table(),
repeat,
cold,
nb_seq as c_int,
(*ctx).workspace,
(*ctx).workspace_size,
(*ctx).bmi2,
)
};
if ERR_isError(ml_size) || ml_size > unsafe { address_distance(end, ip) } {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
ip = unsafe { ip.add(ml_size) };
unsafe { ip.offset_from(start) as usize }
}
#[inline]
unsafe fn init_fse_state(
state: &mut ZSTD_fseState,
dstream: *mut BIT_DStream_t,
table: *const ZSTD_seqSymbol,
) -> Result<(), usize> {
if table.is_null() {
return Err(ERROR(ZstdErrorCode::CorruptionDetected));
}
let header = unsafe { seq_header(table) };
if header.table_log > MAX_FSE_LOG as u32 {
return Err(ERROR(ZstdErrorCode::CorruptionDetected));
}
state.state = unsafe { BIT_readBits(dstream, header.table_log) };
let _ = unsafe { BIT_reloadDStream(dstream) };
state.table = unsafe { table.add(1) };
Ok(())
}
#[inline]
unsafe fn update_fse_state(
state: &mut ZSTD_fseState,
dstream: *mut BIT_DStream_t,
next_state: u16,
nb_bits: u8,
) {
let low_bits = unsafe { BIT_readBits(dstream, nb_bits as u32) };
state.state = next_state as usize + low_bits;
}
unsafe fn decode_sequence(
state: &mut seq_state_t,
long_offsets: bool,
is_last_sequence: bool,
) -> ZSTD_rustSeq {
let ll_info = unsafe { state.state_ll.table.add(state.state_ll.state).read() };
let ml_info = unsafe { state.state_ml.table.add(state.state_ml.state).read() };
let of_info = unsafe { state.state_off.table.add(state.state_off.state).read() };
let ll_bits = ll_info.nb_additional_bits as u32;
let ml_bits = ml_info.nb_additional_bits as u32;
let of_bits = of_info.nb_additional_bits as u32;
let total_bits = ll_bits + ml_bits + of_bits;
let mut sequence = ZSTD_rustSeq {
lit_length: ll_info.base_value as usize,
match_length: ml_info.base_value as usize,
offset: 0,
};
let offset = if of_bits > 1 {
let value = if MEM_32bits() && long_offsets && of_bits >= 25 {
let upper = unsafe { BIT_readBitsFast(&mut state.dstream, of_bits - 5) } << 5;
let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
upper + unsafe { BIT_readBitsFast(&mut state.dstream, 5) }
} else {
let value = unsafe { BIT_readBitsFast(&mut state.dstream, of_bits) };
if MEM_32bits() {
let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
}
value
};
let value = (of_info.base_value as usize).wrapping_add(value);
state.prev_offset[2] = state.prev_offset[1];
state.prev_offset[1] = state.prev_offset[0];
state.prev_offset[0] = value;
value
} else {
let ll_zero = ll_info.base_value == 0;
if of_bits == 0 {
let value = state.prev_offset[usize::from(ll_zero)];
state.prev_offset[1] = state.prev_offset[usize::from(!ll_zero)];
state.prev_offset[0] = value;
value
} else {
let code = (of_info.base_value as usize)
.wrapping_add(usize::from(ll_zero))
.wrapping_add(unsafe { BIT_readBitsFast(&mut state.dstream, 1) });
let mut value = if code == 3 {
state.prev_offset[0].wrapping_sub(1)
} else if code < ZSTD_REP_NUM {
state.prev_offset[code]
} else {
usize::MAX
};
value = value.wrapping_sub(usize::from(value == 0));
if code != 1 {
state.prev_offset[2] = state.prev_offset[1];
}
state.prev_offset[1] = state.prev_offset[0];
state.prev_offset[0] = value;
value
}
};
sequence.offset = offset;
if ml_bits != 0 {
sequence.match_length = sequence
.match_length
.wrapping_add(unsafe { BIT_readBitsFast(&mut state.dstream, ml_bits) });
}
if MEM_32bits() && ml_bits + ll_bits >= 20 {
let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
}
if MEM_64bits() && total_bits >= 30 {
let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
}
if ll_bits != 0 {
sequence.lit_length = sequence
.lit_length
.wrapping_add(unsafe { BIT_readBitsFast(&mut state.dstream, ll_bits) });
}
if MEM_32bits() {
let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
}
if !is_last_sequence {
unsafe {
update_fse_state(
&mut state.state_ll,
&mut state.dstream,
ll_info.next_state,
ll_info.nb_bits,
);
update_fse_state(
&mut state.state_ml,
&mut state.dstream,
ml_info.next_state,
ml_info.nb_bits,
);
}
if MEM_32bits() {
let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
}
unsafe {
update_fse_state(
&mut state.state_off,
&mut state.dstream,
of_info.next_state,
of_info.nb_bits,
);
}
let _ = unsafe { BIT_reloadDStream(&mut state.dstream) };
}
sequence
}
unsafe fn exec_sequence(
op: *mut u8,
oend: *mut u8,
sequence: ZSTD_rustSeq,
lit_ptr: &mut *const u8,
lit_limit: *const u8,
prefix_start: *const u8,
virtual_start: *const u8,
dict_end: *const u8,
split_literals: bool,
) -> usize {
let sequence_length = match sequence.lit_length.checked_add(sequence.match_length) {
Some(length) => length,
None => return ERROR(ZstdErrorCode::DstSizeTooSmall),
};
let output_capacity = unsafe { address_distance(oend.cast_const(), op.cast_const()) };
if sequence_length > output_capacity {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
let literal_available = unsafe { address_distance(lit_limit, *lit_ptr) };
if sequence.lit_length > literal_available {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
let lit_end = unsafe { ptr_add(op, sequence.lit_length) };
if split_literals
&& (op as usize) > (*lit_ptr as usize)
&& (op as usize) < (*lit_ptr as usize).wrapping_add(sequence.lit_length)
{
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
unsafe { copy_bytes(op, *lit_ptr, sequence.lit_length) };
*lit_ptr = unsafe { const_ptr_add(*lit_ptr, sequence.lit_length) };
if sequence.offset == 0 {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
let prefix_history = unsafe { address_distance(lit_end.cast_const(), prefix_start) };
let mut output = lit_end;
let mut match_length = sequence.match_length;
if sequence.offset > prefix_history {
let virtual_history = unsafe { address_distance(lit_end.cast_const(), virtual_start) };
if sequence.offset > virtual_history || dict_end.is_null() {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
let before_prefix = sequence.offset - prefix_history;
let match_ptr = unsafe { dict_end.sub(before_prefix) };
let dict_available = unsafe { address_distance(dict_end, match_ptr) };
let first_length = min(match_length, dict_available);
unsafe { copy_bytes(output, match_ptr, first_length) };
output = unsafe { output.add(first_length) };
match_length -= first_length;
if match_length != 0 {
if prefix_start.is_null() {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
unsafe { copy_match(output, prefix_start, match_length) };
}
} else {
let match_ptr = unsafe { lit_end.sub(sequence.offset) };
if match_ptr.is_null() {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
unsafe { copy_match(output, match_ptr, match_length) };
}
sequence_length
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_execSequenceEnd(
op: *mut u8,
oend: *mut u8,
sequence: ZSTD_rustSeq,
lit_ptr: *mut *const u8,
lit_limit: *const u8,
prefix_start: *const u8,
virtual_start: *const u8,
dict_end: *const u8,
) -> usize {
unsafe {
exec_sequence(
op,
oend,
sequence,
&mut *lit_ptr,
lit_limit,
prefix_start,
virtual_start,
dict_end,
false,
)
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_execSequenceEndSplitLitBuffer(
op: *mut u8,
oend: *mut u8,
_oend_w: *const u8,
sequence: ZSTD_rustSeq,
lit_ptr: *mut *const u8,
lit_limit: *const u8,
prefix_start: *const u8,
virtual_start: *const u8,
dict_end: *const u8,
) -> usize {
unsafe {
exec_sequence(
op,
oend,
sequence,
&mut *lit_ptr,
lit_limit,
prefix_start,
virtual_start,
dict_end,
true,
)
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_execSequence(
op: *mut u8,
oend: *mut u8,
sequence: ZSTD_rustSeq,
lit_ptr: *mut *const u8,
lit_limit: *const u8,
prefix_start: *const u8,
virtual_start: *const u8,
dict_end: *const u8,
) -> usize {
unsafe {
exec_sequence(
op,
oend,
sequence,
&mut *lit_ptr,
lit_limit,
prefix_start,
virtual_start,
dict_end,
false,
)
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_execSequenceSplitLitBuffer(
op: *mut u8,
oend: *mut u8,
_oend_w: *const u8,
sequence: ZSTD_rustSeq,
lit_ptr: *mut *const u8,
lit_limit: *const u8,
prefix_start: *const u8,
virtual_start: *const u8,
dict_end: *const u8,
) -> usize {
unsafe {
exec_sequence(
op,
oend,
sequence,
&mut *lit_ptr,
lit_limit,
prefix_start,
virtual_start,
dict_end,
true,
)
}
}
unsafe fn decompress_sequences(
ctx: *mut ZSTD_rustBlockCtx,
dst: *mut u8,
dst_capacity: usize,
sequence_start: *const u8,
sequence_size: usize,
nb_seq: c_int,
long_offsets: bool,
) -> usize {
if nb_seq < 0 {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
let mut location = unsafe { *(*ctx).lit_buffer_location };
let mut op = dst;
let mut oend = if location == ZSTD_NOT_IN_DST || location == ZSTD_SPLIT {
unsafe { ptr_add(dst, dst_capacity) }
} else {
unsafe { *(*ctx).lit_buffer }
};
let mut lit_ptr = unsafe { *(*ctx).lit_ptr };
let mut lit_limit = if location == ZSTD_SPLIT {
unsafe { *(*ctx).lit_buffer_end }
} else {
unsafe { const_ptr_add(lit_ptr, *(*ctx).lit_size) }
};
let prefix_start = unsafe { *(*ctx).prefix_start };
let virtual_start = unsafe { *(*ctx).virtual_start };
let dict_end = unsafe { *(*ctx).dict_end };
if nb_seq != 0 {
if dst.is_null() {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
let mut state = seq_state_t {
dstream: unsafe { MaybeUninit::zeroed().assume_init() },
state_ll: ZSTD_fseState {
state: 0,
table: ptr::null(),
},
state_off: ZSTD_fseState {
state: 0,
table: ptr::null(),
},
state_ml: ZSTD_fseState {
state: 0,
table: ptr::null(),
},
prev_offset: unsafe { (*entropy(ctx)).rep.map(|value| value as usize) },
};
unsafe { *(*ctx).fse_entropy = 1 };
let init =
unsafe { BIT_initDStream(&mut state.dstream, sequence_start.cast(), sequence_size) };
if ERR_isError(init) {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
if let Err(error) =
unsafe { init_fse_state(&mut state.state_ll, &mut state.dstream, *(*ctx).llt_ptr) }
{
return error;
}
if let Err(error) =
unsafe { init_fse_state(&mut state.state_off, &mut state.dstream, *(*ctx).oft_ptr) }
{
return error;
}
if let Err(error) =
unsafe { init_fse_state(&mut state.state_ml, &mut state.dstream, *(*ctx).mlt_ptr) }
{
return error;
}
for remaining in (1..=nb_seq as usize).rev() {
let mut sequence = unsafe { decode_sequence(&mut state, long_offsets, remaining == 1) };
if location == ZSTD_SPLIT
&& sequence.lit_length > unsafe { address_distance(lit_limit, lit_ptr) }
{
let leftover = unsafe { address_distance(lit_limit, lit_ptr) };
if leftover > unsafe { address_distance(oend.cast_const(), op.cast_const()) } {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
unsafe { copy_bytes(op, lit_ptr, leftover) };
op = unsafe { op.add(leftover) };
sequence.lit_length -= leftover;
lit_ptr = unsafe { (*ctx).lit_extra_buffer.cast_const() };
lit_limit = unsafe {
const_ptr_add(
(*ctx).lit_extra_buffer.cast_const(),
(*ctx).lit_extra_buffer_size,
)
};
location = ZSTD_NOT_IN_DST;
unsafe { *(*ctx).lit_buffer_location = ZSTD_NOT_IN_DST };
}
let decoded = unsafe {
exec_sequence(
op,
oend,
sequence,
&mut lit_ptr,
lit_limit,
prefix_start,
virtual_start,
dict_end,
location == ZSTD_SPLIT,
)
};
if ERR_isError(decoded) {
return decoded;
}
op = unsafe { op.add(decoded) };
}
if unsafe { BIT_endOfDStream(&state.dstream) } == 0 {
return ERROR(ZstdErrorCode::CorruptionDetected);
}
unsafe { (*entropy(ctx)).rep = state.prev_offset.map(|value| value as u32) };
}
if location == ZSTD_SPLIT {
let first_size = unsafe { address_distance(lit_limit, lit_ptr) };
if first_size > unsafe { address_distance(oend.cast_const(), op.cast_const()) } {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
unsafe { copy_bytes(op, lit_ptr, first_size) };
op = unsafe { op.add(first_size) };
lit_ptr = unsafe { (*ctx).lit_extra_buffer.cast_const() };
lit_limit = unsafe {
const_ptr_add(
(*ctx).lit_extra_buffer.cast_const(),
(*ctx).lit_extra_buffer_size,
)
};
unsafe { *(*ctx).lit_buffer_location = ZSTD_NOT_IN_DST };
oend = unsafe { ptr_add(dst, dst_capacity) };
}
let last_size = unsafe { address_distance(lit_limit, lit_ptr) };
if last_size > unsafe { address_distance(oend.cast_const(), op.cast_const()) } {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
unsafe { copy_bytes(op, lit_ptr, last_size) };
op = unsafe { op.add(last_size) };
unsafe { address_distance(op.cast_const(), dst.cast_const()) }
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_prefetchMatch(
prefetch_pos: usize,
sequence: ZSTD_rustSeq,
_prefix_start: *const u8,
_dict_end: *const u8,
) -> usize {
prefetch_pos
.wrapping_add(sequence.lit_length)
.wrapping_add(sequence.match_length)
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_decodeLiteralsBlock_wrapper(
ctx: *mut ZSTD_rustBlockCtx,
src: *const c_void,
src_size: usize,
dst: *mut c_void,
dst_capacity: usize,
) -> usize {
unsafe {
*(*ctx).is_frame_decompression = 0;
decode_literals_block(
ctx,
src.cast(),
src_size,
dst.cast(),
dst_capacity,
NOT_STREAMING,
)
}
}
#[inline]
fn max_short_offset() -> usize {
if MEM_64bits() {
usize::MAX
} else {
((1usize << 26) - 1).wrapping_sub(ZSTD_REP_NUM)
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_decompressBlock_internal(
ctx: *mut ZSTD_rustBlockCtx,
dst: *mut c_void,
dst_capacity: usize,
src: *const c_void,
src_size: usize,
streaming: c_int,
) -> usize {
let block_max = unsafe { block_size_max(ctx) };
if src_size > block_max {
return ERROR(ZstdErrorCode::SrcSizeWrong);
}
let mut input = src.cast::<u8>();
let lit_size =
unsafe { decode_literals_block(ctx, input, src_size, dst.cast(), dst_capacity, streaming) };
if ERR_isError(lit_size) || lit_size > src_size {
return lit_size;
}
input = unsafe { input.add(lit_size) };
let remaining = src_size - lit_size;
let history_end = unsafe { ptr_add(dst.cast(), min(dst_capacity, block_max)) };
let history_size = unsafe { address_distance(history_end.cast_const(), *(*ctx).virtual_start) };
let long_offsets = MEM_32bits() && history_size > max_short_offset();
let mut nb_seq = 0 as c_int;
let header_size =
unsafe { ZSTD_rust_decodeSeqHeaders(ctx, &mut nb_seq, input.cast(), remaining) };
if ERR_isError(header_size) || header_size > remaining {
return header_size;
}
input = unsafe { input.add(header_size) };
let sequence_size = remaining - header_size;
if (dst.is_null() || dst_capacity == 0) && nb_seq > 0 {
return ERROR(ZstdErrorCode::DstSizeTooSmall);
}
unsafe { *(*ctx).ddict_is_cold = 0 };
unsafe {
decompress_sequences(
ctx,
dst.cast(),
dst_capacity,
input,
sequence_size,
nb_seq,
long_offsets,
)
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_checkContinuity(
ctx: *mut ZSTD_rustBlockCtx,
dst: *const c_void,
dst_size: usize,
) {
unsafe {
if dst.cast::<u8>() != *(*ctx).previous_dst_end && dst_size != 0 {
*(*ctx).dict_end = *(*ctx).previous_dst_end;
let prefix = *(*ctx).prefix_start;
let previous = *(*ctx).previous_dst_end;
let distance = address_distance(previous, prefix);
*(*ctx).virtual_start = dst.cast::<u8>().wrapping_sub(distance);
*(*ctx).prefix_start = dst.cast();
*(*ctx).previous_dst_end = dst.cast();
}
}
}
#[no_mangle]
pub unsafe extern "C" fn ZSTD_rust_decompressBlock_deprecated(
ctx: *mut ZSTD_rustBlockCtx,
dst: *mut c_void,
dst_capacity: usize,
src: *const c_void,
src_size: usize,
) -> usize {
unsafe {
*(*ctx).is_frame_decompression = 0;
ZSTD_rust_checkContinuity(ctx, dst.cast(), dst_capacity);
let result = ZSTD_rust_decompressBlock_internal(
ctx,
dst,
dst_capacity,
src,
src_size,
NOT_STREAMING,
);
if !ERR_isError(result) {
*(*ctx).previous_dst_end = ptr_add(dst.cast(), result).cast_const();
}
result
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn offset_code_22_and_23_keep_their_22_and_23_bit_bases() {
assert_eq!(OF_BASE[22], 0x3F_FFFD);
assert_eq!(OF_BASE[23], 0x7F_FFFD);
assert_eq!(OF_BASE[24], 0xFF_FFFD);
let mut normalized = [0i16; MAX_OFF + 1];
normalized[22] = 1 << 5;
let mut table = [ZSTD_seqSymbol::default(); 1 + (1 << 5)];
let mut workspace = [0u32; ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32];
unsafe {
build_fse_table_body(
table.as_mut_ptr(),
normalized.as_ptr(),
22,
OF_BASE.as_ptr(),
OF_BITS.as_ptr(),
5,
workspace.as_mut_ptr(),
std::mem::size_of_val(&workspace),
);
}
assert!(table[1..]
.iter()
.all(|symbol| symbol.base_value == 0x3F_FFFD));
}
}