ddidderr/zstd-rs
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reduce size of dctx by reutilizing dst buffer (#2751)

Browse Source 
* Reduce size of dctx by reutilizing dst buffer

Co-authored-by: Binh Vo <binhvo@fb.com>
This commit is contained in:
binhdvoandBinh Vo authored and GitHub committed 2021-10-25 10:38:01 -04:00
1 parent 0a794f5afe
commit 6a7ede3dfc
7 files changed
+599 -39
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

lib/README.md
+6
View File
@@ -155,6 +155,12 @@ The file structure is designed to make this selection manually achievable for an
- The build macro `ZSTD_NO_INTRINSICS` can be defined to disable all explicit intrinsics. - The build macro `ZSTD_NO_INTRINSICS` can be defined to disable all explicit intrinsics.
Compiler builtins are still used. Compiler builtins are still used.
- The build macro `ZSTD_LITBUFFEREXTRASIZE` can be set to control the amount of extra memory used
during decompression to store literals. This defaults to 64kB. Reducing it can reduce the
memory footprint required for decompression by increasing the portion of the literal buffer that
is stored in the unwritten portion of the dst buffer, at the cost of performance impact for
decompression.
#### Windows : using MinGW+MSYS to create DLL #### Windows : using MinGW+MSYS to create DLL
lib/common/zstd_internal.h
+1 -3
View File
@@ -181,7 +181,7 @@ static void ZSTD_copy16(void* dst, const void* src) {
#if defined(ZSTD_ARCH_ARM_NEON) #if defined(ZSTD_ARCH_ARM_NEON)
vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src)); vst1q_u8((uint8_t*)dst, vld1q_u8((const uint8_t*)src));
#else #else
ZSTD_memcpy(dst, src, 16); ZSTD_memmove(dst, src, 16);
#endif #endif
} }
#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; } #define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; }
@@ -210,8 +210,6 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e
BYTE* op = (BYTE*)dst; BYTE* op = (BYTE*)dst;
BYTE* const oend = op + length; BYTE* const oend = op + length;
assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff <= -WILDCOPY_VECLEN));
if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) { if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) {
/* Handle short offset copies. */ /* Handle short offset copies. */
do { do {
lib/decompress/zstd_decompress.c
+4 -3
View File
@@ -916,7 +916,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
switch(blockProperties.blockType) switch(blockProperties.blockType)
{ {
case bt_compressed: case bt_compressed:
decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1); decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1, not_streaming);
break; break;
case bt_raw : case bt_raw :
decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize); decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize);
@@ -1229,7 +1229,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
{ {
case bt_compressed: case bt_compressed:
DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed");
rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1); rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1, is_streaming);
dctx->expected = 0; /* Streaming not supported */ dctx->expected = 0; /* Streaming not supported */
break; break;
case bt_raw : case bt_raw :
@@ -1824,7 +1824,8 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx)
size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize)
{ {
size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX);
unsigned long long const neededRBSize = windowSize + blockSize + (WILDCOPY_OVERLENGTH * 2); /* space is needed to store the litbuffer after the output of a given block without stomping the extDict of a previous run, as well as to cover both windows against wildcopy*/
unsigned long long const neededRBSize = windowSize + blockSize + ZSTD_BLOCKSIZE_MAX + (WILDCOPY_OVERLENGTH * 2);
unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); unsigned long long const neededSize = MIN(frameContentSize, neededRBSize);
size_t const minRBSize = (size_t) neededSize; size_t const minRBSize = (size_t) neededSize;
RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize,
lib/decompress/zstd_decompress_block.c
+564 -28
View File
@@ -69,15 +69,56 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
} }
} }
/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */
static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize,
const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately)
{
if (streaming == not_streaming && dstCapacity > ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH)
{
/* room for litbuffer to fit without read faulting */
dctx->litBuffer = (BYTE*)dst + ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH;
dctx->litBufferEnd = dctx->litBuffer + litSize;
dctx->litBufferLocation = ZSTD_in_dst;
}
else if (litSize > ZSTD_LITBUFFEREXTRASIZE)
{
/* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */
if (splitImmediately) {
/* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */
dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH;
dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE;
}
else {
/* initially this will be stored entirely in dst during huffman decoding, it will partially shifted to litExtraBuffer after */
dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize;
dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize;
}
dctx->litBufferLocation = ZSTD_split;
}
else
{
/* fits entirely within litExtraBuffer, so no split is necessary */
dctx->litBuffer = dctx->litExtraBuffer;
dctx->litBufferEnd = dctx->litBuffer + litSize;
dctx->litBufferLocation = ZSTD_not_in_dst;
}
}
/* Hidden declaration for fullbench */ /* Hidden declaration for fullbench */
size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
const void* src, size_t srcSize); const void* src, size_t srcSize,
void* dst, size_t dstCapacity, const streaming_operation streaming);
/*! ZSTD_decodeLiteralsBlock() : /*! ZSTD_decodeLiteralsBlock() :
* Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored
* in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current
* block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being
* stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write.
*
* @return : nb of bytes read from src (< srcSize ) * @return : nb of bytes read from src (< srcSize )
* note : symbol not declared but exposed for fullbench */ * note : symbol not declared but exposed for fullbench */
size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */
void* dst, size_t dstCapacity, const streaming_operation streaming)
{ {
DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); DEBUGLOG(5, "ZSTD_decodeLiteralsBlock");
RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, ""); RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, "");
@@ -99,6 +140,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
U32 const lhlCode = (istart[0] >> 2) & 3; U32 const lhlCode = (istart[0] >> 2) & 3;
U32 const lhc = MEM_readLE32(istart); U32 const lhc = MEM_readLE32(istart);
size_t hufSuccess; size_t hufSuccess;
size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
switch(lhlCode) switch(lhlCode)
{ {
case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */
@@ -121,8 +163,11 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); litCSize = (lhc >> 22) + ((size_t)istart[4] << 10);
break; break;
} }
RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, "");
RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, "");
RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, "");
ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0);
/* prefetch huffman table if cold */ /* prefetch huffman table if cold */
if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) {
@@ -159,6 +204,13 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx)); sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx));
} }
} }
if (dctx->litBufferLocation == ZSTD_split)
{
ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE);
ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE);
dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH;
dctx->litBufferEnd -= WILDCOPY_OVERLENGTH;
}
RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, ""); RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, "");
@@ -166,13 +218,13 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
dctx->litSize = litSize; dctx->litSize = litSize;
dctx->litEntropy = 1; dctx->litEntropy = 1;
if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable;
ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return litCSize + lhSize; return litCSize + lhSize;
} }
case set_basic: case set_basic:
{ size_t litSize, lhSize; { size_t litSize, lhSize;
U32 const lhlCode = ((istart[0]) >> 2) & 3; U32 const lhlCode = ((istart[0]) >> 2) & 3;
size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
switch(lhlCode) switch(lhlCode)
{ {
case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */
@@ -189,23 +241,36 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
break; break;
} }
RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, "");
ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1);
if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */
RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, ""); RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, "");
ZSTD_memcpy(dctx->litBuffer, istart+lhSize, litSize); if (dctx->litBufferLocation == ZSTD_split)
{
ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE);
ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE);
}
else
{
ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize);
}
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize; dctx->litSize = litSize;
ZSTD_memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
return lhSize+litSize; return lhSize+litSize;
} }
/* direct reference into compressed stream */ /* direct reference into compressed stream */
dctx->litPtr = istart+lhSize; dctx->litPtr = istart+lhSize;
dctx->litSize = litSize; dctx->litSize = litSize;
dctx->litBufferEnd = dctx->litPtr + litSize;
dctx->litBufferLocation = ZSTD_not_in_dst;
return lhSize+litSize; return lhSize+litSize;
} }
case set_rle: case set_rle:
{ U32 const lhlCode = ((istart[0]) >> 2) & 3; { U32 const lhlCode = ((istart[0]) >> 2) & 3;
size_t litSize, lhSize; size_t litSize, lhSize;
size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
switch(lhlCode) switch(lhlCode)
{ {
case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */
@@ -222,8 +287,19 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4"); RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4");
break; break;
} }
RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, "");
ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, "");
ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1);
if (dctx->litBufferLocation == ZSTD_split)
{
ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE);
ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE);
}
else
{
ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize);
}
dctx->litPtr = dctx->litBuffer; dctx->litPtr = dctx->litBuffer;
dctx->litSize = litSize; dctx->litSize = litSize;
return lhSize+1; return lhSize+1;
@@ -713,7 +789,7 @@ HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) {
* - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart.
* The src buffer must be before the dst buffer. * The src buffer must be before the dst buffer.
*/ */
static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) {
ptrdiff_t const diff = op - ip; ptrdiff_t const diff = op - ip;
BYTE* const oend = op + length; BYTE* const oend = op + length;
@@ -729,6 +805,7 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_
/* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */
assert(length >= 8); assert(length >= 8);
ZSTD_overlapCopy8(&op, &ip, diff); ZSTD_overlapCopy8(&op, &ip, diff);
length -= 8;
assert(op - ip >= 8); assert(op - ip >= 8);
assert(op <= oend); assert(op <= oend);
} }
@@ -743,12 +820,35 @@ static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_
assert(oend > oend_w); assert(oend > oend_w);
ZSTD_wildcopy(op, ip, oend_w - op, ovtype); ZSTD_wildcopy(op, ip, oend_w - op, ovtype);
ip += oend_w - op; ip += oend_w - op;
op = oend_w; op += oend_w - op;
} }
/* Handle the leftovers. */ /* Handle the leftovers. */
while (op < oend) *op++ = *ip++; while (op < oend) *op++ = *ip++;
} }
/* ZSTD_safecopyDstBeforeSrc():
* This version allows overlap with dst before src, or handles the non-overlap case with dst after src
* Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */
static void ZSTD_safecopyDstBeforeSrc(BYTE* op, BYTE const* ip, ptrdiff_t length) {
ptrdiff_t const diff = op - ip;
BYTE* const oend = op + length;
if (length < 8 || diff > -8) {
/* Handle short lengths, close overlaps, and dst not before src. */
while (op < oend) *op++ = *ip++;
return;
}
if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) {
ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap);
ip += oend - WILDCOPY_OVERLENGTH - op;
op += oend - WILDCOPY_OVERLENGTH - op;
}
/* Handle the leftovers. */
while (op < oend) *op++ = *ip++;
}
/* ZSTD_execSequenceEnd(): /* ZSTD_execSequenceEnd():
* This version handles cases that are near the end of the output buffer. It requires * This version handles cases that are near the end of the output buffer. It requires
* more careful checks to make sure there is no overflow. By separating out these hard * more careful checks to make sure there is no overflow. By separating out these hard
@@ -784,7 +884,7 @@ size_t ZSTD_execSequenceEnd(BYTE* op,
if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
/* offset beyond prefix */ /* offset beyond prefix */
RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, "");
match = dictEnd - (prefixStart-match); match = dictEnd - (prefixStart - match);
if (match + sequence.matchLength <= dictEnd) { if (match + sequence.matchLength <= dictEnd) {
ZSTD_memmove(oLitEnd, match, sequence.matchLength); ZSTD_memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength; return sequenceLength;
@@ -795,7 +895,56 @@ size_t ZSTD_execSequenceEnd(BYTE* op,
op = oLitEnd + length1; op = oLitEnd + length1;
sequence.matchLength -= length1; sequence.matchLength -= length1;
match = prefixStart; match = prefixStart;
} } }
}
ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst);
return sequenceLength;
}
/* ZSTD_execSequenceEndSplitLitBuffer():
* This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case.
*/
FORCE_NOINLINE
size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op,
BYTE* const oend, const BYTE* const oend_w, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
const BYTE* match = oLitEnd - sequence.offset;
/* bounds checks : careful of address space overflow in 32-bit mode */
RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer");
RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer");
assert(op < op + sequenceLength);
assert(oLitEnd < op + sequenceLength);
/* copy literals */
RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer");
ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength);
op = oLitEnd;
*litPtr = iLitEnd;
/* copy Match */
if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
/* offset beyond prefix */
RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, "");
match = dictEnd - (prefixStart - match);
if (match + sequence.matchLength <= dictEnd) {
ZSTD_memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength;
}
/* span extDict & currentPrefixSegment */
{ size_t const length1 = dictEnd - match;
ZSTD_memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = prefixStart;
}
}
ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst);
return sequenceLength; return sequenceLength;
} }
@@ -834,6 +983,98 @@ size_t ZSTD_execSequence(BYTE* op,
assert(oLitEnd <= oend_w /* Can wildcopy literals */); assert(oLitEnd <= oend_w /* Can wildcopy literals */);
assert(oMatchEnd <= oend_w /* Can wildcopy matches */); assert(oMatchEnd <= oend_w /* Can wildcopy matches */);
/* Copy Literals:
* Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.
* We likely don't need the full 32-byte wildcopy.
*/
assert(WILDCOPY_OVERLENGTH >= 16);
ZSTD_copy16(op, (*litPtr));
if (UNLIKELY(sequence.litLength > 16)) {
ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap);
}
op = oLitEnd;
*litPtr = iLitEnd; /* update for next sequence */
/* Copy Match */
if (sequence.offset > (size_t)(oLitEnd - prefixStart)) {
/* offset beyond prefix -> go into extDict */
RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, "");
match = dictEnd + (match - prefixStart);
if (match + sequence.matchLength <= dictEnd) {
ZSTD_memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength;
}
/* span extDict & currentPrefixSegment */
{ size_t const length1 = dictEnd - match;
ZSTD_memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = prefixStart;
}
}
/* Match within prefix of 1 or more bytes */
assert(op <= oMatchEnd);
assert(oMatchEnd <= oend_w);
assert(match >= prefixStart);
assert(sequence.matchLength >= 1);
/* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy
* without overlap checking.
*/
if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) {
/* We bet on a full wildcopy for matches, since we expect matches to be
* longer than literals (in general). In silesia, ~10% of matches are longer
* than 16 bytes.
*/
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap);
return sequenceLength;
}
assert(sequence.offset < WILDCOPY_VECLEN);
/* Copy 8 bytes and spread the offset to be >= 8. */
ZSTD_overlapCopy8(&op, &match, sequence.offset);
/* If the match length is > 8 bytes, then continue with the wildcopy. */
if (sequence.matchLength > 8) {
assert(op < oMatchEnd);
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst);
}
return sequenceLength;
}
HINT_INLINE
size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op,
BYTE* const oend, const BYTE* const oend_w, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit,
const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
const BYTE* match = oLitEnd - sequence.offset;
assert(op != NULL /* Precondition */);
assert(oend_w < oend /* No underflow */);
/* Handle edge cases in a slow path:
* - Read beyond end of literals
* - Match end is within WILDCOPY_OVERLIMIT of oend
* - 32-bit mode and the match length overflows
*/
if (UNLIKELY(
iLitEnd > litLimit ||
oMatchEnd > oend_w ||
(MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH)))
return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd);
/* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */
assert(op <= oLitEnd /* No overflow */);
assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */);
assert(oMatchEnd <= oend /* No underflow */);
assert(iLitEnd <= litLimit /* Literal length is in bounds */);
assert(oLitEnd <= oend_w /* Can wildcopy literals */);
assert(oMatchEnd <= oend_w /* Can wildcopy matches */);
/* Copy Literals: /* Copy Literals:
* Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9.
* We likely don't need the full 32-byte wildcopy. * We likely don't need the full 32-byte wildcopy.
@@ -892,6 +1133,7 @@ size_t ZSTD_execSequence(BYTE* op,
return sequenceLength; return sequenceLength;
} }
static void static void
ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt)
{ {
@@ -1073,9 +1315,11 @@ MEM_STATIC void ZSTD_assertValidSequence(
#endif #endif
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
FORCE_INLINE_TEMPLATE size_t FORCE_INLINE_TEMPLATE size_t
DONT_VECTORIZE DONT_VECTORIZE
ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize, void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq, const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset, const ZSTD_longOffset_e isLongOffset,
@@ -1087,11 +1331,11 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx,
BYTE* const oend = ostart + maxDstSize; BYTE* const oend = ostart + maxDstSize;
BYTE* op = ostart; BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr; const BYTE* litPtr = dctx->litPtr;
const BYTE* const litEnd = litPtr + dctx->litSize; const BYTE* litBufferEnd = dctx->litBufferEnd;
const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); const BYTE* const vBase = (const BYTE*) (dctx->virtualStart);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
DEBUGLOG(5, "ZSTD_decompressSequences_body"); DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer");
(void)frame; (void)frame;
/* Regen sequences */ /* Regen sequences */
@@ -1112,7 +1356,9 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx,
BIT_DStream_endOfBuffer < BIT_DStream_completed && BIT_DStream_endOfBuffer < BIT_DStream_completed &&
BIT_DStream_completed < BIT_DStream_overflow); BIT_DStream_completed < BIT_DStream_overflow);
#if defined(__GNUC__) && defined(__x86_64__) /* decompress without overrunning litPtr begins */
{
seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
/* Align the decompression loop to 32 + 16 bytes. /* Align the decompression loop to 32 + 16 bytes.
* *
* zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression
@@ -1145,22 +1391,202 @@ ZSTD_decompressSequences_body( ZSTD_DCtx* dctx,
* - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH
* - Skylake * - Skylake
* *
* Alignment is done for each of the three major decompression loops:
* - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer
* - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer
* - ZSTD_decompressSequences_body
* Alignment choices are made to minimize large swings on bad cases and influence on performance
* from changes external to this code, rather than to overoptimize on the current commit.
*
* If you are seeing performance stability this script can help test. * If you are seeing performance stability this script can help test.
* It tests on 4 commits in zstd where I saw performance change. * It tests on 4 commits in zstd where I saw performance change.
* *
* https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4
*/ */
#if defined(__GNUC__) && defined(__x86_64__)
__asm__(".p2align 6"); __asm__(".p2align 6");
# if __GNUC__ >= 7
/* good for gcc-7, gcc-9, and gcc-11 */
__asm__("nop"); __asm__("nop");
__asm__(".p2align 5"); __asm__(".p2align 5");
__asm__("nop"); __asm__("nop");
# if __GNUC__ >= 9 && __GNUC__ < 11 __asm__(".p2align 4");
/* better for gcc-9 and gcc-10, worse for clang and gcc-8, gcc-11 */ # if __GNUC__ == 8 || __GNUC__ == 10
/* good for gcc-8 and gcc-10 */
__asm__("nop");
__asm__(".p2align 3");
# endif
# endif
#endif
/* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */
for (; litPtr + sequence.litLength <= dctx->litBufferEnd; ) {
size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
#endif
if (UNLIKELY(ZSTD_isError(oneSeqSize)))
return oneSeqSize;
DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
op += oneSeqSize;
if (UNLIKELY(!--nbSeq))
break;
BIT_reloadDStream(&(seqState.DStream));
sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
}
/* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */
if (nbSeq > 0) {
const size_t leftoverLit = dctx->litBufferEnd - litPtr;
if (leftoverLit)
{
RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
sequence.litLength -= leftoverLit;
op += leftoverLit;
}
litPtr = dctx->litExtraBuffer;
litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
dctx->litBufferLocation = ZSTD_not_in_dst;
{
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
#endif
if (UNLIKELY(ZSTD_isError(oneSeqSize)))
return oneSeqSize;
DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
op += oneSeqSize;
if (--nbSeq)
BIT_reloadDStream(&(seqState.DStream));
}
}
}
if (nbSeq > 0) /* there is remaining lit from extra buffer */
{
#if defined(__GNUC__) && defined(__x86_64__)
__asm__(".p2align 6");
__asm__("nop");
# if __GNUC__ != 7
/* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */
__asm__(".p2align 4");
__asm__("nop");
__asm__(".p2align 3");
# elif __GNUC__ >= 11
__asm__(".p2align 3");
# else
__asm__(".p2align 5");
__asm__("nop");
__asm__(".p2align 3");
# endif
#endif
for (; ; ) {
seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase);
#endif
if (UNLIKELY(ZSTD_isError(oneSeqSize)))
return oneSeqSize;
DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
op += oneSeqSize;
if (UNLIKELY(!--nbSeq))
break;
BIT_reloadDStream(&(seqState.DStream));
}
}
/* check if reached exact end */
DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq);
RETURN_ERROR_IF(nbSeq, corruption_detected, "");
RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, "");
/* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
}
/* last literal segment */
if (dctx->litBufferLocation == ZSTD_split) /* split hasn't been reached yet, first get dst then copy litExtraBuffer */
{
size_t const lastLLSize = litBufferEnd - litPtr;
RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
if (op != NULL) {
ZSTD_memmove(op, litPtr, lastLLSize);
op += lastLLSize;
}
litPtr = dctx->litExtraBuffer;
litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
dctx->litBufferLocation = ZSTD_not_in_dst;
}
{ size_t const lastLLSize = litBufferEnd - litPtr;
RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
if (op != NULL) {
ZSTD_memcpy(op, litPtr, lastLLSize);
op += lastLLSize;
}
}
return op-ostart;
}
FORCE_INLINE_TEMPLATE size_t
DONT_VECTORIZE
ZSTD_decompressSequences_body(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset,
const int frame)
{
const BYTE* ip = (const BYTE*)seqStart;
const BYTE* const iend = ip + seqSize;
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ostart + maxDstSize : dctx->litBuffer;
BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr;
const BYTE* const litEnd = litPtr + dctx->litSize;
const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart);
const BYTE* const vBase = (const BYTE*)(dctx->virtualStart);
const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd);
DEBUGLOG(5, "ZSTD_decompressSequences_body");
(void)frame;
/* Regen sequences */
if (nbSeq) {
seqState_t seqState;
dctx->fseEntropy = 1;
{ U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; }
RETURN_ERROR_IF(
ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)),
corruption_detected, "");
ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
assert(dst != NULL);
ZSTD_STATIC_ASSERT(
BIT_DStream_unfinished < BIT_DStream_completed &&
BIT_DStream_endOfBuffer < BIT_DStream_completed &&
BIT_DStream_completed < BIT_DStream_overflow);
#if defined(__GNUC__) && defined(__x86_64__)
__asm__(".p2align 6");
__asm__("nop");
# if __GNUC__ >= 7
__asm__(".p2align 5");
__asm__("nop");
__asm__(".p2align 3"); __asm__(".p2align 3");
# else # else
__asm__(".p2align 4"); __asm__(".p2align 4");
__asm__("nop");
__asm__(".p2align 3");
# endif # endif
#endif #endif
for ( ; ; ) { for ( ; ; ) {
seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd);
@@ -1206,6 +1632,16 @@ ZSTD_decompressSequences_default(ZSTD_DCtx* dctx,
{ {
return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
} }
static size_t
ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset,
const int frame)
{
return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
}
#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
@@ -1241,7 +1677,7 @@ ZSTD_decompressSequencesLong_body(
BYTE* const oend = ostart + maxDstSize; BYTE* const oend = ostart + maxDstSize;
BYTE* op = ostart; BYTE* op = ostart;
const BYTE* litPtr = dctx->litPtr; const BYTE* litPtr = dctx->litPtr;
const BYTE* const litEnd = litPtr + dctx->litSize; const BYTE* litBufferEnd = dctx->litBufferEnd;
const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart);
const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
@@ -1277,13 +1713,29 @@ ZSTD_decompressSequencesLong_body(
} }
RETURN_ERROR_IF(seqNb<seqAdvance, corruption_detected, ""); RETURN_ERROR_IF(seqNb<seqAdvance, corruption_detected, "");
/* decode and decompress */ /* decompress without stomping litBuffer */
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && (seqNb<nbSeq) ; seqNb++) { for (; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && (seqNb < nbSeq); seqNb++) {
seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset);
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd); size_t oneSeqSize;
if (dctx->litBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd)
{
/* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */
const size_t leftoverLit = dctx->litBufferEnd - litPtr;
if (leftoverLit)
{
RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit;
op += leftoverLit;
}
litPtr = dctx->litExtraBuffer;
litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
dctx->litBufferLocation = ZSTD_not_in_dst;
oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize)); assert(!ZSTD_isError(oneSeqSize));
if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb-ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart);
#endif #endif
if (ZSTD_isError(oneSeqSize)) return oneSeqSize; if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
@@ -1291,12 +1743,44 @@ ZSTD_decompressSequencesLong_body(
sequences[seqNb & STORED_SEQS_MASK] = sequence; sequences[seqNb & STORED_SEQS_MASK] = sequence;
op += oneSeqSize; op += oneSeqSize;
} }
else
{
/* lit buffer is either wholly contained in first or second split, or not split at all*/
oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) :
ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart);
#endif
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd);
sequences[seqNb & STORED_SEQS_MASK] = sequence;
op += oneSeqSize;
}
}
RETURN_ERROR_IF(seqNb<nbSeq, corruption_detected, ""); RETURN_ERROR_IF(seqNb<nbSeq, corruption_detected, "");
/* finish queue */ /* finish queue */
seqNb -= seqAdvance; seqNb -= seqAdvance;
for ( ; seqNb<nbSeq ; seqNb++) { for ( ; seqNb<nbSeq ; seqNb++) {
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[seqNb&STORED_SEQS_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd); seq_t *sequence = &(sequences[seqNb&STORED_SEQS_MASK]);
if (dctx->litBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd)
{
const size_t leftoverLit = dctx->litBufferEnd - litPtr;
if (leftoverLit)
{
RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer");
ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit);
sequence->litLength -= leftoverLit;
op += leftoverLit;
}
litPtr = dctx->litExtraBuffer;
litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
dctx->litBufferLocation = ZSTD_not_in_dst;
{
size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) #if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize)); assert(!ZSTD_isError(oneSeqSize));
if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart);
@@ -1304,16 +1788,41 @@ ZSTD_decompressSequencesLong_body(
if (ZSTD_isError(oneSeqSize)) return oneSeqSize; if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize; op += oneSeqSize;
} }
}
else
{
size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ?
ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) :
ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd);
#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE)
assert(!ZSTD_isError(oneSeqSize));
if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart);
#endif
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
}
}
/* save reps for next block */ /* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); } { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
} }
/* last literal segment */ /* last literal segment */
{ size_t const lastLLSize = litEnd - litPtr; if (dctx->litBufferLocation == ZSTD_split) /* first deplete literal buffer in dst, then copy litExtraBuffer */
{
size_t const lastLLSize = litBufferEnd - litPtr;
RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, "");
if (op != NULL) {
ZSTD_memmove(op, litPtr, lastLLSize);
op += lastLLSize;
}
litPtr = dctx->litExtraBuffer;
litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE;
}
{ size_t const lastLLSize = litBufferEnd - litPtr;
RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, "");
if (op != NULL) { if (op != NULL) {
ZSTD_memcpy(op, litPtr, lastLLSize); ZSTD_memmove(op, litPtr, lastLLSize);
op += lastLLSize; op += lastLLSize;
} }
} }
@@ -1347,6 +1856,16 @@ ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx,
{ {
return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
} }
static TARGET_ATTRIBUTE("bmi2") size_t
DONT_VECTORIZE
ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx,
void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset,
const int frame)
{
return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
}
#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
@@ -1385,6 +1904,20 @@ ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
#endif #endif
return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
} }
static size_t
ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize,
const void* seqStart, size_t seqSize, int nbSeq,
const ZSTD_longOffset_e isLongOffset,
const int frame)
{
DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer");
#if DYNAMIC_BMI2
if (ZSTD_DCtx_get_bmi2(dctx)) {
return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
}
#endif
return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame);
}
#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */
@@ -1444,7 +1977,7 @@ ZSTD_getLongOffsetsShare(const ZSTD_seqSymbol* offTable)
size_t size_t
ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const int frame) const void* src, size_t srcSize, const int frame, const streaming_operation streaming)
{ /* blockType == blockCompressed */ { /* blockType == blockCompressed */
const BYTE* ip = (const BYTE*)src; const BYTE* ip = (const BYTE*)src;
/* isLongOffset must be true if there are long offsets. /* isLongOffset must be true if there are long offsets.
@@ -1459,7 +1992,7 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, ""); RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, "");
/* Decode literals section */ /* Decode literals section */
{ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming);
DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize); DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize);
if (ZSTD_isError(litCSize)) return litCSize; if (ZSTD_isError(litCSize)) return litCSize;
ip += litCSize; ip += litCSize;
@@ -1507,6 +2040,9 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
/* else */ /* else */
if (dctx->litBufferLocation == ZSTD_split)
return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame);
else
return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame);
#endif #endif
} }
@@ -1530,7 +2066,7 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
{ {
size_t dSize; size_t dSize;
ZSTD_checkContinuity(dctx, dst, dstCapacity); ZSTD_checkContinuity(dctx, dst, dstCapacity);
dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0); dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0, not_streaming);
dctx->previousDstEnd = (char*)dst + dSize; dctx->previousDstEnd = (char*)dst + dSize;
return dSize; return dSize;
} }
lib/decompress/zstd_decompress_block.h
+7 -1
View File
@@ -33,6 +33,12 @@
*/ */
/* Streaming state is used to inform allocation of the literal buffer */
typedef enum {
not_streaming = 0,
is_streaming = 1
} streaming_operation;
/* ZSTD_decompressBlock_internal() : /* ZSTD_decompressBlock_internal() :
* decompress block, starting at `src`, * decompress block, starting at `src`,
* into destination buffer `dst`. * into destination buffer `dst`.
@@ -41,7 +47,7 @@
*/ */
size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const int frame); const void* src, size_t srcSize, const int frame, const streaming_operation streaming);
/* ZSTD_buildFSETable() : /* ZSTD_buildFSETable() :
* generate FSE decoding table for one symbol (ll, ml or off) * generate FSE decoding table for one symbol (ll, ml or off)
lib/decompress/zstd_decompress_internal.h
+14 -1
View File
@@ -106,6 +106,16 @@ typedef struct {
size_t ddictPtrCount; size_t ddictPtrCount;
} ZSTD_DDictHashSet; } ZSTD_DDictHashSet;
#ifndef ZSTD_LITBUFFEREXTRASIZE
#define ZSTD_LITBUFFEREXTRASIZE (1 << 16) /* extra buffer reduces amount of dst required to store litBuffer */
#endif
typedef enum {
ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */
ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */
ZSTD_split = 2 /* Split between litExtraBuffer and dst */
} ZSTD_litLocation_e;
struct ZSTD_DCtx_s struct ZSTD_DCtx_s
{ {
const ZSTD_seqSymbol* LLTptr; const ZSTD_seqSymbol* LLTptr;
@@ -171,7 +181,10 @@ struct ZSTD_DCtx_s
ZSTD_outBuffer expectedOutBuffer; ZSTD_outBuffer expectedOutBuffer;
/* workspace */ /* workspace */
BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; BYTE* litBuffer;
const BYTE* litBufferEnd;
ZSTD_litLocation_e litBufferLocation;
BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t oversizedDuration; size_t oversizedDuration;
tests/fullbench.c
+3 -3
View File
@@ -123,11 +123,11 @@ static size_t local_ZSTD_decompress(const void* src, size_t srcSize,
static ZSTD_DCtx* g_zdc = NULL; static ZSTD_DCtx* g_zdc = NULL;
#ifndef ZSTD_DLL_IMPORT #ifndef ZSTD_DLL_IMPORT
extern size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* ctx, const void* src, size_t srcSize); extern size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* ctx, const void* src, size_t srcSize, void* dst, size_t dstCapacity);
static size_t local_ZSTD_decodeLiteralsBlock(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) static size_t local_ZSTD_decodeLiteralsBlock(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
{ {
(void)src; (void)srcSize; (void)dst; (void)dstSize; (void)src; (void)srcSize; (void)dst; (void)dstSize;
return ZSTD_decodeLiteralsBlock(g_zdc, buff2, g_cSize); return ZSTD_decodeLiteralsBlock(g_zdc, buff2, g_cSize, dst, dstSize);
} }
static size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) static size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2)
@@ -577,7 +577,7 @@ static int benchMem(unsigned benchNb,
ip += ZSTD_blockHeaderSize; /* skip block header */ ip += ZSTD_blockHeaderSize; /* skip block header */
ZSTD_decompressBegin(g_zdc); ZSTD_decompressBegin(g_zdc);
CONTROL(iend > ip); CONTROL(iend > ip);
ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, (size_t)(iend-ip)); /* skip literal segment */ ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, (size_t)(iend-ip), dstBuff, dstBuffSize); /* skip literal segment */
g_cSize = (size_t)(iend-ip); g_cSize = (size_t)(iend-ip);
memcpy(dstBuff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */ memcpy(dstBuff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */
srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */ srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */