ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix overlap write scenario in presence of incompressible data

Browse Source
This commit is contained in:
Yann Collet
2024-10-23 11:50:56 -07:00
parent fa147cbb4d
commit 83a3402a92
1 changed files with 11 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/compress/zstd_compress.c
+11 -5
View File
@@ -1631,7 +1631,8 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
{ {
ZSTD_compressionParameters cParams; ZSTD_compressionParameters cParams;
if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) {
srcSizeHint = CCtxParams->srcSizeHint; assert(srcSizeHint>=0);
srcSizeHint = (U64)CCtxParams->srcSizeHint;
} }
cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode); cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode);
if (CCtxParams->ldmParams.enableLdm == ZSTD_ps_enable) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; if (CCtxParams->ldmParams.enableLdm == ZSTD_ps_enable) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
@@ -4488,14 +4489,18 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms,
#include "zstd_preSplit.h" #include "zstd_preSplit.h"
static size_t ZSTD_optimalBlockSize(const void* src, size_t srcSize, size_t blockSizeMax, ZSTD_strategy strat) static size_t ZSTD_optimalBlockSize(const void* src, size_t srcSize, size_t blockSizeMax, ZSTD_strategy strat, S64 savings)
{ {
if (srcSize <= 128 KB || blockSizeMax < 128 KB) if (srcSize <= 128 KB || blockSizeMax < 128 KB)
return MIN(srcSize, blockSizeMax); return MIN(srcSize, blockSizeMax);
(void)strat; (void)strat;
if (strat >= ZSTD_btlazy2) if (strat >= ZSTD_btlazy2)
return ZSTD_splitBlock_4k(src, srcSize, blockSizeMax); return ZSTD_splitBlock_4k(src, srcSize, blockSizeMax);
return 92 KB; /* blind split strategy
* heuristic, just tested as being "generally better"
* do not split incompressible data though: just respect the 3 bytes per block overhead limit.
*/
return savings ? 92 KB : 128 KB;
} }
/*! ZSTD_compress_frameChunk() : /*! ZSTD_compress_frameChunk() :
@@ -4516,6 +4521,7 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
BYTE* const ostart = (BYTE*)dst; BYTE* const ostart = (BYTE*)dst;
BYTE* op = ostart; BYTE* op = ostart;
U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog;
S64 savings = (S64)cctx->consumedSrcSize - (S64)cctx->producedCSize;
assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX);
@@ -4526,7 +4532,7 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
while (remaining) { while (remaining) {
ZSTD_matchState_t* const ms = &cctx->blockState.matchState; ZSTD_matchState_t* const ms = &cctx->blockState.matchState;
U32 const lastBlock = lastFrameChunk & (blockSizeMax >= remaining); U32 const lastBlock = lastFrameChunk & (blockSizeMax >= remaining);
size_t blockSize = ZSTD_optimalBlockSize(ip, remaining, blockSizeMax, cctx->appliedParams.cParams.strategy); size_t const blockSize = ZSTD_optimalBlockSize(ip, remaining, blockSizeMax, cctx->appliedParams.cParams.strategy, savings);
assert(blockSize <= remaining); assert(blockSize <= remaining);
/* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding
@@ -4571,7 +4577,7 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
} }
} /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/ } /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/
if (cSize < blockSize) savings += (blockSize - cSize);
ip += blockSize; ip += blockSize;
assert(remaining >= blockSize); assert(remaining >= blockSize);
remaining -= blockSize; remaining -= blockSize;