diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 9fa935d81..18c7cc14b 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2351,6 +2351,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, ZSTD_matchState_t* const ms = &zc->blockState.matchState; DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%zu, dictLimit=%u, nextToUpdate=%u)", dstCapacity, ms->window.dictLimit, ms->nextToUpdate); + assert(srcSize <= ZSTD_BLOCKSIZE_MAX); if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.searchLength); @@ -2478,7 +2479,6 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); - ZSTD_reduceIndex(cctx, correction); if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; else ms->nextToUpdate -= correction; @@ -2597,7 +2597,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, const void* src, size_t srcSize, U32 frame, U32 lastFrameChunk) { - ZSTD_matchState_t* ms = &cctx->blockState.matchState; + ZSTD_matchState_t* const ms = &cctx->blockState.matchState; size_t fhSize = 0; DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u", @@ -2618,8 +2618,25 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, if (!ZSTD_window_update(&ms->window, src, srcSize)) { ms->nextToUpdate = ms->window.dictLimit; } - if (cctx->appliedParams.ldmParams.enableLdm) + if (cctx->appliedParams.ldmParams.enableLdm) { ZSTD_window_update(&cctx->ldmState.window, src, srcSize); + } + + if (!frame) { + /* overflow check and correction for block mode */ + if (ZSTD_window_needOverflowCorrection(ms->window, (const char*)src + srcSize)) { + U32 const cycleLog = ZSTD_cycleLog(cctx->appliedParams.cParams.chainLog, cctx->appliedParams.cParams.strategy); + U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, 1 << cctx->appliedParams.cParams.windowLog, src); + ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + ZSTD_reduceIndex(cctx, correction); + if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; + else ms->nextToUpdate -= correction; + ms->loadedDictEnd = 0; + ms->dictMatchState = NULL; + } + } DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (U32)cctx->blockSize); { size_t const cSize = frame ? @@ -2661,6 +2678,7 @@ size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const { size_t const blockSizeMax = ZSTD_getBlockSize(cctx); if (srcSize > blockSizeMax) return ERROR(srcSize_wrong); + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); } diff --git a/lib/compress/zstd_fast.c b/lib/compress/zstd_fast.c index 7d7efb465..8d2c8452a 100644 --- a/lib/compress/zstd_fast.c +++ b/lib/compress/zstd_fast.c @@ -172,6 +172,7 @@ size_t ZSTD_compressBlock_fast_generic( if (ip <= ilimit) { /* Fill Table */ + assert(base+current+2 > istart); /* check base overflow */ hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2; /* here because current+2 could be > iend-8 */ hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 7191ffcbb..24e00e803 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1291,6 +1291,20 @@ static int basicUnitTests(U32 seed, double compressibility) if (r != blockSize) goto _output_error; } DISPLAYLEVEL(3, "OK \n"); + /* very long stream of block compression */ + DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++); + CHECK( ZSTD_compressBegin(cctx, -99) ); /* we just want to quickly overflow internal U32 index */ + CHECK( ZSTD_getBlockSize(cctx) >= blockSize); + { U64 const toCompress = 5000000000ULL; /* > 4 GB */ + U64 compressed = 0; + while (compressed < toCompress) { + size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); + if (ZSTD_isError(cSize)) goto _output_error; + compressed += blockCSize; + } + } + DISPLAYLEVEL(3, "OK \n"); + /* dictionary block compression */ DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++); CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );