From 6e66bbf5dde09f555a2adb1dd763808c0de3c6c6 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 14 Aug 2018 12:56:21 -0700 Subject: [PATCH 1/7] fixed several minor issues detected by scan-build only notable one : writeNCount() resists better vs invalid distributions (though it should never happen within zstd anyway) --- lib/common/zstd_internal.h | 3 +- lib/compress/fse_compress.c | 65 ++++++++++++++++++-------------- lib/compress/zstd_compress.c | 1 - lib/compress/zstd_opt.c | 2 +- lib/decompress/zstd_decompress.c | 26 ++++++------- lib/dictBuilder/zdict.c | 2 +- lib/legacy/zstd_v04.c | 1 + lib/legacy/zstd_v05.c | 1 + lib/legacy/zstd_v06.c | 2 +- lib/legacy/zstd_v07.c | 20 +++++----- 10 files changed, 67 insertions(+), 56 deletions(-) diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index b4c1af53f..b36a2fbd0 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -79,8 +79,7 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; -#define ZSTD_FRAMEIDSIZE 4 -static const size_t ZSTD_frameIdSize = ZSTD_FRAMEIDSIZE; /* magic number size */ +#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ #define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index 70daae3bc..caa90edec 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -83,7 +83,9 @@ * wkspSize should be sized to handle worst case situation, which is `1< wkspSize) return ERROR(tableLog_tooLarge); tableU16[-2] = (U16) tableLog; tableU16[-1] = (U16) maxSymbolValue; - assert(tableLog < 16); /* required for the threshold strategy to work */ + assert(tableLog < 16); /* required for threshold strategy to work */ /* For explanations on how to distribute symbol values over the table : - * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ + * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ + + #ifdef __clang_analyzer__ + memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */ + #endif /* symbol start positions */ { U32 u; @@ -124,13 +130,15 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi U32 symbol; for (symbol=0; symbol<=maxSymbolValue; symbol++) { int nbOccurences; - for (nbOccurences=0; nbOccurences highThreshold) position = (position + step) & tableMask; /* Low proba area */ + while (position > highThreshold) + position = (position + step) & tableMask; /* Low proba area */ } } - if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */ + assert(position==0); /* Must have initialized all positions */ } /* Build table */ @@ -201,9 +209,10 @@ size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ } -static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, - const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, - unsigned writeIsSafe) +static size_t +FSE_writeNCount_generic (void* header, size_t headerBufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, + unsigned writeIsSafe) { BYTE* const ostart = (BYTE*) header; BYTE* out = ostart; @@ -212,13 +221,12 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, const int tableSize = 1 << tableLog; int remaining; int threshold; - U32 bitStream; - int bitCount; - unsigned charnum = 0; - int previous0 = 0; + U32 bitStream = 0; + int bitCount = 0; + unsigned symbol = 0; + unsigned const alphabetSize = maxSymbolValue + 1; + int previousIs0 = 0; - bitStream = 0; - bitCount = 0; /* Table Size */ bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; bitCount += 4; @@ -228,11 +236,11 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, threshold = tableSize; nbBits = tableLog+1; - while (remaining>1) { /* stops at 1 */ - if (previous0) { - unsigned start = charnum; - while (!normalizedCounter[charnum]) charnum++; - while (charnum >= start+24) { + while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */ + if (previousIs0) { + unsigned start = symbol; + while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++; + while (symbol >= start+24) { start+=24; bitStream += 0xFFFFU << bitCount; if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ @@ -241,12 +249,12 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, out+=2; bitStream>>=16; } - while (charnum >= start+3) { + while (symbol >= start+3) { start+=3; bitStream += 3 << bitCount; bitCount += 2; } - bitStream += (charnum-start) << bitCount; + bitStream += (symbol-start) << bitCount; bitCount += 2; if (bitCount>16) { if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ @@ -256,15 +264,15 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, bitStream >>= 16; bitCount -= 16; } } - { int count = normalizedCounter[charnum++]; - int const max = (2*threshold-1)-remaining; + { int count = normalizedCounter[symbol++]; + int const max = (2*threshold-1) - remaining; remaining -= count < 0 ? -count : count; count++; /* +1 for extra accuracy */ if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ bitStream += count << bitCount; bitCount += nbBits; bitCount -= (count>=1; } } @@ -277,14 +285,15 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, bitCount -= 16; } } + if (remaining != 1) return ERROR(GENERIC); /* incorrect normalized distribution */ + assert(symbol <= alphabetSize); + /* flush remaining bitStream */ if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ out[0] = (BYTE)bitStream; out[1] = (BYTE)(bitStream>>8); out+= (bitCount+7) /8; - if (charnum > maxSymbolValue + 1) return ERROR(GENERIC); - return (out-ostart); } @@ -297,7 +306,7 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalized if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); - return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */); } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index ed3aab871..29dce1250 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1147,7 +1147,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, if (zc->workSpace == NULL) return ERROR(memory_allocation); zc->workSpaceSize = neededSpace; zc->workSpaceOversizedDuration = 0; - ptr = zc->workSpace; /* Statically sized space. * entropyWorkspace never moves, diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c index 476cdc148..c4b9bb13b 100644 --- a/lib/compress/zstd_opt.c +++ b/lib/compress/zstd_opt.c @@ -970,7 +970,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ U32 seqPos = cur; DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)", - last_pos, cur); + last_pos, cur); (void)last_pos; assert(storeEnd < ZSTD_OPT_NUM); DEBUGLOG(6, "last sequence copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", storeEnd, lastSequence.litlen, lastSequence.mlen, lastSequence.off); diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 8f4589d13..16a08fb6b 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -185,7 +185,7 @@ size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } static size_t ZSTD_startingInputLength(ZSTD_format_e format) { size_t const startingInputLength = (format==ZSTD_f_zstd1_magicless) ? - ZSTD_frameHeaderSize_prefix - ZSTD_frameIdSize : + ZSTD_frameHeaderSize_prefix - ZSTD_FRAMEIDSIZE : ZSTD_frameHeaderSize_prefix; ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE); /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */ @@ -278,7 +278,7 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) * Note 3 : Skippable Frame Identifiers are considered valid. */ unsigned ZSTD_isFrame(const void* buffer, size_t size) { - if (size < ZSTD_frameIdSize) return 0; + if (size < ZSTD_FRAMEIDSIZE) return 0; { U32 const magic = MEM_readLE32(buffer); if (magic == ZSTD_MAGICNUMBER) return 1; if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1; @@ -331,6 +331,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s size_t const minInputSize = ZSTD_startingInputLength(format); if (srcSize < minInputSize) return minInputSize; + if (src==NULL) return ERROR(GENERIC); /* invalid parameter */ if ( (format != ZSTD_f_zstd1_magicless) && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { @@ -339,7 +340,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + frame length */ memset(zfhPtr, 0, sizeof(*zfhPtr)); - zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_frameIdSize); + zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); zfhPtr->frameType = ZSTD_skippableFrame; return 0; } @@ -451,7 +452,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) size_t skippableSize; if (srcSize < ZSTD_skippableHeaderSize) return ERROR(srcSize_wrong); - skippableSize = MEM_readLE32((const BYTE *)src + ZSTD_frameIdSize) + skippableSize = MEM_readLE32((const BYTE *)src + ZSTD_FRAMEIDSIZE) + ZSTD_skippableHeaderSize; if (srcSize < skippableSize) { return ZSTD_CONTENTSIZE_ERROR; @@ -1763,7 +1764,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) #endif if ( (srcSize >= ZSTD_skippableHeaderSize) && (MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START ) { - return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + ZSTD_frameIdSize); + return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + ZSTD_FRAMEIDSIZE); } else { const BYTE* ip = (const BYTE*)src; const BYTE* const ipstart = ip; @@ -1797,7 +1798,6 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) if (zfh.checksumFlag) { /* Final frame content checksum */ if (remainingSize < 4) return ERROR(srcSize_wrong); ip += 4; - remainingSize -= 4; } return ip - ipstart; @@ -1932,7 +1932,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, size_t skippableSize; if (srcSize < ZSTD_skippableHeaderSize) return ERROR(srcSize_wrong); - skippableSize = MEM_readLE32((const BYTE*)src + ZSTD_frameIdSize) + skippableSize = MEM_readLE32((const BYTE*)src + ZSTD_FRAMEIDSIZE) + ZSTD_skippableHeaderSize; if (srcSize < skippableSize) return ERROR(srcSize_wrong); @@ -2057,7 +2057,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c case ZSTDds_getFrameHeaderSize : assert(src != NULL); if (dctx->format == ZSTD_f_zstd1) { /* allows header */ - assert(srcSize >= ZSTD_frameIdSize); /* to read skippable magic number */ + assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */ if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ memcpy(dctx->headerBuffer, src, srcSize); dctx->expected = ZSTD_skippableHeaderSize - srcSize; /* remaining to load to get full skippable frame header */ @@ -2167,7 +2167,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c assert(src != NULL); assert(srcSize <= ZSTD_skippableHeaderSize); memcpy(dctx->headerBuffer + (ZSTD_skippableHeaderSize - srcSize), src, srcSize); /* complete skippable header */ - dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_frameIdSize); /* note : dctx->expected can grow seriously large, beyond local buffer size */ + dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ dctx->stage = ZSTDds_skipFrame; return 0; @@ -2268,7 +2268,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict if (magic != ZSTD_MAGIC_DICTIONARY) { return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ } } - dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_frameIdSize); + dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); /* load entropy tables */ { size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize); @@ -2381,7 +2381,7 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict, ZSTD_dictContentType_e return 0; /* pure content mode */ } } - ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_frameIdSize); + ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); /* load entropy tables */ CHECK_E( ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted ); @@ -2510,7 +2510,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) { if (dictSize < 8) return 0; if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0; - return MEM_readLE32((const char*)dict + ZSTD_frameIdSize); + return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); } /*! ZSTD_getDictID_fromDDict() : @@ -2855,7 +2855,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict)); if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ - zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_frameIdSize); + zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); zds->stage = ZSTDds_skipFrame; } else { CHECK_F(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize)); diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index b8a51789a..a4d0a4481 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -698,7 +698,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, short litLengthNCount[MaxLL+1]; U32 repOffset[MAXREPOFFSET]; offsetCount_t bestRepOffset[ZSTD_REP_NUM+1]; - EStats_ress_t esr; + EStats_ress_t esr = { NULL, NULL, NULL }; ZSTD_parameters params; U32 u, huffLog = 11, Offlog = OffFSELog, mlLog = MLFSELog, llLog = LLFSELog, total; size_t pos = 0, errorCode; diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index a2e2cfa80..15000db6d 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -1093,6 +1093,7 @@ static size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, un if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Init, lay down lowprob symbols */ + memset(tableDecode, 0, sizeof(FSE_DECODE_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */ DTableH.tableLog = (U16)tableLog; for (s=0; s<=maxSymbolValue; s++) { diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index a5e1b1ffc..89a5fe7e6 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -1224,6 +1224,7 @@ size_t FSEv05_buildDTable(FSEv05_DTable* dt, const short* normalizedCounter, uns if (tableLog > FSEv05_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Init, lay down lowprob symbols */ + memset(tableDecode, 0, sizeof(FSEv05_FUNCTION_TYPE) * (maxSymbolValue+1) ); /* useless init, but keep static analyzer happy, and we don't need to performance optimize legacy decoders */ DTableH.tableLog = (U16)tableLog; for (s=0; s<=maxSymbolValue; s++) { if (normalizedCounter[s]==-1) { diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 8b068b3e5..d8de77ca9 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -4006,7 +4006,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd, if (ZSTDv06_isError(hSize)) return hSize; if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); - zbd->lhSize += iend-ip; ip = iend; notDone = 0; + zbd->lhSize += iend-ip; *dstCapacityPtr = 0; return (hSize - zbd->lhSize) + ZSTDv06_blockHeaderSize; /* remaining header bytes + next block header */ } diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index 70b170f0f..9f95f62ad 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -3150,10 +3150,10 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, const BYTE* ip = (const BYTE*)src; if (srcSize < ZSTDv07_frameHeaderSize_min) return ZSTDv07_frameHeaderSize_min; + memset(fparamsPtr, 0, sizeof(*fparamsPtr)); if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) { if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTDv07_MAGIC_SKIPPABLE_START) { if (srcSize < ZSTDv07_skippableHeaderSize) return ZSTDv07_skippableHeaderSize; /* magic number + skippable frame length */ - memset(fparamsPtr, 0, sizeof(*fparamsPtr)); fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4); fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */ return 0; @@ -3175,11 +3175,13 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, U32 windowSize = 0; U32 dictID = 0; U64 frameContentSize = 0; - if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */ + if ((fhdByte & 0x08) != 0) /* reserved bits, which must be zero */ + return ERROR(frameParameter_unsupported); if (!directMode) { BYTE const wlByte = ip[pos++]; U32 const windowLog = (wlByte >> 3) + ZSTDv07_WINDOWLOG_ABSOLUTEMIN; - if (windowLog > ZSTDv07_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported); + if (windowLog > ZSTDv07_WINDOWLOG_MAX) + return ERROR(frameParameter_unsupported); windowSize = (1U << windowLog); windowSize += (windowSize >> 3) * (wlByte&7); } @@ -3201,7 +3203,8 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, case 3 : frameContentSize = MEM_readLE64(ip+pos); break; } if (!windowSize) windowSize = (U32)frameContentSize; - if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported); + if (windowSize > windowSizeMax) + return ERROR(frameParameter_unsupported); fparamsPtr->frameContentSize = frameContentSize; fparamsPtr->windowSize = windowSize; fparamsPtr->dictID = dictID; @@ -3220,11 +3223,10 @@ size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, - frame header not completely provided (`srcSize` too small) */ unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize) { - { ZSTDv07_frameParams fparams; - size_t const frResult = ZSTDv07_getFrameParams(&fparams, src, srcSize); - if (frResult!=0) return 0; - return fparams.frameContentSize; - } + ZSTDv07_frameParams fparams; + size_t const frResult = ZSTDv07_getFrameParams(&fparams, src, srcSize); + if (frResult!=0) return 0; + return fparams.frameContentSize; } From 42a02ab745976068255fe1a43915b70b8f9aa104 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 15 Aug 2018 14:35:38 -0700 Subject: [PATCH 2/7] fixed minor warnings issued by scan-build --- Makefile | 4 ++-- contrib/seekable_format/zstdseek_decompress.c | 5 ++++- lib/decompress/zstd_decompress.c | 1 + programs/dibio.c | 6 +++++- programs/dibio.h | 2 +- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 6c4ab9c9b..f8441e60d 100644 --- a/Makefile +++ b/Makefile @@ -114,7 +114,7 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD DragonFly NetBSD MSYS_NT)) HOST_OS = POSIX -CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release +CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON -DCMAKE_BUILD_TYPE=Release .PHONY: list list: @@ -351,7 +351,7 @@ bmi32build: clean $(CC) -v CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test -staticAnalyze: clean +staticAnalyze: $(CC) -v CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all endif diff --git a/contrib/seekable_format/zstdseek_decompress.c b/contrib/seekable_format/zstdseek_decompress.c index b006ff834..2b109b9d0 100644 --- a/contrib/seekable_format/zstdseek_decompress.c +++ b/contrib/seekable_format/zstdseek_decompress.c @@ -56,6 +56,7 @@ #include /* malloc, free */ #include /* FILE* */ +#include #define XXH_STATIC_LINKING_ONLY #define XXH_NAMESPACE ZSTD_ @@ -112,7 +113,7 @@ static int ZSTD_seekable_read_buff(void* opaque, void* buffer, size_t n) static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin) { - buffWrapper_t* buff = (buffWrapper_t*) opaque; + buffWrapper_t* const buff = (buffWrapper_t*) opaque; unsigned long long newOffset; switch (origin) { case SEEK_SET: @@ -124,6 +125,8 @@ static int ZSTD_seekable_seek_buff(void* opaque, long long offset, int origin) case SEEK_END: newOffset = (unsigned long long)buff->size - offset; break; + default: + assert(0); /* not possible */ } if (newOffset > buff->size) { return -1; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 16a08fb6b..5d9f0ba0b 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -330,6 +330,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s const BYTE* ip = (const BYTE*)src; size_t const minInputSize = ZSTD_startingInputLength(format); + memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */ if (srcSize < minInputSize) return minInputSize; if (src==NULL) return ERROR(GENERIC); /* invalid parameter */ diff --git a/programs/dibio.c b/programs/dibio.c index 5d1f6d6c4..fbb8aa6fa 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -27,6 +27,7 @@ #include /* memset */ #include /* fprintf, fopen, ftello64 */ #include /* errno */ +#include #include "mem.h" /* read */ #include "error_private.h" @@ -165,6 +166,7 @@ static U32 DiB_rand(U32* src) static void DiB_shuffle(const char** fileNamesTable, unsigned nbFiles) { U32 seed = 0xFD2FB528; unsigned i; + assert(nbFiles >= 1); for (i = nbFiles - 1; i > 0; --i) { unsigned const j = DiB_rand(&seed) % (i + 1); const char* const tmp = fileNamesTable[j]; @@ -310,7 +312,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, /* Load input buffer */ DISPLAYLEVEL(3, "Shuffling input files\n"); DiB_shuffle(fileNamesTable, nbFiles); - nbFiles = DiB_loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel); + DiB_loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel); { size_t dictSize; if (params) { @@ -319,6 +321,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, srcBuffer, sampleSizes, fs.nbSamples, *params); } else if (optimizeCover) { + assert(coverParams != NULL); dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer, sampleSizes, fs.nbSamples, coverParams); @@ -327,6 +330,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, DISPLAYLEVEL(2, "k=%u\nd=%u\nsteps=%u\nsplit=%u\n", coverParams->k, coverParams->d, coverParams->steps, splitPercentage); } } else { + assert(coverParams != NULL); dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, maxDictSize, srcBuffer, sampleSizes, fs.nbSamples, *coverParams); } diff --git a/programs/dibio.h b/programs/dibio.h index 499e30365..31a6b4bdb 100644 --- a/programs/dibio.h +++ b/programs/dibio.h @@ -33,7 +33,7 @@ */ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, - ZDICT_legacy_params_t *params, ZDICT_cover_params_t *coverParams, + ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams, int optimizeCover); #endif From da55865e47bb8035cbf1675d3d80c61979bfc8d3 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 15 Aug 2018 16:43:13 -0700 Subject: [PATCH 3/7] ensure dependency for zlib wrapper --- Makefile | 14 +++++++------- zlibWrapper/examples/zwrapbench.c | 6 +++--- zlibWrapper/gzlib.c | 4 ++-- zlibWrapper/gzwrite.c | 9 ++++++--- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index f8441e60d..d33757603 100644 --- a/Makefile +++ b/Makefile @@ -30,8 +30,7 @@ default: lib-release zstd-release all: allmost examples manual contrib .PHONY: allmost -allmost: allzstd - $(MAKE) -C $(ZWRAPDIR) all +allmost: allzstd zlibwrapper #skip zwrapper, can't build that on alternate architectures without the proper zlib installed .PHONY: allzstd @@ -44,8 +43,9 @@ all32: $(MAKE) -C $(PRGDIR) zstd32 $(MAKE) -C $(TESTDIR) all32 -.PHONY: lib lib-release -lib lib-release: +.PHONY: lib lib-release libzstd.a +lib : libzstd.a +lib lib-release libzstd.a: @$(MAKE) -C $(ZSTDDIR) $@ .PHONY: zstd zstd-release @@ -59,8 +59,8 @@ zstdmt: cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT) .PHONY: zlibwrapper -zlibwrapper: - $(MAKE) -C $(ZWRAPDIR) test +zlibwrapper: libzstd.a + $(MAKE) -C $(ZWRAPDIR) all .PHONY: test test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror @@ -353,5 +353,5 @@ bmi32build: clean staticAnalyze: $(CC) -v - CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all + CC=$(CC) CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all endif diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index a4dfbb6e8..d2d6073f9 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -573,10 +573,10 @@ static size_t BMK_findMaxMem(U64 requiredMem) do { testmem = (BYTE*)malloc((size_t)requiredMem); requiredMem -= step; - } while (!testmem); + } while (!testmem && requiredMem); /* do not allocate zero bytes */ free(testmem); - return (size_t)(requiredMem); + return (size_t)(requiredMem+1); /* avoid zero */ } static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, @@ -734,7 +734,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; if (benchedSize < totalSizeToLoad) DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); - srcBuffer = malloc(benchedSize); + srcBuffer = malloc(benchedSize + !benchedSize); if (!srcBuffer) EXM_THROW(12, "not enough memory"); /* Load input buffer */ diff --git a/zlibWrapper/gzlib.c b/zlibWrapper/gzlib.c index 8235cff4f..3070dd8b4 100644 --- a/zlibWrapper/gzlib.c +++ b/zlibWrapper/gzlib.c @@ -111,7 +111,7 @@ local gzFile gz_open(path, fd, mode) return NULL; /* allocate gzFile structure to return */ - state = (gz_statep)(gz_state*)malloc(sizeof(gz_state)); + state.state = (gz_state*)malloc(sizeof(gz_state)); if (state.state == NULL) return NULL; state.state->size = 0; /* no buffers allocated yet */ @@ -266,7 +266,7 @@ local gzFile gz_open(path, fd, mode) gz_reset(state); /* return stream */ - return (gzFile)state.file; + return state.file; } /* -- see zlib.h -- */ diff --git a/zlibWrapper/gzwrite.c b/zlibWrapper/gzwrite.c index d1250b900..21d5f8472 100644 --- a/zlibWrapper/gzwrite.c +++ b/zlibWrapper/gzwrite.c @@ -6,6 +6,8 @@ * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html */ +#include + #include "gzguts.h" /* Local functions */ @@ -24,7 +26,7 @@ local int gz_init(state) z_streamp strm = &(state.state->strm); /* allocate input buffer (double size for gzprintf) */ - state.state->in = (unsigned char *)malloc(state.state->want << 1); + state.state->in = (unsigned char*)malloc(state.state->want << 1); if (state.state->in == NULL) { gz_error(state, Z_MEM_ERROR, "out of memory"); return -1; @@ -33,7 +35,7 @@ local int gz_init(state) /* only need output buffer and deflate state if compressing */ if (!state.state->direct) { /* allocate output buffer */ - state.state->out = (unsigned char *)malloc(state.state->want); + state.state->out = (unsigned char*)malloc(state.state->want); if (state.state->out == NULL) { free(state.state->in); gz_error(state, Z_MEM_ERROR, "out of memory"); @@ -284,6 +286,7 @@ z_size_t ZEXPORT gzfwrite(buf, size, nitems, file) gz_statep state; /* get internal structure */ + assert(size != 0); if (file == NULL) return 0; state = (gz_statep)file; @@ -294,7 +297,7 @@ z_size_t ZEXPORT gzfwrite(buf, size, nitems, file) /* compute bytes to read -- error on overflow */ len = nitems * size; - if (size && len / size != nitems) { + if (size && (len / size != nitems)) { gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t"); return 0; } From 5291d9ac31a87b68e717e43bf5b14a71ebaf644d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 15 Aug 2018 17:41:44 -0700 Subject: [PATCH 4/7] fix scope of scan-build tests exclude zlib code --- Makefile | 11 ++++++----- lib/decompress/zstd_decompress.c | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index d33757603..e338c5ca1 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ all: allmost examples manual contrib .PHONY: allmost allmost: allzstd zlibwrapper -#skip zwrapper, can't build that on alternate architectures without the proper zlib installed +# skip zwrapper, can't build that on alternate architectures without the proper zlib installed .PHONY: allzstd allzstd: lib $(MAKE) -C $(PRGDIR) all @@ -44,8 +44,7 @@ all32: $(MAKE) -C $(TESTDIR) all32 .PHONY: lib lib-release libzstd.a -lib : libzstd.a -lib lib-release libzstd.a: +lib lib-release : @$(MAKE) -C $(ZSTDDIR) $@ .PHONY: zstd zstd-release @@ -59,7 +58,7 @@ zstdmt: cp $(PRGDIR)/zstd$(EXT) ./zstdmt$(EXT) .PHONY: zlibwrapper -zlibwrapper: libzstd.a +zlibwrapper: lib $(MAKE) -C $(ZWRAPDIR) all .PHONY: test @@ -351,7 +350,9 @@ bmi32build: clean $(CC) -v CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test +# static analyzer test uses clang's scan-build +# does not analyze zlibWrapper, due to detected issues in zlib source code staticAnalyze: $(CC) -v - CC=$(CC) CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all + CC=$(CC) CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) allzstd examples contrib endif diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 5d9f0ba0b..8fef7e51b 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -2399,6 +2399,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { ddict->dictBuffer = NULL; ddict->dictContent = dict; + if (!dict) dictSize = 0; } else { void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem); ddict->dictBuffer = internalBuffer; From 31224cc126f23d2b7dbac84940aaedbf55ea477b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 15 Aug 2018 17:44:27 -0700 Subject: [PATCH 5/7] added static analyzer tests to travis CI --- .travis.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 36df67052..a61b82876 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,23 +20,25 @@ matrix: - env: Cmd='make gcc8install && CC=gcc-8 CFLAGS="-Werror -O3" make -j all' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd' + - env: Cmd='make staticAnalyze' + - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest' - env: Cmd='make gcc6install libc6install && make clean && CC=gcc-6 CFLAGS=-m32 make uasan-fuzztest' - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest' - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream' - - env: Cmd='make arminstall && make armfuzz' - - env: Cmd='make arminstall && make aarch64fuzz' - - env: Cmd='make ppcinstall && make ppcfuzz' - - env: Cmd='make ppcinstall && make ppc64fuzz' - - env: Cmd='make -j uasanregressiontest && make clean && make -j msanregressiontest' - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest && make clean && make -C tests test-fuzzer-stackmode' + - env: Cmd='make arminstall && make armfuzz' + - env: Cmd='make arminstall && make aarch64fuzz' + - env: Cmd='make ppcinstall && make ppcfuzz' + - env: Cmd='make ppcinstall && make ppc64fuzz' + - env: Cmd='make lz4install && make -C tests test-lz4 && make clean && make -C tests test-pool && make clean && bash tests/libzstd_partial_builds.sh' From 1515f0bb0d73138dd191a9fd5caad09e9868a8a0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 16 Aug 2018 14:40:47 -0700 Subject: [PATCH 6/7] fixed more issues detected by recent version of scan-build test run on Linux --- examples/multiple_streaming_compression.c | 3 ++- examples/streaming_compression.c | 22 ++++++++++++++++------ lib/compress/fse_compress.c | 22 +++++++++++++++------- lib/dictBuilder/divsufsort.c | 6 +++--- 4 files changed, 36 insertions(+), 17 deletions(-) diff --git a/examples/multiple_streaming_compression.c b/examples/multiple_streaming_compression.c index e395aefbc..4308a2e4d 100644 --- a/examples/multiple_streaming_compression.c +++ b/examples/multiple_streaming_compression.c @@ -158,7 +158,8 @@ int main(int argc, const char** argv) } freeResources(ress); - /* success */ + free(ofnBuffer); + printf("compressed %i files \n", argc-1); return 0; diff --git a/examples/streaming_compression.c b/examples/streaming_compression.c index f76364d84..9287ff398 100644 --- a/examples/streaming_compression.c +++ b/examples/streaming_compression.c @@ -73,7 +73,11 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve ZSTD_CStream* const cstream = ZSTD_createCStream(); if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); } size_t const initResult = ZSTD_initCStream(cstream, cLevel); - if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); } + if (ZSTD_isError(initResult)) { + fprintf(stderr, "ZSTD_initCStream() error : %s \n", + ZSTD_getErrorName(initResult)); + exit(11); + } size_t read, toRead = buffInSize; while( (read = fread_orDie(buffIn, toRead, fin)) ) { @@ -81,7 +85,11 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve while (input.pos < input.size) { ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; toRead = ZSTD_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */ - if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); } + if (ZSTD_isError(toRead)) { + fprintf(stderr, "ZSTD_compressStream() error : %s \n", + ZSTD_getErrorName(toRead)); + exit(12); + } if (toRead > buffInSize) toRead = buffInSize; /* Safely handle case when `buffInSize` is manually changed to a value < ZSTD_CStreamInSize()*/ fwrite_orDie(buffOut, output.pos, fout); } @@ -100,15 +108,15 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve } -static const char* createOutFilename_orDie(const char* filename) +static char* createOutFilename_orDie(const char* filename) { size_t const inL = strlen(filename); size_t const outL = inL + 5; - void* outSpace = malloc_orDie(outL); + void* const outSpace = malloc_orDie(outL); memset(outSpace, 0, outL); strcat(outSpace, filename); strcat(outSpace, ".zst"); - return (const char*)outSpace; + return (char*)outSpace; } int main(int argc, const char** argv) @@ -124,8 +132,10 @@ int main(int argc, const char** argv) const char* const inFilename = argv[1]; - const char* const outFilename = createOutFilename_orDie(inFilename); + char* const outFilename = createOutFilename_orDie(inFilename); compressFile_orDie(inFilename, outFilename, 1); + free(outFilename); /* not strictly required, since program execution stops there, + * but some static analyzer main complain otherwise */ return 0; } diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index caa90edec..4075db217 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -240,10 +240,12 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize, if (previousIs0) { unsigned start = symbol; while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++; + if (symbol == alphabetSize) break; /* incorrect distribution */ while (symbol >= start+24) { start+=24; bitStream += 0xFFFFU << bitCount; - if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + if ((!writeIsSafe) && (out > oend-2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ out[0] = (BYTE) bitStream; out[1] = (BYTE)(bitStream>>8); out+=2; @@ -257,7 +259,8 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize, bitStream += (symbol-start) << bitCount; bitCount += 2; if (bitCount>16) { - if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ out[0] = (BYTE)bitStream; out[1] = (BYTE)(bitStream>>8); out += 2; @@ -268,7 +271,8 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize, int const max = (2*threshold-1) - remaining; remaining -= count < 0 ? -count : count; count++; /* +1 for extra accuracy */ - if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ + if (count>=threshold) + count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ bitStream += count << bitCount; bitCount += nbBits; bitCount -= (count>=1; } } if (bitCount>16) { - if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ out[0] = (BYTE)bitStream; out[1] = (BYTE)(bitStream>>8); out += 2; @@ -285,11 +290,13 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize, bitCount -= 16; } } - if (remaining != 1) return ERROR(GENERIC); /* incorrect normalized distribution */ + if (remaining != 1) + return ERROR(GENERIC); /* incorrect normalized distribution */ assert(symbol <= alphabetSize); /* flush remaining bitStream */ - if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ out[0] = (BYTE)bitStream; out[1] = (BYTE)(bitStream>>8); out+= (bitCount+7) /8; @@ -298,7 +305,8 @@ FSE_writeNCount_generic (void* header, size_t headerBufferSize, } -size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +size_t FSE_writeNCount (void* buffer, size_t bufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */ if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ diff --git a/lib/dictBuilder/divsufsort.c b/lib/dictBuilder/divsufsort.c index 60cceb088..ead922044 100644 --- a/lib/dictBuilder/divsufsort.c +++ b/lib/dictBuilder/divsufsort.c @@ -1637,7 +1637,7 @@ construct_SA(const unsigned char *T, int *SA, if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } k = SA + BUCKET_B(c2 = c0, c1); } - assert(k < j); + assert(k < j); assert(k != NULL); *k-- = s; } else { assert(((s == 0) && (T[s] == c1)) || (s < 0)); @@ -1701,7 +1701,7 @@ construct_BWT(const unsigned char *T, int *SA, if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } k = SA + BUCKET_B(c2 = c0, c1); } - assert(k < j); + assert(k < j); assert(k != NULL); *k-- = s; } else if(s != 0) { *j = ~s; @@ -1785,7 +1785,7 @@ construct_BWT_indexes(const unsigned char *T, int *SA, if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } k = SA + BUCKET_B(c2 = c0, c1); } - assert(k < j); + assert(k < j); assert(k != NULL); *k-- = s; } else if(s != 0) { *j = ~s; From 36d6165a2d75ab45ddec7da1f681e4153d3f8153 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 16 Aug 2018 15:41:56 -0700 Subject: [PATCH 7/7] Makefile: added variable SCANBUILD so that a different version of scan-build can be selected --- Makefile | 3 ++- contrib/seekable_format/examples/seekable_compression.c | 7 ++++--- lib/decompress/zstd_decompress.c | 1 + lib/legacy/zstd_v05.c | 1 + lib/legacy/zstd_v06.c | 1 + 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index e338c5ca1..853ceee93 100644 --- a/Makefile +++ b/Makefile @@ -352,7 +352,8 @@ bmi32build: clean # static analyzer test uses clang's scan-build # does not analyze zlibWrapper, due to detected issues in zlib source code +staticAnalyze: SCANBUILD ?= scan-build staticAnalyze: $(CC) -v - CC=$(CC) CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) allzstd examples contrib + CC=$(CC) CPPFLAGS=-g $(SCANBUILD) --status-bugs -v $(MAKE) allzstd examples contrib endif diff --git a/contrib/seekable_format/examples/seekable_compression.c b/contrib/seekable_format/examples/seekable_compression.c index 9485bf26f..9a331a895 100644 --- a/contrib/seekable_format/examples/seekable_compression.c +++ b/contrib/seekable_format/examples/seekable_compression.c @@ -101,7 +101,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve free(buffOut); } -static const char* createOutFilename_orDie(const char* filename) +static char* createOutFilename_orDie(const char* filename) { size_t const inL = strlen(filename); size_t const outL = inL + 5; @@ -109,7 +109,7 @@ static const char* createOutFilename_orDie(const char* filename) memset(outSpace, 0, outL); strcat(outSpace, filename); strcat(outSpace, ".zst"); - return (const char*)outSpace; + return (char*)outSpace; } int main(int argc, const char** argv) { @@ -124,8 +124,9 @@ int main(int argc, const char** argv) { { const char* const inFileName = argv[1]; unsigned const frameSize = (unsigned)atoi(argv[2]); - const char* const outFileName = createOutFilename_orDie(inFileName); + char* const outFileName = createOutFilename_orDie(inFileName); compressFile_orDie(inFileName, outFileName, 5, frameSize); + free(outFileName); } return 0; diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 8fef7e51b..7c4769cd8 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -542,6 +542,7 @@ size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize) { + if (dst==NULL) return ERROR(dstSize_tooSmall); if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall); memcpy(dst, src, srcSize); return srcSize; diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 89a5fe7e6..65f07a3b5 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -2846,6 +2846,7 @@ size_t ZSTDv05_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* static size_t ZSTDv05_copyRawBlock(void* dst, size_t maxDstSize, const void* src, size_t srcSize) { + if (dst==NULL) return ERROR(dstSize_tooSmall); if (srcSize > maxDstSize) return ERROR(dstSize_tooSmall); memcpy(dst, src, srcSize); return srcSize; diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index d8de77ca9..ca982c35e 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -3041,6 +3041,7 @@ size_t ZSTDv06_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* static size_t ZSTDv06_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize) { + if (dst==NULL) return ERROR(dstSize_tooSmall); if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall); memcpy(dst, src, srcSize); return srcSize;