diff --git a/.travis.yml b/.travis.yml index b1ffa799f..6cd3f5f62 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,85 +6,82 @@ matrix: # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - os: linux sudo: false - env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=travis-install + env: PLATFORM="Ubuntu 12.04 container" CMD="make travis-install" - os: linux sudo: false - env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=cmaketest + env: PLATFORM="Ubuntu 12.04 container" CMD="make cmaketest" - os: linux sudo: false - env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=test + env: PLATFORM="Ubuntu 12.04 container" CMD="make test" - os: linux sudo: false - env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM="-C programs test-zstd_nolegacy" + env: PLATFORM="Ubuntu 12.04 container" CMD="make -C programs test-zstd_nolegacy" - os: linux sudo: false - env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=usan + env: PLATFORM="Ubuntu 12.04 container" CMD="make usan" - os: linux sudo: false - env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=asan + env: PLATFORM="Ubuntu 12.04 container" CMD="make asan" - os: linux sudo: false - env: PLATFORM="Ubuntu 12.04 container" MAKE_PARAM=zlibwrapper + env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper" # Standard Ubuntu 12.04 LTS Server Edition 64 bit - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=clangtest + env: PLATFORM="Ubuntu 12.04" CMD="make libc6install clangtest" - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gpptest + env: PLATFORM="Ubuntu 12.04" CMD="make gppinstall gpptest" - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu90test + env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu90test" - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=c99test + env: PLATFORM="Ubuntu 12.04" CMD="make libc6install c99test" - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=gnu99test + env: PLATFORM="Ubuntu 12.04" CMD="make libc6install gnu99test" - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=armtest-w-install + env: PLATFORM="Ubuntu 12.04" CMD="make arminstall armtest" - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs test32" + env: PLATFORM="Ubuntu 12.04" CMD="make libc6install && make -C programs test32" - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C tests versionsTest" + env: PLATFORM="Ubuntu 12.04" CMD="make -C tests versionsTest" - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" MAKE_PARAM=asan32 + env: PLATFORM="Ubuntu 12.04" CMD="make libc6install asan32" - os: linux sudo: required - env: PLATFORM="Ubuntu 12.04" MAKE_PARAM="-C programs valgrindTest" + env: PLATFORM="Ubuntu 12.04" CMD="make valgrindinstall && make -C programs valgrindTest" # Ubuntu 14.04 LTS Server Edition 64 bit - os: linux dist: trusty sudo: required - env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=ppctest-w-install + env: PLATFORM="Ubuntu 14.04" CMD="make gcc5install gcc5test" - os: linux dist: trusty sudo: required - env: PLATFORM="Ubuntu 14.04" MAKE_PARAM=zlibwrapper + env: PLATFORM="Ubuntu 14.04" CMD="make gcc6install gcc6test" + - os: linux + dist: trusty + sudo: required + env: PLATFORM="Ubuntu 14.04" CMD="make ppcinstall ppctest" + - os: linux + dist: trusty + sudo: required + env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper" # OS X Mavericks - os: osx - env: PLATFORM="OS X Mavericks" MAKE_PARAM=travis-install + env: PLATFORM="OS X Mavericks" CMD="make travis-install" - os: osx - env: PLATFORM="OS X Mavericks" MAKE_PARAM=gnu90test + env: PLATFORM="OS X Mavericks" CMD="make gnu90test" - os: osx - env: PLATFORM="OS X Mavericks" MAKE_PARAM=test + env: PLATFORM="OS X Mavericks" CMD="make test" exclude: - compiler: gcc -before_install: - - set -e - - | - if [ "$TRAVIS_OS_NAME" == "linux" ]; then - CAN_I_RUN_SUDO=$(sudo -n uptime 2>&1|grep "load"|wc -l) - echo "CAN_I_RUN_SUDO=$CAN_I_RUN_SUDO\n" - if [ ${CAN_I_RUN_SUDO} -gt 0 ]; then - sudo apt-get install -y -qq clang g++-multilib gcc-multilib valgrind - fi - fi - script: - - make $MAKE_PARAM + - sh -c "$CMD" diff --git a/Makefile b/Makefile index f0d39ea1f..428452880 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ all: zstdprogram: $(MAKE) -C $(PRGDIR) - mv $(PRGDIR)/zstd . + cp $(PRGDIR)/zstd . zlibwrapper: $(MAKE) -C $(ZSTDDIR) all @@ -87,6 +87,14 @@ travis-install: gpptest: clean $(MAKE) all CC=g++ CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" +gcc5test: clean + gcc-5 -v + $(MAKE) all CC=gcc-5 MOREFLAGS="-Werror" + +gcc6test: clean + gcc-6 -v + $(MAKE) all CC=gcc-6 MOREFLAGS="-Werror" + clangtest: clean clang -v $(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion" @@ -95,35 +103,13 @@ armtest: clean $(MAKE) -C $(PRGDIR) datagen # use native, faster $(MAKE) -C $(PRGDIR) test CC=arm-linux-gnueabi-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" -# for Travis CI -arminstall: clean - sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi - -# for Travis CI -armtest-w-install: clean arminstall armtest - ppctest: clean $(MAKE) -C $(PRGDIR) datagen # use native, faster $(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" -# for Travis CI -ppcinstall: clean - # sudo apt-get update -y -q - sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu # doesn't work with Ubuntu 12.04 - -# for Travis CI -ppctest-w-install: clean ppcinstall ppctest - ppc64test: clean $(MAKE) -C $(PRGDIR) datagen # use native, faster - $(MAKE) -C $(PRGDIR) test CC=powerpc64le-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static" - -ppc64install: clean - sudo apt-get update -y -q - sudo apt-get install -y -q qemu-ppc64le binfmt-support qemu-user-static gcc-powerpc64le-linux-gnu - update-binfmts --displ - -ppc64test-w-install: clean ppc64install ppc64test + $(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-m64 -Werror -static" usan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined" @@ -183,3 +169,37 @@ bmix32test: clean bmi32test: clean CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(PRGDIR) test endif + + +#------------------------------------------------------------------------ +# for Travis CI +#------------------------------------------------------------------------ +libc6install: + sudo apt-get install -y -qq libc6-dev-i386 + +gppinstall: + sudo apt-get install -y -qq g++-multilib + +gcc5install: + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update -y -qq + sudo apt-get install -y -qq gcc-5-multilib + +gcc6install: + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update -y -qq + sudo apt-get install -y -qq gcc-6-multilib + +arminstall: clean + sudo apt-get install -y -q qemu binfmt-support qemu-user-static gcc-arm-linux-gnueabi + +ppcinstall: clean + sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu # doesn't work with Ubuntu 12.04 + +ppc64install: clean # compilation works but cannot be executed + sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu + update-binfmts --displ + +valgrindinstall: + echo sudo apt-get install -y -qq valgrind + sudo apt-get install -y -qq valgrind diff --git a/NEWS b/NEWS index 0e8d87d62..a980e80e0 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,6 @@ v0.7.1 +fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier +fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв fixed : corruption issue, reported by cj modified : checksum enabled by default in command line mode diff --git a/lib/common/huf.h b/lib/common/huf.h index ef538df32..3b837f101 100644 --- a/lib/common/huf.h +++ b/lib/common/huf.h @@ -136,7 +136,7 @@ typedef U32 HUF_DTable; size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); +size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ diff --git a/lib/common/zstd.h b/lib/common/zstd.h index b774434b3..d6a1cceea 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -61,7 +61,7 @@ extern "C" { ***************************************/ #define ZSTD_VERSION_MAJOR 0 #define ZSTD_VERSION_MINOR 7 -#define ZSTD_VERSION_RELEASE 0 +#define ZSTD_VERSION_RELEASE 1 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str @@ -132,7 +132,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapa /*! ZSTD_compress_usingDict() : * Compression using a pre-defined Dictionary content (see dictBuilder). * Note 1 : This function load the dictionary, resulting in a significant startup time. -* Note 2 : `dict` must remain valid and unmodified during compression operation. +* Note 2 : `dict` must remain accessible and unmodified during compression operation. * Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_compressCCtx() */ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, @@ -144,7 +144,7 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, * Decompression using a pre-defined Dictionary content (see dictBuilder). * Dictionary must be identical to the one used during compression. * Note 1 : This function load the dictionary, resulting in a significant startup time -* Note 2 : `dict` must remain valid and unmodified during compression operation. +* Note 2 : `dict` must remain accessible and unmodified during compression operation. * Note 3 : `dict` can be `NULL`, in which case, it's equivalent to ZSTD_decompressDCtx() */ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, @@ -192,7 +192,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, /* ==================================================================================== * The definitions in this section are considered experimental. - * They should never be used in association with a dynamic library, as they may change in the future. + * They should never be used with a dynamic library, as they may change in the future. * They are provided for advanced usages. * Use them only in association with static linking. * ==================================================================================== */ @@ -322,15 +322,22 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() Then, consume your input using ZSTD_compressContinue(). - ZSTD_compressContinue() presumes prior data is still accessible and unmodified (up to maximum distance size, see WindowLog). - The interface is synchronous, so input will be entirely consumed and produce associated compressed output. - You must ensure there is enough space in destination buffer to store compressed data under worst case scenario. - Worst case evaluation is provided by ZSTD_compressBound(). + There are some important considerations to keep in mind when using this advanced function : + - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only. + - Interface is synchronous : input will be entirely consumed and produce 1+ compressed blocks. + - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. + Worst case evaluation is provided by ZSTD_compressBound(). + ZSTD_compressContinue() doesn't guarantee recover after a failed compression. + - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). + It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) + - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. + In which case, it will "discard" the relevant memory section from its history. + Finish a frame with ZSTD_compressEnd(), which will write the epilogue. Without epilogue, frames will be considered unfinished (broken) by decoders. - You can then reuse ZSTD_CCtx to compress some new frame. + You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame. */ typedef struct { @@ -357,7 +364,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds A ZSTD_DCtx object can be re-used multiple times. First optional operation is to retrieve frame parameters, using ZSTD_getFrameParams(), which doesn't consume the input. - It can provide the minimum size of rolling buffer required to properly decompress data, + It can provide the minimum size of rolling buffer required to properly decompress data (`windowSize`), and optionally the final size of uncompressed content. (Note : content size is an optional info that may not be present. 0 means : content size unknown) Frame parameters are extracted from the beginning of compressed frame. @@ -373,7 +380,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). ZSTD_decompressContinue() requires this exact amount of bytes, or it will fail. - ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog). + ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize`. They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible. @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index 9c9c8059f..6ed5e52a4 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -79,7 +79,7 @@ static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE; * output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed. * **************************************************/ -typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush } ZBUFF_cStage; +typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush, ZBUFFcs_final } ZBUFF_cStage; /* *** Resources *** */ struct ZBUFF_CCtx_s { @@ -162,7 +162,7 @@ size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, zbc->inToCompress = 0; zbc->inBuffPos = 0; zbc->inBuffTarget = zbc->blockSize; - zbc->outBuffFlushedSize = 0; + zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0; zbc->stage = ZBUFFcs_load; return 0; /* ready to go */ } @@ -196,7 +196,7 @@ MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr, - int flush) /* aggregate : wait for full block before compressing */ + int flush) { U32 notDone = 1; const char* const istart = (const char*)src; @@ -243,17 +243,20 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, } case ZBUFFcs_flush: - /* flush into dst */ { size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize; size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush); op += flushed; zbc->outBuffFlushedSize += flushed; - if (toFlush!=flushed) { notDone = 0; break; } /* not enough space within dst to store compressed block : stop there */ - zbc->outBuffContentSize = 0; - zbc->outBuffFlushedSize = 0; + if (toFlush!=flushed) { notDone = 0; break; } /* dst too small to store flushed data : stop there */ + zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0; zbc->stage = ZBUFFcs_load; break; } + + case ZBUFFcs_final: + notDone = 0; /* do nothing */ + break; + default: return ERROR(GENERIC); /* impossible */ } @@ -291,19 +294,30 @@ size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) BYTE* const ostart = (BYTE*)dst; BYTE* const oend = ostart + *dstCapacityPtr; BYTE* op = ostart; - size_t outSize = *dstCapacityPtr; - size_t epilogueSize, remaining; - ZBUFF_compressFlush(zbc, dst, &outSize); /* flush any remaining inBuff */ - op += outSize; - epilogueSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff + zbc->outBuffContentSize, zbc->outBuffSize - zbc->outBuffContentSize); /* epilogue into outBuff */ - zbc->outBuffContentSize += epilogueSize; - outSize = oend-op; - zbc->stage = ZBUFFcs_flush; - remaining = ZBUFF_compressFlush(zbc, op, &outSize); /* attempt to flush epilogue into dst */ - op += outSize; - if (!remaining) zbc->stage = ZBUFFcs_init; /* close only if nothing left to flush */ - *dstCapacityPtr = op-ostart; /* tells how many bytes were written */ - return remaining; + + if (zbc->stage != ZBUFFcs_final) { + /* flush whatever remains */ + size_t outSize = *dstCapacityPtr; + size_t const remainingToFlush = ZBUFF_compressFlush(zbc, dst, &outSize); + op += outSize; + if (remainingToFlush) { + *dstCapacityPtr = op-ostart; + return remainingToFlush + ZBUFF_endFrameSize; + } + /* create epilogue */ + zbc->stage = ZBUFFcs_final; + zbc->outBuffContentSize = ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize); /* epilogue into outBuff */ + } + + /* flush epilogue */ + { size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize; + size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush); + op += flushed; + zbc->outBuffFlushedSize += flushed; + *dstCapacityPtr = op-ostart; + if (toFlush==flushed) zbc->stage = ZBUFFcs_init; /* end reached */ + return toFlush - flushed; + } } diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 1f94f1968..42cf648a1 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -295,7 +295,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, zc->seqStore.buffer = zc->hashTable3 + h3Size; zc->hufTable = (HUF_CElt*)zc->seqStore.buffer; zc->flagStaticTables = 0; - zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256; + zc->seqStore.buffer = ((U32*)(zc->seqStore.buffer)) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */ zc->nextToUpdate = 1; zc->nextSrc = NULL; @@ -313,14 +313,17 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1); zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1); - zc->seqStore.matchTable = (ZSTD_match_t*)((void*)(zc->seqStore.offCodeFreq + (MaxOff+1))); - zc->seqStore.priceTable = (ZSTD_optimal_t*)((void*)(zc->seqStore.matchTable + ZSTD_OPT_NUM+1)); + zc->seqStore.buffer = zc->seqStore.offCodeFreq + (MaxOff+1); + zc->seqStore.matchTable = (ZSTD_match_t*)zc->seqStore.buffer; + zc->seqStore.buffer = zc->seqStore.matchTable + ZSTD_OPT_NUM+1; + zc->seqStore.priceTable = (ZSTD_optimal_t*)zc->seqStore.buffer; zc->seqStore.buffer = zc->seqStore.priceTable + ZSTD_OPT_NUM+1; zc->seqStore.litLengthSum = 0; } - zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer); - zc->seqStore.litLengthStart = (U16*) (void*)(zc->seqStore.offsetStart + maxNbSeq); - zc->seqStore.matchLengthStart = (U16*) (void*)(zc->seqStore.litLengthStart + maxNbSeq); + zc->seqStore.offsetStart = (U32*)(zc->seqStore.buffer); + zc->seqStore.buffer = zc->seqStore.offsetStart + maxNbSeq; + zc->seqStore.litLengthStart = (U16*)zc->seqStore.buffer; + zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + maxNbSeq; zc->seqStore.llCodeStart = (BYTE*) (zc->seqStore.matchLengthStart + maxNbSeq); zc->seqStore.mlCodeStart = zc->seqStore.llCodeStart + maxNbSeq; zc->seqStore.offCodeStart = zc->seqStore.mlCodeStart + maxNbSeq; @@ -1330,8 +1333,6 @@ static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx, } - - /*-************************************* * Binary Tree search ***************************************/ @@ -1614,7 +1615,7 @@ U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls) const U32 target = (U32)(ip - base); U32 idx = zc->nextToUpdate; - while(idx < target) { + while(idx < target) { /* catch up */ size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls); NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; hashTable[h] = idx; @@ -1651,7 +1652,7 @@ size_t ZSTD_HcFindBestMatch_generic ( /* HC4 match finder */ U32 matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls); - for ( ; (matchIndex>lowLimit) && (nbAttempts) ; nbAttempts--) { + for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) { const BYTE* match; size_t currentMl=0; if ((!extDict) || matchIndex >= dictLimit) { @@ -1665,7 +1666,7 @@ size_t ZSTD_HcFindBestMatch_generic ( } /* save best solution */ - if (currentMl > ml) { ml = currentMl; *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ } + if (currentMl > ml) { ml = currentMl; *offsetPtr = current - matchIndex + ZSTD_REP_MOVE; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ } if (matchIndex <= minChain) break; matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 0814581fe..031509453 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -584,48 +584,51 @@ static void ZDICT_countEStats(EStats_ress_t esr, U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets, const void* src, size_t srcSize) { - const seqStore_t* seqStorePtr; + size_t cSize; if (srcSize > ZSTD_BLOCKSIZE_MAX) srcSize = ZSTD_BLOCKSIZE_MAX; /* protection vs large samples */ ZSTD_copyCCtx(esr.zc, esr.ref); - ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize); - seqStorePtr = ZSTD_getSeqStore(esr.zc); + cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize); + if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; } - /* literals stats */ - { const BYTE* bytePtr; - for(bytePtr = seqStorePtr->litStart; bytePtr < seqStorePtr->lit; bytePtr++) - countLit[*bytePtr]++; - } + if (cSize) { /* if == 0; block is not compressible */ + const seqStore_t* seqStorePtr = ZSTD_getSeqStore(esr.zc); - /* seqStats */ - { size_t const nbSeq = (size_t)(seqStorePtr->offset - seqStorePtr->offsetStart); - ZSTD_seqToCodes(seqStorePtr, nbSeq); - - { const BYTE* codePtr = seqStorePtr->offCodeStart; - size_t u; - for (u=0; ulitStart; bytePtr < seqStorePtr->lit; bytePtr++) + countLit[*bytePtr]++; } - { const BYTE* codePtr = seqStorePtr->mlCodeStart; - size_t u; - for (u=0; uoffset - seqStorePtr->offsetStart); + ZSTD_seqToCodes(seqStorePtr, nbSeq); + + { const BYTE* codePtr = seqStorePtr->offCodeStart; + size_t u; + for (u=0; umlCodeStart; + size_t u; + for (u=0; ullCodeStart; + size_t u; + for (u=0; uoffsetStart; + U32 offset1 = offsetPtr[0] - 3; + U32 offset2 = offsetPtr[1] - 3; + if (offset1 >= MAXREPOFFSET) offset1 = 0; + if (offset2 >= MAXREPOFFSET) offset2 = 0; + repOffsets[offset1] += 3; + repOffsets[offset2] += 1; } - - { const BYTE* codePtr = seqStorePtr->llCodeStart; - size_t u; - for (u=0; uoffsetStart; - U32 offset1 = offsetPtr[0] - 3; - U32 offset2 = offsetPtr[1] - 3; - if (offset1 >= MAXREPOFFSET) offset1 = 0; - if (offset2 >= MAXREPOFFSET) offset2 = 0; - repOffsets[offset1] += 3; - repOffsets[offset2] += 1; } - } /* diff --git a/programs/bench.c b/programs/bench.c index 51d47bde1..04f4276ce 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -42,10 +42,6 @@ /* ************************************* * Constants ***************************************/ -#ifndef ZSTD_VERSION -# define ZSTD_VERSION "" -#endif - #define NBLOOPS 3 #define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ #define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ @@ -363,7 +359,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, memset(&total, 0, sizeof(total)); if (g_displayLevel == 1 && !g_additionalParam) - DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10)); + DISPLAY("bench %s: input %u bytes, %i iterations, %u KB blocks\n", ZSTD_VERSION_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10)); if (cLevelLast < cLevel) cLevelLast = cLevel; diff --git a/programs/fileio.c b/programs/fileio.c index 90041250d..5e7b26dc6 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -413,7 +413,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress, int result; ress.dstFile = FIO_openDstFile(dstFileName); - if (ress.dstFile==0) { fclose(ress.srcFile); return 1; } + if (ress.dstFile==0) return 1; result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, cLevel); diff --git a/programs/playTests.sh b/programs/playTests.sh index a7edbeb94..8afd9cb5a 100755 --- a/programs/playTests.sh +++ b/programs/playTests.sh @@ -22,10 +22,12 @@ roundTripTest() { isWindows=false ECHO="echo" +INTOVOID="/dev/null" case "$OS" in Windows*) isWindows=true ECHO="echo -e" + INTOVOID="nul" ;; esac @@ -53,19 +55,21 @@ $ECHO "test : null-length file roundtrip" $ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout $ECHO "test : decompress file with wrong suffix (must fail)" $ZSTD -d tmpCompressed && die "wrong suffix error not detected!" +$ZSTD -df tmp && die "should have refused : wrong extension" +$ECHO "test : decompress into stdout" $ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout $ZSTD --decompress tmpCompressed -c > tmpResult $ZSTD --decompress tmpCompressed --stdout > tmpResult -if [ "$isWindows" = false ] ; then - $ZSTD -d < tmp.zst > /dev/null # combine decompression, stdin & stdout - $ZSTD -d - < tmp.zst > /dev/null -fi -$ZSTD -dc < tmp.zst > /dev/null -$ZSTD -dc - < tmp.zst > /dev/null +$ECHO "test : decompress from stdin into stdout" +$ZSTD -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout +$ZSTD -dc - < tmp.zst > $INTOVOID +$ZSTD -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used +$ZSTD -d - < tmp.zst > $INTOVOID +$ECHO "test : overwrite protection" $ZSTD -q tmp && die "overwrite check failed!" +$ECHO "test : force overwrite" $ZSTD -q -f tmp $ZSTD -q --force tmp -$ZSTD -df tmp && die "should have refused : wrong extension" $ECHO "test : file removal" $ZSTD -f --rm tmp ls tmp && die "tmp should no longer be present" @@ -135,9 +139,9 @@ rm tmpSparse* $ECHO "\n**** multiple files tests **** " -./datagen -s1 > tmp1 2> /dev/null -./datagen -s2 -g100K > tmp2 2> /dev/null -./datagen -s3 -g1M > tmp3 2> /dev/null +./datagen -s1 > tmp1 2> $INTOVOID +./datagen -s2 -g100K > tmp2 2> $INTOVOID +./datagen -s3 -g1M > tmp3 2> $INTOVOID $ZSTD -f tmp* $ECHO "compress tmp* : " ls -ls tmp* diff --git a/programs/zbufftest.c b/programs/zbufftest.c index b890786cf..41dfa3384 100644 --- a/programs/zbufftest.c +++ b/programs/zbufftest.c @@ -421,11 +421,17 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compres } } /* final frame epilogue */ - { size_t dstBuffSize = cBufferSize - cSize; - size_t const flushError = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize); - CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError)); - cSize += dstBuffSize; - } + { size_t remainingToFlush = (size_t)(-1); + while (remainingToFlush) { + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize); + U32 const enoughDstSize = dstBuffSize >= remainingToFlush; + remainingToFlush = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize); + CHECK (ZBUFF_isError(remainingToFlush), "flush error : %s", ZBUFF_getErrorName(remainingToFlush)); + //DISPLAY("flush %u bytes : still within context : %i \n", (U32)dstBuffSize, (int)remainingToFlush); + CHECK (enoughDstSize && remainingToFlush, "ZBUFF_compressEnd() not fully flushed, but enough space available"); + cSize += dstBuffSize; + } } crcOrig = XXH64_digest(&xxhState); /* multi - fragments decompression test */ diff --git a/tests/test-zstd-speed.py b/tests/test-zstd-speed.py index 9a9a3992d..5cc07fb37 100755 --- a/tests/test-zstd-speed.py +++ b/tests/test-zstd-speed.py @@ -8,9 +8,9 @@ import time import traceback from subprocess import Popen, PIPE -repo_url = 'https://github.com/Cyan4973/zstd.git' +default_repo_url = 'https://github.com/Cyan4973/zstd.git' test_dir_name = 'speedTest' - +email_header = '[ZSTD_speedTest]' def log(text): print time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text @@ -126,14 +126,23 @@ def benchmark_and_compare(branch, commit, resultsFileName, lastCLevel, testFileP return text +def send_simple_email(emails, email_topic, have_mutt, have_mail): + if have_mutt: + execute('mutt -s "' + email_header + ' ' + email_topic + '" ' + emails + '