From ca1d3d423273812c4eebb1ae568a06e6d12278fb Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 28 Feb 2017 08:16:49 +0100 Subject: [PATCH 01/22] updated .travis.yml --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3c77a17e1..8399bd8a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ dist: trusty matrix: fast_finish: true include: - # other feature branches => short tests + # Ubuntu 14.04 - env: Cmd="make libc6install && make -C tests test32" - env: Cmd='make valgrindinstall arminstall ppcinstall arm-ppc-compilation && make clean lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' @@ -25,10 +25,11 @@ script: # cron & master => full tests, as this is the final step towards a Release # pull requests => normal tests (job numbers 1-3) # other feature branches => short tests (job numbers 1-2) + - @echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T7mn sh -c "$Cmd" || travis_terminate 1; else - if [ "$TRAVIS_PULL_REQUEST" == "true" ] && [ $JOB_NUMBER -lt 4 ]; then + if [ "$TRAVIS_PULL_REQUEST" = "true" ] && [ $JOB_NUMBER -lt 4 ]; then sh -c "$Cmd" || travis_terminate 1; else if [ $JOB_NUMBER -lt 3 ]; then From a3352d06bc51ed76d7e4ca592c878918df93eabd Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 28 Feb 2017 08:20:53 +0100 Subject: [PATCH 02/22] updated .travis.yml (2) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8399bd8a0..7d19e36c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ script: # cron & master => full tests, as this is the final step towards a Release # pull requests => normal tests (job numbers 1-3) # other feature branches => short tests (job numbers 1-2) - - @echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST + - echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T7mn sh -c "$Cmd" || travis_terminate 1; else From d1760113eca6244d40603997467a22423e2e1a5b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 Feb 2017 00:14:28 -0800 Subject: [PATCH 03/22] Improved speed of ZSTD_decompressStream() When ZSTD_decompressStream() detects that there is enough space in dst to complete decompression in a single pass, delegates to ZSTD_decompress(), for an extra ~5% speed boost --- lib/decompress/zstd_decompress.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 2646c8028..305a9a876 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -2100,7 +2100,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, { /* pass content and size in case legacy frames are encountered */ return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, - ddict->dictContent, ddict->dictSize, + NULL, 0, ddict); } @@ -2301,6 +2301,20 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB break; } } + /* check for single-pass mode opportunity */ + if (zds->fParams.frameContentSize + && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { + size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart); + if (cSize <= (size_t)(iend-istart)) { + size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend-op, istart, cSize, zds->ddict); + if (ZSTD_isError(decompressedSize)) return decompressedSize; + ip += cSize; + op += decompressedSize; + zds->stage = zdss_init; + someMoreWork = 0; + break; + } } + /* Consume header */ ZSTD_refDDict(zds->dctx, zds->ddict); { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */ From 8b3560e196dc15d64a591fe02c95c414412e9994 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 28 Feb 2017 09:41:23 +0100 Subject: [PATCH 04/22] update gzip tests --- tests/gzip/Makefile | 7 +++++-- tests/gzip/gzip-env.sh | 7 ++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/gzip/Makefile b/tests/gzip/Makefile index ee993934a..b02fb693f 100644 --- a/tests/gzip/Makefile +++ b/tests/gzip/Makefile @@ -12,7 +12,10 @@ VOID = /dev/null export PATH := .:$(PATH) .PHONY: all -all: test-gzip-env test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix +#all: test-gzip-env +all: test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed +all: test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid +all: test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix @echo Testing completed .PHONY: zstd @@ -35,7 +38,7 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD)) test-%: zstd - @./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$*.sh + @./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$*.sh # || echo ignoring error endif diff --git a/tests/gzip/gzip-env.sh b/tests/gzip/gzip-env.sh index f15a8d476..120e52d78 100755 --- a/tests/gzip/gzip-env.sh +++ b/tests/gzip/gzip-env.sh @@ -17,13 +17,10 @@ # along with this program. If not, see . # limit so don't run it by default. -echo XXX=$PATH -gzip --version - . "${srcdir=.}/init.sh"; path_prepend_ . -echo XXX=$PATH -gzip --version +#echo PATH=$PATH +#gzip --version echo a >exp || framework_failure_ gzip in || framework_failure_ From 8e5032a965c38f54cd4e651a011cefb94b94944b Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 28 Feb 2017 09:42:37 +0100 Subject: [PATCH 05/22] cli : fix : --rm is silent when input is stdin (decompression) --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index 7f076aa15..41daa125e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -954,7 +954,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch /* Close file */ if (fclose(srcFile)) EXM_THROW(33, "zstd: %s close error", srcFileName); /* error should never happen */ - if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(34, "zstd: %s: %s", srcFileName, strerror(errno)); }; + if (g_removeSrcFile /* --rm */ && strcmp(srcFileName, stdinmark)) { if (remove(srcFileName)) EXM_THROW(34, "zstd: %s: %s", srcFileName, strerror(errno)); }; return 0; } From c0b1731bce1460df837425e39fc2f3745b2d3908 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 Feb 2017 01:02:46 -0800 Subject: [PATCH 06/22] added test for decompression with NULL dict and NULL DDict previous version of ZSTD_decompressMultiFrame() would fail that test --- tests/fuzzer.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 79516b6cc..6fb69972a 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -108,6 +108,7 @@ static int basicUnitTests(U32 seed, double compressibility) void* const CNBuffer = malloc(CNBuffSize); void* const compressedBuffer = malloc(ZSTD_compressBound(CNBuffSize)); void* const decodedBuffer = malloc(CNBuffSize); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); int testResult = 0; U32 testNb=0; size_t cSize; @@ -155,6 +156,16 @@ static int basicUnitTests(U32 seed, double compressibility) } } DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : decompress with null dict : ", testNb++); + { size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL, 0); + if (r != CNBuffSize) goto _output_error; } + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : decompress with null DDict : ", testNb++); + { size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL); + if (r != CNBuffSize) goto _output_error; } + DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++); { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1); if (!ZSTD_isError(r)) goto _output_error; @@ -210,7 +221,6 @@ static int basicUnitTests(U32 seed, double compressibility) /* Dictionary and CCtx Duplication tests */ { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx(); ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); static const size_t dictSize = 551; DISPLAYLEVEL(4, "test%3i : copy context too soon : ", testNb++); @@ -283,12 +293,10 @@ static int basicUnitTests(U32 seed, double compressibility) ZSTD_freeCCtx(ctxOrig); ZSTD_freeCCtx(ctxDuplicated); - ZSTD_freeDCtx(dctx); } /* Dictionary and dictBuilder tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); size_t dictSize = 16 KB; void* dictBuffer = malloc(dictSize); size_t const totalSampleSize = 1 MB; @@ -370,14 +378,12 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "OK \n"); ZSTD_freeCCtx(cctx); - ZSTD_freeDCtx(dctx); free(dictBuffer); free(samplesSizes); } /* COVER dictionary builder tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); size_t dictSize = 16 KB; size_t optDictSize = dictSize; void* dictBuffer = malloc(dictSize); @@ -414,7 +420,7 @@ static int basicUnitTests(U32 seed, double compressibility) memset(¶ms, 0, sizeof(params)); params.steps = 4; optDictSize = COVER_optimizeTrainFromBuffer(dictBuffer, optDictSize, - CNBuffer, samplesSizes, nbSamples, + CNBuffer, samplesSizes, nbSamples / 4, ¶ms); if (ZDICT_isError(optDictSize)) goto _output_error; DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)optDictSize); @@ -425,7 +431,6 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "OK : %u \n", dictID); ZSTD_freeCCtx(cctx); - ZSTD_freeDCtx(dctx); free(dictBuffer); free(samplesSizes); } @@ -445,7 +450,6 @@ static int basicUnitTests(U32 seed, double compressibility) /* block API tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); static const size_t dictSize = 65 KB; static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */ size_t cSize2; From a33ae6420491e4aa6819041452a108df96977b02 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 Feb 2017 01:15:28 -0800 Subject: [PATCH 07/22] fixed decoding skippable frames --- lib/decompress/zstd_decompress.c | 9 ++++----- tests/zstreamtest.c | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 305a9a876..1252b2c37 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1469,8 +1469,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) if (ZSTD_isError(headerSize)) return headerSize; /* Frame Header */ - { - size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize); + { size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize); if (ZSTD_isError(ret)) return ret; if (ret > 0) return ERROR(srcSize_wrong); } @@ -1503,7 +1502,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) } /*! ZSTD_decompressFrame() : -* `dctx` must be properly initialized */ +* @dctx must be properly initialized */ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void** srcPtr, size_t *srcSizePtr) @@ -1570,7 +1569,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, remainingSize -= 4; } - // Allow caller to get size read + /* Allow caller to get size read */ *srcPtr = ip; *srcSizePtr = remainingSize; return op-ostart; @@ -2302,7 +2301,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } /* check for single-pass mode opportunity */ - if (zds->fParams.frameContentSize + if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */ && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart); if (cSize <= (size_t)(iend-istart)) { diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index c22a284c7..54b890266 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -218,7 +218,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo outBuff.pos = 0; { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); if (r != 0) goto _output_error; } - if (outBuff.pos != 0) goto _output_error; /* skippable frame len is 0 */ + if (outBuff.pos != 0) goto _output_error; /* skippable frame output len is 0 */ DISPLAYLEVEL(3, "OK \n"); /* Basic decompression test */ From 59709d97d9541605c166d02ba055e3814ea634cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milan=20=C5=A0ev=C4=8D=C3=ADk?= Date: Thu, 9 Feb 2017 15:10:24 +0100 Subject: [PATCH 08/22] Support building contrib utils from cmake --- build/cmake/CMakeLists.txt | 4 ++++ build/cmake/contrib/CMakeLists.txt | 16 ++++++++++++++ build/cmake/contrib/pzstd/CMakeLists.txt | 28 ++++++++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 build/cmake/contrib/CMakeLists.txt create mode 100644 build/cmake/contrib/pzstd/CMakeLists.txt diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt index b8f5d18e4..f28f74118 100644 --- a/build/cmake/CMakeLists.txt +++ b/build/cmake/CMakeLists.txt @@ -11,6 +11,7 @@ PROJECT(zstd) CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7) OPTION(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF) +OPTION(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF) IF (ZSTD_LEGACY_SUPPORT) MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT defined!") @@ -23,6 +24,9 @@ ENDIF (ZSTD_LEGACY_SUPPORT) ADD_SUBDIRECTORY(lib) ADD_SUBDIRECTORY(programs) ADD_SUBDIRECTORY(tests) +IF (ZSTD_BUILD_CONTRIB) + ADD_SUBDIRECTORY(contrib) +ENDIF (ZSTD_BUILD_CONTRIB) #----------------------------------------------------------------------------- # Add extra compilation flags diff --git a/build/cmake/contrib/CMakeLists.txt b/build/cmake/contrib/CMakeLists.txt new file mode 100644 index 000000000..68e0881c5 --- /dev/null +++ b/build/cmake/contrib/CMakeLists.txt @@ -0,0 +1,16 @@ +# ################################################################ +# * Copyright (c) 2015-present, Yann Collet, Facebook, Inc. +# * All rights reserved. +# * +# * This source code is licensed under the BSD-style license found in the +# * LICENSE file in the root directory of this source tree. An additional grant +# * of patent rights can be found in the PATENTS file in the same directory. +# +# You can contact the author at : +# - zstd homepage : http://www.zstd.net/ +# ################################################################ + +PROJECT(contrib) + +ADD_SUBDIRECTORY(pzstd) + diff --git a/build/cmake/contrib/pzstd/CMakeLists.txt b/build/cmake/contrib/pzstd/CMakeLists.txt new file mode 100644 index 000000000..6699fcc87 --- /dev/null +++ b/build/cmake/contrib/pzstd/CMakeLists.txt @@ -0,0 +1,28 @@ +# ################################################################ +# * Copyright (c) 2015-present, Yann Collet, Facebook, Inc. +# * All rights reserved. +# * +# * This source code is licensed under the BSD-style license found in the +# * LICENSE file in the root directory of this source tree. An additional grant +# * of patent rights can be found in the PATENTS file in the same directory. +# +# You can contact the author at : +# - zstd homepage : http://www.zstd.net/ +# ################################################################ + +PROJECT(pzstd) + +SET(CMAKE_INCLUDE_CURRENT_DIR TRUE) + +# Define project root directory +SET(ROOT_DIR ../../../..) + +# Define programs directory, where sources and header files are located +SET(LIBRARY_DIR ${ROOT_DIR}/lib) +SET(PROGRAMS_DIR ${ROOT_DIR}/programs) +SET(PZSTD_DIR ${ROOT_DIR}/contrib/pzstd) +INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR}) + +ADD_EXECUTABLE(pzstd ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp) +TARGET_LINK_LIBRARIES(pzstd libzstd_static pthread) + From bf8a30ce0d0e8f2b45a954fcf396220050f9172d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milan=20=C5=A0ev=C4=8D=C3=ADk?= Date: Thu, 9 Feb 2017 15:11:05 +0100 Subject: [PATCH 09/22] Add zstdmt target in cmake --- build/cmake/CMakeLists.txt | 1 + build/cmake/programs/CMakeLists.txt | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt index f28f74118..6b7c28925 100644 --- a/build/cmake/CMakeLists.txt +++ b/build/cmake/CMakeLists.txt @@ -11,6 +11,7 @@ PROJECT(zstd) CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7) OPTION(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF) +OPTION(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ON) OPTION(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF) IF (ZSTD_LEGACY_SUPPORT) diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt index cb3dc6e89..c88ee5cc9 100644 --- a/build/cmake/programs/CMakeLists.txt +++ b/build/cmake/programs/CMakeLists.txt @@ -34,9 +34,17 @@ ENDIF (MSVC) ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources}) TARGET_LINK_LIBRARIES(zstd libzstd_static) - IF (UNIX) ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c) TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static) SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT") ENDIF (UNIX) + +IF (ZSTD_MULTITHREAD_SUPPORT) + ADD_EXECUTABLE(zstdmt ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources}) + SET_TARGET_PROPERTIES(zstdmt PROPERTIES COMPILE_DEFINITIONS "ZSTD_MULTITHREAD") + TARGET_LINK_LIBRARIES(zstdmt libzstd_static) + IF (UNIX) + TARGET_LINK_LIBRARIES(zstdmt pthread) + ENDIF (UNIX) +ENDIF (ZSTD_MULTITHREAD_SUPPORT) From 5a1cc5c22d149135b93c9c48b0c71a8b1f7c0caf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milan=20=C5=A0ev=C4=8D=C3=ADk?= Date: Fri, 10 Feb 2017 12:29:55 +0100 Subject: [PATCH 10/22] Improve handling of library symlinks. Previous method was failing to remove the symlinks when make clean was invoked and wasn't portable. --- build/cmake/lib/CMakeLists.txt | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index 265f7aeb8..1950d97cd 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -166,23 +166,17 @@ IF (UNIX) SET(SHARED_LIBRARY_SYMLINK1_PATH ${CMAKE_CURRENT_BINARY_DIR}/${SHARED_LIBRARY_SYMLINK1}) SET(SHARED_LIBRARY_SYMLINK2_PATH ${CMAKE_CURRENT_BINARY_DIR}/${SHARED_LIBRARY_SYMLINK2}) - if (EXISTS ${SHARED_LIBRARY_SYMLINK1_PATH}) - FILE(REMOVE ${SHARED_LIBRARY_SYMLINK1_PATH}) - endif (EXISTS ${SHARED_LIBRARY_SYMLINK1_PATH}) - - if (EXISTS ${SHARED_LIBRARY_SYMLINK2_PATH}) - FILE(REMOVE ${SHARED_LIBRARY_SYMLINK2_PATH}) - endif (EXISTS ${SHARED_LIBRARY_SYMLINK2_PATH}) + ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK1} + DEPENDS ${SHARED_LIBRARY_LINK_PATH} + COMMENT "Generating symbolic link ${SHARED_LIBRARY_LINK} -> ${SHARED_LIBRARY_SYMLINK1}") ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD - COMMAND ln -s ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK1} - DEPENDS ${SHARED_LIBRARY_LINK_PATH} - COMMENT "Generating symbolic link") + COMMAND ${CMAKE_COMMAND} -E create_symlink ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK2} + DEPENDS ${SHARED_LIBRARY_LINK_PATH} + COMMENT "Generating symbolic link ${SHARED_LIBRARY_LINK} -> ${SHARED_LIBRARY_SYMLINK2}") - ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD - COMMAND ln -s ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK2} - DEPENDS ${SHARED_LIBRARY_LINK_PATH} - COMMENT "Generating symbolic link") + SET_DIRECTORY_PROPERTIES(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${SHARED_LIBRARY_SYMLINK1};${SHARED_LIBRARY_SYMLINK2}") INSTALL(FILES ${SHARED_LIBRARY_SYMLINK1_PATH} DESTINATION ${INSTALL_LIBRARY_DIR}) INSTALL(FILES ${SHARED_LIBRARY_SYMLINK2_PATH} DESTINATION ${INSTALL_LIBRARY_DIR}) From eeb080e6015ecc67a99d912d0376d83e6c81a79c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milan=20=C5=A0ev=C4=8D=C3=ADk?= Date: Mon, 27 Feb 2017 13:56:04 +0100 Subject: [PATCH 11/22] -Wstrict-prototypes is not supported with C++ --- build/cmake/CMakeModules/AddExtraCompilationFlags.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake b/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake index 2d59fabb0..e480c7ead 100644 --- a/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake +++ b/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake @@ -132,10 +132,8 @@ MACRO(ADD_EXTRA_COMPILATION_FLAGS) endif (ACTIVATE_WARNING_CAST_ALIGN) if (ACTIVATE_WARNING_STRICT_PROTOTYPES) - list(APPEND CMAKE_CXX_FLAGS ${WARNING_STRICT_PROTOTYPES}) list(APPEND CMAKE_C_FLAGS ${WARNING_STRICT_PROTOTYPES}) else () - string(REPLACE ${WARNING_STRICT_PROTOTYPES} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") string(REPLACE ${WARNING_STRICT_PROTOTYPES} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") endif (ACTIVATE_WARNING_STRICT_PROTOTYPES) From 4b62f419698d313647f3a111caf2c2b804711a05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milan=20=C5=A0ev=C4=8D=C3=ADk?= Date: Mon, 27 Feb 2017 14:44:49 +0100 Subject: [PATCH 12/22] Added compile flags to pzstd Definition NDEBUG from original Makefile -Wno-shadow silences shadowing in initializers --- build/cmake/contrib/pzstd/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/cmake/contrib/pzstd/CMakeLists.txt b/build/cmake/contrib/pzstd/CMakeLists.txt index 6699fcc87..2a3663f31 100644 --- a/build/cmake/contrib/pzstd/CMakeLists.txt +++ b/build/cmake/contrib/pzstd/CMakeLists.txt @@ -25,4 +25,6 @@ INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD ADD_EXECUTABLE(pzstd ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp) TARGET_LINK_LIBRARIES(pzstd libzstd_static pthread) +SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_DEFINITIONS "NDEBUG") +SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_OPTIONS "-Wno-shadow") From 22d79762ef3e5c3760975742af10e8874958c13b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 Feb 2017 02:12:42 -0800 Subject: [PATCH 13/22] fixed multi frames --- lib/decompress/zstd_decompress.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 1252b2c37..5aa863afc 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -2307,8 +2307,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (cSize <= (size_t)(iend-istart)) { size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend-op, istart, cSize, zds->ddict); if (ZSTD_isError(decompressedSize)) return decompressedSize; - ip += cSize; + ip = istart + cSize; op += decompressedSize; + zds->dctx->expected = 0; zds->stage = zdss_init; someMoreWork = 0; break; From 5c1c80cbb6fe3c71679303e521ec0ed93053899d Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 28 Feb 2017 17:27:41 +0100 Subject: [PATCH 14/22] travis.yml: fixed pull_request --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7d19e36c9..82b92c2a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ script: - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T7mn sh -c "$Cmd" || travis_terminate 1; else - if [ "$TRAVIS_PULL_REQUEST" = "true" ] && [ $JOB_NUMBER -lt 4 ]; then + if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ] && [ $JOB_NUMBER -lt 4 ]; then sh -c "$Cmd" || travis_terminate 1; else if [ $JOB_NUMBER -lt 3 ]; then From a81d4fee5822ed15e15922334197a1f0bd3e09c1 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Tue, 28 Feb 2017 15:28:29 -0800 Subject: [PATCH 15/22] Check to ensure ddict isn't null before dereference --- lib/decompress/zstd_decompress.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 2646c8028..0504778e4 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1576,6 +1576,9 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, return op-ostart; } +static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict); +static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict); + static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -1583,6 +1586,17 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) { void* const dststart = dst; + + if (ddict) { + if (dict) { + /* programmer error, these two cases should be mutually exclusive */ + return ERROR(GENERIC); + } + + dict = ZSTD_DDictDictContent(ddict); + dictSize = ZSTD_DDictDictSize(ddict); + } + while (srcSize >= ZSTD_frameHeaderSize_prefix) { U32 magicNumber; @@ -1938,6 +1952,16 @@ struct ZSTD_DDict_s { ZSTD_customMem cMem; }; /* typedef'd to ZSTD_DDict within "zstd.h" */ +static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict) +{ + return ddict->dictContent; +} + +static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict) +{ + return ddict->dictSize; +} + static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict) { ZSTD_decompressBegin(dstDCtx); /* init */ @@ -2100,7 +2124,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, { /* pass content and size in case legacy frames are encountered */ return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, - ddict->dictContent, ddict->dictSize, + NULL, 0, ddict); } From 43764cdb1dbb2c8889eb5861e8f48ea0a4450990 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 28 Feb 2017 17:44:17 -0800 Subject: [PATCH 16/22] updated NEWS for 1.1.4 cmake, performance --- NEWS | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 7ede5f70c..8b7d83c8a 100644 --- a/NEWS +++ b/NEWS @@ -3,9 +3,11 @@ cli : new : can compress in *.gz format, using --format=gzip command, by Przemys cli : new : advanced benchmark command --priority=rt cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77 cli : fix : --rm remains silent when input is stdin +speed : improved decompression speed in streaming mode for single shot scenarios (+5%) +memory : DDict (decompression dictionary) memory usage down from 150 KB to 20 KB API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell -API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size -build:new : meson build system in contrib/meson, by Dima Krasner +build: new: meson build system in contrib/meson, by Dima Krasner +build: improved cmake script, by @Majlen doc : new : educational decoder, by Sean Purcell v1.1.3 From 4bcc69b7616ea7ec39eab75676905a64b2750bed Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 1 Mar 2017 11:33:25 -0800 Subject: [PATCH 17/22] solves warnings when compiling with global XXH_STATIC_LINKING_ONLY XXH_STATIC_LINKING_ONLY protection macro is intended to be triggered just before the include. The main idea is to keep this setting local : user module shall explicitly understand and accept the static linking restriction which becomes transparent when triggering the macro at project level. Global definition also triggers redefinition warnings for user modules which do locally define the macro. This new version compiles lib and cli without warning when the macro is set globally. That's not a scenario to be recommended, since it trades a local effect for a global one, but it was easy enough to provide from zstd side. --- lib/common/xxhash.c | 4 +++- lib/common/zstd_internal.h | 4 ++++ lib/compress/zstd_compress.c | 2 -- lib/compress/zstdmt_compress.c | 2 -- lib/decompress/zstd_decompress.c | 2 -- lib/legacy/zstd_v07.c | 13 ++++++++----- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/lib/common/xxhash.c b/lib/common/xxhash.c index 29e4fa628..eb44222c5 100644 --- a/lib/common/xxhash.c +++ b/lib/common/xxhash.c @@ -104,7 +104,9 @@ static void XXH_free (void* p) { free(p); } #include static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } -#define XXH_STATIC_LINKING_ONLY +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY +#endif #include "xxhash.h" diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 4b56ce1a2..5c5b28732 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -49,6 +49,10 @@ #include "error_private.h" #define ZSTD_STATIC_LINKING_ONLY #include "zstd.h" +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#endif +#include "xxhash.h" /* XXH_reset, update, digest */ /*-************************************* diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index ec758db00..994e606e6 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -13,8 +13,6 @@ ***************************************/ #include /* memset */ #include "mem.h" -#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#include "xxhash.h" /* XXH_reset, update, digest */ #define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ #include "fse.h" #define HUF_STATIC_LINKING_ONLY diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 483ea157e..45514a81a 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -25,8 +25,6 @@ #include "threading.h" /* mutex */ #include "zstd_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */ #include "zstdmt_compress.h" -#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#include "xxhash.h" /* ====== Debug ====== */ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 61c7936f8..32b1747c6 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -43,8 +43,6 @@ *********************************************************/ #include /* memcpy, memmove, memset */ #include "mem.h" /* low level memory routines */ -#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#include "xxhash.h" /* XXH64_* */ #define FSE_STATIC_LINKING_ONLY #include "fse.h" #define HUF_STATIC_LINKING_ONLY diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index e67916b3c..a54ad0ffd 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -13,12 +13,14 @@ #include /* memcpy */ #include /* malloc, free, qsort */ -#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#include "xxhash.h" /* XXH64_* */ +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#endif +#include "xxhash.h" /* XXH64_* */ #include "zstd_v07.h" -#define FSEv07_STATIC_LINKING_ONLY /* FSEv07_MIN_TABLELOG */ -#define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */ +#define FSEv07_STATIC_LINKING_ONLY /* FSEv07_MIN_TABLELOG */ +#define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */ #define ZSTDv07_STATIC_LINKING_ONLY #include "error_private.h" @@ -4536,7 +4538,8 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd, if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ zbd->outEnd = zbd->outStart + decodedSize; zbd->stage = ZBUFFds_flush; - // break; /* ZBUFFds_flush follows */ + /* break; */ + /* pass-through */ } } case ZBUFFds_flush: From 76f04940898bf062ad4504d95fe741c0c784cb24 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 1 Mar 2017 13:29:29 -0800 Subject: [PATCH 18/22] xxhash can be included twice in any order Previously, followed by : would fail to include the static definitions, because the second include was simply skipped by guard macro. Now it works as intended : the missing static part is included during the second include. --- lib/common/xxhash.h | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/lib/common/xxhash.h b/lib/common/xxhash.h index 2c9b7c61b..9bad1f59f 100644 --- a/lib/common/xxhash.h +++ b/lib/common/xxhash.h @@ -64,16 +64,12 @@ XXH64 13.8 GB/s 1.9 GB/s XXH32 6.8 GB/s 6.0 GB/s */ -#ifndef XXHASH_H_5627135585666179 -#define XXHASH_H_5627135585666179 1 - #if defined (__cplusplus) extern "C" { #endif -#ifndef XXH_NAMESPACE -# define XXH_NAMESPACE ZSTD_ /* Zstandard specific */ -#endif +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 /* **************************** @@ -242,6 +238,11 @@ XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH /* ************************** * Canonical representation ****************************/ +/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. +* The canonical representation uses human-readable write convention, aka big-endian (large digits first). +* These functions allow transformation of hash result into and from its canonical format. +* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. +*/ typedef struct { unsigned char digest[4]; } XXH32_canonical_t; typedef struct { unsigned char digest[8]; } XXH64_canonical_t; @@ -251,14 +252,9 @@ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); -/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. -* The canonical representation uses human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. -*/ +#endif /* XXHASH_H_5627135585666179 */ -#ifdef XXH_STATIC_LINKING_ONLY /* ================================================================================================ This section contains definitions which are not guaranteed to remain stable. @@ -266,6 +262,8 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src They shall only be used with static linking. Never use these definitions in association with dynamic linking ! =================================================================================================== */ +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXH_STATIC_H_3543687687345) +#define XXH_STATIC_H_3543687687345 /* These definitions are only meant to allow allocation of XXH state statically, on stack, or in a struct for example. @@ -299,11 +297,9 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src # include "xxhash.c" /* include xxhash functions as `static`, for inlining */ # endif -#endif /* XXH_STATIC_LINKING_ONLY */ +#endif /* XXH_STATIC_LINKING_ONLY && XXH_STATIC_H_3543687687345 */ #if defined (__cplusplus) } #endif - -#endif /* XXHASH_H_5627135585666179 */ From c1c040eae1c06e66512c604d0fba092a44d3b9fd Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 1 Mar 2017 16:49:20 -0800 Subject: [PATCH 19/22] added gzip tests also : made sure zstd --format=gzip -V would fail if gzip compatibility is not supported --- programs/bench.c | 1 + programs/zstdcli.c | 4 +++- tests/playTests.sh | 35 ++++++++++++++++++++++++++++------- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 663b30743..2dd1cfb0f 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -581,6 +581,7 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* di { double const compressibility = (double)g_compressibilityDefault / 100; + if (cLevel < 1) cLevel = 1; /* minimum compression level */ if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); if (cLevelLast < cLevel) cLevelLast = cLevel; diff --git a/programs/zstdcli.c b/programs/zstdcli.c index a7b4fddc8..050d7a6a2 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -370,10 +370,12 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } +#ifdef ZSTD_GZCOMPRESS if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; } +#endif /* long commands with arguments */ -#ifndef ZSTD_NODICT +#ifndef ZSTD_NODICT if (longCommandWArg(&argument, "--cover=")) { cover=1; if (!parseCoverParameters(argument, &coverParams)) CLEAN_RETURN(badusage(programName)); continue; diff --git a/tests/playTests.sh b/tests/playTests.sh index 88c0ecdfe..5933b14e6 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -304,11 +304,32 @@ $ECHO "\n**** benchmark mode tests **** " $ECHO "bench one file" ./datagen > tmp1 -$ZSTD -bi1 tmp1 +$ZSTD -bi0 tmp1 $ECHO "bench multiple levels" -$ZSTD -i1b1e3 tmp1 +$ZSTD -i0b0e3 tmp1 $ECHO "with recursive and quiet modes" -$ZSTD -rqi1b1e3 tmp1 +$ZSTD -rqi1b1e2 tmp1 + + +$ECHO "\n**** gzip compatibility tests **** " + +GZIPMODE=1 +$ZSTD --format=gzip -V || GZIPMODE=0 +if [ $GZIPMODE -eq 1 ]; then + GZIPEXE=1 + which gzip || GZIPEXE=0 + if [ $GZIPEXE -eq 1 ]; then + ./datagen > tmp + $ZSTD --format=gzip -f tmp + gzip -t -v tmp.gz + gzip -f tmp + $ZSTD -d -f -v tmp.gz + else + $ECHO "gzip binary not detected" + fi +else + $ECHO "gzip mode not supported" +fi $ECHO "\n**** zstd round-trip tests **** " @@ -317,10 +338,10 @@ roundTripTest roundTripTest -g15K # TableID==3 roundTripTest -g127K # TableID==2 roundTripTest -g255K # TableID==1 -roundTripTest -g513K # TableID==0 -roundTripTest -g512K 6 # greedy, hash chain -roundTripTest -g512K 16 # btlazy2 -roundTripTest -g512K 19 # btopt +roundTripTest -g522K # TableID==0 +roundTripTest -g519K 6 # greedy, hash chain +roundTripTest -g517K 16 # btlazy2 +roundTripTest -g516K 19 # btopt rm tmp* From 27526c7201bc4cf1b5ad66ad746e20671cab3ae1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 1 Mar 2017 17:02:49 -0800 Subject: [PATCH 20/22] make : added target shortest shortest only run fast part of playTests.sh . cc @iburinoc --- Makefile | 6 +++++- tests/Makefile | 3 +++ tests/playTests.sh | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c73393cdd..0f97e6c5b 100644 --- a/Makefile +++ b/Makefile @@ -63,6 +63,10 @@ zstdmt: zlibwrapper: $(MAKE) -C $(ZWRAPDIR) test +.PHONY: shortest +shortest: + $(MAKE) -C $(TESTDIR) $@ + .PHONY: test test: $(MAKE) -C $(TESTDIR) $@ @@ -173,7 +177,7 @@ ppcinstall: arminstall: APT_PACKAGES="qemu-system-arm qemu-user-static gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install -valgrindinstall: +valgrindinstall: APT_PACKAGES="valgrind" $(MAKE) apt-install libc6install: diff --git a/tests/Makefile b/tests/Makefile index 30b2a04a3..88c58fce4 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -253,6 +253,9 @@ zstd-playTests: datagen file $(ZSTD) ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST) +shortest: ZSTDRTTEST= +shortest: test-zstd + test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus ifeq ($(QEMU_SYS),) test: test-pool diff --git a/tests/playTests.sh b/tests/playTests.sh index 5933b14e6..653aaf3c8 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -316,8 +316,9 @@ $ECHO "\n**** gzip compatibility tests **** " GZIPMODE=1 $ZSTD --format=gzip -V || GZIPMODE=0 if [ $GZIPMODE -eq 1 ]; then + $ECHO "gzip support detected" GZIPEXE=1 - which gzip || GZIPEXE=0 + gzip -V || GZIPEXE=0 if [ $GZIPEXE -eq 1 ]; then ./datagen > tmp $ZSTD --format=gzip -f tmp From 78208bd8be0a7e19a066214c053f52534d2729f4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 1 Mar 2017 21:02:06 -0800 Subject: [PATCH 21/22] fixed : build zstd cli after libzstd --- programs/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/programs/Makefile b/programs/Makefile index db718d14c..407a4f374 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -24,7 +24,9 @@ else ALIGN_LOOP = endif -CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder +CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ + -I$(ZSTDDIR)/dictBuilder \ + -DXXH_NAMESPACE=ZSTD_ # because xxhash.o already compiled with this macro from library CFLAGS ?= -O3 DEBUGFLAGS = -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ From 3475b9b43100a2a8b8e08ae2b09f8aafc2af758a Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 2 Mar 2017 12:33:02 -0800 Subject: [PATCH 22/22] Set dictID to 0 for content only dictionaries --- lib/decompress/zstd_decompress.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 61c7936f8..b7c668d39 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1989,6 +1989,7 @@ static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict) static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict) { + ddict->dictID = 0; ddict->entropyPresent = 0; if (ddict->dictSize < 8) return 0; { U32 const magic = MEM_readLE32(ddict->dictContent);