diff --git a/.travis.yml b/.travis.yml index 3c77a17e1..82b92c2a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ dist: trusty matrix: fast_finish: true include: - # other feature branches => short tests + # Ubuntu 14.04 - env: Cmd="make libc6install && make -C tests test32" - env: Cmd='make valgrindinstall arminstall ppcinstall arm-ppc-compilation && make clean lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' @@ -25,10 +25,11 @@ script: # cron & master => full tests, as this is the final step towards a Release # pull requests => normal tests (job numbers 1-3) # other feature branches => short tests (job numbers 1-2) + - echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T7mn sh -c "$Cmd" || travis_terminate 1; else - if [ "$TRAVIS_PULL_REQUEST" == "true" ] && [ $JOB_NUMBER -lt 4 ]; then + if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ] && [ $JOB_NUMBER -lt 4 ]; then sh -c "$Cmd" || travis_terminate 1; else if [ $JOB_NUMBER -lt 3 ]; then diff --git a/Makefile b/Makefile index c73393cdd..0f97e6c5b 100644 --- a/Makefile +++ b/Makefile @@ -63,6 +63,10 @@ zstdmt: zlibwrapper: $(MAKE) -C $(ZWRAPDIR) test +.PHONY: shortest +shortest: + $(MAKE) -C $(TESTDIR) $@ + .PHONY: test test: $(MAKE) -C $(TESTDIR) $@ @@ -173,7 +177,7 @@ ppcinstall: arminstall: APT_PACKAGES="qemu-system-arm qemu-user-static gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install -valgrindinstall: +valgrindinstall: APT_PACKAGES="valgrind" $(MAKE) apt-install libc6install: diff --git a/NEWS b/NEWS index 7ede5f70c..8b7d83c8a 100644 --- a/NEWS +++ b/NEWS @@ -3,9 +3,11 @@ cli : new : can compress in *.gz format, using --format=gzip command, by Przemys cli : new : advanced benchmark command --priority=rt cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77 cli : fix : --rm remains silent when input is stdin +speed : improved decompression speed in streaming mode for single shot scenarios (+5%) +memory : DDict (decompression dictionary) memory usage down from 150 KB to 20 KB API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell -API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size -build:new : meson build system in contrib/meson, by Dima Krasner +build: new: meson build system in contrib/meson, by Dima Krasner +build: improved cmake script, by @Majlen doc : new : educational decoder, by Sean Purcell v1.1.3 diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt index b8f5d18e4..6b7c28925 100644 --- a/build/cmake/CMakeLists.txt +++ b/build/cmake/CMakeLists.txt @@ -11,6 +11,8 @@ PROJECT(zstd) CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7) OPTION(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF) +OPTION(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ON) +OPTION(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF) IF (ZSTD_LEGACY_SUPPORT) MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT defined!") @@ -23,6 +25,9 @@ ENDIF (ZSTD_LEGACY_SUPPORT) ADD_SUBDIRECTORY(lib) ADD_SUBDIRECTORY(programs) ADD_SUBDIRECTORY(tests) +IF (ZSTD_BUILD_CONTRIB) + ADD_SUBDIRECTORY(contrib) +ENDIF (ZSTD_BUILD_CONTRIB) #----------------------------------------------------------------------------- # Add extra compilation flags diff --git a/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake b/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake index 2d59fabb0..e480c7ead 100644 --- a/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake +++ b/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake @@ -132,10 +132,8 @@ MACRO(ADD_EXTRA_COMPILATION_FLAGS) endif (ACTIVATE_WARNING_CAST_ALIGN) if (ACTIVATE_WARNING_STRICT_PROTOTYPES) - list(APPEND CMAKE_CXX_FLAGS ${WARNING_STRICT_PROTOTYPES}) list(APPEND CMAKE_C_FLAGS ${WARNING_STRICT_PROTOTYPES}) else () - string(REPLACE ${WARNING_STRICT_PROTOTYPES} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") string(REPLACE ${WARNING_STRICT_PROTOTYPES} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") endif (ACTIVATE_WARNING_STRICT_PROTOTYPES) diff --git a/build/cmake/contrib/CMakeLists.txt b/build/cmake/contrib/CMakeLists.txt new file mode 100644 index 000000000..68e0881c5 --- /dev/null +++ b/build/cmake/contrib/CMakeLists.txt @@ -0,0 +1,16 @@ +# ################################################################ +# * Copyright (c) 2015-present, Yann Collet, Facebook, Inc. +# * All rights reserved. +# * +# * This source code is licensed under the BSD-style license found in the +# * LICENSE file in the root directory of this source tree. An additional grant +# * of patent rights can be found in the PATENTS file in the same directory. +# +# You can contact the author at : +# - zstd homepage : http://www.zstd.net/ +# ################################################################ + +PROJECT(contrib) + +ADD_SUBDIRECTORY(pzstd) + diff --git a/build/cmake/contrib/pzstd/CMakeLists.txt b/build/cmake/contrib/pzstd/CMakeLists.txt new file mode 100644 index 000000000..2a3663f31 --- /dev/null +++ b/build/cmake/contrib/pzstd/CMakeLists.txt @@ -0,0 +1,30 @@ +# ################################################################ +# * Copyright (c) 2015-present, Yann Collet, Facebook, Inc. +# * All rights reserved. +# * +# * This source code is licensed under the BSD-style license found in the +# * LICENSE file in the root directory of this source tree. An additional grant +# * of patent rights can be found in the PATENTS file in the same directory. +# +# You can contact the author at : +# - zstd homepage : http://www.zstd.net/ +# ################################################################ + +PROJECT(pzstd) + +SET(CMAKE_INCLUDE_CURRENT_DIR TRUE) + +# Define project root directory +SET(ROOT_DIR ../../../..) + +# Define programs directory, where sources and header files are located +SET(LIBRARY_DIR ${ROOT_DIR}/lib) +SET(PROGRAMS_DIR ${ROOT_DIR}/programs) +SET(PZSTD_DIR ${ROOT_DIR}/contrib/pzstd) +INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR}) + +ADD_EXECUTABLE(pzstd ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp) +TARGET_LINK_LIBRARIES(pzstd libzstd_static pthread) +SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_DEFINITIONS "NDEBUG") +SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_OPTIONS "-Wno-shadow") + diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt index 265f7aeb8..1950d97cd 100644 --- a/build/cmake/lib/CMakeLists.txt +++ b/build/cmake/lib/CMakeLists.txt @@ -166,23 +166,17 @@ IF (UNIX) SET(SHARED_LIBRARY_SYMLINK1_PATH ${CMAKE_CURRENT_BINARY_DIR}/${SHARED_LIBRARY_SYMLINK1}) SET(SHARED_LIBRARY_SYMLINK2_PATH ${CMAKE_CURRENT_BINARY_DIR}/${SHARED_LIBRARY_SYMLINK2}) - if (EXISTS ${SHARED_LIBRARY_SYMLINK1_PATH}) - FILE(REMOVE ${SHARED_LIBRARY_SYMLINK1_PATH}) - endif (EXISTS ${SHARED_LIBRARY_SYMLINK1_PATH}) - - if (EXISTS ${SHARED_LIBRARY_SYMLINK2_PATH}) - FILE(REMOVE ${SHARED_LIBRARY_SYMLINK2_PATH}) - endif (EXISTS ${SHARED_LIBRARY_SYMLINK2_PATH}) + ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD + COMMAND ${CMAKE_COMMAND} -E create_symlink ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK1} + DEPENDS ${SHARED_LIBRARY_LINK_PATH} + COMMENT "Generating symbolic link ${SHARED_LIBRARY_LINK} -> ${SHARED_LIBRARY_SYMLINK1}") ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD - COMMAND ln -s ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK1} - DEPENDS ${SHARED_LIBRARY_LINK_PATH} - COMMENT "Generating symbolic link") + COMMAND ${CMAKE_COMMAND} -E create_symlink ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK2} + DEPENDS ${SHARED_LIBRARY_LINK_PATH} + COMMENT "Generating symbolic link ${SHARED_LIBRARY_LINK} -> ${SHARED_LIBRARY_SYMLINK2}") - ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD - COMMAND ln -s ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK2} - DEPENDS ${SHARED_LIBRARY_LINK_PATH} - COMMENT "Generating symbolic link") + SET_DIRECTORY_PROPERTIES(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${SHARED_LIBRARY_SYMLINK1};${SHARED_LIBRARY_SYMLINK2}") INSTALL(FILES ${SHARED_LIBRARY_SYMLINK1_PATH} DESTINATION ${INSTALL_LIBRARY_DIR}) INSTALL(FILES ${SHARED_LIBRARY_SYMLINK2_PATH} DESTINATION ${INSTALL_LIBRARY_DIR}) diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt index cb3dc6e89..c88ee5cc9 100644 --- a/build/cmake/programs/CMakeLists.txt +++ b/build/cmake/programs/CMakeLists.txt @@ -34,9 +34,17 @@ ENDIF (MSVC) ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources}) TARGET_LINK_LIBRARIES(zstd libzstd_static) - IF (UNIX) ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c) TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static) SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT") ENDIF (UNIX) + +IF (ZSTD_MULTITHREAD_SUPPORT) + ADD_EXECUTABLE(zstdmt ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources}) + SET_TARGET_PROPERTIES(zstdmt PROPERTIES COMPILE_DEFINITIONS "ZSTD_MULTITHREAD") + TARGET_LINK_LIBRARIES(zstdmt libzstd_static) + IF (UNIX) + TARGET_LINK_LIBRARIES(zstdmt pthread) + ENDIF (UNIX) +ENDIF (ZSTD_MULTITHREAD_SUPPORT) diff --git a/lib/common/xxhash.c b/lib/common/xxhash.c index 29e4fa628..eb44222c5 100644 --- a/lib/common/xxhash.c +++ b/lib/common/xxhash.c @@ -104,7 +104,9 @@ static void XXH_free (void* p) { free(p); } #include static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } -#define XXH_STATIC_LINKING_ONLY +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY +#endif #include "xxhash.h" diff --git a/lib/common/xxhash.h b/lib/common/xxhash.h index 2c9b7c61b..9bad1f59f 100644 --- a/lib/common/xxhash.h +++ b/lib/common/xxhash.h @@ -64,16 +64,12 @@ XXH64 13.8 GB/s 1.9 GB/s XXH32 6.8 GB/s 6.0 GB/s */ -#ifndef XXHASH_H_5627135585666179 -#define XXHASH_H_5627135585666179 1 - #if defined (__cplusplus) extern "C" { #endif -#ifndef XXH_NAMESPACE -# define XXH_NAMESPACE ZSTD_ /* Zstandard specific */ -#endif +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 /* **************************** @@ -242,6 +238,11 @@ XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH /* ************************** * Canonical representation ****************************/ +/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. +* The canonical representation uses human-readable write convention, aka big-endian (large digits first). +* These functions allow transformation of hash result into and from its canonical format. +* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. +*/ typedef struct { unsigned char digest[4]; } XXH32_canonical_t; typedef struct { unsigned char digest[8]; } XXH64_canonical_t; @@ -251,14 +252,9 @@ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); -/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. -* The canonical representation uses human-readable write convention, aka big-endian (large digits first). -* These functions allow transformation of hash result into and from its canonical format. -* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. -*/ +#endif /* XXHASH_H_5627135585666179 */ -#ifdef XXH_STATIC_LINKING_ONLY /* ================================================================================================ This section contains definitions which are not guaranteed to remain stable. @@ -266,6 +262,8 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src They shall only be used with static linking. Never use these definitions in association with dynamic linking ! =================================================================================================== */ +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXH_STATIC_H_3543687687345) +#define XXH_STATIC_H_3543687687345 /* These definitions are only meant to allow allocation of XXH state statically, on stack, or in a struct for example. @@ -299,11 +297,9 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src # include "xxhash.c" /* include xxhash functions as `static`, for inlining */ # endif -#endif /* XXH_STATIC_LINKING_ONLY */ +#endif /* XXH_STATIC_LINKING_ONLY && XXH_STATIC_H_3543687687345 */ #if defined (__cplusplus) } #endif - -#endif /* XXHASH_H_5627135585666179 */ diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index 4b56ce1a2..5c5b28732 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -49,6 +49,10 @@ #include "error_private.h" #define ZSTD_STATIC_LINKING_ONLY #include "zstd.h" +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#endif +#include "xxhash.h" /* XXH_reset, update, digest */ /*-************************************* diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 89f6575dc..5d81f8a38 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -13,8 +13,6 @@ ***************************************/ #include /* memset */ #include "mem.h" -#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#include "xxhash.h" /* XXH_reset, update, digest */ #define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ #include "fse.h" #define HUF_STATIC_LINKING_ONLY diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 483ea157e..45514a81a 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -25,8 +25,6 @@ #include "threading.h" /* mutex */ #include "zstd_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */ #include "zstdmt_compress.h" -#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#include "xxhash.h" /* ====== Debug ====== */ diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index e39bf42bf..4dfdf20fb 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -43,8 +43,6 @@ *********************************************************/ #include /* memcpy, memmove, memset */ #include "mem.h" /* low level memory routines */ -#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#include "xxhash.h" /* XXH64_* */ #define FSE_STATIC_LINKING_ONLY #include "fse.h" #define HUF_STATIC_LINKING_ONLY @@ -1485,8 +1483,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) if (ZSTD_isError(headerSize)) return headerSize; /* Frame Header */ - { - size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize); + { size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize); if (ZSTD_isError(ret)) return ret; if (ret > 0) return ERROR(srcSize_wrong); } @@ -1519,7 +1516,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) } /*! ZSTD_decompressFrame() : -* `dctx` must be properly initialized */ +* @dctx must be properly initialized */ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void** srcPtr, size_t *srcSizePtr) @@ -1586,12 +1583,15 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, remainingSize -= 4; } - // Allow caller to get size read + /* Allow caller to get size read */ *srcPtr = ip; *srcSizePtr = remainingSize; return op-ostart; } +static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict); +static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict); + static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -1599,6 +1599,17 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) { void* const dststart = dst; + + if (ddict) { + if (dict) { + /* programmer error, these two cases should be mutually exclusive */ + return ERROR(GENERIC); + } + + dict = ZSTD_DDictDictContent(ddict); + dictSize = ZSTD_DDictDictSize(ddict); + } + while (srcSize >= ZSTD_frameHeaderSize_prefix) { U32 magicNumber; @@ -1954,6 +1965,16 @@ struct ZSTD_DDict_s { ZSTD_customMem cMem; }; /* typedef'd to ZSTD_DDict within "zstd.h" */ +static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict) +{ + return ddict->dictContent; +} + +static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict) +{ + return ddict->dictSize; +} + static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict) { ZSTD_decompressBegin(dstDCtx); /* init */ @@ -1982,6 +2003,7 @@ static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict) static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict) { + ddict->dictID = 0; ddict->entropyPresent = 0; if (ddict->dictSize < 8) return 0; { U32 const magic = MEM_readLE32(ddict->dictContent); @@ -2116,7 +2138,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, { /* pass content and size in case legacy frames are encountered */ return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, - ddict->dictContent, ddict->dictSize, + NULL, 0, ddict); } @@ -2317,6 +2339,21 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB break; } } + /* check for single-pass mode opportunity */ + if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */ + && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { + size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart); + if (cSize <= (size_t)(iend-istart)) { + size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend-op, istart, cSize, zds->ddict); + if (ZSTD_isError(decompressedSize)) return decompressedSize; + ip = istart + cSize; + op += decompressedSize; + zds->dctx->expected = 0; + zds->stage = zdss_init; + someMoreWork = 0; + break; + } } + /* Consume header */ ZSTD_refDDict(zds->dctx, zds->ddict); { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */ diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index e67916b3c..a54ad0ffd 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -13,12 +13,14 @@ #include /* memcpy */ #include /* malloc, free, qsort */ -#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#include "xxhash.h" /* XXH64_* */ +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#endif +#include "xxhash.h" /* XXH64_* */ #include "zstd_v07.h" -#define FSEv07_STATIC_LINKING_ONLY /* FSEv07_MIN_TABLELOG */ -#define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */ +#define FSEv07_STATIC_LINKING_ONLY /* FSEv07_MIN_TABLELOG */ +#define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */ #define ZSTDv07_STATIC_LINKING_ONLY #include "error_private.h" @@ -4536,7 +4538,8 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd, if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ zbd->outEnd = zbd->outStart + decodedSize; zbd->stage = ZBUFFds_flush; - // break; /* ZBUFFds_flush follows */ + /* break; */ + /* pass-through */ } } case ZBUFFds_flush: diff --git a/programs/Makefile b/programs/Makefile index db718d14c..407a4f374 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -24,7 +24,9 @@ else ALIGN_LOOP = endif -CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder +CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ + -I$(ZSTDDIR)/dictBuilder \ + -DXXH_NAMESPACE=ZSTD_ # because xxhash.o already compiled with this macro from library CFLAGS ?= -O3 DEBUGFLAGS = -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ diff --git a/programs/bench.c b/programs/bench.c index 663b30743..2dd1cfb0f 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -581,6 +581,7 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* di { double const compressibility = (double)g_compressibilityDefault / 100; + if (cLevel < 1) cLevel = 1; /* minimum compression level */ if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel(); if (cLevelLast < cLevel) cLevelLast = cLevel; diff --git a/programs/fileio.c b/programs/fileio.c index 7f076aa15..41daa125e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -954,7 +954,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch /* Close file */ if (fclose(srcFile)) EXM_THROW(33, "zstd: %s close error", srcFileName); /* error should never happen */ - if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(34, "zstd: %s: %s", srcFileName, strerror(errno)); }; + if (g_removeSrcFile /* --rm */ && strcmp(srcFileName, stdinmark)) { if (remove(srcFileName)) EXM_THROW(34, "zstd: %s: %s", srcFileName, strerror(errno)); }; return 0; } diff --git a/programs/zstdcli.c b/programs/zstdcli.c index a7b4fddc8..050d7a6a2 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -370,10 +370,12 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } +#ifdef ZSTD_GZCOMPRESS if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; } +#endif /* long commands with arguments */ -#ifndef ZSTD_NODICT +#ifndef ZSTD_NODICT if (longCommandWArg(&argument, "--cover=")) { cover=1; if (!parseCoverParameters(argument, &coverParams)) CLEAN_RETURN(badusage(programName)); continue; diff --git a/tests/Makefile b/tests/Makefile index 30b2a04a3..88c58fce4 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -253,6 +253,9 @@ zstd-playTests: datagen file $(ZSTD) ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST) +shortest: ZSTDRTTEST= +shortest: test-zstd + test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus ifeq ($(QEMU_SYS),) test: test-pool diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 79516b6cc..6fb69972a 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -108,6 +108,7 @@ static int basicUnitTests(U32 seed, double compressibility) void* const CNBuffer = malloc(CNBuffSize); void* const compressedBuffer = malloc(ZSTD_compressBound(CNBuffSize)); void* const decodedBuffer = malloc(CNBuffSize); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); int testResult = 0; U32 testNb=0; size_t cSize; @@ -155,6 +156,16 @@ static int basicUnitTests(U32 seed, double compressibility) } } DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : decompress with null dict : ", testNb++); + { size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL, 0); + if (r != CNBuffSize) goto _output_error; } + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : decompress with null DDict : ", testNb++); + { size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL); + if (r != CNBuffSize) goto _output_error; } + DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++); { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1); if (!ZSTD_isError(r)) goto _output_error; @@ -210,7 +221,6 @@ static int basicUnitTests(U32 seed, double compressibility) /* Dictionary and CCtx Duplication tests */ { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx(); ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); static const size_t dictSize = 551; DISPLAYLEVEL(4, "test%3i : copy context too soon : ", testNb++); @@ -283,12 +293,10 @@ static int basicUnitTests(U32 seed, double compressibility) ZSTD_freeCCtx(ctxOrig); ZSTD_freeCCtx(ctxDuplicated); - ZSTD_freeDCtx(dctx); } /* Dictionary and dictBuilder tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); size_t dictSize = 16 KB; void* dictBuffer = malloc(dictSize); size_t const totalSampleSize = 1 MB; @@ -370,14 +378,12 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "OK \n"); ZSTD_freeCCtx(cctx); - ZSTD_freeDCtx(dctx); free(dictBuffer); free(samplesSizes); } /* COVER dictionary builder tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); size_t dictSize = 16 KB; size_t optDictSize = dictSize; void* dictBuffer = malloc(dictSize); @@ -414,7 +420,7 @@ static int basicUnitTests(U32 seed, double compressibility) memset(¶ms, 0, sizeof(params)); params.steps = 4; optDictSize = COVER_optimizeTrainFromBuffer(dictBuffer, optDictSize, - CNBuffer, samplesSizes, nbSamples, + CNBuffer, samplesSizes, nbSamples / 4, ¶ms); if (ZDICT_isError(optDictSize)) goto _output_error; DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)optDictSize); @@ -425,7 +431,6 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "OK : %u \n", dictID); ZSTD_freeCCtx(cctx); - ZSTD_freeDCtx(dctx); free(dictBuffer); free(samplesSizes); } @@ -445,7 +450,6 @@ static int basicUnitTests(U32 seed, double compressibility) /* block API tests */ { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); - ZSTD_DCtx* const dctx = ZSTD_createDCtx(); static const size_t dictSize = 65 KB; static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */ size_t cSize2; diff --git a/tests/gzip/Makefile b/tests/gzip/Makefile index ee993934a..b02fb693f 100644 --- a/tests/gzip/Makefile +++ b/tests/gzip/Makefile @@ -12,7 +12,10 @@ VOID = /dev/null export PATH := .:$(PATH) .PHONY: all -all: test-gzip-env test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix +#all: test-gzip-env +all: test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed +all: test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid +all: test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix @echo Testing completed .PHONY: zstd @@ -35,7 +38,7 @@ clean: ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD)) test-%: zstd - @./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$*.sh + @./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$*.sh # || echo ignoring error endif diff --git a/tests/gzip/gzip-env.sh b/tests/gzip/gzip-env.sh index f15a8d476..120e52d78 100755 --- a/tests/gzip/gzip-env.sh +++ b/tests/gzip/gzip-env.sh @@ -17,13 +17,10 @@ # along with this program. If not, see . # limit so don't run it by default. -echo XXX=$PATH -gzip --version - . "${srcdir=.}/init.sh"; path_prepend_ . -echo XXX=$PATH -gzip --version +#echo PATH=$PATH +#gzip --version echo a >exp || framework_failure_ gzip in || framework_failure_ diff --git a/tests/playTests.sh b/tests/playTests.sh index 88c0ecdfe..653aaf3c8 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -304,11 +304,33 @@ $ECHO "\n**** benchmark mode tests **** " $ECHO "bench one file" ./datagen > tmp1 -$ZSTD -bi1 tmp1 +$ZSTD -bi0 tmp1 $ECHO "bench multiple levels" -$ZSTD -i1b1e3 tmp1 +$ZSTD -i0b0e3 tmp1 $ECHO "with recursive and quiet modes" -$ZSTD -rqi1b1e3 tmp1 +$ZSTD -rqi1b1e2 tmp1 + + +$ECHO "\n**** gzip compatibility tests **** " + +GZIPMODE=1 +$ZSTD --format=gzip -V || GZIPMODE=0 +if [ $GZIPMODE -eq 1 ]; then + $ECHO "gzip support detected" + GZIPEXE=1 + gzip -V || GZIPEXE=0 + if [ $GZIPEXE -eq 1 ]; then + ./datagen > tmp + $ZSTD --format=gzip -f tmp + gzip -t -v tmp.gz + gzip -f tmp + $ZSTD -d -f -v tmp.gz + else + $ECHO "gzip binary not detected" + fi +else + $ECHO "gzip mode not supported" +fi $ECHO "\n**** zstd round-trip tests **** " @@ -317,10 +339,10 @@ roundTripTest roundTripTest -g15K # TableID==3 roundTripTest -g127K # TableID==2 roundTripTest -g255K # TableID==1 -roundTripTest -g513K # TableID==0 -roundTripTest -g512K 6 # greedy, hash chain -roundTripTest -g512K 16 # btlazy2 -roundTripTest -g512K 19 # btopt +roundTripTest -g522K # TableID==0 +roundTripTest -g519K 6 # greedy, hash chain +roundTripTest -g517K 16 # btlazy2 +roundTripTest -g516K 19 # btopt rm tmp* diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index c22a284c7..54b890266 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -218,7 +218,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo outBuff.pos = 0; { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); if (r != 0) goto _output_error; } - if (outBuff.pos != 0) goto _output_error; /* skippable frame len is 0 */ + if (outBuff.pos != 0) goto _output_error; /* skippable frame output len is 0 */ DISPLAYLEVEL(3, "OK \n"); /* Basic decompression test */