Merge pull request #1354 from facebook/dev

Zstandard v1.3.6
Merge pull request #1217 from facebook/dev
2018-10-05 09:34:52 -07:00 · 2018-06-28 01:55:26 -07:00 · 2018-06-27 16:04:58 -07:00 · 2018-06-27 14:24:54 -07:00 · 2018-06-27 14:16:52 -07:00 · 2018-03-26 15:19:34 -07:00
19 changed files with 86 additions and 259 deletions
@@ -19,3 +19,6 @@
 # Windows
 *.bat text eol=crlf
 *.cmd text eol=crlf
 # .travis.yml merging
 .travis.yml merge=ours
@@ -47,6 +47,14 @@ matrix:
    - if: tag =~ ^v[0-9]\.[0-9]
      env: Cmd='make -C tests checkTag && tests/checkTag $TRAVIS_BRANCH'
    # additional tests for master branch and cron job
    - env: Cmd="make test"
      os: osx
    - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
    - env: Cmd='make cxxtest && make clean && make gnu90build && make clean && make cmakebuild && make clean && make travis-install'
    - env: Cmd='make gpp6install valgrindinstall && make -C zlibWrapper test && make -C zlibWrapper valgrindTest'
 git:
  depth: 1
@@ -62,38 +62,27 @@ zstdmt:
 zlibwrapper: lib
 	$(MAKE) -C $(ZWRAPDIR) all
 ## test: run long-duration tests
 .PHONY: test
 test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror
 test:
 	MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
 	$(MAKE) -C $(TESTDIR) $@
 ## shortest: same as `make check`
 .PHONY: shortest
 shortest:
 	$(MAKE) -C $(TESTDIR) $@
 ## check: run basic tests for `zstd` cli
 .PHONY: check
 check: shortest
 ## examples: build all examples in `/examples` directory
 .PHONY: examples
 examples: lib
 	CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all
 ## manual: generate API documentation in html format
 .PHONY: manual
 manual:
 	$(MAKE) -C contrib/gen_html $@
 ## man: generate man page
 .PHONY: man
 man:
 	$(MAKE) -C programs $@
 ## contrib: build all supported projects in `/contrib` directory
 .PHONY: contrib
 contrib: lib
 	$(MAKE) -C contrib/pzstd all
@@ -1,10 +1,3 @@
 v1.3.7
 perf: slightly better decompression speed on clang (depending on hardware target)
 fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10
 build: no longer build backtrace by default in release mode; restrict further automatic mode
 build: control backtrace support through build macro BACKTRACE
 misc: added man pages for zstdless and zstdgrep, by @samrussell
 v1.3.6
 perf: much faster dictionary builder, by @jenniferliu
 perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte
@@ -43,12 +43,6 @@ IF (UNIX)
    ADD_CUSTOM_TARGET(zstd.1 ALL
        ${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstd.1 .
        COMMENT "Copying manpage zstd.1")
    ADD_CUSTOM_TARGET(zstdgrep.1 ALL
        ${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstdgrep.1 .
        COMMENT "Copying manpage zstdgrep.1")
    ADD_CUSTOM_TARGET(zstdless.1 ALL
        ${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstdless.1 .
        COMMENT "Copying manpage zstdless.1")
    ADD_CUSTOM_TARGET(zstdcat.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 zstdcat.1 DEPENDS zstd.1 COMMENT "Creating zstdcat.1 symlink")
    ADD_CUSTOM_TARGET(unzstd.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 unzstd.1 DEPENDS zstd.1 COMMENT "Creating unzstd.1 symlink")
@@ -61,8 +55,6 @@ IF (UNIX)
    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstd.1 DESTINATION "${MAN_INSTALL_DIR}")
    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1 DESTINATION "${MAN_INSTALL_DIR}")
    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd.1 DESTINATION "${MAN_INSTALL_DIR}")
    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1 DESTINATION "${MAN_INSTALL_DIR}")
    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1 DESTINATION "${MAN_INSTALL_DIR}")
    ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c)
    TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static)
@@ -1,10 +1,10 @@
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>zstd 1.3.7 Manual</title>
+<title>zstd 1.3.6 Manual</title>
 </head>
 <body>
-<h1>zstd 1.3.7 Manual</h1>
+<h1>zstd 1.3.6 Manual</h1>
 <hr>
 <a name="Contents"></a><h2>Contents</h2>
 <ol>
@@ -313,17 +313,11 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
  The function will update both `pos` fields.
  If `input.pos < input.size`, some input has not been consumed.
  It's up to the caller to present again remaining data.
  The function tries to flush all data decoded immediately, repecting buffer sizes.
  If `output.pos < output.size`, decoder has flushed everything it could.
-  But if `output.pos == output.size`, there is no such guarantee,
+  @return : 0 when a frame is completely decoded and fully flushed,
-  it's likely that some decoded data was not flushed and still remains within internal buffers.
+            an error code, which can be tested using ZSTD_isError(),
-  In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
+            any other value > 0, which means there is still some decoding to do to complete current frame.
-  When no additional input is provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
+            The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
 @return : 0 when a frame is completely decoded and fully flushed,
        or an error code, which can be tested using ZSTD_isError(),
        or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
                                the return value is a suggested next input size (a hint for better latency)
                                that will never load more than the current frame.
 <BR></pre>
@@ -606,40 +600,22 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
 </pre></b><BR>
 <pre><b>size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 </b><p>  start a new compression job, using same parameters from previous job.
-  This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
+  This is typically useful to skip dictionary loading stage, since it will re-use it in-place..
  Note that zcs must be init at least once before using ZSTD_resetCStream().
  If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
  If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
  For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
  but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
- @return : 0, or an error code (which can be tested using ZSTD_isError())
+ @return : 0, or an error code (which can be tested using ZSTD_isError()) 
 </p></pre><BR>
 <pre><b>typedef struct {
-    unsigned long long ingested;   </b>/* nb input bytes read and buffered */<b>
+    unsigned long long ingested;
-    unsigned long long consumed;   </b>/* nb input bytes actually compressed */<b>
+    unsigned long long consumed;
-    unsigned long long produced;   </b>/* nb of compressed bytes generated and buffered */<b>
+    unsigned long long produced;
-    unsigned long long flushed;    </b>/* nb of compressed bytes flushed : not provided; can be tracked from caller side */<b>
+    unsigned currentJobID;
    unsigned currentJobID;         </b>/* MT only : latest started job nb */<b>
    unsigned nbActiveWorkers;      </b>/* MT only : nb of workers actively compressing at probe time */<b>
 } ZSTD_frameProgression;
 </b></pre><BR>
 <pre><b>size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
 </b><p>  Tell how many bytes are ready to be flushed immediately.
  Useful for multithreading scenarios (nbWorkers >= 1).
  Probe the oldest active job, defined as oldest job not yet entirely flushed,
  and check its output buffer.
 @return : amount of data stored in oldest job and ready to be flushed immediately.
  if @return == 0, it means either :
  + there is no active job (could be checked with ZSTD_frameProgression()), or
  + oldest job is still actively compressing data,
    but everything it has produced has also been flushed so far,
    therefore flushing speed is currently limited by production speed of oldest job
    irrespective of the speed of concurrent newer jobs.
 </p></pre><BR>
 <h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
 size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);   </b>/* obsolete : this API will be removed in a future version */<b>
 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); </b>/**< note: no dictionary will be used if dict == NULL or dictSize < 8 */<b>
@@ -1039,13 +1015,9 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx,
 </p></pre><BR>
 <pre><b>typedef enum {
-    ZSTD_e_continue=0, </b>/* collect more data, encoder decides when to output compressed result, for optimal compression ratio */<b>
+    ZSTD_e_continue=0, </b>/* collect more data, encoder decides when to output compressed result, for optimal conditions */<b>
-    ZSTD_e_flush,      </b>/* flush any data provided so far,<b>
+    ZSTD_e_flush,      </b>/* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */<b>
-                        * it creates (at least) one new block, that can be decoded immediately on reception;
+    ZSTD_e_end         </b>/* flush any remaining data and close current frame. Any additional data starts a new frame. */<b>
                        * frame will continue: any future data can still reference previously compressed data, improving compression. */
    ZSTD_e_end         </b>/* flush any remaining data and close current frame.<b>
                        * any additional data starts a new frame.
                        * each frame is independent (does not reference any content from previous frame). */
 } ZSTD_EndDirective;
 </b></pre><BR>
 <pre><b>size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
@@ -339,10 +339,17 @@ MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
 MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
 {
-    U32 const regMask = sizeof(bitContainer)*8 - 1;
+#if defined(__BMI__) && defined(__GNUC__) && __GNUC__*1000+__GNUC_MINOR__ >= 4008  /* experimental */
-    /* if start > regMask, bitstream is corrupted, and result is undefined */
+#  if defined(__x86_64__)
    if (sizeof(bitContainer)==8)
        return _bextr_u64(bitContainer, start, nbBits);
    else
 #  endif
        return _bextr_u32(bitContainer, start, nbBits);
 #else
    assert(nbBits < BIT_MASK_SIZE);
-    return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
+    return (bitContainer >> start) & BIT_mask[nbBits];
 #endif
 }
 MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
@@ -359,13 +366,9 @@ MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
 * @return : value extracted */
 MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
 {
-    /* arbitrate between double-shift and shift+mask */
+#if defined(__BMI__) && defined(__GNUC__)   /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
 #if 1
    /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8,
     * bitstream is likely corrupted, and result is undefined */
    return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
 #else
    /* this code path is slower on my os-x laptop */
    U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
    return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask);
 #endif
@@ -39,10 +39,6 @@ extern "C" {
 #  define MEM_STATIC static  /* this version may generate warnings for unused static functions; disable the relevant warning */
 #endif
 #ifndef __has_builtin
 #  define __has_builtin(x) 0  /* compat. with non-clang compilers */
 #endif
 /* code only tested on 32 and 64 bits systems */
 #define MEM_STATIC_ASSERT(c)   { enum { MEM_static_assert = 1/(int)(!!(c)) }; }
 MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
@@ -202,8 +198,7 @@ MEM_STATIC U32 MEM_swap32(U32 in)
 {
 #if defined(_MSC_VER)     /* Visual Studio */
    return _byteswap_ulong(in);
-#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
+#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
  || (defined(__clang__) && __has_builtin(__builtin_bswap32))
    return __builtin_bswap32(in);
 #else
    return  ((in << 24) & 0xff000000 ) |
@@ -217,8 +212,7 @@ MEM_STATIC U64 MEM_swap64(U64 in)
 {
 #if defined(_MSC_VER)     /* Visual Studio */
    return _byteswap_uint64(in);
-#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
+#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
  || (defined(__clang__) && __has_builtin(__builtin_bswap64))
    return __builtin_bswap64(in);
 #else
    return  ((in << 56) & 0xff00000000000000ULL) |
@@ -147,7 +147,6 @@ ZSTD_DUBT_findBetterDictMatch (
        ZSTD_matchState_t* ms,
        const BYTE* const ip, const BYTE* const iend,
        size_t* offsetPtr,
        size_t bestLength,
        U32 nbCompares,
        U32 const mls,
        const ZSTD_dictMode_e dictMode)
@@ -173,7 +172,8 @@ ZSTD_DUBT_findBetterDictMatch (
    U32         const btMask = (1 << btLog) - 1;
    U32         const btLow = (btMask >= dictHighLimit - dictLowLimit) ? dictLowLimit : dictHighLimit - btMask;
-    size_t commonLengthSmaller=0, commonLengthLarger=0;
+    size_t commonLengthSmaller=0, commonLengthLarger=0, bestLength=0;
    U32 matchEndIdx = current+8+1;
    (void)dictMode;
    assert(dictMode == ZSTD_dictMatchState);
@@ -188,8 +188,10 @@ ZSTD_DUBT_findBetterDictMatch (
        if (matchLength > bestLength) {
            U32 matchIndex = dictMatchIndex + dictIndexDelta;
            if (matchLength > matchEndIdx - matchIndex)
                matchEndIdx = matchIndex + (U32)matchLength;
            if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) {
-                DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)",
+                DEBUGLOG(2, "ZSTD_DUBT_findBestDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)",
                    current, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, ZSTD_REP_MOVE + current - matchIndex, dictMatchIndex, matchIndex);
                bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex;
            }
@@ -198,6 +200,7 @@ ZSTD_DUBT_findBetterDictMatch (
            }
        }
        DEBUGLOG(2, "matchLength:%6zu, match:%p, prefixStart:%p, ip:%p", matchLength, match, prefixStart, ip);
        if (match[matchLength] < ip[matchLength]) {
            if (dictMatchIndex <= btLow) { break; }   /* beyond tree size, stop the search */
            commonLengthSmaller = matchLength;    /* all smaller will now have at least this guaranteed common length */
@@ -212,7 +215,7 @@ ZSTD_DUBT_findBetterDictMatch (
    if (bestLength >= MINMATCH) {
        U32 const mIndex = current - ((U32)*offsetPtr - ZSTD_REP_MOVE); (void)mIndex;
-        DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
+        DEBUGLOG(2, "ZSTD_DUBT_findBestDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
                    current, (U32)bestLength, (U32)*offsetPtr, mIndex);
    }
    return bestLength;
@@ -320,11 +323,6 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
                if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) )
                    bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex;
                if (ip+matchLength == iend) {   /* equal : no way to know if inf or sup */
                    if (dictMode == ZSTD_dictMatchState) {
                        nbCompares = 0; /* in addition to avoiding checking any
                                         * further in this loop, make sure we
                                         * skip checking in the dictionary. */
                    }
                    break;   /* drop, to guarantee consistency (miss a little bit of compression) */
                }
            }
@@ -348,10 +346,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
        *smallerPtr = *largerPtr = 0;
        if (dictMode == ZSTD_dictMatchState && nbCompares) {
-            bestLength = ZSTD_DUBT_findBetterDictMatch(
+            bestLength = ZSTD_DUBT_findBetterDictMatch(ms, ip, iend, offsetPtr, nbCompares, mls, dictMode);
                    ms, ip, iend,
                    offsetPtr, bestLength, nbCompares,
                    mls, dictMode);
        }
        assert(matchEndIdx > current+8); /* ensure nextToUpdate is increased */
@@ -71,7 +71,7 @@ extern "C" {
 /*------   Version   ------*/
 #define ZSTD_VERSION_MAJOR    1
 #define ZSTD_VERSION_MINOR    3
-#define ZSTD_VERSION_RELEASE  7
+#define ZSTD_VERSION_RELEASE  6
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 ZSTDLIB_API unsigned ZSTD_versionNumber(void);   /**< useful to check dll version */
@@ -361,21 +361,15 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output
 *  The function will update both `pos` fields.
 *  If `input.pos < input.size`, some input has not been consumed.
 *  It's up to the caller to present again remaining data.
 *  The function tries to flush all data decoded immediately, repecting buffer sizes.
 *  If `output.pos < output.size`, decoder has flushed everything it could.
-*  But if `output.pos == output.size`, there is no such guarantee,
+*  @return : 0 when a frame is completely decoded and fully flushed,
-*  it's likely that some decoded data was not flushed and still remains within internal buffers.
+*            an error code, which can be tested using ZSTD_isError(),
-*  In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
+*            any other value > 0, which means there is still some decoding to do to complete current frame.
-*  When no additional input is provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
+*            The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
 * @return : 0 when a frame is completely decoded and fully flushed,
 *        or an error code, which can be tested using ZSTD_isError(),
 *        or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
 *                                the return value is a suggested next input size (a hint for better latency)
 *                                that will never load more than the current frame.
 * *******************************************************************************/
 typedef ZSTD_DCtx ZSTD_DStream;  /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
-                                 /* For compatibility with versions <= v1.2.0, prefer differentiating them. */
+                                 /* For compatibility with versions <= v1.2.0, continue to consider them separated. */
 /*===== ZSTD_DStream management functions =====*/
 ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
 ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
@@ -1241,13 +1235,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_resetParameters(ZSTD_CCtx* cctx);
 typedef enum {
-    ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
+    ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal conditions */
-    ZSTD_e_flush,      /* flush any data provided so far,
+    ZSTD_e_flush,      /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
-                        * it creates (at least) one new block, that can be decoded immediately on reception;
+    ZSTD_e_end         /* flush any remaining data and close current frame. Any additional data starts a new frame. */
                        * frame will continue: any future data can still reference previously compressed data, improving compression. */
    ZSTD_e_end         /* flush any remaining data and close current frame.
                        * any additional data starts a new frame.
                        * each frame is independent (does not reference any content from previous frame). */
 } ZSTD_EndDirective;
 /*! ZSTD_compress_generic() :
@@ -134,14 +134,11 @@ else
 LZ4_MSG := $(NO_LZ4_MSG)
 endif
-# explicit backtrace enable/disable for Linux & Darwin
+# enable backtrace symbol names for Linux/Darwin
-ifeq ($(BACKTRACE), 0)
+ALL_SYMBOLS := 0
 DEBUGFLAGS += -DBACKTRACE_ENABLE=0
 endif
 ifeq (,$(filter Windows%, $(OS)))
-ifeq ($(BACKTRACE), 1)
+ifeq ($(ALL_SYMBOLS), 1)
-DEBUGFLAGS += -DBACKTRACE_ENABLE=1
+DEBUGFLAGS_LD+=-rdynamic
 DEBUGFLAGS_LD += -rdynamic
 endif
 endif
@@ -171,12 +168,12 @@ endif
 	$(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT) $(LDFLAGS)
 .PHONY: zstd-release
-zstd-release: DEBUGFLAGS := -DBACKTRACE_ENABLE=0
+zstd-release: DEBUGFLAGS :=
 zstd-release: DEBUGFLAGS_LD :=
 zstd-release: zstd
 zstd32 : CPPFLAGS += $(THREAD_CPP)
-zstd32 : LDFLAGS  += $(THREAD_LD)
+zstd32 : LDFLAGS += $(THREAD_LD)
 zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
 zstd32 : $(ZSTDLIB_FILES) zstdcli.c fileio.c bench.c datagen.c dibio.c
 ifneq (,$(filter Windows%,$(OS)))
@@ -188,17 +185,17 @@ zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) zstdcli.o fileio.c bench.o datagen.
 	$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
 zstd-nomt : THREAD_CPP :=
-zstd-nomt : THREAD_LD  :=
+zstd-nomt : THREAD_LD :=
 zstd-nomt : THREAD_MSG := - multi-threading disabled
 zstd-nomt : zstd
 zstd-nogz : ZLIBCPP :=
-zstd-nogz : ZLIBLD  :=
+zstd-nogz : ZLIBLD :=
 zstd-nogz : ZLIB_MSG := - gzip support is disabled
 zstd-nogz : zstd
 zstd-noxz : LZMACPP :=
-zstd-noxz : LZMALD  :=
+zstd-noxz : LZMALD :=
 zstd-noxz : LZMA_MSG := - xz/lzma support is disabled
 zstd-noxz : zstd
@@ -249,26 +246,16 @@ MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="zstd
 zstd.1: zstd.1.md ../lib/zstd.h
 	cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
 zstdgrep.1: zstdgrep.1.md ../lib/zstd.h
 	cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
 zstdless.1: zstdless.1.md ../lib/zstd.h
 	cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
 .PHONY: man
-man: zstd.1 zstdgrep.1 zstdless.1
+man: zstd.1
 .PHONY: clean-man
 clean-man:
 	rm zstd.1
 	rm zstdgrep.1
 	rm zstdless.1
 .PHONY: preview-man
 preview-man: clean-man man
 	man ./zstd.1
 	man ./zstdgrep.1
 	man ./zstdless.1
 #-----------------------------------------------------------------------------
 # make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
@@ -342,8 +329,6 @@ install: zstd
 	@$(INSTALL_MAN) zstd.1 $(DESTDIR)$(MAN1DIR)/zstd.1
 	@ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/zstdcat.1
 	@ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/unzstd.1
 	@$(INSTALL_MAN) zstdgrep.1 $(DESTDIR)$(MAN1DIR)/zstdgrep.1
 	@$(INSTALL_MAN) zstdless.1 $(DESTDIR)$(MAN1DIR)/zstdless.1
 	@echo zstd installation completed
 .PHONY: uninstall
@@ -353,8 +338,6 @@ uninstall:
 	@$(RM) $(DESTDIR)$(BINDIR)/zstdcat
 	@$(RM) $(DESTDIR)$(BINDIR)/unzstd
 	@$(RM) $(DESTDIR)$(BINDIR)/zstd
 	@$(RM) $(DESTDIR)$(MAN1DIR)/zstdless.1
 	@$(RM) $(DESTDIR)$(MAN1DIR)/zstdgrep.1
 	@$(RM) $(DESTDIR)$(MAN1DIR)/zstdcat.1
 	@$(RM) $(DESTDIR)$(MAN1DIR)/unzstd.1
 	@$(RM) $(DESTDIR)$(MAN1DIR)/zstd.1
@@ -61,12 +61,12 @@ There are however other Makefile targets that create different variations of CLI
  In which case, linking stage will fail if `lz4` library cannot be found.
  This is useful to prevent silent feature disabling.
- __BACKTRACE__ : `zstd` can display a stack backtrace when execution
+- __ALL_SYMBOLS__ : `zstd` can display a stack backtrace if the execution
  generates a runtime exception. By default, this feature may be
  degraded/disabled on some platforms unless additional compiler directives are
-  applied. When triaging a runtime issue, enabling this feature can provide
+  applied. When triaging a runtime issue, enabling this feature can provided
  more context to determine the location of the fault.
-  Example : `make zstd BACKTRACE=1`
+  Example : `make zstd ALL_SYMBOLS=1`
 #### Aggregation of parameters
@@ -20,6 +20,12 @@
 #  define _POSIX_SOURCE 1          /* disable %llu warnings with MinGW on Windows */
 #endif
 #if !defined(BACKTRACES_ENABLE) && \
   (defined(__linux__) || (defined(__APPLE__) && defined(__MACH__)) )
 #  define BACKTRACES_ENABLE 1
 #endif
 /*-*************************************
 *  Includes
 ***************************************/
@@ -31,6 +37,9 @@
 #include <assert.h>
 #include <errno.h>      /* errno */
 #include <signal.h>
 #ifdef BACKTRACES_ENABLE
 #  include <execinfo.h>   /* backtrace, backtrace_symbols */
 #endif
 #if defined (_MSC_VER)
 #  include <sys/stat.h>
@@ -158,30 +167,7 @@ static void clearHandler(void)
 /*-*********************************************************
 *  Termination signal trapping (Print debug stack trace)
 ***********************************************************/
-#if defined(__has_feature) && !defined(BACKTRACE_ENABLE) /* Clang compiler */
+#ifdef BACKTRACES_ENABLE
 #  if (__has_feature(address_sanitizer))
 #    define BACKTRACE_ENABLE 0
 #  endif /* __has_feature(address_sanitizer) */
 #elif defined(__SANITIZE_ADDRESS__) && !defined(BACKTRACE_ENABLE) /* GCC compiler */
 #  define BACKTRACE_ENABLE 0
 #endif
 #if !defined(BACKTRACE_ENABLE)
 /* automatic detector : backtrace enabled by default on linux+glibc and osx */
 #  if (defined(__linux__) && defined(__GLIBC__)) \
     || (defined(__APPLE__) && defined(__MACH__))
 #    define BACKTRACE_ENABLE 1
 #  else
 #    define BACKTRACE_ENABLE 0
 #  endif
 #endif
 /* note : after this point, BACKTRACE_ENABLE is necessarily defined */
 #if BACKTRACE_ENABLE
 #include <execinfo.h>   /* backtrace, backtrace_symbols */
 #define MAX_STACK_FRAMES    50
@@ -222,7 +208,7 @@ static void ABRThandler(int sig) {
 void FIO_addAbortHandler()
 {
-#if BACKTRACE_ENABLE
+#ifdef BACKTRACES_ENABLE
    signal(SIGABRT, ABRThandler);
    signal(SIGFPE, ABRThandler);
    signal(SIGILL, ABRThandler);
@@ -1219,8 +1205,7 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix)
    size_t const sfnSize = strlen(srcFileName);
    size_t const suffixSize = strlen(suffix);
-    if (dfnbCapacity <= sfnSize+suffixSize+1) {
+    if (dfnbCapacity <= sfnSize+suffixSize+1) {  /* resize name buffer */
        /* resize buffer for dstName */
        free(dstFileNameBuffer);
        dfnbCapacity = sfnSize + suffixSize + 30;
        dstFileNameBuffer = (char*)malloc(dfnbCapacity);
@@ -1228,8 +1213,8 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix)
            EXM_THROW(30, "zstd: %s", strerror(errno));
    }   }
    assert(dstFileNameBuffer != NULL);
-    memcpy(dstFileNameBuffer, srcFileName, sfnSize);
+    strncpy(dstFileNameBuffer, srcFileName, sfnSize+1 /* Include null */);
-    memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */);
+    strncat(dstFileNameBuffer, suffix, suffixSize);
    return dstFileNameBuffer;
 }
@@ -1,5 +1,5 @@
 .
-.TH "ZSTD" "1" "October 2018" "zstd 1.3.7" "User Commands"
+.TH "ZSTD" "1" "September 2018" "zstd 1.3.5" "User Commands"
 .
 .SH "NAME"
 \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -123,8 +123,8 @@ Compress using \fB#\fR working threads (default: 1)\. If \fB#\fR is 0, attempt t
 Does not spawn a thread for compression, use a single thread for both I/O and compression\. In this mode, compression is serialized with I/O, which is slightly slower\. (This is different from \fB\-T1\fR, which spawns 1 compression thread in parallel of I/O)\. This mode is the only one available when multithread support is disabled\. Single\-thread mode features lower memory usage\. Final compressed result is slightly different from \fB\-T1\fR\.
 .
 .TP
-\fB\-\-adapt[=min=#,max=#]\fR
+\fB\-\-adapt\fR
-\fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
+\fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
 .
 .TP
 \fB\-D file\fR
@@ -1,23 +0,0 @@
 .
 .TH "ZSTDGREP" "1" "October 2018" "zstd 1.3.7" "User Commands"
 .
 .SH "NAME"
 \fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
 .
 .SH "SYNOPSIS"
 \fBzstdgrep\fR [\fIgrep\-flags\fR] [\-\-] \fIpattern\fR [\fIfiles\fR \.\.\.]
 .
 .SH "DESCRIPTION"
 \fBzstdgrep\fR runs \fBgrep (1)\fR on files or stdin, if no files argument is given, after decompressing them with \fBzstdcat (1)\fR\.
 .
 .P
 The grep\-flags and pattern arguments are passed on to \fBgrep (1)\fR\. If an \fB\-e\fR flag is found in the \fBgrep\-flags\fR, \fBzstdgrep\fR will not look for a pattern argument\.
 .
 .SH "EXIT STATUS"
 In case of missing arguments or missing pattern, 1 will be returned, otherwise 0\.
 .
 .SH "SEE ALSO"
 \fBzstd (1)\fR
 .
 .SH "AUTHORS"
 Thomas Klausner \fIwiz@NetBSD\.org\fR
@@ -1,26 +0,0 @@
 zstdgrep(1) -- print lines matching a pattern in zstandard-compressed files
 ============================================================================
 SYNOPSIS
 --------
 `zstdgrep` [*grep-flags*] [--] _pattern_ [_files_ ...]
 DESCRIPTION
 -----------
 `zstdgrep` runs `grep (1)` on files or stdin, if no files argument is given, after decompressing them with `zstdcat (1)`.
 The grep-flags and pattern arguments are passed on to `grep (1)`.  If an `-e` flag is found in the `grep-flags`, `zstdgrep` will not look for a pattern argument.
 EXIT STATUS
 -----------
 In case of missing arguments or missing pattern, 1 will be returned, otherwise 0.
 SEE ALSO
 --------
 `zstd (1)`
 AUTHORS
 -------
 Thomas Klausner <wiz@NetBSD.org>
@@ -1,14 +0,0 @@
 .
 .TH "ZSTDLESS" "1" "October 2018" "zstd 1.3.7" "User Commands"
 .
 .SH "NAME"
 \fBzstdless\fR \- view zstandard\-compressed files
 .
 .SH "SYNOPSIS"
 \fBzstdless\fR [\fIflags\fR] [\fIfile\fR \.\.\.]
 .
 .SH "DESCRIPTION"
 \fBzstdless\fR runs \fBless (1)\fR on files or stdin, if no files argument is given, after decompressing them with \fBzstdcat (1)\fR\.
 .
 .SH "SEE ALSO"
 \fBzstd (1)\fR
@@ -1,16 +0,0 @@
 zstdless(1) -- view zstandard-compressed files
 ============================================================================
 SYNOPSIS
 --------
 `zstdless` [*flags*] [_file_ ...]
 DESCRIPTION
 -----------
 `zstdless` runs `less (1)` on files or stdin, if no files argument is given, after decompressing them with `zstdcat (1)`.
 SEE ALSO
 --------
 `zstd (1)`
@@ -1,7 +1,6 @@
 # local binary (Makefile)
 fullbench
 fullbench32
 fullbench-lib
 fuzzer
 fuzzer32
 fuzzer-dll
Author	SHA1	Message	Date
Yann ColletandGitHub	4fa456d7f1	Merge pull request #1354 from facebook/dev Zstandard v1.3.6	2018-10-05 09:34:52 -07:00
Yann ColletandGitHub	90ae50224d	Merge pull request #1217 from facebook/dev fix poolTest for v1.3.5	2018-06-28 01:55:26 -07:00
Yann ColletandGitHub	d1745245e0	Merge pull request #1215 from facebook/dev updating doc/images directory	2018-06-27 16:04:58 -07:00
Yann Collet	d947801569	added travis tests for master and cron jobs	2018-06-27 14:24:54 -07:00
Yann Collet	a2263a93ed	Merge 'dev' to become v1.3.5 .travis.yml comes from `dev` a separate commit will add specifics tests for master	2018-06-27 14:16:52 -07:00
Yann ColletandGitHub	255597502c	Merge pull request #1076 from facebook/dev v1.3.4	2018-03-26 15:19:34 -07:00
Yann Collet	11a3e92883	Merge branch 'dev'	2018-03-26 14:19:52 -07:00
Yann ColletandGitHub	f3a8bd553a	Merge pull request #954 from facebook/dev Release v1.3.3	2017-12-21 10:16:40 +01:00
Yann Collet	ed6e36e809	travis yml : added missing ;	2017-10-18 16:11:14 -07:00
Yann Collet	3423a69f7c	removed comment, in case it would help	2017-10-18 12:37:31 -07:00
Yann Collet	1070bbc5f9	reduce cron test time and check fetch duration	2017-10-18 12:35:17 -07:00
Yann Collet	f4340f46b2	make clean before zstd-lz4 compatibility tests to ensure zstd is rebuilt with lz4 support	2017-10-08 02:41:28 -07:00
Yann Collet	d29bf941b9	added cxxtest to master branch C++ compilation will be tested every night	2017-10-08 00:12:17 -07:00
Yann ColletandGitHub	969e6ef44f	Merge pull request #883 from facebook/dev fixed a small error in decodeCorpus	2017-10-08 00:08:12 -07:00
Yann Collet	2b2f3c6f64	Merge branch 'dev' for v1.3.2	2017-10-06 12:13:27 -07:00
Yann Collet	b0a3ad96d3	reduced OS-X tests as they time out regularly	2017-09-26 19:41:11 -07:00
Yann ColletandGitHub	aecf3b479c	Merge pull request #808 from facebook/dev Zstandard v1.3.1	2017-08-20 12:34:18 -07:00
Yann Collet	b72808a49b	changed zstream test time Used to be 3x5mn, resulting in time out issues (make test) Now set to 3x2mn	2017-07-21 11:45:42 -07:00
Yann ColletandGitHub	fe007ba8bd	Merge pull request #750 from facebook/dev fixed wrong assert() condition	2017-07-05 11:05:17 -07:00
Yann ColletandGitHub	7d30795931	Merge pull request #748 from facebook/dev v1.3.0	2017-07-04 15:09:47 -07:00
Yann ColletandGitHub	da0c157fc4	Merge pull request #686 from facebook/dev v1.2.0	2017-05-04 11:18:24 -07:00
Yann ColletandGitHub	39d0c6cde7	Merge pull request #672 from iburinoc/osxlzma Install liblzma on OS X long test	2017-04-21 11:54:05 -07:00
Sean Purcell	02c44a7257	Install liblzma on OS X long test	2017-04-21 10:42:41 -07:00
Yann ColletandGitHub	585bb51413	Merge pull request #617 from iburinoc/longtest-cmake Add cmakebuild to OS X long tests	2017-03-21 14:03:09 -07:00
Sean Purcell	bb8a7601bc	Add cmakebuild to OS X long tests	2017-03-21 13:46:08 -07:00
Yann Collet	ebf275912a	Merge branch 'dev'	2017-03-15 17:31:40 -07:00
Yann Collet	f2ea8932fc	Merge branch 'dev' keep local .travis.yml	2017-03-15 01:02:10 -07:00
Yann ColletandGitHub	2c2ee2b4b0	Merge pull request #599 from iburinoc/ci-master Update .travis.yml for long tests (master branch only)	2017-03-15 00:50:30 -07:00
Sean Purcell	c4f2490050	Update .travis.yml for long tests (master branch only)	2017-03-10 10:38:23 -08:00
Yann ColletandGitHub	cbc5225d38	Merge pull request #531 from facebook/dev Zstandard v1.1.3	2017-02-06 09:17:34 -08:00
Yann ColletandGitHub	817995bb7a	Merge pull request #484 from facebook/dev v1.1.2	2016-12-15 08:57:04 +01:00
Yann ColletandGitHub	99b6d9f1a3	Merge pull request #438 from facebook/dev v1.1.1	2016-11-01 21:05:51 -07:00
Yann ColletandGitHub	b9edd2f9fb	Merge pull request #394 from facebook/dev Zstandard v1.1.0	2016-09-28 05:16:31 +02:00