Merge pull request #1354 from facebook/dev

Zstandard v1.3.6
Merge pull request #1217 from facebook/dev
2018-10-05 09:34:52 -07:00 · 2018-06-28 01:55:26 -07:00 · 2018-06-27 16:04:58 -07:00 · 2018-06-27 14:24:54 -07:00 · 2018-06-27 14:16:52 -07:00 · 2018-03-26 15:19:34 -07:00
19 changed files with 86 additions and 259 deletions
@@ -19,3 +19,6 @@
 # Windows
 *.bat text eol=crlf
 *.cmd text eol=crlf
+
+# .travis.yml merging
+.travis.yml merge=ours
@@ -47,6 +47,14 @@ matrix:
    - if: tag =~ ^v[0-9]\.[0-9]
      env: Cmd='make -C tests checkTag && tests/checkTag $TRAVIS_BRANCH'

+    # additional tests for master branch and cron job
+    - env: Cmd="make test"
+      os: osx
+    - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
+    - env: Cmd='make cxxtest && make clean && make gnu90build && make clean && make cmakebuild && make clean && make travis-install'
+    - env: Cmd='make gpp6install valgrindinstall && make -C zlibWrapper test && make -C zlibWrapper valgrindTest'
+
+
 git:
  depth: 1

@@ -62,38 +62,27 @@ zstdmt:
 zlibwrapper: lib
 	$(MAKE) -C $(ZWRAPDIR) all

-## test: run long-duration tests
 .PHONY: test
 test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror
 test:
 	MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
 	$(MAKE) -C $(TESTDIR) $@

-## shortest: same as `make check`
 .PHONY: shortest
 shortest:
 	$(MAKE) -C $(TESTDIR) $@

-## check: run basic tests for `zstd` cli
 .PHONY: check
 check: shortest

-## examples: build all examples in `/examples` directory
 .PHONY: examples
 examples: lib
 	CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all

-## manual: generate API documentation in html format
 .PHONY: manual
 manual:
 	$(MAKE) -C contrib/gen_html $@

-## man: generate man page
-.PHONY: man
-man:
-	$(MAKE) -C programs $@
-
-## contrib: build all supported projects in `/contrib` directory
 .PHONY: contrib
 contrib: lib
 	$(MAKE) -C contrib/pzstd all
@@ -1,10 +1,3 @@
-v1.3.7
-perf: slightly better decompression speed on clang (depending on hardware target)
-fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10
-build: no longer build backtrace by default in release mode; restrict further automatic mode
-build: control backtrace support through build macro BACKTRACE
-misc: added man pages for zstdless and zstdgrep, by @samrussell
-
 v1.3.6
 perf: much faster dictionary builder, by @jenniferliu
 perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte
@@ -43,12 +43,6 @@ IF (UNIX)
    ADD_CUSTOM_TARGET(zstd.1 ALL
        ${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstd.1 .
        COMMENT "Copying manpage zstd.1")
-    ADD_CUSTOM_TARGET(zstdgrep.1 ALL
-        ${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstdgrep.1 .
-        COMMENT "Copying manpage zstdgrep.1")
-    ADD_CUSTOM_TARGET(zstdless.1 ALL
-        ${CMAKE_COMMAND} -E copy ${PROGRAMS_DIR}/zstdless.1 .
-        COMMENT "Copying manpage zstdless.1")
    ADD_CUSTOM_TARGET(zstdcat.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 zstdcat.1 DEPENDS zstd.1 COMMENT "Creating zstdcat.1 symlink")
    ADD_CUSTOM_TARGET(unzstd.1 ALL ${CMAKE_COMMAND} -E create_symlink zstd.1 unzstd.1 DEPENDS zstd.1 COMMENT "Creating unzstd.1 symlink")

@@ -61,8 +55,6 @@ IF (UNIX)
    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstd.1 DESTINATION "${MAN_INSTALL_DIR}")
    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdcat.1 DESTINATION "${MAN_INSTALL_DIR}")
    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/unzstd.1 DESTINATION "${MAN_INSTALL_DIR}")
-    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdgrep.1 DESTINATION "${MAN_INSTALL_DIR}")
-    INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/zstdless.1 DESTINATION "${MAN_INSTALL_DIR}")

    ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c)
    TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static)
@@ -1,10 +1,10 @@
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
-<title>zstd 1.3.7 Manual</title>
+<title>zstd 1.3.6 Manual</title>
 </head>
 <body>
-<h1>zstd 1.3.7 Manual</h1>
+<h1>zstd 1.3.6 Manual</h1>
 <hr>
 <a name="Contents"></a><h2>Contents</h2>
 <ol>
@@ -313,17 +313,11 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
  The function will update both `pos` fields.
  If `input.pos < input.size`, some input has not been consumed.
  It's up to the caller to present again remaining data.
-  The function tries to flush all data decoded immediately, repecting buffer sizes.
  If `output.pos < output.size`, decoder has flushed everything it could.
-  But if `output.pos == output.size`, there is no such guarantee,
-  it's likely that some decoded data was not flushed and still remains within internal buffers.
-  In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
-  When no additional input is provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
- @return : 0 when a frame is completely decoded and fully flushed,
-        or an error code, which can be tested using ZSTD_isError(),
-        or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
-                                the return value is a suggested next input size (a hint for better latency)
-                                that will never load more than the current frame.
+  @return : 0 when a frame is completely decoded and fully flushed,
+            an error code, which can be tested using ZSTD_isError(),
+            any other value > 0, which means there is still some decoding to do to complete current frame.
+            The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
 
 <BR></pre>

@@ -606,40 +600,22 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
 </pre></b><BR>
 <pre><b>size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 </b><p>  start a new compression job, using same parameters from previous job.
-  This is typically useful to skip dictionary loading stage, since it will re-use it in-place.
+  This is typically useful to skip dictionary loading stage, since it will re-use it in-place..
  Note that zcs must be init at least once before using ZSTD_resetCStream().
  If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN.
  If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end.
  For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs,
  but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead.
- @return : 0, or an error code (which can be tested using ZSTD_isError())
- 
+ @return : 0, or an error code (which can be tested using ZSTD_isError()) 
 </p></pre><BR>

 <pre><b>typedef struct {
-    unsigned long long ingested;   </b>/* nb input bytes read and buffered */<b>
-    unsigned long long consumed;   </b>/* nb input bytes actually compressed */<b>
-    unsigned long long produced;   </b>/* nb of compressed bytes generated and buffered */<b>
-    unsigned long long flushed;    </b>/* nb of compressed bytes flushed : not provided; can be tracked from caller side */<b>
-    unsigned currentJobID;         </b>/* MT only : latest started job nb */<b>
-    unsigned nbActiveWorkers;      </b>/* MT only : nb of workers actively compressing at probe time */<b>
+    unsigned long long ingested;
+    unsigned long long consumed;
+    unsigned long long produced;
+    unsigned currentJobID;
 } ZSTD_frameProgression;
 </b></pre><BR>
-<pre><b>size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
-</b><p>  Tell how many bytes are ready to be flushed immediately.
-  Useful for multithreading scenarios (nbWorkers >= 1).
-  Probe the oldest active job, defined as oldest job not yet entirely flushed,
-  and check its output buffer.
- @return : amount of data stored in oldest job and ready to be flushed immediately.
-  if @return == 0, it means either :
-  + there is no active job (could be checked with ZSTD_frameProgression()), or
-  + oldest job is still actively compressing data,
-    but everything it has produced has also been flushed so far,
-    therefore flushing speed is currently limited by production speed of oldest job
-    irrespective of the speed of concurrent newer jobs.
- 
-</p></pre><BR>
-
 <h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
 size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);   </b>/* obsolete : this API will be removed in a future version */<b>
 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); </b>/**< note: no dictionary will be used if dict == NULL or dictSize < 8 */<b>
@@ -1039,13 +1015,9 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx,
 </p></pre><BR>

 <pre><b>typedef enum {
-    ZSTD_e_continue=0, </b>/* collect more data, encoder decides when to output compressed result, for optimal compression ratio */<b>
-    ZSTD_e_flush,      </b>/* flush any data provided so far,<b>
-                        * it creates (at least) one new block, that can be decoded immediately on reception;
-                        * frame will continue: any future data can still reference previously compressed data, improving compression. */
-    ZSTD_e_end         </b>/* flush any remaining data and close current frame.<b>
-                        * any additional data starts a new frame.
-                        * each frame is independent (does not reference any content from previous frame). */
+    ZSTD_e_continue=0, </b>/* collect more data, encoder decides when to output compressed result, for optimal conditions */<b>
+    ZSTD_e_flush,      </b>/* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */<b>
+    ZSTD_e_end         </b>/* flush any remaining data and close current frame. Any additional data starts a new frame. */<b>
 } ZSTD_EndDirective;
 </b></pre><BR>
 <pre><b>size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
@@ -339,10 +339,17 @@ MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)

 MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
 {
-    U32 const regMask = sizeof(bitContainer)*8 - 1;
-    /* if start > regMask, bitstream is corrupted, and result is undefined */
+#if defined(__BMI__) && defined(__GNUC__) && __GNUC__*1000+__GNUC_MINOR__ >= 4008  /* experimental */
+#  if defined(__x86_64__)
+    if (sizeof(bitContainer)==8)
+        return _bextr_u64(bitContainer, start, nbBits);
+    else
+#  endif
+        return _bextr_u32(bitContainer, start, nbBits);
+#else
    assert(nbBits < BIT_MASK_SIZE);
-    return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
+    return (bitContainer >> start) & BIT_mask[nbBits];
+#endif
 }

 MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
@@ -359,13 +366,9 @@ MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
 * @return : value extracted */
 MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
 {
-    /* arbitrate between double-shift and shift+mask */
-#if 1
-    /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8,
-     * bitstream is likely corrupted, and result is undefined */
+#if defined(__BMI__) && defined(__GNUC__)   /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
    return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
 #else
-    /* this code path is slower on my os-x laptop */
    U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
    return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask);
 #endif
@@ -39,10 +39,6 @@ extern "C" {
 #  define MEM_STATIC static  /* this version may generate warnings for unused static functions; disable the relevant warning */
 #endif

-#ifndef __has_builtin
-#  define __has_builtin(x) 0  /* compat. with non-clang compilers */
-#endif
-
 /* code only tested on 32 and 64 bits systems */
 #define MEM_STATIC_ASSERT(c)   { enum { MEM_static_assert = 1/(int)(!!(c)) }; }
 MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
@@ -202,8 +198,7 @@ MEM_STATIC U32 MEM_swap32(U32 in)
 {
 #if defined(_MSC_VER)     /* Visual Studio */
    return _byteswap_ulong(in);
-#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
-  || (defined(__clang__) && __has_builtin(__builtin_bswap32))
+#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
    return __builtin_bswap32(in);
 #else
    return  ((in << 24) & 0xff000000 ) |
@@ -217,8 +212,7 @@ MEM_STATIC U64 MEM_swap64(U64 in)
 {
 #if defined(_MSC_VER)     /* Visual Studio */
    return _byteswap_uint64(in);
-#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
-  || (defined(__clang__) && __has_builtin(__builtin_bswap64))
+#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
    return __builtin_bswap64(in);
 #else
    return  ((in << 56) & 0xff00000000000000ULL) |
@@ -147,7 +147,6 @@ ZSTD_DUBT_findBetterDictMatch (
        ZSTD_matchState_t* ms,
        const BYTE* const ip, const BYTE* const iend,
        size_t* offsetPtr,
-        size_t bestLength,
        U32 nbCompares,
        U32 const mls,
        const ZSTD_dictMode_e dictMode)
@@ -173,7 +172,8 @@ ZSTD_DUBT_findBetterDictMatch (
    U32         const btMask = (1 << btLog) - 1;
    U32         const btLow = (btMask >= dictHighLimit - dictLowLimit) ? dictLowLimit : dictHighLimit - btMask;

-    size_t commonLengthSmaller=0, commonLengthLarger=0;
+    size_t commonLengthSmaller=0, commonLengthLarger=0, bestLength=0;
+    U32 matchEndIdx = current+8+1;

    (void)dictMode;
    assert(dictMode == ZSTD_dictMatchState);
@@ -188,8 +188,10 @@ ZSTD_DUBT_findBetterDictMatch (

        if (matchLength > bestLength) {
            U32 matchIndex = dictMatchIndex + dictIndexDelta;
+            if (matchLength > matchEndIdx - matchIndex)
+                matchEndIdx = matchIndex + (U32)matchLength;
            if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) {
-                DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)",
+                DEBUGLOG(2, "ZSTD_DUBT_findBestDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)",
                    current, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, ZSTD_REP_MOVE + current - matchIndex, dictMatchIndex, matchIndex);
                bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex;
            }
@@ -198,6 +200,7 @@ ZSTD_DUBT_findBetterDictMatch (
            }
        }

+        DEBUGLOG(2, "matchLength:%6zu, match:%p, prefixStart:%p, ip:%p", matchLength, match, prefixStart, ip);
        if (match[matchLength] < ip[matchLength]) {
            if (dictMatchIndex <= btLow) { break; }   /* beyond tree size, stop the search */
            commonLengthSmaller = matchLength;    /* all smaller will now have at least this guaranteed common length */
@@ -212,7 +215,7 @@ ZSTD_DUBT_findBetterDictMatch (

    if (bestLength >= MINMATCH) {
        U32 const mIndex = current - ((U32)*offsetPtr - ZSTD_REP_MOVE); (void)mIndex;
-        DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
+        DEBUGLOG(2, "ZSTD_DUBT_findBestDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
                    current, (U32)bestLength, (U32)*offsetPtr, mIndex);
    }
    return bestLength;
@@ -320,11 +323,6 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
                if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) )
                    bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex;
                if (ip+matchLength == iend) {   /* equal : no way to know if inf or sup */
-                    if (dictMode == ZSTD_dictMatchState) {
-                        nbCompares = 0; /* in addition to avoiding checking any
-                                         * further in this loop, make sure we
-                                         * skip checking in the dictionary. */
-                    }
                    break;   /* drop, to guarantee consistency (miss a little bit of compression) */
                }
            }
@@ -348,10 +346,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
        *smallerPtr = *largerPtr = 0;

        if (dictMode == ZSTD_dictMatchState && nbCompares) {
-            bestLength = ZSTD_DUBT_findBetterDictMatch(
-                    ms, ip, iend,
-                    offsetPtr, bestLength, nbCompares,
-                    mls, dictMode);
+            bestLength = ZSTD_DUBT_findBetterDictMatch(ms, ip, iend, offsetPtr, nbCompares, mls, dictMode);
        }

        assert(matchEndIdx > current+8); /* ensure nextToUpdate is increased */
@@ -71,7 +71,7 @@ extern "C" {
 /*------   Version   ------*/
 #define ZSTD_VERSION_MAJOR    1
 #define ZSTD_VERSION_MINOR    3
-#define ZSTD_VERSION_RELEASE  7
+#define ZSTD_VERSION_RELEASE  6

 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 ZSTDLIB_API unsigned ZSTD_versionNumber(void);   /**< useful to check dll version */
@@ -361,21 +361,15 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output
 *  The function will update both `pos` fields.
 *  If `input.pos < input.size`, some input has not been consumed.
 *  It's up to the caller to present again remaining data.
-*  The function tries to flush all data decoded immediately, repecting buffer sizes.
 *  If `output.pos < output.size`, decoder has flushed everything it could.
-*  But if `output.pos == output.size`, there is no such guarantee,
-*  it's likely that some decoded data was not flushed and still remains within internal buffers.
-*  In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
-*  When no additional input is provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
-* @return : 0 when a frame is completely decoded and fully flushed,
-*        or an error code, which can be tested using ZSTD_isError(),
-*        or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
-*                                the return value is a suggested next input size (a hint for better latency)
-*                                that will never load more than the current frame.
+*  @return : 0 when a frame is completely decoded and fully flushed,
+*            an error code, which can be tested using ZSTD_isError(),
+*            any other value > 0, which means there is still some decoding to do to complete current frame.
+*            The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
 * *******************************************************************************/

 typedef ZSTD_DCtx ZSTD_DStream;  /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
-                                 /* For compatibility with versions <= v1.2.0, prefer differentiating them. */
+                                 /* For compatibility with versions <= v1.2.0, continue to consider them separated. */
 /*===== ZSTD_DStream management functions =====*/
 ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
 ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
@@ -1241,13 +1235,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_resetParameters(ZSTD_CCtx* cctx);


 typedef enum {
-    ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
-    ZSTD_e_flush,      /* flush any data provided so far,
-                        * it creates (at least) one new block, that can be decoded immediately on reception;
-                        * frame will continue: any future data can still reference previously compressed data, improving compression. */
-    ZSTD_e_end         /* flush any remaining data and close current frame.
-                        * any additional data starts a new frame.
-                        * each frame is independent (does not reference any content from previous frame). */
+    ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal conditions */
+    ZSTD_e_flush,      /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
+    ZSTD_e_end         /* flush any remaining data and close current frame. Any additional data starts a new frame. */
 } ZSTD_EndDirective;

 /*! ZSTD_compress_generic() :
@@ -134,14 +134,11 @@ else
 LZ4_MSG := $(NO_LZ4_MSG)
 endif

-# explicit backtrace enable/disable for Linux & Darwin
-ifeq ($(BACKTRACE), 0)
-DEBUGFLAGS += -DBACKTRACE_ENABLE=0
-endif
+# enable backtrace symbol names for Linux/Darwin
+ALL_SYMBOLS := 0
 ifeq (,$(filter Windows%, $(OS)))
-ifeq ($(BACKTRACE), 1)
-DEBUGFLAGS += -DBACKTRACE_ENABLE=1
-DEBUGFLAGS_LD += -rdynamic
+ifeq ($(ALL_SYMBOLS), 1)
+DEBUGFLAGS_LD+=-rdynamic
 endif
 endif

@@ -171,12 +168,12 @@ endif
 	$(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT) $(LDFLAGS)

 .PHONY: zstd-release
-zstd-release: DEBUGFLAGS := -DBACKTRACE_ENABLE=0
+zstd-release: DEBUGFLAGS :=
 zstd-release: DEBUGFLAGS_LD :=
 zstd-release: zstd

 zstd32 : CPPFLAGS += $(THREAD_CPP)
-zstd32 : LDFLAGS  += $(THREAD_LD)
+zstd32 : LDFLAGS += $(THREAD_LD)
 zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
 zstd32 : $(ZSTDLIB_FILES) zstdcli.c fileio.c bench.c datagen.c dibio.c
 ifneq (,$(filter Windows%,$(OS)))
@@ -188,17 +185,17 @@ zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) zstdcli.o fileio.c bench.o datagen.
 	$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)

 zstd-nomt : THREAD_CPP :=
-zstd-nomt : THREAD_LD  :=
+zstd-nomt : THREAD_LD :=
 zstd-nomt : THREAD_MSG := - multi-threading disabled
 zstd-nomt : zstd

 zstd-nogz : ZLIBCPP :=
-zstd-nogz : ZLIBLD  :=
+zstd-nogz : ZLIBLD :=
 zstd-nogz : ZLIB_MSG := - gzip support is disabled
 zstd-nogz : zstd

 zstd-noxz : LZMACPP :=
-zstd-noxz : LZMALD  :=
+zstd-noxz : LZMALD :=
 zstd-noxz : LZMA_MSG := - xz/lzma support is disabled
 zstd-noxz : zstd

@@ -249,26 +246,16 @@ MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="zstd
 zstd.1: zstd.1.md ../lib/zstd.h
 	cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@

-zstdgrep.1: zstdgrep.1.md ../lib/zstd.h
-	cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
-
-zstdless.1: zstdless.1.md ../lib/zstd.h
-	cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
-
 .PHONY: man
-man: zstd.1 zstdgrep.1 zstdless.1
+man: zstd.1

 .PHONY: clean-man
 clean-man:
 	rm zstd.1
-	rm zstdgrep.1
-	rm zstdless.1

 .PHONY: preview-man
 preview-man: clean-man man
 	man ./zstd.1
-	man ./zstdgrep.1
-	man ./zstdless.1

 #-----------------------------------------------------------------------------
 # make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
@@ -342,8 +329,6 @@ install: zstd
 	@$(INSTALL_MAN) zstd.1 $(DESTDIR)$(MAN1DIR)/zstd.1
 	@ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/zstdcat.1
 	@ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/unzstd.1
-	@$(INSTALL_MAN) zstdgrep.1 $(DESTDIR)$(MAN1DIR)/zstdgrep.1
-	@$(INSTALL_MAN) zstdless.1 $(DESTDIR)$(MAN1DIR)/zstdless.1
 	@echo zstd installation completed

 .PHONY: uninstall
@@ -353,8 +338,6 @@ uninstall:
 	@$(RM) $(DESTDIR)$(BINDIR)/zstdcat
 	@$(RM) $(DESTDIR)$(BINDIR)/unzstd
 	@$(RM) $(DESTDIR)$(BINDIR)/zstd
-	@$(RM) $(DESTDIR)$(MAN1DIR)/zstdless.1
-	@$(RM) $(DESTDIR)$(MAN1DIR)/zstdgrep.1
 	@$(RM) $(DESTDIR)$(MAN1DIR)/zstdcat.1
 	@$(RM) $(DESTDIR)$(MAN1DIR)/unzstd.1
 	@$(RM) $(DESTDIR)$(MAN1DIR)/zstd.1
@@ -61,12 +61,12 @@ There are however other Makefile targets that create different variations of CLI
  In which case, linking stage will fail if `lz4` library cannot be found.
  This is useful to prevent silent feature disabling.

- __BACKTRACE__ : `zstd` can display a stack backtrace when execution
+- __ALL_SYMBOLS__ : `zstd` can display a stack backtrace if the execution
  generates a runtime exception. By default, this feature may be
  degraded/disabled on some platforms unless additional compiler directives are
-  applied. When triaging a runtime issue, enabling this feature can provide
+  applied. When triaging a runtime issue, enabling this feature can provided
  more context to determine the location of the fault.
-  Example : `make zstd BACKTRACE=1`
+  Example : `make zstd ALL_SYMBOLS=1`


 #### Aggregation of parameters
@@ -20,6 +20,12 @@
 #  define _POSIX_SOURCE 1          /* disable %llu warnings with MinGW on Windows */
 #endif

+#if !defined(BACKTRACES_ENABLE) && \
+   (defined(__linux__) || (defined(__APPLE__) && defined(__MACH__)) )
+#  define BACKTRACES_ENABLE 1
+#endif
+
+
 /*-*************************************
 *  Includes
 ***************************************/
@@ -31,6 +37,9 @@
 #include <assert.h>
 #include <errno.h>      /* errno */
 #include <signal.h>
+#ifdef BACKTRACES_ENABLE
+#  include <execinfo.h>   /* backtrace, backtrace_symbols */
+#endif

 #if defined (_MSC_VER)
 #  include <sys/stat.h>
@@ -158,30 +167,7 @@ static void clearHandler(void)
 /*-*********************************************************
 *  Termination signal trapping (Print debug stack trace)
 ***********************************************************/
-#if defined(__has_feature) && !defined(BACKTRACE_ENABLE) /* Clang compiler */
-#  if (__has_feature(address_sanitizer))
-#    define BACKTRACE_ENABLE 0
-#  endif /* __has_feature(address_sanitizer) */
-#elif defined(__SANITIZE_ADDRESS__) && !defined(BACKTRACE_ENABLE) /* GCC compiler */
-#  define BACKTRACE_ENABLE 0
-#endif
-
-#if !defined(BACKTRACE_ENABLE)
-/* automatic detector : backtrace enabled by default on linux+glibc and osx */
-#  if (defined(__linux__) && defined(__GLIBC__)) \
-     || (defined(__APPLE__) && defined(__MACH__))
-#    define BACKTRACE_ENABLE 1
-#  else
-#    define BACKTRACE_ENABLE 0
-#  endif
-#endif
-
-/* note : after this point, BACKTRACE_ENABLE is necessarily defined */
-
-
-#if BACKTRACE_ENABLE
-
-#include <execinfo.h>   /* backtrace, backtrace_symbols */
+#ifdef BACKTRACES_ENABLE

 #define MAX_STACK_FRAMES    50

@@ -222,7 +208,7 @@ static void ABRThandler(int sig) {

 void FIO_addAbortHandler()
 {
-#if BACKTRACE_ENABLE
+#ifdef BACKTRACES_ENABLE
    signal(SIGABRT, ABRThandler);
    signal(SIGFPE, ABRThandler);
    signal(SIGILL, ABRThandler);
@@ -1219,8 +1205,7 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix)
    size_t const sfnSize = strlen(srcFileName);
    size_t const suffixSize = strlen(suffix);

-    if (dfnbCapacity <= sfnSize+suffixSize+1) {
-        /* resize buffer for dstName */
+    if (dfnbCapacity <= sfnSize+suffixSize+1) {  /* resize name buffer */
        free(dstFileNameBuffer);
        dfnbCapacity = sfnSize + suffixSize + 30;
        dstFileNameBuffer = (char*)malloc(dfnbCapacity);
@@ -1228,8 +1213,8 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix)
            EXM_THROW(30, "zstd: %s", strerror(errno));
    }   }
    assert(dstFileNameBuffer != NULL);
-    memcpy(dstFileNameBuffer, srcFileName, sfnSize);
-    memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */);
+    strncpy(dstFileNameBuffer, srcFileName, sfnSize+1 /* Include null */);
+    strncat(dstFileNameBuffer, suffix, suffixSize);

    return dstFileNameBuffer;
 }
@@ -1,5 +1,5 @@
 .
-.TH "ZSTD" "1" "October 2018" "zstd 1.3.7" "User Commands"
+.TH "ZSTD" "1" "September 2018" "zstd 1.3.5" "User Commands"
 .
 .SH "NAME"
 \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -123,8 +123,8 @@ Compress using \fB#\fR working threads (default: 1)\. If \fB#\fR is 0, attempt t
 Does not spawn a thread for compression, use a single thread for both I/O and compression\. In this mode, compression is serialized with I/O, which is slightly slower\. (This is different from \fB\-T1\fR, which spawns 1 compression thread in parallel of I/O)\. This mode is the only one available when multithread support is disabled\. Single\-thread mode features lower memory usage\. Final compressed result is slightly different from \fB\-T1\fR\.
 .
 .TP
-\fB\-\-adapt[=min=#,max=#]\fR
-\fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
+\fB\-\-adapt\fR
+\fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
 .
 .TP
 \fB\-D file\fR
@@ -1,23 +0,0 @@
-.
-.TH "ZSTDGREP" "1" "October 2018" "zstd 1.3.7" "User Commands"
-.
-.SH "NAME"
-\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
-.
-.SH "SYNOPSIS"
-\fBzstdgrep\fR [\fIgrep\-flags\fR] [\-\-] \fIpattern\fR [\fIfiles\fR \.\.\.]
-.
-.SH "DESCRIPTION"
-\fBzstdgrep\fR runs \fBgrep (1)\fR on files or stdin, if no files argument is given, after decompressing them with \fBzstdcat (1)\fR\.
-.
-.P
-The grep\-flags and pattern arguments are passed on to \fBgrep (1)\fR\. If an \fB\-e\fR flag is found in the \fBgrep\-flags\fR, \fBzstdgrep\fR will not look for a pattern argument\.
-.
-.SH "EXIT STATUS"
-In case of missing arguments or missing pattern, 1 will be returned, otherwise 0\.
-.
-.SH "SEE ALSO"
-\fBzstd (1)\fR
-.
-.SH "AUTHORS"
-Thomas Klausner \fIwiz@NetBSD\.org\fR
@@ -1,26 +0,0 @@
-zstdgrep(1) -- print lines matching a pattern in zstandard-compressed files
-============================================================================
-
-SYNOPSIS
--------
-
-`zstdgrep` [*grep-flags*] [--] _pattern_ [_files_ ...]
-
-
-DESCRIPTION
-----------
-`zstdgrep` runs `grep (1)` on files or stdin, if no files argument is given, after decompressing them with `zstdcat (1)`.
-
-The grep-flags and pattern arguments are passed on to `grep (1)`.  If an `-e` flag is found in the `grep-flags`, `zstdgrep` will not look for a pattern argument.
-
-EXIT STATUS
-----------
-In case of missing arguments or missing pattern, 1 will be returned, otherwise 0.
-
-SEE ALSO
--------
-`zstd (1)`
-
-AUTHORS
-------
-Thomas Klausner <wiz@NetBSD.org>
@@ -1,14 +0,0 @@
-.
-.TH "ZSTDLESS" "1" "October 2018" "zstd 1.3.7" "User Commands"
-.
-.SH "NAME"
-\fBzstdless\fR \- view zstandard\-compressed files
-.
-.SH "SYNOPSIS"
-\fBzstdless\fR [\fIflags\fR] [\fIfile\fR \.\.\.]
-.
-.SH "DESCRIPTION"
-\fBzstdless\fR runs \fBless (1)\fR on files or stdin, if no files argument is given, after decompressing them with \fBzstdcat (1)\fR\.
-.
-.SH "SEE ALSO"
-\fBzstd (1)\fR
@@ -1,16 +0,0 @@
-zstdless(1) -- view zstandard-compressed files
-============================================================================
-
-SYNOPSIS
--------
-
-`zstdless` [*flags*] [_file_ ...]
-
-
-DESCRIPTION
-----------
-`zstdless` runs `less (1)` on files or stdin, if no files argument is given, after decompressing them with `zstdcat (1)`.
-
-SEE ALSO
--------
-`zstd (1)`
@@ -1,7 +1,6 @@
 # local binary (Makefile)
 fullbench
 fullbench32
-fullbench-lib
 fuzzer
 fuzzer32
 fuzzer-dll
Author	SHA1	Message	Date
Yann ColletandGitHub	4fa456d7f1	Merge pull request #1354 from facebook/dev Zstandard v1.3.6	2018-10-05 09:34:52 -07:00
Yann ColletandGitHub	90ae50224d	Merge pull request #1217 from facebook/dev fix poolTest for v1.3.5	2018-06-28 01:55:26 -07:00
Yann ColletandGitHub	d1745245e0	Merge pull request #1215 from facebook/dev updating doc/images directory	2018-06-27 16:04:58 -07:00
Yann Collet	d947801569	added travis tests for master and cron jobs	2018-06-27 14:24:54 -07:00
Yann Collet	a2263a93ed	Merge 'dev' to become v1.3.5 .travis.yml comes from `dev` a separate commit will add specifics tests for master	2018-06-27 14:16:52 -07:00
Yann ColletandGitHub	255597502c	Merge pull request #1076 from facebook/dev v1.3.4	2018-03-26 15:19:34 -07:00
Yann Collet	11a3e92883	Merge branch 'dev'	2018-03-26 14:19:52 -07:00
Yann ColletandGitHub	f3a8bd553a	Merge pull request #954 from facebook/dev Release v1.3.3	2017-12-21 10:16:40 +01:00
Yann Collet	ed6e36e809	travis yml : added missing ;	2017-10-18 16:11:14 -07:00
Yann Collet	3423a69f7c	removed comment, in case it would help	2017-10-18 12:37:31 -07:00
Yann Collet	1070bbc5f9	reduce cron test time and check fetch duration	2017-10-18 12:35:17 -07:00
Yann Collet	f4340f46b2	make clean before zstd-lz4 compatibility tests to ensure zstd is rebuilt with lz4 support	2017-10-08 02:41:28 -07:00
Yann Collet	d29bf941b9	added cxxtest to master branch C++ compilation will be tested every night	2017-10-08 00:12:17 -07:00
Yann ColletandGitHub	969e6ef44f	Merge pull request #883 from facebook/dev fixed a small error in decodeCorpus	2017-10-08 00:08:12 -07:00
Yann Collet	2b2f3c6f64	Merge branch 'dev' for v1.3.2	2017-10-06 12:13:27 -07:00
Yann Collet	b0a3ad96d3	reduced OS-X tests as they time out regularly	2017-09-26 19:41:11 -07:00
Yann ColletandGitHub	aecf3b479c	Merge pull request #808 from facebook/dev Zstandard v1.3.1	2017-08-20 12:34:18 -07:00
Yann Collet	b72808a49b	changed zstream test time Used to be 3x5mn, resulting in time out issues (make test) Now set to 3x2mn	2017-07-21 11:45:42 -07:00
Yann ColletandGitHub	fe007ba8bd	Merge pull request #750 from facebook/dev fixed wrong assert() condition	2017-07-05 11:05:17 -07:00
Yann ColletandGitHub	7d30795931	Merge pull request #748 from facebook/dev v1.3.0	2017-07-04 15:09:47 -07:00
Yann ColletandGitHub	da0c157fc4	Merge pull request #686 from facebook/dev v1.2.0	2017-05-04 11:18:24 -07:00
Yann ColletandGitHub	39d0c6cde7	Merge pull request #672 from iburinoc/osxlzma Install liblzma on OS X long test	2017-04-21 11:54:05 -07:00
Sean Purcell	02c44a7257	Install liblzma on OS X long test	2017-04-21 10:42:41 -07:00
Yann ColletandGitHub	585bb51413	Merge pull request #617 from iburinoc/longtest-cmake Add cmakebuild to OS X long tests	2017-03-21 14:03:09 -07:00
Sean Purcell	bb8a7601bc	Add cmakebuild to OS X long tests	2017-03-21 13:46:08 -07:00
Yann Collet	ebf275912a	Merge branch 'dev'	2017-03-15 17:31:40 -07:00
Yann Collet	f2ea8932fc	Merge branch 'dev' keep local .travis.yml	2017-03-15 01:02:10 -07:00
Yann ColletandGitHub	2c2ee2b4b0	Merge pull request #599 from iburinoc/ci-master Update .travis.yml for long tests (master branch only)	2017-03-15 00:50:30 -07:00
Sean Purcell	c4f2490050	Update .travis.yml for long tests (master branch only)	2017-03-10 10:38:23 -08:00
Yann ColletandGitHub	cbc5225d38	Merge pull request #531 from facebook/dev Zstandard v1.1.3	2017-02-06 09:17:34 -08:00
Yann ColletandGitHub	817995bb7a	Merge pull request #484 from facebook/dev v1.1.2	2016-12-15 08:57:04 +01:00
Yann ColletandGitHub	99b6d9f1a3	Merge pull request #438 from facebook/dev v1.1.1	2016-11-01 21:05:51 -07:00
Yann ColletandGitHub	b9edd2f9fb	Merge pull request #394 from facebook/dev Zstandard v1.1.0	2016-09-28 05:16:31 +02:00