diff --git a/Makefile b/Makefile index 9ee0b0ba6..c63db80e9 100644 --- a/Makefile +++ b/Makefile @@ -62,27 +62,38 @@ zstdmt: zlibwrapper: lib $(MAKE) -C $(ZWRAPDIR) all +## test: run long-duration tests .PHONY: test test: MOREFLAGS += -g -DDEBUGLEVEL=1 -Werror test: MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants $(MAKE) -C $(TESTDIR) $@ +## shortest: same as `make check` .PHONY: shortest shortest: $(MAKE) -C $(TESTDIR) $@ +## check: run basic tests for `zstd` cli .PHONY: check check: shortest +## examples: build all examples in `/examples` directory .PHONY: examples examples: lib CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all +## manual: generate API documentation in html format .PHONY: manual manual: $(MAKE) -C contrib/gen_html $@ +## man: generate man page +.PHONY: man +man: + $(MAKE) -C programs $@ + +## contrib: build all supported projects in `/contrib` directory .PHONY: contrib contrib: lib $(MAKE) -C contrib/pzstd all diff --git a/NEWS b/NEWS index 40805c187..fa1a22325 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,9 @@ +v1.3.7 +perf: slightly better decompression speed on clang (depending on hardware target) +fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10 +build: no longer build backtrace by default in release mode; restrict further automatic mode +build: control backtrace support through build macro BACKTRACE + v1.3.6 perf: much faster dictionary builder, by @jenniferliu perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 4a8985f26..f9b1daa8a 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -1,10 +1,10 @@ -zstd 1.3.6 Manual +zstd 1.3.7 Manual -

zstd 1.3.6 Manual

+

zstd 1.3.7 Manual


Contents

    @@ -313,11 +313,17 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); The function will update both `pos` fields. If `input.pos < input.size`, some input has not been consumed. It's up to the caller to present again remaining data. + The function tries to flush all data decoded immediately, repecting buffer sizes. If `output.pos < output.size`, decoder has flushed everything it could. - @return : 0 when a frame is completely decoded and fully flushed, - an error code, which can be tested using ZSTD_isError(), - any other value > 0, which means there is still some decoding to do to complete current frame. - The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame. + But if `output.pos == output.size`, there is no such guarantee, + it's likely that some decoded data was not flushed and still remains within internal buffers. + In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. + When no additional input is provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. + @return : 0 when a frame is completely decoded and fully flushed, + or an error code, which can be tested using ZSTD_isError(), + or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : + the return value is a suggested next input size (a hint for better latency) + that will never load more than the current frame.
    @@ -600,22 +606,40 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict*
    size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
     

    start a new compression job, using same parameters from previous job. - This is typically useful to skip dictionary loading stage, since it will re-use it in-place.. + This is typically useful to skip dictionary loading stage, since it will re-use it in-place. Note that zcs must be init at least once before using ZSTD_resetCStream(). If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. - @return : 0, or an error code (which can be tested using ZSTD_isError()) + @return : 0, or an error code (which can be tested using ZSTD_isError()) +


    typedef struct {
    -    unsigned long long ingested;
    -    unsigned long long consumed;
    -    unsigned long long produced;
    -    unsigned currentJobID;
    +    unsigned long long ingested;   /* nb input bytes read and buffered */
    +    unsigned long long consumed;   /* nb input bytes actually compressed */
    +    unsigned long long produced;   /* nb of compressed bytes generated and buffered */
    +    unsigned long long flushed;    /* nb of compressed bytes flushed : not provided; can be tracked from caller side */
    +    unsigned currentJobID;         /* MT only : latest started job nb */
    +    unsigned nbActiveWorkers;      /* MT only : nb of workers actively compressing at probe time */
     } ZSTD_frameProgression;
     

    +
    size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
    +

    Tell how many bytes are ready to be flushed immediately. + Useful for multithreading scenarios (nbWorkers >= 1). + Probe the oldest active job, defined as oldest job not yet entirely flushed, + and check its output buffer. + @return : amount of data stored in oldest job and ready to be flushed immediately. + if @return == 0, it means either : + + there is no active job (could be checked with ZSTD_frameProgression()), or + + oldest job is still actively compressing data, + but everything it has produced has also been flushed so far, + therefore flushing speed is currently limited by production speed of oldest job + irrespective of the speed of concurrent newer jobs. + +


    +

    Advanced Streaming decompression functions

    typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
     size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);   /* obsolete : this API will be removed in a future version */
     size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */
    @@ -1015,9 +1039,13 @@ size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx,
     


    typedef enum {
    -    ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal conditions */
    -    ZSTD_e_flush,      /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
    -    ZSTD_e_end         /* flush any remaining data and close current frame. Any additional data starts a new frame. */
    +    ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
    +    ZSTD_e_flush,      /* flush any data provided so far,
    +                        * it creates (at least) one new block, that can be decoded immediately on reception;
    +                        * frame will continue: any future data can still reference previously compressed data, improving compression. */
    +    ZSTD_e_end         /* flush any remaining data and close current frame.
    +                        * any additional data starts a new frame.
    +                        * each frame is independent (does not reference any content from previous frame). */
     } ZSTD_EndDirective;
     

    size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
    diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h
    index 2f91460c5..ef89b9878 100644
    --- a/lib/common/bitstream.h
    +++ b/lib/common/bitstream.h
    @@ -339,17 +339,10 @@ MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
     
     MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
     {
    -#if defined(__BMI__) && defined(__GNUC__) && __GNUC__*1000+__GNUC_MINOR__ >= 4008  /* experimental */
    -#  if defined(__x86_64__)
    -    if (sizeof(bitContainer)==8)
    -        return _bextr_u64(bitContainer, start, nbBits);
    -    else
    -#  endif
    -        return _bextr_u32(bitContainer, start, nbBits);
    -#else
    +    U32 const regMask = sizeof(bitContainer)*8 - 1;
    +    /* if start > regMask, bitstream is corrupted, and result is undefined */
         assert(nbBits < BIT_MASK_SIZE);
    -    return (bitContainer >> start) & BIT_mask[nbBits];
    -#endif
    +    return (bitContainer >> (start & regMask)) & BIT_mask[nbBits];
     }
     
     MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
    @@ -366,9 +359,13 @@ MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
      * @return : value extracted */
     MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
     {
    -#if defined(__BMI__) && defined(__GNUC__)   /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
    +    /* arbitrate between double-shift and shift+mask */
    +#if 1
    +    /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8,
    +     * bitstream is likely corrupted, and result is undefined */
         return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
     #else
    +    /* this code path is slower on my os-x laptop */
         U32 const regMask = sizeof(bitD->bitContainer)*8 - 1;
         return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask);
     #endif
    diff --git a/lib/common/mem.h b/lib/common/mem.h
    index 2051bcad1..5da248756 100644
    --- a/lib/common/mem.h
    +++ b/lib/common/mem.h
    @@ -39,6 +39,10 @@ extern "C" {
     #  define MEM_STATIC static  /* this version may generate warnings for unused static functions; disable the relevant warning */
     #endif
     
    +#ifndef __has_builtin
    +#  define __has_builtin(x) 0  /* compat. with non-clang compilers */
    +#endif
    +
     /* code only tested on 32 and 64 bits systems */
     #define MEM_STATIC_ASSERT(c)   { enum { MEM_static_assert = 1/(int)(!!(c)) }; }
     MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
    @@ -198,7 +202,8 @@ MEM_STATIC U32 MEM_swap32(U32 in)
     {
     #if defined(_MSC_VER)     /* Visual Studio */
         return _byteswap_ulong(in);
    -#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
    +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
    +  || (defined(__clang__) && __has_builtin(__builtin_bswap32))
         return __builtin_bswap32(in);
     #else
         return  ((in << 24) & 0xff000000 ) |
    @@ -212,7 +217,8 @@ MEM_STATIC U64 MEM_swap64(U64 in)
     {
     #if defined(_MSC_VER)     /* Visual Studio */
         return _byteswap_uint64(in);
    -#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
    +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
    +  || (defined(__clang__) && __has_builtin(__builtin_bswap64))
         return __builtin_bswap64(in);
     #else
         return  ((in << 56) & 0xff00000000000000ULL) |
    diff --git a/lib/compress/zstd_lazy.c b/lib/compress/zstd_lazy.c
    index 4ca69e3ec..af615e077 100644
    --- a/lib/compress/zstd_lazy.c
    +++ b/lib/compress/zstd_lazy.c
    @@ -147,6 +147,7 @@ ZSTD_DUBT_findBetterDictMatch (
             ZSTD_matchState_t* ms,
             const BYTE* const ip, const BYTE* const iend,
             size_t* offsetPtr,
    +        size_t bestLength,
             U32 nbCompares,
             U32 const mls,
             const ZSTD_dictMode_e dictMode)
    @@ -172,8 +173,7 @@ ZSTD_DUBT_findBetterDictMatch (
         U32         const btMask = (1 << btLog) - 1;
         U32         const btLow = (btMask >= dictHighLimit - dictLowLimit) ? dictLowLimit : dictHighLimit - btMask;
     
    -    size_t commonLengthSmaller=0, commonLengthLarger=0, bestLength=0;
    -    U32 matchEndIdx = current+8+1;
    +    size_t commonLengthSmaller=0, commonLengthLarger=0;
     
         (void)dictMode;
         assert(dictMode == ZSTD_dictMatchState);
    @@ -188,10 +188,8 @@ ZSTD_DUBT_findBetterDictMatch (
     
             if (matchLength > bestLength) {
                 U32 matchIndex = dictMatchIndex + dictIndexDelta;
    -            if (matchLength > matchEndIdx - matchIndex)
    -                matchEndIdx = matchIndex + (U32)matchLength;
                 if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) {
    -                DEBUGLOG(2, "ZSTD_DUBT_findBestDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)",
    +                DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)",
                         current, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, ZSTD_REP_MOVE + current - matchIndex, dictMatchIndex, matchIndex);
                     bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex;
                 }
    @@ -200,7 +198,6 @@ ZSTD_DUBT_findBetterDictMatch (
                 }
             }
     
    -        DEBUGLOG(2, "matchLength:%6zu, match:%p, prefixStart:%p, ip:%p", matchLength, match, prefixStart, ip);
             if (match[matchLength] < ip[matchLength]) {
                 if (dictMatchIndex <= btLow) { break; }   /* beyond tree size, stop the search */
                 commonLengthSmaller = matchLength;    /* all smaller will now have at least this guaranteed common length */
    @@ -215,7 +212,7 @@ ZSTD_DUBT_findBetterDictMatch (
     
         if (bestLength >= MINMATCH) {
             U32 const mIndex = current - ((U32)*offsetPtr - ZSTD_REP_MOVE); (void)mIndex;
    -        DEBUGLOG(2, "ZSTD_DUBT_findBestDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
    +        DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
                         current, (U32)bestLength, (U32)*offsetPtr, mIndex);
         }
         return bestLength;
    @@ -323,6 +320,11 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
                     if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) )
                         bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex;
                     if (ip+matchLength == iend) {   /* equal : no way to know if inf or sup */
    +                    if (dictMode == ZSTD_dictMatchState) {
    +                        nbCompares = 0; /* in addition to avoiding checking any
    +                                         * further in this loop, make sure we
    +                                         * skip checking in the dictionary. */
    +                    }
                         break;   /* drop, to guarantee consistency (miss a little bit of compression) */
                     }
                 }
    @@ -346,7 +348,10 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
             *smallerPtr = *largerPtr = 0;
     
             if (dictMode == ZSTD_dictMatchState && nbCompares) {
    -            bestLength = ZSTD_DUBT_findBetterDictMatch(ms, ip, iend, offsetPtr, nbCompares, mls, dictMode);
    +            bestLength = ZSTD_DUBT_findBetterDictMatch(
    +                    ms, ip, iend,
    +                    offsetPtr, bestLength, nbCompares,
    +                    mls, dictMode);
             }
     
             assert(matchEndIdx > current+8); /* ensure nextToUpdate is increased */
    diff --git a/lib/zstd.h b/lib/zstd.h
    index 7b6964be3..f2af4ac8c 100644
    --- a/lib/zstd.h
    +++ b/lib/zstd.h
    @@ -71,7 +71,7 @@ extern "C" {
     /*------   Version   ------*/
     #define ZSTD_VERSION_MAJOR    1
     #define ZSTD_VERSION_MINOR    3
    -#define ZSTD_VERSION_RELEASE  6
    +#define ZSTD_VERSION_RELEASE  7
     
     #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
     ZSTDLIB_API unsigned ZSTD_versionNumber(void);   /**< useful to check dll version */
    @@ -361,15 +361,21 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output
     *  The function will update both `pos` fields.
     *  If `input.pos < input.size`, some input has not been consumed.
     *  It's up to the caller to present again remaining data.
    +*  The function tries to flush all data decoded immediately, repecting buffer sizes.
     *  If `output.pos < output.size`, decoder has flushed everything it could.
    -*  @return : 0 when a frame is completely decoded and fully flushed,
    -*            an error code, which can be tested using ZSTD_isError(),
    -*            any other value > 0, which means there is still some decoding to do to complete current frame.
    -*            The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
    +*  But if `output.pos == output.size`, there is no such guarantee,
    +*  it's likely that some decoded data was not flushed and still remains within internal buffers.
    +*  In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer.
    +*  When no additional input is provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX.
    +* @return : 0 when a frame is completely decoded and fully flushed,
    +*        or an error code, which can be tested using ZSTD_isError(),
    +*        or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
    +*                                the return value is a suggested next input size (a hint for better latency)
    +*                                that will never load more than the current frame.
     * *******************************************************************************/
     
     typedef ZSTD_DCtx ZSTD_DStream;  /**< DCtx and DStream are now effectively same object (>= v1.3.0) */
    -                                 /* For compatibility with versions <= v1.2.0, continue to consider them separated. */
    +                                 /* For compatibility with versions <= v1.2.0, prefer differentiating them. */
     /*===== ZSTD_DStream management functions =====*/
     ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
     ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
    @@ -1235,9 +1241,13 @@ ZSTDLIB_API size_t ZSTD_CCtx_resetParameters(ZSTD_CCtx* cctx);
     
     
     typedef enum {
    -    ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal conditions */
    -    ZSTD_e_flush,      /* flush any data provided so far - frame will continue, future data can still reference previous data for better compression */
    -    ZSTD_e_end         /* flush any remaining data and close current frame. Any additional data starts a new frame. */
    +    ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */
    +    ZSTD_e_flush,      /* flush any data provided so far,
    +                        * it creates (at least) one new block, that can be decoded immediately on reception;
    +                        * frame will continue: any future data can still reference previously compressed data, improving compression. */
    +    ZSTD_e_end         /* flush any remaining data and close current frame.
    +                        * any additional data starts a new frame.
    +                        * each frame is independent (does not reference any content from previous frame). */
     } ZSTD_EndDirective;
     
     /*! ZSTD_compress_generic() :
    diff --git a/programs/Makefile b/programs/Makefile
    index 8faa5111b..ac17caee8 100644
    --- a/programs/Makefile
    +++ b/programs/Makefile
    @@ -134,11 +134,14 @@ else
     LZ4_MSG := $(NO_LZ4_MSG)
     endif
     
    -# enable backtrace symbol names for Linux/Darwin
    -ALL_SYMBOLS := 0
    +# explicit backtrace enable/disable for Linux & Darwin
    +ifeq ($(BACKTRACE), 0)
    +DEBUGFLAGS += -DBACKTRACE_ENABLE=0
    +endif
     ifeq (,$(filter Windows%, $(OS)))
    -ifeq ($(ALL_SYMBOLS), 1)
    -DEBUGFLAGS_LD+=-rdynamic
    +ifeq ($(BACKTRACE), 1)
    +DEBUGFLAGS += -DBACKTRACE_ENABLE=1
    +DEBUGFLAGS_LD += -rdynamic
     endif
     endif
     
    @@ -168,12 +171,12 @@ endif
     	$(CC) $(FLAGS) $^ $(RES_FILE) -o $@$(EXT) $(LDFLAGS)
     
     .PHONY: zstd-release
    -zstd-release: DEBUGFLAGS :=
    +zstd-release: DEBUGFLAGS := -DBACKTRACE_ENABLE=0
     zstd-release: DEBUGFLAGS_LD :=
     zstd-release: zstd
     
     zstd32 : CPPFLAGS += $(THREAD_CPP)
    -zstd32 : LDFLAGS += $(THREAD_LD)
    +zstd32 : LDFLAGS  += $(THREAD_LD)
     zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
     zstd32 : $(ZSTDLIB_FILES) zstdcli.c fileio.c bench.c datagen.c dibio.c
     ifneq (,$(filter Windows%,$(OS)))
    @@ -185,17 +188,17 @@ zstd-nolegacy : $(ZSTD_FILES) $(ZDICT_FILES) zstdcli.o fileio.c bench.o datagen.
     	$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
     
     zstd-nomt : THREAD_CPP :=
    -zstd-nomt : THREAD_LD :=
    +zstd-nomt : THREAD_LD  :=
     zstd-nomt : THREAD_MSG := - multi-threading disabled
     zstd-nomt : zstd
     
     zstd-nogz : ZLIBCPP :=
    -zstd-nogz : ZLIBLD :=
    +zstd-nogz : ZLIBLD  :=
     zstd-nogz : ZLIB_MSG := - gzip support is disabled
     zstd-nogz : zstd
     
     zstd-noxz : LZMACPP :=
    -zstd-noxz : LZMALD :=
    +zstd-noxz : LZMALD  :=
     zstd-noxz : LZMA_MSG := - xz/lzma support is disabled
     zstd-noxz : zstd
     
    diff --git a/programs/README.md b/programs/README.md
    index 804cb8b0a..ca9056eaa 100644
    --- a/programs/README.md
    +++ b/programs/README.md
    @@ -61,12 +61,12 @@ There are however other Makefile targets that create different variations of CLI
       In which case, linking stage will fail if `lz4` library cannot be found.
       This is useful to prevent silent feature disabling.
     
    -- __ALL_SYMBOLS__ : `zstd` can display a stack backtrace if the execution
    +- __BACKTRACE__ : `zstd` can display a stack backtrace when execution
       generates a runtime exception. By default, this feature may be
       degraded/disabled on some platforms unless additional compiler directives are
    -  applied. When triaging a runtime issue, enabling this feature can provided
    +  applied. When triaging a runtime issue, enabling this feature can provide
       more context to determine the location of the fault.
    -  Example : `make zstd ALL_SYMBOLS=1`
    +  Example : `make zstd BACKTRACE=1`
     
     
     #### Aggregation of parameters
    diff --git a/programs/fileio.c b/programs/fileio.c
    index ed3a29cda..c24f4defb 100644
    --- a/programs/fileio.c
    +++ b/programs/fileio.c
    @@ -20,12 +20,6 @@
     #  define _POSIX_SOURCE 1          /* disable %llu warnings with MinGW on Windows */
     #endif
     
    -#if !defined(BACKTRACES_ENABLE) && \
    -   (defined(__linux__) || (defined(__APPLE__) && defined(__MACH__)) )
    -#  define BACKTRACES_ENABLE 1
    -#endif
    -
    -
     /*-*************************************
     *  Includes
     ***************************************/
    @@ -37,9 +31,6 @@
     #include 
     #include       /* errno */
     #include 
    -#ifdef BACKTRACES_ENABLE
    -#  include    /* backtrace, backtrace_symbols */
    -#endif
     
     #if defined (_MSC_VER)
     #  include 
    @@ -167,7 +158,30 @@ static void clearHandler(void)
     /*-*********************************************************
     *  Termination signal trapping (Print debug stack trace)
     ***********************************************************/
    -#ifdef BACKTRACES_ENABLE
    +#if defined(__has_feature) && !defined(BACKTRACE_ENABLE) /* Clang compiler */
    +#  if (__has_feature(address_sanitizer))
    +#    define BACKTRACE_ENABLE 0
    +#  endif /* __has_feature(address_sanitizer) */
    +#elif defined(__SANITIZE_ADDRESS__) && !defined(BACKTRACE_ENABLE) /* GCC compiler */
    +#  define BACKTRACE_ENABLE 0
    +#endif
    +
    +#if !defined(BACKTRACE_ENABLE)
    +/* automatic detector : backtrace enabled by default on linux+glibc and osx */
    +#  if (defined(__linux__) && defined(__GLIBC__)) \
    +     || (defined(__APPLE__) && defined(__MACH__))
    +#    define BACKTRACE_ENABLE 1
    +#  else
    +#    define BACKTRACE_ENABLE 0
    +#  endif
    +#endif
    +
    +/* note : after this point, BACKTRACE_ENABLE is necessarily defined */
    +
    +
    +#if BACKTRACE_ENABLE
    +
    +#include    /* backtrace, backtrace_symbols */
     
     #define MAX_STACK_FRAMES    50
     
    @@ -208,7 +222,7 @@ static void ABRThandler(int sig) {
     
     void FIO_addAbortHandler()
     {
    -#ifdef BACKTRACES_ENABLE
    +#if BACKTRACE_ENABLE
         signal(SIGABRT, ABRThandler);
         signal(SIGFPE, ABRThandler);
         signal(SIGILL, ABRThandler);
    @@ -1205,7 +1219,8 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix)
         size_t const sfnSize = strlen(srcFileName);
         size_t const suffixSize = strlen(suffix);
     
    -    if (dfnbCapacity <= sfnSize+suffixSize+1) {  /* resize name buffer */
    +    if (dfnbCapacity <= sfnSize+suffixSize+1) {
    +        /* resize buffer for dstName */
             free(dstFileNameBuffer);
             dfnbCapacity = sfnSize + suffixSize + 30;
             dstFileNameBuffer = (char*)malloc(dfnbCapacity);
    @@ -1213,8 +1228,8 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix)
                 EXM_THROW(30, "zstd: %s", strerror(errno));
         }   }
         assert(dstFileNameBuffer != NULL);
    -    strncpy(dstFileNameBuffer, srcFileName, sfnSize+1 /* Include null */);
    -    strncat(dstFileNameBuffer, suffix, suffixSize);
    +    memcpy(dstFileNameBuffer, srcFileName, sfnSize);
    +    memcpy(dstFileNameBuffer+sfnSize, suffix, suffixSize+1 /* Include terminating null */);
     
         return dstFileNameBuffer;
     }
    diff --git a/programs/zstd.1 b/programs/zstd.1
    index 27ee6684c..674f89841 100644
    --- a/programs/zstd.1
    +++ b/programs/zstd.1
    @@ -1,5 +1,5 @@
     .
    -.TH "ZSTD" "1" "September 2018" "zstd 1.3.5" "User Commands"
    +.TH "ZSTD" "1" "October 2018" "zstd 1.3.7" "User Commands"
     .
     .SH "NAME"
     \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
    @@ -123,8 +123,8 @@ Compress using \fB#\fR working threads (default: 1)\. If \fB#\fR is 0, attempt t
     Does not spawn a thread for compression, use a single thread for both I/O and compression\. In this mode, compression is serialized with I/O, which is slightly slower\. (This is different from \fB\-T1\fR, which spawns 1 compression thread in parallel of I/O)\. This mode is the only one available when multithread support is disabled\. Single\-thread mode features lower memory usage\. Final compressed result is slightly different from \fB\-T1\fR\.
     .
     .TP
    -\fB\-\-adapt\fR
    -\fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
    +\fB\-\-adapt[=min=#,max=#]\fR
    +\fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\. \fInote\fR : at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
     .
     .TP
     \fB\-D file\fR
    diff --git a/tests/.gitignore b/tests/.gitignore
    index da536251d..1f08c3995 100644
    --- a/tests/.gitignore
    +++ b/tests/.gitignore
    @@ -1,6 +1,7 @@
     # local binary (Makefile)
     fullbench
     fullbench32
    +fullbench-lib
     fuzzer
     fuzzer32
     fuzzer-dll