Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11

# Conflicts:
#	lib/Makefile

.coverity.yml

@@ -1,5 +0,0 @@
-configurationVersion: 1
-
-filters:
-    # third-party embedded
-    - filePath: lib/dictBuilder/divsufsort.c

contrib/pzstd/Makefile

@@ -170,8 +170,7 @@ roundtrip: test/RoundTripTest$(EXT)
 # Use the static library that zstd builds for simplicity and
 # so we get the compiler options correct
 $(ZSTDDIR)/libzstd.a: $(ZSTD_FILES)
-	$(MAKE) -C $(ZSTDDIR) libzstd CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)"
+	CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)" $(MAKE) -C $(ZSTDDIR) libzstd.a
-
 
 # Rules to build the tests
 test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o Options.o \

doc/zstd_compression_format.md

@@ -1135,7 +1135,8 @@ When `Repeated_Offset2` is used, it's swapped with `Repeated_Offset1`.
 Dictionary format
 -----------------
 
-`zstd` is compatible with "raw content" dictionaries, free of any format restriction.
+`zstd` is compatible with "raw content" dictionaries, free of any format restriction,
+except that they must be at least 8 bytes.
 But dictionaries created by `zstd --train` follow a format, described here.
 
 __Pre-requisites__ : a dictionary has a size,

doc/zstd_manual.html

@@ -58,7 +58,7 @@
 </b><p>    Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
     Hint : compression runs faster if `dstCapacity` >=  `ZSTD_compressBound(srcSize)`.
     @return : compressed size written into `dst` (<= `dstCapacity),
-              or an error code if it fails (which can be tested using ZSTD_isError()) 
+              or an error code if it fails (which can be tested using ZSTD_isError()). 
 </p></pre><BR>
 
 <pre><b>size_t ZSTD_decompress( void* dst, size_t dstCapacity,
@@ -67,7 +67,7 @@
     `dstCapacity` is an upper bound of originalSize.
     If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
     @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
-              or an errorCode if it fails (which can be tested using ZSTD_isError()) 
+              or an errorCode if it fails (which can be tested using ZSTD_isError()). 
 </p></pre><BR>
 
 <pre><b>unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
@@ -96,7 +96,7 @@ const char* ZSTD_getErrorName(size_t code);     </b>/*!< provides readable strin
 <a name="Chapter4"></a><h2>Explicit memory management</h2><pre></pre>
 
 <pre><b>size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
-</b><p>    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()) 
+</b><p>    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). 
 </p></pre><BR>
 
 <h3>Decompression context</h3><pre><b>typedef struct ZSTD_DCtx_s ZSTD_DCtx;
@@ -104,7 +104,7 @@ ZSTD_DCtx* ZSTD_createDCtx(void);
 size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 </b></pre><BR>
 <pre><b>size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-</b><p>   Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) 
+</b><p>   Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). 
 </p></pre><BR>
 
 <a name="Chapter5"></a><h2>Simple dictionary API</h2><pre></pre>
@@ -115,7 +115,8 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
                                      const void* dict,size_t dictSize,
                                            int compressionLevel);
 </b><p>   Compression using a predefined Dictionary (see dictBuilder/zdict.h).
-   Note : This function load the dictionary, resulting in significant startup delay. 
+   Note : This function loads the dictionary, resulting in significant startup delay.
+   Note : When `dict == NULL || dictSize < 8` no dictionary is used. 
 </p></pre><BR>
 
 <pre><b>size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
@@ -124,7 +125,8 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
                                        const void* dict,size_t dictSize);
 </b><p>   Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
    Dictionary must be identical to the one used during compression.
-   Note : This function load the dictionary, resulting in significant startup delay 
+   Note : This function loads the dictionary, resulting in significant startup delay.
+   Note : When `dict == NULL || dictSize < 8` no dictionary is used. 
 </p></pre><BR>
 
 <a name="Chapter6"></a><h2>Fast dictionary API</h2><pre></pre>
@@ -133,11 +135,11 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 </b><p>   When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
    ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
    ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
-   `dict` can be released after ZSTD_CDict creation 
+   `dict` can be released after ZSTD_CDict creation. 
 </p></pre><BR>
 
 <pre><b>size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
-</b><p>   Function frees memory allocated by ZSTD_createCDict() 
+</b><p>   Function frees memory allocated by ZSTD_createCDict(). 
 </p></pre><BR>
 
 <pre><b>size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
@@ -146,12 +148,12 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
                                       const ZSTD_CDict* cdict);
 </b><p>   Compression using a digested Dictionary.
    Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
-   Note that compression level is decided during dictionary creation 
+   Note that compression level is decided during dictionary creation. 
 </p></pre><BR>
 
 <pre><b>ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
 </b><p>   Create a digested dictionary, ready to start decompression operation without startup delay.
-   `dict` can be released after creation 
+   `dict` can be released after creation. 
 </p></pre><BR>
 
 <pre><b>size_t      ZSTD_freeDDict(ZSTD_DDict* ddict);
@@ -162,7 +164,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
                                               void* dst, size_t dstCapacity,
                                         const void* src, size_t srcSize,
                                         const ZSTD_DDict* ddict);
-</b><p>   Decompression using a digested Dictionary
+</b><p>   Decompression using a digested Dictionary.
    Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. 
 </p></pre><BR>
 

lib/Makefile

@@ -75,8 +75,8 @@ $(LIBZSTD): $(ZSTD_FILES)
 	@$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
 ifeq (,$(filter Windows%,$(OS)))
 	@echo creating versioned links
-	@ln -sf $@.$(SHARED_EXT_VER) libzstd.$(SHARED_EXT_MAJOR)
+	@ln -sf $(LIBZSTD) libzstd.$(SHARED_EXT_MAJOR)
-	@ln -sf $@.$(SHARED_EXT_VER) libzstd.$(SHARED_EXT)
+	@ln -sf $(LIBZSTD) libzstd.$(SHARED_EXT)
 endif
 
 libzstd : $(LIBZSTD)

lib/compress/zstd_compress.c

@@ -2695,7 +2695,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
 
 size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
 {
-    ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dictSize);
+    ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dict ? dictSize : 0);
     params.fParams.contentSizeFlag = 1;
     return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
 }

lib/zstd.h

@@ -72,7 +72,7 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void);  /**< returns version number of
     Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
     Hint : compression runs faster if `dstCapacity` >=  `ZSTD_compressBound(srcSize)`.
     @return : compressed size written into `dst` (<= `dstCapacity),
-              or an error code if it fails (which can be tested using ZSTD_isError()) */
+              or an error code if it fails (which can be tested using ZSTD_isError()). */
 ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
                             const void* src, size_t srcSize,
                                   int compressionLevel);
@@ -82,7 +82,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
     `dstCapacity` is an upper bound of originalSize.
     If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
     @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
-              or an errorCode if it fails (which can be tested using ZSTD_isError()) */
+              or an errorCode if it fails (which can be tested using ZSTD_isError()). */
 ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
                               const void* src, size_t compressedSize);
 
@@ -125,7 +125,7 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
 ZSTDLIB_API size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
 
 /*! ZSTD_compressCCtx() :
-    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()) */
+    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */
 ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
 
 /*= Decompression context */
@@ -134,7 +134,7 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
 ZSTDLIB_API size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
 /*! ZSTD_decompressDCtx() :
-*   Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */
+*   Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). */
 ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 
 
@@ -143,7 +143,8 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapa
 ***************************/
 /*! ZSTD_compress_usingDict() :
 *   Compression using a predefined Dictionary (see dictBuilder/zdict.h).
-*   Note : This function load the dictionary, resulting in significant startup delay. */
+*   Note : This function loads the dictionary, resulting in significant startup delay.
+*   Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
 ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
                                            void* dst, size_t dstCapacity,
                                      const void* src, size_t srcSize,
@@ -153,7 +154,8 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
 /*! ZSTD_decompress_usingDict() :
 *   Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
 *   Dictionary must be identical to the one used during compression.
-*   Note : This function load the dictionary, resulting in significant startup delay */
+*   Note : This function loads the dictionary, resulting in significant startup delay.
+*   Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
 ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
                                              void* dst, size_t dstCapacity,
                                        const void* src, size_t srcSize,
@@ -169,17 +171,17 @@ typedef struct ZSTD_CDict_s ZSTD_CDict;
 *   When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
 *   ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
 *   ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
-*   `dict` can be released after ZSTD_CDict creation */
+*   `dict` can be released after ZSTD_CDict creation. */
 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
 
 /*! ZSTD_freeCDict() :
-*   Function frees memory allocated by ZSTD_createCDict() */
+*   Function frees memory allocated by ZSTD_createCDict(). */
 ZSTDLIB_API size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
 
 /*! ZSTD_compress_usingCDict() :
 *   Compression using a digested Dictionary.
 *   Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
-*   Note that compression level is decided during dictionary creation */
+*   Note that compression level is decided during dictionary creation. */
 ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
                                             void* dst, size_t dstCapacity,
                                       const void* src, size_t srcSize,
@@ -190,7 +192,7 @@ typedef struct ZSTD_DDict_s ZSTD_DDict;
 
 /*! ZSTD_createDDict() :
 *   Create a digested dictionary, ready to start decompression operation without startup delay.
-*   `dict` can be released after creation */
+*   `dict` can be released after creation. */
 ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
 
 /*! ZSTD_freeDDict() :
@@ -198,7 +200,7 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
 ZSTDLIB_API size_t      ZSTD_freeDDict(ZSTD_DDict* ddict);
 
 /*! ZSTD_decompress_usingDDict() :
-*   Decompression using a digested Dictionary
+*   Decompression using a digested Dictionary.
 *   Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */
 ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
                                               void* dst, size_t dstCapacity,

programs/zstdcli.c

@@ -238,9 +238,9 @@ int main(int argCount, const char* argv[])
 
     /* init */
     (void)recursive; (void)cLevelLast;    /* not used when ZSTD_NOBENCH set */
-    (void)dictCLevel; (void)dictSelect; (void)dictID;  /* not used when ZSTD_NODICT set */
+    (void)dictCLevel; (void)dictSelect; (void)dictID;  (void)maxDictSize; /* not used when ZSTD_NODICT set */
-    (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */
+    (void)ultra; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */
-    (void)ultra; (void)memLimit;   /* not used when ZSTD_NODECOMPRESS set */
+    (void)memLimit;   /* not used when ZSTD_NODECOMPRESS set */
     if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
     filenameTable[0] = stdinmark;
     displayOut = stderr;