From d82efd8a708633affccfbf8571cb5fd712bc8636 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 2 Nov 2016 16:47:53 -0700 Subject: [PATCH 1/4] ZSTD_compress_usingDict() when dict gets loaded Specify that when `dict == NULL || dictSize < 8` no dictionary gets loaded. Also add some periods. --- doc/zstd_compression_format.md | 3 ++- doc/zstd_manual.html | 24 +++++++++++++----------- lib/compress/zstd_compress.c | 1 + lib/zstd.h | 24 +++++++++++++----------- 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/doc/zstd_compression_format.md b/doc/zstd_compression_format.md index b58b43f5a..b48b39104 100644 --- a/doc/zstd_compression_format.md +++ b/doc/zstd_compression_format.md @@ -1135,7 +1135,8 @@ When `Repeated_Offset2` is used, it's swapped with `Repeated_Offset1`. Dictionary format ----------------- -`zstd` is compatible with "raw content" dictionaries, free of any format restriction. +`zstd` is compatible with "raw content" dictionaries, free of any format restriction, +except that they must be at least 8 bytes. But dictionaries created by `zstd --train` follow a format, described here. __Pre-requisites__ : a dictionary has a size, diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 2470a4555..33ea13c32 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -58,7 +58,7 @@

Compresses `src` content as a single zstd compressed frame into already allocated `dst`. Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. @return : compressed size written into `dst` (<= `dstCapacity), - or an error code if it fails (which can be tested using ZSTD_isError()) + or an error code if it fails (which can be tested using ZSTD_isError()). 


size_t ZSTD_decompress( void* dst, size_t dstCapacity,
@@ -67,7 +67,7 @@
     `dstCapacity` is an upper bound of originalSize.
     If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
     @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
-              or an errorCode if it fails (which can be tested using ZSTD_isError()) 
+              or an errorCode if it fails (which can be tested using ZSTD_isError()). 
 

unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
@@ -96,7 +96,7 @@ const char* ZSTD_getErrorName(size_t code);     /*!< provides readable strin
 
Explicit memory management

 
 
size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
-
    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()) 
+
    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). 
 


 
 
Decompression context
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
@@ -104,7 +104,7 @@ ZSTD_DCtx* ZSTD_createDCtx(void);
 size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 


 
size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-
   Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) 
+
   Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). 
 


 
 
Simple dictionary API

@@ -115,7 +115,8 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
                                      const void* dict,size_t dictSize,
                                            int compressionLevel);
 
   Compression using a predefined Dictionary (see dictBuilder/zdict.h).
-   Note : This function load the dictionary, resulting in significant startup delay. 
+   Note : This function loads the dictionary, resulting in significant startup delay.
+   Note : When `dict == NULL || dictSize < 8` no dictionary is used. 
 


 
 
size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
@@ -124,7 +125,8 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
                                        const void* dict,size_t dictSize);
 
   Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
    Dictionary must be identical to the one used during compression.
-   Note : This function load the dictionary, resulting in significant startup delay 
+   Note : This function loads the dictionary, resulting in significant startup delay.
+   Note : When `dict == NULL || dictSize < 8` no dictionary is used. 
 


 
 
Fast dictionary API

@@ -133,11 +135,11 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
   When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
    ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
    ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
-   `dict` can be released after ZSTD_CDict creation 
+   `dict` can be released after ZSTD_CDict creation. 
 


 
 
size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
-
   Function frees memory allocated by ZSTD_createCDict() 
+
   Function frees memory allocated by ZSTD_createCDict(). 
 


 
 
size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
@@ -146,12 +148,12 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
                                       const ZSTD_CDict* cdict);
 
   Compression using a digested Dictionary.
    Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
-   Note that compression level is decided during dictionary creation 
+   Note that compression level is decided during dictionary creation. 
 


 
 
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
 
   Create a digested dictionary, ready to start decompression operation without startup delay.
-   `dict` can be released after creation 
+   `dict` can be released after creation. 
 


 
 
size_t      ZSTD_freeDDict(ZSTD_DDict* ddict);
@@ -162,7 +164,7 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
                                               void* dst, size_t dstCapacity,
                                         const void* src, size_t srcSize,
                                         const ZSTD_DDict* ddict);
-
   Decompression using a digested Dictionary
+
   Decompression using a digested Dictionary.
    Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. 
 


 
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index e7f7d9911..32059f84d 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2695,6 +2695,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
 
 size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
 {
+    if (!dict) dictSize = 0;
     ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dictSize);
     params.fParams.contentSizeFlag = 1;
     return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
diff --git a/lib/zstd.h b/lib/zstd.h
index ea5caf148..d57c2092b 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -72,7 +72,7 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void);  /**< returns version number of
     Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
     Hint : compression runs faster if `dstCapacity` >=  `ZSTD_compressBound(srcSize)`.
     @return : compressed size written into `dst` (<= `dstCapacity),
-              or an error code if it fails (which can be tested using ZSTD_isError()) */
+              or an error code if it fails (which can be tested using ZSTD_isError()). */
 ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
                             const void* src, size_t srcSize,
                                   int compressionLevel);
@@ -82,7 +82,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
     `dstCapacity` is an upper bound of originalSize.
     If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
     @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
-              or an errorCode if it fails (which can be tested using ZSTD_isError()) */
+              or an errorCode if it fails (which can be tested using ZSTD_isError()). */
 ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
                               const void* src, size_t compressedSize);
 
@@ -125,7 +125,7 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
 ZSTDLIB_API size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
 
 /*! ZSTD_compressCCtx() :
-    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()) */
+    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */
 ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
 
 /*= Decompression context */
@@ -134,7 +134,7 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
 ZSTDLIB_API size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
 /*! ZSTD_decompressDCtx() :
-*   Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */
+*   Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). */
 ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 
 
@@ -143,7 +143,8 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapa
 ***************************/
 /*! ZSTD_compress_usingDict() :
 *   Compression using a predefined Dictionary (see dictBuilder/zdict.h).
-*   Note : This function load the dictionary, resulting in significant startup delay. */
+*   Note : This function loads the dictionary, resulting in significant startup delay.
+*   Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
 ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
                                            void* dst, size_t dstCapacity,
                                      const void* src, size_t srcSize,
@@ -153,7 +154,8 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
 /*! ZSTD_decompress_usingDict() :
 *   Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
 *   Dictionary must be identical to the one used during compression.
-*   Note : This function load the dictionary, resulting in significant startup delay */
+*   Note : This function loads the dictionary, resulting in significant startup delay.
+*   Note : When `dict == NULL || dictSize < 8` no dictionary is used. */
 ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
                                              void* dst, size_t dstCapacity,
                                        const void* src, size_t srcSize,
@@ -169,17 +171,17 @@ typedef struct ZSTD_CDict_s ZSTD_CDict;
 *   When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
 *   ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
 *   ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
-*   `dict` can be released after ZSTD_CDict creation */
+*   `dict` can be released after ZSTD_CDict creation. */
 ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
 
 /*! ZSTD_freeCDict() :
-*   Function frees memory allocated by ZSTD_createCDict() */
+*   Function frees memory allocated by ZSTD_createCDict(). */
 ZSTDLIB_API size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
 
 /*! ZSTD_compress_usingCDict() :
 *   Compression using a digested Dictionary.
 *   Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
-*   Note that compression level is decided during dictionary creation */
+*   Note that compression level is decided during dictionary creation. */
 ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
                                             void* dst, size_t dstCapacity,
                                       const void* src, size_t srcSize,
@@ -190,7 +192,7 @@ typedef struct ZSTD_DDict_s ZSTD_DDict;
 
 /*! ZSTD_createDDict() :
 *   Create a digested dictionary, ready to start decompression operation without startup delay.
-*   `dict` can be released after creation */
+*   `dict` can be released after creation. */
 ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
 
 /*! ZSTD_freeDDict() :
@@ -198,7 +200,7 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
 ZSTDLIB_API size_t      ZSTD_freeDDict(ZSTD_DDict* ddict);
 
 /*! ZSTD_decompress_usingDDict() :
-*   Decompression using a digested Dictionary
+*   Decompression using a digested Dictionary.
 *   Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */
 ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
                                               void* dst, size_t dstCapacity,

From 4bafb5aa9712d5645a6faefc211dd47aa3818118 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 3 Nov 2016 11:32:45 -0700
Subject: [PATCH 2/4] The static library was moved to libzstd.a

---
 contrib/pzstd/Makefile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile
index 2de50416c..ad0e55673 100644
--- a/contrib/pzstd/Makefile
+++ b/contrib/pzstd/Makefile
@@ -170,8 +170,7 @@ roundtrip: test/RoundTripTest$(EXT)
 # Use the static library that zstd builds for simplicity and
 # so we get the compiler options correct
 $(ZSTDDIR)/libzstd.a: $(ZSTD_FILES)
-	$(MAKE) -C $(ZSTDDIR) libzstd CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)"
-
+	CFLAGS="$(ALL_CFLAGS)" LDFLAGS="$(ALL_LDFLAGS)" $(MAKE) -C $(ZSTDDIR) libzstd.a
 
 # Rules to build the tests
 test/RoundTripTest$(EXT): test/RoundTripTest.o $(PROGDIR)/datagen.o Options.o \

From c8a9ac312b11c4c48cc97ff9f21854a60770b5b5 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 3 Nov 2016 12:32:48 -0700
Subject: [PATCH 3/4] Fix dynamic libzstd symlinks

---
 lib/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/Makefile b/lib/Makefile
index 04ebe26bd..a7df44f2f 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -74,8 +74,8 @@ $(LIBZSTD): $(ZSTD_FILES)
 	@echo compiling dynamic library $(LIBVER)
 	@$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
 	@echo creating versioned links
-	@ln -sf $@.$(SHARED_EXT_VER) libzstd.$(SHARED_EXT_MAJOR)
-	@ln -sf $@.$(SHARED_EXT_VER) libzstd.$(SHARED_EXT)
+	@ln -sf $(LIBZSTD) libzstd.$(SHARED_EXT_MAJOR)
+	@ln -sf $(LIBZSTD) libzstd.$(SHARED_EXT)
 
 libzstd : $(LIBZSTD)
 

From 407a11f63e5bb68841ee7b7182bd558ea9472bf8 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 3 Nov 2016 15:52:01 -0700
Subject: [PATCH 4/4] fixed Visual compatibility

---
 .coverity.yml                | 5 -----
 lib/compress/zstd_compress.c | 3 +--
 programs/zstdcli.c           | 6 +++---
 3 files changed, 4 insertions(+), 10 deletions(-)
 delete mode 100644 .coverity.yml

diff --git a/.coverity.yml b/.coverity.yml
deleted file mode 100644
index 907f09601..000000000
--- a/.coverity.yml
+++ /dev/null
@@ -1,5 +0,0 @@
-configurationVersion: 1
-
-filters:
-    # third-party embedded
-    - filePath: lib/dictBuilder/divsufsort.c
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 32059f84d..d152d10dc 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2695,8 +2695,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
 
 size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
 {
-    if (!dict) dictSize = 0;
-    ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dictSize);
+    ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dict ? dictSize : 0);
     params.fParams.contentSizeFlag = 1;
     return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
 }
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index db4d6acc6..38eef2b56 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -238,9 +238,9 @@ int main(int argCount, const char* argv[])
 
     /* init */
     (void)recursive; (void)cLevelLast;    /* not used when ZSTD_NOBENCH set */
-    (void)dictCLevel; (void)dictSelect; (void)dictID;  /* not used when ZSTD_NODICT set */
-    (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */
-    (void)ultra; (void)memLimit;   /* not used when ZSTD_NODECOMPRESS set */
+    (void)dictCLevel; (void)dictSelect; (void)dictID;  (void)maxDictSize; /* not used when ZSTD_NODICT set */
+    (void)ultra; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */
+    (void)memLimit;   /* not used when ZSTD_NODECOMPRESS set */
     if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
     filenameTable[0] = stdinmark;
     displayOut = stderr;