From 1fd5b45c6d9c18e02e481734d892287781eea554 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 31 Oct 2016 10:44:44 +0100 Subject: [PATCH 1/2] updated doc/zstd_manual.html --- doc/zstd_manual.html | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index aac5af9c2..2470a4555 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -29,20 +29,20 @@

Introduction

-  Zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
+  zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
   at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
   decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
-  Levels from 20 to 22 should be used with caution as they require about 300-1300 MB for compression.
+  Levels >= 20, labelled `--ultra`, should be used with caution, as they require more memory.
   Compression can be done in:
     - a single step (described as Simple API)
     - a single step, reusing a context (described as Explicit memory management)
-    - repeated calls of the compression function (described as Streaming compression)
+    - unbounded multiple steps (described as Streaming compression)
   The compression ratio achievable on small data can be highly improved using compression with a dictionary in:
     - a single step (described as Simple dictionary API)
     - a single step, reusing a dictionary (described as Fast dictionary API)
 
-  Advanced and experimantal functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
-  These APIs shall never be used with a dynamic library. 
+  Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
+  These APIs shall never be used with a dynamic library.
   They are not "stable", their definition may change in the future. Only static linking is allowed.
@@ -95,10 +95,6 @@ const char* ZSTD_getErrorName(size_t code); /*!< provides readable strin

Explicit memory management


 
-
Compression context
typedef struct ZSTD_CCtx_s ZSTD_CCtx;
-ZSTD_CCtx* ZSTD_createCCtx(void);
-size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
-


 
size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
 
    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()) 
 


@@ -134,12 +130,14 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
Fast dictionary API

 
 
ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
-
   Create a digested dictionary, ready to start compression operation without startup delay.
+
   When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
+   ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
+   ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
    `dict` can be released after ZSTD_CDict creation 
 


 
 
size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
-
   Function frees memory allocated with ZSTD_createCDict() 
+
   Function frees memory allocated by ZSTD_createCDict() 
 


 
 
size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
@@ -186,8 +184,11 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
   A ZSTD_CStream object is required to track streaming operation.
   Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
   ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
+  It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively,
+  since it will play nicer with system's memory, by re-using already allocated memory.
+  Use one separate ZSTD_CStream per thread for parallel execution.
 
-  Start by initializing ZSTD_CStream.
+  Start a new compression by initializing ZSTD_CStream.
   Use ZSTD_initCStream() to start a new compression operation.
   Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
 
@@ -269,7 +270,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 
 
Advanced types

 
-
typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy;   /* from faster to stronger */
+
typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btopt2 } ZSTD_strategy;   /* from faster to stronger */
 


 
typedef struct {
     unsigned windowLog;      /**< largest match distance : larger == more compression, more memory needed during decompression */
@@ -371,20 +372,22 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
                                              ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be zero == unknown */
-size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  /**< re-use compression parameters from previous init; saves dictionary loading */
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */
 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
 


 
Advanced Streaming decompression functions
typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
 ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
 size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
+size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  /**< note : ddict will just be referenced, and must outlive decompression session */
 size_t ZSTD_resetDStream(ZSTD_DStream* zds);  /**< re-use decompression parameters from previous init; saves dictionary loading */
 size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
 


 
Buffer-less and synchronous inner streaming functions
   This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
   But it's also a complex one, with many restrictions (documented below).
-  Prefer using normal streaming API for an easier experience 
+  Prefer using normal streaming API for an easier experience
  
 

 

From f882e132cfad2579cfc700af9df85dea11469f1c Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Mon, 31 Oct 2016 10:51:58 +0100
Subject: [PATCH 2/2] added gen-zstd-manual.sh

---
 contrib/gen_html/gen-zstd-manual.sh | 9 +++++++++
 contrib/gen_html/gen_html.cpp       | 8 ++++----
 2 files changed, 13 insertions(+), 4 deletions(-)
 create mode 100755 contrib/gen_html/gen-zstd-manual.sh

diff --git a/contrib/gen_html/gen-zstd-manual.sh b/contrib/gen_html/gen-zstd-manual.sh
new file mode 100755
index 000000000..57a8b6ea5
--- /dev/null
+++ b/contrib/gen_html/gen-zstd-manual.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+LIBVER_MAJOR_SCRIPT=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h`
+LIBVER_MINOR_SCRIPT=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h`
+LIBVER_PATCH_SCRIPT=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h`
+LIBVER_SCRIPT=$LIBVER_MAJOR_SCRIPT.$LIBVER_MINOR_SCRIPT.$LIBVER_PATCH_SCRIPT
+
+echo ZSTD_VERSION=$LIBVER_SCRIPT
+./gen_html $LIBVER_SCRIPT ../../lib/zstd.h ./zstd_manual.html
diff --git a/contrib/gen_html/gen_html.cpp b/contrib/gen_html/gen_html.cpp
index 2157829f7..d3ab265a6 100644
--- a/contrib/gen_html/gen_html.cpp
+++ b/contrib/gen_html/gen_html.cpp
@@ -90,7 +90,7 @@ int main(int argc, char *argv[]) {
 
     if (argc < 4) {
         cout << "usage: " << argv[0] << " [zstd_version] [input_file] [output_html]" << endl;
-        exit(0);
+        return 1;
     }
 
     version = "zstd " + string(argv[1]) + " Manual";
@@ -98,14 +98,14 @@ int main(int argc, char *argv[]) {
     istream.open(argv[2], ifstream::in);
     if (!istream.is_open()) {
         cout << "Error opening file " << argv[2] << endl;
-        exit(0);
+        return 1;
     }
 
     ostream.open(argv[3], ifstream::out);
     if (!ostream.is_open()) {
         cout << "Error opening file " << argv[3] << endl;
-        exit(0);
-    }
+        return 1;
+   }
 
     while (getline(istream, line)) {
         input.push_back(line);