Merge pull request #241 from Cyan4973/dev

v0.7.3

NEWS

@@ -1,5 +1,17 @@
+v0.7.3
+New : compression format specification
+New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner.
+New : `ZSTD_getDecompressedSize()`
+New : OpenBSD target, by Juan Francisco Cantero Hurtado
+New : `examples` directory
+fixed : dictBuilder using HC levels, reported by Bartosz Taudul
+fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte
+fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski
+modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section)
+modified : legacy functions no longer need magic number
+
 v0.7.2
-fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski
+fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski.
 fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner.
 fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner.
 

examples/README.md

@@ -0,0 +1,7 @@
+Zstandard library : usage examples
+==================================
+
+- [Dictionary decompression](dictionary_decompression.c)
+  Decompress multiple files using the same dictionary.
+  Compatible with Legacy modes.
+  Introduces usage of : `ZSTD_createDDict()` and `ZSTD_decompress_usingDDict()`

examples/dictionary_decompression.c

@@ -0,0 +1,112 @@
+#include <stdlib.h>    // exit
+#include <stdio.h>     // printf
+#include <string.h>    // strerror
+#include <errno.h>     // errno
+#include <sys/stat.h>  // stat
+#include <zstd.h>
+
+
+static off_t fsizeX(const char *filename)
+{
+    struct stat st;
+    if (stat(filename, &st) == 0) return st.st_size;
+    /* error */
+    printf("stat: %s : %s \n", filename, strerror(errno));
+    exit(1);
+}
+
+static FILE* fopenX(const char *filename, const char *instruction)
+{
+    FILE* const inFile = fopen(filename, instruction);
+    if (inFile) return inFile;
+    /* error */
+    printf("fopen: %s : %s \n", filename, strerror(errno));
+    exit(2);
+}
+
+static void* mallocX(size_t size)
+{
+    void* const buff = malloc(size);
+    if (buff) return buff;
+    /* error */
+    printf("malloc: %s \n", strerror(errno));
+    exit(3);
+}
+
+static void* loadFileX(const char* fileName, size_t* size)
+{
+    off_t const buffSize = fsizeX(fileName);
+    FILE* const inFile = fopenX(fileName, "rb");
+    void* const buffer = mallocX(buffSize);
+    size_t const readSize = fread(buffer, 1, buffSize, inFile);
+    if (readSize != (size_t)buffSize) {
+        printf("fread: %s : %s \n", fileName, strerror(errno));
+        exit(4);
+    }
+    fclose(inFile);
+    *size = buffSize;
+    return buffer;
+}
+
+
+static const ZSTD_DDict* createDict(const char* dictFileName)
+{
+    size_t dictSize;
+    void* const dictBuffer = loadFileX(dictFileName, &dictSize);
+    const ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize);
+    free(dictBuffer);
+    return ddict;
+}
+
+
+/* prototype declared here, as it currently is part of experimental section */
+unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
+
+static void decompress(const char* fname, const ZSTD_DDict* ddict)
+{
+    size_t cSize;
+    void* const cBuff = loadFileX(fname, &cSize);
+    unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize);
+    if (rSize==0) {
+        printf("%s : original size unknown \n", fname);
+        exit(5);
+    }
+    void* const rBuff = mallocX(rSize);
+
+    ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+    size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict);
+
+    if (dSize != rSize) {
+        printf("error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
+        exit(7);
+    }
+
+    /* success */
+    printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
+
+    ZSTD_freeDCtx(dctx);
+    free(rBuff);
+    free(cBuff);
+}
+
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc<3) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s [FILES] dictionary\n", exeName);
+        return 1;
+    }
+
+    /* load dictionary only once */
+    const char* const dictName = argv[argc-1];
+    const ZSTD_DDict* const dictPtr = createDict(dictName);
+
+    int u;
+    for (u=1; u<argc-1; u++) decompress(argv[u], dictPtr);
+
+    printf("All %u files decoded. \n", argc-2);
+}

lib/.gitignore

@@ -0,0 +1,2 @@
+# make install artefact
+libzstd.pc

lib/README.md

@@ -45,14 +45,19 @@ It is used by `zstd` command line utility, and [7zip plugin](http://mcmilk.de/pr
 - compress/zbuff_compress.c
 - decompress/zbuff_decompress.c
 
+
 #### Dictionary builder
 
-To create dictionaries from training sets :
+In order to create dictionaries from some training sets,
+it's needed to include all files from [dictBuilder directory](dictBuilder/)
+
+
+#### Legacy support
+
+Zstandard can decode previous formats, starting from v0.1.
+Support for these format is provided in [folder legacy](legacy/).
+It's also required to compile the library with `ZSTD_LEGACY_SUPPORT = 1`.
 
-- dictBuilder/divsufsort.c
-- dictBuilder/divsufsort.h
-- dictBuilder/zdict.c
-- dictBuilder/zdict.h
 
 #### Miscellaneous
 

lib/common/error_public.h

@@ -63,7 +63,11 @@ typedef enum {
   ZSTD_error_maxCode
 } ZSTD_ErrorCode;
 
-/* note : compare with size_t function results using ZSTD_getError() */
+/*! ZSTD_getErrorCode() :
+    convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
+    which can be used to compare directly with enum list published into "error_public.h" */
+ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
+const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
 
 
 #if defined (__cplusplus)

lib/common/zbuff.h

@@ -185,7 +185,7 @@ ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
 /*--- Advanced Streaming function ---*/
 ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
                                                const void* dict, size_t dictSize,
-                                               ZSTD_parameters params, U64 pledgedSrcSize);
+                                               ZSTD_parameters params, unsigned long long pledgedSrcSize);
 
 #endif /* ZBUFF_STATIC_LINKING_ONLY */
 

lib/common/zstd.h

@@ -61,7 +61,7 @@ extern "C" {
 ***************************************/
 #define ZSTD_VERSION_MAJOR    0
 #define ZSTD_VERSION_MINOR    7
-#define ZSTD_VERSION_RELEASE  2
+#define ZSTD_VERSION_RELEASE  3
 
 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
 #define ZSTD_QUOTE(str) #str
@@ -197,14 +197,13 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
  * Use them only in association with static linking.
  * ==================================================================================== */
 
-/*--- Dependency ---*/
-#include "mem.h"   /* U32 */
-
 /*--- Constants ---*/
 #define ZSTD_MAGICNUMBER            0xFD2FB527   /* v0.7 */
 #define ZSTD_MAGIC_SKIPPABLE_START  0x184D2A50U
 
-#define ZSTD_WINDOWLOG_MAX    ((U32)(MEM_32bits() ? 25 : 27))
+#define ZSTD_WINDOWLOG_MAX_32  25
+#define ZSTD_WINDOWLOG_MAX_64  27
+#define ZSTD_WINDOWLOG_MAX    ((U32)(MEM_32bits() ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
 #define ZSTD_WINDOWLOG_MIN     18
 #define ZSTD_CHAINLOG_MAX     (ZSTD_WINDOWLOG_MAX+1)
 #define ZSTD_CHAINLOG_MIN       4
@@ -229,19 +228,19 @@ static const size_t ZSTD_skippableHeaderSize = 8;  /* magic number + skippable f
 typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy;   /*< from faster to stronger */
 
 typedef struct {
-    U32 windowLog;      /*< largest match distance : larger == more compression, more memory needed during decompression */
+    unsigned windowLog;      /*< largest match distance : larger == more compression, more memory needed during decompression */
-    U32 chainLog;       /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
+    unsigned chainLog;       /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
-    U32 hashLog;        /*< dispatch table : larger == faster, more memory */
+    unsigned hashLog;        /*< dispatch table : larger == faster, more memory */
-    U32 searchLog;      /*< nb of searches : larger == more compression, slower */
+    unsigned searchLog;      /*< nb of searches : larger == more compression, slower */
-    U32 searchLength;   /*< match length searched : larger == faster decompression, sometimes less compression */
+    unsigned searchLength;   /*< match length searched : larger == faster decompression, sometimes less compression */
-    U32 targetLength;   /*< acceptable match size for optimal parser (only) : larger == more compression, slower */
+    unsigned targetLength;   /*< acceptable match size for optimal parser (only) : larger == more compression, slower */
     ZSTD_strategy strategy;
 } ZSTD_compressionParameters;
 
 typedef struct {
-    U32 contentSizeFlag; /*< 1: content size will be in frame header (if known). */
+    unsigned contentSizeFlag; /*< 1: content size will be in frame header (if known). */
-    U32 checksumFlag;    /*< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
+    unsigned checksumFlag;    /*< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
-    U32 noDictIDFlag;    /*< 1: no dict ID will be saved into frame header (if dictionary compression) */
+    unsigned noDictIDFlag;    /*< 1: no dict ID will be saved into frame header (if dictionary compression) */
 } ZSTD_frameParameters;
 
 typedef struct {
@@ -272,12 +271,12 @@ ZSTDLIB_API unsigned ZSTD_maxCLevel (void);
 /*! ZSTD_getParams() :
 *   same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
 *   All fields of `ZSTD_frameParameters` are set to default (0) */
-ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSize, size_t dictSize);
+ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
 
 /*! ZSTD_getCParams() :
 *   @return ZSTD_compressionParameters structure for a selected compression level and srcSize.
 *   `srcSize` value is optional, select 0 if not known */
-ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize);
+ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize);
 
 /*! ZSTD_checkCParams() :
 *   Ensure param values remain within authorized range */
@@ -286,7 +285,7 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
 /*! ZSTD_adjustCParams() :
 *   optimize params for a given `srcSize` and `dictSize`.
 *   both values are optional, select `0` if unknown. */
-ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize);
+ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
 
 /*! ZSTD_compress_advanced() :
 *   Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */
@@ -299,6 +298,15 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
 
 /*--- Advanced Decompression functions ---*/
 
+/** ZSTD_getDecompressedSize() :
+*   compatible with legacy mode
+*   @return : decompressed size if known, 0 otherwise
+              note : 0 can mean any of the following :
+                   - decompressed size is not provided within frame header
+                   - frame header unknown / not supported
+                   - frame header not completely provided (`srcSize` too small) */
+unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
+
 /*! ZSTD_createDCtx_advanced() :
  *  Create a ZSTD decompression context using external alloc and free functions */
 ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
@@ -309,7 +317,7 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
 ******************************************************************/
 ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
-ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize);
+ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
 ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx);
 
 ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
@@ -345,10 +353,10 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
 */
 
 typedef struct {
-    U64 frameContentSize;
+    unsigned long long frameContentSize;
-    U32 windowSize;
+    unsigned windowSize;
-    U32 dictID;
+    unsigned dictID;
-    U32 checksumFlag;
+    unsigned checksumFlag;
 } ZSTD_frameParams;
 
 ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize);   /**< doesn't consume input */
@@ -408,12 +416,14 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
 *  Block functions
 ****************************************/
 /*! Block functions produce and decode raw zstd blocks, without frame metadata.
+    Frame metadata cost is typically ~18 bytes, which is non-negligible on very small blocks.
     User will have to take in charge required information to regenerate data, such as compressed and content sizes.
 
     A few rules to respect :
     - Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB)
-      + If you need to compress more, it's recommended to use ZSTD_compress() instead, since frame metadata costs become negligible.
+      + If you need to compress more, cut data into multiple blocks
-    - Compressing or decompressing requires a context structure
+      + Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large.
+    - Compressing and decompressing require a context structure
       + Use ZSTD_createCCtx() and ZSTD_createDCtx()
     - It is necessary to init context before starting
       + compression : ZSTD_compressBegin()
@@ -423,23 +433,16 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
     - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero.
       In which case, nothing is produced into `dst`.
       + User must test for such outcome and deal directly with uncompressed data
-      + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!
+      + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!!
+      + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history.
+        Use ZSTD_insertBlock() in such a case.
+        Insert block once it's copied into its final position.
 */
 
 #define ZSTD_BLOCKSIZE_MAX (128 * 1024)   /* define, for static allocation */
 ZSTDLIB_API size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-
+ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  /**< insert block into `dctx` history. Useful to track uncompressed blocks */
-
-/*-*************************************
-*  Error management
-***************************************/
-#include "error_public.h"
-/*! ZSTD_getErrorCode() :
-    convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
-    which can be used to compare directly with enum list published into "error_public.h" */
-ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
-ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
 
 
 #endif   /* ZSTD_STATIC_LINKING_ONLY */

lib/compress/zbuff_compress.c

@@ -137,7 +137,7 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
 
 size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
                                    const void* dict, size_t dictSize,
-                                   ZSTD_parameters params, U64 pledgedSrcSize)
+                                   ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
     /* allocate buffers */
     {   size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;

lib/compress/zstd_compress.c

@@ -221,7 +221,7 @@ size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSiz
     Both `srcSize` and `dictSize` are optional (use 0 if unknown),
     but if both are 0, no optimization can be done.
     Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */
-ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize)
+ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize)
 {
     if (srcSize+dictSize == 0) return cPar;   /* no size information available : no adjustment */
 
@@ -2407,7 +2407,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc,
 *   @return : 0, or an error code */
 size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
                              const void* dict, size_t dictSize,
-                                   ZSTD_parameters params, U64 pledgedSrcSize)
+                                   ZSTD_parameters params, unsigned long long pledgedSrcSize)
 {
     /* compression parameters verification and optimization */
     { size_t const errorCode = ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize);
@@ -2744,7 +2744,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
 /*! ZSTD_getCParams() :
 *   @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`.
 *   Size values are optional, provide 0 if not known or unused */
-ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize)
+ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize)
 {
     ZSTD_compressionParameters cp;
     size_t const addedSize = srcSize ? 0 : 500;
@@ -2765,7 +2765,7 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, si
 /*! ZSTD_getParams() :
 *   same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
 *   All fields of `ZSTD_frameParameters` are set to default (0) */
-ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSize, size_t dictSize) {
+ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) {
     ZSTD_parameters params;
     ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize);
     memset(&params, 0, sizeof(params));

lib/decompress/zstd_decompress.c

@@ -195,7 +195,7 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
 /* Frame format description
    Frame Header -  [ Block Header - Block ] - Frame End
    1) Frame Header
-      - 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h)
+      - 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd.h)
       - 1 byte  - Frame Descriptor
    2) Block Header
       - 3 bytes, starting with a 2-bits descriptor
@@ -391,6 +391,26 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
 }
 
 
+/** ZSTD_getDecompressedSize() :
+*   compatible with legacy mode
+*   @return : decompressed size if known, 0 otherwise
+              note : 0 can mean any of the following :
+                   - decompressed size is not provided within frame header
+                   - frame header unknown / not supported
+                   - frame header not completely provided (`srcSize` too small) */
+unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
+{
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
+    if (ZSTD_isLegacy(src, srcSize)) return ZSTD_getDecompressedSize_legacy(src, srcSize);
+#endif
+    {   ZSTD_frameParams fparams;
+        size_t const frResult = ZSTD_getFrameParams(&fparams, src, srcSize);
+        if (frResult!=0) return 0;
+        return fparams.frameContentSize;
+    }
+}
+
+
 /** ZSTD_decodeFrameHeader() :
 *   `srcSize` must be the size provided by ZSTD_frameHeaderSize().
 *   @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
@@ -629,7 +649,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr,
 
     /* FSE table descriptors */
     {   U32 const LLtype  = *ip >> 6;
-        U32 const Offtype = (*ip >> 4) & 3;
+        U32 const OFtype = (*ip >> 4) & 3;
         U32 const MLtype  = (*ip >> 2) & 3;
         ip++;
 
@@ -637,17 +657,17 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr,
         if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */
 
         /* Build DTables */
-        {   size_t const bhSize = ZSTD_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
+        {   size_t const llhSize = ZSTD_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable);
-            if (ZSTD_isError(bhSize)) return ERROR(corruption_detected);
+            if (ZSTD_isError(llhSize)) return ERROR(corruption_detected);
-            ip += bhSize;
+            ip += llhSize;
         }
-        {   size_t const bhSize = ZSTD_buildSeqTable(DTableOffb, Offtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable);
+        {   size_t const ofhSize = ZSTD_buildSeqTable(DTableOffb, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable);
-            if (ZSTD_isError(bhSize)) return ERROR(corruption_detected);
+            if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected);
-            ip += bhSize;
+            ip += ofhSize;
         }
-        {   size_t const bhSize = ZSTD_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable);
+        {   size_t const mlhSize = ZSTD_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable);
-            if (ZSTD_isError(bhSize)) return ERROR(corruption_detected);
+            if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected);
-            ip += bhSize;
+            ip += mlhSize;
     }   }
 
     return ip-istart;
@@ -688,42 +708,37 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
                             0x2000, 0x4000, 0x8000, 0x10000 };
 
     static const U32 ML_base[MaxML+1] = {
-                             0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10,   11,    12,    13,    14,    15,
+                             3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13,   14,    15,    16,    17,    18,
-                            16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,   27,    28,    29,    30,    31,
+                            19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,   30,    31,    32,    33,    34,
-                            32, 34, 36, 38, 40, 44, 48, 56, 64, 80, 96, 0x80, 0x100, 0x200, 0x400, 0x800,
+                            35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
-                            0x1000, 0x2000, 0x4000, 0x8000, 0x10000 };
+                            0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
 
     static const U32 OF_base[MaxOff+1] = {
-                 0,        1,       3,       7,     0xF,     0x1F,     0x3F,     0x7F,
+                 0,        1,       1,       5,     0xD,     0x1D,     0x3D,     0x7D,
-                 0xFF,   0x1FF,   0x3FF,   0x7FF,   0xFFF,   0x1FFF,   0x3FFF,   0x7FFF,
+                 0xFD,   0x1FD,   0x3FD,   0x7FD,   0xFFD,   0x1FFD,   0x3FFD,   0x7FFD,
-                 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,
+                 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
-                 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, /*fake*/ 1, 1 };
+                 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD };
 
     /* sequence */
     {   size_t offset;
         if (!ofCode)
             offset = 0;
         else {
-            offset = OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits);   /* <=  26 bits */
+            offset = OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits);   /* <=  (ZSTD_WINDOWLOG_MAX-1) bits */
             if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream));
         }
 
-        if (offset < ZSTD_REP_NUM) {
+        if (ofCode <= 1) {
-            if (llCode == 0 && offset <= 1) offset = 1-offset;
+            if ((llCode == 0) & (offset <= 1)) offset = 1-offset;
-
+            if (offset) {
-            if (offset != 0) {
+                size_t const temp = seqState->prevOffset[offset];
-                size_t temp = seqState->prevOffset[offset];
+                if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
-                if (offset != 1) {
-                    seqState->prevOffset[2] = seqState->prevOffset[1];
-                }
                 seqState->prevOffset[1] = seqState->prevOffset[0];
                 seqState->prevOffset[0] = offset = temp;
-
             } else {
                 offset = seqState->prevOffset[0];
             }
         } else {
-            offset -= ZSTD_REP_MOVE;
             seqState->prevOffset[2] = seqState->prevOffset[1];
             seqState->prevOffset[1] = seqState->prevOffset[0];
             seqState->prevOffset[0] = offset;
@@ -731,11 +746,11 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
         seq.offset = offset;
     }
 
-    seq.matchLength = ML_base[mlCode] + MINMATCH + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0);   /* <=  16 bits */
+    seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0);   /* <=  16 bits */
     if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream));
 
     seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0);   /* <=  16 bits */
-    if (MEM_32bits() |
+    if (MEM_32bits() ||
        (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream));
 
     /* ANS state update */
@@ -924,6 +939,16 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
 }
 
 
+/** ZSTD_insertBlock() :
+    insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
+ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize)
+{
+    ZSTD_checkContinuity(dctx, blockStart);
+    dctx->previousDstEnd = (const char*)blockStart + blockSize;
+    return blockSize;
+}
+
+
 size_t ZSTD_generateNxByte(void* dst, size_t dstCapacity, BYTE byte, size_t length)
 {
     if (length > dstCapacity) return ERROR(dstSize_tooSmall);
@@ -1020,10 +1045,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
                                  const void* dict, size_t dictSize)
 {
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
-    {   U32 const magicNumber = MEM_readLE32(src);
+    if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize);
-        if (ZSTD_isLegacy(magicNumber))
-            return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize, magicNumber);
-    }
 #endif
     ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
     ZSTD_checkContinuity(dctx, dst);
@@ -1262,8 +1284,8 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t
 
 
 struct ZSTD_DDict_s {
-    void* dictContent;
+    void* dict;
-    size_t dictContentSize;
+    size_t dictSize;
     ZSTD_DCtx* refContext;
 };  /* typedef'd tp ZSTD_CDict within zstd.h */
 
@@ -1295,8 +1317,8 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu
                 return NULL;
         }   }
 
-        ddict->dictContent = dictContent;
+        ddict->dict = dictContent;
-        ddict->dictContentSize = dictSize;
+        ddict->dictSize = dictSize;
         ddict->refContext = dctx;
         return ddict;
     }
@@ -1316,7 +1338,7 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
     ZSTD_freeFunction const cFree = ddict->refContext->customMem.customFree;
     void* const opaque = ddict->refContext->customMem.opaque;
     ZSTD_freeDCtx(ddict->refContext);
-    cFree(opaque, ddict->dictContent);
+    cFree(opaque, ddict->dict);
     cFree(opaque, ddict);
     return 0;
 }
@@ -1329,6 +1351,9 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
                                      const void* src, size_t srcSize,
                                      const ZSTD_DDict* ddict)
 {
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
+    if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize);
+#endif
     return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext,
                                            dst, dstCapacity,
                                            src, srcSize);

lib/dictBuilder/zdict.c

@@ -31,14 +31,15 @@
     - Zstd homepage : https://www.zstd.net
 */
 
+/*-**************************************
+*  Tuning parameters
+****************************************/
+#define ZDICT_MAX_SAMPLES_SIZE (2000U << 20)
+
+
 /*-**************************************
 *  Compiler Options
 ****************************************/
-/* Disable some Visual warning messages */
-#ifdef _MSC_VER
-#  pragma warning(disable : 4127)                /* disable: C4127: conditional expression is constant */
-#endif
-
 /* Unix Large Files support (>4GB) */
 #define _FILE_OFFSET_BITS 64
 #if (defined(__sun__) && (!defined(__LP64__)))   /* Sun Solaris 32-bits requires specific definitions */
@@ -58,13 +59,15 @@
 
 #include "mem.h"           /* read */
 #include "error_private.h"
-#include "fse.h"
+#include "fse.h"           /* FSE_normalizeCount, FSE_writeNCount */
 #define HUF_STATIC_LINKING_ONLY
 #include "huf.h"
 #include "zstd_internal.h" /* includes zstd.h */
 #include "xxhash.h"
 #include "divsufsort.h"
+#ifndef ZDICT_STATIC_LINKING_ONLY
 #  define ZDICT_STATIC_LINKING_ONLY
+#endif
 #include "zdict.h"
 
 
@@ -91,17 +94,19 @@ static const size_t g_min_fast_dictContent = 192;
 /*-*************************************
 *  Console display
 ***************************************/
-#define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
+#define DISPLAY(...)         { fprintf(stderr, __VA_ARGS__); fflush( stderr ); }
 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
 static unsigned g_displayLevel = 0;   /* 0 : no display;   1: errors;   2: default;  4: full information */
 
 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
-            if (ZDICT_GetMilliSpan(g_time) > refreshRate)  \
+            if (ZDICT_clockSpan(g_time) > refreshRate)  \
             { g_time = clock(); DISPLAY(__VA_ARGS__); \
             if (g_displayLevel>=4) fflush(stdout); } }
-static const unsigned refreshRate = 300;
+static const clock_t refreshRate = CLOCKS_PER_SEC * 3 / 10;
 static clock_t g_time = 0;
 
+static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; }
+
 static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length)
 {
     const BYTE* const b = (const BYTE*)ptr;
@@ -117,13 +122,6 @@ static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length)
 /*-********************************************************
 *  Helper functions
 **********************************************************/
-static unsigned ZDICT_GetMilliSpan(clock_t nPrevious)
-{
-    clock_t nCurrent = clock();
-    unsigned nSpan = (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC);
-    return nSpan;
-}
-
 unsigned ZDICT_isError(size_t errorCode) { return ERR_isError(errorCode); }
 
 const char* ZDICT_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
@@ -489,7 +487,7 @@ static U32 ZDICT_dictSize(const dictItem* dictList)
 
 
 static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
-                            const void* const buffer, const size_t bufferSize,   /* buffer must end with noisy guard band */
+                            const void* const buffer, size_t bufferSize,   /* buffer must end with noisy guard band */
                             const size_t* fileSizes, unsigned nbFiles,
                             U32 shiftRatio, unsigned maxDictSize)
 {
@@ -499,7 +497,6 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
     BYTE* doneMarks = (BYTE*)malloc((bufferSize+16)*sizeof(*doneMarks));   /* +16 for overflow security */
     U32* filePos = (U32*)malloc(nbFiles * sizeof(*filePos));
     U32 minRatio = nbFiles >> shiftRatio;
-    int divSuftSortResult;
     size_t result = 0;
 
     /* init */
@@ -511,15 +508,18 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
     if (minRatio < MINRATIO) minRatio = MINRATIO;
     memset(doneMarks, 0, bufferSize+16);
 
+    /* limit sample set size (divsufsort limitation)*/
+    if (bufferSize > ZDICT_MAX_SAMPLES_SIZE) DISPLAYLEVEL(3, "sample set too large : reduced to %u MB ...\n", (U32)(ZDICT_MAX_SAMPLES_SIZE>>20));
+    while (bufferSize > ZDICT_MAX_SAMPLES_SIZE) bufferSize -= fileSizes[--nbFiles];
+
     /* sort */
     DISPLAYLEVEL(2, "sorting %u files of total size %u MB ...\n", nbFiles, (U32)(bufferSize>>20));
-    divSuftSortResult = divsufsort((const unsigned char*)buffer, suffix, (int)bufferSize, 0);
+    { int const divSuftSortResult = divsufsort((const unsigned char*)buffer, suffix, (int)bufferSize, 0);
-    if (divSuftSortResult != 0) { result = ERROR(GENERIC); goto _cleanup; }
+      if (divSuftSortResult != 0) { result = ERROR(GENERIC); goto _cleanup; } }
     suffix[bufferSize] = (int)bufferSize;   /* leads into noise */
     suffix0[0] = (int)bufferSize;           /* leads into noise */
-    {
     /* build reverse suffix sort */
-        size_t pos;
+    {   size_t pos;
         for (pos=0; pos < bufferSize; pos++)
             reverseSuffix[suffix[pos]] = (U32)pos;
         /* build file pos */
@@ -587,7 +587,9 @@ static void ZDICT_countEStats(EStats_ress_t esr,
     size_t cSize;
 
     if (srcSize > ZSTD_BLOCKSIZE_MAX) srcSize = ZSTD_BLOCKSIZE_MAX;   /* protection vs large samples */
-    ZSTD_copyCCtx(esr.zc, esr.ref);
+	{	size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref);
+		if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; }
+	}
     cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize);
     if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; }
 
@@ -709,9 +711,13 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     }
     if (compressionLevel==0) compressionLevel=g_compressionLevel_default;
     params.cParams = ZSTD_getCParams(compressionLevel, averageSampleSize, dictBufferSize);
-    params.cParams.strategy = ZSTD_greedy;
     params.fParams.contentSizeFlag = 0;
-    ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0);
+	{	size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0);
+		if (ZSTD_isError(beginResult)) {
+			eSize = ERROR(GENERIC);
+			DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed ");
+			goto _cleanup;
+	}	}
 
     /* collect stats on all files */
     for (u=0; u<nbFiles; u++) {
@@ -826,6 +832,7 @@ static size_t ZDICT_analyzeEntropy(void*  dstBuffer, size_t maxDstSize,
     MEM_writeLE32(dstPtr+4, repStartValue[1]);
     MEM_writeLE32(dstPtr+8, repStartValue[2]);
 #endif
+    dstPtr += 12;
     eSize += 12;
 
 _cleanup:
@@ -905,7 +912,6 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
 }
 
 #define DIB_MINSAMPLESSIZE (DIB_FASTSEGMENTSIZE*3)
-#define EXIT(e) { dictSize = ERROR(e); goto _cleanup; }
 /*! ZDICT_trainFromBuffer_unsafe() :
 *   `samplesBuffer` must be followed by noisy guard band.
 *   @return : size of dictionary.
@@ -923,12 +929,12 @@ size_t ZDICT_trainFromBuffer_unsafe(
     size_t dictSize = 0;
 
     /* checks */
-    if (maxDictSize <= g_provision_entropySize + g_min_fast_dictContent) EXIT(dstSize_tooSmall);
+    if (maxDictSize <= g_provision_entropySize + g_min_fast_dictContent) return ERROR(dstSize_tooSmall);
     if (!dictList) return ERROR(memory_allocation);
 
     /* init */
     { unsigned u; for (u=0, sBuffSize=0; u<nbSamples; u++) sBuffSize += samplesSizes[u]; }
-    if (sBuffSize < DIB_MINSAMPLESSIZE) EXIT(no_error);   /* not enough source to create dictionary */
+    if (sBuffSize < DIB_MINSAMPLESSIZE) return 0;   /* not enough source to create dictionary */
     ZDICT_initDictItem(dictList);
     g_displayLevel = params.notificationLevel;
     if (selectivity==0) selectivity = g_selectivity_default;
@@ -948,9 +954,9 @@ size_t ZDICT_trainFromBuffer_unsafe(
             DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", dictList[0].pos, dictContentSize);
             DISPLAYLEVEL(3, "list %u best segments \n", nb);
             for (u=1; u<=nb; u++) {
-                U32 const p = dictList[u].pos;
+                U32 p = dictList[u].pos;
-                U32 const l = dictList[u].length;
+                U32 l = dictList[u].length;
-                U32 const d = MIN(40, l);
+                U32 d = MIN(40, l);
                 DISPLAYLEVEL(3, "%3u:%3u bytes at pos %8u, savings %7u bytes |",
                              u, l, p, dictList[u].savings);
                 ZDICT_printHex(3, (const char*)samplesBuffer+p, d);
@@ -966,7 +972,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
             for (u=1; u<dictList->pos; u++) {
                 U32 l = dictList[u].length;
                 ptr -= l;
-                if (ptr<(BYTE*)dictBuffer) EXIT(GENERIC);   /* should not happen */
+                if (ptr<(BYTE*)dictBuffer) return ERROR(GENERIC);   /* should not happen */
                 memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l);
         }   }
 
@@ -983,7 +989,7 @@ size_t ZDICT_trainFromBuffer_unsafe(
                                                              params);
     }
 
-_cleanup :
+    /* clean up */
     free(dictList);
     return dictSize;
 }

lib/legacy/zstd_legacy.h

@@ -54,8 +54,11 @@ extern "C" {
     @return : > 0 if supported by legacy decoder. 0 otherwise.
               return value is the version.
 */
-MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE)
+MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize)
 {
+    U32 magicNumberLE;
+    if (srcSize<4) return 0;
+    magicNumberLE = MEM_readLE32(src);
     switch(magicNumberLE)
     {
         case ZSTDv01_magicNumberLE:return 1;
@@ -69,23 +72,45 @@ MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE)
 }
 
 
+MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, size_t srcSize)
+{
+    if (srcSize < 4) return 0;
+
+    {   U32 const version = ZSTD_isLegacy(src, srcSize);
+        if (version < 5) return 0;  /* no decompressed size in frame header, or not a legacy format */
+        if (version==5) {
+            ZSTDv05_parameters fParams;
+            size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize);
+            if (frResult != 0) return 0;
+            return fParams.srcSize;
+        }
+        if (version==6) {
+            ZSTDv06_frameParams fParams;
+            size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize);
+            if (frResult != 0) return 0;
+            return fParams.frameContentSize;
+        }
+        return 0;   /* should not be possible */
+    }
+}
+
 MEM_STATIC size_t ZSTD_decompressLegacy(
                      void* dst, size_t dstCapacity,
                const void* src, size_t compressedSize,
-               const void* dict,size_t dictSize,
+               const void* dict,size_t dictSize)
-                     U32 magicNumberLE)
 {
-    switch(magicNumberLE)
+    U32 const version = ZSTD_isLegacy(src, compressedSize);
+    switch(version)
     {
-        case ZSTDv01_magicNumberLE :
+        case 1 :
             return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize);
-        case ZSTDv02_magicNumber :
+        case 2 :
             return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize);
-        case ZSTDv03_magicNumber :
+        case 3 :
             return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize);
-        case ZSTDv04_magicNumber :
+        case 4 :
             return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize);
-        case ZSTDv05_MAGICNUMBER :
+        case 5 :
             {   size_t result;
                 ZSTDv05_DCtx* const zd = ZSTDv05_createDCtx();
                 if (zd==NULL) return ERROR(memory_allocation);
@@ -93,7 +118,7 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
                 ZSTDv05_freeDCtx(zd);
                 return result;
             }
-        case ZSTDv06_MAGICNUMBER :
+        case 6 :
             {   size_t result;
                 ZSTDv06_DCtx* const zd = ZSTDv06_createDCtx();
                 if (zd==NULL) return ERROR(memory_allocation);

lib/legacy/zstd_v06.c

@@ -535,8 +535,6 @@ ZSTDLIB_API size_t ZSTDv06_decompress_usingPreparedDCtx(
 
 
 
-struct ZSTDv06_frameParams_s { U64 frameContentSize; U32 windowLog; };
-
 #define ZSTDv06_FRAMEHEADERSIZE_MAX 13    /* for static allocation */
 static const size_t ZSTDv06_frameHeaderSize_min = 5;
 static const size_t ZSTDv06_frameHeaderSize_max = ZSTDv06_FRAMEHEADERSIZE_MAX;

lib/legacy/zstd_v06.h

@@ -107,7 +107,7 @@ ZSTDLIB_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx,
 /*-************************
 *  Advanced Streaming API
 ***************************/
-
+struct ZSTDv06_frameParams_s { unsigned long long frameContentSize; unsigned windowLog; };
 typedef struct ZSTDv06_frameParams_s ZSTDv06_frameParams;
 
 ZSTDLIB_API size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src, size_t srcSize);   /**< doesn't consume input */

programs/Makefile

@@ -154,10 +154,10 @@ clean:
 	@echo Cleaning completed
 
 
-#------------------------------------------------------------------------
+#---------------------------------------------------------------------------------
-#make install is validated only for Linux, OSX, kFreeBSD and Hurd targets
+#make install is validated only for Linux, OSX, kFreeBSD, Hurd and OpenBSD targets
-#------------------------------------------------------------------------
+#---------------------------------------------------------------------------------
-ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU))
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD))
 HOST_OS = POSIX
 install: zstd
 	@echo Installing binaries

programs/dibio.c

@@ -30,6 +30,7 @@
 #include <string.h>         /* memset */
 #include <stdio.h>          /* fprintf, fopen, ftello64 */
 #include <time.h>           /* clock_t, clock, CLOCKS_PER_SEC */
+#include <errno.h>          /* errno */
 
 #include "mem.h"            /* read */
 #include "error_private.h"
@@ -43,13 +44,10 @@
 #define MB *(1 <<20)
 #define GB *(1U<<30)
 
-#define DICTLISTSIZE 10000
 #define MEMMULT 11
 static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t));
 
 #define NOISELENGTH 32
-#define PRIME1   2654435761U
-#define PRIME2   2246822519U
 
 
 /*-*************************************
@@ -60,17 +58,13 @@ static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_
 static unsigned g_displayLevel = 0;   /* 0 : no display;   1: errors;   2: default;  4: full information */
 
 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
-            if ((DIB_GetMilliSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \
+            if ((DIB_clockSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \
             { g_time = clock(); DISPLAY(__VA_ARGS__); \
             if (g_displayLevel>=4) fflush(stdout); } }
-static const unsigned refreshRate = 150;
+static const clock_t refreshRate = CLOCKS_PER_SEC * 2 / 10;
 static clock_t g_time = 0;
 
-static unsigned DIB_GetMilliSpan(clock_t nPrevious)
+static clock_t DIB_clockSpan(clock_t nPrevious) { return clock() - nPrevious; }
-{
-    clock_t const nCurrent = clock();
-    return (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC);
-}
 
 
 /*-*************************************
@@ -97,13 +91,15 @@ unsigned DiB_isError(size_t errorCode) { return ERR_isError(errorCode); }
 
 const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
 
+#define MIN(a,b)   ( (a) < (b) ? (a) : (b) )
+
 
 /* ********************************************************
 *  File related operations
 **********************************************************/
 /** DiB_loadFiles() :
 *   @return : nb of files effectively loaded into `buffer` */
-static unsigned DiB_loadFiles(void* buffer, size_t bufferSize,
+static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr,
                               size_t* fileSizes,
                               const char** fileNamesTable, unsigned nbFiles)
 {
@@ -112,18 +108,20 @@ static unsigned DiB_loadFiles(void* buffer, size_t bufferSize,
     unsigned n;
 
     for (n=0; n<nbFiles; n++) {
-        unsigned long long const fs64 = UTIL_getFileSize(fileNamesTable[n]);
+        const char* const fileName = fileNamesTable[n];
-        size_t const fileSize = (size_t)(fs64 > bufferSize-pos ? 0 : fs64);
+        unsigned long long const fs64 = UTIL_getFileSize(fileName);
-        FILE* const f = fopen(fileNamesTable[n], "rb");
+        size_t const fileSize = (size_t) MIN(fs64, 128 KB);
-        if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
+        if (fileSize > *bufferSizePtr-pos) break;
-        DISPLAYUPDATE(2, "Loading %s...       \r", fileNamesTable[n]);
+        {   FILE* const f = fopen(fileName, "rb");
+            if (f==NULL) EXM_THROW(10, "zstd: dictBuilder: %s %s ", fileName, strerror(errno));
+            DISPLAYUPDATE(2, "Loading %s...       \r", fileName);
             { size_t const readSize = fread(buff+pos, 1, fileSize, f);
-          if (readSize != fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
+              if (readSize != fileSize) EXM_THROW(11, "Pb reading %s", fileName);
               pos += readSize; }
             fileSizes[n] = fileSize;
             fclose(f);
-        if (fileSize == 0) break;  /* stop there, not enough memory to load all files */
+    }   }
-    }
+    *bufferSizePtr = pos;
     return n;
 }
 
@@ -137,26 +135,28 @@ static size_t DiB_findMaxMem(unsigned long long requiredMem)
     void* testmem = NULL;
 
     requiredMem = (((requiredMem >> 23) + 1) << 23);
-    requiredMem += 2 * step;
+    requiredMem += step;
     if (requiredMem > maxMemory) requiredMem = maxMemory;
 
     while (!testmem) {
-        requiredMem -= step;
         testmem = malloc((size_t)requiredMem);
+        requiredMem -= step;
     }
 
     free(testmem);
-    return (size_t)(requiredMem - step);
+    return (size_t)requiredMem;
 }
 
 
 static void DiB_fillNoise(void* buffer, size_t length)
 {
-    unsigned acc = PRIME1;
+    unsigned const prime1 = 2654435761U;
+    unsigned const prime2 = 2246822519U;
+    unsigned acc = prime1;
     size_t p=0;;
 
     for (p=0; p<length; p++) {
-        acc *= PRIME2;
+        acc *= prime2;
         ((unsigned char*)buffer)[p] = (unsigned char)(acc >> 21);
     }
 }
@@ -188,7 +188,6 @@ size_t ZDICT_trainFromBuffer_unsafe(void* dictBuffer, size_t dictBufferCapacity,
                               ZDICT_params_t parameters);
 
 
-#define MIN(a,b)  ((a)<(b)?(a):(b))
 int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
                        const char** fileNamesTable, unsigned nbFiles,
                        ZDICT_params_t params)
@@ -197,7 +196,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
     size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
     unsigned long long const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
     size_t const maxMem =  DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
-    size_t const benchedSize = MIN (maxMem, (size_t)totalSizeToLoad);
+    size_t benchedSize = MIN (maxMem, (size_t)totalSizeToLoad);
     void* const srcBuffer = malloc(benchedSize+NOISELENGTH);
     int result = 0;
 
@@ -210,7 +209,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
         DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(benchedSize >> 20));
 
     /* Load input buffer */
-    nbFiles = DiB_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
+    nbFiles = DiB_loadFiles(srcBuffer, &benchedSize, fileSizes, fileNamesTable, nbFiles);
     DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH);   /* guard band, for end of buffer condition */
 
     {   size_t const dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize,

programs/fileio.c

@@ -700,7 +700,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName)
         if (sizeCheck != toRead) EXM_THROW(31, "zstd: %s read error : cannot read header", srcFileName);
         {   U32 const magic = MEM_readLE32(ress.srcBuffer);
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
-            if (ZSTD_isLegacy(magic)) {
+            if (ZSTD_isLegacy(ress.srcBuffer, 4)) {
                 filesize += FIO_decompressLegacyFrame(dstFile, srcFile, ress.dictBuffer, ress.dictBufferSize, magic);
                 continue;
             }

programs/fuzzer.c

@@ -41,7 +41,8 @@
 #include <string.h>      /* strcmp */
 #include <time.h>        /* clock_t */
 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_compressContinue, ZSTD_compressBlock */
-#include "zstd.h"        /* ZSTD_VERSION_STRING, ZSTD_getErrorCode */
+#include "zstd.h"        /* ZSTD_VERSION_STRING */
+#include "error_public.h" /* ZSTD_getErrorCode */
 #include "zdict.h"       /* ZDICT_trainFromBuffer */
 #include "datagen.h"     /* RDG_genBuffer */
 #include "mem.h"
@@ -137,6 +138,12 @@ static int basicUnitTests(U32 seed, double compressibility)
               cSize=r );
     DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
 
+    DISPLAYLEVEL(4, "test%3i : decompressed size test : ", testNb++);
+    {   unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize);
+        if (rSize != CNBuffSize) goto _output_error;
+    }
+    DISPLAYLEVEL(4, "OK \n");
+
     DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize);
     CHECKPLUS( r , ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize),
                if (r != CNBuffSize) goto _output_error);
@@ -338,13 +345,18 @@ static int basicUnitTests(U32 seed, double compressibility)
         if (ZSTD_isError(cSize)) goto _output_error;
         cSize2 = ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize);
         if (ZSTD_isError(cSize2)) goto _output_error;
+        memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize);   /* fake non-compressed block */
+        cSize2 = ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize),
+                                          (char*)CNBuffer+dictSize+2*blockSize, blockSize);
+        if (ZSTD_isError(cSize2)) goto _output_error;
         DISPLAYLEVEL(4, "OK \n");
 
         DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++);
         CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
         { CHECK_V( r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
           if (r != blockSize) goto _output_error; }
-        { CHECK_V( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+blockSize, CNBuffSize, (char*)compressedBuffer+cSize, cSize2) );
+        ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize);   /* insert non-compressed block into dctx history */
+        { CHECK_V( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, CNBuffSize, (char*)compressedBuffer+cSize+blockSize, cSize2) );
           if (r != blockSize) goto _output_error; }
         DISPLAYLEVEL(4, "OK \n");
 
@@ -390,14 +402,16 @@ static int basicUnitTests(U32 seed, double compressibility)
         U32 rSeed = 1;
 
         /* create batch of 3-bytes sequences */
-        { int i; for (i=0; i < NB3BYTESSEQ; i++) {
+        {   int i;
+            for (i=0; i < NB3BYTESSEQ; i++) {
                 _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255);
                 _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255);
                 _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255);
         }   }
 
         /* randomly fills CNBuffer with prepared 3-bytes sequences */
-        { int i; for (i=0; i < _3BYTESTESTLENGTH; i += 3) {   /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
+        {   int i;
+            for (i=0; i < _3BYTESTESTLENGTH; i += 3) {   /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
                 U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK;
                 ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
                 ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];
@@ -556,6 +570,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD
                   CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); }
         }   }
 
+        /* Decompressed size test */
+        {   unsigned long long const rSize = ZSTD_getDecompressedSize(cBuffer, cSize);
+            CHECK(rSize != sampleSize, "decompressed size incorrect");
+        }
+
         /* frame header decompression test */
         {   ZSTD_frameParams dParams;
             size_t const check = ZSTD_getFrameParams(&dParams, cBuffer, cSize);

programs/zstd.1

@@ -39,7 +39,6 @@ It also features a very fast decoder, with speed > 500 MB/s per core.
      Use \fB-q\fR to turn them off
 
 
-\fBzstd\fR supports the following options :
 
 .SH OPTIONS
 .TP
@@ -132,9 +131,6 @@ Typical gains range from ~10% (at 64KB) to x5 better (at <1KB).
 .TP
 .B \-B#
  cut file into independent blocks of size # (default: no block)
-.TP
-.B \-r#
- test all compression levels from 1 to # (default: disabled)
 
 
 .SH BUGS

programs/zstdcli.c

@@ -34,6 +34,7 @@
 #include "util.h"     /* Compiler options, UTIL_HAS_CREATEFILELIST */
 #include <string.h>   /* strcmp, strlen */
 #include <ctype.h>    /* toupper */
+#include <errno.h>    /* errno */
 #include "fileio.h"
 #ifndef ZSTD_NOBENCH
 #  include "bench.h"  /* BMK_benchFiles, BMK_SetNbIterations */
@@ -45,7 +46,6 @@
 #include "zstd.h"     /* ZSTD_VERSION_STRING */
 
 
-
 /*-************************************
 *  OS-specific Includes
 **************************************/
@@ -116,6 +116,7 @@ static int usage(const char* programName)
     DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n");
     DISPLAY( " -f     : overwrite output without prompting \n");
     DISPLAY( "--rm    : remove source file(s) after successful de/compression \n");
+    DISPLAY( " -k     : preserve source file(s) (default) \n");
     DISPLAY( " -h/-H  : display help/long help and exit\n");
     return 0;
 }
@@ -169,7 +170,6 @@ static int badusage(const char* programName)
     return 1;
 }
 
-
 static void waitEnter(void)
 {
     int unused;
@@ -205,7 +205,8 @@ int main(int argCount, const char** argv)
         dictBuild=0,
         nextArgumentIsOutFileName=0,
         nextArgumentIsMaxDict=0,
-        nextArgumentIsDictID=0;
+        nextArgumentIsDictID=0,
+        nextArgumentIsFile=0;
     unsigned cLevel = 1;
     unsigned cLevelLast = 1;
     unsigned recursive = 0;
@@ -229,7 +230,7 @@ int main(int argCount, const char** argv)
     (void)recursive; (void)cLevelLast;    /* not used when ZSTD_NOBENCH set */
     (void)dictCLevel; (void)dictSelect; (void)dictID;  /* not used when ZSTD_NODICT set */
     (void)decode; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */
-    if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); }
+    if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
     filenameTable[0] = stdinmark;
     displayOut = stderr;
     /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */
@@ -247,7 +248,10 @@ int main(int argCount, const char** argv)
         const char* argument = argv[argNb];
         if(!argument) continue;   /* Protection if argument empty */
 
+        if (nextArgumentIsFile==0) {
+
             /* long commands (--long-word) */
+            if (!strcmp(argument, "--")) { nextArgumentIsFile=1; continue; }
             if (!strcmp(argument, "--decompress")) { decode=1; continue; }
             if (!strcmp(argument, "--force")) {  FIO_overwriteMode(); continue; }
             if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); }
@@ -388,19 +392,6 @@ int main(int argCount, const char** argv)
                 continue;
             }   /* if (argument[0]=='-') */
 
-        if (nextEntryIsDictionary) {
-            nextEntryIsDictionary = 0;
-            dictFileName = argument;
-            continue;
-        }
-
-        if (nextArgumentIsOutFileName) {
-            nextArgumentIsOutFileName = 0;
-            outFileName = argument;
-            if (!strcmp(outFileName, "-")) outFileName = stdoutmark;
-            continue;
-        }
-
             if (nextArgumentIsMaxDict) {
                 nextArgumentIsMaxDict = 0;
                 maxDictSize = readU32FromChar(&argument);
@@ -415,6 +406,21 @@ int main(int argCount, const char** argv)
                 continue;
             }
 
+        }   /* if (nextArgumentIsAFile==0) */
+
+        if (nextEntryIsDictionary) {
+            nextEntryIsDictionary = 0;
+            dictFileName = argument;
+            continue;
+        }
+
+        if (nextArgumentIsOutFileName) {
+            nextArgumentIsOutFileName = 0;
+            outFileName = argument;
+            if (!strcmp(outFileName, "-")) outFileName = stdoutmark;
+            continue;
+        }
+
         /* add filename to list */
         filenameTable[filenameIdx++] = argument;
     }

zstd_compression_format.md

@@ -16,7 +16,7 @@ Distribution of this document is unlimited.
 
 ### Version
 
-0.0.1 (30/06/2016 - Work in progress - unfinished)
+0.1.0 (08/07/16)
 
 
 Introduction
@@ -25,7 +25,7 @@ Introduction
 The purpose of this document is to define a lossless compressed data format,
 that is independent of CPU type, operating system,
 file system and character set, suitable for
-File compression, Pipe and streaming compression
+file compression, pipe and streaming compression,
 using the [Zstandard algorithm](http://www.zstandard.org).
 
 The data can be produced or consumed,
@@ -76,8 +76,8 @@ allowing streaming operations.
 General Structure of Zstandard Frame format
 -------------------------------------------
 
-| MagicNb |  F. Header | Block | (More blocks) | EndMark |
+| MagicNb |  Frame Header | Block | (More blocks) | EndMark |
-|:-------:|:----------:| ----- | ------------- | ------- |
+|:-------:|:-------------:| ----- | ------------- | ------- |
 | 4 bytes |  2-14 bytes   |       |               | 3 bytes |
 
 __Magic Number__
@@ -87,11 +87,11 @@ Value : 0xFD2FB527
 
 __Frame Header__
 
-2 to 14 Bytes, to be detailed in the next part.
+2 to 14 Bytes, detailed in [next part](#frame-header).
 
 __Data Blocks__
 
-To be detailed later on.
+Detailed in [next chapter](#data-blocks).
 That’s where compressed data is stored.
 
 __EndMark__
@@ -99,16 +99,17 @@ __EndMark__
 The flow of blocks ends when the last block header brings an _end signal_ .
 This last block header may optionally host a __Content Checksum__ .
 
-__Content Checksum__
+##### __Content Checksum__
 
-Content Checksum verify that frame content has been regenrated correctly.
+Content Checksum verify that frame content has been regenerated correctly.
 The content checksum is the result
 of [xxh64() hash function](https://www.xxHash.com)
 digesting the original (decoded) data as input, and a seed of zero.
 Bits from 11 to 32 (included) are extracted to form a 22 bits checksum
-stored into the last block header.
+stored into the endmark body.
 ```
-contentChecksum = (XXH64(content, size, 0) >> 11) & (1<<22)-1);
+mask22bits = (1<<22)-1;
+contentChecksum = (XXH64(content, size, 0) >> 11) & mask22bits;
 ```
 Content checksum is only present when its associated flag
 is set in the frame descriptor.
@@ -134,9 +135,9 @@ delivering the final decompressed result as if it was a single content.
 Frame Header
 -------------
 
-| FHD     | (WD)      | (Content Size) | (dictID)  |
+| FHD     | (WD)      | (dictID)  | (Content Size) |
-| ------- | --------- |:--------------:| --------- |
+| ------- | --------- | --------- |:--------------:|
-| 1 byte  | 0-1 byte  |  0 - 8 bytes   | 0-4 bytes |
+| 1 byte  | 0-1 byte  | 0-4 bytes |  0 - 8 bytes   |
 
 Frame header has a variable size, which uses a minimum of 2 bytes,
 and up to 14 bytes depending on optional parameters.
@@ -145,10 +146,10 @@ __FHD byte__ (Frame Header Descriptor)
 
 The first Header's byte is called the Frame Header Descriptor.
 It tells which other fields are present.
-Decoding this byte is enough to get the full size of the Frame Header.
+Decoding this byte is enough to tell the size of Frame Header.
 
 |  BitNb  |   7-6  |    5    |   4    |    3     |    2     |  1-0   |
-| ------- | ------ | ------- | ------ | -------- | -------- | -------- |
+| ------- | ------ | ------- | ------ | -------- | -------- | ------ |
 |FieldName| FCSize | Segment | Unused | Reserved | Checksum | dictID |
 
 In this table, bit 7 is highest bit, while bit 0 is lowest.
@@ -162,28 +163,28 @@ specifying if decompressed data size is provided within the header.
 | ------- | --- | --- | --- | --- |
 |FieldSize| 0-1 |  2  |  4  |  8  |
 
-Value 0 has a double meaning :
+Value 0 meaning depends on _single segment_ mode :
 it either means `0` (size not provided) _if_ the `WD` byte is present,
-or it means `1` byte (size <= 255 bytes).
+or `1` (frame content size <= 255 bytes) otherwise.
 
 __Single Segment__
 
 If this flag is set,
 data shall be regenerated within a single continuous memory segment.
+
 In which case, `WD` byte __is not present__,
 but `Frame Content Size` field necessarily is.
-
 As a consequence, the decoder must allocate a memory segment
 of size `>= Frame Content Size`.
 
 In order to preserve the decoder from unreasonable memory requirement,
-a decoder can refuse a compressed frame
+a decoder can reject a compressed frame
 which requests a memory size beyond decoder's authorized range.
 
 For broader compatibility, decoders are recommended to support
-memory sizes of 8 MB at least.
+memory sizes of at least 8 MB.
-However, this is merely a recommendation,
+This is just a recommendation,
-and each decoder is free to support higher or lower limits,
+each decoder is free to support higher or lower limits,
 depending on local limitations.
 
 __Unused bit__
@@ -204,13 +205,14 @@ to signal a feature that must be interpreted in order to decode the frame.
 __Content checksum flag__
 
 If this flag is set, a content checksum will be present into the EndMark.
-The checksum is a 22 bits value extracted from the XXH64() of data.
+The checksum is a 22 bits value extracted from the XXH64() of data,
-See __Content Checksum__ .
+and stored into endMark. See [__Content Checksum__](#content-checksum) .
 
 __Dictionary ID flag__
 
 This is a 2-bits flag (`= FHD & 3`),
-telling if a dictionary ID is provided within the header
+telling if a dictionary ID is provided within the header.
+It also specifies the size of this field.
 
 |  Value  |  0  |  1  |  2  |  3  |
 | ------- | --- | --- | --- | --- |
@@ -222,6 +224,10 @@ Provides guarantees on maximum back-reference distance
 that will be present within compressed data.
 This information is useful for decoders to allocate enough memory.
 
+`WD` byte is optional. It's not present in `single segment` mode.
+In which case, the maximum back-reference distance is the content size itself,
+which can be any value from 1 to 2^64-1 bytes (16 EB).
+
 |   BitNb   |    7-3   |    0-2   |
 | --------- | -------- | -------- |
 | FieldName | Exponent | Mantissa |
@@ -234,26 +240,37 @@ windowAdd = (windowBase / 8) * Mantissa;
 windowSize = windowBase + windowAdd;
 ```
 The minimum window size is 1 KB.
-The maximum size is (15*(2^38))-1 bytes, which is almost 1.875 TB.
+The maximum size is `15*(1<<38)` bytes, which is 1.875 TB.
 
 To properly decode compressed data,
 a decoder will need to allocate a buffer of at least `windowSize` bytes.
 
-Note that `WD` byte is optional. It's not present in `single segment` mode.
-In which case, the maximum back-reference distance is the content size itself,
-which can be any value from 1 to 2^64-1 bytes (16 EB).
-
 In order to preserve decoder from unreasonable memory requirements,
 a decoder can refuse a compressed frame
 which requests a memory size beyond decoder's authorized range.
 
-For better interoperability,
+For improved interoperability,
 decoders are recommended to be compatible with window sizes of 8 MB.
 Encoders are recommended to not request more than 8 MB.
 It's merely a recommendation though,
 decoders are free to support larger or lower limits,
 depending on local limitations.
 
+__Dictionary ID__
+
+This is a variable size field, which contains an ID.
+It checks if the correct dictionary is used for decoding.
+Note that this field is optional. If it's not present,
+it's up to the caller to make sure it uses the correct dictionary.
+
+Field size depends on __Dictionary ID flag__.
+1 byte can represent an ID 0-255.
+2 bytes can represent an ID 0-65535.
+4 bytes can represent an ID 0-4294967295.
+
+It's allowed to represent a small ID (for example `13`)
+with a large 4-bytes dictionary ID, losing some compacity in the process.
+
 __Frame Content Size__
 
 This is the original (uncompressed) size.
@@ -271,30 +288,15 @@ Format is Little endian.
 
 When field size is 1, 4 or 8 bytes, the value is read directly.
 When field size is 2, _an offset of 256 is added_.
-It's allowed to represent a small size (ex: `18`) using the 8-bytes variant.
+It's allowed to represent a small size (ex: `18`) using any compatible variant.
 A size of `0` means `content size is unknown`.
 In which case, the `WD` byte will necessarily be present,
-and becomes the only hint to determine memory allocation.
+and becomes the only hint to guide memory allocation.
 
 In order to preserve decoder from unreasonable memory requirement,
 a decoder can refuse a compressed frame
 which requests a memory size beyond decoder's authorized range.
 
-__Dictionary ID__
-
-This is a variable size field, which contains a single ID.
-It checks if the correct dictionary is used for decoding.
-Note that this field is optional. If it's not present,
-it's up to the caller to make sure it uses the correct dictionary.
-
-Field size depends on __Dictionary ID flag__.
-1 byte can represent an ID 0-255.
-2 bytes can represent an ID 0-65535.
-4 bytes can represent an ID 0-(2^32-1).
-
-It's allowed to represent a small ID (for example `13`)
-with a large 4-bytes dictionary ID, losing some efficiency in the process.
-
 
 Data Blocks
 -----------
@@ -317,8 +319,8 @@ There are 4 block types :
 | ---------- | ---------- | --- | --- | ------- |
 | Block Type | Compressed | Raw | RLE | EndMark |
 
-- Compressed : this is a Zstandard compressed block,
+- Compressed : this is a [Zstandard compressed block](#compressed-block-format),
-  detailed in a later part of this specification.
+  detailed in another section of this specification.
   "block size" is the compressed size.
   Decompressed size is unknown,
   but its maximum possible value is guaranteed (see below)
@@ -329,12 +331,12 @@ There are 4 block types :
   while the "compressed" block is just 1 byte (the byte to repeat).
 - EndMark : this is not a block. Signal the end of the frame.
   The rest of the field may be optionally filled by a checksum
-  (see frame checksum).
+  (see [Content Checksum](#content-checksum)).
 
 Block sizes must respect a few rules :
-- In compressed mode, compressed size if always strictly `< contentSize`.
+- In compressed mode, compressed size if always strictly `< decompressed size`.
-- Block decompressed size is necessarily <= maximum back-reference distance .
+- Block decompressed size is always <= maximum back-reference distance .
-- Block decompressed size is necessarily <= 128 KB
+- Block decompressed size is always <= 128 KB
 
 
 __Data__
@@ -343,9 +345,9 @@ Where the actual data to decode stands.
 It might be compressed or not, depending on previous field indications.
 A data block is not necessarily "full" :
 since an arbitrary “flush” may happen anytime,
-block content can be any size, up to Block Maximum Size.
+block decompressed content can be any size,
-Block Maximum Size is the smallest of :
+up to Block Maximum Decompressed Size, which is the smallest of :
-- Max back-reference distance
+- Maximum back-reference distance
 - 128 KB
 
 
@@ -362,8 +364,9 @@ Its design is pretty straightforward,
 with the sole objective to allow the decoder to quickly skip
 over user-defined data and continue decoding.
 
-Skippable frames defined in this specification are compatible with LZ4 ones.
+Skippable frames defined in this specification are compatible with [LZ4] ones.
 
+[LZ4]:http://www.lz4.org
 
 __Magic Number__ :
 
@@ -386,29 +389,31 @@ User Data can be anything. Data will just be skipped by the decoder.
 Compressed block format
 -----------------------
 This specification details the content of a _compressed block_.
-A compressed block has a size, which must be known in order to decode it.
+A compressed block has a size, which must be known.
 It also has a guaranteed maximum regenerated size,
 in order to properly allocate destination buffer.
-See "Frame format" for more details.
+See [Data Blocks](#data-blocks) for more details.
 
 A compressed block consists of 2 sections :
-- Literals section
+- [Literals section](#literals-section)
-- Sequences section
+- [Sequences section](#sequences-section)
 
-### Prerequisite
+### Prerequisites
-For proper decoding, a compressed block requires access to following elements :
+To decode a compressed block, the following elements are necessary :
 - Previous decoded blocks, up to a distance of `windowSize`,
-  or all previous blocks in the same frame "single segment" mode.
+  or all previous blocks in "single segment" mode.
 - List of "recent offsets" from previous compressed block.
+- Decoding tables of previous compressed block for each symbol type
+  (literals, litLength, matchLength, offset).
 
 
-### Compressed Literals
+### Literals section
 
-Literals are compressed using order-0 huffman compression.
+Literals are compressed using Huffman prefix codes.
 During sequence phase, literals will be entangled with match copy operations.
 All literals are regrouped in the first part of the block.
 They can be decoded first, and then copied during sequence operations,
-or they can be decoded on the flow, as needed by sequences.
+or they can be decoded on the flow, as needed by sequence commands.
 
 | Header | (Tree Description) | Stream1 | (Stream2) | (Stream3) | (Stream4) |
 | ------ | ------------------ | ------- | --------- | --------- | --------- |
@@ -417,9 +422,9 @@ Literals can be compressed, or uncompressed.
 When compressed, an optional tree description can be present,
 followed by 1 or 4 streams.
 
-#### Block Literal Header
+#### Literals section header
 
-Header is in charge of describing precisely how literals are packed.
+Header is in charge of describing how literals are packed.
 It's a byte-aligned variable-size bitfield, ranging from 1 to 5 bytes,
 using big-endian convention.
 
@@ -439,9 +444,8 @@ This is a 2-bits field, describing 4 different block types :
         starting with a huffman tree description.
         See details below.
 - Repeat Stats : This is a huffman-compressed block,
-               using huffman tree from previous huffman-compressed block.
+        using huffman tree _from previous huffman-compressed literals block_.
         Huffman tree description will be skipped.
-               Compressed stream is equivalent to "compressed" block type.
 - Raw : Literals are stored uncompressed.
 - RLE : Literals consist of a single byte value repeated N times.
 
@@ -455,7 +459,7 @@ Sizes format are divided into 2 families :
 
 For values spanning several bytes, convention is Big-endian.
 
-__Sizes format for Raw or RLE block__ :
+__Sizes format for Raw or RLE literals block__ :
 
 - Value : 0x : Regenerated size uses 5 bits (0-31).
                Total literal header size is 1 byte.
@@ -464,39 +468,58 @@ __Sizes format for Raw or RLE block__ :
                Total literal header size is 2 bytes.
                `size = ((h[0] & 15) << 8) + h[1];`
 - Value : 11 : Regenerated size uses 20 bits (0-1048575).
-               Total literal header size is 2 bytes.
+               Total literal header size is 3 bytes.
                `size = ((h[0] & 15) << 16) + (h[1]<<8) + h[2];`
 
 Note : it's allowed to represent a short value (ex : `13`)
 using a long format, accepting the reduced compacity.
 
-__Sizes format for Compressed Block__ :
+__Sizes format for Compressed literals block__ :
 
 Note : also applicable to "repeat-stats" blocks.
-- Value : 00 : 4 streams
+- Value : 00 : 4 streams.
-               Compressed and regenerated sizes use 10 bits (0-1023)
+               Compressed and regenerated sizes use 10 bits (0-1023).
-               Total literal header size is 3 bytes
+               Total literal header size is 3 bytes.
-- Value : 01 : _Single stream_
+- Value : 01 : _Single stream_.
-               Compressed and regenerated sizes use 10 bits (0-1023)
+               Compressed and regenerated sizes use 10 bits (0-1023).
-               Total literal header size is 3 bytes
+               Total literal header size is 3 bytes.
-- Value : 10 : 4 streams
+- Value : 10 : 4 streams.
-               Compressed and regenerated sizes use 14 bits (0-16383)
+               Compressed and regenerated sizes use 14 bits (0-16383).
-               Total literal header size is 4 bytes
+               Total literal header size is 4 bytes.
-- Value : 10 : 4 streams
+- Value : 10 : 4 streams.
-               Compressed and regenerated sizes use 18 bits (0-262143)
+               Compressed and regenerated sizes use 18 bits (0-262143).
-               Total literal header size is 5 bytes
+               Total literal header size is 5 bytes.
 
 Compressed and regenerated size fields follow big endian convention.
 
 #### Huffman Tree description
 
-This section is only present when block type is _compressed_ (`0`).
+This section is only present when literals block type is `Compressed` (`0`).
-It describes the different leaf nodes of the huffman tree,
+
-and their relative weights.
+Prefix coding represents symbols from an a priori known alphabet
+by bit sequences (codes), one code for each symbol,
+in a manner such that different symbols may be represented
+by bit sequences of different lengths,
+but a parser can always parse an encoded string
+unambiguously symbol-by-symbol.
+
+Given an alphabet with known symbol frequencies,
+the Huffman algorithm allows the construction of an optimal prefix code
+using the fewest bits of any possible prefix codes for that alphabet.
+Such a code is called a Huffman code.
+
+Prefix code must not exceed a maximum code length.
+More bits improve accuracy but cost more header size,
+and require more memory for decoding operations.
+
+The current format limits the maximum depth to 15 bits.
+The reference decoder goes further, by limiting it to 11 bits.
+It is recommended to remain compatible with reference decoder.
+
 
 ##### Representation
 
-All byte values from zero (included) to last present one (excluded)
+All literal values from zero (included) to last present one (excluded)
 are represented by `weight` values, from 0 to `maxBits`.
 Transformation from `weight` to `nbBits` follows this formulae :
 `nbBits = weight ? maxBits + 1 - weight : 0;` .
@@ -507,8 +530,8 @@ This power of 2 gives `maxBits`, the depth of the current tree.
 __Example__ :
 Let's presume the following huffman tree must be described :
 
-|  Value | 0 | 1 | 2 | 3 | 4 | 5 |
+| literal |  0  |  1  |  2  |  3  |  4  |  5  |
-| ------ | - | - | - | - | - | - |
+| ------- | --- | --- | --- | --- | --- | --- |
 | nbBits  |  1  |  2  |  3  |  0  |  4  |  4  |
 
 The tree depth is 4, since its smallest element uses 4 bits.
@@ -517,75 +540,616 @@ Values from `0` to `4` will be listed using `weight` instead of `nbBits`.
 Weight formula is : `weight = nbBits ? maxBits + 1 - nbBits : 0;`
 It gives the following serie of weights :
 
-| weight | 4 | 3 | 2 | 0 | 1 |
+| weights |  4  |  3  |  2  |  0  |  1  |
-| ------ | - | - | - | - | - |
+| ------- | --- | --- | --- | --- | --- |
-|  Value | 0 | 1 | 2 | 3 | 4 |
+| literal |  0  |  1  |  2  |  3  |  4  |
 
 The decoder will do the inverse operation :
-having collected weights of symbols from `0` to `4`,
+having collected weights of literals from `0` to `4`,
-it knows the last symbol, `5`, is present with a non-zero weight.
+it knows the last literal, `5`, is present with a non-zero weight.
-The weight of `5` can be deduced by joining to the nearest power of 2.
+The weight of `5` can be deducted by joining to the nearest power of 2.
 Sum of 2^(weight-1) (excluding 0) is :
-8 + 4 + 2 + 0 + 1 = 15
+`8 + 4 + 2 + 0 + 1 = 15`
 Nearest power of 2 is 16.
 Therefore, `maxBits = 4` and `weight[5] = 1`.
-It can then proceed to transform back weights into nbBits :
-`weight = nbBits ? maxBits + 1 - nbBits : 0;` .
 
 ##### Huffman Tree header
 
-This is a single byte value (0-255), which tells how to decode the tree.
+This is a single byte value (0-255),
+which tells how to decode the list of weights.
 
 - if headerByte >= 242 : this is one of 14 pre-defined weight distributions :
-  + 242 :  1x1 (+ 1x1)
+
-  + 243 :  2x1 (+ 1x2)
+| value    |242|243|244|245|246|247|248|249|250|251|252|253|254|255|
-  + 244 :  3x1 (+ 1x1)
+| -------- |---|---|---|---|---|---|---|---|---|---|---|---|---|---|
-  + 245 :  4x1 (+ 1x4)
+| Nb of 1s | 1 | 2 | 3 | 4 | 7 | 8 | 15| 16| 31| 32| 63| 64|127|128|
-  + 246 :  7x1 (+ 1x1)
+|Complement| 1 | 2 | 1 | 4 | 1 | 8 | 1 | 16| 1 | 32| 1 | 64| 1 |128|
-  + 247 :  8x1 (+ 1x8)
+
-  + 248 : 15x1 (+ 1x1)
+_Note_ : complement is found by using "join to nearest power of 2" rule.
-  + 249 : 16x1 (+ 1x16)
-  + 250 : 31x1 (+ 1x1)
-  + 251 : 32x1 (+ 1x32)
-  + 252 : 63x1 (+ 1x1)
-  + 253 : 64x1 (+ 1x64)
-  + 254 :127x1 (+ 1x1)
-  + 255 :128x1 (+ 1x128)
 
 - if headerByte >= 128 : this is a direct representation,
   where each weight is written directly as a 4 bits field (0-15).
-  The full representation occupies (nbSymbols+1/2) bytes,
+  The full representation occupies `((nbSymbols+1)/2)` bytes,
   meaning it uses a last full byte even if nbSymbols is odd.
-  `nbSymbols = headerByte - 127;`
+  `nbSymbols = headerByte - 127;`.
+  Note that maximum nbSymbols is 241-127 = 114.
+  A larger serie must necessarily use FSE compression.
 
 - if headerByte < 128 :
   the serie of weights is compressed by FSE.
-  The length of the compressed serie is `headerByte` (0-127).
+  The length of the FSE-compressed serie is `headerByte` (0-127).
 
 ##### FSE (Finite State Entropy) compression of huffman weights
 
-The serie of weights is compressed using standard FSE compression.
+The serie of weights is compressed using FSE compression.
 It's a single bitstream with 2 interleaved states,
-using a single distribution table.
+sharing a single distribution table.
 
 To decode an FSE bitstream, it is necessary to know its compressed size.
 Compressed size is provided by `headerByte`.
-It's also necessary to know its maximum decompressed size.
+It's also necessary to know its maximum decompressed size,
-In this case, it's `255`, since literal values range from `0` to `255`,
+which is `255`, since literal values span from `0` to `255`,
-and the last symbol value is not represented.
+and last symbol value is not represented.
 
 An FSE bitstream starts by a header, describing probabilities distribution.
-Result will create a Decoding Table.
+It will create a Decoding Table.
-It is necessary to know the maximum accuracy of distribution
+Table must be pre-allocated, which requires to support a maximum accuracy.
-to properly allocate space for the Table.
+For a list of huffman weights, recommended maximum is 7 bits.
-For a list of huffman weights, this maximum is 8 bits.
+
+FSE header is [described in relevant chapter](#fse-distribution-table--condensed-format),
+and so is [FSE bitstream](#bitstream).
+The main difference is that Huffman header compression uses 2 states,
+which share the same FSE distribution table.
+Bitstream contains only FSE symbols, there are no interleaved "raw bitfields".
+The number of symbols to decode is discovered
+by tracking bitStream overflow condition.
+When both states have overflowed the bitstream, end is reached.
 
-FSE header and bitstreams are described in a separated chapter.
 
 ##### Conversion from weights to huffman prefix codes
 
+All present symbols shall now have a `weight` value.
+A `weight` directly represents a `range` of prefix codes,
+following the formulae : `range = weight ? 1 << (weight-1) : 0 ;`
+Symbols are sorted by weight.
+Within same weight, symbols keep natural order.
+Starting from lowest weight,
+symbols are being allocated to a range of prefix codes.
+Symbols with a weight of zero are not present.
 
+It is then possible to transform weights into nbBits :
+`nbBits = nbBits ? maxBits + 1 - weight : 0;` .
+
+
+__Example__ :
+Let's presume the following huffman tree has been decoded :
+
+| Literal |  0  |  1  |  2  |  3  |  4  |  5  |
+| ------- | --- | --- | --- | --- | --- | --- |
+|  weight |  4  |  3  |  2  |  0  |  1  |  1  |
+
+Sorted by weight and then natural order,
+it gives the following distribution :
+
+| Literal      |  3  |  4  |  5  |  2  |  1  |   0  |
+| ------------ | --- | --- | --- | --- | --- | ---- |
+| weight       |  0  |  1  |  1  |  2  |  3  |   4  |
+| range        |  0  |  1  |  1  |  2  |  4  |   8  |
+| prefix codes | N/A |  0  |  1  | 2-3 | 4-7 | 8-15 |
+| nb bits      |  0  |  4  |  4  |  3  |  2  |   1  |
+
+
+#### Literals bitstreams
+
+##### Bitstreams sizes
+
+As seen in a previous paragraph,
+there are 2 flavors of huffman-compressed literals :
+single stream, and 4-streams.
+
+4-streams is useful for CPU with multiple execution units and OoO operations.
+Since each stream can be decoded independently,
+it's possible to decode them up to 4x faster than a single stream,
+presuming the CPU has enough parallelism available.
+
+For single stream, header provides both the compressed and regenerated size.
+For 4-streams though,
+header only provides compressed and regenerated size of all 4 streams combined.
+In order to properly decode the 4 streams,
+it's necessary to know the compressed and regenerated size of each stream.
+
+Regenerated size is easiest :
+each stream has a size of `(totalSize+3)/4`,
+except the last one, which is up to 3 bytes smaller, to reach `totalSize`.
+
+Compressed size must be provided explicitly : in the 4-streams variant,
+bitstreams are preceded by 3 unsigned Little Endian 16-bits values.
+Each value represents the compressed size of one stream, in order.
+The last stream size is deducted from total compressed size
+and from already known stream sizes :
+`stream4CSize = totalCSize - 6 - stream1CSize - stream2CSize - stream3CSize;`
+
+##### Bitstreams read and decode
+
+Each bitstream must be read _backward_,
+that is starting from the end down to the beginning.
+Therefore it's necessary to know the size of each bitstream.
+
+It's also necessary to know exactly which _bit_ is the latest.
+This is detected by a final bit flag :
+the highest bit of latest byte is a final-bit-flag.
+Consequently, a last byte of `0` is not possible.
+And the final-bit-flag itself is not part of the useful bitstream.
+Hence, the last byte contain between 0 and 7 useful bits.
+
+Starting from the end,
+it's possible to read the bitstream in a little-endian fashion,
+keeping track of already used bits.
+
+Reading the last `maxBits` bits,
+it's then possible to compare extracted value to the prefix codes table,
+determining the symbol to decode and number of bits to discard.
+
+The process continues up to reading the required number of symbols per stream.
+If a bitstream is not entirely and exactly consumed,
+hence reaching exactly its beginning position with all bits consumed,
+the decoding process is considered faulty.
+
+
+### Sequences section
+
+A compressed block is a succession of _sequences_ .
+A sequence is a literal copy command, followed by a match copy command.
+A literal copy command specifies a length.
+It is the number of bytes to be copied (or extracted) from the literal section.
+A match copy command specifies an offset and a length.
+The offset gives the position to copy from,
+which can stand within a previous block.
+
+There are 3 symbol types, `literalLength`, `matchLength` and `offset`,
+which are encoded together, interleaved in a single _bitstream_.
+
+Each symbol is a _code_ in its own context,
+which specifies a baseline and a number of bits to add.
+_Codes_ are FSE compressed,
+and interleaved with raw additional bits in the same bitstream.
+
+The Sequences section starts by a header,
+followed by optional Probability tables for each symbol type,
+followed by the bitstream.
+
+| Header | (LitLengthTable) | (OffsetTable) | (MatchLengthTable) | bitStream |
+| ------ | ---------------- | ------------- | ------------------ | --------- |
+
+To decode the Sequence section, it's required to know its size.
+This size is deducted from `blockSize - literalSectionSize`.
+
+
+#### Sequences section header
+
+Consists in 2 items :
+- Nb of Sequences
+- Flags providing Symbol compression types
+
+__Nb of Sequences__
+
+This is a variable size field, `nbSeqs`, using between 1 and 3 bytes.
+Let's call its first byte `byte0`.
+- `if (byte0 == 0)` : there are no sequences.
+            The sequence section stops there.
+            Regenerated content is defined entirely by literals section.
+- `if (byte0 < 128)` : `nbSeqs = byte0;` . Uses 1 byte.
+- `if (byte0 < 255)` : `nbSeqs = ((byte0-128) << 8) + byte1;` . Uses 2 bytes.
+- `if (byte0 == 255)`: `nbSeqs = byte1 + (byte2<<8) + 0x7F00;` . Uses 3 bytes.
+
+__Symbol compression modes__
+
+This is a single byte, defining the compression mode of each symbol type.
+
+|  BitNb  |   7-6  |   5-4  |   3-2  |    1-0   |
+| ------- | ------ | ------ | ------ | -------- |
+|FieldName| LLtype | OFType | MLType | Reserved |
+
+The last field, `Reserved`, must be all-zeroes.
+
+`LLtype`, `OFType` and `MLType` define the compression mode of
+Literal Lengths, Offsets and Match Lengths respectively.
+
+They follow the same enumeration :
+
+|       Value      |    0   |  1  |    2   |  3  |
+| ---------------- | ------ | --- | ------ | --- |
+| Compression Mode | predef | RLE | Repeat | FSE |
+
+- "predef" : uses a pre-defined distribution table.
+- "RLE" : it's a single code, repeated `nbSeqs` times.
+- "Repeat" : re-use distribution table from previous compressed block.
+- "FSE" : standard FSE compression.
+          A distribution table will be present.
+          It will be described in [next part](#distribution-tables).
+
+#### Symbols decoding
+
+##### Literal Lengths codes
+
+Literal lengths codes are values ranging from `0` to `35` included.
+They define lengths from 0 to 131071 bytes.
+
+|  Code  | 0-15 |
+| ------ | ---- |
+| length | Code |
+| nbBits |   0  |
+
+
+|   Code   |  16  |  17  |  18  |  19  |  20  |  21  |  22  |  23  |
+| -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
+| Baseline |  16  |  18  |  20  |  22  |  24  |  28  |  32  |  40  |
+| nb Bits  |   1  |   1  |   1  |   1  |   2  |   2  |   3  |   3  |
+
+|   Code   |  24  |  25  |  26  |  27  |  28  |  29  |  30  |  31  |
+| -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
+| Baseline |  48  |  64  |  128 |  256 |  512 | 1024 | 2048 | 4096 |
+| nb Bits  |   4  |   6  |   7  |   8  |   9  |  10  |  11  |  12  |
+
+|   Code   |  32  |  33  |  34  |  35  |
+| -------- | ---- | ---- | ---- | ---- |
+| Baseline | 8192 |16384 |32768 |65536 |
+| nb Bits  |  13  |  14  |  15  |  16  |
+
+__Default distribution__
+
+When "compression mode" is "predef"",
+a pre-defined distribution is used for FSE compression.
+
+Below is its definition. It uses an accuracy of 6 bits (64 states).
+```
+short literalLengths_defaultDistribution[36] =
+        { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
+          2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
+         -1,-1,-1,-1 };
+```
+
+##### Match Lengths codes
+
+Match lengths codes are values ranging from `0` to `52` included.
+They define lengths from 3 to 131074 bytes.
+
+|  Code  |   0-31   |
+| ------ | -------- |
+| value  | Code + 3 |
+| nbBits |     0    |
+
+|   Code   |  32  |  33  |  34  |  35  |  36  |  37  |  38  |  39  |
+| -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
+| Baseline |  35  |  37  |  39  |  41  |  43  |  47  |  51  |  59  |
+| nb Bits  |   1  |   1  |   1  |   1  |   2  |   2  |   3  |   3  |
+
+|   Code   |  40  |  41  |  42  |  43  |  44  |  45  |  46  |  47  |
+| -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
+| Baseline |  67  |  83  |  99  |  131 |  258 |  514 | 1026 | 2050 |
+| nb Bits  |   4  |   4  |   5  |   7  |   8  |   9  |  10  |  11  |
+
+|   Code   |  48  |  49  |  50  |  51  |  52  |
+| -------- | ---- | ---- | ---- | ---- | ---- |
+| Baseline | 4098 | 8194 |16486 |32770 |65538 |
+| nb Bits  |  12  |  13  |  14  |  15  |  16  |
+
+__Default distribution__
+
+When "compression mode" is defined as "predef",
+a pre-defined distribution is used for FSE compression.
+
+Here is its definition. It uses an accuracy of 6 bits (64 states).
+```
+short matchLengths_defaultDistribution[53] =
+        { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,
+         -1,-1,-1,-1,-1 };
+```
+
+##### Offset codes
+
+Offset codes are values ranging from `0` to `N`,
+with `N` being limited by maximum backreference distance.
+
+A decoder is free to limit its maximum `N` supported.
+Recommendation is to support at least up to `22`.
+For information, at the time of this writing.
+the reference decoder supports a maximum `N` value of `28` in 64-bits mode.
+
+An offset code is also the nb of additional bits to read,
+and can be translated into an `OFValue` using the following formulae :
+
+```
+OFValue = (1 << offsetCode) + readNBits(offsetCode);
+if (OFValue > 3) offset = OFValue - 3;
+```
+
+OFValue from 1 to 3 are special : they define "repeat codes",
+which means one of the previous offsets will be repeated.
+They are sorted in recency order, with 1 meaning the most recent one.
+See [Repeat offsets](#repeat-offsets) paragraph.
+
+__Default distribution__
+
+When "compression mode" is defined as "predef",
+a pre-defined distribution is used for FSE compression.
+
+Here is its definition. It uses an accuracy of 5 bits (32 states),
+and supports a maximum `N` of 28, allowing offset values up to 536,870,908 .
+
+If any sequence in the compressed block requires an offset larger than this,
+it's not possible to use the default distribution to represent it.
+
+```
+short offsetCodes_defaultDistribution[53] =
+        { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
+          1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 };
+```
+
+#### Distribution tables
+
+Following the header, up to 3 distribution tables can be described.
+They are, in order :
+- Literal lengthes
+- Offsets
+- Match Lengthes
+
+The content to decode depends on their respective compression mode :
+- Repeat mode : no content. Re-use distribution from previous compressed block.
+- Predef : no content. Use pre-defined distribution table.
+- RLE : 1 byte. This is the only code to use across the whole compressed block.
+- FSE : A distribution table is present.
+
+##### FSE distribution table : condensed format
+
+An FSE distribution table describes the probabilities of all symbols
+from `0` to the last present one (included)
+on a normalized scale of `1 << AccuracyLog` .
+
+It's a bitstream which is read forward, in little-endian fashion.
+It's not necessary to know its exact size,
+since it will be discovered and reported by the decoding process.
+
+The bitstream starts by reporting on which scale it operates.
+`AccuracyLog = low4bits + 5;`
+In theory, it can define a scale from 5 to 20.
+In practice, decoders are allowed to limit the maximum supported `AccuracyLog`.
+Recommended maximum are `9` for literal and match lengthes, and `8` for offsets.
+The reference decoder uses these limits.
+
+Then follow each symbol value, from `0` to last present one.
+The nb of bits used by each field is variable.
+It depends on :
+
+- Remaining probabilities + 1 :
+  __example__ :
+  Presuming an AccuracyLog of 8,
+  and presuming 100 probabilities points have already been distributed,
+  the decoder may read any value from `0` to `255 - 100 + 1 == 156` (included).
+  Therefore, it must read `log2sup(156) == 8` bits.
+
+- Value decoded : small values use 1 less bit :
+  __example__ :
+  Presuming values from 0 to 156 (included) are possible,
+  255-156 = 99 values are remaining in an 8-bits field.
+  They are used this way :
+  first 99 values (hence from 0 to 98) use only 7 bits,
+  values from 99 to 156 use 8 bits.
+  This is achieved through this scheme :
+
+  | Value read | Value decoded | nb Bits used |
+  | ---------- | ------------- | ------------ |
+  |   0 -  98  |   0 -  98     |  7           |
+  |  99 - 127  |  99 - 127     |  8           |
+  | 128 - 226  |   0 -  98     |  7           |
+  | 227 - 255  | 128 - 156     |  8           |
+
+Symbols probabilities are read one by one, in order.
+
+Probability is obtained from Value decoded by following formulae :
+`Proba = value - 1;`
+
+It means value `0` becomes negative probability `-1`.
+`-1` is a special probability, which means `less than 1`.
+Its effect on distribution table is described in [next paragraph].
+For the purpose of calculating cumulated distribution, it counts as one.
+
+[next paragraph]:#fse-decoding--from-normalized-distribution-to-decoding-tables
+
+When a symbol has a probability of `zero`,
+it is followed by a 2-bits repeat flag.
+This repeat flag tells how many probabilities of zeroes follow the current one.
+It provides a number ranging from 0 to 3.
+If it is a 3, another 2-bits repeat flag follows, and so on.
+
+When last symbol reaches cumulated total of `1 << AccuracyLog`,
+decoding is complete.
+Then the decoder can tell how many bytes were used in this process,
+and how many symbols are present.
+
+The bitstream consumes a round number of bytes.
+Any remaining bit within the last byte is just unused.
+
+If the last symbol makes cumulated total go above `1 << AccuracyLog`,
+distribution is considered corrupted.
+
+##### FSE decoding : from normalized distribution to decoding tables
+
+The distribution of normalized probabilities is enough
+to create a unique decoding table.
+
+It follows the following build rule :
+
+The table has a size of `tableSize = 1 << AccuracyLog;`.
+Each cell describes the symbol decoded,
+and instructions to get the next state.
+
+Symbols are scanned in their natural order for `less than 1` probabilities.
+Symbols with this probability are being attributed a single cell,
+starting from the end of the table.
+These symbols define a full state reset, reading `AccuracyLog` bits.
+
+All remaining symbols are sorted in their natural order.
+Starting from symbol `0` and table position `0`,
+each symbol gets attributed as many cells as its probability.
+Cell allocation is spreaded, not linear :
+each successor position follow this rule :
+
+```
+position += (tableSize>>1) + (tableSize>>3) + 3;
+position &= tableSize-1;
+```
+
+A position is skipped if already occupied,
+typically by a "less than 1" probability symbol.
+
+The result is a list of state values.
+Each state will decode the current symbol.
+
+To get the Number of bits and baseline required for next state,
+it's first necessary to sort all states in their natural order.
+The lower states will need 1 more bit than higher ones.
+
+__Example__ :
+Presuming a symbol has a probability of 5.
+It receives 5 state values. States are sorted in natural order.
+
+Next power of 2 is 8.
+Space of probabilities is divided into 8 equal parts.
+Presuming the AccuracyLog is 7, it defines 128 states.
+Divided by 8, each share is 16 large.
+
+In order to reach 8, 8-5=3 lowest states will count "double",
+taking shares twice larger,
+requiring one more bit in the process.
+
+Numbering starts from higher states using less bits.
+
+| state order |   0   |   1   |    2   |   3  |   4   |
+| ----------- | ----- | ----- | ------ | ---- | ----- |
+| width       |  32   |  32   |   32   |  16  |  16   |
+| nb Bits     |   5   |   5   |    5   |   4  |   4   |
+| range nb    |   2   |   4   |    6   |   0  |   1   |
+| baseline    |  32   |  64   |   96   |   0  |  16   |
+| range       | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 |
+
+Next state is determined from current state
+by reading the required number of bits, and adding the specified baseline.
+
+
+#### Bitstream
+
+All sequences are stored in a single bitstream, read _backward_.
+It is therefore necessary to know the bitstream size,
+which is deducted from compressed block size.
+
+The last useful bit of the stream is followed by an end-bit-flag.
+Highest bit of last byte is this flag.
+It does not belong to the useful part of the bitstream.
+Therefore, last byte has 0-7 useful bits.
+Note that it also means that last byte cannot be `0`.
+
+##### Starting states
+
+The bitstream starts with initial state values,
+each using the required number of bits in their respective _accuracy_,
+decoded previously from their normalized distribution.
+
+It starts by `Literal Length State`,
+followed by `Offset State`,
+and finally `Match Length State`.
+
+Reminder : always keep in mind that all values are read _backward_.
+
+##### Decoding a sequence
+
+A state gives a code.
+A code provides a baseline and number of bits to add.
+See [Symbol Decoding] section for details on each symbol.
+
+Decoding starts by reading the nb of bits required to decode offset.
+It then does the same for match length,
+and then for literal length.
+
+Offset / matchLength / litLength define a sequence.
+It starts by inserting the number of literals defined by `litLength`,
+then continue by copying `matchLength` bytes from `currentPos - offset`.
+
+The next operation is to update states.
+Using rules pre-calculated in the decoding tables,
+`Literal Length State` is updated,
+followed by `Match Length State`,
+and then `Offset State`.
+
+This operation will be repeated `NbSeqs` times.
+At the end, the bitstream shall be entirely consumed,
+otherwise bitstream is considered corrupted.
+
+[Symbol Decoding]:#symbols-decoding
+
+##### Repeat offsets
+
+As seen in [Offset Codes], the first 3 values define a repeated offset.
+They are sorted in recency order, with 1 meaning "most recent one".
+
+There is an exception though, when current sequence's literal length is `0`.
+In which case, 1 would just make previous match longer.
+Therefore, in such case, 1 means in fact 2, and 2 is impossible.
+Meaning of 3 is unmodified.
+
+Repeat offsets start with the following values : 1, 4 and 8 (in order).
+
+Then each block receives its start value from previous compressed block.
+Note that non-compressed blocks are skipped,
+they do not contribute to offset history.
+
+[Offset Codes]: #offset-codes
+
+###### Offset updates rules
+
+When the new offset is a normal one,
+offset history is simply translated by one position,
+with the new offset taking first spot.
+
+- When repeat offset 1 (most recent) is used, history is unmodified.
+- When repeat offset 2 is used, it's swapped with offset 1.
+- When repeat offset 3 is used, it takes first spot,
+  pushing the other ones by one position.
+
+
+Dictionary format
+-----------------
+
+`zstd` is compatible with "pure content" dictionaries, free of any format restriction.
+But dictionaries created by `zstd --train` follow a format, described here.
+
+__Pre-requisites__ : a dictionary has a known length,
+                     defined either by a buffer limit, or a file size.
+
+| Header | DictID | Stats | Content |
+| ------ | ------ | ----- | ------- |
+
+__Header__ : 4 bytes ID, value 0xEC30A437, Little Endian format
+
+__Dict_ID__ : 4 bytes, stored in Little Endian format.
+              DictID can be any value, except 0 (which means no DictID).
+              It's used by decoders to check if they use the correct dictionary.
+
+__Stats__ : Entropy tables, following the same format as a [compressed blocks].
+            They are stored in following order :
+            Huffman tables for literals, FSE table for offset,
+            FSE table for matchLenth, and FSE table for litLength.
+            It's finally followed by 3 offset values, populating recent offsets,
+            stored in order, 4-bytes little endian each, for a total of 12 bytes.
+
+__Content__ : Where the actual dictionary content is.
+              Content size depends on Dictionary size.
+
+[compressed blocks]: #compressed-block-format
 
 
 Version changes
 ---------------
+0.1.0 initial release