From fb5c59fc89a4fd1ed0d98f0558008dc87cdcdfa8 Mon Sep 17 00:00:00 2001 From: Juan Francisco Cantero Hurtado Date: Sun, 3 Jul 2016 02:20:39 +0200 Subject: [PATCH 01/33] Redundant entry for options in the man page. ".SH OPTIONS" is enough. --- programs/zstd.1 | 1 - 1 file changed, 1 deletion(-) diff --git a/programs/zstd.1 b/programs/zstd.1 index 56be346ca..bba6fa122 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -39,7 +39,6 @@ It also features a very fast decoder, with speed > 500 MB/s per core. Use \fB-q\fR to turn them off -\fBzstd\fR supports the following options : .SH OPTIONS .TP From 06ad6f19115a2440e1904d1e187bc97ac830c7f1 Mon Sep 17 00:00:00 2001 From: Juan Francisco Cantero Hurtado Date: Sun, 3 Jul 2016 02:22:31 +0200 Subject: [PATCH 02/33] Add OpenBSD to the Makefile test. --- programs/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index 52a7ca076..a55268a01 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -154,10 +154,10 @@ clean: @echo Cleaning completed -#------------------------------------------------------------------------ -#make install is validated only for Linux, OSX, kFreeBSD and Hurd targets -#------------------------------------------------------------------------ -ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU)) +#--------------------------------------------------------------------------------- +#make install is validated only for Linux, OSX, kFreeBSD, Hurd and OpenBSD targets +#--------------------------------------------------------------------------------- +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD)) HOST_OS = POSIX install: zstd @echo Installing binaries From d916c908e031dc5c911b4188886deb9f1ccf43b5 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 4 Jul 2016 00:42:58 +0200 Subject: [PATCH 03/33] updated doc --- NEWS | 3 + lib/common/zstd.h | 2 +- programs/zstdcli.c | 19 +++---- zstd_compression_format.md | 113 ++++++++++++++++++++++++++++++++----- 4 files changed, 113 insertions(+), 24 deletions(-) diff --git a/NEWS b/NEWS index f1a8eff84..5d6d0a869 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +v0.7.3 +added : OpenBSD target, by Juan Francisco Cantero Hurtado + v0.7.2 fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner. diff --git a/lib/common/zstd.h b/lib/common/zstd.h index de2ec1eb9..3dcd533db 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -61,7 +61,7 @@ extern "C" { ***************************************/ #define ZSTD_VERSION_MAJOR 0 #define ZSTD_VERSION_MINOR 7 -#define ZSTD_VERSION_RELEASE 2 +#define ZSTD_VERSION_RELEASE 3 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str diff --git a/programs/zstdcli.c b/programs/zstdcli.c index c63655c43..d1693a3f8 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -31,7 +31,7 @@ /*-************************************ * Includes **************************************/ -#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */ +#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST, errno */ #include /* strcmp, strlen */ #include /* toupper */ #include "fileio.h" @@ -45,7 +45,6 @@ #include "zstd.h" /* ZSTD_VERSION_STRING */ - /*-************************************ * OS-specific Includes **************************************/ @@ -53,12 +52,12 @@ # include /* _isatty */ # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) #else -#if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) -# include /* isatty */ -# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) -#else -# define IS_CONSOLE(stdStream) 0 -#endif +# if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) +# include /* isatty */ +# define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +# else +# define IS_CONSOLE(stdStream) 0 +# endif #endif @@ -116,6 +115,7 @@ static int usage(const char* programName) DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n"); DISPLAY( " -f : overwrite output without prompting \n"); DISPLAY( "--rm : remove source file(s) after successful de/compression \n"); + DISPLAY( " -k : preserve source file(s) (default) \n"); DISPLAY( " -h/-H : display help/long help and exit\n"); return 0; } @@ -169,7 +169,6 @@ static int badusage(const char* programName) return 1; } - static void waitEnter(void) { int unused; @@ -229,7 +228,7 @@ int main(int argCount, const char** argv) (void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */ (void)dictCLevel; (void)dictSelect; (void)dictID; /* not used when ZSTD_NODICT set */ (void)decode; (void)cLevel; /* not used when ZSTD_NOCOMPRESS set */ - if (filenameTable==NULL) { DISPLAY("not enough memory\n"); exit(1); } + if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); } filenameTable[0] = stdinmark; displayOut = stderr; /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */ diff --git a/zstd_compression_format.md b/zstd_compression_format.md index c05c237cc..7816d1cb9 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -16,7 +16,7 @@ Distribution of this document is unlimited. ### Version -0.0.1 (30/06/2016 - Work in progress - unfinished) +0.0.2 (July 2016 - Work in progress - unfinished) Introduction @@ -464,7 +464,7 @@ __Sizes format for Raw or RLE block__ : Total literal header size is 2 bytes. `size = ((h[0] & 15) << 8) + h[1];` - Value : 11 : Regenerated size uses 20 bits (0-1048575). - Total literal header size is 2 bytes. + Total literal header size is 3 bytes. `size = ((h[0] & 15) << 16) + (h[1]<<8) + h[2];` Note : it's allowed to represent a short value (ex : `13`) @@ -507,9 +507,9 @@ This power of 2 gives `maxBits`, the depth of the current tree. __Example__ : Let's presume the following huffman tree must be described : -| Value | 0 | 1 | 2 | 3 | 4 | 5 | -| ------ | - | - | - | - | - | - | -| nbBits | 1 | 2 | 3 | 0 | 4 | 4 | +| literal | 0 | 1 | 2 | 3 | 4 | 5 | +| ------- | --- | --- | --- | --- | --- | --- | +| nbBits | 1 | 2 | 3 | 0 | 4 | 4 | The tree depth is 4, since its smallest element uses 4 bits. Value `5` will not be listed, nor will values above `5`. @@ -517,20 +517,18 @@ Values from `0` to `4` will be listed using `weight` instead of `nbBits`. Weight formula is : `weight = nbBits ? maxBits + 1 - nbBits : 0;` It gives the following serie of weights : -| weight | 4 | 3 | 2 | 0 | 1 | -| ------ | - | - | - | - | - | -| Value | 0 | 1 | 2 | 3 | 4 | +| weights | 4 | 3 | 2 | 0 | 1 | +| ------- | --- | --- | --- | --- | --- | +| literal | 0 | 1 | 2 | 3 | 4 | The decoder will do the inverse operation : -having collected weights of symbols from `0` to `4`, -it knows the last symbol, `5`, is present with a non-zero weight. +having collected weights of literals from `0` to `4`, +it knows the last literal, `5`, is present with a non-zero weight. The weight of `5` can be deduced by joining to the nearest power of 2. Sum of 2^(weight-1) (excluding 0) is : -8 + 4 + 2 + 0 + 1 = 15 +`8 + 4 + 2 + 0 + 1 = 15` Nearest power of 2 is 16. Therefore, `maxBits = 4` and `weight[5] = 1`. -It can then proceed to transform back weights into nbBits : -`weight = nbBits ? maxBits + 1 - nbBits : 0;` . ##### Huffman Tree header @@ -584,6 +582,95 @@ FSE header and bitstreams are described in a separated chapter. ##### Conversion from weights to huffman prefix codes +All present symbols shall now have a `weight` value. +A `weight` directly represent a `range` of prefix codes, +following the formulae : `range = weight ? 1 << (weight-1) : 0 ;` +Symbols are sorted by weight. +Within same weight, symbols keep natural order. +Starting from lowest weight, +symbols are being allocated to a range of prefix codes. +Symbols with a weight of zero are not present. + +It can then proceed to transform weights into nbBits : +`nbBits = nbBits ? maxBits + 1 - weight : 0;` . + + +__Example__ : +Let's presume the following huffman tree has been decoded : + +| Literal | 0 | 1 | 2 | 3 | 4 | 5 | +| ------- | --- | --- | --- | --- | --- | --- | +| weight | 4 | 3 | 2 | 0 | 1 | 1 | + +Sorted by weight and then natural order, +it gives the following distribution : + +| Literal | 3 | 4 | 5 | 2 | 1 | 0 | +| ------------ | --- | --- | --- | --- | --- | ---- | +| weight | 0 | 1 | 1 | 2 | 3 | 4 | +| range | 0 | 1 | 1 | 2 | 4 | 8 | +| prefix codes | N/A | 0 | 1 | 2-3 | 4-7 | 8-15 | +| nb bits | 0 | 4 | 4 | 3 | 2 | 1 | + + + +#### Literals bitstreams + +##### Bitstreams sizes + +As seen in a previous paragraph, +there are 2 flavors of huffman-compressed literals : +single stream, and 4-streams. + +4-streams is useful for CPU with multiple execution units and OoO operations. +Since each stream can be decoded independently, +it's possible to decode them up to 4x faster than a single stream, +presuming the CPU has enough parallelism available. + +For single stream, header provides both the compressed and regenerated size. +For 4-streams though, +header only provides compressed and regenerated size of all 4 streams combined. + +In order to properly decode the 4 streams, +it's necessary to know the compressed and regenerated size of each stream. + +Regenerated size is easiest : +each stream has a size of `(totalSize+3)/4`, +except the last one, which is up to 3 bytes smaller, to reach totalSize. + +Compressed size must be provided explicitly : in the 4-streams variant, +bitstream is preceded by 3 unsigned short values using Little Endian convention. +Each value represent the compressed size of one stream, in order. +The last stream size is deducted from total compressed size +and from already known stream sizes : +`stream4CSize = totalCSize - 6 - stream1CSize - stream2CSize - stream3CSize;` + +##### Bitstreams reading + +Each bitstream must be read _backward_, +that is starting from the end down to the beginning. +Therefore it's necessary to know the size of each bitstream. + +It's also necessary to know exactly which _bit_ is the latest. +This is detected by a final bit flag : +the highest bit of latest byte is a final-bit-flag. +Consequently, a last byte of `0` is not possible. +And the final-bit-flag itself is not part of the useful bitstream. +Hence, the last byte contain between 0 and 7 useful bits. + +Starting from the end, +it's possible to read the bitstream in a little-endian fashion, +keeping track of already used bits. + +Extracting `maxBits`, +it's then possible to compare extracted value to the prefix codes table, +determining the symbol to decode and number of bits to discard. + +The process continues up to reading the required number of symbols per stream. +If a bitstream is not entirely and exactly consumed, +hence reaching exactly its beginning position with all bits consumed, +the decoding process is considered faulty. + From 00d44abe718cf7f9803517732ed4831a1891d438 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 4 Jul 2016 01:29:47 +0200 Subject: [PATCH 04/33] updated doc --- NEWS | 2 +- zstd_compression_format.md | 83 +++++++++++++++++++++++++++++--------- 2 files changed, 66 insertions(+), 19 deletions(-) diff --git a/NEWS b/NEWS index 5d6d0a869..bb520df7d 100644 --- a/NEWS +++ b/NEWS @@ -2,7 +2,7 @@ v0.7.3 added : OpenBSD target, by Juan Francisco Cantero Hurtado v0.7.2 -fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski +fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski. fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner. fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner. diff --git a/zstd_compression_format.md b/zstd_compression_format.md index 7816d1cb9..a01305019 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -386,7 +386,7 @@ User Data can be anything. Data will just be skipped by the decoder. Compressed block format ----------------------- This specification details the content of a _compressed block_. -A compressed block has a size, which must be known in order to decode it. +A compressed block has a size, which must be known. It also has a guaranteed maximum regenerated size, in order to properly allocate destination buffer. See "Frame format" for more details. @@ -396,19 +396,21 @@ A compressed block consists of 2 sections : - Sequences section ### Prerequisite -For proper decoding, a compressed block requires access to following elements : +To decode a compressed block, it's required to access to following elements : - Previous decoded blocks, up to a distance of `windowSize`, - or all previous blocks in the same frame "single segment" mode. + or all frame's previous blocks in "single segment" mode. - List of "recent offsets" from previous compressed block. +- Decoding tables of previous compressed block for each symbol type + (literals, litLength, matchLength, offset) -### Compressed Literals +### Literals section Literals are compressed using order-0 huffman compression. During sequence phase, literals will be entangled with match copy operations. All literals are regrouped in the first part of the block. They can be decoded first, and then copied during sequence operations, -or they can be decoded on the flow, as needed by sequences. +or they can be decoded on the flow, as needed by sequence commands. | Header | (Tree Description) | Stream1 | (Stream2) | (Stream3) | (Stream4) | | ------ | ------------------ | ------- | --------- | --------- | --------- | @@ -417,9 +419,9 @@ Literals can be compressed, or uncompressed. When compressed, an optional tree description can be present, followed by 1 or 4 streams. -#### Block Literal Header +#### Literals section header -Header is in charge of describing precisely how literals are packed. +Header is in charge of describing how literals are packed. It's a byte-aligned variable-size bitfield, ranging from 1 to 5 bytes, using big-endian convention. @@ -491,12 +493,31 @@ Compressed and regenerated size fields follow big endian convention. #### Huffman Tree description This section is only present when block type is _compressed_ (`0`). -It describes the different leaf nodes of the huffman tree, -and their relative weights. + +Prefix coding represents symbols from an a priori known +alphabet by bit sequences (codes), one code for each symbol, in +a manner such that different symbols may be represented by bit +sequences of different lengths, but a parser can always parse +an encoded string unambiguously symbol-by-symbol. + +Given an alphabet with known symbol frequencies, the Huffman +algorithm allows the construction of an optimal prefix code +(one which represents strings with those symbol frequencies +using the fewest bits of any possible prefix codes for that +alphabet). Such a code is called a Huffman code. + +Huffman code must not exceed a maximum code length. +More bits improve accuracy but cost more header size, +and requires more memory for decoding operations. + +The current format limits the maximum depth to 15 bits. +The reference decoder goes further, by limiting it to 11 bits. +It is recommended to remain compatible with reference decoder. + ##### Representation -All byte values from zero (included) to last present one (excluded) +All literal values from zero (included) to last present one (excluded) are represented by `weight` values, from 0 to `maxBits`. Transformation from `weight` to `nbBits` follows this formulae : `nbBits = weight ? maxBits + 1 - weight : 0;` . @@ -552,7 +573,7 @@ This is a single byte value (0-255), which tells how to decode the tree. - if headerByte >= 128 : this is a direct representation, where each weight is written directly as a 4 bits field (0-15). - The full representation occupies (nbSymbols+1/2) bytes, + The full representation occupies ((nbSymbols+1)/2) bytes, meaning it uses a last full byte even if nbSymbols is odd. `nbSymbols = headerByte - 127;` @@ -573,7 +594,7 @@ In this case, it's `255`, since literal values range from `0` to `255`, and the last symbol value is not represented. An FSE bitstream starts by a header, describing probabilities distribution. -Result will create a Decoding Table. +It will create a Decoding Table. It is necessary to know the maximum accuracy of distribution to properly allocate space for the Table. For a list of huffman weights, this maximum is 8 bits. @@ -583,7 +604,7 @@ FSE header and bitstreams are described in a separated chapter. ##### Conversion from weights to huffman prefix codes All present symbols shall now have a `weight` value. -A `weight` directly represent a `range` of prefix codes, +A `weight` directly represents a `range` of prefix codes, following the formulae : `range = weight ? 1 << (weight-1) : 0 ;` Symbols are sorted by weight. Within same weight, symbols keep natural order. @@ -591,7 +612,7 @@ Starting from lowest weight, symbols are being allocated to a range of prefix codes. Symbols with a weight of zero are not present. -It can then proceed to transform weights into nbBits : +It is then possible to transform weights into nbBits : `nbBits = nbBits ? maxBits + 1 - weight : 0;` . @@ -639,13 +660,13 @@ each stream has a size of `(totalSize+3)/4`, except the last one, which is up to 3 bytes smaller, to reach totalSize. Compressed size must be provided explicitly : in the 4-streams variant, -bitstream is preceded by 3 unsigned short values using Little Endian convention. -Each value represent the compressed size of one stream, in order. +bitstreams are preceded by 3 unsigned Little Endian 16-bits values. +Each value represents the compressed size of one stream, in order. The last stream size is deducted from total compressed size and from already known stream sizes : `stream4CSize = totalCSize - 6 - stream1CSize - stream2CSize - stream3CSize;` -##### Bitstreams reading +##### Bitstreams read and decode Each bitstream must be read _backward_, that is starting from the end down to the beginning. @@ -662,7 +683,7 @@ Starting from the end, it's possible to read the bitstream in a little-endian fashion, keeping track of already used bits. -Extracting `maxBits`, +Reading the last `maxBits` bits, it's then possible to compare extracted value to the prefix codes table, determining the symbol to decode and number of bits to discard. @@ -672,6 +693,32 @@ hence reaching exactly its beginning position with all bits consumed, the decoding process is considered faulty. +### Sequences section + +A compressed block is a succession of _sequences_ . +A sequence is a literal copy command, followed by a match copy command. +A literal copy command specifies a length. +It is the number of bytes to be copied (or extracted) from the literal section. +A match copy command specifies an offset and a length. +The offset gives the position to copy from, +which can stand within a previous block. + +These are 3 symbol types, `literalLength`, `matchLength` and `offset`, +which are encoded together, interleaved in a single _bitstream_. + +Each symbol decoding consists of a _code_, +which specifies a baseline and a number of additional bits. +_Codes_ are FSE compressed, +and interleaved with raw additional bits in the same bitstream. + +The Sequence section starts by a header, +followed by an optional Probability table for each symbol type, +followed by the bitstream. + +#### Sequences section header + + + Version changes From 92c986b4e84f9ee58ab689c1ee1e67d4356af7c0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 4 Jul 2016 01:37:30 +0200 Subject: [PATCH 05/33] fixed cmake error (missing errno) --- programs/zstdcli.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index d1693a3f8..129936e15 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -31,9 +31,10 @@ /*-************************************ * Includes **************************************/ -#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST, errno */ +#include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */ #include /* strcmp, strlen */ #include /* toupper */ +#include #include "fileio.h" #ifndef ZSTD_NOBENCH # include "bench.h" /* BMK_benchFiles, BMK_SetNbIterations */ From 23f05ccc6bfb1dd72851495aceebfae6da08654a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 4 Jul 2016 16:13:11 +0200 Subject: [PATCH 06/33] updated specifications --- lib/decompress/zstd_decompress.c | 22 +-- zstd_compression_format.md | 323 +++++++++++++++++++++++++++---- 2 files changed, 298 insertions(+), 47 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 001a19ae8..228205824 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -195,7 +195,7 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) /* Frame format description Frame Header - [ Block Header - Block ] - Frame End 1) Frame Header - - 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h) + - 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd.h) - 1 byte - Frame Descriptor 2) Block Header - 3 bytes, starting with a 2-bits descriptor @@ -629,7 +629,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr, /* FSE table descriptors */ { U32 const LLtype = *ip >> 6; - U32 const Offtype = (*ip >> 4) & 3; + U32 const OFtype = (*ip >> 4) & 3; U32 const MLtype = (*ip >> 2) & 3; ip++; @@ -637,17 +637,17 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr, if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */ /* Build DTables */ - { size_t const bhSize = ZSTD_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable); - if (ZSTD_isError(bhSize)) return ERROR(corruption_detected); - ip += bhSize; + { size_t const llhSize = ZSTD_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable); + if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); + ip += llhSize; } - { size_t const bhSize = ZSTD_buildSeqTable(DTableOffb, Offtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable); - if (ZSTD_isError(bhSize)) return ERROR(corruption_detected); - ip += bhSize; + { size_t const ofhSize = ZSTD_buildSeqTable(DTableOffb, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable); + if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); + ip += ofhSize; } - { size_t const bhSize = ZSTD_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable); - if (ZSTD_isError(bhSize)) return ERROR(corruption_detected); - ip += bhSize; + { size_t const mlhSize = ZSTD_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable); + if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); + ip += mlhSize; } } return ip-istart; diff --git a/zstd_compression_format.md b/zstd_compression_format.md index a01305019..dbadac758 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -134,9 +134,9 @@ delivering the final decompressed result as if it was a single content. Frame Header ------------- -| FHD | (WD) | (Content Size) | (dictID) | -| ------- | --------- |:--------------:| --------- | -| 1 byte | 0-1 byte | 0 - 8 bytes | 0-4 bytes | +| FHD | (WD) | (dictID) | (Content Size) | +| ------- | --------- | --------- |:--------------:| +| 1 byte | 0-1 byte | 0-4 bytes | 0 - 8 bytes | Frame header has a variable size, which uses a minimum of 2 bytes, and up to 14 bytes depending on optional parameters. @@ -145,11 +145,11 @@ __FHD byte__ (Frame Header Descriptor) The first Header's byte is called the Frame Header Descriptor. It tells which other fields are present. -Decoding this byte is enough to get the full size of the Frame Header. +Decoding this byte is enough to tell the size of Frame Header. -| BitNb | 7-6 | 5 | 4 | 3 | 2 | 1-0 | -| ------- | ------ | ------- | ------ | -------- | -------- | -------- | -|FieldName| FCSize | Segment | Unused | Reserved | Checksum | dictID | +| BitNb | 7-6 | 5 | 4 | 3 | 2 | 1-0 | +| ------- | ------ | ------- | ------ | -------- | -------- | ------ | +|FieldName| FCSize | Segment | Unused | Reserved | Checksum | dictID | In this table, bit 7 is highest bit, while bit 0 is lowest. @@ -162,28 +162,28 @@ specifying if decompressed data size is provided within the header. | ------- | --- | --- | --- | --- | |FieldSize| 0-1 | 2 | 4 | 8 | -Value 0 has a double meaning : +Value 0 meaning depends on _single segment_ mode : it either means `0` (size not provided) _if_ the `WD` byte is present, -or it means `1` byte (size <= 255 bytes). +or `1` (frame content size <= 255 bytes) otherwise. __Single Segment__ If this flag is set, data shall be regenerated within a single continuous memory segment. + In which case, `WD` byte __is not present__, but `Frame Content Size` field necessarily is. - As a consequence, the decoder must allocate a memory segment of size `>= Frame Content Size`. In order to preserve the decoder from unreasonable memory requirement, -a decoder can refuse a compressed frame +a decoder can reject a compressed frame which requests a memory size beyond decoder's authorized range. For broader compatibility, decoders are recommended to support -memory sizes of 8 MB at least. -However, this is merely a recommendation, -and each decoder is free to support higher or lower limits, +memory sizes of at least 8 MB. +This is just a recommendation, +as each decoder is free to support higher or lower limits, depending on local limitations. __Unused bit__ @@ -254,6 +254,21 @@ It's merely a recommendation though, decoders are free to support larger or lower limits, depending on local limitations. +__Dictionary ID__ + +This is a variable size field, which contains an ID. +It checks if the correct dictionary is used for decoding. +Note that this field is optional. If it's not present, +it's up to the caller to make sure it uses the correct dictionary. + +Field size depends on __Dictionary ID flag__. +1 byte can represent an ID 0-255. +2 bytes can represent an ID 0-65535. +4 bytes can represent an ID 0-(2^32-1). + +It's allowed to represent a small ID (for example `13`) +with a large 4-bytes dictionary ID, losing some efficiency in the process. + __Frame Content Size__ This is the original (uncompressed) size. @@ -274,27 +289,12 @@ When field size is 2, _an offset of 256 is added_. It's allowed to represent a small size (ex: `18`) using the 8-bytes variant. A size of `0` means `content size is unknown`. In which case, the `WD` byte will necessarily be present, -and becomes the only hint to determine memory allocation. +and becomes the only hint to help memory allocation. In order to preserve decoder from unreasonable memory requirement, a decoder can refuse a compressed frame which requests a memory size beyond decoder's authorized range. -__Dictionary ID__ - -This is a variable size field, which contains a single ID. -It checks if the correct dictionary is used for decoding. -Note that this field is optional. If it's not present, -it's up to the caller to make sure it uses the correct dictionary. - -Field size depends on __Dictionary ID flag__. -1 byte can represent an ID 0-255. -2 bytes can represent an ID 0-65535. -4 bytes can represent an ID 0-(2^32-1). - -It's allowed to represent a small ID (for example `13`) -with a large 4-bytes dictionary ID, losing some efficiency in the process. - Data Blocks ----------- @@ -364,7 +364,6 @@ over user-defined data and continue decoding. Skippable frames defined in this specification are compatible with LZ4 ones. - __Magic Number__ : 4 Bytes, Little endian format. @@ -395,8 +394,8 @@ A compressed block consists of 2 sections : - Literals section - Sequences section -### Prerequisite -To decode a compressed block, it's required to access to following elements : +### Prerequisites +To decode a compressed block, the following elements are necessary : - Previous decoded blocks, up to a distance of `windowSize`, or all frame's previous blocks in "single segment" mode. - List of "recent offsets" from previous compressed block. @@ -634,7 +633,6 @@ it gives the following distribution : | nb bits | 0 | 4 | 4 | 3 | 2 | 1 | - #### Literals bitstreams ##### Bitstreams sizes @@ -711,12 +709,265 @@ which specifies a baseline and a number of additional bits. _Codes_ are FSE compressed, and interleaved with raw additional bits in the same bitstream. -The Sequence section starts by a header, -followed by an optional Probability table for each symbol type, +The Sequences section starts by a header, +followed by optional Probability tables for each symbol type, followed by the bitstream. +To decode the Sequence section, it's required to know its size. +This size is deducted from "blockSize - literalSectionSize". + + #### Sequences section header +Consists in 2 items : +- Nb of Sequences +- Flags providing Symbol compression types + +__Nb of Sequences__ + +This is a variable size field, `nbSeqs`, using between 1 and 3 bytes. +Let's call its first byte `byte0`. +- `if (byte0 == 0)` : there are no sequences. + The sequence section stops there. + Regenerated content is defined entirely by literals section. +- `if (byte0 < 128)` : nbSeqs = byte0 . Uses 1 byte. +- `if (byte0 < 255)` : nbSeqs = ((byte0-128) << 8) + byte1 . Uses 2 bytes. +- `if (byte0 == 255)`: nbSeqs = byte1 + (byte2<<8) + 0x7F00 . Uses 3 bytes. + +__Symbol compression modes__ + +This is a single byte, defining the compression mode of each symbol type. + +| BitNb | 7-6 | 5-4 | 3-2 | 1-0 | +| ------- | ------ | ------ | ------ | -------- | +|FieldName| LLtype | OFType | MLType | Reserved | + +The last field, `Reserved`, must be all-zeroes. + +`LLtype`, `OFType` and `MLType` define the compression mode of +Literal Lengths, Offsets and Match Lengths respectively. + +They follow the same enumeration : + +| Value | 0 | 1 | 2 | 3 | +| ---------------- | ------ | --- | ------ | --- | +| Compression Mode | predef | RLE | Repeat | FSE | + +- "predef" : uses a pre-defined distribution table. +- "RLE" : it's a single code, repeated `nbSeqs` times. +- "Repeat" : re-use distribution table from previous compressed block. +- "FSE" : standard FSE compression. + Symbol type requires a distribution table, + which will be described in next part. + +#### Symbols decoding + +##### Literal Lengths codes + +Literal lengths codes are values ranging from `0` to `35` included. +They define lengths from 0 to 131071 bytes. + +| Code | 0-15 | +| ------ | ---- | +| nbBits | 0 | +| value | Code | + +| Code | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | +| -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | +| Baseline | 16 | 18 | 20 | 22 | 24 | 28 | 32 | 40 | +| nb Bits | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 3 | + +| Code | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | +| -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | +| Baseline | 48 | 64 | 128 | 256 | 512 | 1024 | 2048 | 4096 | +| nb Bits | 4 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | + +| Code | 32 | 33 | 34 | 35 | +| -------- | ---- | ---- | ---- | ---- | +| Baseline | 8192 |16384 |32768 |65536 | +| nb Bits | 13 | 14 | 15 | 16 | + +__Default distribution__ + +When "compression mode" is defined as "default distribution", +a pre-defined distribution is used for FSE compression. + +Here is its definition. It uses an accuracy of 6 bits (64 states). +``` +short literalLengths_defaultDistribution[36] = + { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, + -1,-1,-1,-1 }; +``` + +##### Match Lengths codes + +Match lengths codes are values ranging from `0` to `52` included. +They define lengths from 3 to 131074 bytes. + +| Code | 0-31 | +| ------ | -------- | +| nbBits | 0 | +| value | Code + 3 | + +| Code | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | +| -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | +| Baseline | 35 | 37 | 39 | 41 | 43 | 47 | 51 | 59 | +| nb Bits | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 3 | + +| Code | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | +| -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | +| Baseline | 67 | 83 | 99 | 131 | 258 | 514 | 1026 | 2050 | +| nb Bits | 4 | 4 | 5 | 7 | 8 | 9 | 10 | 11 | + +| Code | 48 | 49 | 50 | 51 | 52 | +| -------- | ---- | ---- | ---- | ---- | ---- | +| Baseline | 4098 | 8194 |16486 |32770 |65538 | +| nb Bits | 12 | 13 | 14 | 15 | 16 | + +__Default distribution__ + +When "compression mode" is defined as "default distribution", +a pre-defined distribution is used for FSE compression. + +Here is its definition. It uses an accuracy of 6 bits (64 states). +``` +short matchLengths_defaultDistribution[53] = + { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1, + -1,-1,-1,-1,-1 }; +``` + +##### Offset codes + +Offset codes are values ranging from `0` to `N`, +with `N` being limited by maximum backreference distance. + +A decoder is free to limit its maximum `N` supported, +although the recommendation is to support at least up to `22`. +For information, at the time of this writing. +the reference decoder supports a maximum `N` value of `28` in 64-bits mode. + +An offset code is also the nb of additional bits to read, +and can be translated into an `OFValue` using the following formulae : + +``` +OFValue = (1 << offsetCode) + readNBits(offsetCode); +if (OFValue > 3) offset = OFValue - 3; +``` + +OFValue from 1 to 3 are special : they define "repeat codes", +which means one of the previous offsets will be repeated. +They are sorted in recency order, with 1 meaning the most recent one. + +__Default distribution__ + +When "compression mode" is defined as "default distribution", +a pre-defined distribution is used for FSE compression. + +Here is its definition. It uses an accuracy of 5 bits (32 states), +and support a maximum `N` of 28, allowing offset values up to 536,870,908 . + +If any sequence in the compressed block requires an offset larger than this, +it's not possible to use the default distribution to represent it. + +``` +short offsetCodes_defaultDistribution[53] = + { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 }; +``` + +#### Distribution tables + +Following the header, up to 3 distribution tables can be described. +They are, in order : +- Literal lengthes +- Offsets +- Match Lengthes + +The content to decode depends on their respective compression mode : +- Repeat mode : no content. Re-use distribution from previous compressed block. +- Predef : no content. Use pre-defined distribution table. +- RLE : 1 byte. This is the only code to use across the whole compressed block. +- FSE : A distribution table is present. + +##### FSE distribution table : condensed format + +An FSE distribution table describes the probabilities of all symbols +from `0` to the last present one (included) +on a normalized scale of `2^AccuracyLog` . + +It's a bitstream which is read forward, in little-endian fashion. +It's not necessary to know its exact size, +since it will be discovered and reported by the decoding process. + +The bitstream starts by reporting on which scale it operates. +`AccuracyLog = low4bits + 5;` +In theory, it can define a scale from 5 to 20. +In practice, decoders are allowed to limit the maximum supported `AccuracyLog`. +Recommended maximum are `9` for literal and match lengthes, and `8` for offsets. +The reference decoder uses these limits. + +Then follow each symbol value, from `0` to last present one. +The nb of bits used by each field is variable. +It depends on : + +- Remaining probabilities + 1 : + __example__ : + Presuming an AccuracyLog of 8, + and presuming 100 probabilities points have already been distributed, + the decoder may discover value from `0` to `255 - 100 + 1 == 156` (included). + Therefore, it must read `log2sup(156) == 8` bits. + +- Value decoded : small values use 1 less bit : + __example__ : + Presuming values from 0 to 156 (included) are possible, + 255-156 = 99 values are remaining in an 8-bits field. + They are used this way : + first 99 values (hence from 0 to 98) use only 7 bits, + values from 99 to 156 use 8 bits. + This is achieved through this scheme : + + | Value read | Value decoded | nb Bits used | + | ---------- | ------------- | ------------ | + | 0 - 98 | 0 - 98 | 7 | + | 99 - 127 | 99 - 127 | 8 | + | 128 - 226 | 0 - 98 | 7 | + | 227 - 255 | 128 - 156 | 8 | + +Symbols probabilities are read one by one, in order. + +Probability is obtained from Value decoded by following formulae : +`Proba = value - 1;` + +It means value `0` becomes negative probability `-1`. +`-1` is a special probability, which means `less than 1`. +Its effect on distribution table is described in a later paragraph. +For the purpose of calculating cumulated distribution, it counts as one. + +When a symbol has a probability of `zero`, +it is followed by a 2-bits repeat flag. +This repeat flag tells how many probabilities of zeroes follow the current one. +It provides a number ranging from 0 to 3. +If it is a 3, another 2-bits repeat flag follows, and so on. + +When last symbol reaches cumulated total of `2^AccuracyLog`, +decoding is complete. +Then the decoder can tell how many bytes were used in this process, +and how many symbols are present. + +The bitstream consumes a round number of bytes. +Any remaining bit within the last byte is just unused. + +If the last symbol makes cumulated total go above `2^AccuracyLog`, +distribution is considered corrupted. + +##### FSE decoding : from normalized distribution to decoding tables + + + +#### Bitstream From f9cac7a734886bca1e1bc1f6450640ade42a4534 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 4 Jul 2016 18:16:16 +0200 Subject: [PATCH 07/33] Added GNU separator `--`, to specifies that all following arguments are necessary file names (and not commands). Suggested by @chipturner (#230) --- NEWS | 1 + lib/decompress/zstd_decompress.c | 2 +- programs/zstdcli.c | 272 ++++++++++++++++--------------- 3 files changed, 141 insertions(+), 134 deletions(-) diff --git a/NEWS b/NEWS index bb520df7d..88e8c3b1f 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ v0.7.3 +added : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner. added : OpenBSD target, by Juan Francisco Cantero Hurtado v0.7.2 diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 228205824..366637c0a 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -735,7 +735,7 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream)); seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */ - if (MEM_32bits() | + if (MEM_32bits() || (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream)); /* ANS state update */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 129936e15..24fc33b7d 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -34,7 +34,7 @@ #include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */ #include /* strcmp, strlen */ #include /* toupper */ -#include +#include /* errno */ #include "fileio.h" #ifndef ZSTD_NOBENCH # include "bench.h" /* BMK_benchFiles, BMK_SetNbIterations */ @@ -205,7 +205,8 @@ int main(int argCount, const char** argv) dictBuild=0, nextArgumentIsOutFileName=0, nextArgumentIsMaxDict=0, - nextArgumentIsDictID=0; + nextArgumentIsDictID=0, + nextArgumentIsFile=0; unsigned cLevel = 1; unsigned cLevelLast = 1; unsigned recursive = 0; @@ -247,146 +248,165 @@ int main(int argCount, const char** argv) const char* argument = argv[argNb]; if(!argument) continue; /* Protection if argument empty */ - /* long commands (--long-word) */ - if (!strcmp(argument, "--decompress")) { decode=1; continue; } - if (!strcmp(argument, "--force")) { FIO_overwriteMode(); continue; } - if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); } - if (!strcmp(argument, "--help")) { displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); } - if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; } - if (!strcmp(argument, "--quiet")) { displayLevel--; continue; } - if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); continue; } - if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; } - if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; } - if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; } - if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } - if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; } - if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } - if (!strcmp(argument, "--test")) { decode=1; outFileName=nulmark; FIO_overwriteMode(); continue; } - if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; } - if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; } - if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; } - if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } - if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } + if (nextArgumentIsFile==0) { - /* '-' means stdin/stdout */ - if (!strcmp(argument, "-")){ - if (!filenameIdx) { - filenameIdx=1, filenameTable[0]=stdinmark; - outFileName=stdoutmark; - displayLevel-=(displayLevel==2); - continue; - } } + /* long commands (--long-word) */ + if (!strcmp(argument, "--")) { nextArgumentIsFile=1; continue; } + if (!strcmp(argument, "--decompress")) { decode=1; continue; } + if (!strcmp(argument, "--force")) { FIO_overwriteMode(); continue; } + if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); } + if (!strcmp(argument, "--help")) { displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); } + if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; } + if (!strcmp(argument, "--quiet")) { displayLevel--; continue; } + if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); continue; } + if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; } + if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; } + if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; } + if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } + if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; } + if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } + if (!strcmp(argument, "--test")) { decode=1; outFileName=nulmark; FIO_overwriteMode(); continue; } + if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; } + if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; } + if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; } + if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } + if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } - /* Decode commands (note : aggregated commands are allowed) */ - if (argument[0]=='-') { - argument++; - - while (argument[0]!=0) { -#ifndef ZSTD_NOCOMPRESS - /* compression Level */ - if ((*argument>='0') && (*argument<='9')) { - cLevel = readU32FromChar(&argument); - dictCLevel = cLevel; - if (dictCLevel > ZSTD_maxCLevel()) - CLEAN_RETURN(badusage(programName)); + /* '-' means stdin/stdout */ + if (!strcmp(argument, "-")){ + if (!filenameIdx) { + filenameIdx=1, filenameTable[0]=stdinmark; + outFileName=stdoutmark; + displayLevel-=(displayLevel==2); continue; - } -#endif + } } - switch(argument[0]) - { - /* Display help */ - case 'V': displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); /* Version Only */ - case 'H': - case 'h': displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); + /* Decode commands (note : aggregated commands are allowed) */ + if (argument[0]=='-') { + argument++; - /* Decoding */ - case 'd': decode=1; argument++; break; + while (argument[0]!=0) { + #ifndef ZSTD_NOCOMPRESS + /* compression Level */ + if ((*argument>='0') && (*argument<='9')) { + cLevel = readU32FromChar(&argument); + dictCLevel = cLevel; + if (dictCLevel > ZSTD_maxCLevel()) + CLEAN_RETURN(badusage(programName)); + continue; + } + #endif - /* Force stdout, even if stdout==console */ - case 'c': forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); argument++; break; + switch(argument[0]) + { + /* Display help */ + case 'V': displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); /* Version Only */ + case 'H': + case 'h': displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); - /* Use file content as dictionary */ - case 'D': nextEntryIsDictionary = 1; argument++; break; + /* Decoding */ + case 'd': decode=1; argument++; break; - /* Overwrite */ - case 'f': FIO_overwriteMode(); forceStdout=1; argument++; break; + /* Force stdout, even if stdout==console */ + case 'c': forceStdout=1; outFileName=stdoutmark; displayLevel-=(displayLevel==2); argument++; break; - /* Verbose mode */ - case 'v': displayLevel=4; argument++; break; + /* Use file content as dictionary */ + case 'D': nextEntryIsDictionary = 1; argument++; break; - /* Quiet mode */ - case 'q': displayLevel--; argument++; break; + /* Overwrite */ + case 'f': FIO_overwriteMode(); forceStdout=1; argument++; break; - /* keep source file (default); for gzip/xz compatibility */ - case 'k': FIO_setRemoveSrcFile(0); argument++; break; + /* Verbose mode */ + case 'v': displayLevel=4; argument++; break; - /* Checksum */ - case 'C': argument++; FIO_setChecksumFlag(2); break; + /* Quiet mode */ + case 'q': displayLevel--; argument++; break; - /* test compressed file */ - case 't': decode=1; outFileName=nulmark; argument++; break; + /* keep source file (default); for gzip/xz compatibility */ + case 'k': FIO_setRemoveSrcFile(0); argument++; break; - /* dictionary name */ - case 'o': nextArgumentIsOutFileName=1; argument++; break; + /* Checksum */ + case 'C': argument++; FIO_setChecksumFlag(2); break; - /* recursive */ - case 'r': recursive=1; argument++; break; + /* test compressed file */ + case 't': decode=1; outFileName=nulmark; argument++; break; -#ifndef ZSTD_NOBENCH - /* Benchmark */ - case 'b': bench=1; argument++; break; + /* dictionary name */ + case 'o': nextArgumentIsOutFileName=1; argument++; break; - /* range bench (benchmark only) */ - case 'e': - /* compression Level */ + /* recursive */ + case 'r': recursive=1; argument++; break; + + #ifndef ZSTD_NOBENCH + /* Benchmark */ + case 'b': bench=1; argument++; break; + + /* range bench (benchmark only) */ + case 'e': + /* compression Level */ + argument++; + cLevelLast = readU32FromChar(&argument); + break; + + /* Modify Nb Iterations (benchmark only) */ + case 'i': argument++; - cLevelLast = readU32FromChar(&argument); + { U32 const iters = readU32FromChar(&argument); + BMK_setNotificationLevel(displayLevel); + BMK_SetNbIterations(iters); + } break; - /* Modify Nb Iterations (benchmark only) */ - case 'i': - argument++; - { U32 const iters = readU32FromChar(&argument); - BMK_setNotificationLevel(displayLevel); - BMK_SetNbIterations(iters); + /* cut input into blocks (benchmark only) */ + case 'B': + argument++; + { size_t bSize = readU32FromChar(&argument); + if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */ + if (toupper(*argument)=='M') bSize<<=20, argument++; + if (toupper(*argument)=='B') argument++; + BMK_setNotificationLevel(displayLevel); + BMK_SetBlockSize(bSize); + } + break; + #endif /* ZSTD_NOBENCH */ + + /* Dictionary Selection level */ + case 's': + argument++; + dictSelect = readU32FromChar(&argument); + break; + + /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */ + case 'p': argument++; + #ifndef ZSTD_NOBENCH + if ((*argument>='0') && (*argument<='9')) { + BMK_setAdditionalParam(readU32FromChar(&argument)); + } else + #endif + main_pause=1; + break; + /* unknown command */ + default : CLEAN_RETURN(badusage(programName)); } - break; - - /* cut input into blocks (benchmark only) */ - case 'B': - argument++; - { size_t bSize = readU32FromChar(&argument); - if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */ - if (toupper(*argument)=='M') bSize<<=20, argument++; - if (toupper(*argument)=='B') argument++; - BMK_setNotificationLevel(displayLevel); - BMK_SetBlockSize(bSize); - } - break; -#endif /* ZSTD_NOBENCH */ - - /* Dictionary Selection level */ - case 's': - argument++; - dictSelect = readU32FromChar(&argument); - break; - - /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */ - case 'p': argument++; -#ifndef ZSTD_NOBENCH - if ((*argument>='0') && (*argument<='9')) { - BMK_setAdditionalParam(readU32FromChar(&argument)); - } else -#endif - main_pause=1; - break; - /* unknown command */ - default : CLEAN_RETURN(badusage(programName)); } + continue; + } /* if (argument[0]=='-') */ + + if (nextArgumentIsMaxDict) { + nextArgumentIsMaxDict = 0; + maxDictSize = readU32FromChar(&argument); + if (toupper(*argument)=='K') maxDictSize <<= 10; + if (toupper(*argument)=='M') maxDictSize <<= 20; + continue; } - continue; - } /* if (argument[0]=='-') */ + + if (nextArgumentIsDictID) { + nextArgumentIsDictID = 0; + dictID = readU32FromChar(&argument); + continue; + } + + } /* if (nextArgumentIsAFile==0) */ if (nextEntryIsDictionary) { nextEntryIsDictionary = 0; @@ -401,20 +421,6 @@ int main(int argCount, const char** argv) continue; } - if (nextArgumentIsMaxDict) { - nextArgumentIsMaxDict = 0; - maxDictSize = readU32FromChar(&argument); - if (toupper(*argument)=='K') maxDictSize <<= 10; - if (toupper(*argument)=='M') maxDictSize <<= 20; - continue; - } - - if (nextArgumentIsDictID) { - nextArgumentIsDictID = 0; - dictID = readU32FromChar(&argument); - continue; - } - /* add filename to list */ filenameTable[filenameIdx++] = argument; } From 9ca73364e6f4c1eb3fa55692b1edfd6f81ad163c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 5 Jul 2016 10:53:38 +0200 Subject: [PATCH 08/33] updated spec --- lib/decompress/zstd_decompress.c | 2 +- programs/zstd.1 | 3 - zstd_compression_format.md | 215 +++++++++++++++++++++++++------ 3 files changed, 176 insertions(+), 44 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 366637c0a..5a0d97657 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -712,7 +712,7 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) if (llCode == 0 && offset <= 1) offset = 1-offset; if (offset != 0) { - size_t temp = seqState->prevOffset[offset]; + size_t const temp = seqState->prevOffset[offset]; if (offset != 1) { seqState->prevOffset[2] = seqState->prevOffset[1]; } diff --git a/programs/zstd.1 b/programs/zstd.1 index bba6fa122..7201f76c3 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -131,9 +131,6 @@ Typical gains range from ~10% (at 64KB) to x5 better (at <1KB). .TP .B \-B# cut file into independent blocks of size # (default: no block) -.TP -.B \-r# - test all compression levels from 1 to # (default: disabled) .SH BUGS diff --git a/zstd_compression_format.md b/zstd_compression_format.md index dbadac758..ce17f569f 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -25,7 +25,7 @@ Introduction The purpose of this document is to define a lossless compressed data format, that is independent of CPU type, operating system, file system and character set, suitable for -File compression, Pipe and streaming compression +file compression, pipe and streaming compression, using the [Zstandard algorithm](http://www.zstandard.org). The data can be produced or consumed, @@ -99,7 +99,7 @@ __EndMark__ The flow of blocks ends when the last block header brings an _end signal_ . This last block header may optionally host a __Content Checksum__ . -__Content Checksum__ +##### __Content Checksum__ Content Checksum verify that frame content has been regenrated correctly. The content checksum is the result @@ -108,7 +108,8 @@ digesting the original (decoded) data as input, and a seed of zero. Bits from 11 to 32 (included) are extracted to form a 22 bits checksum stored into the last block header. ``` -contentChecksum = (XXH64(content, size, 0) >> 11) & (1<<22)-1); +mask22bits = (1<<22)-1; +contentChecksum = (XXH64(content, size, 0) >> 11) & mask22bits; ``` Content checksum is only present when its associated flag is set in the frame descriptor. @@ -183,7 +184,7 @@ which requests a memory size beyond decoder's authorized range. For broader compatibility, decoders are recommended to support memory sizes of at least 8 MB. This is just a recommendation, -as each decoder is free to support higher or lower limits, +each decoder is free to support higher or lower limits, depending on local limitations. __Unused bit__ @@ -204,13 +205,14 @@ to signal a feature that must be interpreted in order to decode the frame. __Content checksum flag__ If this flag is set, a content checksum will be present into the EndMark. -The checksum is a 22 bits value extracted from the XXH64() of data. -See __Content Checksum__ . +The checksum is a 22 bits value extracted from the XXH64() of data, +and stored into endMark. See [__Content Checksum__](#content-checksum) . __Dictionary ID flag__ This is a 2-bits flag (`= FHD & 3`), -telling if a dictionary ID is provided within the header +telling if a dictionary ID is provided within the header. +It also specifies the size of this field. | Value | 0 | 1 | 2 | 3 | | ------- | --- | --- | --- | --- | @@ -286,10 +288,10 @@ Format is Little endian. When field size is 1, 4 or 8 bytes, the value is read directly. When field size is 2, _an offset of 256 is added_. -It's allowed to represent a small size (ex: `18`) using the 8-bytes variant. +It's allowed to represent a small size (ex: `18`) using any compatible variant. A size of `0` means `content size is unknown`. In which case, the `WD` byte will necessarily be present, -and becomes the only hint to help memory allocation. +and becomes the only hint to guide memory allocation. In order to preserve decoder from unreasonable memory requirement, a decoder can refuse a compressed frame @@ -317,8 +319,8 @@ There are 4 block types : | ---------- | ---------- | --- | --- | ------- | | Block Type | Compressed | Raw | RLE | EndMark | -- Compressed : this is a Zstandard compressed block, - detailed in a later part of this specification. +- Compressed : this is a [Zstandard compressed block](#compressed-block-format), + detailed in another section of this specification. "block size" is the compressed size. Decompressed size is unknown, but its maximum possible value is guaranteed (see below) @@ -329,12 +331,12 @@ There are 4 block types : while the "compressed" block is just 1 byte (the byte to repeat). - EndMark : this is not a block. Signal the end of the frame. The rest of the field may be optionally filled by a checksum - (see frame checksum). + (see [__Content Checksum__]). Block sizes must respect a few rules : -- In compressed mode, compressed size if always strictly `< contentSize`. -- Block decompressed size is necessarily <= maximum back-reference distance . -- Block decompressed size is necessarily <= 128 KB +- In compressed mode, compressed size if always strictly `< decompressed size`. +- Block decompressed size is always <= maximum back-reference distance . +- Block decompressed size is always <= 128 KB __Data__ @@ -343,8 +345,8 @@ Where the actual data to decode stands. It might be compressed or not, depending on previous field indications. A data block is not necessarily "full" : since an arbitrary “flush” may happen anytime, -block content can be any size, up to Block Maximum Size. -Block Maximum Size is the smallest of : +block decompressed content can be any size, up to Block Maximum Size. +Block Maximum Decompressed Size is the smallest of : - Max back-reference distance - 128 KB @@ -388,7 +390,7 @@ This specification details the content of a _compressed block_. A compressed block has a size, which must be known. It also has a guaranteed maximum regenerated size, in order to properly allocate destination buffer. -See "Frame format" for more details. +See [Data Blocks](#data-blocks) for more details. A compressed block consists of 2 sections : - Literals section @@ -397,15 +399,15 @@ A compressed block consists of 2 sections : ### Prerequisites To decode a compressed block, the following elements are necessary : - Previous decoded blocks, up to a distance of `windowSize`, - or all frame's previous blocks in "single segment" mode. + or all previous blocks in "single segment" mode. - List of "recent offsets" from previous compressed block. - Decoding tables of previous compressed block for each symbol type - (literals, litLength, matchLength, offset) + (literals, litLength, matchLength, offset). ### Literals section -Literals are compressed using order-0 huffman compression. +Literals are compressed using huffman compression. During sequence phase, literals will be entangled with match copy operations. All literals are regrouped in the first part of the block. They can be decoded first, and then copied during sequence operations, @@ -456,7 +458,7 @@ Sizes format are divided into 2 families : For values spanning several bytes, convention is Big-endian. -__Sizes format for Raw or RLE block__ : +__Sizes format for Raw or RLE literals block__ : - Value : 0x : Regenerated size uses 5 bits (0-31). Total literal header size is 1 byte. @@ -471,7 +473,7 @@ __Sizes format for Raw or RLE block__ : Note : it's allowed to represent a short value (ex : `13`) using a long format, accepting the reduced compacity. -__Sizes format for Compressed Block__ : +__Sizes format for Compressed literals block__ : Note : also applicable to "repeat-stats" blocks. - Value : 00 : 4 streams @@ -491,23 +493,23 @@ Compressed and regenerated size fields follow big endian convention. #### Huffman Tree description -This section is only present when block type is _compressed_ (`0`). +This section is only present when block type is `Compressed` (`0`). -Prefix coding represents symbols from an a priori known -alphabet by bit sequences (codes), one code for each symbol, in -a manner such that different symbols may be represented by bit -sequences of different lengths, but a parser can always parse -an encoded string unambiguously symbol-by-symbol. +Prefix coding represents symbols from an a priori known alphabet +by bit sequences (codes), one code for each symbol, +in a manner such that different symbols may be represented +by bit sequences of different lengths, +but a parser can always parse an encoded string +unambiguously symbol-by-symbol. -Given an alphabet with known symbol frequencies, the Huffman -algorithm allows the construction of an optimal prefix code -(one which represents strings with those symbol frequencies -using the fewest bits of any possible prefix codes for that -alphabet). Such a code is called a Huffman code. +Given an alphabet with known symbol frequencies, +the Huffman algorithm allows the construction of an optimal prefix code +using the fewest bits of any possible prefix codes for that alphabet. +Such a code is called a Huffman code. -Huffman code must not exceed a maximum code length. +Prefix code must not exceed a maximum code length. More bits improve accuracy but cost more header size, -and requires more memory for decoding operations. +and require more memory for decoding operations. The current format limits the maximum depth to 15 bits. The reference decoder goes further, by limiting it to 11 bits. @@ -552,7 +554,8 @@ Therefore, `maxBits = 4` and `weight[5] = 1`. ##### Huffman Tree header -This is a single byte value (0-255), which tells how to decode the tree. +This is a single byte value (0-255), +which tells how to decode the list of weights. - if headerByte >= 242 : this is one of 14 pre-defined weight distributions : + 242 : 1x1 (+ 1x1) @@ -896,7 +899,7 @@ The content to decode depends on their respective compression mode : An FSE distribution table describes the probabilities of all symbols from `0` to the last present one (included) -on a normalized scale of `2^AccuracyLog` . +on a normalized scale of `1 << AccuracyLog` . It's a bitstream which is read forward, in little-endian fashion. It's not necessary to know its exact size, @@ -952,7 +955,7 @@ This repeat flag tells how many probabilities of zeroes follow the current one. It provides a number ranging from 0 to 3. If it is a 3, another 2-bits repeat flag follows, and so on. -When last symbol reaches cumulated total of `2^AccuracyLog`, +When last symbol reaches cumulated total of `1 << AccuracyLog`, decoding is complete. Then the decoder can tell how many bytes were used in this process, and how many symbols are present. @@ -960,15 +963,147 @@ and how many symbols are present. The bitstream consumes a round number of bytes. Any remaining bit within the last byte is just unused. -If the last symbol makes cumulated total go above `2^AccuracyLog`, +If the last symbol makes cumulated total go above `1 << AccuracyLog`, distribution is considered corrupted. ##### FSE decoding : from normalized distribution to decoding tables +The distribution of normalized probabilities is enough +to create a unique decoding table. + +It follows the following build rule : + +The table has a size of `tableSize = 1 << AccuracyLog;`. +Each cell describes the symbol decoded, +and instructions to get the next state. + +Symbols are scanned in their natural order for `less than 1` probabilities. +Symbols with this probability are being attributed a single cell, +starting from the end of the table. +These symbols define a full state reset, reading `AccuracyLog` bits. + +All remaining symbols are sorted in their natural order. +Starting from symbol `0` and table position `0`, +each symbol gets attributed as many cells as its probability. +Cell allocation is spreaded, not linear : +each successor position follow this rule : + +`position += (tableSize>>1) + (tableSize>>3) + 3); +position &= tableSize-1;` + +A position is skipped if already occupied, +typically by a "less than 1" probability symbol. + +The result is a list of state values. +Each state will decode the current symbol. + +To get the Number of bits and baseline required for next state, +it's first necessary to sort all states in their natural order. +The lower states will need 1 bit more than higher ones. + +__Example__ : +Presuming a symbol has a probability of 5. +It receives 5 state values. + +Next power of 2 is 8. +Space of probabilities is divided into 8 equal parts. +Presuming the AccuracyLog is 7, it defines 128 states. +Divided by 8, each share is 16 large. + +In order to reach 8, 8-5=3 lowest states will count "double", +taking shares twice larger, +requiring one more bit in the process. + +Numbering starts from higher states using less bits. + +| state order | 0 | 1 | 2 | 3 | 4 | +| ----------- | ----- | ----- | ------ | ---- | ----- | +| width | 32 | 32 | 32 | 16 | 16 | +| nb Bits | 5 | 5 | 5 | 4 | 4 | +| range nb | 2 | 4 | 6 | 0 | 1 | +| baseline | 32 | 64 | 96 | 0 | 16 | +| range | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 | + +Next state is determined from current state +by reading the required number of bits, and adding the specified baseline. #### Bitstream +All sequences are stored in a single bitstream, read _backward_. +It is therefore necessary to know the bitstream size, +which is deducted from compressed block size. + +The exact last bit of the stream is followed by a set-bit-flag. +The highest bit of last byte is this flag. +It does not belong to the useful part of the bitstream. +Therefore, last byte has 0-7 useful bits. +Note that it also means that last byte cannot be `0`. + +##### Starting states + +The bitstream starts with initial state values, +each using the required number of bits in their respective _accuracy_, +decoded previously from their normalized distribution. + +It starts by `Literal Length State`, +followed by `Offset State`, +and finally `Match Length State`. + +Reminder : always keep in mind that all values are read _backward_. + +##### Decoding a sequence + +A state gives a code. +A code provides a baseline and number of bits to add. +See [Symbol Decoding] section for details on each symbol. + +Decoding starts by reading the nb of bits required to decode offset. +It then does the same for match length, +and then for literal length. + +Offset / matchLength / litLength define a sequence, which can be applied. + +The next operation is to update states. +Using rules pre-calculated in the decoding tables, +`Literal Length State` is updated, +followed by `Match Length State`, +and then `Offset State`. + +This operation will be repeated `NbSeqs` times. +At the end, the bitstream shall be entirely consumed, +otherwise bitstream is considered corrupted. + +[Symbol Decoding]:#symbols-decoding + +##### Repeat offsets + +As seen in [Offset Codes], the first 3 values define a repeated offset. +They are sorted in recency order, with 1 meaning "most recent one". + +There is an exception though, when current sequence's literal length is `0`. +In which case, 1 would just make previous match longer. +Therefore, in such case, 1 means in fact 2, and 2 is impossible. +Meaning of 3 is unmodified. + +Repeat offsets start with the following values : 1, 4 and 8 (in order). + +Then each block receives its start value from previous compressed block. +Note that non-compressed blocks are skipped, +they do not contribute to offset history. + +[Offset Codes]: #offset-codes + +###### Offset updates rules + +When the new offset is a normal one, +offset history is simply translated by one position, +with the new offset taking first spot. + +- When repeat offset 1 (most recent) is used, history is unmodified. +- When repeat offset 2 is used, it's swapped with offset 1. +- When repeat offset 3 is used, it takes first spot, + pushing the other ones by one position. From cd25a917411aabfa295b2bc9687a0136fc500880 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 5 Jul 2016 11:50:37 +0200 Subject: [PATCH 09/33] updated format spec --- zstd_compression_format.md | 165 ++++++++++++++++++------------------- 1 file changed, 81 insertions(+), 84 deletions(-) diff --git a/zstd_compression_format.md b/zstd_compression_format.md index ce17f569f..94b1dc5cd 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -76,9 +76,9 @@ allowing streaming operations. General Structure of Zstandard Frame format ------------------------------------------- -| MagicNb | F. Header | Block | (More blocks) | EndMark | -|:-------:|:----------:| ----- | ------------- | ------- | -| 4 bytes | 2-14 bytes | | | 3 bytes | +| MagicNb | Frame Header | Block | (More blocks) | EndMark | +|:-------:|:-------------:| ----- | ------------- | ------- | +| 4 bytes | 2-14 bytes | | | 3 bytes | __Magic Number__ @@ -87,11 +87,11 @@ Value : 0xFD2FB527 __Frame Header__ -2 to 14 Bytes, to be detailed in the next part. +2 to 14 Bytes, detailed in [next part](#frame-header). __Data Blocks__ -To be detailed later on. +Detailed in [next chapter](#data-blocks). That’s where compressed data is stored. __EndMark__ @@ -101,12 +101,12 @@ This last block header may optionally host a __Content Checksum__ . ##### __Content Checksum__ -Content Checksum verify that frame content has been regenrated correctly. +Content Checksum verify that frame content has been regenerated correctly. The content checksum is the result of [xxh64() hash function](https://www.xxHash.com) digesting the original (decoded) data as input, and a seed of zero. Bits from 11 to 32 (included) are extracted to form a 22 bits checksum -stored into the last block header. +stored into the endmark body. ``` mask22bits = (1<<22)-1; contentChecksum = (XXH64(content, size, 0) >> 11) & mask22bits; @@ -224,6 +224,10 @@ Provides guarantees on maximum back-reference distance that will be present within compressed data. This information is useful for decoders to allocate enough memory. +`WD` byte is optional. It's not present in `single segment` mode. +In which case, the maximum back-reference distance is the content size itself, +which can be any value from 1 to 2^64-1 bytes (16 EB). + | BitNb | 7-3 | 0-2 | | --------- | -------- | -------- | | FieldName | Exponent | Mantissa | @@ -236,20 +240,16 @@ windowAdd = (windowBase / 8) * Mantissa; windowSize = windowBase + windowAdd; ``` The minimum window size is 1 KB. -The maximum size is (15*(2^38))-1 bytes, which is almost 1.875 TB. +The maximum size is `15*(1<<38)` bytes, which is 1.875 TB. To properly decode compressed data, a decoder will need to allocate a buffer of at least `windowSize` bytes. -Note that `WD` byte is optional. It's not present in `single segment` mode. -In which case, the maximum back-reference distance is the content size itself, -which can be any value from 1 to 2^64-1 bytes (16 EB). - In order to preserve decoder from unreasonable memory requirements, a decoder can refuse a compressed frame which requests a memory size beyond decoder's authorized range. -For better interoperability, +For improved interoperability, decoders are recommended to be compatible with window sizes of 8 MB. Encoders are recommended to not request more than 8 MB. It's merely a recommendation though, @@ -266,10 +266,10 @@ it's up to the caller to make sure it uses the correct dictionary. Field size depends on __Dictionary ID flag__. 1 byte can represent an ID 0-255. 2 bytes can represent an ID 0-65535. -4 bytes can represent an ID 0-(2^32-1). +4 bytes can represent an ID 0-4294967295. It's allowed to represent a small ID (for example `13`) -with a large 4-bytes dictionary ID, losing some efficiency in the process. +with a large 4-bytes dictionary ID, losing some compacity in the process. __Frame Content Size__ @@ -331,7 +331,7 @@ There are 4 block types : while the "compressed" block is just 1 byte (the byte to repeat). - EndMark : this is not a block. Signal the end of the frame. The rest of the field may be optionally filled by a checksum - (see [__Content Checksum__]). + (see [Content Checksum](#content-checksum)). Block sizes must respect a few rules : - In compressed mode, compressed size if always strictly `< decompressed size`. @@ -345,9 +345,9 @@ Where the actual data to decode stands. It might be compressed or not, depending on previous field indications. A data block is not necessarily "full" : since an arbitrary “flush” may happen anytime, -block decompressed content can be any size, up to Block Maximum Size. -Block Maximum Decompressed Size is the smallest of : -- Max back-reference distance +block decompressed content can be any size, +up to Block Maximum Decompressed Size, which is the smallest of : +- Maximum back-reference distance - 128 KB @@ -364,7 +364,9 @@ Its design is pretty straightforward, with the sole objective to allow the decoder to quickly skip over user-defined data and continue decoding. -Skippable frames defined in this specification are compatible with LZ4 ones. +Skippable frames defined in this specification are compatible with [LZ4] ones. + +[LZ4]:http://www.lz4.org __Magic Number__ : @@ -393,8 +395,8 @@ in order to properly allocate destination buffer. See [Data Blocks](#data-blocks) for more details. A compressed block consists of 2 sections : -- Literals section -- Sequences section +- [Literals section](#literals-section) +- [Sequences section](#sequences-section) ### Prerequisites To decode a compressed block, the following elements are necessary : @@ -439,12 +441,11 @@ This is a 2-bits field, describing 4 different block types : | Block Type | Compressed | Repeat | Raw | RLE | - Compressed : This is a standard huffman-compressed block, - starting with a huffman tree description. - See details below. + starting with a huffman tree description. + See details below. - Repeat Stats : This is a huffman-compressed block, - using huffman tree from previous huffman-compressed block. - Huffman tree description will be skipped. - Compressed stream is equivalent to "compressed" block type. + using huffman tree _from previous huffman-compressed literals block_. + Huffman tree description will be skipped. - Raw : Literals are stored uncompressed. - RLE : Literals consist of a single byte value repeated N times. @@ -476,24 +477,24 @@ using a long format, accepting the reduced compacity. __Sizes format for Compressed literals block__ : Note : also applicable to "repeat-stats" blocks. -- Value : 00 : 4 streams - Compressed and regenerated sizes use 10 bits (0-1023) - Total literal header size is 3 bytes -- Value : 01 : _Single stream_ - Compressed and regenerated sizes use 10 bits (0-1023) - Total literal header size is 3 bytes -- Value : 10 : 4 streams - Compressed and regenerated sizes use 14 bits (0-16383) - Total literal header size is 4 bytes -- Value : 10 : 4 streams - Compressed and regenerated sizes use 18 bits (0-262143) - Total literal header size is 5 bytes +- Value : 00 : 4 streams. + Compressed and regenerated sizes use 10 bits (0-1023). + Total literal header size is 3 bytes. +- Value : 01 : _Single stream_. + Compressed and regenerated sizes use 10 bits (0-1023). + Total literal header size is 3 bytes. +- Value : 10 : 4 streams. + Compressed and regenerated sizes use 14 bits (0-16383). + Total literal header size is 4 bytes. +- Value : 10 : 4 streams. + Compressed and regenerated sizes use 18 bits (0-262143). + Total literal header size is 5 bytes. Compressed and regenerated size fields follow big endian convention. #### Huffman Tree description -This section is only present when block type is `Compressed` (`0`). +This section is only present when literals block type is `Compressed` (`0`). Prefix coding represents symbols from an a priori known alphabet by bit sequences (codes), one code for each symbol, @@ -546,7 +547,7 @@ It gives the following serie of weights : The decoder will do the inverse operation : having collected weights of literals from `0` to `4`, it knows the last literal, `5`, is present with a non-zero weight. -The weight of `5` can be deduced by joining to the nearest power of 2. +The weight of `5` can be deducted by joining to the nearest power of 2. Sum of 2^(weight-1) (excluding 0) is : `8 + 4 + 2 + 0 + 1 = 15` Nearest power of 2 is 16. @@ -558,24 +559,17 @@ This is a single byte value (0-255), which tells how to decode the list of weights. - if headerByte >= 242 : this is one of 14 pre-defined weight distributions : - + 242 : 1x1 (+ 1x1) - + 243 : 2x1 (+ 1x2) - + 244 : 3x1 (+ 1x1) - + 245 : 4x1 (+ 1x4) - + 246 : 7x1 (+ 1x1) - + 247 : 8x1 (+ 1x8) - + 248 : 15x1 (+ 1x1) - + 249 : 16x1 (+ 1x16) - + 250 : 31x1 (+ 1x1) - + 251 : 32x1 (+ 1x32) - + 252 : 63x1 (+ 1x1) - + 253 : 64x1 (+ 1x64) - + 254 :127x1 (+ 1x1) - + 255 :128x1 (+ 1x128) + +| value |242|243|244|245|246|247|248|249|250|251|252|253|254|255| +| -------- | +| Nb of 1s | 1 | 2 | 3 | 4 | 7 | 8 | 15| 16| 31| 32| 63| 64|127|128| +|Complement| 1 | 2 | 1 | 4 | 1 | 8 | 1 | 16| 1 | 32| 1 | 64| 1 |128| + +_Note_ : complement is by using the "join to nearest power of 2" rule. - if headerByte >= 128 : this is a direct representation, where each weight is written directly as a 4 bits field (0-15). - The full representation occupies ((nbSymbols+1)/2) bytes, + The full representation occupies `((nbSymbols+1)/2)` bytes, meaning it uses a last full byte even if nbSymbols is odd. `nbSymbols = headerByte - 127;` @@ -593,13 +587,13 @@ To decode an FSE bitstream, it is necessary to know its compressed size. Compressed size is provided by `headerByte`. It's also necessary to know its maximum decompressed size. In this case, it's `255`, since literal values range from `0` to `255`, -and the last symbol value is not represented. +and last symbol value is not represented. An FSE bitstream starts by a header, describing probabilities distribution. It will create a Decoding Table. It is necessary to know the maximum accuracy of distribution to properly allocate space for the Table. -For a list of huffman weights, this maximum is 8 bits. +For a list of huffman weights, this maximum is 7 bits. FSE header and bitstreams are described in a separated chapter. @@ -652,13 +646,12 @@ presuming the CPU has enough parallelism available. For single stream, header provides both the compressed and regenerated size. For 4-streams though, header only provides compressed and regenerated size of all 4 streams combined. - In order to properly decode the 4 streams, it's necessary to know the compressed and regenerated size of each stream. Regenerated size is easiest : each stream has a size of `(totalSize+3)/4`, -except the last one, which is up to 3 bytes smaller, to reach totalSize. +except the last one, which is up to 3 bytes smaller, to reach `totalSize`. Compressed size must be provided explicitly : in the 4-streams variant, bitstreams are preceded by 3 unsigned Little Endian 16-bits values. @@ -704,11 +697,11 @@ A match copy command specifies an offset and a length. The offset gives the position to copy from, which can stand within a previous block. -These are 3 symbol types, `literalLength`, `matchLength` and `offset`, +There are 3 symbol types, `literalLength`, `matchLength` and `offset`, which are encoded together, interleaved in a single _bitstream_. -Each symbol decoding consists of a _code_, -which specifies a baseline and a number of additional bits. +Each symbol is a _code_ in its own context, +which specifies a baseline and a number of bits to add. _Codes_ are FSE compressed, and interleaved with raw additional bits in the same bitstream. @@ -717,7 +710,7 @@ followed by optional Probability tables for each symbol type, followed by the bitstream. To decode the Sequence section, it's required to know its size. -This size is deducted from "blockSize - literalSectionSize". +This size is deducted from `blockSize - literalSectionSize`. #### Sequences section header @@ -733,9 +726,9 @@ Let's call its first byte `byte0`. - `if (byte0 == 0)` : there are no sequences. The sequence section stops there. Regenerated content is defined entirely by literals section. -- `if (byte0 < 128)` : nbSeqs = byte0 . Uses 1 byte. -- `if (byte0 < 255)` : nbSeqs = ((byte0-128) << 8) + byte1 . Uses 2 bytes. -- `if (byte0 == 255)`: nbSeqs = byte1 + (byte2<<8) + 0x7F00 . Uses 3 bytes. +- `if (byte0 < 128)` : `nbSeqs = byte0;` . Uses 1 byte. +- `if (byte0 < 255)` : `nbSeqs = ((byte0-128) << 8) + byte1;` . Uses 2 bytes. +- `if (byte0 == 255)`: `nbSeqs = byte1 + (byte2<<8) + 0x7F00;` . Uses 3 bytes. __Symbol compression modes__ @@ -760,8 +753,8 @@ They follow the same enumeration : - "RLE" : it's a single code, repeated `nbSeqs` times. - "Repeat" : re-use distribution table from previous compressed block. - "FSE" : standard FSE compression. - Symbol type requires a distribution table, - which will be described in next part. + A distribution table will be present. + It will be described in [next part](#distribution-tables). #### Symbols decoding @@ -772,8 +765,9 @@ They define lengths from 0 to 131071 bytes. | Code | 0-15 | | ------ | ---- | -| nbBits | 0 | | value | Code | +| nbBits | 0 | + | Code | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | | -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | @@ -792,7 +786,7 @@ They define lengths from 0 to 131071 bytes. __Default distribution__ -When "compression mode" is defined as "default distribution", +When "compression mode" is "predef"", a pre-defined distribution is used for FSE compression. Here is its definition. It uses an accuracy of 6 bits (64 states). @@ -810,8 +804,8 @@ They define lengths from 3 to 131074 bytes. | Code | 0-31 | | ------ | -------- | -| nbBits | 0 | | value | Code + 3 | +| nbBits | 0 | | Code | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | | -------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | @@ -847,8 +841,8 @@ short matchLengths_defaultDistribution[53] = Offset codes are values ranging from `0` to `N`, with `N` being limited by maximum backreference distance. -A decoder is free to limit its maximum `N` supported, -although the recommendation is to support at least up to `22`. +A decoder is free to limit its maximum `N` supported. +Recommendation is to support at least up to `22`. For information, at the time of this writing. the reference decoder supports a maximum `N` value of `28` in 64-bits mode. @@ -863,14 +857,15 @@ if (OFValue > 3) offset = OFValue - 3; OFValue from 1 to 3 are special : they define "repeat codes", which means one of the previous offsets will be repeated. They are sorted in recency order, with 1 meaning the most recent one. +See [Repeat offsets](#repeat-offsets) paragraph. __Default distribution__ -When "compression mode" is defined as "default distribution", +When "compression mode" is defined as "predef", a pre-defined distribution is used for FSE compression. Here is its definition. It uses an accuracy of 5 bits (32 states), -and support a maximum `N` of 28, allowing offset values up to 536,870,908 . +and supports a maximum `N` of 28, allowing offset values up to 536,870,908 . If any sequence in the compressed block requires an offset larger than this, it's not possible to use the default distribution to represent it. @@ -920,7 +915,7 @@ It depends on : __example__ : Presuming an AccuracyLog of 8, and presuming 100 probabilities points have already been distributed, - the decoder may discover value from `0` to `255 - 100 + 1 == 156` (included). + the decoder may read any value from `0` to `255 - 100 + 1 == 156` (included). Therefore, it must read `log2sup(156) == 8` bits. - Value decoded : small values use 1 less bit : @@ -946,7 +941,7 @@ Probability is obtained from Value decoded by following formulae : It means value `0` becomes negative probability `-1`. `-1` is a special probability, which means `less than 1`. -Its effect on distribution table is described in a later paragraph. +Its effect on distribution table is described in next paragraph. For the purpose of calculating cumulated distribution, it counts as one. When a symbol has a probability of `zero`, @@ -988,8 +983,10 @@ each symbol gets attributed as many cells as its probability. Cell allocation is spreaded, not linear : each successor position follow this rule : -`position += (tableSize>>1) + (tableSize>>3) + 3); -position &= tableSize-1;` +``` +position += (tableSize>>1) + (tableSize>>3) + 3; +position &= tableSize-1; +``` A position is skipped if already occupied, typically by a "less than 1" probability symbol. @@ -999,11 +996,11 @@ Each state will decode the current symbol. To get the Number of bits and baseline required for next state, it's first necessary to sort all states in their natural order. -The lower states will need 1 bit more than higher ones. +The lower states will need 1 more bit than higher ones. __Example__ : Presuming a symbol has a probability of 5. -It receives 5 state values. +It receives 5 state values. States are sorted in natural order. Next power of 2 is 8. Space of probabilities is divided into 8 equal parts. @@ -1034,8 +1031,8 @@ All sequences are stored in a single bitstream, read _backward_. It is therefore necessary to know the bitstream size, which is deducted from compressed block size. -The exact last bit of the stream is followed by a set-bit-flag. -The highest bit of last byte is this flag. +The bit of the stream is followed by a set-bit-flag. +Highest bit of last byte is this flag. It does not belong to the useful part of the bitstream. Therefore, last byte has 0-7 useful bits. Note that it also means that last byte cannot be `0`. From e0ce5b094b8ee223e3582bf4b8bcf2326156e842 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 6 Jul 2016 01:50:44 +0200 Subject: [PATCH 10/33] updated spec --- zstd_compression_format.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zstd_compression_format.md b/zstd_compression_format.md index 94b1dc5cd..2fbe3fa4c 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -561,7 +561,7 @@ which tells how to decode the list of weights. - if headerByte >= 242 : this is one of 14 pre-defined weight distributions : | value |242|243|244|245|246|247|248|249|250|251|252|253|254|255| -| -------- | +| -------- |---|---|---|---|---|---|---|---|---|---|---|---|---|---| | Nb of 1s | 1 | 2 | 3 | 4 | 7 | 8 | 15| 16| 31| 32| 63| 64|127|128| |Complement| 1 | 2 | 1 | 4 | 1 | 8 | 1 | 16| 1 | 32| 1 | 64| 1 |128| From fe07eaa972f44f71443806c13d0286fad15153e2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 6 Jul 2016 02:25:44 +0200 Subject: [PATCH 11/33] simplified ZSTD_decodeSequence() --- lib/decompress/zstd_decompress.c | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 5a0d97657..9786ee171 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -688,16 +688,16 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) 0x2000, 0x4000, 0x8000, 0x10000 }; static const U32 ML_base[MaxML+1] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 34, 36, 38, 40, 44, 48, 56, 64, 80, 96, 0x80, 0x100, 0x200, 0x400, 0x800, - 0x1000, 0x2000, 0x4000, 0x8000, 0x10000 }; + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; static const U32 OF_base[MaxOff+1] = { - 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, - 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, - 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, - 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, /*fake*/ 1, 1 }; + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, /*fake*/ 1, 1 }; /* sequence */ { size_t offset; @@ -708,22 +708,18 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); } - if (offset < ZSTD_REP_NUM) { - if (llCode == 0 && offset <= 1) offset = 1-offset; + if (ofCode <= 1) { + if ((llCode == 0) & (offset <= 1)) offset = 1-offset; - if (offset != 0) { + if (offset) { size_t const temp = seqState->prevOffset[offset]; - if (offset != 1) { - seqState->prevOffset[2] = seqState->prevOffset[1]; - } + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[0] = offset = temp; - } else { offset = seqState->prevOffset[0]; } } else { - offset -= ZSTD_REP_MOVE; seqState->prevOffset[2] = seqState->prevOffset[1]; seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[0] = offset; @@ -731,7 +727,7 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) seq.offset = offset; } - seq.matchLength = ML_base[mlCode] + MINMATCH + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */ + seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */ if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream)); seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */ From 517e1ba623a2c922d41ffc45cb4916080edacf6b Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 6 Jul 2016 12:35:09 +0200 Subject: [PATCH 12/33] fixed dictBuilder issue with HC levels. Reported by Bartosz Taudul. --- lib/decompress/zstd_decompress.c | 1 - lib/dictBuilder/zdict.c | 52 +++++++++++++++++--------------- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 9786ee171..02498aea0 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -710,7 +710,6 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) if (ofCode <= 1) { if ((llCode == 0) & (offset <= 1)) offset = 1-offset; - if (offset) { size_t const temp = seqState->prevOffset[offset]; if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index dfcb4ab48..fa09acb36 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -34,11 +34,6 @@ /*-************************************** * Compiler Options ****************************************/ -/* Disable some Visual warning messages */ -#ifdef _MSC_VER -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#endif - /* Unix Large Files support (>4GB) */ #define _FILE_OFFSET_BITS 64 #if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */ @@ -58,13 +53,15 @@ #include "mem.h" /* read */ #include "error_private.h" -#include "fse.h" +#include "fse.h" /* FSE_normalizeCount, FSE_writeNCount */ #define HUF_STATIC_LINKING_ONLY #include "huf.h" #include "zstd_internal.h" /* includes zstd.h */ #include "xxhash.h" #include "divsufsort.h" -#define ZDICT_STATIC_LINKING_ONLY +#ifndef ZDICT_STATIC_LINKING_ONLY +# define ZDICT_STATIC_LINKING_ONLY +#endif #include "zdict.h" @@ -91,15 +88,15 @@ static const size_t g_min_fast_dictContent = 192; /*-************************************* * Console display ***************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); } #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */ #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ - if (ZDICT_GetMilliSpan(g_time) > refreshRate) \ + if (ZDICT_clockSpan(g_time) > refreshRate) \ { g_time = clock(); DISPLAY(__VA_ARGS__); \ if (g_displayLevel>=4) fflush(stdout); } } -static const unsigned refreshRate = 300; +static const unsigned refreshRate = CLOCKS_PER_SEC * 3 / 10; static clock_t g_time = 0; static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length) @@ -117,11 +114,9 @@ static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length) /*-******************************************************** * Helper functions **********************************************************/ -static unsigned ZDICT_GetMilliSpan(clock_t nPrevious) +static unsigned ZDICT_clockSpan(clock_t nPrevious) { - clock_t nCurrent = clock(); - unsigned nSpan = (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC); - return nSpan; + return clock() - nPrevious; } unsigned ZDICT_isError(size_t errorCode) { return ERR_isError(errorCode); } @@ -587,7 +582,9 @@ static void ZDICT_countEStats(EStats_ress_t esr, size_t cSize; if (srcSize > ZSTD_BLOCKSIZE_MAX) srcSize = ZSTD_BLOCKSIZE_MAX; /* protection vs large samples */ - ZSTD_copyCCtx(esr.zc, esr.ref); + { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref); + if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; } + } cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize); if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; } @@ -709,9 +706,14 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, } if (compressionLevel==0) compressionLevel=g_compressionLevel_default; params.cParams = ZSTD_getCParams(compressionLevel, averageSampleSize, dictBufferSize); - params.cParams.strategy = ZSTD_greedy; + //params.cParams.strategy = ZSTD_greedy; params.fParams.contentSizeFlag = 0; - ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0); + { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0); + if (ZSTD_isError(beginResult)) { + eSize = ERROR(GENERIC); + DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed "); + goto _cleanup; + } } /* collect stats on all files */ for (u=0; upos; u++) { U32 l = dictList[u].length; ptr -= l; - if (ptr<(BYTE*)dictBuffer) EXIT(GENERIC); /* should not happen */ + if (ptr<(BYTE*)dictBuffer) return ERROR(GENERIC); /* should not happen */ memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l); } } @@ -983,7 +985,7 @@ size_t ZDICT_trainFromBuffer_unsafe( params); } -_cleanup : + /* clean up */ free(dictList); return dictSize; } From a295b3170fa6b05f93faaf60ab152f1f50310dbe Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 6 Jul 2016 13:13:12 +0200 Subject: [PATCH 13/33] fixed conversion warning --- NEWS | 1 + lib/dictBuilder/zdict.c | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index 88e8c3b1f..4569e12fc 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ v0.7.3 added : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner. added : OpenBSD target, by Juan Francisco Cantero Hurtado +fixed : dictBuilder using HC levels, reported by Bartosz Taudul v0.7.2 fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski. diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index fa09acb36..04b20ec7c 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -99,6 +99,8 @@ static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: defau static const unsigned refreshRate = CLOCKS_PER_SEC * 3 / 10; static clock_t g_time = 0; +static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; } + static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length) { const BYTE* const b = (const BYTE*)ptr; @@ -114,11 +116,6 @@ static void ZDICT_printHex(U32 dlevel, const void* ptr, size_t length) /*-******************************************************** * Helper functions **********************************************************/ -static unsigned ZDICT_clockSpan(clock_t nPrevious) -{ - return clock() - nPrevious; -} - unsigned ZDICT_isError(size_t errorCode) { return ERR_isError(errorCode); } const char* ZDICT_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } From 445d49d8980a8c7dc64e20184ff49379dced8feb Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 6 Jul 2016 13:27:22 +0200 Subject: [PATCH 14/33] fixed conversion warning --- lib/dictBuilder/zdict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 04b20ec7c..e81cdb3ae 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -96,7 +96,7 @@ static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: defau if (ZDICT_clockSpan(g_time) > refreshRate) \ { g_time = clock(); DISPLAY(__VA_ARGS__); \ if (g_displayLevel>=4) fflush(stdout); } } -static const unsigned refreshRate = CLOCKS_PER_SEC * 3 / 10; +static const clock_t refreshRate = CLOCKS_PER_SEC * 3 / 10; static clock_t g_time = 0; static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; } From bcb5f77efa245deb5a5c62dc3518ab746cae370c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 6 Jul 2016 15:41:03 +0200 Subject: [PATCH 15/33] dictBuilder manages better samples of null size 0 and large size > 128 KB --- programs/dibio.c | 60 +++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/programs/dibio.c b/programs/dibio.c index d23476e31..73a63bf1d 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -43,13 +43,10 @@ #define MB *(1 <<20) #define GB *(1U<<30) -#define DICTLISTSIZE 10000 #define MEMMULT 11 static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_t)(512 MB) << sizeof(size_t)); #define NOISELENGTH 32 -#define PRIME1 2654435761U -#define PRIME2 2246822519U /*-************************************* @@ -60,17 +57,13 @@ static const size_t maxMemory = (sizeof(size_t) == 4) ? (2 GB - 64 MB) : ((size_ static unsigned g_displayLevel = 0; /* 0 : no display; 1: errors; 2: default; 4: full information */ #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ - if ((DIB_GetMilliSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \ + if ((DIB_clockSpan(g_time) > refreshRate) || (g_displayLevel>=4)) \ { g_time = clock(); DISPLAY(__VA_ARGS__); \ if (g_displayLevel>=4) fflush(stdout); } } -static const unsigned refreshRate = 150; +static const clock_t refreshRate = CLOCKS_PER_SEC * 2 / 10; static clock_t g_time = 0; -static unsigned DIB_GetMilliSpan(clock_t nPrevious) -{ - clock_t const nCurrent = clock(); - return (unsigned)(((nCurrent - nPrevious) * 1000) / CLOCKS_PER_SEC); -} +static clock_t DIB_clockSpan(clock_t nPrevious) { return clock() - nPrevious; } /*-************************************* @@ -97,13 +90,15 @@ unsigned DiB_isError(size_t errorCode) { return ERR_isError(errorCode); } const char* DiB_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + /* ******************************************************** * File related operations **********************************************************/ /** DiB_loadFiles() : * @return : nb of files effectively loaded into `buffer` */ -static unsigned DiB_loadFiles(void* buffer, size_t bufferSize, +static unsigned DiB_loadFiles(void* buffer, size_t* bufferSizePtr, size_t* fileSizes, const char** fileNamesTable, unsigned nbFiles) { @@ -112,18 +107,20 @@ static unsigned DiB_loadFiles(void* buffer, size_t bufferSize, unsigned n; for (n=0; n bufferSize-pos ? 0 : fs64); - FILE* const f = fopen(fileNamesTable[n], "rb"); - if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]); - DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]); - { size_t const readSize = fread(buff+pos, 1, fileSize, f); - if (readSize != fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); - pos += readSize; } - fileSizes[n] = fileSize; - fclose(f); - if (fileSize == 0) break; /* stop there, not enough memory to load all files */ - } + const char* const fileName = fileNamesTable[n]; + unsigned long long const fs64 = UTIL_getFileSize(fileName); + size_t const fileSize = (size_t) MIN(fs64, 128 KB); + if (fileSize > *bufferSizePtr-pos) break; + { FILE* const f = fopen(fileName, "rb"); + if (f==NULL) EXM_THROW(10, "zstd: dictBuilder: %s %s ", fileName, strerror(errno)); + DISPLAYUPDATE(2, "Loading %s... \r", fileName); + { size_t const readSize = fread(buff+pos, 1, fileSize, f); + if (readSize != fileSize) EXM_THROW(11, "Pb reading %s", fileName); + pos += readSize; } + fileSizes[n] = fileSize; + fclose(f); + } } + *bufferSizePtr = pos; return n; } @@ -137,26 +134,28 @@ static size_t DiB_findMaxMem(unsigned long long requiredMem) void* testmem = NULL; requiredMem = (((requiredMem >> 23) + 1) << 23); - requiredMem += 2 * step; + requiredMem += step; if (requiredMem > maxMemory) requiredMem = maxMemory; while (!testmem) { - requiredMem -= step; testmem = malloc((size_t)requiredMem); + requiredMem -= step; } free(testmem); - return (size_t)(requiredMem - step); + return (size_t)requiredMem; } static void DiB_fillNoise(void* buffer, size_t length) { - unsigned acc = PRIME1; + unsigned const prime1 = 2654435761U; + unsigned const prime2 = 2246822519U; + unsigned acc = prime1; size_t p=0;; for (p=0; p> 21); } } @@ -188,7 +187,6 @@ size_t ZDICT_trainFromBuffer_unsafe(void* dictBuffer, size_t dictBufferCapacity, ZDICT_params_t parameters); -#define MIN(a,b) ((a)<(b)?(a):(b)) int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, const char** fileNamesTable, unsigned nbFiles, ZDICT_params_t params) @@ -197,7 +195,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); unsigned long long const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); size_t const maxMem = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT; - size_t const benchedSize = MIN (maxMem, (size_t)totalSizeToLoad); + size_t benchedSize = MIN (maxMem, (size_t)totalSizeToLoad); void* const srcBuffer = malloc(benchedSize+NOISELENGTH); int result = 0; @@ -210,7 +208,7 @@ int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(benchedSize >> 20)); /* Load input buffer */ - nbFiles = DiB_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); + nbFiles = DiB_loadFiles(srcBuffer, &benchedSize, fileSizes, fileNamesTable, nbFiles); DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH); /* guard band, for end of buffer condition */ { size_t const dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize, From 99b045b70a08dcf73c924934105a3bd55da3cf6e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 6 Jul 2016 16:12:38 +0200 Subject: [PATCH 16/33] dictBuilder protection vs huge sample sets (>2 GB) --- lib/dictBuilder/zdict.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index e81cdb3ae..f559f5583 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -31,6 +31,12 @@ - Zstd homepage : https://www.zstd.net */ +/*-************************************** +* Tuning parameters +****************************************/ +#define ZDICT_MAX_SAMPLES_SIZE (1500U << 20) + + /*-************************************** * Compiler Options ****************************************/ @@ -481,7 +487,7 @@ static U32 ZDICT_dictSize(const dictItem* dictList) static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize, - const void* const buffer, const size_t bufferSize, /* buffer must end with noisy guard band */ + const void* const buffer, size_t bufferSize, /* buffer must end with noisy guard band */ const size_t* fileSizes, unsigned nbFiles, U32 shiftRatio, unsigned maxDictSize) { @@ -503,6 +509,10 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize, if (minRatio < MINRATIO) minRatio = MINRATIO; memset(doneMarks, 0, bufferSize+16); + /* limit sample set size (divsufsort limitation)*/ + if (bufferSize > ZDICT_MAX_SAMPLES_SIZE) DISPLAYLEVEL(3, "sample set too large : reduce to %u MB ...\n", (U32)(ZDICT_MAX_SAMPLES_SIZE>>20)); + while (bufferSize > ZDICT_MAX_SAMPLES_SIZE) bufferSize -= fileSizes[--nbFiles]; + /* sort */ DISPLAYLEVEL(2, "sorting %u files of total size %u MB ...\n", nbFiles, (U32)(bufferSize>>20)); divSuftSortResult = divsufsort((const unsigned char*)buffer, suffix, (int)bufferSize, 0); @@ -703,7 +713,6 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, } if (compressionLevel==0) compressionLevel=g_compressionLevel_default; params.cParams = ZSTD_getCParams(compressionLevel, averageSampleSize, dictBufferSize); - //params.cParams.strategy = ZSTD_greedy; params.fParams.contentSizeFlag = 0; { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0); if (ZSTD_isError(beginResult)) { From 29652e26189ccb2a41e7d93b4162231daa6b011d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 6 Jul 2016 16:25:46 +0200 Subject: [PATCH 17/33] sample set limitation closer to 2 GB --- lib/dictBuilder/zdict.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index f559f5583..f1af41962 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -34,7 +34,7 @@ /*-************************************** * Tuning parameters ****************************************/ -#define ZDICT_MAX_SAMPLES_SIZE (1500U << 20) +#define ZDICT_MAX_SAMPLES_SIZE (2000U << 20) /*-************************************** @@ -497,7 +497,6 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize, BYTE* doneMarks = (BYTE*)malloc((bufferSize+16)*sizeof(*doneMarks)); /* +16 for overflow security */ U32* filePos = (U32*)malloc(nbFiles * sizeof(*filePos)); U32 minRatio = nbFiles >> shiftRatio; - int divSuftSortResult; size_t result = 0; /* init */ @@ -510,18 +509,17 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize, memset(doneMarks, 0, bufferSize+16); /* limit sample set size (divsufsort limitation)*/ - if (bufferSize > ZDICT_MAX_SAMPLES_SIZE) DISPLAYLEVEL(3, "sample set too large : reduce to %u MB ...\n", (U32)(ZDICT_MAX_SAMPLES_SIZE>>20)); + if (bufferSize > ZDICT_MAX_SAMPLES_SIZE) DISPLAYLEVEL(3, "sample set too large : reduced to %u MB ...\n", (U32)(ZDICT_MAX_SAMPLES_SIZE>>20)); while (bufferSize > ZDICT_MAX_SAMPLES_SIZE) bufferSize -= fileSizes[--nbFiles]; /* sort */ DISPLAYLEVEL(2, "sorting %u files of total size %u MB ...\n", nbFiles, (U32)(bufferSize>>20)); - divSuftSortResult = divsufsort((const unsigned char*)buffer, suffix, (int)bufferSize, 0); - if (divSuftSortResult != 0) { result = ERROR(GENERIC); goto _cleanup; } + { int const divSuftSortResult = divsufsort((const unsigned char*)buffer, suffix, (int)bufferSize, 0); + if (divSuftSortResult != 0) { result = ERROR(GENERIC); goto _cleanup; } } suffix[bufferSize] = (int)bufferSize; /* leads into noise */ suffix0[0] = (int)bufferSize; /* leads into noise */ - { - /* build reverse suffix sort */ - size_t pos; + /* build reverse suffix sort */ + { size_t pos; for (pos=0; pos < bufferSize; pos++) reverseSuffix[suffix[pos]] = (U32)pos; /* build file pos */ From a3d03a3973818f1f64865224a4f1c03a9cc098a4 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 6 Jul 2016 16:27:17 +0200 Subject: [PATCH 18/33] added dependency --- programs/dibio.c | 1 + 1 file changed, 1 insertion(+) diff --git a/programs/dibio.c b/programs/dibio.c index 73a63bf1d..a61ea9cc6 100644 --- a/programs/dibio.c +++ b/programs/dibio.c @@ -30,6 +30,7 @@ #include /* memset */ #include /* fprintf, fopen, ftello64 */ #include /* clock_t, clock, CLOCKS_PER_SEC */ +#include /* errno */ #include "mem.h" /* read */ #include "error_private.h" From f246cf5423e0efea72df3b8fac321de471b52110 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 6 Jul 2016 20:30:52 +0200 Subject: [PATCH 19/33] ZSTD_decompress_usingDDict() compatible with Legacy mode --- NEWS | 1 + lib/common/zstd.h | 12 +++++++++--- lib/decompress/zstd_decompress.c | 16 ++++++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 4569e12fc..3793ed44b 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,7 @@ v0.7.3 added : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner. added : OpenBSD target, by Juan Francisco Cantero Hurtado fixed : dictBuilder using HC levels, reported by Bartosz Taudul +fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte v0.7.2 fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski. diff --git a/lib/common/zstd.h b/lib/common/zstd.h index 3dcd533db..1906c1859 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -408,12 +408,14 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds * Block functions ****************************************/ /*! Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~18 bytes, which is non-negligible on very small blocks. User will have to take in charge required information to regenerate data, such as compressed and content sizes. A few rules to respect : - Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB) - + If you need to compress more, it's recommended to use ZSTD_compress() instead, since frame metadata costs become negligible. - - Compressing or decompressing requires a context structure + + If you need to compress more, cut data into multiple blocks + + Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large. + - Compressing and decompressing require a context structure + Use ZSTD_createCCtx() and ZSTD_createDCtx() - It is necessary to init context before starting + compression : ZSTD_compressBegin() @@ -423,12 +425,16 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. In which case, nothing is produced into `dst`. + User must test for such outcome and deal directly with uncompressed data - + ZSTD_decompressBlock() doesn't accept uncompressed data as input !! + + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!! + + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history. + Use ZSTD_insertBlock() in such a case. + Insert block once it's copied into its final position. */ #define ZSTD_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */ ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful to track uncompressed blocks */ /*-************************************* diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 02498aea0..3cc38cd06 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -919,6 +919,16 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, } +/** ZSTD_insertBlock() : + insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ +ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) +{ + ZSTD_checkContinuity(dctx, blockStart); + dctx->previousDstEnd = (const char*)blockStart + blockSize; + return blockSize; +} + + size_t ZSTD_generateNxByte(void* dst, size_t dstCapacity, BYTE byte, size_t length) { if (length > dstCapacity) return ERROR(dstSize_tooSmall); @@ -1324,6 +1334,12 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, const void* src, size_t srcSize, const ZSTD_DDict* ddict) { +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) + { U32 const magicNumber = MEM_readLE32(src); + if (ZSTD_isLegacy(magicNumber)) + return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dictContent, ddict->dictContentSize, magicNumber); + } +#endif return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext, dst, dstCapacity, src, srcSize); From 52c04fe58f152455eac7fa27c46edf9741e6ae62 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 7 Jul 2016 11:53:18 +0200 Subject: [PATCH 20/33] removed `mem.h` dependency from `zstd.h` (experimental section) --- NEWS | 1 + lib/common/zstd.h | 37 +++++++++++++++++------------------- lib/compress/zstd_compress.c | 8 ++++---- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/NEWS b/NEWS index 3793ed44b..a62160edf 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,7 @@ added : `--` separator, stating that all following arguments are file names. Sug added : OpenBSD target, by Juan Francisco Cantero Hurtado fixed : dictBuilder using HC levels, reported by Bartosz Taudul fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte +modified : removed `mem.h` dependency from `zstd.h` (experimental section) v0.7.2 fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski. diff --git a/lib/common/zstd.h b/lib/common/zstd.h index 1906c1859..4de1b3dce 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -197,9 +197,6 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, * Use them only in association with static linking. * ==================================================================================== */ -/*--- Dependency ---*/ -#include "mem.h" /* U32 */ - /*--- Constants ---*/ #define ZSTD_MAGICNUMBER 0xFD2FB527 /* v0.7 */ #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U @@ -229,19 +226,19 @@ static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable f typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /*< from faster to stronger */ typedef struct { - U32 windowLog; /*< largest match distance : larger == more compression, more memory needed during decompression */ - U32 chainLog; /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ - U32 hashLog; /*< dispatch table : larger == faster, more memory */ - U32 searchLog; /*< nb of searches : larger == more compression, slower */ - U32 searchLength; /*< match length searched : larger == faster decompression, sometimes less compression */ - U32 targetLength; /*< acceptable match size for optimal parser (only) : larger == more compression, slower */ + unsigned windowLog; /*< largest match distance : larger == more compression, more memory needed during decompression */ + unsigned chainLog; /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + unsigned hashLog; /*< dispatch table : larger == faster, more memory */ + unsigned searchLog; /*< nb of searches : larger == more compression, slower */ + unsigned searchLength; /*< match length searched : larger == faster decompression, sometimes less compression */ + unsigned targetLength; /*< acceptable match size for optimal parser (only) : larger == more compression, slower */ ZSTD_strategy strategy; } ZSTD_compressionParameters; typedef struct { - U32 contentSizeFlag; /*< 1: content size will be in frame header (if known). */ - U32 checksumFlag; /*< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ - U32 noDictIDFlag; /*< 1: no dict ID will be saved into frame header (if dictionary compression) */ + unsigned contentSizeFlag; /*< 1: content size will be in frame header (if known). */ + unsigned checksumFlag; /*< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ + unsigned noDictIDFlag; /*< 1: no dict ID will be saved into frame header (if dictionary compression) */ } ZSTD_frameParameters; typedef struct { @@ -272,12 +269,12 @@ ZSTDLIB_API unsigned ZSTD_maxCLevel (void); /*! ZSTD_getParams() : * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`. * All fields of `ZSTD_frameParameters` are set to default (0) */ -ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSize, size_t dictSize); +ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize); /*! ZSTD_getCParams() : * @return ZSTD_compressionParameters structure for a selected compression level and srcSize. * `srcSize` value is optional, select 0 if not known */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize); +ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize); /*! ZSTD_checkCParams() : * Ensure param values remain within authorized range */ @@ -286,7 +283,7 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); /*! ZSTD_adjustCParams() : * optimize params for a given `srcSize` and `dictSize`. * both values are optional, select `0` if unknown. */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize); +ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); /*! ZSTD_compress_advanced() : * Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */ @@ -309,7 +306,7 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); ******************************************************************/ ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize); +ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx); ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); @@ -345,10 +342,10 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci */ typedef struct { - U64 frameContentSize; - U32 windowSize; - U32 dictID; - U32 checksumFlag; + unsigned long long frameContentSize; + unsigned windowSize; + unsigned dictID; + unsigned checksumFlag; } ZSTD_frameParams; ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 037466171..cd55c7ae7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -221,7 +221,7 @@ size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSiz Both `srcSize` and `dictSize` are optional (use 0 if unknown), but if both are 0, no optimization can be done. Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */ -ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize) +ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) { if (srcSize+dictSize == 0) return cPar; /* no size information available : no adjustment */ @@ -2407,7 +2407,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, * @return : 0, or an error code */ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, - ZSTD_parameters params, U64 pledgedSrcSize) + ZSTD_parameters params, unsigned long long pledgedSrcSize) { /* compression parameters verification and optimization */ { size_t const errorCode = ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize); @@ -2744,7 +2744,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV /*! ZSTD_getCParams() : * @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`. * Size values are optional, provide 0 if not known or unused */ -ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize) +ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) { ZSTD_compressionParameters cp; size_t const addedSize = srcSize ? 0 : 500; @@ -2765,7 +2765,7 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, si /*! ZSTD_getParams() : * same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`. * All fields of `ZSTD_frameParameters` are set to default (0) */ -ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSize, size_t dictSize) { +ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) { ZSTD_parameters params; ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize); memset(¶ms, 0, sizeof(params)); From f323bf7d327cdda51802f7c3bad5f5b8bc2401d2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 7 Jul 2016 13:14:21 +0200 Subject: [PATCH 21/33] added : ZSTD_getDecompressedSize() --- NEWS | 1 + lib/common/zstd.h | 9 ++++++++ lib/decompress/zstd_decompress.c | 22 ++++++++++++++++++++ lib/legacy/zstd_legacy.h | 24 ++++++++++++++++++++++ lib/legacy/zstd_v06.c | 4 +--- lib/legacy/zstd_v06.h | 2 +- programs/fuzzer.c | 35 ++++++++++++++++++++++---------- 7 files changed, 82 insertions(+), 15 deletions(-) diff --git a/NEWS b/NEWS index a62160edf..cc9a59d2e 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ v0.7.3 added : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner. +added : `ZSTD_getDecompressedSize()` added : OpenBSD target, by Juan Francisco Cantero Hurtado fixed : dictBuilder using HC levels, reported by Bartosz Taudul fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte diff --git a/lib/common/zstd.h b/lib/common/zstd.h index 4de1b3dce..0ba4ca812 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -296,6 +296,15 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, /*--- Advanced Decompression functions ---*/ +/** ZSTD_getDecompressedSize() : +* compatible with legacy mode +* @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - decompressed size is not provided within frame header + - frame header unknown / not supported + - frame header not completely provided (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + /*! ZSTD_createDCtx_advanced() : * Create a ZSTD decompression context using external alloc and free functions */ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 3cc38cd06..a43e80359 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -391,6 +391,28 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t } +/** ZSTD_getDecompressedSize() : +* compatible with legacy mode +* @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - decompressed size is not provided within frame header + - frame header unknown / not supported + - frame header not completely provided (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) { +#if ZSTD_LEGACY_SUPPORT + if (srcSize < 4) return 0; + { U32 const magic = MEM_readLE32(src); + if (ZSTD_isLegacy(magic)) return ZSTD_getDecompressedSize_legacy(src, srcSize); + } +#endif + { ZSTD_frameParams fparams; + size_t const frResult = ZSTD_getFrameParams(&fparams, src, srcSize); + if (frResult!=0) return 0; + return fparams.frameContentSize; + } +} + + /** ZSTD_decodeFrameHeader() : * `srcSize` must be the size provided by ZSTD_frameHeaderSize(). * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h index 22921beca..56b3e8215 100644 --- a/lib/legacy/zstd_legacy.h +++ b/lib/legacy/zstd_legacy.h @@ -69,6 +69,30 @@ MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE) } +MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, size_t srcSize) +{ + if (srcSize < 4) return 0; + + { U32 const magic = MEM_readLE32(src); + U32 const version = ZSTD_isLegacy(magic); + if (!version) return 0; /* not a supported legacy format */ + if (version < 5) return 0; /* no decompressed size in frame header */ + if (version==5) { + ZSTDv05_parameters fParams; + size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize); + if (frResult != 0) return 0; + return fParams.srcSize; + } + if (version==6) { + ZSTDv06_frameParams fParams; + size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize); + if (frResult != 0) return 0; + return fParams.frameContentSize; + } + return 0; /* should not be possible */ + } +} + MEM_STATIC size_t ZSTD_decompressLegacy( void* dst, size_t dstCapacity, const void* src, size_t compressedSize, diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 2640c86b3..ce6967eb8 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -36,7 +36,7 @@ #include "zstd_v06.h" #include /* size_t, ptrdiff_t */ #include /* memcpy */ -#include /* malloc, free, qsort */ +#include /* malloc, free, qsort */ @@ -535,8 +535,6 @@ ZSTDLIB_API size_t ZSTDv06_decompress_usingPreparedDCtx( -struct ZSTDv06_frameParams_s { U64 frameContentSize; U32 windowLog; }; - #define ZSTDv06_FRAMEHEADERSIZE_MAX 13 /* for static allocation */ static const size_t ZSTDv06_frameHeaderSize_min = 5; static const size_t ZSTDv06_frameHeaderSize_max = ZSTDv06_FRAMEHEADERSIZE_MAX; diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h index 55619bef2..177f14834 100644 --- a/lib/legacy/zstd_v06.h +++ b/lib/legacy/zstd_v06.h @@ -107,7 +107,7 @@ ZSTDLIB_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx, /*-************************ * Advanced Streaming API ***************************/ - +struct ZSTDv06_frameParams_s { unsigned long long frameContentSize; unsigned windowLog; }; typedef struct ZSTDv06_frameParams_s ZSTDv06_frameParams; ZSTDLIB_API size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ diff --git a/programs/fuzzer.c b/programs/fuzzer.c index cd87775ef..2126e124f 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -137,6 +137,12 @@ static int basicUnitTests(U32 seed, double compressibility) cSize=r ); DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100); + DISPLAYLEVEL(4, "test%3i : decompressed size test : ", testNb++); + { unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize); + if (rSize != CNBuffSize) goto _output_error; + } + DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize); CHECKPLUS( r , ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize), if (r != CNBuffSize) goto _output_error); @@ -390,19 +396,21 @@ static int basicUnitTests(U32 seed, double compressibility) U32 rSeed = 1; /* create batch of 3-bytes sequences */ - { int i; for (i=0; i < NB3BYTESSEQ; i++) { - _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255); - _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255); - _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255); - }} + { int i; + for (i=0; i < NB3BYTESSEQ; i++) { + _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255); + _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255); + _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255); + } } /* randomly fills CNBuffer with prepared 3-bytes sequences */ - { int i; for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */ - U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK; - ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0]; - ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1]; - ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2]; - } }} + { int i; + for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */ + U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK; + ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0]; + ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1]; + ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2]; + } } } DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++); { CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), CNBuffer, _3BYTESTESTLENGTH, 19) ); @@ -556,6 +564,11 @@ static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, U32 const maxD CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); } } } + /* Decompressed size test */ + { unsigned long long const rSize = ZSTD_getDecompressedSize(cBuffer, cSize); + CHECK(rSize != sampleSize, "decompressed size incorrect"); + } + /* frame header decompression test */ { ZSTD_frameParams dParams; size_t const check = ZSTD_getFrameParams(&dParams, cBuffer, cSize); From e09d38e921996784d1794bf436bde51e920e824f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 7 Jul 2016 13:17:37 +0200 Subject: [PATCH 22/33] removed `mem.h` dependency from `zbuff.h` (experimental section) --- lib/common/zbuff.h | 2 +- lib/compress/zbuff_compress.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/common/zbuff.h b/lib/common/zbuff.h index 65f01e563..7820db26d 100644 --- a/lib/common/zbuff.h +++ b/lib/common/zbuff.h @@ -185,7 +185,7 @@ ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); /*--- Advanced Streaming function ---*/ ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, - ZSTD_parameters params, U64 pledgedSrcSize); + ZSTD_parameters params, unsigned long long pledgedSrcSize); #endif /* ZBUFF_STATIC_LINKING_ONLY */ diff --git a/lib/compress/zbuff_compress.c b/lib/compress/zbuff_compress.c index ef8448ce5..837d22cf7 100644 --- a/lib/compress/zbuff_compress.c +++ b/lib/compress/zbuff_compress.c @@ -137,7 +137,7 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, - ZSTD_parameters params, U64 pledgedSrcSize) + ZSTD_parameters params, unsigned long long pledgedSrcSize) { /* allocate buffers */ { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; From 974f52fc5d2ad0c22f46ee0e0b1702d44fcf6e8f Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 7 Jul 2016 14:08:00 +0200 Subject: [PATCH 23/33] Added "dictionary decompression" example --- examples/README.md | 7 ++ examples/dictionary_decompression.c | 112 ++++++++++++++++++++++++++++ lib/.gitignore | 2 + 3 files changed, 121 insertions(+) create mode 100644 examples/README.md create mode 100644 examples/dictionary_decompression.c create mode 100644 lib/.gitignore diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 000000000..a3d59315e --- /dev/null +++ b/examples/README.md @@ -0,0 +1,7 @@ +Zstandard library : usage examples +================================== + +- [Dictionary decompression](dictionary_decompression.c) + Decompress multiple files using the same dictionary. + Compatible with Legacy modes. + Introduces usage of : `ZSTD_createDDict()` and `ZSTD_decompress_usingDDict()` diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c new file mode 100644 index 000000000..e307bea5b --- /dev/null +++ b/examples/dictionary_decompression.c @@ -0,0 +1,112 @@ +#include // exit +#include // printf +#include // strerror +#include // errno +#include // stat +#include + + +static off_t fsizeX(const char *filename) +{ + struct stat st; + if (stat(filename, &st) == 0) return st.st_size; + /* error */ + printf("stat: %s : %s \n", filename, strerror(errno)); + exit(1); +} + +static FILE* fopenX(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + printf("fopen: %s : %s \n", filename, strerror(errno)); + exit(2); +} + +static void* mallocX(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + printf("malloc: %s \n", strerror(errno)); + exit(3); +} + +static void* loadFileX(const char* fileName, size_t* size) +{ + off_t const buffSize = fsizeX(fileName); + FILE* const inFile = fopenX(fileName, "rb"); + void* const buffer = mallocX(buffSize); + size_t const readSize = fread(buffer, 1, buffSize, inFile); + if (readSize != (size_t)buffSize) { + printf("fread: %s : %s \n", fileName, strerror(errno)); + exit(4); + } + fclose(inFile); + *size = buffSize; + return buffer; +} + + +static const ZSTD_DDict* createDict(const char* dictFileName) +{ + size_t dictSize; + void* const dictBuffer = loadFileX(dictFileName, &dictSize); + const ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize); + free(dictBuffer); + return ddict; +} + + +/* prototype declared here, as it currently is part of experimental section */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + +static void decompress(const char* fname, const ZSTD_DDict* ddict) +{ + size_t cSize; + void* const cBuff = loadFileX(fname, &cSize); + unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize); + if (rSize==0) { + printf("%s : original size unknown \n", fname); + exit(5); + } + void* const rBuff = mallocX(rSize); + + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict); + + if (dSize != rSize) { + printf("error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize)); + exit(7); + } + + /* success */ + printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize); + + ZSTD_freeDCtx(dctx); + free(rBuff); + free(cBuff); +} + + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc<3) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s [FILES] dictionary\n", exeName); + return 1; + } + + /* load dictionary only once */ + const char* const dictName = argv[argc-1]; + const ZSTD_DDict* const dictPtr = createDict(dictName); + + int u; + for (u=1; u Date: Thu, 7 Jul 2016 14:17:40 +0200 Subject: [PATCH 24/33] removed "error_public.h" dependency from "zstd.h" --- lib/common/error_public.h | 6 +++++- lib/common/zstd.h | 11 ----------- programs/fuzzer.c | 3 ++- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/lib/common/error_public.h b/lib/common/error_public.h index e8cfcc918..29050b3b3 100644 --- a/lib/common/error_public.h +++ b/lib/common/error_public.h @@ -63,7 +63,11 @@ typedef enum { ZSTD_error_maxCode } ZSTD_ErrorCode; -/* note : compare with size_t function results using ZSTD_getError() */ +/*! ZSTD_getErrorCode() : + convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, + which can be used to compare directly with enum list published into "error_public.h" */ +ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +const char* ZSTD_getErrorString(ZSTD_ErrorCode code); #if defined (__cplusplus) diff --git a/lib/common/zstd.h b/lib/common/zstd.h index 0ba4ca812..31d596092 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -443,17 +443,6 @@ ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCa ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful to track uncompressed blocks */ -/*-************************************* -* Error management -***************************************/ -#include "error_public.h" -/*! ZSTD_getErrorCode() : - convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, - which can be used to compare directly with enum list published into "error_public.h" */ -ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); -ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); - - #endif /* ZSTD_STATIC_LINKING_ONLY */ #if defined (__cplusplus) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index 2126e124f..f38a48bf4 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -41,7 +41,8 @@ #include /* strcmp */ #include /* clock_t */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ -#include "zstd.h" /* ZSTD_VERSION_STRING, ZSTD_getErrorCode */ +#include "zstd.h" /* ZSTD_VERSION_STRING */ +#include "error_public.h" /* ZSTD_getErrorCode */ #include "zdict.h" /* ZDICT_trainFromBuffer */ #include "datagen.h" /* RDG_genBuffer */ #include "mem.h" From 19c27d27f1ede1cd6616821fe6c02a27321c6a8c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 7 Jul 2016 14:40:13 +0200 Subject: [PATCH 25/33] simplified legacy functions, no longer need magic number --- NEWS | 10 ++++++---- lib/decompress/zstd_decompress.c | 20 ++++++-------------- lib/legacy/zstd_legacy.h | 29 +++++++++++++++-------------- programs/fileio.c | 2 +- 4 files changed, 28 insertions(+), 33 deletions(-) diff --git a/NEWS b/NEWS index cc9a59d2e..06c94ce9c 100644 --- a/NEWS +++ b/NEWS @@ -1,10 +1,12 @@ v0.7.3 -added : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner. -added : `ZSTD_getDecompressedSize()` -added : OpenBSD target, by Juan Francisco Cantero Hurtado +New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner. +New : `ZSTD_getDecompressedSize()` +New : OpenBSD target, by Juan Francisco Cantero Hurtado +New : `examples` directory fixed : dictBuilder using HC levels, reported by Bartosz Taudul fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte -modified : removed `mem.h` dependency from `zstd.h` (experimental section) +modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section) +modified : legacy functions no longer need magic number v0.7.2 fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski. diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index a43e80359..c1d9197d4 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -398,12 +398,10 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t - decompressed size is not provided within frame header - frame header unknown / not supported - frame header not completely provided (`srcSize` too small) */ -unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) { -#if ZSTD_LEGACY_SUPPORT - if (srcSize < 4) return 0; - { U32 const magic = MEM_readLE32(src); - if (ZSTD_isLegacy(magic)) return ZSTD_getDecompressedSize_legacy(src, srcSize); - } +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) + if (ZSTD_isLegacy(src, srcSize)) return ZSTD_getDecompressedSize_legacy(src, srcSize); #endif { ZSTD_frameParams fparams; size_t const frResult = ZSTD_getFrameParams(&fparams, src, srcSize); @@ -1047,10 +1045,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) - { U32 const magicNumber = MEM_readLE32(src); - if (ZSTD_isLegacy(magicNumber)) - return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize, magicNumber); - } + if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize); #endif ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); ZSTD_checkContinuity(dctx, dst); @@ -1357,10 +1352,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) - { U32 const magicNumber = MEM_readLE32(src); - if (ZSTD_isLegacy(magicNumber)) - return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dictContent, ddict->dictContentSize, magicNumber); - } + if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dictContent, ddict->dictContentSize); #endif return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext, dst, dstCapacity, diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h index 56b3e8215..ab9634b32 100644 --- a/lib/legacy/zstd_legacy.h +++ b/lib/legacy/zstd_legacy.h @@ -54,8 +54,11 @@ extern "C" { @return : > 0 if supported by legacy decoder. 0 otherwise. return value is the version. */ -MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE) +MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize) { + U32 magicNumberLE; + if (srcSize<4) return 0; + magicNumberLE = MEM_readLE32(src); switch(magicNumberLE) { case ZSTDv01_magicNumberLE:return 1; @@ -73,10 +76,8 @@ MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, s { if (srcSize < 4) return 0; - { U32 const magic = MEM_readLE32(src); - U32 const version = ZSTD_isLegacy(magic); - if (!version) return 0; /* not a supported legacy format */ - if (version < 5) return 0; /* no decompressed size in frame header */ + { U32 const version = ZSTD_isLegacy(src, srcSize); + if (version < 5) return 0; /* no decompressed size in frame header, or not a legacy format */ if (version==5) { ZSTDv05_parameters fParams; size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize); @@ -96,20 +97,20 @@ MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, s MEM_STATIC size_t ZSTD_decompressLegacy( void* dst, size_t dstCapacity, const void* src, size_t compressedSize, - const void* dict,size_t dictSize, - U32 magicNumberLE) + const void* dict,size_t dictSize) { - switch(magicNumberLE) + U32 const version = ZSTD_isLegacy(src, compressedSize); + switch(version) { - case ZSTDv01_magicNumberLE : + case 1 : return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize); - case ZSTDv02_magicNumber : + case 2 : return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize); - case ZSTDv03_magicNumber : + case 3 : return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize); - case ZSTDv04_magicNumber : + case 4 : return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize); - case ZSTDv05_MAGICNUMBER : + case 5 : { size_t result; ZSTDv05_DCtx* const zd = ZSTDv05_createDCtx(); if (zd==NULL) return ERROR(memory_allocation); @@ -117,7 +118,7 @@ MEM_STATIC size_t ZSTD_decompressLegacy( ZSTDv05_freeDCtx(zd); return result; } - case ZSTDv06_MAGICNUMBER : + case 6 : { size_t result; ZSTDv06_DCtx* const zd = ZSTDv06_createDCtx(); if (zd==NULL) return ERROR(memory_allocation); diff --git a/programs/fileio.c b/programs/fileio.c index 6081e4af7..3eb8d881e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -700,7 +700,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* srcFileName) if (sizeCheck != toRead) EXM_THROW(31, "zstd: %s read error : cannot read header", srcFileName); { U32 const magic = MEM_readLE32(ress.srcBuffer); #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) - if (ZSTD_isLegacy(magic)) { + if (ZSTD_isLegacy(ress.srcBuffer, 4)) { filesize += FIO_decompressLegacyFrame(dstFile, srcFile, ress.dictBuffer, ress.dictBufferSize, magic); continue; } From aa2628da30759007c14dba4d5dee5d0a9ee86f48 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 7 Jul 2016 15:28:41 +0200 Subject: [PATCH 26/33] added : ZSTD_insertBlock(), basic tests --- programs/fuzzer.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/programs/fuzzer.c b/programs/fuzzer.c index f38a48bf4..f4ffdad04 100644 --- a/programs/fuzzer.c +++ b/programs/fuzzer.c @@ -345,13 +345,18 @@ static int basicUnitTests(U32 seed, double compressibility) if (ZSTD_isError(cSize)) goto _output_error; cSize2 = ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize); if (ZSTD_isError(cSize2)) goto _output_error; + memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* fake non-compressed block */ + cSize2 = ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize), + (char*)CNBuffer+dictSize+2*blockSize, blockSize); + if (ZSTD_isError(cSize2)) goto _output_error; DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++); CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) ); { CHECK_V( r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); if (r != blockSize) goto _output_error; } - { CHECK_V( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+blockSize, CNBuffSize, (char*)compressedBuffer+cSize, cSize2) ); + ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */ + { CHECK_V( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, CNBuffSize, (char*)compressedBuffer+cSize+blockSize, cSize2) ); if (r != blockSize) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); From 26f681451f5d5c8d71066d87f8942a1b87d81a63 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 8 Jul 2016 10:42:59 +0200 Subject: [PATCH 27/33] updated doc --- lib/README.md | 15 ++++++++++----- zstd_compression_format.md | 31 ++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/lib/README.md b/lib/README.md index 45e8e6fdc..935706506 100644 --- a/lib/README.md +++ b/lib/README.md @@ -45,14 +45,19 @@ It is used by `zstd` command line utility, and [7zip plugin](http://mcmilk.de/pr - compress/zbuff_compress.c - decompress/zbuff_decompress.c + #### Dictionary builder -To create dictionaries from training sets : +In order to create dictionaries from some training sets, +it's needed to include all files from [dictBuilder directory](dictBuilder/) + + +#### Legacy support + +Zstandard can decode previous formats, starting from v0.1. +Support for these format is provided in [folder legacy](legacy/). +It's also required to compile the library with `ZSTD_LEGACY_SUPPORT = 1`. -- dictBuilder/divsufsort.c -- dictBuilder/divsufsort.h -- dictBuilder/zdict.c -- dictBuilder/zdict.h #### Miscellaneous diff --git a/zstd_compression_format.md b/zstd_compression_format.md index 2fbe3fa4c..75cf4a83b 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -565,37 +565,46 @@ which tells how to decode the list of weights. | Nb of 1s | 1 | 2 | 3 | 4 | 7 | 8 | 15| 16| 31| 32| 63| 64|127|128| |Complement| 1 | 2 | 1 | 4 | 1 | 8 | 1 | 16| 1 | 32| 1 | 64| 1 |128| -_Note_ : complement is by using the "join to nearest power of 2" rule. +_Note_ : complement is found by using "join to nearest power of 2" rule. - if headerByte >= 128 : this is a direct representation, where each weight is written directly as a 4 bits field (0-15). The full representation occupies `((nbSymbols+1)/2)` bytes, meaning it uses a last full byte even if nbSymbols is odd. - `nbSymbols = headerByte - 127;` + `nbSymbols = headerByte - 127;`. + Note that maximum nbSymbols is 241-127 = 114. + A larger serie must necessarily use FSE compression. - if headerByte < 128 : the serie of weights is compressed by FSE. - The length of the compressed serie is `headerByte` (0-127). + The length of the FSE-compressed serie is `headerByte` (0-127). ##### FSE (Finite State Entropy) compression of huffman weights -The serie of weights is compressed using standard FSE compression. +The serie of weights is compressed using FSE compression. It's a single bitstream with 2 interleaved states, -using a single distribution table. +sharing a single distribution table. To decode an FSE bitstream, it is necessary to know its compressed size. Compressed size is provided by `headerByte`. -It's also necessary to know its maximum decompressed size. -In this case, it's `255`, since literal values range from `0` to `255`, +It's also necessary to know its maximum decompressed size, +which is `255`, since literal values span from `0` to `255`, and last symbol value is not represented. An FSE bitstream starts by a header, describing probabilities distribution. It will create a Decoding Table. -It is necessary to know the maximum accuracy of distribution -to properly allocate space for the Table. -For a list of huffman weights, this maximum is 7 bits. +Table must be pre-allocated, which requires to support a maximum accuracy. +For a list of huffman weights, recommended maximum is 7 bits. + +FSE header is [described in relevant chapter](#fse-distribution-table--condensed-format), +and so is [FSE bitstream](#bitstream). +The main difference is that Huffman header compression uses 2 states, +which share the same FSE distribution table. +Bitstream contains only FSE symbols, there are no interleaved "raw bitfields". +The number of symbols to decode is discovered +by tracking bitStream overflow condition. +When both states have overflowed the bitstream, end is reached. -FSE header and bitstreams are described in a separated chapter. ##### Conversion from weights to huffman prefix codes From ed3845d3fa6769824f57c5fb5bd0c96368e32137 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 8 Jul 2016 12:57:10 +0200 Subject: [PATCH 28/33] introduced ZSTD_WINDOWLOG_MAX_32 (#239), suggested by @GregSlazinski --- lib/common/zstd.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/common/zstd.h b/lib/common/zstd.h index 31d596092..304edd364 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -201,7 +201,9 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, #define ZSTD_MAGICNUMBER 0xFD2FB527 /* v0.7 */ #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U -#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? 25 : 27)) +#define ZSTD_WINDOWLOG_MAX_32 25 +#define ZSTD_WINDOWLOG_MAX_64 27 +#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) #define ZSTD_WINDOWLOG_MIN 18 #define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1) #define ZSTD_CHAINLOG_MIN 4 From c5fb5b7fcdf7adc13020d90172fdd74671a90ad0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 8 Jul 2016 13:13:37 +0200 Subject: [PATCH 29/33] support offset > 128 MB --- lib/decompress/zstd_decompress.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index c1d9197d4..a72a244ff 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -717,14 +717,14 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, - 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, /*fake*/ 1, 1 }; + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD }; /* sequence */ { size_t offset; if (!ofCode) offset = 0; else { - offset = OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits); /* <= 26 bits */ + offset = OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); } From c40ba718d735af6467de83eaf038f60bd5e1fe2a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 8 Jul 2016 15:39:02 +0200 Subject: [PATCH 30/33] updated spec --- zstd_compression_format.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/zstd_compression_format.md b/zstd_compression_format.md index 75cf4a83b..bb5d71be9 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -409,7 +409,7 @@ To decode a compressed block, the following elements are necessary : ### Literals section -Literals are compressed using huffman compression. +Literals are compressed using Huffman prefix codes. During sequence phase, literals will be entangled with match copy operations. All literals are regrouped in the first part of the block. They can be decoded first, and then copied during sequence operations, @@ -718,6 +718,9 @@ The Sequences section starts by a header, followed by optional Probability tables for each symbol type, followed by the bitstream. +| Header | (LitLengthTable) | (OffsetTable) | (MatchLengthTable) | bitStream | +| ------ | ---------------- | ------------- | ------------------ | --------- | + To decode the Sequence section, it's required to know its size. This size is deducted from `blockSize - literalSectionSize`. @@ -774,7 +777,7 @@ They define lengths from 0 to 131071 bytes. | Code | 0-15 | | ------ | ---- | -| value | Code | +| length | Code | | nbBits | 0 | @@ -798,7 +801,7 @@ __Default distribution__ When "compression mode" is "predef"", a pre-defined distribution is used for FSE compression. -Here is its definition. It uses an accuracy of 6 bits (64 states). +Below is its definition. It uses an accuracy of 6 bits (64 states). ``` short literalLengths_defaultDistribution[36] = { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, @@ -833,7 +836,7 @@ They define lengths from 3 to 131074 bytes. __Default distribution__ -When "compression mode" is defined as "default distribution", +When "compression mode" is defined as "predef", a pre-defined distribution is used for FSE compression. Here is its definition. It uses an accuracy of 6 bits (64 states). @@ -950,9 +953,11 @@ Probability is obtained from Value decoded by following formulae : It means value `0` becomes negative probability `-1`. `-1` is a special probability, which means `less than 1`. -Its effect on distribution table is described in next paragraph. +Its effect on distribution table is described in [next paragraph]. For the purpose of calculating cumulated distribution, it counts as one. +[next paragraph]:#fse-decoding--from-normalized-distribution-to-decoding-tables + When a symbol has a probability of `zero`, it is followed by a 2-bits repeat flag. This repeat flag tells how many probabilities of zeroes follow the current one. @@ -1040,7 +1045,7 @@ All sequences are stored in a single bitstream, read _backward_. It is therefore necessary to know the bitstream size, which is deducted from compressed block size. -The bit of the stream is followed by a set-bit-flag. +The last useful bit of the stream is followed by an end-bit-flag. Highest bit of last byte is this flag. It does not belong to the useful part of the bitstream. Therefore, last byte has 0-7 useful bits. @@ -1068,7 +1073,9 @@ Decoding starts by reading the nb of bits required to decode offset. It then does the same for match length, and then for literal length. -Offset / matchLength / litLength define a sequence, which can be applied. +Offset / matchLength / litLength define a sequence. +It starts by inserting the number of literals defined by `litLength`, +then continue by copying `matchLength` bytes from `currentPos - offset`. The next operation is to update states. Using rules pre-calculated in the decoding tables, From bd106070632a8b5fb40147f73e10d344be0ad86a Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 8 Jul 2016 19:16:57 +0200 Subject: [PATCH 31/33] updated spec --- lib/decompress/zstd_decompress.c | 10 +++++----- zstd_compression_format.md | 33 +++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index a72a244ff..42acd0d6a 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1284,8 +1284,8 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t struct ZSTD_DDict_s { - void* dictContent; - size_t dictContentSize; + void* dict; + size_t dictSize; ZSTD_DCtx* refContext; }; /* typedef'd tp ZSTD_CDict within zstd.h */ @@ -1317,8 +1317,8 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu return NULL; } } - ddict->dictContent = dictContent; - ddict->dictContentSize = dictSize; + ddict->dict = dictContent; + ddict->dictSize = dictSize; ddict->refContext = dctx; return ddict; } @@ -1338,7 +1338,7 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict) ZSTD_freeFunction const cFree = ddict->refContext->customMem.customFree; void* const opaque = ddict->refContext->customMem.opaque; ZSTD_freeDCtx(ddict->refContext); - cFree(opaque, ddict->dictContent); + cFree(opaque, ddict->dict); cFree(opaque, ddict); return 0; } diff --git a/zstd_compression_format.md b/zstd_compression_format.md index bb5d71be9..0db184cc3 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -16,7 +16,7 @@ Distribution of this document is unlimited. ### Version -0.0.2 (July 2016 - Work in progress - unfinished) +0.1.0 (08/07/16) Introduction @@ -1119,6 +1119,37 @@ with the new offset taking first spot. pushing the other ones by one position. +Dictionary format +----------------- + +`zstd` is compatible with "pure content" dictionaries, free of any format restriction. +But dictionaries created by `zstd --train` follow a format, described here. + +__Pre-requisites__ : a dictionary has a known length, + defined either by a buffer limit, or a file size. + +| Header | DictID | Stats | Content | +| ------ | ------ | ----- | ------- | + +__Header__ : 4 bytes ID, value 0xEC30A437, Little Endian format + +__Dict_ID__ : 4 bytes, stored in Little Endian format. + DictID can be any value, except 0 (which means no DictID). + It's used by decoders to check they use the correct dictionary. + +__Stats__ : Entropy tables, following the same format as a [compressed blocks]. + They are stored in following order : + Huffman tables for literals, FSE table for offset, + FSE table for matchLenth, and finally FSE table for litLength. + It's then followed by 3 offset values, populating recent offsets, + stored in order 4-bytes little endian each, for a total of 12 bytes. + +__Content__ : Where the actual dictionary content is. + Content depends on Dictionary size. + +[compressed blocks]: #compressed-block-format + Version changes --------------- +0.1.0 initial release From d4e103a04e9efc0073cc6e7d318928e16c74eb4d Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 8 Jul 2016 19:18:30 +0200 Subject: [PATCH 32/33] updated doc --- NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS b/NEWS index 06c94ce9c..6a27ae2cf 100644 --- a/NEWS +++ b/NEWS @@ -1,10 +1,12 @@ v0.7.3 +New : compression format specification New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner. New : `ZSTD_getDecompressedSize()` New : OpenBSD target, by Juan Francisco Cantero Hurtado New : `examples` directory fixed : dictBuilder using HC levels, reported by Bartosz Taudul fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte +fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section) modified : legacy functions no longer need magic number From 722e14bb654ebe34bf2c736f009e0ac81c5aba72 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Fri, 8 Jul 2016 19:22:16 +0200 Subject: [PATCH 33/33] fixed compilation error in decompression module --- lib/decompress/zstd_decompress.c | 2 +- zstd_compression_format.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 42acd0d6a..2645342c7 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1352,7 +1352,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) - if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dictContent, ddict->dictContentSize); + if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize); #endif return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext, dst, dstCapacity, diff --git a/zstd_compression_format.md b/zstd_compression_format.md index 0db184cc3..9f2227406 100644 --- a/zstd_compression_format.md +++ b/zstd_compression_format.md @@ -1135,17 +1135,17 @@ __Header__ : 4 bytes ID, value 0xEC30A437, Little Endian format __Dict_ID__ : 4 bytes, stored in Little Endian format. DictID can be any value, except 0 (which means no DictID). - It's used by decoders to check they use the correct dictionary. + It's used by decoders to check if they use the correct dictionary. __Stats__ : Entropy tables, following the same format as a [compressed blocks]. They are stored in following order : Huffman tables for literals, FSE table for offset, - FSE table for matchLenth, and finally FSE table for litLength. - It's then followed by 3 offset values, populating recent offsets, - stored in order 4-bytes little endian each, for a total of 12 bytes. + FSE table for matchLenth, and FSE table for litLength. + It's finally followed by 3 offset values, populating recent offsets, + stored in order, 4-bytes little endian each, for a total of 12 bytes. __Content__ : Where the actual dictionary content is. - Content depends on Dictionary size. + Content size depends on Dictionary size. [compressed blocks]: #compressed-block-format