clarified dictionary in format description
This commit is contained in:
@@ -2,6 +2,7 @@ v1.0.1
|
|||||||
New : contrib/pzstd, parallel version of zstd, by Nick Terrell
|
New : contrib/pzstd, parallel version of zstd, by Nick Terrell
|
||||||
Fixed : CLI -d output to stdout by default when input is stdin (#322)
|
Fixed : CLI -d output to stdout by default when input is stdin (#322)
|
||||||
Fixed : CLI correctly detects console on Mac OS-X
|
Fixed : CLI correctly detects console on Mac OS-X
|
||||||
|
Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319)
|
||||||
Fixed : zstd-pgo, reported by octoploid (#329)
|
Fixed : zstd-pgo, reported by octoploid (#329)
|
||||||
|
|
||||||
v1.0.0
|
v1.0.0
|
||||||
|
|||||||
@@ -8,7 +8,6 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/*-*******************************************************
|
/*-*******************************************************
|
||||||
* Compiler specifics
|
* Compiler specifics
|
||||||
*********************************************************/
|
*********************************************************/
|
||||||
|
|||||||
@@ -463,12 +463,6 @@ static U32 ZDICT_dictSize(const dictItem* dictList)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
|
|
||||||
if (ZDICT_clockSpan(displayClock) > refreshRate) \
|
|
||||||
{ displayClock = clock(); DISPLAY(__VA_ARGS__); \
|
|
||||||
if (g_displayLevel>=4) fflush(stdout); } }
|
|
||||||
static const clock_t refreshRate = CLOCKS_PER_SEC * 3 / 10;
|
|
||||||
|
|
||||||
static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
|
static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
|
||||||
const void* const buffer, size_t bufferSize, /* buffer must end with noisy guard band */
|
const void* const buffer, size_t bufferSize, /* buffer must end with noisy guard band */
|
||||||
const size_t* fileSizes, unsigned nbFiles,
|
const size_t* fileSizes, unsigned nbFiles,
|
||||||
@@ -481,6 +475,12 @@ static size_t ZDICT_trainBuffer(dictItem* dictList, U32 dictListSize,
|
|||||||
U32* filePos = (U32*)malloc(nbFiles * sizeof(*filePos));
|
U32* filePos = (U32*)malloc(nbFiles * sizeof(*filePos));
|
||||||
size_t result = 0;
|
size_t result = 0;
|
||||||
clock_t displayClock = 0;
|
clock_t displayClock = 0;
|
||||||
|
clock_t const refreshRate = CLOCKS_PER_SEC * 3 / 10;
|
||||||
|
|
||||||
|
# define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
|
||||||
|
if (ZDICT_clockSpan(displayClock) > refreshRate) \
|
||||||
|
{ displayClock = clock(); DISPLAY(__VA_ARGS__); \
|
||||||
|
if (g_displayLevel>=4) fflush(stdout); } }
|
||||||
|
|
||||||
/* init */
|
/* init */
|
||||||
DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
|
DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
|
||||||
|
|||||||
@@ -1126,10 +1126,10 @@ When `Repeated_Offset2` is used, it's swapped with `Repeated_Offset1`.
|
|||||||
Dictionary format
|
Dictionary format
|
||||||
-----------------
|
-----------------
|
||||||
|
|
||||||
`zstd` is compatible with "pure content" dictionaries, free of any format restriction.
|
`zstd` is compatible with "raw content" dictionaries, free of any format restriction.
|
||||||
But dictionaries created by `zstd --train` follow a format, described here.
|
But dictionaries created by `zstd --train` follow a format, described here.
|
||||||
|
|
||||||
__Pre-requisites__ : a dictionary has a known length,
|
__Pre-requisites__ : a dictionary has a size,
|
||||||
defined either by a buffer limit, or a file size.
|
defined either by a buffer limit, or a file size.
|
||||||
|
|
||||||
| `Magic_Number` | `Dictionary_ID` | `Entropy_Tables` | `Content` |
|
| `Magic_Number` | `Dictionary_ID` | `Entropy_Tables` | `Content` |
|
||||||
@@ -1157,14 +1157,15 @@ __`Entropy_Tables`__ : following the same format as a [compressed blocks].
|
|||||||
It's finally followed by 3 offset values, populating recent offsets,
|
It's finally followed by 3 offset values, populating recent offsets,
|
||||||
stored in order, 4-bytes little-endian each, for a total of 12 bytes.
|
stored in order, 4-bytes little-endian each, for a total of 12 bytes.
|
||||||
|
|
||||||
__`Content`__ : Where the actual dictionary content is.
|
__`Content`__ : The rest of the dictionary is its content.
|
||||||
Content size depends on Dictionary size.
|
The content act as a "past" in front of data to compress or decompress.
|
||||||
|
|
||||||
[compressed blocks]: #the-format-of-compressed_block
|
[compressed blocks]: #the-format-of-compressed_block
|
||||||
|
|
||||||
|
|
||||||
Version changes
|
Version changes
|
||||||
---------------
|
---------------
|
||||||
|
- 0.2.1 : clarify field names, by Przemyslaw Skibinski
|
||||||
- 0.2.0 : numerous format adjustments for zstd v0.8
|
- 0.2.0 : numerous format adjustments for zstd v0.8
|
||||||
- 0.1.2 : limit Huffman tree depth to 11 bits
|
- 0.1.2 : limit Huffman tree depth to 11 bits
|
||||||
- 0.1.1 : reserved dictID ranges
|
- 0.1.1 : reserved dictID ranges
|
||||||
|
|||||||
Reference in New Issue
Block a user