ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added constant ZSTD_MAGIC_SKIPPABLE_MASK

Browse Source 
and updated several API comments
This commit is contained in:
Yann Collet
2018-11-13 17:36:35 -08:00
parent b83d1e7714
commit 2c8fde538f
4 changed files with 106 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/zstd_manual.html
+52 -36
View File
@@ -20,16 +20,17 @@
<li><a href="#Chapter10">Streaming decompression - HowTo</a></li> <li><a href="#Chapter10">Streaming decompression - HowTo</a></li>
<li><a href="#Chapter11">ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li> <li><a href="#Chapter11">ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li>
<li><a href="#Chapter12">Candidate API for promotion into stable</a></li> <li><a href="#Chapter12">Candidate API for promotion into stable</a></li>
<li><a href="#Chapter13">Frame size functions</a></li> <li><a href="#Chapter13">Purely experimental API</a></li>
<li><a href="#Chapter14">Memory management</a></li> <li><a href="#Chapter14">Frame size functions</a></li>
<li><a href="#Chapter15">Advanced compression functions</a></li> <li><a href="#Chapter15">Memory management</a></li>
<li><a href="#Chapter16">Advanced decompression functions</a></li> <li><a href="#Chapter16">Advanced compression functions</a></li>
<li><a href="#Chapter17">Advanced streaming functions</a></li> <li><a href="#Chapter17">Advanced decompression functions</a></li>
<li><a href="#Chapter18">Buffer-less and synchronous inner streaming functions</a></li> <li><a href="#Chapter18">Advanced streaming functions</a></li>
<li><a href="#Chapter19">Buffer-less streaming compression (synchronous mode)</a></li> <li><a href="#Chapter19">Buffer-less and synchronous inner streaming functions</a></li>
<li><a href="#Chapter20">Buffer-less streaming decompression (synchronous mode)</a></li> <li><a href="#Chapter20">Buffer-less streaming compression (synchronous mode)</a></li>
<li><a href="#Chapter21">New advanced API (experimental)</a></li> <li><a href="#Chapter21">Buffer-less streaming decompression (synchronous mode)</a></li>
<li><a href="#Chapter22">Block level API</a></li> <li><a href="#Chapter22">New advanced API (experimental)</a></li>
<li><a href="#Chapter23">Block level API</a></li>
</ol> </ol>
<hr> <hr>
<a name="Chapter1"></a><h2>Introduction</h2><pre> <a name="Chapter1"></a><h2>Introduction</h2><pre>
@@ -356,23 +357,38 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
<pre><b>int ZSTD_minCLevel(void); </b>/*!< minimum negative compression level allowed */<b> <pre><b>int ZSTD_minCLevel(void); </b>/*!< minimum negative compression level allowed */<b>
</b></pre><BR> </b></pre><BR>
<pre><b>typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, <a name="Chapter13"></a><h2>Purely experimental API</h2><pre>
ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; </b>/* from faster to stronger */<b> The following symbols and constants are not planned
to join "stable API" status anytime soon.
Some of them might even be removed in the future.
<BR></pre>
<pre><b>typedef enum { ZSTD_fast=1,
ZSTD_dfast=2,
ZSTD_greedy=3,
ZSTD_lazy=4,
ZSTD_lazy2=5,
ZSTD_btlazy2=6,
ZSTD_btopt=7,
ZSTD_btultra=8
</b>/* note : new strategies might be added in the future */<b>
} ZSTD_strategy;
</b></pre><BR> </b></pre><BR>
<pre><b>typedef struct { <pre><b>typedef struct {
unsigned windowLog; </b>/**< largest match distance : larger == more compression, more memory needed during decompression */<b> unsigned windowLog; </b>/**< largest match distance : larger == more compression, more memory needed during decompression */<b>
unsigned chainLog; </b>/**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */<b> unsigned chainLog; </b>/**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */<b>
unsigned hashLog; </b>/**< dispatch table : larger == faster, more memory */<b> unsigned hashLog; </b>/**< dispatch table : larger == faster, more memory */<b>
unsigned searchLog; </b>/**< nb of searches : larger == more compression, slower */<b> unsigned searchLog; </b>/**< nb of searches : larger == more compression, slower */<b>
unsigned searchLength; </b>/**< match length searched : larger == faster decompression, sometimes less compression */<b> unsigned searchLength; </b>/**< match length searched : larger == faster decompression, sometimes less compression */<b>
unsigned targetLength; </b>/**< acceptable match size for optimal parser (only) : larger == more compression, slower */<b> unsigned targetLength; </b>/**< acceptable match size for optimal parser (only) : larger == more compression, slower */<b>
ZSTD_strategy strategy; ZSTD_strategy strategy; </b>/**< see ZSTD_strategy definition above */<b>
} ZSTD_compressionParameters; } ZSTD_compressionParameters;
</b></pre><BR> </b></pre><BR>
<pre><b>typedef struct { <pre><b>typedef struct {
unsigned contentSizeFlag; </b>/**< 1: content size will be in frame header (when known) */<b> unsigned contentSizeFlag; </b>/**< 1: content size will be in frame header (when known) */<b>
unsigned checksumFlag; </b>/**< 1: generate a 32-bits checksum at end of frame, for error detection */<b> unsigned checksumFlag; </b>/**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */<b>
unsigned noDictIDFlag; </b>/**< 1: no dictID will be saved into frame header (if dictionary compression) */<b> unsigned noDictIDFlag; </b>/**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */<b>
} ZSTD_frameParameters; } ZSTD_frameParameters;
</b></pre><BR> </b></pre><BR>
<pre><b>typedef struct { <pre><b>typedef struct {
@@ -381,17 +397,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} ZSTD_parameters; } ZSTD_parameters;
</b></pre><BR> </b></pre><BR>
<pre><b>typedef enum { <pre><b>typedef enum {
ZSTD_dct_auto = 0, </b>/* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */<b> ZSTD_dct_auto = 0, </b>/* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */<b>
ZSTD_dct_rawContent, </b>/* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */<b> ZSTD_dct_rawContent = 1, </b>/* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */<b>
ZSTD_dct_fullDict </b>/* refuses to load a dictionary if it does not respect Zstandard's specification */<b> ZSTD_dct_fullDict = 2 </b>/* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */<b>
} ZSTD_dictContentType_e; } ZSTD_dictContentType_e;
</b></pre><BR> </b></pre><BR>
<pre><b>typedef enum { <pre><b>typedef enum {
ZSTD_dlm_byCopy = 0, </b>/**< Copy dictionary content internally */<b> ZSTD_dlm_byCopy = 0, </b>/**< Copy dictionary content internally */<b>
ZSTD_dlm_byRef, </b>/**< Reference dictionary content -- the dictionary buffer must outlive its users. */<b> ZSTD_dlm_byRef = 1, </b>/**< Reference dictionary content -- the dictionary buffer must outlive its users. */<b>
} ZSTD_dictLoadMethod_e; } ZSTD_dictLoadMethod_e;
</b></pre><BR> </b></pre><BR>
<a name="Chapter13"></a><h2>Frame size functions</h2><pre></pre> <a name="Chapter14"></a><h2>Frame size functions</h2><pre></pre>
<pre><b>size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); <pre><b>size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
</b><p> `src` should point to the start of a ZSTD encoded frame or skippable frame </b><p> `src` should point to the start of a ZSTD encoded frame or skippable frame
@@ -430,7 +446,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
or an error code (if srcSize is too small) or an error code (if srcSize is too small)
</p></pre><BR> </p></pre><BR>
<a name="Chapter14"></a><h2>Memory management</h2><pre></pre> <a name="Chapter15"></a><h2>Memory management</h2><pre></pre>
<pre><b>size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); <pre><b>size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
@@ -520,7 +536,7 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< t
</p></pre><BR> </p></pre><BR>
<a name="Chapter15"></a><h2>Advanced compression functions</h2><pre></pre> <a name="Chapter16"></a><h2>Advanced compression functions</h2><pre></pre>
<pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); <pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
</b><p> Create a digested dictionary for compression </b><p> Create a digested dictionary for compression
@@ -562,7 +578,7 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< t
</b><p> Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters </b><p> Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters
</p></pre><BR> </p></pre><BR>
<a name="Chapter16"></a><h2>Advanced decompression functions</h2><pre></pre> <a name="Chapter17"></a><h2>Advanced decompression functions</h2><pre></pre>
<pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size); <pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size);
</b><p> Tells if the content of `buffer` starts with a valid Frame Identifier. </b><p> Tells if the content of `buffer` starts with a valid Frame Identifier.
@@ -602,7 +618,7 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< t
When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code.
</p></pre><BR> </p></pre><BR>
<a name="Chapter17"></a><h2>Advanced streaming functions</h2><pre></pre> <a name="Chapter18"></a><h2>Advanced streaming functions</h2><pre></pre>
<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */<b> <h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */<b>
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/<b> size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/<b>
@@ -653,14 +669,14 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); </b>/**< note : ddict is referenced, it must outlive decompression session */<b> size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); </b>/**< note : ddict is referenced, it must outlive decompression session */<b>
size_t ZSTD_resetDStream(ZSTD_DStream* zds); </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b> size_t ZSTD_resetDStream(ZSTD_DStream* zds); </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b>
</pre></b><BR> </pre></b><BR>
<a name="Chapter18"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre> <a name="Chapter19"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre>
This is an advanced API, giving full control over buffer management, for users which need direct control over memory. This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
But it's also a complex one, with several restrictions, documented below. But it's also a complex one, with several restrictions, documented below.
Prefer normal streaming API for an easier experience. Prefer normal streaming API for an easier experience.
<BR></pre> <BR></pre>
<a name="Chapter19"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre> <a name="Chapter20"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre>
A ZSTD_CCtx object is required to track streaming operations. A ZSTD_CCtx object is required to track streaming operations.
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
ZSTD_CCtx object can be re-used multiple times within successive compression operations. ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@@ -696,7 +712,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b> size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b>
size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b> size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b>
</pre></b><BR> </pre></b><BR>
<a name="Chapter20"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre> <a name="Chapter21"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre>
A ZSTD_DCtx object is required to track streaming operations. A ZSTD_DCtx object is required to track streaming operations.
Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
A ZSTD_DCtx object can be re-used multiple times. A ZSTD_DCtx object can be re-used multiple times.
@@ -787,7 +803,7 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long
</pre></b><BR> </pre></b><BR>
<pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; <pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
</b></pre><BR> </b></pre><BR>
<a name="Chapter21"></a><h2>New advanced API (experimental)</h2><pre></pre> <a name="Chapter22"></a><h2>New advanced API (experimental)</h2><pre></pre>
<pre><b>typedef enum { <pre><b>typedef enum {
</b>/* Opened question : should we have a format ZSTD_f_auto ?<b> </b>/* Opened question : should we have a format ZSTD_f_auto ?<b>
@@ -1283,7 +1299,7 @@ size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx,
</p></pre><BR> </p></pre><BR>
<a name="Chapter22"></a><h2>Block level API</h2><pre></pre> <a name="Chapter23"></a><h2>Block level API</h2><pre></pre>
<pre><b></b><p> Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). <pre><b></b><p> Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
User will have to take in charge required information to regenerate data, such as compressed and content sizes. User will have to take in charge required information to regenerate data, such as compressed and content sizes.
lib/decompress/zstd_decompress.c
+7 -7
View File
@@ -187,7 +187,7 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size)
if (size < ZSTD_FRAMEIDSIZE) return 0; if (size < ZSTD_FRAMEIDSIZE) return 0;
{ U32 const magic = MEM_readLE32(buffer); { U32 const magic = MEM_readLE32(buffer);
if (magic == ZSTD_MAGICNUMBER) return 1; if (magic == ZSTD_MAGICNUMBER) return 1;
if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1; if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1;
} }
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
if (ZSTD_isLegacy(buffer, size)) return 1; if (ZSTD_isLegacy(buffer, size)) return 1;
@@ -242,7 +242,7 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s
if ( (format != ZSTD_f_zstd1_magicless) if ( (format != ZSTD_f_zstd1_magicless)
&& (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) {
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
/* skippable frame */ /* skippable frame */
if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) if (srcSize < ZSTD_SKIPPABLEHEADERSIZE)
return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */
@@ -355,7 +355,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) { while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) {
U32 const magicNumber = MEM_readLE32(src); U32 const magicNumber = MEM_readLE32(src);
if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
size_t skippableSize; size_t skippableSize;
if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) if (srcSize < ZSTD_SKIPPABLEHEADERSIZE)
return ERROR(srcSize_wrong); return ERROR(srcSize_wrong);
@@ -435,7 +435,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
return ZSTD_findFrameCompressedSizeLegacy(src, srcSize); return ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
#endif #endif
if ( (srcSize >= ZSTD_SKIPPABLEHEADERSIZE) if ( (srcSize >= ZSTD_SKIPPABLEHEADERSIZE)
&& (MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START ) { && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START ) {
return ZSTD_SKIPPABLEHEADERSIZE + MEM_readLE32((const BYTE*)src + ZSTD_FRAMEIDSIZE); return ZSTD_SKIPPABLEHEADERSIZE + MEM_readLE32((const BYTE*)src + ZSTD_FRAMEIDSIZE);
} else { } else {
const BYTE* ip = (const BYTE*)src; const BYTE* ip = (const BYTE*)src;
@@ -659,7 +659,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
{ U32 const magicNumber = MEM_readLE32(src); { U32 const magicNumber = MEM_readLE32(src);
DEBUGLOG(4, "reading magic number %08X (expecting %08X)", DEBUGLOG(4, "reading magic number %08X (expecting %08X)",
(U32)magicNumber, (U32)ZSTD_MAGICNUMBER); (U32)magicNumber, (U32)ZSTD_MAGICNUMBER);
if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
size_t skippableSize; size_t skippableSize;
if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) if (srcSize < ZSTD_SKIPPABLEHEADERSIZE)
return ERROR(srcSize_wrong); return ERROR(srcSize_wrong);
@@ -790,7 +790,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
assert(src != NULL); assert(src != NULL);
if (dctx->format == ZSTD_f_zstd1) { /* allows header */ if (dctx->format == ZSTD_f_zstd1) { /* allows header */
assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */ assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
memcpy(dctx->headerBuffer, src, srcSize); memcpy(dctx->headerBuffer, src, srcSize);
dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */ dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */
dctx->stage = ZSTDds_decodeSkippableHeader; dctx->stage = ZSTDds_decodeSkippableHeader;
@@ -1419,7 +1419,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
DEBUGLOG(4, "Consume header"); DEBUGLOG(4, "Consume header");
CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict)); CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE);
zds->stage = ZSTDds_skipFrame; zds->stage = ZSTDds_skipFrame;
} else { } else {
lib/zstd.h
+46 -19
View File
@@ -415,9 +415,12 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */ ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */
/* --- Constants ---*/ /* --- Constants ---*/
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* v0.8+ */
#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* v0.7+ */ /* all magic numbers are supposed read/written to/from files/memory using little-endian convention */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U #define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */
#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, are understood as skippable frames */
#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0
#define ZSTD_BLOCKSIZELOG_MAX 17 #define ZSTD_BLOCKSIZELOG_MAX 17
#define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) /* define, for static allocation */ #define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX) /* define, for static allocation */
@@ -439,6 +442,20 @@ ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level a
#define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */ #define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */
#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX #define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX
#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ #define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */
/****************************************************************************************
* Purely experimental API
****************************************************************************************
* The following symbols and constants are not planned
* to join "stable API" status anytime soon.
* Some of them might even be removed in the future.
* ***************************************************************************************/
#define ZSTD_LDM_MINMATCH_MAX 4096 #define ZSTD_LDM_MINMATCH_MAX 4096
#define ZSTD_LDM_MINMATCH_MIN 4 #define ZSTD_LDM_MINMATCH_MIN 4
#define ZSTD_LDM_BUCKETSIZELOG_MAX 8 #define ZSTD_LDM_BUCKETSIZELOG_MAX 8
@@ -450,23 +467,33 @@ ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level a
/* --- Advanced types --- */ /* --- Advanced types --- */
typedef enum { ZSTD_fast=1, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2,
ZSTD_btlazy2, ZSTD_btopt, ZSTD_btultra } ZSTD_strategy; /* from faster to stronger */ /* Compression strategies, listed from fastest to strongest */
typedef enum { ZSTD_fast=1,
ZSTD_dfast=2,
ZSTD_greedy=3,
ZSTD_lazy=4,
ZSTD_lazy2=5,
ZSTD_btlazy2=6,
ZSTD_btopt=7,
ZSTD_btultra=8
/* note : new strategies might be added in the future */
} ZSTD_strategy;
typedef struct { typedef struct {
unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */
unsigned hashLog; /**< dispatch table : larger == faster, more memory */ unsigned hashLog; /**< dispatch table : larger == faster, more memory */
unsigned searchLog; /**< nb of searches : larger == more compression, slower */ unsigned searchLog; /**< nb of searches : larger == more compression, slower */
unsigned searchLength; /**< match length searched : larger == faster decompression, sometimes less compression */ unsigned searchLength; /**< match length searched : larger == faster decompression, sometimes less compression */
unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */
ZSTD_strategy strategy; ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */
} ZSTD_compressionParameters; } ZSTD_compressionParameters;
typedef struct { typedef struct {
unsigned contentSizeFlag; /**< 1: content size will be in frame header (when known) */ unsigned contentSizeFlag; /**< 1: content size will be in frame header (when known) */
unsigned checksumFlag; /**< 1: generate a 32-bits checksum at end of frame, for error detection */ unsigned checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */
unsigned noDictIDFlag; /**< 1: no dictID will be saved into frame header (if dictionary compression) */ unsigned noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */
} ZSTD_frameParameters; } ZSTD_frameParameters;
typedef struct { typedef struct {
@@ -477,14 +504,14 @@ typedef struct {
typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params; typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params;
typedef enum { typedef enum {
ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */
ZSTD_dct_rawContent, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
ZSTD_dct_fullDict /* refuses to load a dictionary if it does not respect Zstandard's specification */ ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */
} ZSTD_dictContentType_e; } ZSTD_dictContentType_e;
typedef enum { typedef enum {
ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */
ZSTD_dlm_byRef, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ ZSTD_dlm_byRef = 1, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */
} ZSTD_dictLoadMethod_e; } ZSTD_dictLoadMethod_e;
programs/fileio.c
+1 -1
View File
@@ -2164,7 +2164,7 @@ FIO_analyzeFrames(fileInfo_t* info, FILE* const srcFile)
info->numActualFrames++; info->numActualFrames++;
} }
/* Skippable frame */ /* Skippable frame */
else if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { else if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
U32 const frameSize = MEM_readLE32(headerBuffer + 4); U32 const frameSize = MEM_readLE32(headerBuffer + 4);
long const seek = (long)(8 + frameSize - numBytesRead); long const seek = (long)(8 + frameSize - numBytesRead);
ERROR_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0, ERROR_IF(LONG_SEEK(srcFile, seek, SEEK_CUR) != 0,