ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #739 from facebook/refPrefix

Browse Source 
ZSTD_refPrefix
This commit is contained in:
Yann Collet
2017-06-29 04:36:03 -07:00
committed by GitHub
parent 811deaea6f 037466245f
commit 590937df20
8 changed files with 221 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/common/zstd_internal.h
+11 -7
View File
@@ -71,12 +71,17 @@
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2)
# include <stdio.h>
static unsigned g_debugLevel = ZSTD_DEBUG;
# define DEBUGLOG(l, ...) { \
if (l<=g_debugLevel) { \
fprintf(stderr, __FILE__ ": "); \
fprintf(stderr, __VA_ARGS__); \
fprintf(stderr, " \n"); \
/* recommended values for ZSTD_DEBUG display levels :
* 2 : reserved for currently active debugging path
* 3 : events once per object lifetime (CCtx, CDict)
* 4 : events once per frame
* 5 : events once per block
* 6 : events once per sequence (*very* verbose) */
# define DEBUGLOG(l, ...) { \
if (l<=ZSTD_DEBUG) { \
fprintf(stderr, __FILE__ ": "); \
fprintf(stderr, __VA_ARGS__); \
fprintf(stderr, " \n"); \
} }
#else
# define DEBUGLOG(l, ...) {} /* disabled */
@@ -98,7 +103,6 @@
* Common constants
***************************************/
#define ZSTD_OPT_NUM (1<<12)
#define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */
#define ZSTD_REP_NUM 3 /* number of repcodes */
#define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */
lib/compress/zstd_compress.c
+61 -62
View File
@@ -88,7 +88,6 @@ struct ZSTD_CCtx_s {
U32 hashLog3; /* dispatch table : larger == faster, more memory */
U32 loadedDictEnd; /* index of end of dictionary */
U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */
ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent" or "fullDict" only */
ZSTD_compressionStage_e stage;
U32 rep[ZSTD_REP_NUM];
U32 repToConfirm[ZSTD_REP_NUM];
@@ -118,8 +117,6 @@ struct ZSTD_CCtx_s {
unsigned* entropyScratchSpace;
/* streaming */
ZSTD_CDict* cdictLocal;
const ZSTD_CDict* cdict;
char* inBuff;
size_t inBuffSize;
size_t inToCompress;
@@ -132,6 +129,14 @@ struct ZSTD_CCtx_s {
ZSTD_cStreamStage streamStage;
U32 frameEnded;
/* Dictionary */
ZSTD_dictMode_e dictMode; /* select restricting dictionary to "rawContent" or "fullDict" only */
U32 dictContentByRef;
ZSTD_CDict* cdictLocal;
const ZSTD_CDict* cdict;
const void* prefix;
size_t prefixSize;
/* Multi-threading */
U32 nbThreads;
ZSTDMT_CCtx* mtctx;
@@ -322,11 +327,6 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
cctx->requestedParams.cParams.strategy = (ZSTD_strategy)value;
return 0;
#if 0
case ZSTD_p_windowSize : /* to be done later */
return ERROR(compressionParameter_unsupported);
#endif
case ZSTD_p_contentSizeFlag :
DEBUGLOG(5, "set content size flag = %u", (value>0));
/* Content size written in frame header _when known_ (default:1) */
@@ -343,10 +343,9 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
cctx->requestedParams.fParams.noDictIDFlag = (value==0);
return 0;
case ZSTD_p_refDictContent : /* to be done later */
return ERROR(compressionParameter_unsupported);
/* Dictionary parameters */
case ZSTD_p_dictMode :
if (cctx->cdict) return ERROR(stage_wrong); /* must be set before loading */
/* restrict dictionary mode, to "rawContent" or "fullDict" only */
ZSTD_STATIC_ASSERT((U32)ZSTD_dm_fullDict > (U32)ZSTD_dm_rawContent);
if (value > (unsigned)ZSTD_dm_fullDict)
@@ -354,6 +353,12 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v
cctx->dictMode = (ZSTD_dictMode_e)value;
return 0;
case ZSTD_p_refDictContent :
if (cctx->cdict) return ERROR(stage_wrong); /* must be set before loading */
/* dictionary content will be referenced, instead of copied */
cctx->dictContentByRef = value>0;
return 0;
case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize,
* even when referencing into Dictionary content
* default : 0 when using a CDict, 1 when using a Prefix */
@@ -417,7 +422,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
ZSTD_getCParams(cctx->compressionLevel, 0, dictSize);
cctx->cdictLocal = ZSTD_createCDict_advanced(
dict, dictSize,
0 /* byReference */, cctx->dictMode,
cctx->dictContentByRef, cctx->dictMode,
cParams, cctx->customMem);
cctx->cdict = cctx->cdictLocal;
if (cctx->cdictLocal == NULL)
@@ -426,19 +431,22 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
return 0;
}
/* Not ready yet ! */
size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize)
{
(void)cctx; (void)prefix; (void)prefixSize; /* to be done later */
if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
return ERROR(compressionParameter_unsupported);
}
size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
{
if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
cctx->cdict = cdict;
return ERROR(compressionParameter_unsupported);
cctx->prefix = NULL; /* exclusive */
cctx->prefixSize = 0;
return 0;
}
size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize)
{
if (cctx->streamStage != zcss_init) return ERROR(stage_wrong);
cctx->cdict = NULL; /* prefix discards any prior cdict */
cctx->prefix = prefix;
cctx->prefixSize = prefixSize;
return 0;
}
static void ZSTD_startNewCompression(ZSTD_CCtx* cctx)
@@ -3180,7 +3188,7 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* cctx,
if (dictMode==ZSTD_dm_rawContent)
return ZSTD_loadDictionaryContent(cctx, dict, dictSize);
if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) {
if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) {
if (dictMode == ZSTD_dm_auto) {
DEBUGLOG(5, "raw content dictionary detected");
return ZSTD_loadDictionaryContent(cctx, dict, dictSize);
@@ -3621,14 +3629,18 @@ size_t ZSTD_CStreamOutSize(void)
}
static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs,
ZSTD_parameters params,
unsigned long long pledgedSrcSize)
const void* dict, size_t dictSize, ZSTD_dictMode_e dictMode,
const ZSTD_CDict* cdict,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
{
DEBUGLOG(5, "ZSTD_resetCStream_internal");
/* params are supposed to be fully validated at this point */
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
assert(!((dict) && (cdict))); /* either dict or cdict, not both */
CHECK_F( ZSTD_compressBegin_internal(zcs,
NULL, 0, ZSTD_dm_auto,
zcs->cdict,
dict, dictSize, dictMode,
cdict,
params, pledgedSrcSize,
ZSTDb_buffered) );
@@ -3649,10 +3661,11 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
if (zcs->compressionLevel != ZSTD_CLEVEL_CUSTOM) {
params.cParams = ZSTD_getCParams(zcs->compressionLevel, pledgedSrcSize, 0 /* dictSize */);
}
return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->dictMode, zcs->cdict, params, pledgedSrcSize);
}
/*! ZSTD_initCStream_internal() :
* Note : not static, but hidden (not exposed). Used by zstdmt_compress.c
* Assumption 1 : params are valid
* Assumption 2 : either dict, or cdict, is defined, not both */
size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
@@ -3671,7 +3684,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
}
ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
0 /* byReference */, ZSTD_dm_auto,
zcs->dictContentByRef, zcs->dictMode,
params.cParams, zcs->customMem);
zcs->cdict = zcs->cdictLocal;
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
@@ -3687,7 +3700,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
zcs->requestedParams = params;
zcs->compressionLevel = ZSTD_CLEVEL_CUSTOM;
return ZSTD_resetCStream_internal(zcs, params, pledgedSrcSize);
return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->dictMode, zcs->cdict, params, pledgedSrcSize);
}
/* ZSTD_initCStream_usingCDict_advanced() :
@@ -3911,26 +3924,30 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
if (cctx->streamStage == zcss_init) {
/* transparent reset */
const void* const prefix = cctx->prefix;
size_t const prefixSize = cctx->prefixSize;
ZSTD_parameters params = cctx->requestedParams;
if (cctx->compressionLevel != ZSTD_CLEVEL_CUSTOM)
params.cParams = ZSTD_getCParams(cctx->compressionLevel,
cctx->pledgedSrcSizePlusOne-1, 0 /* dictSize */);
cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/);
cctx->prefix = NULL; cctx->prefixSize = 0; /* single usage */
assert(prefix==NULL || cctx->cdict==NULL); /* only one can be set */
#ifdef ZSTD_MULTITHREAD
if (cctx->nbThreads > 1) {
DEBUGLOG(4, "call ZSTDMT_initCStream_internal");
CHECK_F( ZSTDMT_initCStream_internal(cctx->mtctx, NULL, 0, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbThreads=%u", cctx->nbThreads);
CHECK_F( ZSTDMT_initCStream_internal(cctx->mtctx, prefix, prefixSize, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
cctx->streamStage = zcss_load;
} else
#endif
{
CHECK_F( ZSTD_resetCStream_internal(cctx, params, cctx->pledgedSrcSizePlusOne-1) );
CHECK_F( ZSTD_resetCStream_internal(cctx, prefix, prefixSize, cctx->dictMode, cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) );
} }
#ifdef ZSTD_MULTITHREAD
if (cctx->nbThreads > 1) {
size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
DEBUGLOG(4, "ZSTDMT_compressStream_generic : %u", (U32)flushMin);
DEBUGLOG(5, "ZSTDMT_compressStream_generic : %u", (U32)flushMin);
if ( ZSTD_isError(flushMin)
|| (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */
ZSTD_startNewCompression(cctx);
@@ -4098,42 +4115,24 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
},
};
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
/* This function just controls
* the monotonic memory budget increase of ZSTD_defaultCParameters[0].
* Run only once, on first ZSTD_getCParams() usage, when ZSTD_DEBUG is enabled
*/
MEM_STATIC void ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget(void)
{
# define ZSTD_TABLECOST(h,c) ((1<<(h)) + (1<<(c)))
# define ZDCP_FIELD(l,field) (ZSTD_defaultCParameters[0][l].field)
# define ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(l) { \
assert(ZDCP_FIELD(l,windowLog) <= ZDCP_FIELD(l+1,windowLog) ); \
assert(ZSTD_TABLECOST(ZDCP_FIELD(l,hashLog), ZDCP_FIELD(l,chainLog)) <= ZSTD_TABLECOST(ZDCP_FIELD(l+1,hashLog), ZDCP_FIELD(l+1,chainLog)) ); \
int level;
for (level=1; level<ZSTD_maxCLevel(); level++) {
ZSTD_compressionParameters const c1 = ZSTD_defaultCParameters[0][level];
ZSTD_compressionParameters const c2 = ZSTD_defaultCParameters[0][level+1];
DEBUGLOG(3, "controlling compression params level %i", level);
assert(c1.windowLog <= c2.windowLog);
# define ZSTD_TABLECOST(h,c) ((1<<(h)) + (1<<(c)))
assert(ZSTD_TABLECOST(c1.hashLog, c1.chainLog) <= ZSTD_TABLECOST(c2.hashLog, c2.chainLog));
}
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(1);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(2);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(3);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(4);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(5);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(6);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(7);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(8);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(9);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(10);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(11);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(12);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(13);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(14);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(15);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(16);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(17);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(18);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(19);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(20);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(21);
assert(ZSTD_maxCLevel()==22);
}
#endif
/*! ZSTD_getCParams() :
* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`.
lib/compress/zstdmt_compress.c
+1 -1
View File
@@ -586,7 +586,7 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs,
if (dict) {
ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize,
0 /* byRef */, ZSTD_dm_auto,
0 /* byRef */, ZSTD_dm_auto, /* note : a loadPrefix becomes an internal CDict */
params.cParams, zcs->cMem);
zcs->cdict = zcs->cdictLocal;
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
lib/decompress/zstd_decompress.c
+31 -13
View File
@@ -750,6 +750,7 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
const BYTE* const istart = (const BYTE* const)src;
const BYTE* const iend = istart + srcSize;
const BYTE* ip = istart;
DEBUGLOG(5, "ZSTD_decodeSeqHeaders");
/* check */
if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong);
@@ -930,12 +931,18 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
seq.offset = offset;
}
seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */
seq.matchLength = ML_base[mlCode]
+ ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */
if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream);
seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */
if (MEM_32bits() ||
(totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&seqState->DStream);
seq.litLength = LL_base[llCode]
+ ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */
if ( MEM_32bits()
|| (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) )
BIT_reloadDStream(&seqState->DStream);
DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u",
(U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset);
/* ANS state update */
FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */
@@ -975,7 +982,8 @@ size_t ZSTD_execSequence(BYTE* op,
/* copy Match */
if (sequence.offset > (size_t)(oLitEnd - base)) {
/* offset beyond prefix -> go into extDict */
if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
if (sequence.offset > (size_t)(oLitEnd - vBase))
return ERROR(corruption_detected);
match = dictEnd + (match - base);
if (match + sequence.matchLength <= dictEnd) {
memmove(oLitEnd, match, sequence.matchLength);
@@ -1043,9 +1051,12 @@ static size_t ZSTD_decompressSequences(
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
int nbSeq;
DEBUGLOG(5, "ZSTD_decompressSequences");
/* Build Decoding Tables */
{ size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
DEBUGLOG(5, "ZSTD_decodeSeqHeaders: size=%u, nbSeq=%i",
(U32)seqHSize, nbSeq);
if (ZSTD_isError(seqHSize)) return seqHSize;
ip += seqHSize;
}
@@ -1064,11 +1075,13 @@ static size_t ZSTD_decompressSequences(
nbSeq--;
{ seq_t const sequence = ZSTD_decodeSequence(&seqState);
size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd);
DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize);
if (ZSTD_isError(oneSeqSize)) return oneSeqSize;
op += oneSeqSize;
} }
/* check if reached exact end */
DEBUGLOG(5, "after decode loop, remaining nbSeq : %i", nbSeq);
if (nbSeq) return ERROR(corruption_detected);
/* save reps for next block */
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) dctx->entropy.rep[i] = (U32)(seqState.prevOffset[i]); }
@@ -1355,11 +1368,13 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
const void* src, size_t srcSize)
{ /* blockType == blockCompressed */
const BYTE* ip = (const BYTE*)src;
DEBUGLOG(5, "ZSTD_decompressBlock_internal");
if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);
/* Decode literals section */
{ size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize);
if (ZSTD_isError(litCSize)) return litCSize;
ip += litCSize;
srcSize -= litCSize;
@@ -1697,6 +1712,7 @@ static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skip
* or an error code, which can be tested using ZSTD_isError() */
size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
DEBUGLOG(5, "ZSTD_decompressContinue");
/* Sanity check */
if (srcSize != dctx->expected) return ERROR(srcSize_wrong); /* unauthorized */
if (dstCapacity) ZSTD_checkContinuity(dctx, dst);
@@ -1756,10 +1772,12 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
}
case ZSTDds_decompressLastBlock:
case ZSTDds_decompressBlock:
DEBUGLOG(5, "case ZSTDds_decompressBlock");
{ size_t rSize;
switch(dctx->bType)
{
case bt_compressed:
DEBUGLOG(5, "case bt_compressed");
rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize);
break;
case bt_raw :
@@ -1884,7 +1902,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
{
if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize);
{ U32 const magic = MEM_readLE32(dict);
if (magic != ZSTD_DICT_MAGIC) {
if (magic != ZSTD_MAGIC_DICTIONARY) {
return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */
} }
dctx->dictID = MEM_readLE32((const char*)dict + 4);
@@ -1964,7 +1982,7 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
ddict->entropyPresent = 0;
if (ddict->dictSize < 8) return 0;
{ U32 const magic = MEM_readLE32(ddict->dictContent);
if (magic != ZSTD_DICT_MAGIC) return 0; /* pure content mode */
if (magic != ZSTD_MAGIC_DICTIONARY) return 0; /* pure content mode */
}
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + 4);
@@ -2083,7 +2101,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
{
if (dictSize < 8) return 0;
if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return 0;
if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0;
return MEM_readLE32((const char*)dict + 4);
}
@@ -2323,7 +2341,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} }
/* Consume header (see ZSTDds_decodeFrameHeader) */
DEBUGLOG(5, "Consume header");
DEBUGLOG(4, "Consume header");
CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict));
if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
@@ -2336,7 +2354,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
}
/* control buffer memory usage */
DEBUGLOG(5, "Control max buffer memory usage");
DEBUGLOG(4, "Control max buffer memory usage");
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
@@ -2346,12 +2364,12 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
zds->blockSize = blockSize;
if ((zds->inBuffSize < blockSize) || (zds->outBuffSize < neededOutSize)) {
size_t const bufferSize = blockSize + neededOutSize;
DEBUGLOG(5, "inBuff : from %u to %u",
DEBUGLOG(4, "inBuff : from %u to %u",
(U32)zds->inBuffSize, (U32)blockSize);
DEBUGLOG(5, "outBuff : from %u to %u",
DEBUGLOG(4, "outBuff : from %u to %u",
(U32)zds->outBuffSize, (U32)neededOutSize);
if (zds->staticSize) { /* static DCtx */
DEBUGLOG(5, "staticSize : %u", (U32)zds->staticSize);
DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize);
assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */
if (bufferSize > zds->staticSize - sizeof(ZSTD_DCtx))
return ERROR(memory_allocation);
lib/dictBuilder/zdict.c
+3 -3
View File
@@ -94,7 +94,7 @@ const char* ZDICT_getErrorName(size_t errorCode) { return ERR_getErrorName(error
unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize)
{
if (dictSize < 8) return 0;
if (MEM_readLE32(dictBuffer) != ZSTD_DICT_MAGIC) return 0;
if (MEM_readLE32(dictBuffer) != ZSTD_MAGIC_DICTIONARY) return 0;
return MEM_readLE32((const char*)dictBuffer + 4);
}
@@ -854,7 +854,7 @@ size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity,
if (dictBufferCapacity < ZDICT_DICTSIZE_MIN) return ERROR(dstSize_tooSmall);
/* dictionary header */
MEM_writeLE32(header, ZSTD_DICT_MAGIC);
MEM_writeLE32(header, ZSTD_MAGIC_DICTIONARY);
{ U64 const randomID = XXH64(customDictContent, dictContentSize, 0);
U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768;
U32 const dictID = params.dictID ? params.dictID : compliantID;
@@ -906,7 +906,7 @@ size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictCo
}
/* add dictionary header (after entropy tables) */
MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC);
MEM_writeLE32(dictBuffer, ZSTD_MAGIC_DICTIONARY);
{ U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0);
U32 const compliantID = (randomID % ((1U<<31)-32768)) + 32768;
U32 const dictID = params.dictID ? params.dictID : compliantID;
lib/zstd.h
+25 -28
View File
@@ -360,6 +360,7 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
/* --- Constants ---*/
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* v0.7+ */
#define ZSTD_WINDOWLOG_MAX_32 27
#define ZSTD_WINDOWLOG_MAX_64 27
@@ -379,7 +380,7 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
#define ZSTD_FRAMEHEADERSIZE_MIN 6
static const size_t ZSTD_frameHeaderSize_prefix = 5;
static const size_t ZSTD_frameHeaderSize_prefix = 5; /* minimum input size to know frame header size */
static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
@@ -574,7 +575,10 @@ ZSTDLIB_API size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter par
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
typedef enum { ZSTD_dm_auto=0, ZSTD_dm_rawContent, ZSTD_dm_fullDict } ZSTD_dictMode_e;
typedef enum { ZSTD_dm_auto=0, /* dictionary is "full" if it starts with ZSTD_MAGIC_DICTIONARY, rawContent otherwize */
ZSTD_dm_rawContent, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */
ZSTD_dm_fullDict /* refuses to load a dictionary if it does not respect Zstandard's specification */
} ZSTD_dictMode_e;
/*! ZSTD_createCDict_advanced() :
* Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
@@ -944,26 +948,17 @@ typedef enum {
* The higher the value of selected strategy, the more complex it is,
* resulting in stronger and slower compression.
* Special: value 0 means "do not change strategy". */
#if 0
ZSTD_p_windowSize, /* Maximum allowed back-reference distance.
* Can be set to a more precise value than windowLog.
* Will be transparently reduced to closest possible inferior value
* (see Zstandard compression format) */
/* Not ready yet ! */
#endif
/* frame parameters */
ZSTD_p_contentSizeFlag=200, /* Content size is written into frame header _whenever known_ (default:1) */
ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */
ZSTD_p_dictIDFlag, /* When applicable, dictID of dictionary is provided in frame header (default:1) */
/* dictionary parameters */
ZSTD_p_refDictContent=300, /* Content of dictionary content will be referenced, instead of copied (default:0).
* This avoids duplicating dictionary content.
* But it also requires that dictionary buffer outlives its users */
/* Not ready yet ! <=================================== */
ZSTD_p_dictMode, /* Select how dictionary must be interpreted. Value must be from type ZSTD_dictMode_e.
/* dictionary parameters (must be set before ZSTD_CCtx_loadDictionary) */
ZSTD_p_dictMode=300, /* Select how dictionary content must be interpreted. Value must be from type ZSTD_dictMode_e.
* default : 0==auto : dictionary will be "full" if it respects specification, otherwise it will be "rawContent" */
ZSTD_p_refDictContent, /* Dictionary content will be referenced, instead of copied (default:0==byCopy).
* It requires that dictionary buffer outlives its users */
/* multi-threading parameters */
ZSTD_p_nbThreads=400, /* Select how many threads a compression job can spawn (default:1)
@@ -978,9 +973,9 @@ typedef enum {
* 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */
/* advanced parameters - may not remain available after API update */
ZSTD_p_forceMaxWindow=1100, /* Force back-references to remain < windowSize,
* even when referencing into Dictionary content.
* default : 0 when using a CDict, 1 when using a Prefix */
ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize,
* even when referencing into Dictionary content (default:0) */
} ZSTD_cParameter;
@@ -1007,8 +1002,8 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
* Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
* meaning "return to no-dictionary mode".
* Note 1 : Dictionary content will be copied internally,
* except if ZSTD_p_refDictContent is set.
* Note 1 : `dict` content will be copied internally,
* except if ZSTD_p_refDictContent is set before loading.
* Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters.
* For this reason, compression parameters cannot be changed anymore after loading a dictionary.
* It's also a CPU-heavy operation, with non-negligible impact on latency.
@@ -1017,7 +1012,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long lo
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
/*! ZSTD_CCtx_refCDict() :
* Ref a prepared dictionary, to be used for all next compression jobs.
* Reference a prepared dictionary, to be used for all next compression jobs.
* Note that compression parameters are enforced from within CDict,
* and supercede any compression parameter previously set within CCtx.
* The dictionary will remain valid for future compression jobs using same CCtx.
@@ -1030,16 +1025,18 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
/*! ZSTD_CCtx_refPrefix() :
* Reference a prefix (content-only dictionary) to bootstrap next compression job.
* Decompression will have to use same prefix.
* Prefix is only used once. Tables are discarded at end of compression job.
* If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict.
* Reference a prefix (single-usage dictionary) for next compression job.
* Decompression need same prefix to properly regenerate data.
* Prefix is **only used once**. Tables are discarded at end of compression job.
* Subsequent compression jobs will be done without prefix (if none is explicitly referenced).
* If there is a need to use same prefix multiple times, consider embedding it into a ZSTD_CDict instead.
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
* Special : Adding a NULL (or 0-size) dictionary invalidates any previous prefix, meaning "return to no-dictionary mode".
* Special : Adding any prefix (including NULL) invalidates any previous prefix or dictionary
* Note 1 : Prefix buffer is referenced. It must outlive compression job.
* Note 2 : Referencing a prefix involves building tables, which are dependent on compression parameters.
* It's a CPU-heavy operation, with non-negligible impact on latency. */
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); /* Not ready yet ! <===================================== */
* It's a CPU-heavy operation, with non-negligible impact on latency.
* Note 3 : it's possible to alter ZSTD_p_dictMode using ZSTD_CCtx_setParameter() */
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize);