ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1316 from facebook/coldDict

Browse Source 
Cold dictionary mitigation
This commit is contained in:
Yann Collet
2018-09-14 10:37:46 -07:00
committed by GitHub
parent b287b8f7c6 5512400677
commit 0e5b447aaa
6 changed files with 189 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEWS
+5
View File
@@ -1,3 +1,8 @@
v1.3.6
perf: much faster dictionary builder, by @jenniferliu
api : reduced DDict size by 2 KB
misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97
v1.3.5 v1.3.5
perf: much faster dictionary compression, by @felixhandte perf: much faster dictionary compression, by @felixhandte
perf: small quality improvement for dictionary generation, by @terrelln perf: small quality improvement for dictionary generation, by @terrelln
contrib/largeNbDicts/largeNbDicts.c
+37 -23
View File
@@ -19,8 +19,8 @@
/*--- Dependencies ---*/ /*--- Dependencies ---*/
#include <stddef.h> /* size_t */ #include <stddef.h> /* size_t */
#include <stdlib.h> /* malloc, free */ #include <stdlib.h> /* malloc, free, abort */
#include <stdio.h> /* printf */ #include <stdio.h> /* fprintf */
#include <assert.h> /* assert */ #include <assert.h> /* assert */
#include "util.h" #include "util.h"
@@ -49,9 +49,9 @@
/*--- Macros ---*/ /*--- Macros ---*/
#define CONTROL(c) assert(c) #define CONTROL(c) { if (!(c)) abort(); }
#undef MIN #undef MIN
#define MIN(a,b) ((a) < (b) ? (a) : (b)) #define MIN(a,b) ((a) < (b) ? (a) : (b))
/*--- Display Macros ---*/ /*--- Display Macros ---*/
@@ -226,42 +226,50 @@ void shrinkSizes(slice_collection_t collection,
} }
slice_collection_t splitSlices(slice_collection_t srcSlices, size_t blockSize) /* splitSlices() :
* nbSlices : if == 0, nbSlices is automatically determined from srcSlices and blockSize.
* otherwise, creates exactly nbSlices slices,
* by either truncating input (when smaller)
* or repeating input from beginning */
static slice_collection_t
splitSlices(slice_collection_t srcSlices, size_t blockSize, size_t nbSlices)
{ {
if (blockSize==0) blockSize = (size_t)(-1); /* means "do not cut" */ if (blockSize==0) blockSize = (size_t)(-1); /* means "do not cut" */
size_t nbBlocks = 0; size_t nbSrcBlocks = 0;
for (size_t ssnb=0; ssnb < srcSlices.nbSlices; ssnb++) { for (size_t ssnb=0; ssnb < srcSlices.nbSlices; ssnb++) {
size_t pos = 0; size_t pos = 0;
while (pos <= srcSlices.capacities[ssnb]) { while (pos <= srcSlices.capacities[ssnb]) {
nbBlocks++; nbSrcBlocks++;
pos += blockSize; pos += blockSize;
} }
} }
void** const sliceTable = (void**)malloc(nbBlocks * sizeof(*sliceTable)); if (nbSlices == 0) nbSlices = nbSrcBlocks;
size_t* const capacities = (size_t*)malloc(nbBlocks * sizeof(*capacities));
void** const sliceTable = (void**)malloc(nbSlices * sizeof(*sliceTable));
size_t* const capacities = (size_t*)malloc(nbSlices * sizeof(*capacities));
if (sliceTable == NULL || capacities == NULL) { if (sliceTable == NULL || capacities == NULL) {
free(sliceTable); free(sliceTable);
free(capacities); free(capacities);
return kNullCollection; return kNullCollection;
} }
size_t blockNb = 0; size_t ssnb = 0;
for (size_t ssnb=0; ssnb < srcSlices.nbSlices; ssnb++) { for (size_t sliceNb=0; sliceNb < nbSlices; ) {
ssnb = (ssnb + 1) % srcSlices.nbSlices;
size_t pos = 0; size_t pos = 0;
char* const ptr = (char*)srcSlices.slicePtrs[ssnb]; char* const ptr = (char*)srcSlices.slicePtrs[ssnb];
while (pos < srcSlices.capacities[ssnb]) { while (pos < srcSlices.capacities[ssnb] && sliceNb < nbSlices) {
size_t const size = MIN(blockSize, srcSlices.capacities[ssnb] - pos); size_t const size = MIN(blockSize, srcSlices.capacities[ssnb] - pos);
sliceTable[blockNb] = ptr + pos; sliceTable[sliceNb] = ptr + pos;
capacities[blockNb] = size; capacities[sliceNb] = size;
blockNb++; sliceNb++;
pos += blockSize; pos += blockSize;
} }
} }
assert(blockNb == nbBlocks);
slice_collection_t result; slice_collection_t result;
result.nbSlices = nbBlocks; result.nbSlices = nbSlices;
result.slicePtrs = sliceTable; result.slicePtrs = sliceTable;
result.capacities = capacities; result.capacities = capacities;
return result; return result;
@@ -329,6 +337,7 @@ static buffer_collection_t
createBufferCollection_fromFiles(const char* const * fileNamesTable, unsigned nbFiles) createBufferCollection_fromFiles(const char* const * fileNamesTable, unsigned nbFiles)
{ {
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
assert(totalSizeToLoad != UTIL_FILESIZE_UNKNOWN);
assert(totalSizeToLoad <= BENCH_SIZE_MAX); assert(totalSizeToLoad <= BENCH_SIZE_MAX);
size_t const loadedSize = (size_t)totalSizeToLoad; size_t const loadedSize = (size_t)totalSizeToLoad;
assert(loadedSize > 0); assert(loadedSize > 0);
@@ -565,7 +574,9 @@ static int benchMem(slice_collection_t dstBlocks,
* @return : 0 is success, 1+ otherwise */ * @return : 0 is success, 1+ otherwise */
int bench(const char** fileNameTable, unsigned nbFiles, int bench(const char** fileNameTable, unsigned nbFiles,
const char* dictionary, const char* dictionary,
size_t blockSize, int clevel, unsigned nbDictMax, int nbRounds) size_t blockSize, int clevel,
unsigned nbDictMax, unsigned nbBlocks,
int nbRounds)
{ {
int result = 0; int result = 0;
@@ -577,8 +588,8 @@ int bench(const char** fileNameTable, unsigned nbFiles,
DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n", DISPLAYLEVEL(3, "created src buffer of size %.1f MB \n",
(double)srcSize / (1 MB)); (double)srcSize / (1 MB));
slice_collection_t const srcSlices = splitSlices(srcs.slices, blockSize); slice_collection_t const srcSlices = splitSlices(srcs.slices, blockSize, nbBlocks);
unsigned const nbBlocks = (unsigned)(srcSlices.nbSlices); nbBlocks = (unsigned)(srcSlices.nbSlices);
DISPLAYLEVEL(3, "split input into %u blocks ", nbBlocks); DISPLAYLEVEL(3, "split input into %u blocks ", nbBlocks);
if (blockSize) if (blockSize)
DISPLAYLEVEL(3, "of max size %u bytes ", (unsigned)blockSize); DISPLAYLEVEL(3, "of max size %u bytes ", (unsigned)blockSize);
@@ -596,10 +607,10 @@ int bench(const char** fileNameTable, unsigned nbFiles,
buffer_t dstBuffer = createBuffer(dstBufferCapacity); buffer_t dstBuffer = createBuffer(dstBufferCapacity);
CONTROL(dstBuffer.ptr != NULL); CONTROL(dstBuffer.ptr != NULL);
void** const sliceTable = (void**)malloc(nbBlocks * sizeof(*sliceTable)); void** const sliceTable = malloc(nbBlocks * sizeof(*sliceTable));
CONTROL(sliceTable != NULL); CONTROL(sliceTable != NULL);
{ char* const ptr = (char*)dstBuffer.ptr; { char* const ptr = dstBuffer.ptr;
size_t pos = 0; size_t pos = 0;
for (size_t snb=0; snb < nbBlocks; snb++) { for (size_t snb=0; snb < nbBlocks; snb++) {
sliceTable[snb] = ptr + pos; sliceTable[snb] = ptr + pos;
@@ -727,6 +738,7 @@ int usage(const char* exeName)
DISPLAY ("-# : use compression level # (default: %u) \n", CLEVEL_DEFAULT); DISPLAY ("-# : use compression level # (default: %u) \n", CLEVEL_DEFAULT);
DISPLAY ("-D # : use # as a dictionary (default: create one) \n"); DISPLAY ("-D # : use # as a dictionary (default: create one) \n");
DISPLAY ("-i# : nb benchmark rounds (default: %u) \n", BENCH_TIME_DEFAULT_S); DISPLAY ("-i# : nb benchmark rounds (default: %u) \n", BENCH_TIME_DEFAULT_S);
DISPLAY ("--nbBlocks=#: use # blocks for bench (default: one per file) \n");
DISPLAY ("--nbDicts=# : create # dictionaries for bench (default: one per block) \n"); DISPLAY ("--nbDicts=# : create # dictionaries for bench (default: one per block) \n");
DISPLAY ("-h : help (this text) \n"); DISPLAY ("-h : help (this text) \n");
return 0; return 0;
@@ -755,6 +767,7 @@ int main (int argc, const char** argv)
int cLevel = CLEVEL_DEFAULT; int cLevel = CLEVEL_DEFAULT;
size_t blockSize = BLOCKSIZE_DEFAULT; size_t blockSize = BLOCKSIZE_DEFAULT;
size_t nbDicts = 0; /* determine nbDicts automatically: 1 dictionary per block */ size_t nbDicts = 0; /* determine nbDicts automatically: 1 dictionary per block */
size_t nbBlocks = 0; /* determine nbBlocks automatically, from source and blockSize */
for (int argNb = 1; argNb < argc ; argNb++) { for (int argNb = 1; argNb < argc ; argNb++) {
const char* argument = argv[argNb]; const char* argument = argv[argNb];
@@ -766,6 +779,7 @@ int main (int argc, const char** argv)
if (longCommandWArg(&argument, "-B")) { blockSize = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "-B")) { blockSize = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--blockSize=")) { blockSize = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--blockSize=")) { blockSize = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--nbDicts=")) { nbDicts = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--nbDicts=")) { nbDicts = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--nbBlocks=")) { nbBlocks = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "--clevel=")) { cLevel = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "--clevel=")) { cLevel = readU32FromChar(&argument); continue; }
if (longCommandWArg(&argument, "-")) { cLevel = readU32FromChar(&argument); continue; } if (longCommandWArg(&argument, "-")) { cLevel = readU32FromChar(&argument); continue; }
/* anything that's not a command is a filename */ /* anything that's not a command is a filename */
@@ -783,7 +797,7 @@ int main (int argc, const char** argv)
filenameTable = UTIL_createFileList(nameTable, nameIdx, &buffer_containing_filenames, &nbFiles, 1 /* follow_links */); filenameTable = UTIL_createFileList(nameTable, nameIdx, &buffer_containing_filenames, &nbFiles, 1 /* follow_links */);
} }
int result = bench(filenameTable, nbFiles, dictionary, blockSize, cLevel, nbDicts, nbRounds); int result = bench(filenameTable, nbFiles, dictionary, blockSize, cLevel, nbDicts, nbBlocks, nbRounds);
free(buffer_containing_filenames); free(buffer_containing_filenames);
free(nameTable); free(nameTable);
doc/zstd_manual.html
+3
View File
@@ -182,6 +182,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only.
`dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict `dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict
Note : A ZSTD_CDict can be created with an empty dictionary, but it is inefficient for small data.
</p></pre><BR> </p></pre><BR>
<pre><b>size_t ZSTD_freeCDict(ZSTD_CDict* CDict); <pre><b>size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
@@ -196,6 +197,8 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
Note that compression level is decided during dictionary creation. Note that compression level is decided during dictionary creation.
Frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) Frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no)
Note : ZSTD_compress_usingCDict() can be used with a ZSTD_CDict created from an empty dictionary.
But it is inefficient for small data, and it is recommended to use ZSTD_compressCCtx().
</p></pre><BR> </p></pre><BR>
<pre><b>ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); <pre><b>ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
lib/common/compiler.h
+22 -5
View File
@@ -89,20 +89,37 @@
#endif #endif
/* prefetch /* prefetch
* can be disabled, by declaring NO_PREFETCH macro */ * can be disabled, by declaring NO_PREFETCH macro
* All prefetch invocations use a single default locality 2,
* generating instruction prefetcht1,
* which, according to Intel, means "load data into L2 cache".
* This is a good enough "middle ground" for the time being,
* though in theory, it would be better to specialize locality depending on data being prefetched.
* Tests could not determine any sensible difference based on locality value. */
#if defined(NO_PREFETCH) #if defined(NO_PREFETCH)
# define PREFETCH(ptr) /* disabled */ # define PREFETCH(ptr) (void)(ptr) /* disabled */
#else #else
# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ # if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */
# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ # include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
# define PREFETCH(ptr) _mm_prefetch((const char*)ptr, _MM_HINT_T0) # define PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) # elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
# define PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0) # define PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
# else # else
# define PREFETCH(ptr) /* disabled */ # define PREFETCH(ptr) (void)(ptr) /* disabled */
# endif # endif
#endif /* NO_PREFETCH */ #endif /* NO_PREFETCH */
#define CACHELINE_SIZE 64
#define PREFETCH_AREA(p, s) { \
const char* const _ptr = (const char*)(p); \
size_t const _size = (size_t)(s); \
size_t _pos; \
for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \
PREFETCH(_ptr + _pos); \
} \
}
/* disable warnings */ /* disable warnings */
#ifdef _MSC_VER /* Visual Studio */ #ifdef _MSC_VER /* Visual Studio */
# include <intrin.h> /* For Visual 2005 */ # include <intrin.h> /* For Visual 2005 */
lib/decompress/zstd_decompress.c
+106 -62
View File
@@ -40,7 +40,6 @@
# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_DEFAULTMAX) + 1) # define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_DEFAULTMAX) + 1)
#endif #endif
/*! /*!
* NO_FORWARD_PROGRESS_MAX : * NO_FORWARD_PROGRESS_MAX :
* maximum allowed nb of calls to ZSTD_decompressStream() and ZSTD_decompress_generic() * maximum allowed nb of calls to ZSTD_decompressStream() and ZSTD_decompress_generic()
@@ -52,11 +51,13 @@
# define ZSTD_NO_FORWARD_PROGRESS_MAX 16 # define ZSTD_NO_FORWARD_PROGRESS_MAX 16
#endif #endif
/*-******************************************************* /*-*******************************************************
* Dependencies * Dependencies
*********************************************************/ *********************************************************/
#include <string.h> /* memcpy, memmove, memset */ #include <string.h> /* memcpy, memmove, memset */
#include "cpu.h" #include "compiler.h" /* prefetch */
#include "cpu.h" /* bmi2 */
#include "mem.h" /* low level memory routines */ #include "mem.h" /* low level memory routines */
#define FSE_STATIC_LINKING_ONLY #define FSE_STATIC_LINKING_ONLY
#include "fse.h" #include "fse.h"
@@ -68,6 +69,9 @@
# include "zstd_legacy.h" # include "zstd_legacy.h"
#endif #endif
static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict);
static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict);
/*-************************************* /*-*************************************
* Errors * Errors
@@ -138,7 +142,6 @@ struct ZSTD_DCtx_s
U32 fseEntropy; U32 fseEntropy;
XXH64_state_t xxhState; XXH64_state_t xxhState;
size_t headerSize; size_t headerSize;
U32 dictID;
ZSTD_format_e format; ZSTD_format_e format;
const BYTE* litPtr; const BYTE* litPtr;
ZSTD_customMem customMem; ZSTD_customMem customMem;
@@ -147,9 +150,13 @@ struct ZSTD_DCtx_s
size_t staticSize; size_t staticSize;
int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
/* streaming */ /* dictionary */
ZSTD_DDict* ddictLocal; ZSTD_DDict* ddictLocal;
const ZSTD_DDict* ddict; const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */
U32 dictID;
int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
/* streaming */
ZSTD_dStreamStage streamStage; ZSTD_dStreamStage streamStage;
char* inBuff; char* inBuff;
size_t inBuffSize; size_t inBuffSize;
@@ -200,6 +207,8 @@ static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
dctx->ddict = NULL; dctx->ddict = NULL;
dctx->ddictLocal = NULL; dctx->ddictLocal = NULL;
dctx->dictEnd = NULL;
dctx->ddictIsCold = 0;
dctx->inBuff = NULL; dctx->inBuff = NULL;
dctx->inBuffSize = 0; dctx->inBuffSize = 0;
dctx->outBuffSize = 0; dctx->outBuffSize = 0;
@@ -575,6 +584,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
case set_repeat: case set_repeat:
if (dctx->litEntropy==0) return ERROR(dictionary_corrupted); if (dctx->litEntropy==0) return ERROR(dictionary_corrupted);
/* fall-through */ /* fall-through */
case set_compressed: case set_compressed:
if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */
{ size_t lhSize, litSize, litCSize; { size_t lhSize, litSize, litCSize;
@@ -606,6 +616,11 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected); if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected);
if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
/* prefetch huffman table if cold */
if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) {
PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable));
}
if (HUF_isError((litEncType==set_repeat) ? if (HUF_isError((litEncType==set_repeat) ?
( singleStream ? ( singleStream ?
HUF_decompress1X_usingDTable_bmi2(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr, dctx->bmi2) : HUF_decompress1X_usingDTable_bmi2(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr, dctx->bmi2) :
@@ -886,7 +901,8 @@ static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymb
symbolEncodingType_e type, U32 max, U32 maxLog, symbolEncodingType_e type, U32 max, U32 maxLog,
const void* src, size_t srcSize, const void* src, size_t srcSize,
const U32* baseValue, const U32* nbAdditionalBits, const U32* baseValue, const U32* nbAdditionalBits,
const ZSTD_seqSymbol* defaultTable, U32 flagRepeatTable) const ZSTD_seqSymbol* defaultTable, U32 flagRepeatTable,
int ddictIsCold, int nbSeq)
{ {
switch(type) switch(type)
{ {
@@ -905,6 +921,12 @@ static size_t ZSTD_buildSeqTable(ZSTD_seqSymbol* DTableSpace, const ZSTD_seqSymb
return 0; return 0;
case set_repeat: case set_repeat:
if (!flagRepeatTable) return ERROR(corruption_detected); if (!flagRepeatTable) return ERROR(corruption_detected);
/* prefetch FSE table if used */
if (ddictIsCold && (nbSeq > 24 /* heuristic */)) {
const void* const pStart = *DTablePtr;
size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog));
PREFETCH_AREA(pStart, pSize);
}
return 0; return 0;
case set_compressed : case set_compressed :
{ U32 tableLog; { U32 tableLog;
@@ -957,25 +979,25 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
const BYTE* const istart = (const BYTE* const)src; const BYTE* const istart = (const BYTE* const)src;
const BYTE* const iend = istart + srcSize; const BYTE* const iend = istart + srcSize;
const BYTE* ip = istart; const BYTE* ip = istart;
int nbSeq;
DEBUGLOG(5, "ZSTD_decodeSeqHeaders"); DEBUGLOG(5, "ZSTD_decodeSeqHeaders");
/* check */ /* check */
if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong); if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong);
/* SeqHead */ /* SeqHead */
{ int nbSeq = *ip++; nbSeq = *ip++;
if (!nbSeq) { *nbSeqPtr=0; return 1; } if (!nbSeq) { *nbSeqPtr=0; return 1; }
if (nbSeq > 0x7F) { if (nbSeq > 0x7F) {
if (nbSeq == 0xFF) { if (nbSeq == 0xFF) {
if (ip+2 > iend) return ERROR(srcSize_wrong); if (ip+2 > iend) return ERROR(srcSize_wrong);
nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2;
} else { } else {
if (ip >= iend) return ERROR(srcSize_wrong); if (ip >= iend) return ERROR(srcSize_wrong);
nbSeq = ((nbSeq-0x80)<<8) + *ip++; nbSeq = ((nbSeq-0x80)<<8) + *ip++;
}
} }
*nbSeqPtr = nbSeq;
} }
*nbSeqPtr = nbSeq;
/* FSE table descriptors */ /* FSE table descriptors */
if (ip+4 > iend) return ERROR(srcSize_wrong); /* minimum possible size */ if (ip+4 > iend) return ERROR(srcSize_wrong); /* minimum possible size */
@@ -989,7 +1011,8 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
LLtype, MaxLL, LLFSELog, LLtype, MaxLL, LLFSELog,
ip, iend-ip, ip, iend-ip,
LL_base, LL_bits, LL_base, LL_bits,
LL_defaultDTable, dctx->fseEntropy); LL_defaultDTable, dctx->fseEntropy,
dctx->ddictIsCold, nbSeq);
if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); if (ZSTD_isError(llhSize)) return ERROR(corruption_detected);
ip += llhSize; ip += llhSize;
} }
@@ -998,7 +1021,8 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
OFtype, MaxOff, OffFSELog, OFtype, MaxOff, OffFSELog,
ip, iend-ip, ip, iend-ip,
OF_base, OF_bits, OF_base, OF_bits,
OF_defaultDTable, dctx->fseEntropy); OF_defaultDTable, dctx->fseEntropy,
dctx->ddictIsCold, nbSeq);
if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected);
ip += ofhSize; ip += ofhSize;
} }
@@ -1007,12 +1031,23 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
MLtype, MaxML, MLFSELog, MLtype, MaxML, MLFSELog,
ip, iend-ip, ip, iend-ip,
ML_base, ML_bits, ML_base, ML_bits,
ML_defaultDTable, dctx->fseEntropy); ML_defaultDTable, dctx->fseEntropy,
dctx->ddictIsCold, nbSeq);
if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected);
ip += mlhSize; ip += mlhSize;
} }
} }
/* prefetch dictionary content */
if (dctx->ddictIsCold) {
size_t const dictSize = (const char*)dctx->prefixStart - (const char*)dctx->virtualStart;
size_t const psmin = MIN(dictSize, (size_t)(64*nbSeq) /* heuristic */ );
size_t const pSize = MIN(psmin, 128 KB /* protection */ );
const void* const pStart = (const char*)dctx->dictEnd - pSize;
PREFETCH_AREA(pStart, pSize);
dctx->ddictIsCold = 0;
}
return ip-istart; return ip-istart;
} }
@@ -1679,7 +1714,8 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
/* isLongOffset must be true if there are long offsets. /* isLongOffset must be true if there are long offsets.
* Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN. * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN.
* We don't expect that to be the case in 64-bit mode. * We don't expect that to be the case in 64-bit mode.
* In block mode, window size is not known, so we have to be conservative. (note: but it could be evaluated from current-lowLimit) * In block mode, window size is not known, so we have to be conservative.
* (note: but it could be evaluated from current-lowLimit)
*/ */
ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN))); ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN)));
DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize);
@@ -1887,9 +1923,6 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
return op-ostart; return op-ostart;
} }
static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict);
static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict);
static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const void* src, size_t srcSize,
@@ -1898,6 +1931,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
{ {
void* const dststart = dst; void* const dststart = dst;
int moreThan1Frame = 0; int moreThan1Frame = 0;
DEBUGLOG(5, "ZSTD_decompressMultiFrame");
assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */ assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */
if (ddict) { if (ddict) {
@@ -2193,8 +2228,8 @@ static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dict
return 0; return 0;
} }
/* ZSTD_loadEntropy() : /*! ZSTD_loadEntropy() :
* dict : must point at beginning of a valid zstd dictionary * dict : must point at beginning of a valid zstd dictionary.
* @return : size of entropy tables read */ * @return : size of entropy tables read */
static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy, static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy,
const void* const dict, size_t const dictSize) const void* const dict, size_t const dictSize)
@@ -2206,13 +2241,11 @@ static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy,
assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */ assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */
dictPtr += 8; /* skip header = magic + dictID */ dictPtr += 8; /* skip header = magic + dictID */
ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, LLTable) == 0); ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == offsetof(ZSTD_entropyDTables_t, LLTable) + sizeof(entropy->LLTable));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == sizeof(entropy->LLTable)); ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == offsetof(ZSTD_entropyDTables_t, OFTable) + sizeof(entropy->OFTable));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == sizeof(entropy->LLTable) + sizeof(entropy->OFTable)); ZSTD_STATIC_ASSERT(sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE);
ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, hufTable) == sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable)); { void* const workspace = &entropy->LLTable; /* use fse tables as temporary workspace; implies fse tables are grouped together */
ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, hufTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE); size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable);
{ void* const workspace = entropy; /* use fse tables as temporary workspace; implies fse table precede huffTable at beginning of entropy */
size_t const workspaceSize = offsetof(ZSTD_entropyDTables_t, hufTable);
size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable,
dictPtr, dictEnd - dictPtr, dictPtr, dictEnd - dictPtr,
workspace, workspaceSize); workspace, workspaceSize);
@@ -2292,7 +2325,6 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
return ZSTD_refDictContent(dctx, dict, dictSize); return ZSTD_refDictContent(dctx, dict, dictSize);
} }
/* Note : this function cannot fail */
size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
{ {
assert(dctx != NULL); assert(dctx != NULL);
@@ -2338,36 +2370,45 @@ struct ZSTD_DDict_s {
static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict) static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict)
{ {
assert(ddict != NULL);
return ddict->dictContent; return ddict->dictContent;
} }
static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict) static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict)
{ {
assert(ddict != NULL);
return ddict->dictSize; return ddict->dictSize;
} }
size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict) size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
{ {
CHECK_F( ZSTD_decompressBegin(dstDCtx) ); DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict");
if (ddict) { /* support begin on NULL */ assert(dctx != NULL);
dstDCtx->dictID = ddict->dictID; if (ddict) {
dstDCtx->prefixStart = ddict->dictContent; dctx->ddictIsCold = (dctx->dictEnd != (const char*)ddict->dictContent + ddict->dictSize);
dstDCtx->virtualStart = ddict->dictContent; DEBUGLOG(4, "DDict is %s",
dstDCtx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; dctx->ddictIsCold ? "~cold~" : "hot!");
dstDCtx->previousDstEnd = dstDCtx->dictEnd; }
CHECK_F( ZSTD_decompressBegin(dctx) );
if (ddict) { /* NULL ddict is equivalent to no dictionary */
dctx->dictID = ddict->dictID;
dctx->prefixStart = ddict->dictContent;
dctx->virtualStart = ddict->dictContent;
dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
dctx->previousDstEnd = dctx->dictEnd;
if (ddict->entropyPresent) { if (ddict->entropyPresent) {
dstDCtx->litEntropy = 1; dctx->litEntropy = 1;
dstDCtx->fseEntropy = 1; dctx->fseEntropy = 1;
dstDCtx->LLTptr = ddict->entropy.LLTable; dctx->LLTptr = ddict->entropy.LLTable;
dstDCtx->MLTptr = ddict->entropy.MLTable; dctx->MLTptr = ddict->entropy.MLTable;
dstDCtx->OFTptr = ddict->entropy.OFTable; dctx->OFTptr = ddict->entropy.OFTable;
dstDCtx->HUFptr = ddict->entropy.hufTable; dctx->HUFptr = ddict->entropy.hufTable;
dstDCtx->entropy.rep[0] = ddict->entropy.rep[0]; dctx->entropy.rep[0] = ddict->entropy.rep[0];
dstDCtx->entropy.rep[1] = ddict->entropy.rep[1]; dctx->entropy.rep[1] = ddict->entropy.rep[1];
dstDCtx->entropy.rep[2] = ddict->entropy.rep[2]; dctx->entropy.rep[2] = ddict->entropy.rep[2];
} else { } else {
dstDCtx->litEntropy = 0; dctx->litEntropy = 0;
dstDCtx->fseEntropy = 0; dctx->fseEntropy = 0;
} }
} }
return 0; return 0;
@@ -2604,12 +2645,15 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds)
} }
/* *** Initialization *** */ /* *** Initialization *** */
size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; } size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; }
size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; }
size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType) size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx,
const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType)
{ {
if (dctx->streamStage != zdss_init) return ERROR(stage_wrong); if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
ZSTD_freeDDict(dctx->ddictLocal); ZSTD_freeDDict(dctx->ddictLocal);
@@ -2663,13 +2707,6 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
return ZSTD_initDStream_usingDict(zds, NULL, 0); return ZSTD_initDStream_usingDict(zds, NULL, 0);
} }
size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
{
if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
dctx->ddict = ddict;
return 0;
}
/* ZSTD_initDStream_usingDDict() : /* ZSTD_initDStream_usingDDict() :
* ddict will just be referenced, and must outlive decompression session * ddict will just be referenced, and must outlive decompression session
* this function cannot fail */ * this function cannot fail */
@@ -2708,6 +2745,13 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* dctx,
return 0; return 0;
} }
size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
{
if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
dctx->ddict = ddict;
return 0;
}
size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize)
{ {
if (dctx->streamStage != zdss_init) return ERROR(stage_wrong); if (dctx->streamStage != zdss_init) return ERROR(stage_wrong);
tests/zstreamtest.c
+13 -13
View File
@@ -135,34 +135,34 @@ typedef struct {
size_t filled; size_t filled;
} buffer_t; } buffer_t;
static const buffer_t g_nullBuffer = { NULL, 0 , 0 }; static const buffer_t kBuffNull = { NULL, 0 , 0 };
static void FUZ_freeDictionary(buffer_t dict)
{
free(dict.start);
}
static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blockSize, size_t requestedDictSize) static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blockSize, size_t requestedDictSize)
{ {
buffer_t dict = { NULL, 0, 0 }; buffer_t dict = kBuffNull;
size_t const nbBlocks = (srcSize + (blockSize-1)) / blockSize; size_t const nbBlocks = (srcSize + (blockSize-1)) / blockSize;
size_t* const blockSizes = (size_t*) malloc(nbBlocks * sizeof(size_t)); size_t* const blockSizes = (size_t*)malloc(nbBlocks * sizeof(size_t));
if (!blockSizes) return dict; if (!blockSizes) return kBuffNull;
dict.start = malloc(requestedDictSize); dict.start = malloc(requestedDictSize);
if (!dict.start) { free(blockSizes); return dict; } if (!dict.start) { free(blockSizes); return kBuffNull; }
{ size_t nb; { size_t nb;
for (nb=0; nb<nbBlocks-1; nb++) blockSizes[nb] = blockSize; for (nb=0; nb<nbBlocks-1; nb++) blockSizes[nb] = blockSize;
blockSizes[nbBlocks-1] = srcSize - (blockSize * (nbBlocks-1)); blockSizes[nbBlocks-1] = srcSize - (blockSize * (nbBlocks-1));
} }
{ size_t const dictSize = ZDICT_trainFromBuffer(dict.start, requestedDictSize, src, blockSizes, (unsigned)nbBlocks); { size_t const dictSize = ZDICT_trainFromBuffer(dict.start, requestedDictSize, src, blockSizes, (unsigned)nbBlocks);
free(blockSizes); free(blockSizes);
if (ZDICT_isError(dictSize)) { free(dict.start); return g_nullBuffer; } if (ZDICT_isError(dictSize)) { FUZ_freeDictionary(dict); return kBuffNull; }
dict.size = requestedDictSize; dict.size = requestedDictSize;
dict.filled = dictSize; dict.filled = dictSize;
return dict; /* how to return dictSize ? */ return dict;
} }
} }
static void FUZ_freeDictionary(buffer_t dict)
{
free(dict.start);
}
/* Round trips data and updates xxh with the decompressed data produced */ /* Round trips data and updates xxh with the decompressed data produced */
static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
XXH64_state_t* xxh, void* data, size_t size, XXH64_state_t* xxh, void* data, size_t size,
@@ -276,7 +276,7 @@ static int basicUnitTests(U32 seed, double compressibility)
ZSTD_inBuffer inBuff, inBuff2; ZSTD_inBuffer inBuff, inBuff2;
ZSTD_outBuffer outBuff; ZSTD_outBuffer outBuff;
buffer_t dictionary = g_nullBuffer; buffer_t dictionary = kBuffNull;
size_t const dictSize = 128 KB; size_t const dictSize = 128 KB;
unsigned dictID = 0; unsigned dictID = 0;