ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

uncoupled maxdlog and chainlog

Browse Source
This commit is contained in:
Yann Collet
2015-10-22 16:55:40 +01:00
parent f3eca25322
commit eeb8ba18d1
4 changed files with 62 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/zstdhc.c
+29 -29
View File
@@ -52,17 +52,18 @@ static const U32 ZSTD_HC_compressionLevel_default = 9;
* Local Constants * Local Constants
***************************************/ ***************************************/
#define MINMATCH 4 #define MINMATCH 4
#define MAX_DISTANCE 65535 /* <=== To be changed (dynamic ?) */ #define MAXD_LOG 19
#define MAX_DISTANCE (1 << MAXD_LOG) /* <=== dynamic ? */
#define DICTIONARY_LOGSIZE 16 #define CHAIN_LOG 16
#define MAXD (1<<DICTIONARY_LOGSIZE) #define CHAIN_SIZE (1<<CHAIN_LOG)
#define MAXD_MASK (MAXD - 1) #define CHAIN_MASK (CHAIN_SIZE - 1)
#define HASH_LOG (DICTIONARY_LOGSIZE-1) #define HASH_LOG (CHAIN_LOG-1)
#define HASHTABLESIZE (1 << HASH_LOG) #define HASHTABLESIZE (1 << HASH_LOG)
#define HASH_MASK (HASHTABLESIZE - 1) #define HASH_MASK (HASHTABLESIZE - 1)
static const U32 g_maxCompressionLevel = 19; static const U32 g_maxCompressionLevel = MAXD_LOG;
#define KB *1024 #define KB *1024
#define MB *1024*1024 #define MB *1024*1024
@@ -77,16 +78,15 @@ static const U32 g_maxCompressionLevel = 19;
struct ZSTD_HC_CCtx_s struct ZSTD_HC_CCtx_s
{ {
U32 hashTable[HASHTABLESIZE]; U32 hashTable[HASHTABLESIZE];
U16 chainTable[MAXD]; U16 chainTable[CHAIN_SIZE];
const BYTE* end; /* next block here to continue on current prefix */ const BYTE* end; /* next block here to continue on current prefix */
const BYTE* base; /* All index relative to this position */ const BYTE* base; /* All regular indexes relative to this position */
const BYTE* dictBase; /* alternate base for extDict */ const BYTE* dictBase; /* extDict indexes relative to this position */
BYTE* inputBuffer; /* deprecated */
U32 dictLimit; /* below that point, need extDict */ U32 dictLimit; /* below that point, need extDict */
U32 lowLimit; /* below that point, no more dict */ U32 lowLimit; /* below that point, no more data */
U32 nextToUpdate; /* index from which to continue dictionary update */ U32 nextToUpdate; /* index from which to continue dictionary update */
U32 compressionLevel; U32 compressionLevel;
seqStore_t seqStore; seqStore_t seqStore; /* sequences storage ptrs */
BYTE buffer[WORKPLACESIZE]; BYTE buffer[WORKPLACESIZE];
}; };
@@ -98,18 +98,18 @@ ZSTD_HC_CCtx* ZSTD_HC_createCCtx(void)
size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx) { free(cctx); return 0; } size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx) { free(cctx); return 0; }
static void ZSTD_HC_resetCCtx (ZSTD_HC_CCtx* zc, U32 compressionLevel, const BYTE* start) static void ZSTD_HC_resetCCtx (ZSTD_HC_CCtx* zc, U32 compressionLevel, const void* start)
{ {
if (compressionLevel==0) compressionLevel = ZSTD_HC_compressionLevel_default; if (compressionLevel==0) compressionLevel = ZSTD_HC_compressionLevel_default;
if (compressionLevel > g_maxCompressionLevel) compressionLevel = g_maxCompressionLevel; if (compressionLevel > g_maxCompressionLevel) compressionLevel = g_maxCompressionLevel;
memset(zc->hashTable, 0, sizeof(zc->hashTable)); memset(zc->hashTable, 0, sizeof(zc->hashTable));
memset(zc->chainTable, 0xFF, sizeof(zc->chainTable)); memset(zc->chainTable, 0xFF, sizeof(zc->chainTable));
zc->nextToUpdate = 64 KB; zc->nextToUpdate = MAX_DISTANCE;
zc->base = start - 64 KB; zc->end = (const BYTE*)start;
zc->end = start; zc->base = zc->end - MAX_DISTANCE;
zc->dictBase = start - 64 KB; zc->dictBase = zc->base;
zc->dictLimit = 64 KB; zc->dictLimit = MAX_DISTANCE;
zc->lowLimit = 64 KB; zc->lowLimit = MAX_DISTANCE;
zc->compressionLevel = compressionLevel; zc->compressionLevel = compressionLevel;
zc->seqStore.buffer = zc->buffer; zc->seqStore.buffer = zc->buffer;
zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer); zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer);
@@ -124,8 +124,8 @@ static void ZSTD_HC_resetCCtx (ZSTD_HC_CCtx* zc, U32 compressionLevel, const BYT
* Local Macros * Local Macros
***************************************/ ***************************************/
#define HASH_FUNCTION(u) (((u) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG)) #define HASH_FUNCTION(u) (((u) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))
//#define DELTANEXTU16(p) chainTable[(p) & MAXD_MASK] /* flexible, MAXD dependent */ //#define DELTANEXTU16(d) chainTable[(d) & MAXD_MASK] /* flexible, CHAINSIZE dependent */
#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ #define DELTANEXTU16(d) chainTable[(U16)(d)] /* faster, specific to CHAINLOG==16 */
static U32 ZSTD_HC_hashPtr(const void* ptr) { return HASH_FUNCTION(MEM_read32(ptr)); } static U32 ZSTD_HC_hashPtr(const void* ptr) { return HASH_FUNCTION(MEM_read32(ptr)); }
@@ -167,7 +167,7 @@ static size_t ZSTD_HC_insertAndFindBestMatch (
const BYTE* const base = zc->base; const BYTE* const base = zc->base;
const BYTE* const dictBase = zc->dictBase; const BYTE* const dictBase = zc->dictBase;
const U32 dictLimit = zc->dictLimit; const U32 dictLimit = zc->dictLimit;
const U32 lowLimit = (zc->lowLimit + 64 KB > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (64 KB - 1); const U32 lowLimit = (zc->lowLimit + MAX_DISTANCE > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (MAX_DISTANCE - 1);
U32 matchIndex; U32 matchIndex;
const BYTE* match; const BYTE* match;
int nbAttempts=maxNbAttempts; int nbAttempts=maxNbAttempts;
@@ -204,6 +204,7 @@ static size_t ZSTD_HC_insertAndFindBestMatch (
if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */
} }
} }
if (base + matchIndex <= ip - CHAIN_SIZE) break;
matchIndex -= DELTANEXTU16(matchIndex); matchIndex -= DELTANEXTU16(matchIndex);
} }
@@ -226,7 +227,7 @@ size_t ZSTD_HC_InsertAndGetWiderMatch (
const BYTE* const base = zc->base; const BYTE* const base = zc->base;
const U32 dictLimit = zc->dictLimit; const U32 dictLimit = zc->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit; const BYTE* const lowPrefixPtr = base + dictLimit;
const U32 lowLimit = (zc->lowLimit + 64 KB > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (64 KB - 1); const U32 lowLimit = (zc->lowLimit + MAX_DISTANCE > (U32)(ip-base)) ? zc->lowLimit : (U32)(ip - base) - (MAX_DISTANCE - 1);
const BYTE* const dictBase = zc->dictBase; const BYTE* const dictBase = zc->dictBase;
U32 matchIndex; U32 matchIndex;
int nbAttempts = maxNbAttempts; int nbAttempts = maxNbAttempts;
@@ -281,6 +282,8 @@ size_t ZSTD_HC_InsertAndGetWiderMatch (
if (mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } if (mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
} }
} }
if (base + matchIndex <= ip - CHAIN_SIZE)
matchIndex -= MAX_DISTANCE; /* ensures it gets eliminated on next test */
matchIndex -= DELTANEXTU16(matchIndex); matchIndex -= DELTANEXTU16(matchIndex);
} }
@@ -409,8 +412,8 @@ static void ZSTD_HC_setExternalDict(ZSTD_HC_CCtx* ctxPtr, const void* newBlock)
ctxPtr->lowLimit = ctxPtr->dictLimit; ctxPtr->lowLimit = ctxPtr->dictLimit;
ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
ctxPtr->dictBase = ctxPtr->base; ctxPtr->dictBase = ctxPtr->base;
ctxPtr->base = newBlock - ctxPtr->dictLimit; ctxPtr->base = (const BYTE*)newBlock - ctxPtr->dictLimit;
ctxPtr->end = newBlock; ctxPtr->end = (const BYTE*)newBlock;
ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */
} }
@@ -422,7 +425,7 @@ size_t ZSTD_HC_compress_continue (ZSTD_HC_CCtx* ctxPtr,
if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB)
{ {
size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit;
if (dictSize > 64 KB) dictSize = 64 KB; if (dictSize > MAX_DISTANCE) dictSize = MAX_DISTANCE;
ZSTD_HC_loadDict(ctxPtr, ctxPtr->end - dictSize, dictSize); ZSTD_HC_loadDict(ctxPtr, ctxPtr->end - dictSize, dictSize);
} }
@@ -498,9 +501,6 @@ size_t ZSTD_HC_compress(void* dst, size_t maxDstSize, const void* src, size_t sr
/************************************** /**************************************
* Streaming Functions * Streaming Functions
**************************************/ **************************************/
/* dictionary saving */ /* dictionary saving */
size_t ZSTD_HC_saveDict (ZSTD_HC_CCtx* ctx, void* safeBuffer, size_t dictSize) size_t ZSTD_HC_saveDict (ZSTD_HC_CCtx* ctx, void* safeBuffer, size_t dictSize)
programs/Makefile
+1 -1
View File
@@ -61,7 +61,7 @@ all: zstd zstd32 fullbench fullbench32 fuzzer fuzzer32 datagen
zstd : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c zstd : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
$(CC) $(FLAGS) $^ -o $@$(EXT) $(CC) $(FLAGS) $^ -o $@$(EXT)
zstd32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c zstd32: $(ZSTDDIR)/zstd.c $(ZSTDDIR)/zstdhc.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c xxhash.c bench.c fileio.c zstdcli.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT) $(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
fullbench : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c datagen.c fullbench.c fullbench : $(ZSTDDIR)/zstd.c $(ZSTDDIR)/fse.c $(ZSTDDIR)/huff0.c $(ZSTDDIR)/legacy/zstd_v01.c datagen.c fullbench.c
programs/bench.c
+10 -14
View File
@@ -48,15 +48,15 @@
***************************************/ ***************************************/
#include <stdlib.h> /* malloc, free */ #include <stdlib.h> /* malloc, free */
#include <string.h> /* memset */ #include <string.h> /* memset */
#include <stdio.h> // fprintf, fopen, ftello64 #include <stdio.h> /* fprintf, fopen, ftello64 */
#include <sys/types.h> // stat64 #include <sys/types.h> /* stat64 */
#include <sys/stat.h> // stat64 #include <sys/stat.h> /* stat64 */
// Use ftime() if gettimeofday() is not available on your target /* Use ftime() if gettimeofday() is not available */
#if defined(BMK_LEGACY_TIMER) #if defined(BMK_LEGACY_TIMER)
# include <sys/timeb.h> // timeb, ftime # include <sys/timeb.h> /* timeb, ftime */
#else #else
# include <sys/time.h> // gettimeofday # include <sys/time.h> /* gettimeofday */
#endif #endif
#include "mem.h" #include "mem.h"
@@ -249,13 +249,9 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize); const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
void* const compressedBuffer = malloc(maxCompressedSize); void* const compressedBuffer = malloc(maxCompressedSize);
void* const resultBuffer = malloc(srcSize); void* const resultBuffer = malloc(srcSize);
compressor_t compressor; const compressor_t compressor = (cLevel <= 1) ? local_compress_fast : ZSTD_HC_compress;
U64 crcOrig; U64 crcOrig;
/* Init */
if (cLevel <= 1) compressor = local_compress_fast;
else compressor = ZSTD_HC_compress;
/* Memory allocation & restrictions */ /* Memory allocation & restrictions */
if (!compressedBuffer || !resultBuffer || !blockTable) if (!compressedBuffer || !resultBuffer || !blockTable)
{ {
@@ -356,7 +352,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
if (crcOrig!=crcCheck) if (crcOrig!=crcCheck)
{ {
unsigned u; unsigned u;
unsigned eBlockSize = MIN(65536*2, blockSize); unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize));
DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", fileName, (unsigned)crcOrig, (unsigned)crcCheck); DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", fileName, (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++) for (u=0; u<srcSize; u++)
{ {
@@ -465,10 +461,10 @@ static int BMK_benchOneFile(char* inFileName, int cLevel)
return 13; return 13;
} }
// Bench /* Bench */
result = BMK_benchMem(srcBuffer, benchedSize, inFileName, cLevel); result = BMK_benchMem(srcBuffer, benchedSize, inFileName, cLevel);
// End /* clean up */
free(srcBuffer); free(srcBuffer);
DISPLAY("\n"); DISPLAY("\n");
return result; return result;
programs/fileio.c
+22 -20
View File
@@ -29,9 +29,9 @@
The license of this file is GPLv2. The license of this file is GPLv2.
*/ */
/************************************** /* *************************************
* Tuning options * Tuning options
**************************************/ ***************************************/
#ifndef ZSTD_LEGACY_SUPPORT #ifndef ZSTD_LEGACY_SUPPORT
/**LEGACY_SUPPORT : /**LEGACY_SUPPORT :
* decompressor can decode older formats (starting from Zstd 0.1+) */ * decompressor can decode older formats (starting from Zstd 0.1+) */
@@ -39,9 +39,9 @@
#endif // ZSTD_LEGACY_SUPPORT #endif // ZSTD_LEGACY_SUPPORT
/************************************** /* *************************************
* Compiler Options * Compiler Options
**************************************/ ***************************************/
/* Disable some Visual warning messages */ /* Disable some Visual warning messages */
#ifdef _MSC_VER #ifdef _MSC_VER
# define _CRT_SECURE_NO_WARNINGS # define _CRT_SECURE_NO_WARNINGS
@@ -55,9 +55,9 @@
#define _POSIX_SOURCE 1 /* enable fileno() within <stdio.h> on unix */ #define _POSIX_SOURCE 1 /* enable fileno() within <stdio.h> on unix */
/************************************** /* *************************************
* Includes * Includes
**************************************/ ***************************************/
#include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */ #include <stdio.h> /* fprintf, fopen, fread, _fileno, stdin, stdout */
#include <stdlib.h> /* malloc, free */ #include <stdlib.h> /* malloc, free */
#include <string.h> /* strcmp, strlen */ #include <string.h> /* strcmp, strlen */
@@ -69,12 +69,12 @@
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
# include "zstd_v01.h" /* legacy */ # include "zstd_v01.h" /* legacy */
#endif // ZSTD_LEGACY_SUPPORT #endif
/************************************** /* *************************************
* OS-specific Includes * OS-specific Includes
**************************************/ ***************************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__) #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <fcntl.h> /* _O_BINARY */ # include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _isatty */ # include <io.h> /* _setmode, _isatty */
@@ -90,9 +90,9 @@
#endif #endif
/************************************** /* *************************************
* Constants * Constants
**************************************/ ***************************************/
#define KB *(1U<<10) #define KB *(1U<<10)
#define MB *(1U<<20) #define MB *(1U<<20)
#define GB *(1U<<30) #define GB *(1U<<30)
@@ -116,9 +116,9 @@ static const unsigned FIO_blockHeaderSize = 3;
#define CACHELINE 64 #define CACHELINE 64
/************************************** /* *************************************
* Macros * Macros
**************************************/ ***************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
@@ -131,19 +131,21 @@ static const unsigned refreshRate = 150;
static clock_t g_time = 0; static clock_t g_time = 0;
/************************************** /* *************************************
* Local Parameters * Local Parameters
**************************************/ ***************************************/
static U32 g_overwrite = 0; static U32 g_overwrite = 0;
void FIO_overwriteMode(void) { g_overwrite=1; } void FIO_overwriteMode(void) { g_overwrite=1; }
void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; } void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
/************************************** /* *************************************
* Exceptions * Exceptions
**************************************/ ***************************************/
#define DEBUG 0 #ifndef DEBUG
# define DEBUG 0
#endif
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); #define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
#define EXM_THROW(error, ...) \ #define EXM_THROW(error, ...) \
{ \ { \
@@ -155,9 +157,9 @@ void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
} }
/************************************** /* *************************************
* Functions * Functions
**************************************/ ***************************************/
static unsigned FIO_GetMilliSpan(clock_t nPrevious) static unsigned FIO_GetMilliSpan(clock_t nPrevious)
{ {
clock_t nCurrent = clock(); clock_t nCurrent = clock();